Add metrics from SNTPv4 packet to ntp collector & add ntpd sanity check (#655)
* Add metrics from SNTPv4 packet to ntp collector & add ntpd sanity check 1. Checking local clock against remote NTP daemon is bad idea, local ntpd acting as a client should do it better and avoid excessive load on remote NTP server so the collector is refactored to query local NTP server. 2. Checking local clock against remote one does not check local ntpd itself. Local ntpd may be down or out of sync due to network issues, but clock will be OK. 3. Checking NTP server using sanity of it's response is tricky and depends on ntpd implementation, that's why common `node_ntp_sanity` variable is exported. * `govendor add golang.org/x/net/ipv4`, it is dependency of github.com/beevik/ntp * Update github.com/beevik/ntp to include boring SNTP fix * Use variable name from RFC5905 * ntp: move code to make export of raw metrics more explicit * Move NTP math to `github.com/beevik/ntp` * Make `golint` happy * Add some brief docs explaining `ntp` #655 and `timex` #664 modules * ntp: drop XXX comment that got its decision * ntp: add `_seconds` suffix to relevant metrics * Better `node_ntp_leap` comment * s/node_ntp_reftime/node_ntp_reference_timestamp_seconds/ as requested by @discordianfish * Extract subsystem name to const as suggested by @SuperQ
This commit is contained in:
Leonid Evdokimov
Johannes 'fish' Ziemke
parent
b0d5c00832
commit
c169b4b1c5
169 changed files with 10650 additions and 64 deletions
128
collector/ntp.go
128
collector/ntp.go
|
|
@ -17,62 +17,146 @@ package collector
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/beevik/ntp"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/common/log"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
)
|
||||
|
||||
const (
|
||||
hour24 = 24 * time.Hour // `time` does not export `Day` as Day != 24h because of DST
|
||||
ntpSubsystem = "ntp"
|
||||
)
|
||||
|
||||
var (
|
||||
ntpServer = kingpin.Flag("collector.ntp.server", "NTP server to use for ntp collector.").Default("").String()
|
||||
ntpServer = kingpin.Flag("collector.ntp.server", "NTP server to use for ntp collector").Default("127.0.0.1").String()
|
||||
ntpProtocolVersion = kingpin.Flag("collector.ntp.protocol-version", "NTP protocol version").Default("4").Int()
|
||||
ntpServerIsLocal = kingpin.Flag("collector.ntp.server-is-local", "Certify that collector.ntp.server address is the same local host as this collector.").Default("false").Bool()
|
||||
ntpIPTTL = kingpin.Flag("collector.ntp.ip-ttl", "IP TTL to use while sending NTP query").Default("1").Int()
|
||||
// 3.46608s ~ 1.5s + PHI * (1 << maxPoll), where 1.5s is MAXDIST from ntp.org, it is 1.0 in RFC5905
|
||||
// max-distance option is used as-is without phi*(1<<poll)
|
||||
ntpMaxDistance = kingpin.Flag("collector.ntp.max-distance", "Max accumulated distance to the root").Default("3.46608s").Duration()
|
||||
ntpOffsetTolerance = kingpin.Flag("collector.ntp.local-offset-tolerance", "Offset between local clock and local ntpd time to tolerate").Default("1ms").Duration()
|
||||
|
||||
leapMidnight time.Time
|
||||
)
|
||||
|
||||
type ntpCollector struct {
|
||||
drift, stratum typedDesc
|
||||
stratum, leap, rtt, offset, reftime, rootDelay, rootDispersion, sanity typedDesc
|
||||
}
|
||||
|
||||
func init() {
|
||||
Factories["ntp"] = NewNtpCollector
|
||||
}
|
||||
|
||||
// NewNtpCollector returns a new Collector exposing the offset between ntp and
|
||||
// the current system time.
|
||||
// NewNtpCollector returns a new Collector exposing sanity of local NTP server.
|
||||
// Default definition of "local" is:
|
||||
// - collector.ntp.server address is a loopback address (or collector.ntp.server-is-mine flag is turned on)
|
||||
// - the server is reachable with outgoin IP_TTL = 1
|
||||
func NewNtpCollector() (Collector, error) {
|
||||
warnDeprecated("ntp")
|
||||
if *ntpServer == "" {
|
||||
return nil, fmt.Errorf("no NTP server specified, see -collector.ntp.server")
|
||||
ipaddr := net.ParseIP(*ntpServer)
|
||||
if !*ntpServerIsLocal && (ipaddr == nil || !ipaddr.IsLoopback()) {
|
||||
return nil, fmt.Errorf("only IP address of local NTP server is valid for -collector.ntp.server")
|
||||
}
|
||||
|
||||
if *ntpProtocolVersion < 2 || *ntpProtocolVersion > 4 {
|
||||
return nil, fmt.Errorf("invalid NTP protocol version %d; must be 2, 3, or 4", *ntpProtocolVersion)
|
||||
}
|
||||
|
||||
if *ntpOffsetTolerance < 0 {
|
||||
return nil, fmt.Errorf("Offset tolerance must be non-negative")
|
||||
}
|
||||
|
||||
return &ntpCollector{
|
||||
drift: typedDesc{prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, "ntp", "drift_seconds"),
|
||||
"Time between system time and ntp time.",
|
||||
stratum: typedDesc{prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, ntpSubsystem, "stratum"),
|
||||
"NTPD stratum.",
|
||||
nil, nil,
|
||||
), prometheus.GaugeValue},
|
||||
stratum: typedDesc{prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, "ntp", "stratum"),
|
||||
"NTP server stratum.",
|
||||
leap: typedDesc{prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, ntpSubsystem, "leap"),
|
||||
"NTPD leap second indicator, 2 bits.",
|
||||
nil, nil,
|
||||
), prometheus.GaugeValue},
|
||||
rtt: typedDesc{prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, ntpSubsystem, "rtt_seconds"),
|
||||
"RTT to NTPD.",
|
||||
nil, nil,
|
||||
), prometheus.GaugeValue},
|
||||
offset: typedDesc{prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, ntpSubsystem, "offset_seconds"),
|
||||
"ClockOffset between NTP and local clock.",
|
||||
nil, nil,
|
||||
), prometheus.GaugeValue},
|
||||
reftime: typedDesc{prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, ntpSubsystem, "reference_timestamp_seconds"),
|
||||
"NTPD ReferenceTime, UNIX timestamp.",
|
||||
nil, nil,
|
||||
), prometheus.GaugeValue},
|
||||
rootDelay: typedDesc{prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, ntpSubsystem, "root_delay_seconds"),
|
||||
"NTPD RootDelay.",
|
||||
nil, nil,
|
||||
), prometheus.GaugeValue},
|
||||
rootDispersion: typedDesc{prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, ntpSubsystem, "root_dispersion_seconds"),
|
||||
"NTPD RootDispersion.",
|
||||
nil, nil,
|
||||
), prometheus.GaugeValue},
|
||||
sanity: typedDesc{prometheus.NewDesc(
|
||||
prometheus.BuildFQName(Namespace, ntpSubsystem, "sanity"),
|
||||
"NTPD sanity according to RFC5905 heuristics and configured limits.",
|
||||
nil, nil,
|
||||
), prometheus.GaugeValue},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *ntpCollector) Update(ch chan<- prometheus.Metric) error {
|
||||
resp, err := ntp.Query(*ntpServer, *ntpProtocolVersion)
|
||||
resp, err := ntp.QueryWithOptions(*ntpServer, ntp.QueryOptions{
|
||||
Version: *ntpProtocolVersion,
|
||||
TTL: *ntpIPTTL,
|
||||
Timeout: time.Second, // default `ntpdate` timeout
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't get NTP drift: %s", err)
|
||||
return fmt.Errorf("couldn't get SNTP reply: %s", err)
|
||||
}
|
||||
|
||||
ch <- c.stratum.mustNewConstMetric(float64(resp.Stratum))
|
||||
ch <- c.leap.mustNewConstMetric(float64(resp.Leap))
|
||||
ch <- c.rtt.mustNewConstMetric(resp.RTT.Seconds())
|
||||
ch <- c.offset.mustNewConstMetric(resp.ClockOffset.Seconds())
|
||||
if resp.ReferenceTime.Unix() > 0 {
|
||||
// Go Zero is 0001-01-01 00:00:00 UTC
|
||||
// NTP Zero is 1900-01-01 00:00:00 UTC
|
||||
// UNIX Zero is 1970-01-01 00:00:00 UTC
|
||||
// so let's keep ALL ancient `reftime` values as zero
|
||||
ch <- c.reftime.mustNewConstMetric(float64(resp.ReferenceTime.UnixNano()) / 1e9)
|
||||
} else {
|
||||
ch <- c.reftime.mustNewConstMetric(0)
|
||||
}
|
||||
ch <- c.rootDelay.mustNewConstMetric(resp.RootDelay.Seconds())
|
||||
ch <- c.rootDispersion.mustNewConstMetric(resp.RootDispersion.Seconds())
|
||||
|
||||
// Here is SNTP packet sanity check that is exposed to move burden of
|
||||
// configuration from node_exporter user to the developer.
|
||||
|
||||
maxerr := *ntpOffsetTolerance
|
||||
if resp.Leap == ntp.LeapAddSecond || resp.Leap == ntp.LeapDelSecond {
|
||||
// state of leapMidnight is cached as leap flag is dropped right after midnight
|
||||
leapMidnight = resp.Time.Truncate(hour24).Add(hour24)
|
||||
}
|
||||
if leapMidnight.Add(-hour24).Before(resp.Time) && resp.Time.Before(leapMidnight.Add(hour24)) {
|
||||
// tolerate leap smearing
|
||||
maxerr += time.Second
|
||||
}
|
||||
|
||||
if resp.Validate() && resp.RootDistance <= *ntpMaxDistance && resp.CausalityViolation <= maxerr {
|
||||
ch <- c.sanity.mustNewConstMetric(1)
|
||||
} else {
|
||||
ch <- c.sanity.mustNewConstMetric(0)
|
||||
}
|
||||
driftSeconds := resp.ClockOffset.Seconds()
|
||||
log.Debugf("Set ntp_drift_seconds: %f", driftSeconds)
|
||||
ch <- c.drift.mustNewConstMetric(driftSeconds)
|
||||
|
||||
stratum := float64(resp.Stratum)
|
||||
log.Debugf("Set ntp_stratum: %f", stratum)
|
||||
ch <- c.stratum.mustNewConstMetric(stratum)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue