ston1th/node_exporter
0
0
Fork 0
Code Issues Releases Activity

Fix rollover bug in mountstats collector (#1364)

Browse source 
* Update procfs vendor to pull in github.com/prometheus/procfs/pull/165
* Update mountstats collector to use new types.
* Rollover counter automatically to avoid float64 accuracy issues.
* Update e2e test.

Signed-off-by: Ben Kochie <superq@gmail.com>
This commit is contained in:
Ben Kochie 2019-05-31 18:30:37 +02:00 • committed by GitHub
commit 8146998945
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 771 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

8
collector/fixtures/e2e-64k-page-output.txt
View file

@ -1577,6 +1577,7 @@ node_mountstats_nfs_event_write_extension_total{export="192.168.1.1:/srv/test",p
node_mountstats_nfs_event_write_extension_total{export="192.168.1.1:/srv/test",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_major_timeouts_total Number of times a request has had a major timeout for a given operation.
# TYPE node_mountstats_nfs_operations_major_timeouts_total counter
node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 0
node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 0
@ -1585,6 +1586,7 @@ node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/tes
node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_queue_time_seconds_total Duration all requests spent queued for transmission for a given operation before they were sent, in seconds.
# TYPE node_mountstats_nfs_operations_queue_time_seconds_total counter
node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 9.007044786793922e+12
node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 0.006
@ -1593,6 +1595,7 @@ node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv
node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_received_bytes_total Number of bytes received for a given operation, including RPC headers and payload.
# TYPE node_mountstats_nfs_operations_received_bytes_total counter
node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 3.62996810236e+11
node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 1.210292152e+09
@ -1601,6 +1604,7 @@ node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/tes
node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_request_time_seconds_total Duration all requests took from when a request was enqueued to when it was completely handled for a given operation, in seconds.
# TYPE node_mountstats_nfs_operations_request_time_seconds_total counter
node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 1.953587717e+06
node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 79.407
@ -1609,6 +1613,7 @@ node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/s
node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_requests_total Number of requests performed for a given operation.
# TYPE node_mountstats_nfs_operations_requests_total counter
node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 2.927395007e+09
node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 1298
@ -1617,6 +1622,7 @@ node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",ope
node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_response_time_seconds_total Duration all requests took to get a reply back after a request for a given operation was transmitted, in seconds.
# TYPE node_mountstats_nfs_operations_response_time_seconds_total counter
node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 1.667369447e+06
node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 79.386
@ -1625,6 +1631,7 @@ node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/
node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_sent_bytes_total Number of bytes sent for a given operation, including RPC headers and payload.
# TYPE node_mountstats_nfs_operations_sent_bytes_total counter
node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 5.26931094212e+11
node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 207680
@ -1633,6 +1640,7 @@ node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",o
node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_transmissions_total Number of times an actual RPC request has been transmitted for a given operation.
# TYPE node_mountstats_nfs_operations_transmissions_total counter
node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 2.927394995e+09
node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 1298

8
collector/fixtures/e2e-output.txt
View file

@ -1577,6 +1577,7 @@ node_mountstats_nfs_event_write_extension_total{export="192.168.1.1:/srv/test",p
node_mountstats_nfs_event_write_extension_total{export="192.168.1.1:/srv/test",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_major_timeouts_total Number of times a request has had a major timeout for a given operation.
# TYPE node_mountstats_nfs_operations_major_timeouts_total counter
node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 0
node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 0
@ -1585,6 +1586,7 @@ node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/tes
node_mountstats_nfs_operations_major_timeouts_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_queue_time_seconds_total Duration all requests spent queued for transmission for a given operation before they were sent, in seconds.
# TYPE node_mountstats_nfs_operations_queue_time_seconds_total counter
node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 9.007044786793922e+12
node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 0.006
@ -1593,6 +1595,7 @@ node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv
node_mountstats_nfs_operations_queue_time_seconds_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_received_bytes_total Number of bytes received for a given operation, including RPC headers and payload.
# TYPE node_mountstats_nfs_operations_received_bytes_total counter
node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 3.62996810236e+11
node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 1.210292152e+09
@ -1601,6 +1604,7 @@ node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/tes
node_mountstats_nfs_operations_received_bytes_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_request_time_seconds_total Duration all requests took from when a request was enqueued to when it was completely handled for a given operation, in seconds.
# TYPE node_mountstats_nfs_operations_request_time_seconds_total counter
node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 1.953587717e+06
node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 79.407
@ -1609,6 +1613,7 @@ node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/s
node_mountstats_nfs_operations_request_time_seconds_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_requests_total Number of requests performed for a given operation.
# TYPE node_mountstats_nfs_operations_requests_total counter
node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 2.927395007e+09
node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 1298
@ -1617,6 +1622,7 @@ node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",ope
node_mountstats_nfs_operations_requests_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_response_time_seconds_total Duration all requests took to get a reply back after a request for a given operation was transmitted, in seconds.
# TYPE node_mountstats_nfs_operations_response_time_seconds_total counter
node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 1.667369447e+06
node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 79.386
@ -1625,6 +1631,7 @@ node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/
node_mountstats_nfs_operations_response_time_seconds_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_sent_bytes_total Number of bytes sent for a given operation, including RPC headers and payload.
# TYPE node_mountstats_nfs_operations_sent_bytes_total counter
node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 5.26931094212e+11
node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 207680
@ -1633,6 +1640,7 @@ node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",o
node_mountstats_nfs_operations_sent_bytes_total{export="192.168.1.1:/srv/test",operation="WRITE",protocol="udp"} 0
# HELP node_mountstats_nfs_operations_transmissions_total Number of times an actual RPC request has been transmitted for a given operation.
# TYPE node_mountstats_nfs_operations_transmissions_total counter
node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="ACCESS",protocol="udp"} 2.927394995e+09
node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="tcp"} 0
node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="NULL",protocol="udp"} 0
node_mountstats_nfs_operations_transmissions_total{export="192.168.1.1:/srv/test",operation="READ",protocol="tcp"} 1298

2
collector/fixtures/proc/10/mountstats
View file

@ -31,6 +31,7 @@ device 192.168.1.1:/srv/test mounted on /mnt/nfs/test-dupe with fstype nfs4 stat
NULL: 0 0 0 0 0 0 0 0
READ: 1298 1298 0 207680 1210292152 6 79386 79407
WRITE: 0 0 0 0 0 0 0 0
ACCESS: 2927395007 2927394995 0 526931094212 362996810236 18446743919241604546 1667369447 1953587717
device 192.168.1.1:/srv/test mounted on /mnt/nfs/test-dupe with fstype nfs4 statvers=1.1
opts: rw,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=udp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=192.168.1.5,local_lock=none
@ -46,3 +47,4 @@ device 192.168.1.1:/srv/test mounted on /mnt/nfs/test-dupe with fstype nfs4 stat
NULL: 0 0 0 0 0 0 0 0
READ: 1298 1298 0 207680 1210292152 6 79386 79407
WRITE: 0 0 0 0 0 0 0 0
ACCESS: 2927395007 2927394995 0 526931094212 362996810236 18446743919241604546 1667369447 1953587717

13
collector/mountstats_linux.go
View file

@ -21,6 +21,11 @@ import (
"github.com/prometheus/procfs"
)
var (
// 64-bit float mantissa: https://en.wikipedia.org/wiki/Double-precision_floating-point_format
float64Mantissa uint64 = 9007199254740992
)
type mountStatsCollector struct {
// General statistics
NFSAgeSecondsTotal *prometheus.Desc
@ -618,7 +623,7 @@ func (c *mountStatsCollector) updateNFSStats(ch chan<- prometheus.Metric, export
ch <- prometheus.MustNewConstMetric(
c.NFSTransportIdleTimeSeconds,
prometheus.GaugeValue,
s.Transport.IdleTime.Seconds(),
float64(s.Transport.IdleTimeSeconds%float64Mantissa),
export,
protocol,
)
@ -728,7 +733,7 @@ func (c *mountStatsCollector) updateNFSStats(ch chan<- prometheus.Metric, export
ch <- prometheus.MustNewConstMetric(
c.NFSOperationsQueueTimeSecondsTotal,
prometheus.CounterValue,
op.CumulativeQueueTime.Seconds(),
float64(op.CumulativeQueueMilliseconds%float64Mantissa)/1000.0,
export,
protocol,
op.Operation,
@ -737,7 +742,7 @@ func (c *mountStatsCollector) updateNFSStats(ch chan<- prometheus.Metric, export
ch <- prometheus.MustNewConstMetric(
c.NFSOperationsResponseTimeSecondsTotal,
prometheus.CounterValue,
op.CumulativeTotalResponseTime.Seconds(),
float64(op.CumulativeTotalResponseMilliseconds%float64Mantissa)/1000.0,
export,
protocol,
op.Operation,
@ -746,7 +751,7 @@ func (c *mountStatsCollector) updateNFSStats(ch chan<- prometheus.Metric, export
ch <- prometheus.MustNewConstMetric(
c.NFSOperationsRequestTimeSecondsTotal,
prometheus.CounterValue,
op.CumulativeTotalRequestTime.Seconds(),
float64(op.CumulativeTotalRequestMilliseconds%float64Mantissa)/1000.0,
export,
protocol,
op.Operation,