ston1th/node_exporter
0
0
Fork 0
Code Issues Releases Activity

Beginnings of a node-exporter monitoring mixin.

Browse source 
Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>
This commit is contained in:
Tom Wilkie 2018-05-08 12:10:29 +02:00 • committed by Matthias Loibl
commit bafe1707f1
No known key found for this signature in database
GPG key ID: B1C7DF661ABB2C1A
11 changed files with 767 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

165
node-mixin/alerts/alerts.libsonnet Normal file
View file

@ -0,0 +1,165 @@
{
prometheusAlerts+:: {
groups+: [
{
name: 'node',
rules: [
{
alert: 'NodeFilesystemSpaceFillingUp',
expr: |||
predict_linear(node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s}[6h], 24*60*60) < 0
AND
node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_size{%(nodeExporterSelector)s,%(fsSelectors)s} < 0.4
AND
node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} is predicted to run out of space within the next 24 hours.',
},
},
{
alert: 'NodeFilesystemSpaceFillingUp',
expr: |||
predict_linear(node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s}[6h], 4*60*60) < 0
AND
node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_size{%(nodeExporterSelector)s,%(fsSelectors)s} < 0.2
AND
node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
||| % $._config,
'for': '1h',
labels: {
severity: 'critical',
},
annotations: {
message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} is predicted to run out of space within the next 4 hours.',
},
},
{
alert: 'NodeFilesystemOutOfSpace',
expr: |||
node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_size{%(nodeExporterSelector)s,%(fsSelectors)s} * 100 < 5
AND
node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ $value }}% available space left.',
},
},
{
alert: 'NodeFilesystemOutOfSpace',
expr: |||
node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_size{%(nodeExporterSelector)s,%(fsSelectors)s} * 100 < 3
AND
node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
||| % $._config,
'for': '1h',
labels: {
severity: 'critical',
},
annotations: {
message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ $value }}% available space left.',
},
},
{
alert: 'NodeFilesystemFilesFillingUp',
expr: |||
predict_linear(node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s}[6h], 24*60*60) < 0
AND
node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelectors)s} < 0.4
AND
node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} is predicted to run out of files within the next 24 hours.',
},
},
{
alert: 'NodeFilesystemFilesFillingUp',
expr: |||
predict_linear(node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s}[6h], 4*60*60) < 0
AND
node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelectors)s} < 0.2
AND
node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} is predicted to run out of files within the next 4 hours.',
},
},
{
alert: 'NodeFilesystemOutOfFiles',
expr: |||
node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelectors)s} * 100 < 5
AND
node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ $value }}% available inodes left.',
},
},
{
alert: 'NodeFilesystemOutOfSpace',
expr: |||
node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelectors)s} * 100 < 3
AND
node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
||| % $._config,
'for': '1h',
labels: {
severity: 'critical',
},
annotations: {
message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ $value }}% available space left.',
},
},
{
alert: 'NodeNetworkReceiveErrs',
expr: |||
increase(node_network_receive_errs[2m]) > 10
||| % $._config,
'for': '1h',
labels: {
severity: 'critical',
},
annotations: {
message: '{{ $labels.instance }} interface {{ $labels.device }} shows errors while receiving packets ({{ $value }} errors in two minutes).',
},
},
{
alert: 'NodeNetworkTransmitErrs',
expr: |||
increase(node_network_transmit_errs[2m]) > 10
||| % $._config,
'for': '1h',
labels: {
severity: 'critical',
},
annotations: {
message: '{{ $labels.instance }} interface {{ $labels.device }} shows errors while transmitting packets ({{ $value }} errors in two minutes).',
},
},
],
},
],
},
}