Scrape CPU latency stats from /proc/schedstat (#1389)

These are useful as a direct indication of CPU contention and task
scheduler latency.

Handy references:
 - https://github.com/torvalds/linux/blob/master/Documentation/scheduler/sched-stats.txt
 - https://doc.opensuse.org/documentation/leap/tuning/html/book.sle.tuning/cha.tuning.taskscheduler.html

procfs is updated to pull in the enabling change:
https://github.com/prometheus/procfs/pull/186

Signed-off-by: Phil Frost <phil@postmates.com>

collector/schedstat_linux.go

@@ -0,0 +1,94 @@
+// Copyright 2019 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs"
+)
+
+var (
+	runningSecondsTotal = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "schedstat", "running_seconds_total"),
+		"Number of seconds CPU spent running a process.",
+		[]string{"cpu"},
+		nil,
+	)
+
+	waitingSecondsTotal = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "schedstat", "waiting_seconds_total"),
+		"Number of seconds spent by processing waiting for this CPU.",
+		[]string{"cpu"},
+		nil,
+	)
+
+	timeslicesTotal = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "schedstat", "timeslices_total"),
+		"Number of timeslices executed by CPU.",
+		[]string{"cpu"},
+		nil,
+	)
+)
+
+// NewSchedstatCollector returns a new Collector exposing task scheduler statistics
+func NewSchedstatCollector() (Collector, error) {
+	fs, err := procfs.NewFS(*procPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open procfs: %v", err)
+	}
+
+	return &schedstatCollector{fs: fs}, nil
+}
+
+type schedstatCollector struct {
+	fs procfs.FS
+}
+
+func init() {
+	registerCollector("schedstat", defaultEnabled, NewSchedstatCollector)
+}
+
+func (c *schedstatCollector) Update(ch chan<- prometheus.Metric) error {
+	stats, err := c.fs.Schedstat()
+	if err != nil {
+		return err
+	}
+
+	for _, cpu := range stats.CPUs {
+		ch <- prometheus.MustNewConstMetric(
+			runningSecondsTotal,
+			prometheus.CounterValue,
+			cpu.RunningSeconds(),
+			cpu.CPUNum,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			waitingSecondsTotal,
+			prometheus.CounterValue,
+			cpu.WaitingSeconds(),
+			cpu.CPUNum,
+		)
+
+		ch <- prometheus.MustNewConstMetric(
+			timeslicesTotal,
+			prometheus.CounterValue,
+			float64(cpu.RunTimeslices),
+			cpu.CPUNum,
+		)
+	}
+
+	return nil
+}