1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package raft
import (
"math"
"time"
"github.com/armon/go-metrics"
)
// saturationMetric measures the saturation (percentage of time spent working vs
// waiting for work) of an event processing loop, such as runFSM. It reports the
// saturation as a gauge metric (at most) once every reportInterval.
//
// Callers must instrument their loop with calls to sleeping and working, starting
// with a call to sleeping.
//
// Note: the caller must be single-threaded and saturationMetric is not safe for
// concurrent use by multiple goroutines.
type saturationMetric struct {
reportInterval time.Duration
// slept contains time for which the event processing loop was sleeping rather
// than working in the period since lastReport.
slept time.Duration
// lost contains time that is considered lost due to incorrect use of
// saturationMetricBucket (e.g. calling sleeping() or working() multiple
// times in succession) in the period since lastReport.
lost time.Duration
lastReport, sleepBegan, workBegan time.Time
// These are overwritten in tests.
nowFn func() time.Time
reportFn func(float32)
}
// newSaturationMetric creates a saturationMetric that will update the gauge
// with the given name at the given reportInterval. keepPrev determines the
// number of previous measurements that will be used to smooth out spikes.
func newSaturationMetric(name []string, reportInterval time.Duration) *saturationMetric {
m := &saturationMetric{
reportInterval: reportInterval,
nowFn: time.Now,
lastReport: time.Now(),
reportFn: func(sat float32) { metrics.AddSample(name, sat) },
}
return m
}
// sleeping records the time at which the loop began waiting for work. After the
// initial call it must always be proceeded by a call to working.
func (s *saturationMetric) sleeping() {
now := s.nowFn()
if !s.sleepBegan.IsZero() {
// sleeping called twice in succession. Count that time as lost rather than
// measuring nonsense.
s.lost += now.Sub(s.sleepBegan)
}
s.sleepBegan = now
s.workBegan = time.Time{}
s.report()
}
// working records the time at which the loop began working. It must always be
// proceeded by a call to sleeping.
func (s *saturationMetric) working() {
now := s.nowFn()
if s.workBegan.IsZero() {
if s.sleepBegan.IsZero() {
// working called before the initial call to sleeping. Count that time as
// lost rather than measuring nonsense.
s.lost += now.Sub(s.lastReport)
} else {
s.slept += now.Sub(s.sleepBegan)
}
} else {
// working called twice in succession. Count that time as lost rather than
// measuring nonsense.
s.lost += now.Sub(s.workBegan)
}
s.workBegan = now
s.sleepBegan = time.Time{}
s.report()
}
// report updates the gauge if reportInterval has passed since our last report.
func (s *saturationMetric) report() {
now := s.nowFn()
timeSinceLastReport := now.Sub(s.lastReport)
if timeSinceLastReport < s.reportInterval {
return
}
var saturation float64
total := timeSinceLastReport - s.lost
if total != 0 {
saturation = float64(total-s.slept) / float64(total)
saturation = math.Round(saturation*100) / 100
}
s.reportFn(float32(saturation))
s.slept = 0
s.lost = 0
s.lastReport = now
}
|
