1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
package metrics
import (
"fmt"
"io"
"log"
"math"
"runtime"
runtimemetrics "runtime/metrics"
"strings"
"github.com/valyala/histogram"
)
// See https://pkg.go.dev/runtime/metrics#hdr-Supported_metrics
var runtimeMetrics = [][2]string{
{"/sched/latencies:seconds", "go_sched_latencies_seconds"},
{"/sync/mutex/wait/total:seconds", "go_mutex_wait_seconds_total"},
{"/cpu/classes/gc/mark/assist:cpu-seconds", "go_gc_mark_assist_cpu_seconds_total"},
{"/cpu/classes/gc/total:cpu-seconds", "go_gc_cpu_seconds_total"},
{"/gc/pauses:seconds", "go_gc_pauses_seconds"},
{"/cpu/classes/scavenge/total:cpu-seconds", "go_scavenge_cpu_seconds_total"},
{"/gc/gomemlimit:bytes", "go_memlimit_bytes"},
}
var supportedRuntimeMetrics = initSupportedRuntimeMetrics(runtimeMetrics)
func initSupportedRuntimeMetrics(rms [][2]string) [][2]string {
exposedMetrics := make(map[string]struct{})
for _, d := range runtimemetrics.All() {
exposedMetrics[d.Name] = struct{}{}
}
var supportedMetrics [][2]string
for _, rm := range rms {
metricName := rm[0]
if _, ok := exposedMetrics[metricName]; ok {
supportedMetrics = append(supportedMetrics, rm)
} else {
log.Printf("github.com/VictoriaMetrics/metrics: do not expose %s metric, since the corresponding metric %s isn't supported in the current Go runtime", rm[1], metricName)
}
}
return supportedMetrics
}
func writeGoMetrics(w io.Writer) {
writeRuntimeMetrics(w)
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
WriteGaugeUint64(w, "go_memstats_alloc_bytes", ms.Alloc)
WriteCounterUint64(w, "go_memstats_alloc_bytes_total", ms.TotalAlloc)
WriteGaugeUint64(w, "go_memstats_buck_hash_sys_bytes", ms.BuckHashSys)
WriteCounterUint64(w, "go_memstats_frees_total", ms.Frees)
WriteGaugeFloat64(w, "go_memstats_gc_cpu_fraction", ms.GCCPUFraction)
WriteGaugeUint64(w, "go_memstats_gc_sys_bytes", ms.GCSys)
WriteGaugeUint64(w, "go_memstats_heap_alloc_bytes", ms.HeapAlloc)
WriteGaugeUint64(w, "go_memstats_heap_idle_bytes", ms.HeapIdle)
WriteGaugeUint64(w, "go_memstats_heap_inuse_bytes", ms.HeapInuse)
WriteGaugeUint64(w, "go_memstats_heap_objects", ms.HeapObjects)
WriteGaugeUint64(w, "go_memstats_heap_released_bytes", ms.HeapReleased)
WriteGaugeUint64(w, "go_memstats_heap_sys_bytes", ms.HeapSys)
WriteGaugeFloat64(w, "go_memstats_last_gc_time_seconds", float64(ms.LastGC)/1e9)
WriteCounterUint64(w, "go_memstats_lookups_total", ms.Lookups)
WriteCounterUint64(w, "go_memstats_mallocs_total", ms.Mallocs)
WriteGaugeUint64(w, "go_memstats_mcache_inuse_bytes", ms.MCacheInuse)
WriteGaugeUint64(w, "go_memstats_mcache_sys_bytes", ms.MCacheSys)
WriteGaugeUint64(w, "go_memstats_mspan_inuse_bytes", ms.MSpanInuse)
WriteGaugeUint64(w, "go_memstats_mspan_sys_bytes", ms.MSpanSys)
WriteGaugeUint64(w, "go_memstats_next_gc_bytes", ms.NextGC)
WriteGaugeUint64(w, "go_memstats_other_sys_bytes", ms.OtherSys)
WriteGaugeUint64(w, "go_memstats_stack_inuse_bytes", ms.StackInuse)
WriteGaugeUint64(w, "go_memstats_stack_sys_bytes", ms.StackSys)
WriteGaugeUint64(w, "go_memstats_sys_bytes", ms.Sys)
WriteCounterUint64(w, "go_cgo_calls_count", uint64(runtime.NumCgoCall()))
WriteGaugeUint64(w, "go_cpu_count", uint64(runtime.NumCPU()))
gcPauses := histogram.NewFast()
for _, pauseNs := range ms.PauseNs[:] {
gcPauses.Update(float64(pauseNs) / 1e9)
}
phis := []float64{0, 0.25, 0.5, 0.75, 1}
quantiles := make([]float64, 0, len(phis))
WriteMetadataIfNeeded(w, "go_gc_duration_seconds", "summary")
for i, q := range gcPauses.Quantiles(quantiles[:0], phis) {
fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %g`+"\n", phis[i], q)
}
fmt.Fprintf(w, "go_gc_duration_seconds_sum %g\n", float64(ms.PauseTotalNs)/1e9)
fmt.Fprintf(w, "go_gc_duration_seconds_count %d\n", ms.NumGC)
WriteCounterUint64(w, "go_gc_forced_count", uint64(ms.NumForcedGC))
WriteGaugeUint64(w, "go_gomaxprocs", uint64(runtime.GOMAXPROCS(0)))
WriteGaugeUint64(w, "go_goroutines", uint64(runtime.NumGoroutine()))
numThread, _ := runtime.ThreadCreateProfile(nil)
WriteGaugeUint64(w, "go_threads", uint64(numThread))
// Export build details.
WriteMetadataIfNeeded(w, "go_info", "gauge")
fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version())
WriteMetadataIfNeeded(w, "go_info_ext", "gauge")
fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n",
runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT())
}
func writeRuntimeMetrics(w io.Writer) {
samples := make([]runtimemetrics.Sample, len(supportedRuntimeMetrics))
for i, rm := range supportedRuntimeMetrics {
samples[i].Name = rm[0]
}
runtimemetrics.Read(samples)
for i, rm := range supportedRuntimeMetrics {
writeRuntimeMetric(w, rm[1], &samples[i])
}
}
func writeRuntimeMetric(w io.Writer, name string, sample *runtimemetrics.Sample) {
kind := sample.Value.Kind()
switch kind {
case runtimemetrics.KindBad:
panic(fmt.Errorf("BUG: unexpected runtimemetrics.KindBad for sample.Name=%q", sample.Name))
case runtimemetrics.KindUint64:
v := sample.Value.Uint64()
if strings.HasSuffix(name, "_total") {
WriteCounterUint64(w, name, v)
} else {
WriteGaugeUint64(w, name, v)
}
case runtimemetrics.KindFloat64:
v := sample.Value.Float64()
if isCounterName(name) {
WriteCounterFloat64(w, name, v)
} else {
WriteGaugeFloat64(w, name, v)
}
case runtimemetrics.KindFloat64Histogram:
h := sample.Value.Float64Histogram()
writeRuntimeHistogramMetric(w, name, h)
default:
panic(fmt.Errorf("unexpected metric kind=%d", kind))
}
}
func writeRuntimeHistogramMetric(w io.Writer, name string, h *runtimemetrics.Float64Histogram) {
buckets := h.Buckets
counts := h.Counts
if len(buckets) != len(counts)+1 {
panic(fmt.Errorf("the number of buckets must be bigger than the number of counts by 1 in histogram %s; got buckets=%d, counts=%d", name, len(buckets), len(counts)))
}
tailCount := uint64(0)
if strings.HasSuffix(name, "_seconds") {
// Limit the maximum bucket to 1 second, since Go runtime exposes buckets with 10K seconds,
// which have little sense. At the same time such buckets may lead to high cardinality issues
// at the scraper side.
for len(buckets) > 0 && buckets[len(buckets)-1] > 1 {
buckets = buckets[:len(buckets)-1]
tailCount += counts[len(counts)-1]
counts = counts[:len(counts)-1]
}
}
iStep := float64(len(buckets)) / maxRuntimeHistogramBuckets
totalCount := uint64(0)
iNext := 0.0
WriteMetadataIfNeeded(w, name, "histogram")
for i, count := range counts {
totalCount += count
if float64(i) >= iNext {
iNext += iStep
le := buckets[i+1]
if !math.IsInf(le, 1) {
fmt.Fprintf(w, `%s_bucket{le="%g"} %d`+"\n", name, le, totalCount)
}
}
}
totalCount += tailCount
fmt.Fprintf(w, `%s_bucket{le="+Inf"} %d`+"\n", name, totalCount)
}
// Limit the number of buckets for Go runtime histograms in order to prevent from high cardinality issues at scraper side.
const maxRuntimeHistogramBuckets = 30
|
