1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
|
load 5m
http_requests{job="api-server", instance="0", group="production"} 0+10x10
http_requests{job="api-server", instance="1", group="production"} 0+20x10
http_requests{job="api-server", instance="0", group="canary"} 0+30x10
http_requests{job="api-server", instance="1", group="canary"} 0+40x10
http_requests{job="app-server", instance="0", group="production"} 0+50x10
http_requests{job="app-server", instance="1", group="production"} 0+60x10
http_requests{job="app-server", instance="0", group="canary"} 0+70x10
http_requests{job="app-server", instance="1", group="canary"} 0+80x10
load 5m
vector_matching_a{l="x"} 0+1x100
vector_matching_a{l="y"} 0+2x50
vector_matching_b{l="x"} 0+4x25
eval instant at 50m SUM(http_requests) BY (job) - COUNT(http_requests) BY (job)
{job="api-server"} 996
{job="app-server"} 2596
eval instant at 50m 2 - SUM(http_requests) BY (job)
{job="api-server"} -998
{job="app-server"} -2598
eval instant at 50m -http_requests{job="api-server",instance="0",group="production"}
{job="api-server",instance="0",group="production"} -100
eval instant at 50m +http_requests{job="api-server",instance="0",group="production"}
http_requests{job="api-server",instance="0",group="production"} 100
eval instant at 50m - - - SUM(http_requests) BY (job)
{job="api-server"} -1000
{job="app-server"} -2600
eval instant at 50m - - - 1
-1
eval instant at 50m 1000 / SUM(http_requests) BY (job)
{job="api-server"} 1
{job="app-server"} 0.38461538461538464
eval instant at 50m SUM(http_requests) BY (job) - 2
{job="api-server"} 998
{job="app-server"} 2598
eval instant at 50m SUM(http_requests) BY (job) % 3
{job="api-server"} 1
{job="app-server"} 2
eval instant at 50m SUM(http_requests) BY (job) % 0.3
{job="api-server"} 0.1
{job="app-server"} 0.2
eval instant at 50m SUM(http_requests) BY (job) ^ 2
{job="api-server"} 1000000
{job="app-server"} 6760000
eval instant at 50m SUM(http_requests) BY (job) % 3 ^ 2
{job="api-server"} 1
{job="app-server"} 8
eval instant at 50m SUM(http_requests) BY (job) % 2 ^ (3 ^ 2)
{job="api-server"} 488
{job="app-server"} 40
eval instant at 50m SUM(http_requests) BY (job) % 2 ^ 3 ^ 2
{job="api-server"} 488
{job="app-server"} 40
eval instant at 50m SUM(http_requests) BY (job) % 2 ^ 3 ^ 2 ^ 2
{job="api-server"} 1000
{job="app-server"} 2600
eval instant at 50m COUNT(http_requests) BY (job) ^ COUNT(http_requests) BY (job)
{job="api-server"} 256
{job="app-server"} 256
eval instant at 50m SUM(http_requests) BY (job) / 0
{job="api-server"} +Inf
{job="app-server"} +Inf
eval instant at 50m SUM(http_requests) BY (job) + SUM(http_requests) BY (job)
{job="api-server"} 2000
{job="app-server"} 5200
eval instant at 50m http_requests{job="api-server", group="canary"}
http_requests{group="canary", instance="0", job="api-server"} 300
http_requests{group="canary", instance="1", job="api-server"} 400
eval instant at 50m http_requests{job="api-server", group="canary"} + rate(http_requests{job="api-server"}[5m]) * 5 * 60
{group="canary", instance="0", job="api-server"} 330
{group="canary", instance="1", job="api-server"} 440
eval instant at 50m rate(http_requests[25m]) * 25 * 60
{group="canary", instance="0", job="api-server"} 150
{group="canary", instance="0", job="app-server"} 350
{group="canary", instance="1", job="api-server"} 200
{group="canary", instance="1", job="app-server"} 400
{group="production", instance="0", job="api-server"} 50
{group="production", instance="0", job="app-server"} 249.99999999999997
{group="production", instance="1", job="api-server"} 100
{group="production", instance="1", job="app-server"} 300
eval instant at 50m http_requests{group="canary"} and http_requests{instance="0"}
http_requests{group="canary", instance="0", job="api-server"} 300
http_requests{group="canary", instance="0", job="app-server"} 700
eval instant at 50m (http_requests{group="canary"} + 1) and http_requests{instance="0"}
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
eval instant at 50m (http_requests{group="canary"} + 1) and on(instance, job) http_requests{instance="0", group="production"}
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
eval instant at 50m (http_requests{group="canary"} + 1) and on(instance) http_requests{instance="0", group="production"}
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
eval instant at 50m (http_requests{group="canary"} + 1) and ignoring(group) http_requests{instance="0", group="production"}
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
eval instant at 50m (http_requests{group="canary"} + 1) and ignoring(group, job) http_requests{instance="0", group="production"}
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
eval instant at 50m http_requests{group="canary"} or http_requests{group="production"}
http_requests{group="canary", instance="0", job="api-server"} 300
http_requests{group="canary", instance="0", job="app-server"} 700
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="canary", instance="1", job="app-server"} 800
http_requests{group="production", instance="0", job="api-server"} 100
http_requests{group="production", instance="0", job="app-server"} 500
http_requests{group="production", instance="1", job="api-server"} 200
http_requests{group="production", instance="1", job="app-server"} 600
# On overlap the rhs samples must be dropped.
eval instant at 50m (http_requests{group="canary"} + 1) or http_requests{instance="1"}
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
{group="canary", instance="1", job="api-server"} 401
{group="canary", instance="1", job="app-server"} 801
http_requests{group="production", instance="1", job="api-server"} 200
http_requests{group="production", instance="1", job="app-server"} 600
# Matching only on instance excludes everything that has instance=0/1 but includes
# entries without the instance label.
eval instant at 50m (http_requests{group="canary"} + 1) or on(instance) (http_requests or cpu_count or vector_matching_a)
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
{group="canary", instance="1", job="api-server"} 401
{group="canary", instance="1", job="app-server"} 801
vector_matching_a{l="x"} 10
vector_matching_a{l="y"} 20
eval instant at 50m (http_requests{group="canary"} + 1) or ignoring(l, group, job) (http_requests or cpu_count or vector_matching_a)
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
{group="canary", instance="1", job="api-server"} 401
{group="canary", instance="1", job="app-server"} 801
vector_matching_a{l="x"} 10
vector_matching_a{l="y"} 20
eval instant at 50m http_requests{group="canary"} unless http_requests{instance="0"}
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="canary", instance="1", job="app-server"} 800
eval instant at 50m http_requests{group="canary"} unless on(job) http_requests{instance="0"}
eval instant at 50m http_requests{group="canary"} unless on(job, instance) http_requests{instance="0"}
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="canary", instance="1", job="app-server"} 800
eval instant at 50m http_requests{group="canary"} / on(instance,job) http_requests{group="production"}
{instance="0", job="api-server"} 3
{instance="0", job="app-server"} 1.4
{instance="1", job="api-server"} 2
{instance="1", job="app-server"} 1.3333333333333333
eval instant at 50m http_requests{group="canary"} unless ignoring(group, instance) http_requests{instance="0"}
eval instant at 50m http_requests{group="canary"} unless ignoring(group) http_requests{instance="0"}
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="canary", instance="1", job="app-server"} 800
eval instant at 50m http_requests{group="canary"} / ignoring(group) http_requests{group="production"}
{instance="0", job="api-server"} 3
{instance="0", job="app-server"} 1.4
{instance="1", job="api-server"} 2
{instance="1", job="app-server"} 1.3333333333333333
# https://github.com/prometheus/prometheus/issues/1489
eval instant at 50m http_requests AND ON (dummy) vector(1)
http_requests{group="canary", instance="0", job="api-server"} 300
http_requests{group="canary", instance="0", job="app-server"} 700
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="canary", instance="1", job="app-server"} 800
http_requests{group="production", instance="0", job="api-server"} 100
http_requests{group="production", instance="0", job="app-server"} 500
http_requests{group="production", instance="1", job="api-server"} 200
http_requests{group="production", instance="1", job="app-server"} 600
eval instant at 50m http_requests AND IGNORING (group, instance, job) vector(1)
http_requests{group="canary", instance="0", job="api-server"} 300
http_requests{group="canary", instance="0", job="app-server"} 700
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="canary", instance="1", job="app-server"} 800
http_requests{group="production", instance="0", job="api-server"} 100
http_requests{group="production", instance="0", job="app-server"} 500
http_requests{group="production", instance="1", job="api-server"} 200
http_requests{group="production", instance="1", job="app-server"} 600
# Comparisons.
eval instant at 50m SUM(http_requests) BY (job) > 1000
{job="app-server"} 2600
eval instant at 50m 1000 < SUM(http_requests) BY (job)
{job="app-server"} 1000
eval instant at 50m SUM(http_requests) BY (job) <= 1000
{job="api-server"} 1000
eval instant at 50m SUM(http_requests) BY (job) != 1000
{job="app-server"} 2600
eval instant at 50m SUM(http_requests) BY (job) == 1000
{job="api-server"} 1000
eval instant at 50m SUM(http_requests) BY (job) == bool 1000
{job="api-server"} 1
{job="app-server"} 0
eval instant at 50m SUM(http_requests) BY (job) == bool SUM(http_requests) BY (job)
{job="api-server"} 1
{job="app-server"} 1
eval instant at 50m SUM(http_requests) BY (job) != bool SUM(http_requests) BY (job)
{job="api-server"} 0
{job="app-server"} 0
eval instant at 50m 0 == bool 1
0
eval instant at 50m 1 == bool 1
1
eval instant at 50m http_requests{job="api-server", instance="0", group="production"} == bool 100
{job="api-server", instance="0", group="production"} 1
# group_left/group_right.
clear
load 5m
node_var{instance="abc",job="node"} 2
node_role{instance="abc",job="node",role="prometheus"} 1
load 5m
node_cpu{instance="abc",job="node",mode="idle"} 3
node_cpu{instance="abc",job="node",mode="user"} 1
node_cpu{instance="def",job="node",mode="idle"} 8
node_cpu{instance="def",job="node",mode="user"} 2
load 5m
random{foo="bar"} 1
load 5m
threshold{instance="abc",job="node",target="a@b.com"} 0
# Copy machine role to node variable.
eval instant at 5m node_role * on (instance) group_right (role) node_var
{instance="abc",job="node",role="prometheus"} 2
eval instant at 5m node_var * on (instance) group_left (role) node_role
{instance="abc",job="node",role="prometheus"} 2
eval instant at 5m node_var * ignoring (role) group_left (role) node_role
{instance="abc",job="node",role="prometheus"} 2
eval instant at 5m node_role * ignoring (role) group_right (role) node_var
{instance="abc",job="node",role="prometheus"} 2
# Copy machine role to node variable with instrumentation labels.
eval instant at 5m node_cpu * ignoring (role, mode) group_left (role) node_role
{instance="abc",job="node",mode="idle",role="prometheus"} 3
{instance="abc",job="node",mode="user",role="prometheus"} 1
eval instant at 5m node_cpu * on (instance) group_left (role) node_role
{instance="abc",job="node",mode="idle",role="prometheus"} 3
{instance="abc",job="node",mode="user",role="prometheus"} 1
# Ratio of total.
eval instant at 5m node_cpu / on (instance) group_left sum by (instance,job)(node_cpu)
{instance="abc",job="node",mode="idle"} .75
{instance="abc",job="node",mode="user"} .25
{instance="def",job="node",mode="idle"} .80
{instance="def",job="node",mode="user"} .20
eval instant at 5m sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu)
{job="node",mode="idle"} 0.7857142857142857
{job="node",mode="user"} 0.21428571428571427
eval instant at 5m sum(sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu))
{} 1.0
eval instant at 5m node_cpu / ignoring (mode) group_left sum without (mode)(node_cpu)
{instance="abc",job="node",mode="idle"} .75
{instance="abc",job="node",mode="user"} .25
{instance="def",job="node",mode="idle"} .80
{instance="def",job="node",mode="user"} .20
eval instant at 5m node_cpu / ignoring (mode) group_left(dummy) sum without (mode)(node_cpu)
{instance="abc",job="node",mode="idle"} .75
{instance="abc",job="node",mode="user"} .25
{instance="def",job="node",mode="idle"} .80
{instance="def",job="node",mode="user"} .20
eval instant at 5m sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu)
{job="node",mode="idle"} 0.7857142857142857
{job="node",mode="user"} 0.21428571428571427
eval instant at 5m sum(sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu))
{} 1.0
# Copy over label from metric with no matching labels, without having to list cross-job target labels ('job' here).
eval instant at 5m node_cpu + on(dummy) group_left(foo) random*0
{instance="abc",job="node",mode="idle",foo="bar"} 3
{instance="abc",job="node",mode="user",foo="bar"} 1
{instance="def",job="node",mode="idle",foo="bar"} 8
{instance="def",job="node",mode="user",foo="bar"} 2
# Use threshold from metric, and copy over target.
eval instant at 5m node_cpu > on(job, instance) group_left(target) threshold
node_cpu{instance="abc",job="node",mode="idle",target="a@b.com"} 3
node_cpu{instance="abc",job="node",mode="user",target="a@b.com"} 1
# Use threshold from metric, and a default (1) if it's not present.
eval instant at 5m node_cpu > on(job, instance) group_left(target) (threshold or on (job, instance) (sum by (job, instance)(node_cpu) * 0 + 1))
node_cpu{instance="abc",job="node",mode="idle",target="a@b.com"} 3
node_cpu{instance="abc",job="node",mode="user",target="a@b.com"} 1
node_cpu{instance="def",job="node",mode="idle"} 8
node_cpu{instance="def",job="node",mode="user"} 2
clear
load 5m
random{foo="bar"} 2
metricA{baz="meh"} 3
metricB{baz="meh"} 4
# On with no labels, for metrics with no common labels.
eval instant at 5m random + on() metricA
{} 5
# Ignoring with no labels is the same as no ignoring.
eval instant at 5m metricA + ignoring() metricB
{baz="meh"} 7
eval instant at 5m metricA + metricB
{baz="meh"} 7
clear
# Test duplicate labelset in promql output.
load 5m
testmetric1{src="a",dst="b"} 0
testmetric2{src="a",dst="b"} 1
eval_fail instant at 0m -{__name__=~'testmetric1|testmetric2'}
|
