1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
|
// SPDX-License-Identifier: GPL-3.0-or-later
// Heavily inspired from proc_net_dev.c
#include "plugin_proc.h"
#define PLUGIN_PROC_MODULE_INFINIBAND_NAME "/sys/class/infiniband"
#define CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND \
"plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_INFINIBAND_NAME
// ib_device::name[IB_DEVICE_NAME_MAX(64)] + "-" + ib_device::phys_port_cnt[u8 = 3 chars]
#define IBNAME_MAX 68
// ----------------------------------------------------------------------------
// infiniband & omnipath standard counters
// I use macro as there's no single file acting as summary, but a lot of different files, so can't use helpers like
// procfile(). Also, omnipath generates other counters, that are not provided by infiniband
#define FOREACH_COUNTER(GEN, ...) \
FOREACH_COUNTER_BYTES(GEN, __VA_ARGS__) \
FOREACH_COUNTER_PACKETS(GEN, __VA_ARGS__) \
FOREACH_COUNTER_ERRORS(GEN, __VA_ARGS__)
#define FOREACH_COUNTER_BYTES(GEN, ...) \
GEN(port_rcv_data, bytes, "Received", 1, __VA_ARGS__) \
GEN(port_xmit_data, bytes, "Sent", -1, __VA_ARGS__)
#define FOREACH_COUNTER_PACKETS(GEN, ...) \
GEN(port_rcv_packets, packets, "Received", 1, __VA_ARGS__) \
GEN(port_xmit_packets, packets, "Sent", -1, __VA_ARGS__) \
GEN(multicast_rcv_packets, packets, "Mcast rcvd", 1, __VA_ARGS__) \
GEN(multicast_xmit_packets, packets, "Mcast sent", -1, __VA_ARGS__) \
GEN(unicast_rcv_packets, packets, "Ucast rcvd", 1, __VA_ARGS__) \
GEN(unicast_xmit_packets, packets, "Ucast sent", -1, __VA_ARGS__)
#define FOREACH_COUNTER_ERRORS(GEN, ...) \
GEN(port_rcv_errors, errors, "Pkts malformated", 1, __VA_ARGS__) \
GEN(port_rcv_constraint_errors, errors, "Pkts rcvd discarded ", 1, __VA_ARGS__) \
GEN(port_xmit_discards, errors, "Pkts sent discarded", 1, __VA_ARGS__) \
GEN(port_xmit_wait, errors, "Tick Wait to send", 1, __VA_ARGS__) \
GEN(VL15_dropped, errors, "Pkts missed resource", 1, __VA_ARGS__) \
GEN(excessive_buffer_overrun_errors, errors, "Buffer overrun", 1, __VA_ARGS__) \
GEN(link_downed, errors, "Link Downed", 1, __VA_ARGS__) \
GEN(link_error_recovery, errors, "Link recovered", 1, __VA_ARGS__) \
GEN(local_link_integrity_errors, errors, "Link integrity err", 1, __VA_ARGS__) \
GEN(symbol_error, errors, "Link minor errors", 1, __VA_ARGS__) \
GEN(port_rcv_remote_physical_errors, errors, "Pkts rcvd with EBP", 1, __VA_ARGS__) \
GEN(port_rcv_switch_relay_errors, errors, "Pkts rcvd discarded by switch", 1, __VA_ARGS__) \
GEN(port_xmit_constraint_errors, errors, "Pkts sent discarded by switch", 1, __VA_ARGS__)
//
// Hardware Counters
//
// IMPORTANT: These are vendor-specific fields.
// If you want to add a new vendor, search this for for 'VENDORS:' keyword and
// add your definition as 'VENDOR-<key>:' where <key> if the string part that
// is shown in /sys/class/infiniband/<key>X_Y
// EG: for Mellanox, shown as mlx0_1, it's 'mlx'
// for Intel, shown as hfi1_1, it's 'hfi'
// VENDORS: List of implemented hardware vendors
#define FOREACH_HWCOUNTER_NAME(GEN, ...) GEN(mlx, __VA_ARGS__)
// VENDOR-MLX: HW Counters for Mellanox ConnectX Devices
#define FOREACH_HWCOUNTER_MLX(GEN, ...) \
FOREACH_HWCOUNTER_MLX_PACKETS(GEN, __VA_ARGS__) \
FOREACH_HWCOUNTER_MLX_ERRORS(GEN, __VA_ARGS__)
#define FOREACH_HWCOUNTER_MLX_PACKETS(GEN, ...) \
GEN(np_cnp_sent, hwpackets, "RoCEv2 Congestion sent", 1, __VA_ARGS__) \
GEN(np_ecn_marked_roce_packets, hwpackets, "RoCEv2 Congestion rcvd", -1, __VA_ARGS__) \
GEN(rp_cnp_handled, hwpackets, "IB Congestion handled", 1, __VA_ARGS__) \
GEN(rx_atomic_requests, hwpackets, "ATOMIC req. rcvd", 1, __VA_ARGS__) \
GEN(rx_dct_connect, hwpackets, "Connection req. rcvd", 1, __VA_ARGS__) \
GEN(rx_read_requests, hwpackets, "Read req. rcvd", 1, __VA_ARGS__) \
GEN(rx_write_requests, hwpackets, "Write req. rcvd", 1, __VA_ARGS__) \
GEN(roce_adp_retrans, hwpackets, "RoCE retrans adaptive", 1, __VA_ARGS__) \
GEN(roce_adp_retrans_to, hwpackets, "RoCE retrans timeout", 1, __VA_ARGS__) \
GEN(roce_slow_restart, hwpackets, "RoCE slow restart", 1, __VA_ARGS__) \
GEN(roce_slow_restart_cnps, hwpackets, "RoCE slow restart congestion", 1, __VA_ARGS__) \
GEN(roce_slow_restart_trans, hwpackets, "RoCE slow restart count", 1, __VA_ARGS__)
#define FOREACH_HWCOUNTER_MLX_ERRORS(GEN, ...) \
GEN(duplicate_request, hwerrors, "Duplicated packets", -1, __VA_ARGS__) \
GEN(implied_nak_seq_err, hwerrors, "Pkt Seq Num gap", 1, __VA_ARGS__) \
GEN(local_ack_timeout_err, hwerrors, "Ack timer expired", 1, __VA_ARGS__) \
GEN(out_of_buffer, hwerrors, "Drop missing buffer", 1, __VA_ARGS__) \
GEN(out_of_sequence, hwerrors, "Drop out of sequence", 1, __VA_ARGS__) \
GEN(packet_seq_err, hwerrors, "NAK sequence rcvd", 1, __VA_ARGS__) \
GEN(req_cqe_error, hwerrors, "CQE err Req", 1, __VA_ARGS__) \
GEN(resp_cqe_error, hwerrors, "CQE err Resp", 1, __VA_ARGS__) \
GEN(req_cqe_flush_error, hwerrors, "CQE Flushed err Req", 1, __VA_ARGS__) \
GEN(resp_cqe_flush_error, hwerrors, "CQE Flushed err Resp", 1, __VA_ARGS__) \
GEN(req_remote_access_errors, hwerrors, "Remote access err Req", 1, __VA_ARGS__) \
GEN(resp_remote_access_errors, hwerrors, "Remote access err Resp", 1, __VA_ARGS__) \
GEN(req_remote_invalid_request, hwerrors, "Remote invalid req", 1, __VA_ARGS__) \
GEN(resp_local_length_error, hwerrors, "Local length err Resp", 1, __VA_ARGS__) \
GEN(rnr_nak_retry_err, hwerrors, "RNR NAK Packets", 1, __VA_ARGS__) \
GEN(rp_cnp_ignored, hwerrors, "CNP Pkts ignored", 1, __VA_ARGS__) \
GEN(rx_icrc_encapsulated, hwerrors, "RoCE ICRC Errors", 1, __VA_ARGS__)
// Common definitions used more than once
#define GEN_RRD_DIM_ADD(NAME, GRP, DESC, DIR, PORT) \
GEN_RRD_DIM_ADD_CUSTOM(NAME, GRP, DESC, DIR, PORT, 1, 1, RRD_ALGORITHM_INCREMENTAL)
#define GEN_RRD_DIM_ADD_CUSTOM(NAME, GRP, DESC, DIR, PORT, MULT, DIV, ALGO) \
PORT->rd_##NAME = rrddim_add(PORT->st_##GRP, DESC, NULL, DIR * MULT, DIV, ALGO);
#define GEN_RRD_DIM_ADD_HW(NAME, GRP, DESC, DIR, PORT, HW) \
HW->rd_##NAME = rrddim_add(PORT->st_##GRP, DESC, NULL, DIR, 1, RRD_ALGORITHM_INCREMENTAL);
#define GEN_RRD_DIM_SETP(NAME, GRP, DESC, DIR, PORT) \
rrddim_set_by_pointer(PORT->st_##GRP, PORT->rd_##NAME, (collected_number)PORT->NAME);
#define GEN_RRD_DIM_SETP_HW(NAME, GRP, DESC, DIR, PORT, HW) \
rrddim_set_by_pointer(PORT->st_##GRP, HW->rd_##NAME, (collected_number)HW->NAME);
// https://community.mellanox.com/s/article/understanding-mlx5-linux-counters-and-status-parameters
// https://community.mellanox.com/s/article/infiniband-port-counters
static struct ibport {
char *name;
char *counters_path;
char *hwcounters_path;
int len;
// flags
int configured;
int enabled;
int discovered;
int do_bytes;
int do_packets;
int do_errors;
int do_hwpackets;
int do_hwerrors;
const char *chart_type_bytes;
const char *chart_type_packets;
const char *chart_type_errors;
const char *chart_type_hwpackets;
const char *chart_type_hwerrors;
const char *chart_id_bytes;
const char *chart_id_packets;
const char *chart_id_errors;
const char *chart_id_hwpackets;
const char *chart_id_hwerrors;
const char *chart_family;
unsigned long priority;
// Port details using drivers/infiniband/core/sysfs.c :: rate_show()
RRDDIM *rd_speed;
uint64_t speed;
uint64_t width;
// Stats from /$device/ports/$portid/counters
// as drivers/infiniband/hw/qib/qib_verbs.h
// All uint64 except vl15_dropped, local_link_integrity_errors, excessive_buffer_overrun_errors uint32
// Will generate 2 elements for each counter:
// - uint64_t to store the value
// - char* to store the filename path
// - RRDDIM* to store the RRD Dimension
#define GEN_DEF_COUNTER(NAME, ...) \
uint64_t NAME; \
char *file_##NAME; \
RRDDIM *rd_##NAME;
FOREACH_COUNTER(GEN_DEF_COUNTER)
// Vendor specific hwcounters from /$device/ports/$portid/hw_counters
// We will generate one struct pointer per vendor to avoid future casting
#define GEN_DEF_HWCOUNTER_PTR(VENDOR, ...) struct ibporthw_##VENDOR *hwcounters_##VENDOR;
FOREACH_HWCOUNTER_NAME(GEN_DEF_HWCOUNTER_PTR)
// Function pointer to the "infiniband_hwcounters_parse_<vendor>" function
void (*hwcounters_parse)(struct ibport *);
void (*hwcounters_dorrd)(struct ibport *);
// charts and dim
RRDSET *st_bytes;
RRDSET *st_packets;
RRDSET *st_errors;
RRDSET *st_hwpackets;
RRDSET *st_hwerrors;
const RRDSETVAR_ACQUIRED *stv_speed;
usec_t speed_last_collected_usec;
struct ibport *next;
} *ibport_root = NULL, *ibport_last_used = NULL;
// VENDORS: reading / calculation functions
#define GEN_DEF_HWCOUNTER(NAME, ...) \
uint64_t NAME; \
char *file_##NAME; \
RRDDIM *rd_##NAME;
#define GEN_DO_HWCOUNTER_READ(NAME, GRP, DESC, DIR, PORT, HW, ...) \
if (HW->file_##NAME) { \
if (read_single_number_file(HW->file_##NAME, (unsigned long long *)&HW->NAME)) { \
error("cannot read iface '%s' hwcounter '" #HW "'", PORT->name); \
HW->file_##NAME = NULL; \
} \
}
// VENDOR-MLX: Mellanox
struct ibporthw_mlx {
FOREACH_HWCOUNTER_MLX(GEN_DEF_HWCOUNTER)
};
void infiniband_hwcounters_parse_mlx(struct ibport *port)
{
if (port->do_hwerrors != CONFIG_BOOLEAN_NO)
FOREACH_HWCOUNTER_MLX_ERRORS(GEN_DO_HWCOUNTER_READ, port, port->hwcounters_mlx)
if (port->do_hwpackets != CONFIG_BOOLEAN_NO)
FOREACH_HWCOUNTER_MLX_PACKETS(GEN_DO_HWCOUNTER_READ, port, port->hwcounters_mlx)
}
void infiniband_hwcounters_dorrd_mlx(struct ibport *port)
{
if (port->do_hwerrors != CONFIG_BOOLEAN_NO) {
FOREACH_HWCOUNTER_MLX_ERRORS(GEN_RRD_DIM_SETP_HW, port, port->hwcounters_mlx)
rrdset_done(port->st_hwerrors);
}
if (port->do_hwpackets != CONFIG_BOOLEAN_NO) {
FOREACH_HWCOUNTER_MLX_PACKETS(GEN_RRD_DIM_SETP_HW, port, port->hwcounters_mlx)
rrdset_done(port->st_hwpackets);
}
}
// ----------------------------------------------------------------------------
static struct ibport *get_ibport(const char *dev, const char *port)
{
struct ibport *p;
char name[IBNAME_MAX + 1];
snprintfz(name, IBNAME_MAX, "%s-%s", dev, port);
// search it, resuming from the last position in sequence
for (p = ibport_last_used; p; p = p->next) {
if (unlikely(!strcmp(name, p->name))) {
ibport_last_used = p->next;
return p;
}
}
// new round, from the beginning to the last position used this time
for (p = ibport_root; p != ibport_last_used; p = p->next) {
if (unlikely(!strcmp(name, p->name))) {
ibport_last_used = p->next;
return p;
}
}
// create a new one
p = callocz(1, sizeof(struct ibport));
p->name = strdupz(name);
p->len = strlen(p->name);
p->chart_type_bytes = strdupz("infiniband_cnt_bytes");
p->chart_type_packets = strdupz("infiniband_cnt_packets");
p->chart_type_errors = strdupz("infiniband_cnt_errors");
p->chart_type_hwpackets = strdupz("infiniband_hwc_packets");
p->chart_type_hwerrors = strdupz("infiniband_hwc_errors");
char buffer[RRD_ID_LENGTH_MAX + 1];
snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cntbytes_%s", p->name);
p->chart_id_bytes = strdupz(buffer);
snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cntpackets_%s", p->name);
p->chart_id_packets = strdupz(buffer);
snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cnterrors_%s", p->name);
p->chart_id_errors = strdupz(buffer);
snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_hwcntpackets_%s", p->name);
p->chart_id_hwpackets = strdupz(buffer);
snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_hwcnterrors_%s", p->name);
p->chart_id_hwerrors = strdupz(buffer);
p->chart_family = strdupz(p->name);
p->priority = NETDATA_CHART_PRIO_INFINIBAND;
// Link current ibport to last one in the list
if (ibport_root) {
struct ibport *t;
for (t = ibport_root; t->next; t = t->next)
;
t->next = p;
} else
ibport_root = p;
return p;
}
int do_sys_class_infiniband(int update_every, usec_t dt)
{
(void)dt;
static SIMPLE_PATTERN *disabled_list = NULL;
static int initialized = 0;
static int enable_new_ports = -1, enable_only_active = CONFIG_BOOLEAN_YES;
static int do_bytes = -1, do_packets = -1, do_errors = -1, do_hwpackets = -1, do_hwerrors = -1;
static char *sys_class_infiniband_dirname = NULL;
static long long int dt_to_refresh_ports = 0, last_refresh_ports_usec = 0;
if (unlikely(enable_new_ports == -1)) {
char dirname[FILENAME_MAX + 1];
snprintfz(dirname, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/class/infiniband");
sys_class_infiniband_dirname =
config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "dirname to monitor", dirname);
do_bytes = config_get_boolean_ondemand(
CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "bandwidth counters", CONFIG_BOOLEAN_YES);
do_packets = config_get_boolean_ondemand(
CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "packets counters", CONFIG_BOOLEAN_YES);
do_errors = config_get_boolean_ondemand(
CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "errors counters", CONFIG_BOOLEAN_YES);
do_hwpackets = config_get_boolean_ondemand(
CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "hardware packets counters", CONFIG_BOOLEAN_AUTO);
do_hwerrors = config_get_boolean_ondemand(
CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "hardware errors counters", CONFIG_BOOLEAN_AUTO);
enable_only_active = config_get_boolean_ondemand(
CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "monitor only active ports", CONFIG_BOOLEAN_AUTO);
disabled_list = simple_pattern_create(
config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "disable by default interfaces matching", ""), NULL,
SIMPLE_PATTERN_EXACT);
dt_to_refresh_ports =
config_get_number(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "refresh ports state every seconds", 30) *
USEC_PER_SEC;
if (dt_to_refresh_ports < 0)
dt_to_refresh_ports = 0;
}
// init listing of /sys/class/infiniband/ (or rediscovery)
if (unlikely(!initialized) || unlikely(last_refresh_ports_usec >= dt_to_refresh_ports)) {
// If folder does not exists, return 1 to disable
DIR *devices_dir = opendir(sys_class_infiniband_dirname);
if (unlikely(!devices_dir))
return 1;
// Work on all device available
struct dirent *dev_dent;
while ((dev_dent = readdir(devices_dir))) {
// Skip special folders
if (!strcmp(dev_dent->d_name, "..") || !strcmp(dev_dent->d_name, "."))
continue;
// /sys/class/infiniband/<dev>/ports
char ports_dirname[FILENAME_MAX + 1];
snprintfz(ports_dirname, FILENAME_MAX, "%s/%s/%s", sys_class_infiniband_dirname, dev_dent->d_name, "ports");
DIR *ports_dir = opendir(ports_dirname);
if (unlikely(!ports_dir))
continue;
struct dirent *port_dent;
while ((port_dent = readdir(ports_dir))) {
// Skip special folders
if (!strcmp(port_dent->d_name, "..") || !strcmp(port_dent->d_name, "."))
continue;
char buffer[FILENAME_MAX + 1];
// Check if counters are available (mandatory)
// /sys/class/infiniband/<device>/ports/<port>/counters
char counters_dirname[FILENAME_MAX + 1];
snprintfz(counters_dirname, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "counters");
DIR *counters_dir = opendir(counters_dirname);
// Standard counters are mandatory
if (!counters_dir)
continue;
closedir(counters_dir);
// Hardware Counters are optional, used later
char hwcounters_dirname[FILENAME_MAX + 1];
snprintfz(
hwcounters_dirname, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "hw_counters");
// Get new or existing ibport
struct ibport *p = get_ibport(dev_dent->d_name, port_dent->d_name);
if (!p)
continue;
// Prepare configuration
if (!p->configured) {
p->configured = 1;
// Enable by default, will be filtered out later
p->enabled = 1;
p->counters_path = strdupz(counters_dirname);
p->hwcounters_path = strdupz(hwcounters_dirname);
snprintfz(buffer, FILENAME_MAX, "plugin:proc:/sys/class/infiniband:%s", p->name);
// Standard counters
p->do_bytes = config_get_boolean_ondemand(buffer, "bytes", do_bytes);
p->do_packets = config_get_boolean_ondemand(buffer, "packets", do_packets);
p->do_errors = config_get_boolean_ondemand(buffer, "errors", do_errors);
// Gen filename allocation and concatenation
#define GEN_DO_COUNTER_NAME(NAME, GRP, DESC, DIR, PORT, ...) \
PORT->file_##NAME = callocz(1, strlen(PORT->counters_path) + sizeof(#NAME) + 3); \
strcat(PORT->file_##NAME, PORT->counters_path); \
strcat(PORT->file_##NAME, "/" #NAME);
FOREACH_COUNTER(GEN_DO_COUNTER_NAME, p)
// Check HW Counters vendor dependent
DIR *hwcounters_dir = opendir(hwcounters_dirname);
if (hwcounters_dir) {
// By default set standard
p->do_hwpackets = config_get_boolean_ondemand(buffer, "hwpackets", do_hwpackets);
p->do_hwerrors = config_get_boolean_ondemand(buffer, "hwerrors", do_hwerrors);
// VENDORS: Set your own
// Allocate the chars to the filenames
#define GEN_DO_HWCOUNTER_NAME(NAME, GRP, DESC, DIR, PORT, HW, ...) \
HW->file_##NAME = callocz(1, strlen(PORT->hwcounters_path) + sizeof(#NAME) + 3); \
strcat(HW->file_##NAME, PORT->hwcounters_path); \
strcat(HW->file_##NAME, "/" #NAME);
// VENDOR-MLX: Mellanox
if (strncmp(dev_dent->d_name, "mlx", 3) == 0) {
// Allocate the vendor specific struct
p->hwcounters_mlx = callocz(1, sizeof(struct ibporthw_mlx));
FOREACH_HWCOUNTER_MLX(GEN_DO_HWCOUNTER_NAME, p, p->hwcounters_mlx)
// Set the function pointer for hwcounter parsing
p->hwcounters_parse = &infiniband_hwcounters_parse_mlx;
p->hwcounters_dorrd = &infiniband_hwcounters_dorrd_mlx;
}
// VENDOR: Unknown
else {
p->do_hwpackets = CONFIG_BOOLEAN_NO;
p->do_hwerrors = CONFIG_BOOLEAN_NO;
}
closedir(hwcounters_dir);
}
}
// Check port state to keep activation
if (enable_only_active) {
snprintfz(buffer, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "state");
unsigned long long active;
// File is "1: DOWN" or "4: ACTIVE", but str2ull will stop on first non-decimal char
read_single_number_file(buffer, &active);
// Want "IB_PORT_ACTIVE" == "4", as defined by drivers/infiniband/core/sysfs.c::state_show()
if (active == 4)
p->enabled = 1;
else
p->enabled = 0;
}
if (p->enabled)
p->enabled = !simple_pattern_matches(disabled_list, p->name);
// Check / Update the link speed & width frm "rate" file
// Sample output: "100 Gb/sec (4X EDR)"
snprintfz(buffer, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "rate");
char buffer_rate[65];
if (read_file(buffer, buffer_rate, 64)) {
error("Unable to read '%s'", buffer);
p->width = 1;
} else {
char *buffer_width = strstr(buffer_rate, "(");
buffer_width++;
// str2ull will stop on first non-decimal value
p->speed = str2ull(buffer_rate);
p->width = str2ull(buffer_width);
}
if (!p->discovered)
info(
"Infiniband card %s port %s at speed %" PRIu64 " width %" PRIu64 "",
dev_dent->d_name,
port_dent->d_name,
p->speed,
p->width);
p->discovered = 1;
}
closedir(ports_dir);
}
closedir(devices_dir);
initialized = 1;
last_refresh_ports_usec = 0;
}
last_refresh_ports_usec += dt;
// Update all ports values
struct ibport *port;
for (port = ibport_root; port; port = port->next) {
if (!port->enabled)
continue;
//
// Read values from system to struct
//
// counter from file and place it in ibport struct
#define GEN_DO_COUNTER_READ(NAME, GRP, DESC, DIR, PORT, ...) \
if (PORT->file_##NAME) { \
if (read_single_number_file(PORT->file_##NAME, (unsigned long long *)&PORT->NAME)) { \
error("cannot read iface '%s' counter '" #NAME "'", PORT->name); \
PORT->file_##NAME = NULL; \
} \
}
// Update charts
if (port->do_bytes != CONFIG_BOOLEAN_NO) {
// Read values from sysfs
FOREACH_COUNTER_BYTES(GEN_DO_COUNTER_READ, port)
// First creation of RRD Set (charts)
if (unlikely(!port->st_bytes)) {
port->st_bytes = rrdset_create_localhost(
"Infiniband",
port->chart_id_bytes,
NULL,
port->chart_family,
"ib.bytes",
"Bandwidth usage",
"kilobits/s",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_INFINIBAND_NAME,
port->priority + 1,
update_every,
RRDSET_TYPE_AREA);
// Create Dimensions
rrdset_flag_set(port->st_bytes, RRDSET_FLAG_DETAIL);
// On this chart, we want to have a KB/s so the dashboard will autoscale it
// The reported values are also per-lane, so we must multiply it by the width
// x4 lanes multiplier as per Documentation/ABI/stable/sysfs-class-infiniband
FOREACH_COUNTER_BYTES(GEN_RRD_DIM_ADD_CUSTOM, port, 4 * 8 * port->width, 1024, RRD_ALGORITHM_INCREMENTAL)
port->stv_speed = rrdsetvar_custom_chart_variable_add_and_acquire(port->st_bytes, "link_speed");
}
// Link read values to dimensions
FOREACH_COUNTER_BYTES(GEN_RRD_DIM_SETP, port)
// For link speed set only variable
rrdsetvar_custom_chart_variable_set(port->st_bytes, port->stv_speed, port->speed);
rrdset_done(port->st_bytes);
}
if (port->do_packets != CONFIG_BOOLEAN_NO) {
// Read values from sysfs
FOREACH_COUNTER_PACKETS(GEN_DO_COUNTER_READ, port)
// First creation of RRD Set (charts)
if (unlikely(!port->st_packets)) {
port->st_packets = rrdset_create_localhost(
"Infiniband",
port->chart_id_packets,
NULL,
port->chart_family,
"ib.packets",
"Packets Statistics",
"packets/s",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_INFINIBAND_NAME,
port->priority + 2,
update_every,
RRDSET_TYPE_AREA);
// Create Dimensions
rrdset_flag_set(port->st_packets, RRDSET_FLAG_DETAIL);
FOREACH_COUNTER_PACKETS(GEN_RRD_DIM_ADD, port)
}
// Link read values to dimensions
FOREACH_COUNTER_PACKETS(GEN_RRD_DIM_SETP, port)
rrdset_done(port->st_packets);
}
if (port->do_errors != CONFIG_BOOLEAN_NO) {
// Read values from sysfs
FOREACH_COUNTER_ERRORS(GEN_DO_COUNTER_READ, port)
// First creation of RRD Set (charts)
if (unlikely(!port->st_errors)) {
port->st_errors = rrdset_create_localhost(
"Infiniband",
port->chart_id_errors,
NULL,
port->chart_family,
"ib.errors",
"Error Counters",
"errors/s",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_INFINIBAND_NAME,
port->priority + 3,
update_every,
RRDSET_TYPE_LINE);
// Create Dimensions
rrdset_flag_set(port->st_errors, RRDSET_FLAG_DETAIL);
FOREACH_COUNTER_ERRORS(GEN_RRD_DIM_ADD, port)
}
// Link read values to dimensions
FOREACH_COUNTER_ERRORS(GEN_RRD_DIM_SETP, port)
rrdset_done(port->st_errors);
}
//
// HW Counters
//
// Call the function for parsing and setting hwcounters
if (port->hwcounters_parse && port->hwcounters_dorrd) {
// Read all values (done by vendor-specific function)
(*port->hwcounters_parse)(port);
if (port->do_hwerrors != CONFIG_BOOLEAN_NO) {
// First creation of RRD Set (charts)
if (unlikely(!port->st_hwerrors)) {
port->st_hwerrors = rrdset_create_localhost(
"Infiniband",
port->chart_id_hwerrors,
NULL,
port->chart_family,
"ib.hwerrors",
"Hardware Errors",
"errors/s",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_INFINIBAND_NAME,
port->priority + 4,
update_every,
RRDSET_TYPE_LINE);
rrdset_flag_set(port->st_hwerrors, RRDSET_FLAG_DETAIL);
// VENDORS: Set your selection
// VENDOR: Mellanox
if (strncmp(port->name, "mlx", 3) == 0) {
FOREACH_HWCOUNTER_MLX_ERRORS(GEN_RRD_DIM_ADD_HW, port, port->hwcounters_mlx)
}
// Unknown vendor, should not happen
else {
error(
"Unmanaged vendor for '%s', do_hwerrors should have been set to no. Please report this bug",
port->name);
port->do_hwerrors = CONFIG_BOOLEAN_NO;
}
}
}
if (port->do_hwpackets != CONFIG_BOOLEAN_NO) {
// First creation of RRD Set (charts)
if (unlikely(!port->st_hwpackets)) {
port->st_hwpackets = rrdset_create_localhost(
"Infiniband",
port->chart_id_hwpackets,
NULL,
port->chart_family,
"ib.hwpackets",
"Hardware Packets Statistics",
"packets/s",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_INFINIBAND_NAME,
port->priority + 5,
update_every,
RRDSET_TYPE_LINE);
rrdset_flag_set(port->st_hwpackets, RRDSET_FLAG_DETAIL);
// VENDORS: Set your selection
// VENDOR: Mellanox
if (strncmp(port->name, "mlx", 3) == 0) {
FOREACH_HWCOUNTER_MLX_PACKETS(GEN_RRD_DIM_ADD_HW, port, port->hwcounters_mlx)
}
// Unknown vendor, should not happen
else {
error(
"Unmanaged vendor for '%s', do_hwpackets should have been set to no. Please report this bug",
port->name);
port->do_hwpackets = CONFIG_BOOLEAN_NO;
}
}
}
// Update values to rrd (done by vendor-specific function)
(*port->hwcounters_dorrd)(port);
}
}
return 0;
}
|
