1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
|
// SPDX-License-Identifier: GPL-2.0
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <libgen.h>
#include "trace-local.h"
#include "version.h"
struct usage_help {
char *name;
char *short_help;
char *long_help;
};
static struct usage_help usage_help[] = {
{
"record",
"record a trace into a trace.dat file",
" %s record [-v][-e event [-f filter]][-p plugin][-F][-d][-D][-o file] \\\n"
" [-q][-s usecs][-O option ][-l func][-g func][-n func] \\\n"
" [-P pid][-N host:port][-t][-r prio][-b size][-B buf] \\\n"
" [--proxy vsock][command ...]\n"
" [-m max][-C clock]\n"
" -e run command with event enabled\n"
" -f filter for previous -e event\n"
" -R trigger for previous -e event\n"
" -p run command with plugin enabled\n"
" -F filter only on the given process\n"
" -P trace the given pid like -F for the command\n"
" -c also trace the children of -F (or -P if kernel supports it)\n"
" -C set the trace clock\n"
" -T do a stacktrace on all events\n"
" -l filter function name\n"
" -g set graph function\n"
" -n do not trace function\n"
" -m max size per CPU in kilobytes\n"
" -M set CPU mask to trace\n"
" -v will negate all -e (disable those events) and -B (delete those instances) after it\n"
" -d disable function tracer when running\n"
" -D Full disable of function tracing (for all users)\n"
" -o data output file [default trace.dat]\n"
" -O option to enable (or disable)\n"
" -r real time priority to run the capture threads\n"
" -s sleep interval between recording (in usecs) [default: 1000]\n"
" -S used with --profile, to enable only events in command line\n"
" -N host:port to connect to (see listen)\n"
" -V cid:port to connect to via vsocket (see listen)\n"
" -t used with -N, forces use of tcp in live trace\n"
" -b change kernel buffersize (in kilobytes per CPU)\n"
" -B create sub buffer and following events will be enabled here\n"
" -k do not reset the buffers after tracing.\n"
" -i do not fail if an event is not found\n"
" -q print no output to the screen\n"
" -G when profiling, set soft and hard irqs as global\n"
" --quiet print no output to the screen\n"
" --temp specify a directory to store the temp files used to create trace.dat\n"
" --subbuf-size to specify the sub-buffer size in kilobytes\n"
" --module filter module name\n"
" --by-comm used with --profile, merge events for related comms\n"
" --profile enable tracing options needed for report --profile\n"
" --func-stack perform a stack trace for function tracer\n"
" (use with caution)\n"
" --max-graph-depth limit function_graph depth\n"
" --cmdlines-size change kernel saved_cmdlines_size\n"
" --no-filter include trace-cmd threads in the trace\n"
" --proc-map save the traced processes address map into the trace.dat file\n"
" --user execute the specified [command ...] as given user\n"
" --tsc2nsec Convert the current clock to nanoseconds, using tsc multiplier and shift from the Linux"
" kernel's perf interface\n"
" --tsync-interval set the loop interval, in ms, for timestamps synchronization with guests:"
" If a negative number is specified, timestamps synchronization is disabled"
" If 0 is specified, no loop is performed - timestamps offset is calculated only twice,"
" at the beginnig and at the end of the trace\n"
" --poll don't block while reading from the trace buffer\n"
" --name used with -A to give the agent a specific name\n"
" --file-version set the desired trace file version\n"
" --compression compress the trace output file, one of these strings can be passed:\n"
" any - auto select the best available compression algorithm\n"
" none - do not compress the trace file\n"
" name - the name of the desired compression algorithms\n"
" available algorithms can be listed with trace-cmd list -c\n"
" --proxy vsocket to reach the agent. Acts the same as -A (for an agent)\n"
" but will send the proxy connection to the agent.\n"
" --daemonize run trace-cmd in the background as a daemon after recording has started.\n"
" creates a pidfile at /var/run/trace-cmd-record.pid with the pid of trace-cmd\n"
" during the recording.\n"
},
{
"set",
"set a ftrace configuration parameter",
" %s set [-v][-e event [-f filter]][-p plugin][-F][-d][-D] \\\n"
" [-q][-s usecs][-O option ][-l func][-g func][-n func] \\\n"
" [-P pid][-b size][-B buf][-m max][-C clock][command ...]\n"
" -e enable event\n"
" -f filter for previous -e event\n"
" -R trigger for previous -e event\n"
" -p set ftrace plugin\n"
" -P set PIDs to be traced\n"
" -c also trace the children of -F (or -P if kernel supports it)\n"
" -C set the trace clock\n"
" -T do a stacktrace on all events\n"
" -l filter function name\n"
" -g set graph function\n"
" -n do not trace function\n"
" -m max size per CPU in kilobytes\n"
" -M set CPU mask to trace\n"
" -v will negate all -e (disable those events) and -B (delete those instances) after it\n"
" -d disable function tracer when running\n"
" -D Full disable of function tracing (for all users)\n"
" -O option to enable (or disable)\n"
" -b change kernel buffersize (in kilobytes per CPU)\n"
" -B create sub buffer and following events will be enabled here\n"
" -i do not fail if an event is not found\n"
" -q print no output to the screen\n"
" --quiet print no output to the screen\n"
" --module filter module name\n"
" --func-stack perform a stack trace for function tracer\n"
" (use with caution)\n"
" --max-graph-depth limit function_graph depth\n"
" --cmdlines-size change kernel saved_cmdlines_size\n"
" --user execute the specified [command ...] as given user\n"
" --fork return immediately if a command is specified\n"
" --verbose 'level' Set the desired log level\n"
},
{
"start",
"start tracing without recording into a file",
" %s start [-e event][-p plugin][-d][-O option ][-P pid]\n"
" Uses same options as record.\n"
" It only enables the tracing and exits\n"
"\n"
" --fork: If a command is specified, then return right after it forks\n"
" --verbose 'level' Set the desired log level\n"
},
{
"extract",
"extract a trace from the kernel",
" %s extract [-p plugin][-O option][-o file][-B buf][-s][-a][-t]\n"
" Uses similar options as record, but only reads an existing trace.\n"
" -s : extract the snapshot instead of the main buffer\n"
" -B : extract a given buffer (more than one may be specified)\n"
" -a : extract all buffers (except top one)\n"
" -t : extract the top level buffer (useful with -B and -a)\n"
" --verbose 'level' Set the desired log level\n"
},
{
"stop",
"stop the kernel from recording trace data",
" %s stop [-B buf [-B buf]..] [-a] [-t]\n"
" Stops the tracer from recording more data.\n"
" Used in conjunction with start\n"
" -B stop a given buffer (more than one may be specified)\n"
" -a stop all buffers (except top one)\n"
" -t stop the top level buffer (useful with -B or -a)\n"
},
{
"restart",
"restart the kernel trace data recording",
" %s restart [-B buf [-B buf]..] [-a] [-t]\n"
" Restarts recording after a trace-cmd stop.\n"
" Used in conjunction with stop\n"
" -B restart a given buffer (more than one may be specified)\n"
" -a restart all buffers (except top one)\n"
" -t restart the top level buffer (useful with -B or -a)\n"
},
{
"show",
"show the contents of the kernel tracing buffer",
" %s show [-p|-s][-c cpu][-B buf][options]\n"
" Basically, this is a cat of the trace file.\n"
" -p read the trace_pipe file instead\n"
" -s read the snapshot file instance\n"
" (Can't have both -p and -s)\n"
" -c just show the file associated with a given CPU\n"
" -B read from a tracing buffer instance.\n"
" -f display the file path that is being dumped\n"
" The following options shows the corresponding file name\n"
" and then exits.\n"
" --tracing_on\n"
" --current_tracer\n"
" --buffer_size (for buffer_size_kb)\n"
" --buffer_total_size (for buffer_total_size_kb)\n"
" --buffer_subbuf_size (for buffer_subbuf_size_kb)\n"
" --buffer_percent (for buffer_percent)\n"
" --ftrace_filter (for set_ftrace_filter)\n"
" --ftrace_notrace (for set_ftrace_notrace)\n"
" --ftrace_pid (for set_ftrace_pid)\n"
" --graph_function (for set_graph_function)\n"
" --graph_notrace (for set_graph_notrace)\n"
" --cpumask (for tracing_cpumask)\n"
},
{
"reset",
"disable all kernel tracing and clear the trace buffers",
" %s reset [-b size][-B buf][-a][-d][-t]\n"
" Disables the tracer (may reset trace file)\n"
" Used in conjunction with start\n"
" -b change the kernel buffer size (in kilobytes per CPU)\n"
" -d delete the previous specified instance\n"
" -B reset the given buffer instance (may specify multiple -B)\n"
" -a reset all instances (except top one)\n"
" -t reset the top level instance (useful with -B or -a)\n"
},
{
"clear",
"clear the trace buffers",
" %s clear [-B buf][-a]\n"
" -B clear the given buffer (may specify multiple -B)\n"
" -a clear all existing buffers, including the top level one\n"
},
{
"report",
"read out the trace stored in a trace.dat file",
" %s report [-i file] [--cpu cpu] [-e][-f][-l][-P][-L][-N][-R][-E]\\\n"
" [-r events][-n events][-F filter][-v][-V[1-6]][-T][-O option]\n"
" [-H [start_system:]start_event,start_match[,pid]/[end_system:]end_event,end_match[,flags]\n"
" [-G]\n"
" -i input file [default trace.dat]\n"
" -e show file endianess\n"
" -f show function mapping list\n"
" -P show printk list\n"
" -E show event files stored\n"
" -F filter to filter output on\n"
" -I filter out events with the HARDIRQ flag set\n"
" -S filter out events with the SOFTIRQ flag set\n"
" -t print out full timestamp. Do not truncate to 6 places.\n"
" -R raw format: ignore print format and only show field data\n"
" -r raw format the events that match the option\n"
" -v will negate all -F after it (Not show matches)\n"
" -T print out the filter strings created and exit\n"
" -V[level] verbose (shows plugins being loaded)\n"
" With optional level (see --verbose numbers)\n"
" -L load only local (~/.trace-cmd/plugins) plugins\n"
" -N do not load any plugins\n"
" -n ignore plugin handlers for events that match the option\n"
" -w show wakeup latencies\n"
" -l show latency format (default with latency tracers)\n"
" -O plugin option -O [plugin:]var[=val]\n"
" --cpu <cpu1,cpu2,...> - filter events according to the given cpu list.\n"
" A range of CPUs can be specified using 'cpuX-cpuY' notation.\n"
" --cpus - List the CPUs that have content in it then exit.\n"
" --first-event - Show the timestamp of the first event for all CPUs.\n"
" --last-event - Show the timestamp of the last event for all CPUs.\n"
" --check-events return whether all event formats can be parsed\n"
" --stat - show the buffer stats that were reported at the end of the record.\n"
" --uname - show uname of the record, if it was saved\n"
" --version - show version used to build the trace-cmd exec that created the file\n"
" --profile report stats on where tasks are blocked and such\n"
" -G when profiling, set soft and hard irqs as global\n"
" -H Allows users to hook two events together for timings\n"
" (used with --profile)\n"
" --by-comm used with --profile, merge events for related comms\n"
" --ts-offset will add amount to timestamp of all events of the\n"
" previous data file.\n"
" --ts2secs HZ, pass in the timestamp frequency (per second)\n"
" to convert the displayed timestamps to seconds\n"
" Affects the previous data file, unless there was no\n"
" previous data file, in which case it becomes default\n"
" --ts-diff Show the delta timestamp between events.\n"
" --ts-check Check to make sure no time stamp on any CPU goes backwards.\n"
" --nodate Ignore the --date processing of trace-cmd record.\n"
" --raw-ts Display raw timestamps, without any corrections.\n"
" --align-ts Display timestamps aligned to the first event.\n"
" --verbose[=level] Set the desired log level\n"
" 0 or none - no error messages\n"
" 1 or crit - only critical messages\n"
" 2 or err - 'crit' and error messages\n"
" 3 or warn - 'err' and warning messages\n"
" 4 or info - 'warn' and informational messages\n"
" 5 or debug - 'info' and debugging messages\n"
" 6 or all - same as debug\n"
},
{
"stream",
"Start tracing and read the output directly",
" %s stream [-e event][-p plugin][-d][-O option ][-P pid]\n"
" Uses same options as record but does not write to files or the network.\n"
" --verbose 'level' Set the desired log level\n"
},
{
"profile",
"Start profiling and read the output directly",
" %s profile [-e event][-p plugin][-d][-O option ][-P pid][-G][-S][-o output]\n"
" [-H [start_system:]start_event,start_match[,pid]/[end_system:]end_event,end_match[,flags]\n\n"
" Uses same options as record --profile.\n"
" -H Allows users to hook two events together for timings\n"
" --verbose 'level' Set the desired log level\n"
},
{
"hist",
"show a histogram of the trace.dat information",
" %s hist [-i file][-P] [file]"
" -P ignore pids (compact all functions)\n"
},
{
"stat",
"show the status of the running tracing (ftrace) system",
" %s stat [-B buf][-t][-o]"
" -B show the status of a instance buffer\n"
" -t show the top level status along with buffer specified by -B\n"
" -o list tracing options\n"
},
{
"split",
"parse a trace.dat file into smaller file(s)",
" %s split [options] -o file [start [end]]\n"
" -o output file to write to (file.1, file.2, etc)\n"
" -s n split file up by n seconds\n"
" -m n split file up by n milliseconds\n"
" -u n split file up by n microseconds\n"
" -e n split file up by n events\n"
" -p n split file up by n pages\n"
" -C n select CPU n\n"
" -B buffer keep buffer in resulting .dat file\n"
" Use -t to promote the buffer to the top instance.\n"
" -t promote preceding buffer to the top instance.\n"
" Must follow -B.\n"
" --top keep top buffer in resulting .dat file.\n"
" -b new name of the top instance. Must follow --top.\n"
" -r repeat from start to end\n"
" -c per cpu, that is -p 2 will be 2 pages for each CPU\n"
" if option is specified, it will split the file\n"
" up starting at start, and ending at end\n"
" start - decimal start time in seconds (ex: 75678.923853)\n"
" if left out, will start at beginning of file\n"
" end - decimal end time in seconds\n"
},
{
"options",
"list the plugin options available for trace-cmd report",
" %s options\n"
},
{
"listen",
"listen on a network socket for trace clients",
" %s listen -p port[-D][-o file][-d dir][-l logfile]\n"
" Creates a socket to listen for clients.\n"
" -p port number to listen on.\n"
" -D run in daemon mode.\n"
" -V listen on a vsocket instead.\n"
" -o file name to use for clients.\n"
" -d directory to store client files.\n"
" -l logfile to write messages to.\n"
" --verbose 'level' Set the desired log level\n"
},
{
"agent",
"listen on a vsocket for trace clients",
" %s agent -p port[-D][-N IP][-P cid]\n"
" Creates a vsocket to listen for clients.\n"
" -N Connect to IP via TCP instead of vsockets\n"
" *** Insecure setting, only use on a trusted network ***\n"
" *** Only use if the client is totally trusted. ***\n"
" -p port number to listen on.\n"
" -D run in daemon mode.\n"
" -P Also act as a proxy server, with a single client denoted\n"
" by a context ID (cid).\n"
" --verbose 'level' Set the desired log level\n"
},
{
"setup-guest",
"create FIFOs for tracing guest VMs",
" %s setup-guest [-c cpus][-p perm][-g group][-a] guest\n"
" -c number of guest virtual CPUs\n"
" -p FIFOs permissions (default: 0660)\n"
" -g FIFOs group owner\n"
" -a Attach FIFOs to guest VM config\n"
},
{
"list",
"list the available events, plugins or options",
" %s list [-e [regex]][-t][-o][-f [regex]]\n"
" -e list available events\n"
" -F show event format\n"
" --full show the print fmt with -F\n"
" -R show event triggers\n"
" -l show event filters\n"
" -t list available tracers\n"
" -o list available options\n"
" -f [regex] list available functions to filter on\n"
" -P list loaded plugin files (by path)\n"
" -O list plugin options\n"
" -B list defined buffer instances\n"
" -C list the defined clocks (and active one)\n"
" -c list the supported trace file compression algorithms\n"
},
{
"restore",
"restore a crashed record",
" %s restore [-c][-o file][-i file] cpu-file [cpu-file ...]\n"
" -c create a partial trace.dat file only\n"
" -o output file\n"
" -i partial trace.dat file for input\n"
},
{
"snapshot",
"take snapshot of running trace",
" %s snapshot [-s][-r][-f][-B buf][-c cpu]\n"
" -s take a snapshot of the trace buffer\n"
" -r reset current snapshot\n"
" -f free the snapshot buffer\n"
" without the above three options, display snapshot\n"
" -c operate on the snapshot buffer for the given CPU\n"
" -B operate on the snapshot buffer for a tracing buffer instance.\n"
},
{
"stack",
"output, enable or disable kernel stack tracing",
" %s stack [--start][--stop][--reset]\n"
" --start enable the stack tracer\n"
" --stop disable the stack tracer\n"
" --reset reset the maximum stack found\n"
" --verbose 'level' Set the desired log level\n"
},
{
"check-events",
"parse trace event formats",
" %s check-events [-N]\n"
" -N do not load any plugins\n"
" --verbose 'level' Set the desired log level\n"
},
{
"dump",
"read out the meta data from a trace file",
" %s dump [options]\n"
" -i input file, default is trace.dat\n"
" -v validate a trace file\n"
" --all print all meta data from a trace file\n"
" --summary print a meta data summary\n"
" --head-page print header page information\n"
" --head-event print header event information\n"
" --ftrace-events print ftrace events format\n"
" --systems print recorded event systems\n"
" --events print format of recorded events\n"
" --kallsyms print information of the mapping of function addresses to the function names\n"
" --printk print trace_printk() format strings\n"
" --cmd-lines print information mapping a PID to a process name\n"
" --options print options\n"
" --flyrecord information of offset and count of recorded events per CPU\n"
" --clock trace clock, saved in the file\n"
" -h, --help show usage information\n"
" --verbose 'level' Set the desired log level\n"
},
{
"attach",
"Attach a host and guest trace.dat file",
" %s attach [options] host_file guest_file vcpu_pid,...\n"
" -s offset,scale,fraction[,timestamp] conversion to sync guest timestamp\n"
" host_file The trace.dat file from the host\n"
" guest_file The trace.dat file from the guest\n"
" vcpu_pid list of process ids from the host that represent the vCPUs of the guest\n"
},
{
"convert",
"convert trace file to different version",
" %s convert [options]\n"
" -i input file, default is trace.dat\n"
" -o output file, mandatory parameter.\n"
" The output file can be specified also as last argument of the command\n"
" --file-version set the desired trace file version\n"
" --compression compress the trace output file, one of these strings can be passed:\n"
" any - auto select the best available compression algorithm\n"
" none - do not compress the trace file\n"
" name - the name of the desired compression algorithms\n"
" available algorithms can be listed with trace-cmd list -c\n"
},
{
"sqlhist",
"Run a SQL like query to create histogram or synthetic events (see man tracefs_sql(3))\n",
"%s sql [-n name][-e][-s][-S fields][-m var][-c var][-T][-t dir][-f file | 'sql-command-line']\n"
" -n name - name of synthetic event 'Anonymous' if left off\n"
" -t dir - use dir instead of /sys/kernel/tracing\n"
" -e - execute the commands to create the synthetic event\n"
" -m - trigger the action when var is a new max.\n"
" -c - trigger the action when var changes.\n"
" -s - used with -m or -c to do a snapshot of the tracing buffer\n"
" -S - used with -m or -c to save fields of the end event (comma deliminated)\n"
" -T - used with -m or -c to do both a snapshot and a trace\n"
" -f file - read sql lines from file otherwise from the command line\n"
" if file is '-' then read from standard input.\n\n"
" See man tracefs_sql(3) for sql-command-line\n"
},
{
NULL, NULL, NULL
}
};
static struct usage_help *find_help(char *cmd)
{
struct usage_help *help;
help = usage_help;
while (help->name) {
if (strcmp(cmd, help->name) == 0)
return help;
help++;
}
return NULL;
}
void usage(char **argv)
{
struct usage_help *help = NULL;
char *arg = argv[0];
char *p;
p = basename(arg);
printf("\n"
"%s version %s (%s)\n\n"
"usage:\n", p, VERSION_STRING, VERSION_GIT);
if (argv[1])
help = find_help(argv[1]);
if (help) {
printf(help->long_help, p);
goto out;
}
printf(" %s [COMMAND] ...\n\n"
" commands:\n", p);
help = usage_help;
while (help->name) {
printf(" %s - %s\n", help->name, help->short_help);
help++;
}
out:
printf("\n");
exit(-1);
}
void trace_usage(int argc, char **argv)
{
usage(argv);
}
|
