1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
|
#ifndef FIO_STAT_H
#define FIO_STAT_H
struct group_run_stats {
uint64_t max_run[2], min_run[2];
uint64_t max_bw[2], min_bw[2];
uint64_t io_kb[2];
uint64_t agg[2];
uint32_t kb_base;
uint32_t groupid;
};
/*
* How many depth levels to log
*/
#define FIO_IO_U_MAP_NR 7
#define FIO_IO_U_LAT_U_NR 10
#define FIO_IO_U_LAT_M_NR 12
/*
* Aggregate clat samples to report percentile(s) of them.
*
* EXECUTIVE SUMMARY
*
* FIO_IO_U_PLAT_BITS determines the maximum statistical error on the
* value of resulting percentiles. The error will be approximately
* 1/2^(FIO_IO_U_PLAT_BITS+1) of the value.
*
* FIO_IO_U_PLAT_GROUP_NR and FIO_IO_U_PLAT_BITS determine the maximum
* range being tracked for latency samples. The maximum value tracked
* accurately will be 2^(GROUP_NR + PLAT_BITS -1) microseconds.
*
* FIO_IO_U_PLAT_GROUP_NR and FIO_IO_U_PLAT_BITS determine the memory
* requirement of storing those aggregate counts. The memory used will
* be (FIO_IO_U_PLAT_GROUP_NR * 2^FIO_IO_U_PLAT_BITS) * sizeof(int)
* bytes.
*
* FIO_IO_U_PLAT_NR is the total number of buckets.
*
* DETAILS
*
* Suppose the clat varies from 0 to 999 (usec), the straightforward
* method is to keep an array of (999 + 1) buckets, in which a counter
* keeps the count of samples which fall in the bucket, e.g.,
* {[0],[1],...,[999]}. However this consumes a huge amount of space,
* and can be avoided if an approximation is acceptable.
*
* One such method is to let the range of the bucket to be greater
* than one. This method has low accuracy when the value is small. For
* example, let the buckets be {[0,99],[100,199],...,[900,999]}, and
* the represented value of each bucket be the mean of the range. Then
* a value 0 has an round-off error of 49.5. To improve on this, we
* use buckets with non-uniform ranges, while bounding the error of
* each bucket within a ratio of the sample value. A simple example
* would be when error_bound = 0.005, buckets are {
* {[0],[1],...,[99]}, {[100,101],[102,103],...,[198,199]},..,
* {[900,909],[910,919]...} }. The total range is partitioned into
* groups with different ranges, then buckets with uniform ranges. An
* upper bound of the error is (range_of_bucket/2)/value_of_bucket
*
* For better efficiency, we implement this using base two. We group
* samples by their Most Significant Bit (MSB), extract the next M bit
* of them as an index within the group, and discard the rest of the
* bits.
*
* E.g., assume a sample 'x' whose MSB is bit n (starting from bit 0),
* and use M bit for indexing
*
* | n | M bits | bit (n-M-1) ... bit 0 |
*
* Because x is at least 2^n, and bit 0 to bit (n-M-1) is at most
* (2^(n-M) - 1), discarding bit 0 to (n-M-1) makes the round-off
* error
*
* 2^(n-M)-1 2^(n-M) 1
* e <= --------- <= ------- = ---
* 2^n 2^n 2^M
*
* Furthermore, we use "mean" of the range to represent the bucket,
* the error e can be lowered by half to 1 / 2^(M+1). By using M bits
* as the index, each group must contains 2^M buckets.
*
* E.g. Let M (FIO_IO_U_PLAT_BITS) be 6
* Error bound is 1/2^(6+1) = 0.0078125 (< 1%)
*
* Group MSB #discarded range of #buckets
* error_bits value
* ----------------------------------------------------------------
* 0* 0~5 0 [0,63] 64
* 1* 6 0 [64,127] 64
* 2 7 1 [128,255] 64
* 3 8 2 [256,511] 64
* 4 9 3 [512,1023] 64
* ... ... ... [...,...] ...
* 18 23 17 [8838608,+inf]** 64
*
* * Special cases: when n < (M-1) or when n == (M-1), in both cases,
* the value cannot be rounded off. Use all bits of the sample as
* index.
*
* ** If a sample's MSB is greater than 23, it will be counted as 23.
*/
#define FIO_IO_U_PLAT_BITS 6
#define FIO_IO_U_PLAT_VAL (1 << FIO_IO_U_PLAT_BITS)
#define FIO_IO_U_PLAT_GROUP_NR 19
#define FIO_IO_U_PLAT_NR (FIO_IO_U_PLAT_GROUP_NR * FIO_IO_U_PLAT_VAL)
#define FIO_IO_U_LIST_MAX_LEN 20 /* The size of the default and user-specified
list of percentiles */
#define MAX_PATTERN_SIZE 512
#define FIO_JOBNAME_SIZE 128
#define FIO_VERROR_SIZE 128
struct thread_stat {
char name[FIO_JOBNAME_SIZE];
char verror[FIO_VERROR_SIZE];
uint32_t error;
uint32_t groupid;
uint32_t pid;
char description[FIO_JOBNAME_SIZE];
uint32_t members;
/*
* bandwidth and latency stats
*/
struct io_stat clat_stat[2]; /* completion latency */
struct io_stat slat_stat[2]; /* submission latency */
struct io_stat lat_stat[2]; /* total latency */
struct io_stat bw_stat[2]; /* bandwidth stats */
struct io_stat iops_stat[2]; /* IOPS stats */
/*
* fio system usage accounting
*/
uint64_t usr_time;
uint64_t sys_time;
uint64_t ctx;
uint64_t minf, majf;
/*
* IO depth and latency stats
*/
uint64_t clat_percentiles;
fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN];
uint32_t io_u_map[FIO_IO_U_MAP_NR];
uint32_t io_u_submit[FIO_IO_U_MAP_NR];
uint32_t io_u_complete[FIO_IO_U_MAP_NR];
uint32_t io_u_lat_u[FIO_IO_U_LAT_U_NR];
uint32_t io_u_lat_m[FIO_IO_U_LAT_M_NR];
uint32_t io_u_plat[2][FIO_IO_U_PLAT_NR];
uint64_t total_io_u[3];
uint64_t short_io_u[3];
uint64_t total_submit;
uint64_t total_complete;
uint64_t io_bytes[2];
uint64_t runtime[2];
uint64_t total_run_time;
/*
* IO Error related stats
*/
uint16_t continue_on_error;
uint64_t total_err_count;
uint32_t first_error;
uint32_t kb_base;
};
struct jobs_eta {
uint32_t nr_running;
uint32_t nr_ramp;
uint32_t nr_pending;
uint32_t files_open;
uint32_t m_rate, t_rate;
uint32_t m_iops, t_iops;
uint32_t rate[2];
uint32_t iops[2];
uint64_t elapsed_sec;
uint64_t eta_sec;
uint32_t is_pow2;
/*
* Network 'copy' of run_str[]
*/
uint32_t nr_threads;
uint8_t run_str[0];
};
extern void show_thread_status(struct thread_stat *ts, struct group_run_stats *rs);
extern void show_group_stats(struct group_run_stats *rs);
extern int calc_thread_status(struct jobs_eta *je, int force);
extern void display_thread_status(struct jobs_eta *je);
extern void show_run_stats(void);
extern void show_running_run_stats(void);
extern void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src, int nr);
extern void sum_group_stats(struct group_run_stats *dst, struct group_run_stats *src);
extern void init_thread_stat(struct thread_stat *ts);
extern void init_group_run_stat(struct group_run_stats *gs);
#endif
|
