1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
|
#ifndef FIO_IO_U
#define FIO_IO_U
#include "compiler/compiler.h"
#include "os/os.h"
#include "io_ddir.h"
#include "debug.h"
#include "file.h"
#include "workqueue.h"
#ifdef CONFIG_LIBAIO
#include <libaio.h>
#endif
enum {
IO_U_F_FREE = 1 << 0,
IO_U_F_FLIGHT = 1 << 1,
IO_U_F_NO_FILE_PUT = 1 << 2,
IO_U_F_IN_CUR_DEPTH = 1 << 3,
IO_U_F_BUSY_OK = 1 << 4,
IO_U_F_TRIMMED = 1 << 5,
IO_U_F_BARRIER = 1 << 6,
IO_U_F_VER_LIST = 1 << 7,
IO_U_F_PATTERN_DONE = 1 << 8,
IO_U_F_DEVICE_ERROR = 1 << 9,
IO_U_F_VER_IN_DEV = 1 << 10, /* Verify data in device */
};
/*
* The io unit
*/
struct io_u {
struct timespec start_time;
struct timespec issue_time;
struct fio_file *file;
unsigned int flags;
enum fio_ddir ddir;
/*
* For replay workloads, we may want to account as a different
* IO type than what is being submitted.
*/
enum fio_ddir acct_ddir;
/*
* Write generation
*/
uint64_t numberio;
/*
* IO priority.
*/
unsigned short ioprio;
unsigned short clat_prio_index;
/*
* number of trim ranges for this IO.
*/
unsigned int number_trim;
/*
* Allocated/set buffer and length
*/
unsigned long long buflen;
unsigned long long offset; /* is really ->xfer_offset... */
unsigned long long verify_offset; /* is really ->offset */
void *buf;
/*
* Initial seed for generating the buffer contents
*/
uint64_t rand_seed;
/*
* IO engine state, may be different from above when we get
* partial transfers / residual data counts
*/
void *xfer_buf;
unsigned long long xfer_buflen;
/*
* Parameter related to pre-filled buffers and
* their size to handle variable block sizes.
*/
unsigned long long buf_filled_len;
struct io_piece *ipo;
unsigned long long resid;
unsigned int error;
int inflight_idx;
/*
* io engine private data
*/
union {
unsigned int index;
unsigned int seen;
};
void *engine_data;
union {
struct flist_head verify_list;
struct workqueue_work work;
};
/*
* ZBD mode zbd_queue_io callback: called after engine->queue operation
* to advance a zone write pointer and eventually unlock the I/O zone.
* @q indicates the I/O queue status (busy, queued or completed).
* @success == true means that the I/O operation has been queued or
* completed successfully.
*/
void (*zbd_queue_io)(struct thread_data *td, struct io_u *, int *q);
/*
* ZBD mode zbd_put_io callback: called in after completion of an I/O
* or commit of an async I/O to unlock the I/O target zone.
*/
void (*zbd_put_io)(struct thread_data *td, const struct io_u *);
/*
* Callback for io completion
*/
int (*end_io)(struct thread_data *, struct io_u **);
uint32_t dtype;
uint32_t dspec;
union {
#ifdef CONFIG_LIBAIO
struct iocb iocb;
#endif
#ifdef CONFIG_POSIXAIO
os_aiocb_t aiocb;
#endif
#ifdef FIO_HAVE_SGIO
struct sg_io_hdr hdr;
#endif
#ifdef CONFIG_SOLARISAIO
aio_result_t resultp;
#endif
#ifdef CONFIG_RDMA
struct ibv_mr *mr;
#endif
void *mmap_data;
};
void *pi_attr;
};
/*
* io unit handling
*/
extern struct io_u *__get_io_u(struct thread_data *);
extern struct io_u *get_io_u(struct thread_data *);
extern void put_io_u(struct thread_data *, struct io_u *);
extern void clear_io_u(struct thread_data *, struct io_u *);
extern void requeue_io_u(struct thread_data *, struct io_u **);
extern int __must_check io_u_sync_complete(struct thread_data *, struct io_u *);
extern int __must_check io_u_queued_complete(struct thread_data *, int);
extern void io_u_queued(struct thread_data *, struct io_u *);
extern int io_u_quiesce(struct thread_data *);
extern void io_u_log_error(struct thread_data *, struct io_u *);
extern void io_u_mark_depth(struct thread_data *, unsigned int);
extern void fill_io_buffer(struct thread_data *, void *, unsigned long long, unsigned long long);
extern void io_u_fill_buffer(struct thread_data *td, struct io_u *, unsigned long long, unsigned long long);
void io_u_mark_complete(struct thread_data *, unsigned int);
void io_u_mark_submit(struct thread_data *, unsigned int);
bool queue_full(const struct thread_data *);
int do_io_u_sync(const struct thread_data *, struct io_u *);
int do_io_u_trim(struct thread_data *, struct io_u *);
#ifdef FIO_INC_DEBUG
static inline void dprint_io_u(struct io_u *io_u, const char *p)
{
struct fio_file *f = io_u->file;
if (f)
dprint(FD_IO, "%s: io_u %p: off=0x%llx,len=0x%llx,ddir=%d,file=%s\n",
p, io_u,
(unsigned long long) io_u->offset,
io_u->buflen, io_u->ddir,
f->file_name);
else
dprint(FD_IO, "%s: io_u %p: off=0x%llx,len=0x%llx,ddir=%d\n",
p, io_u,
(unsigned long long) io_u->offset,
io_u->buflen, io_u->ddir);
}
#else
#define dprint_io_u(io_u, p)
#endif
static inline enum fio_ddir acct_ddir(struct io_u *io_u)
{
if (io_u->acct_ddir != -1)
return io_u->acct_ddir;
return io_u->ddir;
}
#define io_u_clear(td, io_u, val) \
td_flags_clear((td), &(io_u->flags), (val))
#define io_u_set(td, io_u, val) \
td_flags_set((td), &(io_u)->flags, (val))
#endif
|
