File: fio.h

package info (click to toggle)
fio 3.12-2
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, sid
  • size: 4,488 kB
  • sloc: ansic: 65,165; sh: 3,284; python: 1,978; makefile: 657; yacc: 204; lex: 184
file content (863 lines) | stat: -rw-r--r-- 20,002 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
#ifndef FIO_H
#define FIO_H

#include <sched.h>
#include <limits.h>
#include <pthread.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <errno.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <inttypes.h>
#include <assert.h>

#include "compiler/compiler.h"
#include "thread_options.h"
#include "flist.h"
#include "fifo.h"
#include "arch/arch.h"
#include "os/os.h"
#include "log.h"
#include "debug.h"
#include "file.h"
#include "io_ddir.h"
#include "ioengines.h"
#include "iolog.h"
#include "helpers.h"
#include "minmax.h"
#include "options.h"
#include "profile.h"
#include "fio_time.h"
#include "gettime.h"
#include "oslib/getopt.h"
#include "lib/rand.h"
#include "lib/rbtree.h"
#include "lib/num2str.h"
#include "client.h"
#include "server.h"
#include "stat.h"
#include "flow.h"
#include "io_u.h"
#include "io_u_queue.h"
#include "workqueue.h"
#include "steadystate.h"
#include "lib/nowarn_snprintf.h"

#ifdef CONFIG_SOLARISAIO
#include <sys/asynch.h>
#endif

#ifdef CONFIG_LIBNUMA
#include <linux/mempolicy.h>
#include <numa.h>

/*
 * "local" is pseudo-policy
 */
#ifndef MPOL_LOCAL
#define MPOL_LOCAL 4
#endif
#endif

#ifdef CONFIG_CUDA
#include <cuda.h>
#endif

struct fio_sem;

/*
 * offset generator types
 */
enum {
	RW_SEQ_SEQ	= 0,
	RW_SEQ_IDENT,
};

enum {
	__TD_F_VER_BACKLOG	= 0,
	__TD_F_TRIM_BACKLOG,
	__TD_F_READ_IOLOG,
	__TD_F_REFILL_BUFFERS,
	__TD_F_SCRAMBLE_BUFFERS,
	__TD_F_DO_VERIFY,
	__TD_F_PROFILE_OPS,
	__TD_F_COMPRESS,
	__TD_F_COMPRESS_LOG,
	__TD_F_VSTATE_SAVED,
	__TD_F_NEED_LOCK,
	__TD_F_CHILD,
	__TD_F_NO_PROGRESS,
	__TD_F_REGROW_LOGS,
	__TD_F_MMAP_KEEP,
	__TD_F_DIRS_CREATED,
	__TD_F_CHECK_RATE,
	__TD_F_LAST,		/* not a real bit, keep last */
};

enum {
	TD_F_VER_BACKLOG	= 1U << __TD_F_VER_BACKLOG,
	TD_F_TRIM_BACKLOG	= 1U << __TD_F_TRIM_BACKLOG,
	TD_F_READ_IOLOG		= 1U << __TD_F_READ_IOLOG,
	TD_F_REFILL_BUFFERS	= 1U << __TD_F_REFILL_BUFFERS,
	TD_F_SCRAMBLE_BUFFERS	= 1U << __TD_F_SCRAMBLE_BUFFERS,
	TD_F_DO_VERIFY		= 1U << __TD_F_DO_VERIFY,
	TD_F_PROFILE_OPS	= 1U << __TD_F_PROFILE_OPS,
	TD_F_COMPRESS		= 1U << __TD_F_COMPRESS,
	TD_F_COMPRESS_LOG	= 1U << __TD_F_COMPRESS_LOG,
	TD_F_VSTATE_SAVED	= 1U << __TD_F_VSTATE_SAVED,
	TD_F_NEED_LOCK		= 1U << __TD_F_NEED_LOCK,
	TD_F_CHILD		= 1U << __TD_F_CHILD,
	TD_F_NO_PROGRESS        = 1U << __TD_F_NO_PROGRESS,
	TD_F_REGROW_LOGS	= 1U << __TD_F_REGROW_LOGS,
	TD_F_MMAP_KEEP		= 1U << __TD_F_MMAP_KEEP,
	TD_F_DIRS_CREATED	= 1U << __TD_F_DIRS_CREATED,
	TD_F_CHECK_RATE		= 1U << __TD_F_CHECK_RATE,
};

enum {
	FIO_RAND_BS_OFF		= 0,
	FIO_RAND_BS1_OFF,
	FIO_RAND_BS2_OFF,
	FIO_RAND_VER_OFF,
	FIO_RAND_MIX_OFF,
	FIO_RAND_FILE_OFF,
	FIO_RAND_BLOCK_OFF,
	FIO_RAND_FILE_SIZE_OFF,
	FIO_RAND_TRIM_OFF,
	FIO_RAND_BUF_OFF,
	FIO_RAND_SEQ_RAND_READ_OFF,
	FIO_RAND_SEQ_RAND_WRITE_OFF,
	FIO_RAND_SEQ_RAND_TRIM_OFF,
	FIO_RAND_START_DELAY,
	FIO_DEDUPE_OFF,
	FIO_RAND_POISSON_OFF,
	FIO_RAND_ZONE_OFF,
	FIO_RAND_POISSON2_OFF,
	FIO_RAND_POISSON3_OFF,
	FIO_RAND_NR_OFFS,
};

enum {
	IO_MODE_INLINE = 0,
	IO_MODE_OFFLOAD = 1,

	RATE_PROCESS_LINEAR = 0,
	RATE_PROCESS_POISSON = 1,
};

enum {
	F_ADV_NONE = 0,
	F_ADV_TYPE,
	F_ADV_RANDOM,
	F_ADV_SEQUENTIAL,
};

/*
 * Per-thread/process specific data. Only used for the network client
 * for now.
 */
void sk_out_assign(struct sk_out *);
void sk_out_drop(void);

struct zone_split_index {
	uint8_t size_perc;
	uint8_t size_perc_prev;
	uint64_t size;
	uint64_t size_prev;
};

#define FIO_MAX_OPEN_ZBD_ZONES 128

/*
 * This describes a single thread/process executing a fio job.
 */
struct thread_data {
	struct flist_head opt_list;
	unsigned long flags;
	struct thread_options o;
	void *eo;
	pthread_t thread;
	unsigned int thread_number;
	unsigned int subjob_number;
	unsigned int groupid;
	struct thread_stat ts __attribute__ ((aligned(8)));

	int client_type;

	struct io_log *slat_log;
	struct io_log *clat_log;
	struct io_log *clat_hist_log;
	struct io_log *lat_log;
	struct io_log *bw_log;
	struct io_log *iops_log;

	struct workqueue log_compress_wq;

	struct thread_data *parent;

	uint64_t stat_io_bytes[DDIR_RWDIR_CNT];
	struct timespec bw_sample_time;

	uint64_t stat_io_blocks[DDIR_RWDIR_CNT];
	struct timespec iops_sample_time;

	volatile int update_rusage;
	struct fio_sem *rusage_sem;
	struct rusage ru_start;
	struct rusage ru_end;

	struct fio_file **files;
	unsigned char *file_locks;
	unsigned int files_size;
	unsigned int files_index;
	unsigned int nr_open_files;
	unsigned int nr_done_files;
	union {
		unsigned int next_file;
		struct frand_state next_file_state;
	};
	union {
		struct zipf_state next_file_zipf;
		struct gauss_state next_file_gauss;
	};
	union {
		double zipf_theta;
		double pareto_h;
		double gauss_dev;
	};
	int error;
	int sig;
	int done;
	int stop_io;
	pid_t pid;
	char *orig_buffer;
	size_t orig_buffer_size;
	volatile int runstate;
	volatile bool terminate;
	bool last_was_sync;
	enum fio_ddir last_ddir;

	int mmapfd;

	void *iolog_buf;
	FILE *iolog_f;

	unsigned long rand_seeds[FIO_RAND_NR_OFFS];

	struct frand_state bsrange_state[DDIR_RWDIR_CNT];
	struct frand_state verify_state;
	struct frand_state trim_state;
	struct frand_state delay_state;

	struct frand_state buf_state;
	struct frand_state buf_state_prev;
	struct frand_state dedupe_state;
	struct frand_state zone_state;

	struct zone_split_index **zone_state_index;

	unsigned int verify_batch;
	unsigned int trim_batch;

	struct thread_io_list *vstate;

	int shm_id;

	/*
	 * IO engine hooks, contains everything needed to submit an io_u
	 * to any of the available IO engines.
	 */
	struct ioengine_ops *io_ops;
	int io_ops_init;

	/*
	 * IO engine private data and dlhandle.
	 */
	void *io_ops_data;
	void *io_ops_dlhandle;

	/*
	 * Queue depth of io_u's that fio MIGHT do
	 */
	unsigned int cur_depth;

	/*
	 * io_u's about to be committed
	 */
	unsigned int io_u_queued;

	/*
	 * io_u's submitted but not completed yet
	 */
	unsigned int io_u_in_flight;

	/*
	 * List of free and busy io_u's
	 */
	struct io_u_ring io_u_requeues;
	struct io_u_queue io_u_freelist;
	struct io_u_queue io_u_all;
	pthread_mutex_t io_u_lock;
	pthread_cond_t free_cond;

	/*
	 * async verify offload
	 */
	struct flist_head verify_list;
	pthread_t *verify_threads;
	unsigned int nr_verify_threads;
	pthread_cond_t verify_cond;
	int verify_thread_exit;

	/*
	 * Rate state
	 */
	uint64_t rate_bps[DDIR_RWDIR_CNT];
	uint64_t rate_next_io_time[DDIR_RWDIR_CNT];
	unsigned long rate_bytes[DDIR_RWDIR_CNT];
	unsigned long rate_blocks[DDIR_RWDIR_CNT];
	unsigned long long rate_io_issue_bytes[DDIR_RWDIR_CNT];
	struct timespec lastrate[DDIR_RWDIR_CNT];
	int64_t last_usec[DDIR_RWDIR_CNT];
	struct frand_state poisson_state[DDIR_RWDIR_CNT];

	/*
	 * Enforced rate submission/completion workqueue
	 */
	struct workqueue io_wq;

	uint64_t total_io_size;
	uint64_t fill_device_size;

	/*
	 * Issue side
	 */
	uint64_t io_issues[DDIR_RWDIR_CNT];
	uint64_t io_issue_bytes[DDIR_RWDIR_CNT];
	uint64_t loops;

	/*
	 * Completions
	 */
	uint64_t io_blocks[DDIR_RWDIR_CNT];
	uint64_t this_io_blocks[DDIR_RWDIR_CNT];
	uint64_t io_bytes[DDIR_RWDIR_CNT];
	uint64_t this_io_bytes[DDIR_RWDIR_CNT];
	uint64_t io_skip_bytes;
	uint64_t zone_bytes;
	struct fio_sem *sem;
	uint64_t bytes_done[DDIR_RWDIR_CNT];

	/*
	 * State for random io, a bitmap of blocks done vs not done
	 */
	struct frand_state random_state;

	struct timespec start;	/* start of this loop */
	struct timespec epoch;	/* time job was started */
	unsigned long long unix_epoch; /* Time job was started, unix epoch based. */
	struct timespec last_issue;
	long time_offset;
	struct timespec ts_cache;
	struct timespec terminate_time;
	unsigned int ts_cache_nr;
	unsigned int ts_cache_mask;
	bool ramp_time_over;

	/*
	 * Time since last latency_window was started
	 */
	struct timespec latency_ts;
	unsigned int latency_qd;
	unsigned int latency_qd_high;
	unsigned int latency_qd_low;
	unsigned int latency_failed;
	uint64_t latency_ios;
	int latency_end_run;

	/*
	 * read/write mixed workload state
	 */
	struct frand_state rwmix_state;
	unsigned long rwmix_issues;
	enum fio_ddir rwmix_ddir;
	unsigned int ddir_seq_nr;

	/*
	 * rand/seq mixed workload state
	 */
	struct frand_state seq_rand_state[DDIR_RWDIR_CNT];

	/*
	 * IO history logs for verification. We use a tree for sorting,
	 * if we are overwriting. Otherwise just use a fifo.
	 */
	struct rb_root io_hist_tree;
	struct flist_head io_hist_list;
	unsigned long io_hist_len;

	/*
	 * For IO replaying
	 */
	struct flist_head io_log_list;
	FILE *io_log_rfile;
	unsigned int io_log_current;
	unsigned int io_log_checkmark;
	unsigned int io_log_highmark;
	struct timespec io_log_highmark_time;

	/*
	 * For tracking/handling discards
	 */
	struct flist_head trim_list;
	unsigned long trim_entries;

	/*
	 * for fileservice, how often to switch to a new file
	 */
	unsigned int file_service_nr;
	unsigned int file_service_left;
	struct fio_file *file_service_file;

	unsigned int sync_file_range_nr;

	/*
	 * For generating file sizes
	 */
	struct frand_state file_size_state;

	/*
	 * Error counts
	 */
	unsigned int total_err_count;
	int first_error;

	struct fio_flow *flow;

	/*
	 * Can be overloaded by profiles
	 */
	struct prof_io_ops prof_io_ops;
	void *prof_data;

	void *pinned_mem;

	struct steadystate_data ss;

	char verror[FIO_VERROR_SIZE];

#ifdef CONFIG_CUDA
	/*
	 * for GPU memory management
	 */
	int gpu_dev_cnt;
	int gpu_dev_id;
	CUdevice  cu_dev;
	CUcontext cu_ctx;
	CUdeviceptr dev_mem_ptr;
#endif	

};

/*
 * when should interactive ETA output be generated
 */
enum {
	FIO_ETA_AUTO,
	FIO_ETA_ALWAYS,
	FIO_ETA_NEVER,
};

#define __td_verror(td, err, msg, func)					\
	do {								\
		unsigned int ____e = (err);				\
		if ((td)->error)					\
			break;						\
		(td)->error = ____e;					\
		if (!(td)->first_error)					\
			nowarn_snprintf(td->verror, sizeof(td->verror),	\
					"file:%s:%d, func=%s, error=%s", \
					__FILE__, __LINE__, (func), (msg)); \
	} while (0)


#define td_clear_error(td)		do {		\
	(td)->error = 0;				\
	if ((td)->parent)				\
		(td)->parent->error = 0;		\
} while (0)

#define td_verror(td, err, func)	do {			\
	__td_verror((td), (err), strerror((err)), (func));	\
	if ((td)->parent)					\
		__td_verror((td)->parent, (err), strerror((err)), (func)); \
} while (0)

#define td_vmsg(td, err, msg, func)	do {			\
	__td_verror((td), (err), (msg), (func));		\
	if ((td)->parent)					\
		__td_verror((td)->parent, (err), (msg), (func));	\
} while (0)

#define __fio_stringify_1(x)	#x
#define __fio_stringify(x)	__fio_stringify_1(x)

extern bool exitall_on_terminate;
extern unsigned int thread_number;
extern unsigned int stat_number;
extern int shm_id;
extern int groupid;
extern int output_format;
extern int append_terse_output;
extern int temp_stall_ts;
extern uintptr_t page_mask, page_size;
extern bool read_only;
extern int eta_print;
extern int eta_new_line;
extern unsigned int eta_interval_msec;
extern unsigned long done_secs;
extern int fio_gtod_offload;
extern int fio_gtod_cpu;
extern enum fio_cs fio_clock_source;
extern int fio_clock_source_set;
extern int warnings_fatal;
extern int terse_version;
extern bool is_backend;
extern bool is_local_backend;
extern int nr_clients;
extern bool log_syslog;
extern int status_interval;
extern const char fio_version_string[];
extern char *trigger_file;
extern char *trigger_cmd;
extern char *trigger_remote_cmd;
extern long long trigger_timeout;
extern char *aux_path;

extern struct thread_data *threads;

static inline bool is_running_backend(void)
{
	return is_backend || is_local_backend;
}

extern bool eta_time_within_slack(unsigned int time);

static inline void fio_ro_check(const struct thread_data *td, struct io_u *io_u)
{
	assert(!(io_u->ddir == DDIR_WRITE && !td_write(td)) &&
	       !(io_u->ddir == DDIR_TRIM && !td_trim(td)));
}

#define REAL_MAX_JOBS		4096

static inline bool should_fsync(struct thread_data *td)
{
	if (td->last_was_sync)
		return false;
	if (td_write(td) || td->o.override_sync)
		return true;

	return false;
}

/*
 * Init/option functions
 */
extern int __must_check fio_init_options(void);
extern int __must_check parse_options(int, char **);
extern int parse_jobs_ini(char *, int, int, int);
extern int parse_cmd_line(int, char **, int);
extern int fio_backend(struct sk_out *);
extern void reset_fio_state(void);
extern void clear_io_state(struct thread_data *, int);
extern int fio_options_parse(struct thread_data *, char **, int);
extern void fio_keywords_init(void);
extern void fio_keywords_exit(void);
extern int fio_cmd_option_parse(struct thread_data *, const char *, char *);
extern int fio_cmd_ioengine_option_parse(struct thread_data *, const char *, char *);
extern void fio_fill_default_options(struct thread_data *);
extern int fio_show_option_help(const char *);
extern void fio_options_set_ioengine_opts(struct option *long_options, struct thread_data *td);
extern void fio_options_dup_and_init(struct option *);
extern char *fio_option_dup_subs(const char *);
extern void fio_options_mem_dupe(struct thread_data *);
extern void td_fill_rand_seeds(struct thread_data *);
extern void td_fill_verify_state_seed(struct thread_data *);
extern void add_job_opts(const char **, int);
extern int ioengine_load(struct thread_data *);
extern bool parse_dryrun(void);
extern int fio_running_or_pending_io_threads(void);
extern int fio_set_fd_nonblocking(int, const char *);
extern void sig_show_status(int sig);
extern struct thread_data *get_global_options(void);

extern uintptr_t page_mask;
extern uintptr_t page_size;
extern int initialize_fio(char *envp[]);
extern void deinitialize_fio(void);

#define FIO_GETOPT_JOB		0x89000000
#define FIO_GETOPT_IOENGINE	0x98000000
#define FIO_NR_OPTIONS		(FIO_MAX_OPTS + 128)

/*
 * ETA/status stuff
 */
extern void print_thread_status(void);
extern void print_status_init(int);
extern char *fio_uint_to_kmg(unsigned int val);

/*
 * Thread life cycle. Once a thread has a runstate beyond TD_INITIALIZED, it
 * will never back again. It may cycle between running/verififying/fsyncing.
 * Once the thread reaches TD_EXITED, it is just waiting for the core to
 * reap it.
 */
enum {
	TD_NOT_CREATED = 0,
	TD_CREATED,
	TD_INITIALIZED,
	TD_RAMP,
	TD_SETTING_UP,
	TD_RUNNING,
	TD_PRE_READING,
	TD_VERIFYING,
	TD_FSYNCING,
	TD_FINISHING,
	TD_EXITED,
	TD_REAPED,
	TD_LAST,
	TD_NR,
};

#define TD_ENG_FLAG_SHIFT	17
#define TD_ENG_FLAG_MASK	((1U << 17) - 1)

static inline void td_set_ioengine_flags(struct thread_data *td)
{
	td->flags = (~(TD_ENG_FLAG_MASK << TD_ENG_FLAG_SHIFT) & td->flags) |
		    (td->io_ops->flags << TD_ENG_FLAG_SHIFT);
}

static inline bool td_ioengine_flagged(struct thread_data *td,
				       enum fio_ioengine_flags flags)
{
	return ((td->flags >> TD_ENG_FLAG_SHIFT) & flags) != 0;
}

extern void td_set_runstate(struct thread_data *, int);
extern int td_bump_runstate(struct thread_data *, int);
extern void td_restore_runstate(struct thread_data *, int);
extern const char *runstate_to_name(int runstate);

/*
 * Allow 60 seconds for a job to quit on its own, otherwise reap with
 * a vengeance.
 */
#define FIO_REAP_TIMEOUT	300

#define TERMINATE_ALL		(-1U)
extern void fio_terminate_threads(unsigned int);
extern void fio_mark_td_terminate(struct thread_data *);

/*
 * Memory helpers
 */
extern int __must_check fio_pin_memory(struct thread_data *);
extern void fio_unpin_memory(struct thread_data *);
extern int __must_check allocate_io_mem(struct thread_data *);
extern void free_io_mem(struct thread_data *);
extern void free_threads_shm(void);

#ifdef FIO_INTERNAL
#define PTR_ALIGN(ptr, mask)	\
	(char *) (((uintptr_t) (ptr) + (mask)) & ~(mask))
#endif

/*
 * Reset stats after ramp time completes
 */
extern void reset_all_stats(struct thread_data *);

extern int io_queue_event(struct thread_data *td, struct io_u *io_u, int *ret,
		   enum fio_ddir ddir, uint64_t *bytes_issued, int from_verify,
		   struct timespec *comp_time);

/*
 * Latency target helpers
 */
extern void lat_target_check(struct thread_data *);
extern void lat_target_init(struct thread_data *);
extern void lat_target_reset(struct thread_data *);

/*
 * Iterates all threads/processes within all the defined jobs
 */
#define for_each_td(td, i)	\
	for ((i) = 0, (td) = &threads[0]; (i) < (int) thread_number; (i)++, (td)++)
#define for_each_file(td, f, i)	\
	if ((td)->files_index)						\
		for ((i) = 0, (f) = (td)->files[0];			\
	    	 (i) < (td)->o.nr_files && ((f) = (td)->files[i]) != NULL; \
		 (i)++)

#define fio_assert(td, cond)	do {	\
	if (!(cond)) {			\
		int *__foo = NULL;	\
		fprintf(stderr, "file:%s:%d, assert %s failed\n", __FILE__, __LINE__, #cond);	\
		td_set_runstate((td), TD_EXITED);	\
		(td)->error = EFAULT;		\
		*__foo = 0;			\
	}	\
} while (0)

static inline bool fio_fill_issue_time(struct thread_data *td)
{
	if (td->o.read_iolog_file ||
	    !td->o.disable_clat || !td->o.disable_slat || !td->o.disable_bw)
		return true;

	return false;
}

static inline bool option_check_rate(struct thread_data *td, enum fio_ddir ddir)
{
	struct thread_options *o = &td->o;

	/*
	 * If some rate setting was given, we need to check it
	 */
	if (o->rate[ddir] || o->ratemin[ddir] || o->rate_iops[ddir] ||
	    o->rate_iops_min[ddir])
		return true;

	return false;
}

static inline bool __should_check_rate(struct thread_data *td)
{
	return (td->flags & TD_F_CHECK_RATE) != 0;
}

static inline bool should_check_rate(struct thread_data *td)
{
	if (!__should_check_rate(td))
		return false;

	return ddir_rw_sum(td->bytes_done) != 0;
}

static inline unsigned long long td_max_bs(struct thread_data *td)
{
	unsigned long long max_bs;

	max_bs = max(td->o.max_bs[DDIR_READ], td->o.max_bs[DDIR_WRITE]);
	return max(td->o.max_bs[DDIR_TRIM], max_bs);
}

static inline unsigned long long td_min_bs(struct thread_data *td)
{
	unsigned long long min_bs;

	min_bs = min(td->o.min_bs[DDIR_READ], td->o.min_bs[DDIR_WRITE]);
	return min(td->o.min_bs[DDIR_TRIM], min_bs);
}

static inline bool td_async_processing(struct thread_data *td)
{
	return (td->flags & TD_F_NEED_LOCK) != 0;
}

static inline bool td_offload_overlap(struct thread_data *td)
{
	return td->o.serialize_overlap && td->o.io_submit_mode == IO_MODE_OFFLOAD;
}

/*
 * We currently only need to do locking if we have verifier threads
 * accessing our internal structures too
 */
static inline void __td_io_u_lock(struct thread_data *td)
{
	pthread_mutex_lock(&td->io_u_lock);
}

static inline void __td_io_u_unlock(struct thread_data *td)
{
	pthread_mutex_unlock(&td->io_u_lock);
}

static inline void td_io_u_free_notify(struct thread_data *td)
{
	if (td_async_processing(td))
		pthread_cond_signal(&td->free_cond);
}

static inline void td_flags_clear(struct thread_data *td, unsigned int *flags,
				  unsigned int value)
{
	if (!td_async_processing(td))
		*flags &= ~value;
	else
		__sync_fetch_and_and(flags, ~value);
}

static inline void td_flags_set(struct thread_data *td, unsigned int *flags,
				unsigned int value)
{
	if (!td_async_processing(td))
		*flags |= value;
	else
		__sync_fetch_and_or(flags, value);
}

extern const char *fio_get_arch_string(int);
extern const char *fio_get_os_string(int);

enum {
	__FIO_OUTPUT_TERSE	= 0,
	__FIO_OUTPUT_JSON	= 1,
	__FIO_OUTPUT_NORMAL	= 2,
        __FIO_OUTPUT_JSON_PLUS  = 3,
	FIO_OUTPUT_NR		= 4,

	FIO_OUTPUT_TERSE	= 1U << __FIO_OUTPUT_TERSE,
	FIO_OUTPUT_JSON		= 1U << __FIO_OUTPUT_JSON,
	FIO_OUTPUT_NORMAL	= 1U << __FIO_OUTPUT_NORMAL,
	FIO_OUTPUT_JSON_PLUS    = 1U << __FIO_OUTPUT_JSON_PLUS,
};

enum {
	FIO_RAND_DIST_RANDOM	= 0,
	FIO_RAND_DIST_ZIPF,
	FIO_RAND_DIST_PARETO,
	FIO_RAND_DIST_GAUSS,
	FIO_RAND_DIST_ZONED,
	FIO_RAND_DIST_ZONED_ABS,
};

#define FIO_DEF_ZIPF		1.1
#define FIO_DEF_PARETO		0.2

enum {
	FIO_RAND_GEN_TAUSWORTHE = 0,
	FIO_RAND_GEN_LFSR,
	FIO_RAND_GEN_TAUSWORTHE64,
};

enum {
	FIO_CPUS_SHARED		= 0,
	FIO_CPUS_SPLIT,
};

extern void exec_trigger(const char *);
extern void check_trigger_file(void);

extern bool in_flight_overlap(struct io_u_queue *q, struct io_u *io_u);
extern pthread_mutex_t overlap_check;

#endif