1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
|
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008-2020 University of Houston. All rights reserved.
* Copyright (c) 2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2018 DataDirect Networks. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_COMMON_OMPIO_H
#define MCA_COMMON_OMPIO_H
#include <fcntl.h>
#include "mpi.h"
#include "opal/class/opal_list.h"
#include "ompi/errhandler/errhandler.h"
#include "opal/mca/threads/mutex.h"
#include "ompi/file/file.h"
#include "ompi/mca/io/io.h"
#include "ompi/mca/fs/fs.h"
#include "ompi/mca/fcoll/fcoll.h"
#include "ompi/mca/fbtl/fbtl.h"
#include "ompi/mca/sharedfp/sharedfp.h"
#include "ompi/communicator/communicator.h"
#include "ompi/info/info.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/datatype/ompi_datatype.h"
#include "ompi/request/request.h"
#define OMPIO_MIN(a, b) (((a) < (b)) ? (a) : (b))
#define OMPIO_MAX(a, b) (((a) < (b)) ? (b) : (a))
#define OMPIO_MCA_GET(fh, name) ((fh)->f_get_mca_parameter_value(#name, strlen(#name)+1))
#define OMPIO_MCA_PRINT_INFO(_fh,_infostr,_infoval, _msg ) { \
int _verbose = _fh->f_get_mca_parameter_value("verbose_info_parsing", strlen("verbose_info_parsing")); \
if ( 1==_verbose && 0==_fh->f_rank ) printf("File: %s info: %s value %s %s\n", _fh->f_filename, _infostr, _infoval, _msg); \
if ( 2==_verbose ) printf("File: %s info: %s value %s %s\n", _fh->f_filename, _infostr, _infoval, _msg); \
}
/*
* Flags
*/
#define OMPIO_CONTIGUOUS_MEMORY 0x00000001
#define OMPIO_UNIFORM_FVIEW 0x00000002
#define OMPIO_FILE_IS_OPEN 0x00000004
#define OMPIO_FILE_VIEW_IS_SET 0x00000008
#define OMPIO_CONTIGUOUS_FVIEW 0x00000010
#define OMPIO_AGGREGATOR_IS_SET 0x00000020
#define OMPIO_SHAREDFP_IS_SET 0x00000040
#define OMPIO_LOCK_ENTIRE_FILE 0x00000080
#define OMPIO_LOCK_NEVER 0x00000100
#define OMPIO_LOCK_NOT_THIS_OP 0x00000200
#define OMPIO_DATAREP_NATIVE 0x00000400
#define OMPIO_COLLECTIVE_OP 0x00000800
#define OMPIO_ROOT 0
/*AGGREGATOR GROUPING DECISIONS*/
#define OMPIO_MERGE 1
#define OMPIO_SPLIT 2
#define OMPIO_RETAIN 3
#define DATA_VOLUME 1
#define UNIFORM_DISTRIBUTION 2
#define CONTIGUITY 3
#define OPTIMIZE_GROUPING 4
#define SIMPLE 5
#define NO_REFINEMENT 6
#define SIMPLE_PLUS 7
#define OMPIO_LOCK_ENTIRE_REGION 10
#define OMPIO_LOCK_SELECTIVE 11
#define OMPIO_FCOLL_WANT_TIME_BREAKDOWN 0
#define MCA_IO_DEFAULT_FILE_VIEW_SIZE 6*1024*1024
#define OMPIO_UNIFORM_DIST_THRESHOLD 0.5
#define OMPIO_CONTG_THRESHOLD 1048576
#define OMPIO_CONTG_FACTOR 8
#define OMPIO_DEFAULT_STRIPE_SIZE 1048576
#define OMPIO_PROCS_PER_GROUP_TAG 0
#define OMPIO_PROCS_IN_GROUP_TAG 1
#define OMPIO_MERGE_THRESHOLD 0.5
#define OMPIO_PERM_NULL -1
#define OMPIO_IOVEC_INITIAL_SIZE 100
enum ompio_fs_type
{
NONE = 0,
UFS = 1,
PVFS2 = 2,
LUSTRE = 3,
PLFS = 4,
IME = 5,
GPFS = 6
};
typedef struct mca_common_ompio_io_array_t {
void *memory_address;
/* we need that of type OMPI_MPI_OFFSET_TYPE */
void *offset;
size_t length;
/*mca_common_ompio_server_t io_server;*/
} mca_common_ompio_io_array_t;
typedef struct mca_common_ompio_access_array_t{
OMPI_MPI_OFFSET_TYPE *offsets;
int *lens;
MPI_Aint *mem_ptrs;
int count;
} mca_common_ompio_access_array_t;
/* forward declaration to keep the compiler happy. */
struct ompio_file_t;
typedef int (*mca_common_ompio_generate_current_file_view_fn_t) (struct ompio_file_t *fh,
size_t max_data,
struct iovec **f_iov,
int *iov_count);
/* functions to retrieve the number of aggregators and the size of the
temporary buffer on aggregators from the fcoll modules */
typedef int (*mca_common_ompio_get_mca_parameter_value_fn_t) ( char *mca_parameter_name, int name_length );
struct mca_common_ompio_print_queue;
/**
* Back-end structure for MPI_File
*/
struct ompio_file_t {
/* General parameters */
int fd;
struct ompi_file_t *f_fh; /* pointer back to the file_t structure */
OMPI_MPI_OFFSET_TYPE f_offset; /* byte offset of current position */
OMPI_MPI_OFFSET_TYPE f_disp; /* file_view displacement */
int f_rank;
int f_size;
int f_amode;
int f_perm;
ompi_communicator_t *f_comm;
const char *f_filename;
char *f_fullfilename;
char *f_datarep;
opal_convertor_t *f_mem_convertor;
opal_convertor_t *f_file_convertor;
opal_info_t *f_info;
int32_t f_flags;
void *f_fs_ptr;
int f_fs_block_size;
int f_atomicity;
size_t f_stripe_size;
int f_stripe_count;
size_t f_cc_size;
size_t f_avg_view_size;
int f_bytes_per_agg;
enum ompio_fs_type f_fstype;
ompi_request_t *f_split_coll_req;
bool f_split_coll_in_use;
/* Place for selected sharedfp module to hang it's data.
Note: Neither f_sharedfp nor f_sharedfp_component seemed appropriate for this.
*/
void *f_sharedfp_data;
/* File View parameters */
struct iovec *f_decoded_iov;
uint32_t f_iov_count;
ompi_datatype_t *f_iov_type;
size_t f_position_in_file_view; /* in bytes */
size_t f_total_bytes; /* total bytes read/written within 1 Fview*/
int f_index_in_file_view;
ptrdiff_t f_view_extent;
size_t f_view_size;
ompi_datatype_t *f_etype;
ompi_datatype_t *f_filetype;
ompi_datatype_t *f_orig_filetype; /* the fileview passed by the user to us */
size_t f_etype_size;
/* contains IO requests that needs to be read/written */
mca_common_ompio_io_array_t *f_io_array;
int f_num_of_io_entries;
/* Hooks for modules to hang things */
mca_base_component_t *f_fs_component;
mca_base_component_t *f_fcoll_component;
mca_base_component_t *f_fbtl_component;
mca_base_component_t *f_sharedfp_component;
/* structure of function pointers */
mca_fs_base_module_t *f_fs;
mca_fcoll_base_module_t *f_fcoll;
mca_fbtl_base_module_t *f_fbtl;
mca_sharedfp_base_module_t *f_sharedfp;
/* Timing information */
struct mca_common_ompio_print_queue *f_coll_write_time;
struct mca_common_ompio_print_queue *f_coll_read_time;
/*initial list of aggregators and groups*/
int *f_init_aggr_list;
int f_init_num_aggrs;
int f_init_procs_per_group;
int *f_init_procs_in_group;
/* final of aggregators and groups*/
int *f_aggr_list;
int f_num_aggrs;
int *f_procs_in_group;
int f_procs_per_group;
/* internal ompio functions required by fbtl and fcoll */
mca_common_ompio_generate_current_file_view_fn_t f_generate_current_file_view;
mca_common_ompio_get_mca_parameter_value_fn_t f_get_mca_parameter_value;
};
typedef struct ompio_file_t ompio_file_t;
struct mca_common_ompio_data_t {
ompio_file_t ompio_fh;
};
typedef struct mca_common_ompio_data_t mca_common_ompio_data_t;
#include "common_ompio_print_queue.h"
#include "common_ompio_aggregators.h"
OMPI_DECLSPEC int mca_common_ompio_file_write (ompio_file_t *fh, const void *buf, int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status);
OMPI_DECLSPEC int mca_common_ompio_file_write_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
int count, struct ompi_datatype_t *datatype,
ompi_status_public_t *status);
OMPI_DECLSPEC int mca_common_ompio_file_iwrite (ompio_file_t *fh, const void *buf, int count,
struct ompi_datatype_t *datatype, ompi_request_t **request);
OMPI_DECLSPEC int mca_common_ompio_file_iwrite_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,
const void *buf, int count, struct ompi_datatype_t *datatype,
ompi_request_t **request);
OMPI_DECLSPEC int mca_common_ompio_file_write_all (ompio_file_t *fh, const void *buf,
int count, struct ompi_datatype_t *datatype,
ompi_status_public_t *status);
OMPI_DECLSPEC int mca_common_ompio_file_write_at_all (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
int count, struct ompi_datatype_t *datatype,
ompi_status_public_t *status);
OMPI_DECLSPEC int mca_common_ompio_file_iwrite_all (ompio_file_t *fp, const void *buf,
int count, struct ompi_datatype_t *datatype, ompi_request_t **request);
OMPI_DECLSPEC int mca_common_ompio_file_iwrite_at_all (ompio_file_t *fp, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
int count, struct ompi_datatype_t *datatype, ompi_request_t **request);
OMPI_DECLSPEC int mca_common_ompio_build_io_array ( ompio_file_t *fh, int index, int cycles,
size_t bytes_per_cycle, size_t max_data, uint32_t iov_count,
struct iovec *decoded_iov, int *ii, int *jj, size_t *tbw,
size_t *spc, mca_common_ompio_io_array_t **io_array,
int *num_io_entries );
OMPI_DECLSPEC int mca_common_ompio_file_read (ompio_file_t *fh, void *buf, int count,
struct ompi_datatype_t *datatype, ompi_status_public_t *status);
OMPI_DECLSPEC int mca_common_ompio_file_read_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, void *buf,
int count, struct ompi_datatype_t *datatype,
ompi_status_public_t * status);
OMPI_DECLSPEC int mca_common_ompio_file_iread (ompio_file_t *fh, void *buf, int count,
struct ompi_datatype_t *datatype, ompi_request_t **request);
OMPI_DECLSPEC int mca_common_ompio_file_iread_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,
void *buf, int count, struct ompi_datatype_t *datatype,
ompi_request_t **request);
OMPI_DECLSPEC int mca_common_ompio_file_read_all (ompio_file_t *fh, void *buf, int count, struct ompi_datatype_t *datatype,
ompi_status_public_t * status);
OMPI_DECLSPEC int mca_common_ompio_file_read_at_all (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,
void *buf, int count, struct ompi_datatype_t *datatype,
ompi_status_public_t * status);
OMPI_DECLSPEC int mca_common_ompio_file_iread_all (ompio_file_t *fp, void *buf, int count, struct ompi_datatype_t *datatype,
ompi_request_t **request);
OMPI_DECLSPEC int mca_common_ompio_file_iread_at_all (ompio_file_t *fp, OMPI_MPI_OFFSET_TYPE offset,
void *buf, int count, struct ompi_datatype_t *datatype,
ompi_request_t **request);
OMPI_DECLSPEC int mca_common_ompio_file_open (ompi_communicator_t *comm, const char *filename,
int amode, opal_info_t *info,
ompio_file_t *ompio_fh, bool use_sharedfp);
OMPI_DECLSPEC int mca_common_ompio_file_delete (const char *filename,
struct opal_info_t *info);
OMPI_DECLSPEC int mca_common_ompio_create_incomplete_file_handle (const char *filename,
ompio_file_t **fh);
OMPI_DECLSPEC int mca_common_ompio_file_close (ompio_file_t *ompio_fh);
OMPI_DECLSPEC int mca_common_ompio_file_get_size (ompio_file_t *ompio_fh, OMPI_MPI_OFFSET_TYPE *size);
OMPI_DECLSPEC int mca_common_ompio_file_get_position (ompio_file_t *fh,OMPI_MPI_OFFSET_TYPE *offset);
OMPI_DECLSPEC int mca_common_ompio_set_explicit_offset (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset);
OMPI_DECLSPEC int mca_common_ompio_set_file_defaults (ompio_file_t *fh);
OMPI_DECLSPEC int mca_common_ompio_set_view (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE disp,
ompi_datatype_t *etype, ompi_datatype_t *filetype, const char *datarep,
opal_info_t *info);
OMPI_DECLSPEC int mca_common_ompio_base_file_read_all (struct ompio_file_t *fh, void *buf, int count,
struct ompi_datatype_t *datatype, ompi_status_public_t *status);
/*
* Function that takes in a datatype and buffer, and decodes that datatype
* into an iovec using the convertor_raw function
*/
OMPI_DECLSPEC int mca_common_ompio_decode_datatype (struct ompio_file_t *fh,
struct ompi_datatype_t *datatype,
int count,
const void *buf,
size_t *max_data,
opal_convertor_t *convertor,
struct iovec **iov,
uint32_t *iov_count);
OMPI_DECLSPEC int mca_common_ompio_set_callbacks(mca_common_ompio_generate_current_file_view_fn_t generate_current_file_view,
mca_common_ompio_get_mca_parameter_value_fn_t get_mca_parameter_value);
#endif /* MCA_COMMON_OMPIO_H */
|
