1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 #ifndef MCA_COMMON_OMPIO_H
25 #define MCA_COMMON_OMPIO_H
26
27 #include <fcntl.h>
28
29 #include "mpi.h"
30 #include "opal/class/opal_list.h"
31 #include "ompi/errhandler/errhandler.h"
32 #include "opal/threads/mutex.h"
33 #include "ompi/file/file.h"
34 #include "ompi/mca/io/io.h"
35 #include "ompi/mca/fs/fs.h"
36 #include "ompi/mca/fcoll/fcoll.h"
37 #include "ompi/mca/fbtl/fbtl.h"
38 #include "ompi/mca/sharedfp/sharedfp.h"
39 #include "ompi/communicator/communicator.h"
40 #include "ompi/info/info.h"
41 #include "opal/datatype/opal_convertor.h"
42 #include "ompi/datatype/ompi_datatype.h"
43 #include "ompi/request/request.h"
44
45 #define OMPIO_MIN(a, b) (((a) < (b)) ? (a) : (b))
46 #define OMPIO_MAX(a, b) (((a) < (b)) ? (b) : (a))
47
48 #define OMPIO_MCA_GET(fh, name) ((fh)->f_get_mca_parameter_value(#name, strlen(#name)+1))
49 #define OMPIO_MCA_PRINT_INFO(_fh,_infostr,_infoval, _msg ) { \
50 int _verbose = _fh->f_get_mca_parameter_value("verbose_info_parsing", strlen("verbose_info_parsing")); \
51 if ( 1==_verbose && 0==_fh->f_rank ) printf("File: %s info: %s value %s %s\n", _fh->f_filename, _infostr, _infoval, _msg); \
52 if ( 2==_verbose ) printf("File: %s info: %s value %s %s\n", _fh->f_filename, _infostr, _infoval, _msg); \
53 }
54
55
56
57
58
59 #define OMPIO_CONTIGUOUS_MEMORY 0x00000001
60 #define OMPIO_UNIFORM_FVIEW 0x00000002
61 #define OMPIO_FILE_IS_OPEN 0x00000004
62 #define OMPIO_FILE_VIEW_IS_SET 0x00000008
63 #define OMPIO_CONTIGUOUS_FVIEW 0x00000010
64 #define OMPIO_AGGREGATOR_IS_SET 0x00000020
65 #define OMPIO_SHAREDFP_IS_SET 0x00000040
66 #define OMPIO_LOCK_ENTIRE_FILE 0x00000080
67 #define OMPIO_LOCK_NEVER 0x00000100
68 #define OMPIO_LOCK_NOT_THIS_OP 0x00000200
69 #define OMPIO_DATAREP_NATIVE 0x00000400
70
71 #define OMPIO_ROOT 0
72
73
74 #define OMPIO_MERGE 1
75 #define OMPIO_SPLIT 2
76 #define OMPIO_RETAIN 3
77
78 #define DATA_VOLUME 1
79 #define UNIFORM_DISTRIBUTION 2
80 #define CONTIGUITY 3
81 #define OPTIMIZE_GROUPING 4
82 #define SIMPLE 5
83 #define NO_REFINEMENT 6
84 #define SIMPLE_PLUS 7
85
86 #define OMPIO_LOCK_ENTIRE_REGION 10
87 #define OMPIO_LOCK_SELECTIVE 11
88
89 #define OMPIO_FCOLL_WANT_TIME_BREAKDOWN 0
90 #define MCA_IO_DEFAULT_FILE_VIEW_SIZE 4*1024*1024
91
92 #define OMPIO_UNIFORM_DIST_THRESHOLD 0.5
93 #define OMPIO_CONTG_THRESHOLD 1048576
94 #define OMPIO_CONTG_FACTOR 8
95 #define OMPIO_DEFAULT_STRIPE_SIZE 1048576
96 #define OMPIO_PROCS_PER_GROUP_TAG 0
97 #define OMPIO_PROCS_IN_GROUP_TAG 1
98 #define OMPIO_MERGE_THRESHOLD 0.5
99
100 #define OMPIO_PERM_NULL -1
101 #define OMPIO_IOVEC_INITIAL_SIZE 100
102
103 enum ompio_fs_type
104 {
105 NONE = 0,
106 UFS = 1,
107 PVFS2 = 2,
108 LUSTRE = 3,
109 PLFS = 4,
110 IME = 5
111 };
112
113 typedef struct mca_common_ompio_io_array_t {
114 void *memory_address;
115
116 void *offset;
117 size_t length;
118
119 } mca_common_ompio_io_array_t;
120
121
122 typedef struct mca_common_ompio_access_array_t{
123 OMPI_MPI_OFFSET_TYPE *offsets;
124 int *lens;
125 MPI_Aint *mem_ptrs;
126 int count;
127 } mca_common_ompio_access_array_t;
128
129
130
131 struct ompio_file_t;
132 typedef int (*mca_common_ompio_generate_current_file_view_fn_t) (struct ompio_file_t *fh,
133 size_t max_data,
134 struct iovec **f_iov,
135 int *iov_count);
136
137
138
139 typedef int (*mca_common_ompio_get_mca_parameter_value_fn_t) ( char *mca_parameter_name, int name_length );
140
141
142 struct mca_common_ompio_print_queue;
143
144
145
146
147 struct ompio_file_t {
148
149 int fd;
150 struct ompi_file_t *f_fh;
151 OMPI_MPI_OFFSET_TYPE f_offset;
152 OMPI_MPI_OFFSET_TYPE f_disp;
153 int f_rank;
154 int f_size;
155 int f_amode;
156 int f_perm;
157 ompi_communicator_t *f_comm;
158 const char *f_filename;
159 char *f_datarep;
160 opal_convertor_t *f_mem_convertor;
161 opal_convertor_t *f_file_convertor;
162 opal_info_t *f_info;
163 int32_t f_flags;
164 void *f_fs_ptr;
165 int f_fs_block_size;
166 int f_atomicity;
167 size_t f_stripe_size;
168 int f_stripe_count;
169 size_t f_cc_size;
170 int f_bytes_per_agg;
171 enum ompio_fs_type f_fstype;
172 ompi_request_t *f_split_coll_req;
173 bool f_split_coll_in_use;
174
175
176
177 void *f_sharedfp_data;
178
179
180
181 struct iovec *f_decoded_iov;
182 uint32_t f_iov_count;
183 ompi_datatype_t *f_iov_type;
184 size_t f_position_in_file_view;
185 size_t f_total_bytes;
186 int f_index_in_file_view;
187 ptrdiff_t f_view_extent;
188 size_t f_view_size;
189 ompi_datatype_t *f_etype;
190 ompi_datatype_t *f_filetype;
191 ompi_datatype_t *f_orig_filetype;
192 size_t f_etype_size;
193
194
195 mca_common_ompio_io_array_t *f_io_array;
196 int f_num_of_io_entries;
197
198
199 mca_base_component_t *f_fs_component;
200 mca_base_component_t *f_fcoll_component;
201 mca_base_component_t *f_fbtl_component;
202 mca_base_component_t *f_sharedfp_component;
203
204
205 mca_fs_base_module_t *f_fs;
206 mca_fcoll_base_module_t *f_fcoll;
207 mca_fbtl_base_module_t *f_fbtl;
208 mca_sharedfp_base_module_t *f_sharedfp;
209
210
211 struct mca_common_ompio_print_queue *f_coll_write_time;
212 struct mca_common_ompio_print_queue *f_coll_read_time;
213
214
215 int *f_init_aggr_list;
216 int f_init_num_aggrs;
217 int f_init_procs_per_group;
218 int *f_init_procs_in_group;
219
220
221 int *f_aggr_list;
222 int f_num_aggrs;
223 int *f_procs_in_group;
224 int f_procs_per_group;
225
226
227 mca_common_ompio_generate_current_file_view_fn_t f_generate_current_file_view;
228
229 mca_common_ompio_get_mca_parameter_value_fn_t f_get_mca_parameter_value;
230 };
231 typedef struct ompio_file_t ompio_file_t;
232
233 struct mca_common_ompio_data_t {
234 ompio_file_t ompio_fh;
235 };
236 typedef struct mca_common_ompio_data_t mca_common_ompio_data_t;
237
238
239 #include "common_ompio_print_queue.h"
240 #include "common_ompio_aggregators.h"
241
242 OMPI_DECLSPEC int mca_common_ompio_file_write (ompio_file_t *fh, const void *buf, int count,
243 struct ompi_datatype_t *datatype,
244 ompi_status_public_t *status);
245
246 OMPI_DECLSPEC int mca_common_ompio_file_write_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
247 int count, struct ompi_datatype_t *datatype,
248 ompi_status_public_t *status);
249
250 OMPI_DECLSPEC int mca_common_ompio_file_iwrite (ompio_file_t *fh, const void *buf, int count,
251 struct ompi_datatype_t *datatype, ompi_request_t **request);
252
253 OMPI_DECLSPEC int mca_common_ompio_file_iwrite_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,
254 const void *buf, int count, struct ompi_datatype_t *datatype,
255 ompi_request_t **request);
256
257 OMPI_DECLSPEC int mca_common_ompio_file_write_all (ompio_file_t *fh, const void *buf,
258 int count, struct ompi_datatype_t *datatype,
259 ompi_status_public_t *status);
260
261 OMPI_DECLSPEC int mca_common_ompio_file_write_at_all (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
262 int count, struct ompi_datatype_t *datatype,
263 ompi_status_public_t *status);
264
265 OMPI_DECLSPEC int mca_common_ompio_file_iwrite_all (ompio_file_t *fp, const void *buf,
266 int count, struct ompi_datatype_t *datatype, ompi_request_t **request);
267
268 OMPI_DECLSPEC int mca_common_ompio_file_iwrite_at_all (ompio_file_t *fp, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
269 int count, struct ompi_datatype_t *datatype, ompi_request_t **request);
270
271 OMPI_DECLSPEC int mca_common_ompio_build_io_array ( ompio_file_t *fh, int index, int cycles,
272 size_t bytes_per_cycle, int max_data, uint32_t iov_count,
273 struct iovec *decoded_iov, int *ii, int *jj, size_t *tbw,
274 size_t *spc, mca_common_ompio_io_array_t **io_array,
275 int *num_io_entries );
276
277
278 OMPI_DECLSPEC int mca_common_ompio_file_read (ompio_file_t *fh, void *buf, int count,
279 struct ompi_datatype_t *datatype, ompi_status_public_t *status);
280
281 OMPI_DECLSPEC int mca_common_ompio_file_read_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, void *buf,
282 int count, struct ompi_datatype_t *datatype,
283 ompi_status_public_t * status);
284
285 OMPI_DECLSPEC int mca_common_ompio_file_iread (ompio_file_t *fh, void *buf, int count,
286 struct ompi_datatype_t *datatype, ompi_request_t **request);
287
288 OMPI_DECLSPEC int mca_common_ompio_file_iread_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,
289 void *buf, int count, struct ompi_datatype_t *datatype,
290 ompi_request_t **request);
291
292 OMPI_DECLSPEC int mca_common_ompio_file_read_all (ompio_file_t *fh, void *buf, int count, struct ompi_datatype_t *datatype,
293 ompi_status_public_t * status);
294
295 OMPI_DECLSPEC int mca_common_ompio_file_read_at_all (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,
296 void *buf, int count, struct ompi_datatype_t *datatype,
297 ompi_status_public_t * status);
298
299 OMPI_DECLSPEC int mca_common_ompio_file_iread_all (ompio_file_t *fp, void *buf, int count, struct ompi_datatype_t *datatype,
300 ompi_request_t **request);
301
302 OMPI_DECLSPEC int mca_common_ompio_file_iread_at_all (ompio_file_t *fp, OMPI_MPI_OFFSET_TYPE offset,
303 void *buf, int count, struct ompi_datatype_t *datatype,
304 ompi_request_t **request);
305
306 OMPI_DECLSPEC int mca_common_ompio_file_open (ompi_communicator_t *comm, const char *filename,
307 int amode, opal_info_t *info,
308 ompio_file_t *ompio_fh, bool use_sharedfp);
309
310 OMPI_DECLSPEC int mca_common_ompio_file_delete (const char *filename,
311 struct opal_info_t *info);
312 OMPI_DECLSPEC int mca_common_ompio_create_incomplete_file_handle (const char *filename,
313 ompio_file_t **fh);
314
315 OMPI_DECLSPEC int mca_common_ompio_file_close (ompio_file_t *ompio_fh);
316 OMPI_DECLSPEC int mca_common_ompio_file_get_size (ompio_file_t *ompio_fh, OMPI_MPI_OFFSET_TYPE *size);
317 OMPI_DECLSPEC int mca_common_ompio_file_get_position (ompio_file_t *fh,OMPI_MPI_OFFSET_TYPE *offset);
318 OMPI_DECLSPEC int mca_common_ompio_set_explicit_offset (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset);
319 OMPI_DECLSPEC int mca_common_ompio_set_file_defaults (ompio_file_t *fh);
320 OMPI_DECLSPEC int mca_common_ompio_set_view (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE disp,
321 ompi_datatype_t *etype, ompi_datatype_t *filetype, const char *datarep,
322 opal_info_t *info);
323
324
325
326
327
328
329 OMPI_DECLSPEC int mca_common_ompio_decode_datatype (struct ompio_file_t *fh,
330 struct ompi_datatype_t *datatype,
331 int count,
332 const void *buf,
333 size_t *max_data,
334 opal_convertor_t *convertor,
335 struct iovec **iov,
336 uint32_t *iov_count);
337
338 OMPI_DECLSPEC int mca_common_ompio_set_callbacks(mca_common_ompio_generate_current_file_view_fn_t generate_current_file_view,
339 mca_common_ompio_get_mca_parameter_value_fn_t get_mca_parameter_value);
340 #endif