1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*-
2 * vim: ts=8 sts=4 sw=4 noexpandtab
3 *
4 * Copyright (C) 2008 University of Chicago.
5 * See COPYRIGHT notice in top-level directory.
6 */
7
8 #include "adio.h"
9 #include "adio_extern.h"
10 #include "ad_pvfs2.h"
11
12 #include "ad_pvfs2_common.h"
13
14 void ADIOI_PVFS2_OldWriteStrided(ADIO_File fd, const void *buf, int count,
15 MPI_Datatype datatype, int file_ptr_type,
16 ADIO_Offset offset, ADIO_Status *status,
17 int *error_code)
18 {
19 /* as with all the other WriteStrided functions, offset is in units of
20 * etype relative to the filetype */
21
22 /* Since PVFS2 does not support file locking, can't do buffered writes
23 as on Unix */
24
25 ADIOI_Flatlist_node *flat_buf, *flat_file;
26 int i, j, k, bwr_size, fwr_size=0, st_index=0;
27 int sum, n_etypes_in_filetype, size_in_filetype;
28 MPI_Count bufsize;
29 int n_filetypes, etype_in_filetype;
30 ADIO_Offset abs_off_in_filetype=0;
31 MPI_Count filetype_size, etype_size, buftype_size;
32 MPI_Aint filetype_extent, buftype_extent, filetype_lb, buftype_lb;
33 int buf_count, buftype_is_contig, filetype_is_contig;
34 ADIO_Offset off, disp, start_off, initial_off;
35 int flag, st_fwr_size, st_n_filetypes;
36 int err_flag=0;
37
38 int mem_list_count, file_list_count;
39 PVFS_size * mem_offsets;
40 int64_t *file_offsets;
41 int *mem_lengths;
42 int32_t *file_lengths;
43 int total_blks_to_write;
44
45 int max_mem_list, max_file_list;
46
47 int b_blks_wrote;
48 int f_data_wrote;
49 int size_wrote=0, n_write_lists, extra_blks;
50
51 int end_bwr_size, end_fwr_size;
52 int start_k, start_j, new_file_write, new_buffer_write;
53 int start_mem_offset;
54 PVFS_Request mem_req, file_req;
55 ADIOI_PVFS2_fs * pvfs_fs;
56 PVFS_sysresp_io resp_io;
57 MPI_Offset total_bytes_written=0;
58 static char myname[] = "ADIOI_PVFS2_WRITESTRIDED";
59
60 /* note: don't increase this: several parts of PVFS2 now
61 * assume this limit*/
62 #define MAX_ARRAY_SIZE 64
63
64 /* --BEGIN ERROR HANDLING-- */
65 if (fd->atomicity) {
66 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
67 MPIR_ERR_RECOVERABLE,
68 myname, __LINE__,
69 MPI_ERR_ARG,
70 "Atomic noncontiguous writes are not supported by PVFS2", 0);
71 return;
72 }
73 /* --END ERROR HANDLING-- */
74
75 ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
76 ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
77
78 /* the HDF5 tests showed a bug in this list processing code (see many many
79 * lines down below). We added a workaround, but common HDF5 file types
80 * are actually contiguous and do not need the expensive workarond */
81 if (!filetype_is_contig) {
82 flat_file = ADIOI_Flatlist;
83 while (flat_file->type != fd->filetype) flat_file = flat_file->next;
84 if (flat_file->count == 1 && !buftype_is_contig)
85 filetype_is_contig = 1;
86 }
87
88 MPI_Type_size_x(fd->filetype, &filetype_size);
89 if ( ! filetype_size ) {
90 #ifdef HAVE_STATUS_SET_BYTES
91 MPIR_Status_set_bytes(status, datatype, 0);
92 #endif
93 *error_code = MPI_SUCCESS;
94 return;
95 }
96
97 MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
98 MPI_Type_size_x(datatype, &buftype_size);
99 MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
100 etype_size = fd->etype_size;
101
102 bufsize = buftype_size * count;
103
104 pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
105
106 if (!buftype_is_contig && filetype_is_contig) {
107
108 /* noncontiguous in memory, contiguous in file. */
109 int64_t file_offset;
110 int32_t file_length;
111
112 flat_buf = ADIOI_Flatten_and_find(datatype);
113
114 if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
115 off = fd->disp + etype_size * offset;
116 }
117 else off = fd->fp_ind;
118
119 file_list_count = 1;
120 file_offset = off;
121 file_length = 0;
122 total_blks_to_write = count*flat_buf->count;
123 b_blks_wrote = 0;
124
125 /* allocate arrays according to max usage */
126 if (total_blks_to_write > MAX_ARRAY_SIZE)
127 mem_list_count = MAX_ARRAY_SIZE;
128 else mem_list_count = total_blks_to_write;
129 mem_offsets = (PVFS_size*)ADIOI_Malloc(mem_list_count*sizeof(PVFS_size));
130 mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
131
132 j = 0;
133 /* step through each block in memory, filling memory arrays */
134 while (b_blks_wrote < total_blks_to_write) {
135 for (i=0; i<flat_buf->count; i++) {
136 mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] =
137 /* TODO: fix this warning by casting to an integer that's
138 * the same size as a char * and /then/ casting to
139 * PVFS_size */
140 ((PVFS_size)buf + j*buftype_extent + flat_buf->indices[i]);
141 mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] =
142 flat_buf->blocklens[i];
143 file_length += flat_buf->blocklens[i];
144 b_blks_wrote++;
145 if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
146 (b_blks_wrote == total_blks_to_write)) {
147
148 /* in the case of the last write list call,
149 adjust mem_list_count */
150 if (b_blks_wrote == total_blks_to_write) {
151 mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
152 /* in case last write list call fills max arrays */
153 if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
154 }
155 err_flag = PVFS_Request_hindexed(mem_list_count,
156 mem_lengths, mem_offsets,
157 PVFS_BYTE, &mem_req);
158 /* --BEGIN ERROR HANDLING-- */
159 if (err_flag != 0) {
160 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
161 MPIR_ERR_RECOVERABLE,
162 myname, __LINE__,
163 ADIOI_PVFS2_error_convert(err_flag),
164 "Error in PVFS_Request_hindexed (memory)", 0);
165 break;
166 }
167 /* --END ERROR HANDLING-- */
168
169 err_flag = PVFS_Request_contiguous(file_length,
170 PVFS_BYTE, &file_req);
171 /* --BEGIN ERROR HANDLING-- */
172 if (err_flag != 0) {
173 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
174 MPIR_ERR_RECOVERABLE,
175 myname, __LINE__,
176 ADIOI_PVFS2_error_convert(err_flag),
177 "Error in PVFS_Request_contiguous (file)", 0);
178 break;
179 }
180 /* --END ERROR HANDLING-- */
181
182 #ifdef ADIOI_MPE_LOGGING
183 MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
184 #endif
185 err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req,
186 file_offset, PVFS_BOTTOM,
187 mem_req,
188 &(pvfs_fs->credentials),
189 &resp_io);
190 #ifdef ADIOI_MPE_LOGGING
191 MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
192 #endif
193 total_bytes_written += resp_io.total_completed;
194
195 /* in the case of error or the last write list call,
196 * leave here */
197 /* --BEGIN ERROR HANDLING-- */
198 if (err_flag) {
199 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
200 MPIR_ERR_RECOVERABLE,
201 myname, __LINE__,
202 ADIOI_PVFS2_error_convert(err_flag),
203 "Error in PVFS_sys_write", 0);
204 break;
205 }
206 /* --END ERROR HANDLING-- */
207 if (b_blks_wrote == total_blks_to_write) break;
208
209 file_offset += file_length;
210 file_length = 0;
211 PVFS_Request_free(&mem_req);
212 PVFS_Request_free(&file_req);
213 }
214 } /* for (i=0; i<flat_buf->count; i++) */
215 j++;
216 } /* while (b_blks_wrote < total_blks_to_write) */
217 ADIOI_Free(mem_offsets);
218 ADIOI_Free(mem_lengths);
219
220 if (file_ptr_type == ADIO_INDIVIDUAL)
221 fd->fp_ind += total_bytes_written;
222
223 if (!err_flag) *error_code = MPI_SUCCESS;
224
225 fd->fp_sys_posn = -1; /* clear this. */
226
227 #ifdef HAVE_STATUS_SET_BYTES
228 MPIR_Status_set_bytes(status, datatype, bufsize);
229 /* This is a temporary way of filling in status. The right way is to
230 keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
231 #endif
232
233 ADIOI_Delete_flattened(datatype);
234 return;
235 } /* if (!buftype_is_contig && filetype_is_contig) */
236
237 /* already know that file is noncontiguous from above */
238 /* noncontiguous in file */
239
240 /* filetype already flattened in ADIO_Open */
241 flat_file = ADIOI_Flatlist;
242 while (flat_file->type != fd->filetype) flat_file = flat_file->next;
243
244 disp = fd->disp;
245 initial_off = offset;
246
247 /* for each case - ADIO_Individual pointer or explicit, find offset
248 (file offset in bytes), n_filetypes (how many filetypes into file
249 to start), fwr_size (remaining amount of data in present file
250 block), and st_index (start point in terms of blocks in starting
251 filetype) */
252 if (file_ptr_type == ADIO_INDIVIDUAL) {
253 offset = fd->fp_ind; /* in bytes */
254 n_filetypes = -1;
255 flag = 0;
256 while (!flag) {
257 n_filetypes++;
258 for (i=0; i<flat_file->count; i++) {
259 if (disp + flat_file->indices[i] +
260 ((ADIO_Offset) n_filetypes)*filetype_extent +
261 flat_file->blocklens[i] >= offset) {
262 st_index = i;
263 fwr_size = disp + flat_file->indices[i] +
264 ((ADIO_Offset) n_filetypes)*filetype_extent
265 + flat_file->blocklens[i] - offset;
266 flag = 1;
267 break;
268 }
269 }
270 } /* while (!flag) */
271 } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
272 else {
273 n_etypes_in_filetype = filetype_size/etype_size;
274 n_filetypes = (int) (offset / n_etypes_in_filetype);
275 etype_in_filetype = (int) (offset % n_etypes_in_filetype);
276 size_in_filetype = etype_in_filetype * etype_size;
277
278 sum = 0;
279 for (i=0; i<flat_file->count; i++) {
280 sum += flat_file->blocklens[i];
281 if (sum > size_in_filetype) {
282 st_index = i;
283 fwr_size = sum - size_in_filetype;
284 abs_off_in_filetype = flat_file->indices[i] +
285 size_in_filetype - (sum - flat_file->blocklens[i]);
286 break;
287 }
288 }
289
290 /* abs. offset in bytes in the file */
291 offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent +
292 abs_off_in_filetype;
293 } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
294
295 start_off = offset;
296 st_fwr_size = fwr_size;
297 st_n_filetypes = n_filetypes;
298
299 if (buftype_is_contig && !filetype_is_contig) {
300
301 /* contiguous in memory, noncontiguous in file. should be the most
302 common case. */
303
304 int mem_length;
305 intptr_t mem_offset;
306
307 i = 0;
308 j = st_index;
309 off = offset;
310 n_filetypes = st_n_filetypes;
311
312 mem_list_count = 1;
313
314 /* determine how many blocks in file to write */
315 f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
316 total_blks_to_write = 1;
317 if (j < (flat_file->count -1)) j++;
318 else {
319 j = 0;
320 n_filetypes++;
321 }
322 while (f_data_wrote < bufsize) {
323 f_data_wrote += flat_file->blocklens[j];
324 total_blks_to_write++;
325 if (j<(flat_file->count-1)) j++;
326 else j = 0;
327 }
328
329 j = st_index;
330 n_filetypes = st_n_filetypes;
331 n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
332 extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
333
334 mem_offset = (intptr_t) buf;
335 mem_length = 0;
336
337 /* if at least one full writelist, allocate file arrays
338 at max array size and don't free until very end */
339 if (n_write_lists) {
340 file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
341 sizeof(int64_t));
342 file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
343 sizeof(int32_t));
344 }
345 /* if there's no full writelist allocate file arrays according
346 to needed size (extra_blks) */
347 else {
348 file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
349 sizeof(int64_t));
350 file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
351 sizeof(int32_t));
352 }
353
354 /* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
355 for (i=0; i<n_write_lists; i++) {
356 file_list_count = MAX_ARRAY_SIZE;
357 if(!i) {
358 file_offsets[0] = offset;
359 file_lengths[0] = st_fwr_size;
360 mem_length = st_fwr_size;
361 }
362 for (k=0; k<MAX_ARRAY_SIZE; k++) {
363 if (i || k) {
364 file_offsets[k] = disp +
365 ((ADIO_Offset)n_filetypes)*filetype_extent
366 + flat_file->indices[j];
367 file_lengths[k] = flat_file->blocklens[j];
368 mem_length += file_lengths[k];
369 }
370 if (j<(flat_file->count - 1)) j++;
371 else {
372 j = 0;
373 n_filetypes++;
374 }
375 } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
376
377 err_flag = PVFS_Request_contiguous(mem_length,
378 PVFS_BYTE, &mem_req);
379 /* --BEGIN ERROR HANDLING-- */
380 if (err_flag != 0) {
381 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
382 MPIR_ERR_RECOVERABLE,
383 myname, __LINE__,
384 ADIOI_PVFS2_error_convert(err_flag),
385 "Error in PVFS_Request_contiguous (memory)", 0);
386 goto error_state;
387 }
388 /* --END ERROR HANDLING-- */
389
390 err_flag = PVFS_Request_hindexed(file_list_count, file_lengths,
391 file_offsets, PVFS_BYTE,
392 &file_req);
393 /* --BEGIN ERROR HANDLING-- */
394 if (err_flag != 0) {
395 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
396 MPIR_ERR_RECOVERABLE,
397 myname, __LINE__,
398 ADIOI_PVFS2_error_convert(err_flag),
399 "Error in PVFS_Request_hindexed (file)", 0);
400 goto error_state;
401 }
402 /* --END ERROR HANDLING-- */
403
404 /* PVFS_Request_hindexed already expresses the offsets into the
405 * file, so we should not pass in an offset if we are using
406 * hindexed for the file type */
407 #ifdef ADIOI_MPE_LOGGING
408 MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
409 #endif
410 err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0,
411 (void *)mem_offset, mem_req,
412 &(pvfs_fs->credentials), &resp_io);
413 #ifdef ADIOI_MPE_LOGGING
414 MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
415 #endif
416 /* --BEGIN ERROR HANDLING-- */
417 if (err_flag != 0) {
418 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
419 MPIR_ERR_RECOVERABLE,
420 myname, __LINE__,
421 ADIOI_PVFS2_error_convert(err_flag),
422 "Error in PVFS_sys_write", 0);
423 goto error_state;
424 }
425 /* --END ERROR HANDLING-- */
426 total_bytes_written += resp_io.total_completed;
427
428 mem_offset += mem_length;
429 mem_lengths = 0;
430 PVFS_Request_free(&file_req);
431 PVFS_Request_free(&mem_req);
432
433 } /* for (i=0; i<n_write_lists; i++) */
434
435 /* for file arrays smaller than MAX_ARRAY_SIZE (last write_list call) */
436 if (extra_blks) {
437 file_list_count = extra_blks;
438 if(!i) {
439 file_offsets[0] = offset;
440 file_lengths[0] = ADIOI_MIN(st_fwr_size, bufsize);
441 }
442 for (k=0; k<extra_blks; k++) {
443 if(i || k) {
444 file_offsets[k] = disp +
445 ((ADIO_Offset)n_filetypes)*filetype_extent +
446 flat_file->indices[j];
447 if (k == (extra_blks - 1)) {
448 file_lengths[k] = bufsize - (int32_t) mem_lengths
449 - mem_offset + (int32_t) buf;
450 }
451 else file_lengths[k] = flat_file->blocklens[j];
452 } /* if(i || k) */
453 mem_lengths += file_lengths[k];
454 if (j<(flat_file->count - 1)) j++;
455 else {
456 j = 0;
457 n_filetypes++;
458 }
459 } /* for (k=0; k<extra_blks; k++) */
460
461 err_flag = PVFS_Request_contiguous(mem_length,
462 PVFS_BYTE, &mem_req);
463 /* --BEGIN ERROR HANDLING-- */
464 if (err_flag != 0) {
465 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
466 MPIR_ERR_RECOVERABLE,
467 myname, __LINE__,
468 ADIOI_PVFS2_error_convert(err_flag),
469 "Error in PVFS_Request_contiguous (memory)", 0);
470 goto error_state;
471 }
472 /* --END ERROR HANDLING-- */
473
474 err_flag = PVFS_Request_hindexed(file_list_count, file_lengths,
475 file_offsets, PVFS_BYTE,
476 &file_req);
477 /* --BEGIN ERROR HANDLING-- */
478 if (err_flag != 0) {
479 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
480 MPIR_ERR_RECOVERABLE,
481 myname, __LINE__,
482 ADIOI_PVFS2_error_convert(err_flag),
483 "Error in PVFS_Request_hindexed(file)", 0);
484 goto error_state;
485 }
486 /* --END ERROR HANDLING-- */
487
488 /* as above, use 0 for 'offset' when using hindexed file type*/
489 #ifdef ADIOI_MPE_LOGGING
490 MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
491 #endif
492 err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0,
493 (void *)mem_offset, mem_req,
494 &(pvfs_fs->credentials), &resp_io);
495 #ifdef ADIOI_MPE_LOGGING
496 MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
497 #endif
498 /* --BEGIN ERROR HANDLING-- */
499 if (err_flag != 0) {
500 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
501 MPIR_ERR_RECOVERABLE,
502 myname, __LINE__,
503 ADIOI_PVFS2_error_convert(err_flag),
504 "Error in PVFS_sys_write", 0);
505 goto error_state;
506 }
507 /* --END ERROR HANDLING-- */
508 total_bytes_written += resp_io.total_completed;
509 PVFS_Request_free(&mem_req);
510 PVFS_Request_free(&file_req);
511 }
512 }
513 else {
514 /* noncontiguous in memory as well as in file */
515
516 flat_buf = ADIOI_Flatten_and_find(datatype);
517
518 size_wrote = 0;
519 n_filetypes = st_n_filetypes;
520 fwr_size = st_fwr_size;
521 bwr_size = flat_buf->blocklens[0];
522 buf_count = 0;
523 start_mem_offset = 0;
524 start_k = k = 0;
525 start_j = st_index;
526 max_mem_list = 0;
527 max_file_list = 0;
528
529 /* run through and file max_file_list and max_mem_list so that you
530 can allocate the file and memory arrays less than MAX_ARRAY_SIZE
531 if possible */
532
533 while (size_wrote < bufsize) {
534 k = start_k;
535 new_buffer_write = 0;
536 mem_list_count = 0;
537 while ((mem_list_count < MAX_ARRAY_SIZE) &&
538 (new_buffer_write < bufsize-size_wrote)) {
539 /* find mem_list_count and file_list_count such that both are
540 less than MAX_ARRAY_SIZE, the sum of their lengths are
541 equal, and the sum of all the data written and data to be
542 written in the next immediate write list is less than
543 bufsize */
544 if(mem_list_count) {
545 if((new_buffer_write + flat_buf->blocklens[k] +
546 size_wrote) > bufsize) {
547 end_bwr_size = new_buffer_write +
548 flat_buf->blocklens[k] - (bufsize - size_wrote);
549 new_buffer_write = bufsize - size_wrote;
550 }
551 else {
552 new_buffer_write += flat_buf->blocklens[k];
553 end_bwr_size = flat_buf->blocklens[k];
554 }
555 }
556 else {
557 if (bwr_size > (bufsize - size_wrote)) {
558 new_buffer_write = bufsize - size_wrote;
559 bwr_size = new_buffer_write;
560 }
561 else new_buffer_write = bwr_size;
562 }
563 mem_list_count++;
564 k = (k + 1)%flat_buf->count;
565 } /* while ((mem_list_count < MAX_ARRAY_SIZE) &&
566 (new_buffer_write < bufsize-size_wrote)) */
567 j = start_j;
568 new_file_write = 0;
569 file_list_count = 0;
570 while ((file_list_count < MAX_ARRAY_SIZE) &&
571 (new_file_write < new_buffer_write)) {
572 if(file_list_count) {
573 if((new_file_write + flat_file->blocklens[j]) >
574 new_buffer_write) {
575 end_fwr_size = new_buffer_write - new_file_write;
576 new_file_write = new_buffer_write;
577 j--;
578 }
579 else {
580 new_file_write += flat_file->blocklens[j];
581 end_fwr_size = flat_file->blocklens[j];
582 }
583 }
584 else {
585 if (fwr_size > new_buffer_write) {
586 new_file_write = new_buffer_write;
587 fwr_size = new_file_write;
588 }
589 else new_file_write = fwr_size;
590 }
591 file_list_count++;
592 if (j < (flat_file->count - 1)) j++;
593 else j = 0;
594
595 k = start_k;
596 if ((new_file_write < new_buffer_write) &&
597 (file_list_count == MAX_ARRAY_SIZE)) {
598 new_buffer_write = 0;
599 mem_list_count = 0;
600 while (new_buffer_write < new_file_write) {
601 if(mem_list_count) {
602 if((new_buffer_write + flat_buf->blocklens[k]) >
603 new_file_write) {
604 end_bwr_size = new_file_write -
605 new_buffer_write;
606 new_buffer_write = new_file_write;
607 k--;
608 }
609 else {
610 new_buffer_write += flat_buf->blocklens[k];
611 end_bwr_size = flat_buf->blocklens[k];
612 }
613 }
614 else {
615 new_buffer_write = bwr_size;
616 if (bwr_size > (bufsize - size_wrote)) {
617 new_buffer_write = bufsize - size_wrote;
618 bwr_size = new_buffer_write;
619 }
620 }
621 mem_list_count++;
622 k = (k + 1)%flat_buf->count;
623 } /* while (new_buffer_write < new_file_write) */
624 } /* if ((new_file_write < new_buffer_write) &&
625 (file_list_count == MAX_ARRAY_SIZE)) */
626 } /* while ((mem_list_count < MAX_ARRAY_SIZE) &&
627 (new_buffer_write < bufsize-size_wrote)) */
628
629 /* fakes filling the writelist arrays of lengths found above */
630 k = start_k;
631 j = start_j;
632 for (i=0; i<mem_list_count; i++) {
633 if(i) {
634 if (i == (mem_list_count - 1)) {
635 if (flat_buf->blocklens[k] == end_bwr_size)
636 bwr_size = flat_buf->blocklens[(k+1)%
637 flat_buf->count];
638 else {
639 bwr_size = flat_buf->blocklens[k] - end_bwr_size;
640 k--;
641 buf_count--;
642 }
643 }
644 }
645 buf_count++;
646 k = (k + 1)%flat_buf->count;
647 } /* for (i=0; i<mem_list_count; i++) */
648 for (i=0; i<file_list_count; i++) {
649 if (i) {
650 if (i == (file_list_count - 1)) {
651 if (flat_file->blocklens[j] == end_fwr_size)
652 fwr_size = flat_file->blocklens[(j+1)%
653 flat_file->count];
654 else {
655 fwr_size = flat_file->blocklens[j] - end_fwr_size;
656 j--;
657 }
658 }
659 }
660 if (j < flat_file->count - 1) j++;
661 else {
662 j = 0;
663 n_filetypes++;
664 }
665 } /* for (i=0; i<file_list_count; i++) */
666 size_wrote += new_buffer_write;
667 start_k = k;
668 start_j = j;
669 if (max_mem_list < mem_list_count)
670 max_mem_list = mem_list_count;
671 if (max_file_list < file_list_count)
672 max_file_list = file_list_count;
673 } /* while (size_wrote < bufsize) */
674
675 /* one last check before we actually carry out the operation:
676 * this code has hard-to-fix bugs when a noncontiguous file type has
677 * such large pieces that the sum of the lengths of the memory type is
678 * not larger than one of those pieces (and vice versa for large memory
679 * types and many pices of file types. In these cases, give up and
680 * fall back to naive reads and writes. The testphdf5 test created a
681 * type with two very large memory regions and 600 very small file
682 * regions. The same test also created a type with one very large file
683 * region and many (700) very small memory regions. both cases caused
684 * problems for this code */
685
686 if ( ( (file_list_count == 1) &&
687 (new_file_write < flat_file->blocklens[0] ) ) ||
688 ((mem_list_count == 1) &&
689 (new_buffer_write < flat_buf->blocklens[0]) ) ||
690 ((file_list_count == MAX_ARRAY_SIZE) &&
691 (new_file_write < flat_buf->blocklens[0]) ) ||
692 ( (mem_list_count == MAX_ARRAY_SIZE) &&
693 (new_buffer_write < flat_file->blocklens[0])) )
694 {
695 ADIOI_Delete_flattened(datatype);
696 ADIOI_GEN_WriteStrided_naive(fd, buf, count, datatype,
697 file_ptr_type, initial_off, status, error_code);
698 return;
699 }
700
701
702 mem_offsets = (PVFS_size*)ADIOI_Malloc(max_mem_list*sizeof(PVFS_size));
703 mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
704 file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
705 file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
706
707 size_wrote = 0;
708 n_filetypes = st_n_filetypes;
709 fwr_size = st_fwr_size;
710 bwr_size = flat_buf->blocklens[0];
711 buf_count = 0;
712 start_mem_offset = 0;
713 start_k = k = 0;
714 start_j = st_index;
715
716 /* this section calculates mem_list_count and file_list_count
717 and also finds the possibly odd sized last array elements
718 in new_fwr_size and new_bwr_size */
719
720 while (size_wrote < bufsize) {
721 k = start_k;
722 new_buffer_write = 0;
723 mem_list_count = 0;
724 while ((mem_list_count < MAX_ARRAY_SIZE) &&
725 (new_buffer_write < bufsize-size_wrote)) {
726 /* find mem_list_count and file_list_count such that both are
727 less than MAX_ARRAY_SIZE, the sum of their lengths are
728 equal, and the sum of all the data written and data to be
729 written in the next immediate write list is less than
730 bufsize */
731 if(mem_list_count) {
732 if((new_buffer_write + flat_buf->blocklens[k] +
733 size_wrote) > bufsize) {
734 end_bwr_size = new_buffer_write +
735 flat_buf->blocklens[k] - (bufsize - size_wrote);
736 new_buffer_write = bufsize - size_wrote;
737 }
738 else {
739 new_buffer_write += flat_buf->blocklens[k];
740 end_bwr_size = flat_buf->blocklens[k];
741 }
742 }
743 else {
744 if (bwr_size > (bufsize - size_wrote)) {
745 new_buffer_write = bufsize - size_wrote;
746 bwr_size = new_buffer_write;
747 }
748 else new_buffer_write = bwr_size;
749 }
750 mem_list_count++;
751 k = (k + 1)%flat_buf->count;
752 } /* while ((mem_list_count < MAX_ARRAY_SIZE) &&
753 (new_buffer_write < bufsize-size_wrote)) */
754 j = start_j;
755 new_file_write = 0;
756 file_list_count = 0;
757 while ((file_list_count < MAX_ARRAY_SIZE) &&
758 (new_file_write < new_buffer_write)) {
759 if(file_list_count) {
760 if((new_file_write + flat_file->blocklens[j]) >
761 new_buffer_write) {
762 end_fwr_size = new_buffer_write - new_file_write;
763 new_file_write = new_buffer_write;
764 j--;
765 }
766 else {
767 new_file_write += flat_file->blocklens[j];
768 end_fwr_size = flat_file->blocklens[j];
769 }
770 }
771 else {
772 if (fwr_size > new_buffer_write) {
773 new_file_write = new_buffer_write;
774 fwr_size = new_file_write;
775 }
776 else new_file_write = fwr_size;
777 }
778 file_list_count++;
779 if (j < (flat_file->count - 1)) j++;
780 else j = 0;
781
782 k = start_k;
783 if ((new_file_write < new_buffer_write) &&
784 (file_list_count == MAX_ARRAY_SIZE)) {
785 new_buffer_write = 0;
786 mem_list_count = 0;
787 while (new_buffer_write < new_file_write) {
788 if(mem_list_count) {
789 if((new_buffer_write + flat_buf->blocklens[k]) >
790 new_file_write) {
791 end_bwr_size = new_file_write -
792 new_buffer_write;
793 new_buffer_write = new_file_write;
794 k--;
795 }
796 else {
797 new_buffer_write += flat_buf->blocklens[k];
798 end_bwr_size = flat_buf->blocklens[k];
799 }
800 }
801 else {
802 new_buffer_write = bwr_size;
803 if (bwr_size > (bufsize - size_wrote)) {
804 new_buffer_write = bufsize - size_wrote;
805 bwr_size = new_buffer_write;
806 }
807 }
808 mem_list_count++;
809 k = (k + 1)%flat_buf->count;
810 } /* while (new_buffer_write < new_file_write) */
811 } /* if ((new_file_write < new_buffer_write) &&
812 (file_list_count == MAX_ARRAY_SIZE)) */
813 } /* while ((mem_list_count < MAX_ARRAY_SIZE) &&
814 (new_buffer_write < bufsize-size_wrote)) */
815
816 /* fills the allocated writelist arrays */
817 k = start_k;
818 j = start_j;
819 for (i=0; i<mem_list_count; i++) {
820 /* TODO: fix this warning by casting to an integer that's the
821 * same size as a char * and /then/ casting to PVFS_size */
822 mem_offsets[i] = ((PVFS_size)buf + buftype_extent*
823 (buf_count/flat_buf->count) +
824 (int)flat_buf->indices[k]);
825
826 if(!i) {
827 mem_lengths[0] = bwr_size;
828 mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
829 }
830 else {
831 if (i == (mem_list_count - 1)) {
832 mem_lengths[i] = end_bwr_size;
833 if (flat_buf->blocklens[k] == end_bwr_size)
834 bwr_size = flat_buf->blocklens[(k+1)%
835 flat_buf->count];
836 else {
837 bwr_size = flat_buf->blocklens[k] - end_bwr_size;
838 k--;
839 buf_count--;
840 }
841 }
842 else {
843 mem_lengths[i] = flat_buf->blocklens[k];
844 }
845 }
846 buf_count++;
847 k = (k + 1)%flat_buf->count;
848 } /* for (i=0; i<mem_list_count; i++) */
849 for (i=0; i<file_list_count; i++) {
850 file_offsets[i] = disp + flat_file->indices[j] +
851 ((ADIO_Offset)n_filetypes) * filetype_extent;
852 if (!i) {
853 file_lengths[0] = fwr_size;
854 file_offsets[0] += flat_file->blocklens[j] - fwr_size;
855 }
856 else {
857 if (i == (file_list_count - 1)) {
858 file_lengths[i] = end_fwr_size;
859 if (flat_file->blocklens[j] == end_fwr_size)
860 fwr_size = flat_file->blocklens[(j+1)%
861 flat_file->count];
862 else {
863 fwr_size = flat_file->blocklens[j] - end_fwr_size;
864 j--;
865 }
866 }
867 else file_lengths[i] = flat_file->blocklens[j];
868 }
869 if (j < flat_file->count - 1) j++;
870 else {
871 j = 0;
872 n_filetypes++;
873 }
874 } /* for (i=0; i<file_list_count; i++) */
875
876 err_flag = PVFS_Request_hindexed(mem_list_count, mem_lengths,
877 mem_offsets, PVFS_BYTE, &mem_req);
878 /* --BEGIN ERROR HANDLING-- */
879 if (err_flag != 0 ) {
880 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
881 MPIR_ERR_RECOVERABLE,
882 myname, __LINE__,
883 ADIOI_PVFS2_error_convert(err_flag),
884 "Error in PVFS_Request_hindexed (memory)", 0);
885 goto error_state;
886 }
887 /* --END ERROR HANDLING-- */
888
889 err_flag = PVFS_Request_hindexed(file_list_count, file_lengths,
890 file_offsets, PVFS_BYTE,
891 &file_req);
892 /* --BEGIN ERROR HANDLING-- */
893 if (err_flag != 0) {
894 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
895 MPIR_ERR_RECOVERABLE,
896 myname, __LINE__,
897 ADIOI_PVFS2_error_convert(err_flag),
898 "Error in PVFS_Request_hindexed", 0);
899 goto error_state;
900 }
901 /* --END ERROR HANDLING-- */
902
903 /* offset will be expressed in memory and file datatypes */
904
905 #ifdef ADIOI_MPE_LOGGING
906 MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
907 #endif
908 err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0,
909 PVFS_BOTTOM, mem_req,
910 &(pvfs_fs->credentials), &resp_io);
911 #ifdef ADIOI_MPE_LOGGING
912 MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
913 #endif
914 /* --BEGIN ERROR HANDLING-- */
915 if (err_flag != 0) {
916 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
917 MPIR_ERR_RECOVERABLE,
918 myname, __LINE__,
919 ADIOI_PVFS2_error_convert(err_flag),
920 "Error in PVFS_sys_write", 0);
921 goto error_state;
922 }
923 /* --END ERROR HANDLING-- */
924
925 size_wrote += new_buffer_write;
926 total_bytes_written += resp_io.total_completed;
927 start_k = k;
928 start_j = j;
929 PVFS_Request_free(&mem_req);
930 PVFS_Request_free(&file_req);
931 } /* while (size_wrote < bufsize) */
932 ADIOI_Free(mem_offsets);
933 ADIOI_Free(mem_lengths);
934 }
935 /* when incrementing fp_ind, need to also take into account the file type:
936 * consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
937 * if we wrote N elements, offset needs to point at beginning of type, not
938 * at empty region at offset N+1).
939 *
940 * As we discussed on mpich-discuss in may/june 2009, the code below might
941 * look wierd, but by putting fp_ind at the last byte written, the next
942 * time we run through the strided code we'll update the fp_ind to the
943 * right location. */
944 if (file_ptr_type == ADIO_INDIVIDUAL) {
945 fd->fp_ind = file_offsets[file_list_count-1]+
946 file_lengths[file_list_count-1];
947 }
948 ADIOI_Free(file_offsets);
949 ADIOI_Free(file_lengths);
950
951 *error_code = MPI_SUCCESS;
952
953 error_state:
954 fd->fp_sys_posn = -1; /* set it to null. */
955
956 #ifdef HAVE_STATUS_SET_BYTES
957 MPIR_Status_set_bytes(status, datatype, bufsize);
958 /* This is a temporary way of filling in status. The right way is to
959 keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
960 #endif
961
962 if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
963 }