ompi/mca/io/romio321/romio/adio/ad_zoidfs/ad_zoidfs_write

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
ADIOI_ZOIDFS_WriteStrided
   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
   2  * vim: ts=8 sts=4 sw=4 noexpandtab 
   3  * 
   4  *   Copyright (C) 2008 University of Chicago. 
   5  *   See COPYRIGHT notice in top-level directory.
   6  */
   7 
   8 #include "adio.h"
   9 #include "adio_extern.h"
  10 #include "ad_zoidfs.h"
  11 
  12 #include "ad_zoidfs_common.h"
  13 
  14 /* Copied from ADIOI_PVFS2_OldWriteStrided.  It would be good to have fewer
  15  * copies of this code... */
  16 void ADIOI_ZOIDFS_WriteStrided(ADIO_File fd, void *buf, int count,
  17                         MPI_Datatype datatype, int file_ptr_type,
  18                         ADIO_Offset offset, ADIO_Status *status,
  19                         int *error_code)
  20 {
  21     /* as with all the other WriteStrided functions, offset is in units of
  22      * etype relative to the filetype */
  23 
  24     /* Since zoidfs does not support file locking, can't do buffered writes
  25        as on Unix */
  26 
  27     ADIOI_Flatlist_node *flat_buf, *flat_file;
  28     int i, j, k, bwr_size, fwr_size=0, st_index=0;
  29     int sum, n_etypes_in_filetype, size_in_filetype;
  30     MPI_Count bufsize;
  31     int n_filetypes, etype_in_filetype;
  32     ADIO_Offset abs_off_in_filetype=0;
  33     MPI_Count filetype_size, etype_size, buftype_size;
  34     MPI_Aint filetype_extent, buftype_extent, filetype_lb, buftype_lb;
  35     int buf_count, buftype_is_contig, filetype_is_contig;
  36     ADIO_Offset off, disp, start_off, initial_off;
  37     int flag, st_fwr_size, st_n_filetypes;
  38     int err_flag=0;
  39 
  40     size_t mem_list_count, file_list_count;
  41     const void ** mem_offsets;
  42     uint64_t *file_offsets;
  43     size_t *mem_lengths;
  44     uint64_t *file_lengths;
  45     int total_blks_to_write;
  46 
  47     int max_mem_list, max_file_list;
  48 
  49     int b_blks_wrote;
  50     int f_data_wrote;
  51     int size_wrote=0, n_write_lists, extra_blks;
  52 
  53     int end_bwr_size, end_fwr_size;
  54     int start_k, start_j, new_file_write, new_buffer_write;
  55     int start_mem_offset;
  56     ADIOI_ZOIDFS_object *zoidfs_obj_ptr;
  57     MPI_Offset total_bytes_written=0;
  58     static char myname[] = "ADIOI_ZOIDFS_WRITESTRIDED";
  59 
  60     /* note: I don't know what zoidfs will do if you pass it a super-long list,
  61      * so let's keep with the PVFS limit for now */
  62 #define MAX_ARRAY_SIZE 64
  63 
  64     /* --BEGIN ERROR HANDLING-- */
  65     if (fd->atomicity) {
  66         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
  67                                            MPIR_ERR_RECOVERABLE,
  68                                            myname, __LINE__,
  69                                            MPI_ERR_ARG,
  70                                            "Atomic noncontiguous writes are not supported by ZOIDFS", 0);
  71         return;
  72     }
  73     /* --END ERROR HANDLING-- */
  74 
  75     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
  76     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
  77 
  78     /* the HDF5 tests showed a bug in this list processing code (see many many
  79      * lines down below).  We added a workaround, but common HDF5 file types
  80      * are actually contiguous and do not need the expensive workarond */
  81     if (!filetype_is_contig) {
  82         flat_file = ADIOI_Flatlist;
  83         while (flat_file->type != fd->filetype) flat_file = flat_file->next;
  84         if (flat_file->count == 1 && !buftype_is_contig)
  85             filetype_is_contig = 1;
  86     }
  87 
  88     MPI_Type_size_x(fd->filetype, &filetype_size);
  89     if ( ! filetype_size ) {
  90 #ifdef HAVE_STATUS_SET_BYTES
  91         MPIR_Status_set_bytes(status, datatype, 0);
  92 #endif
  93         *error_code = MPI_SUCCESS; 
  94         return;
  95     }
  96 
  97     MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
  98     MPI_Type_size_x(datatype, &buftype_size);
  99     MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
 100     etype_size = fd->etype_size;
 101     
 102     bufsize = buftype_size * count;
 103 
 104     zoidfs_obj_ptr = (ADIOI_ZOIDFS_object*)fd->fs_ptr;
 105 
 106     if (!buftype_is_contig && filetype_is_contig) {
 107 
 108 /* noncontiguous in memory, contiguous in file.  */
 109         uint64_t file_offsets;
 110         uint64_t file_lengths;
 111 
 112         flat_buf = ADIOI_Flatten_and_find(datatype);
 113         
 114         if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
 115             off = fd->disp + etype_size * offset;
 116         }
 117         else off = fd->fp_ind;
 118 
 119         file_list_count = 1;
 120         file_offsets = off;
 121         file_lengths = 0;
 122         total_blks_to_write = count*flat_buf->count;
 123         b_blks_wrote = 0;
 124 
 125         /* allocate arrays according to max usage */
 126         if (total_blks_to_write > MAX_ARRAY_SIZE)
 127             mem_list_count = MAX_ARRAY_SIZE;
 128         else mem_list_count = total_blks_to_write;
 129         mem_offsets = (void*)ADIOI_Malloc(mem_list_count*sizeof(void*));
 130         mem_lengths = (size_t*)ADIOI_Malloc(mem_list_count*sizeof(size_t));
 131 
 132         j = 0;
 133         /* step through each block in memory, filling memory arrays */
 134         while (b_blks_wrote < total_blks_to_write) {
 135             for (i=0; i<flat_buf->count; i++) {
 136                 mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] = 
 137                     buf + 
 138                      j*buftype_extent + 
 139                      flat_buf->indices[i];
 140                 mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] = 
 141                     flat_buf->blocklens[i];
 142                 file_lengths += flat_buf->blocklens[i];
 143                 b_blks_wrote++;
 144                 if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
 145                     (b_blks_wrote == total_blks_to_write)) {
 146 
 147                     /* in the case of the last write list call,
 148                        adjust mem_list_count */
 149                     if (b_blks_wrote == total_blks_to_write) {
 150                         mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
 151                         /* in case last write list call fills max arrays */
 152                         if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
 153                     }
 154 #ifdef ADIOI_MPE_LOGGING
 155                     MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 156 #endif
 157                     NO_STALE(err_flag, fd, zoidfs_obj_ptr,
 158                                     zoidfs_write(zoidfs_obj_ptr, 
 159                                             mem_list_count,
 160                                             mem_offsets, mem_lengths, 
 161                                             1, &file_offsets, &file_lengths, ZOIDFS_NO_OP_HINT));
 162 
 163                     /* --BEGIN ERROR HANDLING-- */
 164                     if (err_flag != ZFS_OK) {
 165                         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 166                                                            MPIR_ERR_RECOVERABLE,
 167                                                            myname, __LINE__,
 168                                                            ADIOI_ZOIDFS_error_convert(err_flag),
 169                                                            "Error in zoidfs_write", 0);
 170                         break;
 171                     }
 172 #ifdef ADIOI_MPE_LOGGING
 173                     MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 174 #endif
 175                     total_bytes_written += file_lengths;
 176                   
 177                     /* in the case of error or the last write list call, 
 178                      * leave here */
 179                     /* --BEGIN ERROR HANDLING-- */
 180                     if (err_flag) {
 181                         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 182                                                            MPIR_ERR_RECOVERABLE,
 183                                                            myname, __LINE__,
 184                                                            ADIOI_ZOIDFS_error_convert(err_flag),
 185                                                            "Error in zoidfs_write", 0);
 186                         break;
 187                     }
 188                     /* --END ERROR HANDLING-- */
 189                     if (b_blks_wrote == total_blks_to_write) break;
 190 
 191                     file_offsets += file_lengths;
 192                     file_lengths = 0;
 193                 } 
 194             } /* for (i=0; i<flat_buf->count; i++) */
 195             j++;
 196         } /* while (b_blks_wrote < total_blks_to_write) */
 197         ADIOI_Free(mem_offsets);
 198         ADIOI_Free(mem_lengths);
 199 
 200         if (file_ptr_type == ADIO_INDIVIDUAL) 
 201             fd->fp_ind += total_bytes_written;
 202 
 203         if (!err_flag)  *error_code = MPI_SUCCESS;
 204 
 205         fd->fp_sys_posn = -1;   /* clear this. */
 206 
 207 #ifdef HAVE_STATUS_SET_BYTES
 208         MPIR_Status_set_bytes(status, datatype, bufsize);
 209 /* This is a temporary way of filling in status. The right way is to 
 210    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
 211 #endif
 212 
 213         ADIOI_Delete_flattened(datatype);
 214         return;
 215     } /* if (!buftype_is_contig && filetype_is_contig) */
 216 
 217     /* already know that file is noncontiguous from above */
 218     /* noncontiguous in file */
 219 
 220 /* filetype already flattened in ADIO_Open */
 221     flat_file = ADIOI_Flatlist;
 222     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
 223 
 224     disp = fd->disp;
 225     initial_off = offset;
 226 
 227     /* for each case - ADIO_Individual pointer or explicit, find offset
 228        (file offset in bytes), n_filetypes (how many filetypes into file 
 229        to start), fwr_size (remaining amount of data in present file
 230        block), and st_index (start point in terms of blocks in starting
 231        filetype) */
 232     if (file_ptr_type == ADIO_INDIVIDUAL) {
 233         offset = fd->fp_ind; /* in bytes */
 234         n_filetypes = -1;
 235         flag = 0;
 236         while (!flag) {
 237             n_filetypes++;
 238             for (i=0; i<flat_file->count; i++) {
 239                 if (disp + flat_file->indices[i] + 
 240                     ((ADIO_Offset) n_filetypes)*filetype_extent +
 241                       flat_file->blocklens[i] >= offset) {
 242                   st_index = i;
 243                   fwr_size = disp + flat_file->indices[i] + 
 244                     ((ADIO_Offset) n_filetypes)*filetype_extent
 245                     + flat_file->blocklens[i] - offset;
 246                   flag = 1;
 247                   break;
 248                 }
 249             }
 250         } /* while (!flag) */
 251     } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
 252     else {
 253         n_etypes_in_filetype = filetype_size/etype_size;
 254         n_filetypes = (int) (offset / n_etypes_in_filetype);
 255         etype_in_filetype = (int) (offset % n_etypes_in_filetype);
 256         size_in_filetype = etype_in_filetype * etype_size;
 257         
 258         sum = 0;
 259         for (i=0; i<flat_file->count; i++) {
 260             sum += flat_file->blocklens[i];
 261             if (sum > size_in_filetype) {
 262                 st_index = i;
 263                 fwr_size = sum - size_in_filetype;
 264                 abs_off_in_filetype = flat_file->indices[i] +
 265                     size_in_filetype - (sum - flat_file->blocklens[i]);
 266                 break;
 267             }
 268         }
 269 
 270         /* abs. offset in bytes in the file */
 271         offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent +
 272             abs_off_in_filetype;
 273     } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
 274 
 275     start_off = offset;
 276     st_fwr_size = fwr_size;
 277     st_n_filetypes = n_filetypes;
 278     
 279     if (buftype_is_contig && !filetype_is_contig) {
 280 
 281 /* contiguous in memory, noncontiguous in file. should be the most
 282    common case. */
 283 
 284         /* only one memory off-len pair, so no array */
 285         size_t mem_lengths;
 286         size_t mem_offsets;
 287         
 288         i = 0;
 289         j = st_index;
 290         off = offset;
 291         n_filetypes = st_n_filetypes;
 292         
 293         mem_list_count = 1;
 294         
 295         /* determine how many blocks in file to write */
 296         f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
 297         total_blks_to_write = 1;
 298         if (j < (flat_file->count -1)) j++;
 299         else {
 300             j = 0;
 301             n_filetypes++;
 302         }
 303         while (f_data_wrote < bufsize) {
 304             f_data_wrote += flat_file->blocklens[j];
 305             total_blks_to_write++;
 306             if (j<(flat_file->count-1)) j++;
 307             else j = 0; 
 308         }
 309             
 310         j = st_index;
 311         n_filetypes = st_n_filetypes;
 312         n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
 313         extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
 314         
 315         mem_offsets = (size_t)buf;
 316         mem_lengths = 0;
 317         
 318         /* if at least one full writelist, allocate file arrays
 319            at max array size and don't free until very end */
 320         if (n_write_lists) {
 321             file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
 322                                                   sizeof(int64_t));
 323             file_lengths = (uint64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
 324                                                   sizeof(uint64_t));
 325         }
 326         /* if there's no full writelist allocate file arrays according
 327            to needed size (extra_blks) */
 328         else {
 329             file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
 330                                                   sizeof(int64_t));
 331             file_lengths = (uint64_t*)ADIOI_Malloc(extra_blks*
 332                                                   sizeof(uint64_t));
 333         }
 334         
 335         /* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
 336         for (i=0; i<n_write_lists; i++) {
 337             file_list_count = MAX_ARRAY_SIZE;
 338             if(!i) {
 339                 file_offsets[0] = offset;
 340                 file_lengths[0] = st_fwr_size;
 341                 mem_lengths = st_fwr_size;
 342             }
 343             for (k=0; k<MAX_ARRAY_SIZE; k++) {
 344                 if (i || k) {
 345                     file_offsets[k] = disp + 
 346                         ((ADIO_Offset)n_filetypes)*filetype_extent
 347                         + flat_file->indices[j];
 348                     file_lengths[k] = flat_file->blocklens[j];
 349                     mem_lengths += file_lengths[k];
 350                 }
 351                 if (j<(flat_file->count - 1)) j++;
 352                 else {
 353                     j = 0;
 354                     n_filetypes++;
 355                 }
 356             } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
 357 #ifdef ADIOI_MPE_LOGGING
 358             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 359 #endif
 360             NO_STALE(err_flag, fd, zoidfs_obj_ptr,
 361                             zoidfs_write(zoidfs_obj_ptr,
 362                                     1, buf, &mem_lengths,
 363                                     file_list_count, 
 364                                     file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
 365 
 366 #ifdef ADIOI_MPE_LOGGING
 367             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 368 #endif
 369             /* --BEGIN ERROR HANDLING-- */
 370             if (err_flag != ZFS_OK) {
 371                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 372                                                    MPIR_ERR_RECOVERABLE,
 373                                                    myname, __LINE__,
 374                                                    ADIOI_ZOIDFS_error_convert(err_flag),
 375                                                    "Error in zoidfs_write", 0);
 376                 goto error_state;
 377             }
 378             /* --END ERROR HANDLING-- */
 379             total_bytes_written += mem_lengths;
 380 
 381             mem_offsets += mem_lengths;
 382             mem_lengths = 0;
 383 
 384         } /* for (i=0; i<n_write_lists; i++) */
 385 
 386         /* for file arrays smaller than MAX_ARRAY_SIZE (last write_list call) */
 387         if (extra_blks) {
 388             file_list_count = extra_blks;
 389             if(!i) {
 390                 file_offsets[0] = offset;
 391                 file_lengths[0] = ADIOI_MIN(st_fwr_size, bufsize);
 392             }
 393             for (k=0; k<extra_blks; k++) {
 394                 if(i || k) {
 395                     file_offsets[k] = disp + 
 396                         ((ADIO_Offset)n_filetypes)*filetype_extent +
 397                         flat_file->indices[j];
 398                     /* XXX: double-check these casts  */
 399                     if (k == (extra_blks - 1)) {
 400                         file_lengths[k] = bufsize 
 401                                 - mem_lengths - mem_offsets +  (size_t)buf;
 402                     }
 403                     else file_lengths[k] = flat_file->blocklens[j];
 404                 } /* if(i || k) */
 405                 mem_lengths += file_lengths[k];
 406                 if (j<(flat_file->count - 1)) j++;
 407                 else {
 408                     j = 0;
 409                     n_filetypes++;
 410                 }
 411             } /* for (k=0; k<extra_blks; k++) */
 412 
 413 #ifdef ADIOI_MPE_LOGGING
 414             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 415 #endif
 416             NO_STALE(err_flag, fd, zoidfs_obj_ptr, 
 417                             zoidfs_write(zoidfs_obj_ptr, 1, 
 418                                     (const void **)&mem_offsets, 
 419                                     &mem_lengths,
 420                                     file_list_count, 
 421                                     file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
 422 
 423 #ifdef ADIOI_MPE_LOGGING
 424             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 425 #endif
 426             /* --BEGIN ERROR HANDLING-- */
 427             if (err_flag != 0) {
 428                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 429                                                    MPIR_ERR_RECOVERABLE,
 430                                                    myname, __LINE__,
 431                                                    ADIOI_ZOIDFS_error_convert(err_flag),
 432                                                    "Error in zoidfs_write", 0);
 433                 goto error_state;
 434             }
 435             /* --END ERROR HANDLING-- */
 436             total_bytes_written += mem_lengths;
 437         }
 438     } 
 439     else {
 440         /* noncontiguous in memory as well as in file */
 441 
 442         flat_buf = ADIOI_Flatten_and_find(datatype);
 443 
 444         size_wrote = 0;
 445         n_filetypes = st_n_filetypes;
 446         fwr_size = st_fwr_size;
 447         bwr_size = flat_buf->blocklens[0];
 448         buf_count = 0;
 449         start_mem_offset = 0;
 450         start_k = k = 0;
 451         start_j = st_index;
 452         max_mem_list = 0;
 453         max_file_list = 0;
 454 
 455         /* run through and file max_file_list and max_mem_list so that you 
 456            can allocate the file and memory arrays less than MAX_ARRAY_SIZE
 457            if possible */
 458 
 459         while (size_wrote < bufsize) {
 460             k = start_k;
 461             new_buffer_write = 0;
 462             mem_list_count = 0;
 463             while ((mem_list_count < MAX_ARRAY_SIZE) && 
 464                    (new_buffer_write < bufsize-size_wrote)) {
 465                 /* find mem_list_count and file_list_count such that both are
 466                    less than MAX_ARRAY_SIZE, the sum of their lengths are
 467                    equal, and the sum of all the data written and data to be
 468                    written in the next immediate write list is less than
 469                    bufsize */
 470                 if(mem_list_count) {
 471                     if((new_buffer_write + flat_buf->blocklens[k] + 
 472                         size_wrote) > bufsize) {
 473                         end_bwr_size = new_buffer_write + 
 474                             flat_buf->blocklens[k] - (bufsize - size_wrote);
 475                         new_buffer_write = bufsize - size_wrote;
 476                     }
 477                     else {
 478                         new_buffer_write += flat_buf->blocklens[k];
 479                         end_bwr_size = flat_buf->blocklens[k];
 480                     }
 481                 }
 482                 else {
 483                     if (bwr_size > (bufsize - size_wrote)) {
 484                         new_buffer_write = bufsize - size_wrote;
 485                         bwr_size = new_buffer_write;
 486                     }
 487                     else new_buffer_write = bwr_size;
 488                 }
 489                 mem_list_count++;
 490                 k = (k + 1)%flat_buf->count;
 491              } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
 492                (new_buffer_write < bufsize-size_wrote)) */
 493             j = start_j;
 494             new_file_write = 0;
 495             file_list_count = 0;
 496             while ((file_list_count < MAX_ARRAY_SIZE) && 
 497                    (new_file_write < new_buffer_write)) { 
 498                 if(file_list_count) {
 499                     if((new_file_write + flat_file->blocklens[j]) > 
 500                        new_buffer_write) {
 501                         end_fwr_size = new_buffer_write - new_file_write;
 502                         new_file_write = new_buffer_write;
 503                         j--;
 504                     }
 505                     else {
 506                         new_file_write += flat_file->blocklens[j];
 507                         end_fwr_size = flat_file->blocklens[j];
 508                     }
 509                 }
 510                 else {
 511                     if (fwr_size > new_buffer_write) {
 512                         new_file_write = new_buffer_write;
 513                         fwr_size = new_file_write;
 514                     }
 515                     else new_file_write = fwr_size;
 516                 }
 517                 file_list_count++;
 518                 if (j < (flat_file->count - 1)) j++;
 519                 else j = 0;
 520                 
 521                 k = start_k;
 522                 if ((new_file_write < new_buffer_write) && 
 523                     (file_list_count == MAX_ARRAY_SIZE)) {
 524                     new_buffer_write = 0;
 525                     mem_list_count = 0;
 526                     while (new_buffer_write < new_file_write) {
 527                         if(mem_list_count) {
 528                             if((new_buffer_write + flat_buf->blocklens[k]) >
 529                                new_file_write) {
 530                                 end_bwr_size = new_file_write - 
 531                                     new_buffer_write;
 532                                 new_buffer_write = new_file_write;
 533                                 k--;
 534                             }
 535                             else {
 536                                 new_buffer_write += flat_buf->blocklens[k];
 537                                 end_bwr_size = flat_buf->blocklens[k];
 538                             }
 539                         }
 540                         else {
 541                             new_buffer_write = bwr_size;
 542                             if (bwr_size > (bufsize - size_wrote)) {
 543                                 new_buffer_write = bufsize - size_wrote;
 544                                 bwr_size = new_buffer_write;
 545                             }
 546                         }
 547                         mem_list_count++;
 548                         k = (k + 1)%flat_buf->count;
 549                     } /* while (new_buffer_write < new_file_write) */
 550                 } /* if ((new_file_write < new_buffer_write) &&
 551                      (file_list_count == MAX_ARRAY_SIZE)) */
 552             } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
 553                  (new_buffer_write < bufsize-size_wrote)) */
 554 
 555             /*  fakes filling the writelist arrays of lengths found above  */
 556             k = start_k;
 557             j = start_j;
 558             for (i=0; i<mem_list_count; i++) {       
 559                 if(i) {
 560                     if (i == (mem_list_count - 1)) {
 561                         if (flat_buf->blocklens[k] == end_bwr_size)
 562                             bwr_size = flat_buf->blocklens[(k+1)%
 563                                                           flat_buf->count];
 564                         else {
 565                             bwr_size = flat_buf->blocklens[k] - end_bwr_size;
 566                             k--;
 567                             buf_count--;
 568                         }
 569                     }
 570                 }
 571                 buf_count++;
 572                 k = (k + 1)%flat_buf->count;
 573             } /* for (i=0; i<mem_list_count; i++) */
 574             for (i=0; i<file_list_count; i++) {
 575                 if (i) {
 576                     if (i == (file_list_count - 1)) {
 577                         if (flat_file->blocklens[j] == end_fwr_size)
 578                             fwr_size = flat_file->blocklens[(j+1)%
 579                                                           flat_file->count];   
 580                         else {
 581                             fwr_size = flat_file->blocklens[j] - end_fwr_size;
 582                             j--;
 583                         }
 584                     }
 585                 }
 586                 if (j < flat_file->count - 1) j++;
 587                 else {
 588                     j = 0;
 589                     n_filetypes++;
 590                 }
 591             } /* for (i=0; i<file_list_count; i++) */
 592             size_wrote += new_buffer_write;
 593             start_k = k;
 594             start_j = j;
 595             if (max_mem_list < mem_list_count)
 596                 max_mem_list = mem_list_count;
 597             if (max_file_list < file_list_count)
 598                 max_file_list = file_list_count;
 599         } /* while (size_wrote < bufsize) */
 600 
 601         /* one last check before we actually carry out the operation:
 602          * this code has hard-to-fix bugs when a noncontiguous file type has
 603          * such large pieces that the sum of the lengths of the memory type is
 604          * not larger than one of those pieces (and vice versa for large memory
 605          * types and many pices of file types.  In these cases, give up and
 606          * fall back to naive reads and writes.  The testphdf5 test created a
 607          * type with two very large memory regions and 600 very small file
 608          * regions.  The same test also created a type with one very large file
 609          * region and many (700) very small memory regions.  both cases caused
 610          * problems for this code */
 611 
 612         if ( ( (file_list_count == 1) && 
 613                     (new_file_write < flat_file->blocklens[0] ) ) ||
 614                 ((mem_list_count == 1) && 
 615                     (new_buffer_write < flat_buf->blocklens[0]) ) ||
 616                 ((file_list_count == MAX_ARRAY_SIZE) && 
 617                     (new_file_write < flat_buf->blocklens[0]) ) ||
 618                 ( (mem_list_count == MAX_ARRAY_SIZE) &&
 619                     (new_buffer_write < flat_file->blocklens[0])) )
 620         {
 621             ADIOI_Delete_flattened(datatype);
 622             ADIOI_GEN_WriteStrided_naive(fd, buf, count, datatype,
 623                     file_ptr_type, initial_off, status, error_code);
 624             return;
 625         }
 626 
 627 
 628         mem_offsets = (void *)ADIOI_Malloc(max_mem_list*sizeof(void *));
 629         mem_lengths = (size_t*)ADIOI_Malloc(max_mem_list*sizeof(size_t));
 630         file_offsets = (uint64_t *)ADIOI_Malloc(max_file_list*sizeof(uint64_t));
 631         file_lengths = (uint64_t*)ADIOI_Malloc(max_file_list*sizeof(uint64_t));
 632             
 633         size_wrote = 0;
 634         n_filetypes = st_n_filetypes;
 635         fwr_size = st_fwr_size;
 636         bwr_size = flat_buf->blocklens[0];
 637         buf_count = 0;
 638         start_mem_offset = 0;
 639         start_k = k = 0;
 640         start_j = st_index;
 641 
 642         /*  this section calculates mem_list_count and file_list_count
 643             and also finds the possibly odd sized last array elements
 644             in new_fwr_size and new_bwr_size  */
 645         
 646         while (size_wrote < bufsize) {
 647             k = start_k;
 648             new_buffer_write = 0;
 649             mem_list_count = 0;
 650             while ((mem_list_count < MAX_ARRAY_SIZE) && 
 651                    (new_buffer_write < bufsize-size_wrote)) {
 652                 /* find mem_list_count and file_list_count such that both are
 653                    less than MAX_ARRAY_SIZE, the sum of their lengths are
 654                    equal, and the sum of all the data written and data to be
 655                    written in the next immediate write list is less than
 656                    bufsize */
 657                 if(mem_list_count) {
 658                     if((new_buffer_write + flat_buf->blocklens[k] + 
 659                         size_wrote) > bufsize) {
 660                         end_bwr_size = new_buffer_write + 
 661                             flat_buf->blocklens[k] - (bufsize - size_wrote);
 662                         new_buffer_write = bufsize - size_wrote;
 663                     }
 664                     else {
 665                         new_buffer_write += flat_buf->blocklens[k];
 666                         end_bwr_size = flat_buf->blocklens[k];
 667                     }
 668                 }
 669                 else {
 670                     if (bwr_size > (bufsize - size_wrote)) {
 671                         new_buffer_write = bufsize - size_wrote;
 672                         bwr_size = new_buffer_write;
 673                     }
 674                     else new_buffer_write = bwr_size;
 675                 }
 676                 mem_list_count++;
 677                 k = (k + 1)%flat_buf->count;
 678              } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
 679                (new_buffer_write < bufsize-size_wrote)) */
 680             j = start_j;
 681             new_file_write = 0;
 682             file_list_count = 0;
 683             while ((file_list_count < MAX_ARRAY_SIZE) && 
 684                    (new_file_write < new_buffer_write)) {
 685                 if(file_list_count) {
 686                     if((new_file_write + flat_file->blocklens[j]) > 
 687                        new_buffer_write) {
 688                         end_fwr_size = new_buffer_write - new_file_write;
 689                         new_file_write = new_buffer_write;
 690                         j--;
 691                     }
 692                     else {
 693                         new_file_write += flat_file->blocklens[j];
 694                         end_fwr_size = flat_file->blocklens[j];
 695                     }
 696                 }
 697                 else {
 698                     if (fwr_size > new_buffer_write) {
 699                         new_file_write = new_buffer_write;
 700                         fwr_size = new_file_write;
 701                     }
 702                     else new_file_write = fwr_size;
 703                 }
 704                 file_list_count++;
 705                 if (j < (flat_file->count - 1)) j++;
 706                 else j = 0;
 707                 
 708                 k = start_k;
 709                 if ((new_file_write < new_buffer_write) && 
 710                     (file_list_count == MAX_ARRAY_SIZE)) {
 711                     new_buffer_write = 0;
 712                     mem_list_count = 0;
 713                     while (new_buffer_write < new_file_write) {
 714                         if(mem_list_count) {
 715                             if((new_buffer_write + flat_buf->blocklens[k]) >
 716                                new_file_write) {
 717                                 end_bwr_size = new_file_write -
 718                                   new_buffer_write;
 719                                 new_buffer_write = new_file_write;
 720                                 k--;
 721                             }
 722                             else {
 723                                 new_buffer_write += flat_buf->blocklens[k];
 724                                 end_bwr_size = flat_buf->blocklens[k];
 725                             }
 726                         }
 727                         else {
 728                             new_buffer_write = bwr_size;
 729                             if (bwr_size > (bufsize - size_wrote)) {
 730                                 new_buffer_write = bufsize - size_wrote;
 731                                 bwr_size = new_buffer_write;
 732                             }
 733                         }
 734                         mem_list_count++;
 735                         k = (k + 1)%flat_buf->count;
 736                     } /* while (new_buffer_write < new_file_write) */
 737                 } /* if ((new_file_write < new_buffer_write) &&
 738                      (file_list_count == MAX_ARRAY_SIZE)) */
 739             } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
 740                  (new_buffer_write < bufsize-size_wrote)) */
 741 
 742             /*  fills the allocated writelist arrays  */
 743             k = start_k;
 744             j = start_j;
 745             for (i=0; i<mem_list_count; i++) {       
 746                 mem_offsets[i] = buf + 
 747                         buftype_extent* (buf_count/flat_buf->count) +
 748                                   flat_buf->indices[k];
 749                 
 750                 if(!i) {
 751                     mem_lengths[0] = bwr_size;
 752                     mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
 753                 }
 754                 else {
 755                     if (i == (mem_list_count - 1)) {
 756                         mem_lengths[i] = end_bwr_size;
 757                         if (flat_buf->blocklens[k] == end_bwr_size)
 758                             bwr_size = flat_buf->blocklens[(k+1)%
 759                                                           flat_buf->count];
 760                         else {
 761                             bwr_size = flat_buf->blocklens[k] - end_bwr_size;
 762                             k--;
 763                             buf_count--;
 764                         }
 765                     }
 766                     else {
 767                         mem_lengths[i] = flat_buf->blocklens[k];
 768                     }
 769                 }
 770                 buf_count++;
 771                 k = (k + 1)%flat_buf->count;
 772             } /* for (i=0; i<mem_list_count; i++) */
 773             for (i=0; i<file_list_count; i++) {
 774                 file_offsets[i] = disp + flat_file->indices[j] + 
 775                     ((ADIO_Offset)n_filetypes) * filetype_extent;
 776                 if (!i) {
 777                     file_lengths[0] = fwr_size;
 778                     file_offsets[0] += flat_file->blocklens[j] - fwr_size;
 779                 }
 780                 else {
 781                     if (i == (file_list_count - 1)) {
 782                         file_lengths[i] = end_fwr_size;
 783                         if (flat_file->blocklens[j] == end_fwr_size)
 784                             fwr_size = flat_file->blocklens[(j+1)%
 785                                                           flat_file->count];   
 786                         else {
 787                             fwr_size = flat_file->blocklens[j] - end_fwr_size;
 788                             j--;
 789                         }
 790                     }
 791                     else file_lengths[i] = flat_file->blocklens[j];
 792                 }
 793                 if (j < flat_file->count - 1) j++;
 794                 else {
 795                     j = 0;
 796                     n_filetypes++;
 797                 }
 798             } /* for (i=0; i<file_list_count; i++) */
 799 
 800 #ifdef ADIOI_MPE_LOGGING
 801             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 802 #endif
 803             NO_STALE(err_flag, fd, zoidfs_obj_ptr,
 804                             zoidfs_write(zoidfs_obj_ptr, 
 805                                     mem_list_count, mem_offsets, mem_lengths, 
 806                                     file_list_count, 
 807                                     file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
 808             /* --BEGIN ERROR HANDLING-- */
 809             if (err_flag != ZFS_OK) {
 810                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 811                                                    MPIR_ERR_RECOVERABLE,
 812                                                    myname, __LINE__,
 813                                                    ADIOI_ZOIDFS_error_convert(err_flag),
 814                                                    "Error in zoidfs_write", 0);
 815                 goto error_state;
 816             }
 817             /* --END ERROR HANDLING-- */
 818 #ifdef ADIOI_MPE_LOGGING
 819             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 820 #endif
 821             size_wrote += new_buffer_write;
 822             total_bytes_written += new_buffer_write; /* XXX: is this right? */
 823             start_k = k;
 824             start_j = j;
 825         } /* while (size_wrote < bufsize) */
 826         ADIOI_Free(mem_offsets);
 827         ADIOI_Free(mem_lengths);
 828     }
 829     /* when incrementing fp_ind, need to also take into account the file type:
 830      * consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
 831      * if we wrote N elements, offset needs to point at beginning of type, not
 832      * at empty region at offset N+1).  
 833      *
 834      * As we discussed on mpich-discuss in may/june 2009, the code below might
 835      * look wierd, but by putting fp_ind at the last byte written, the next
 836      * time we run through the strided code we'll update the fp_ind to the
 837      * right location. */
 838     if (file_ptr_type == ADIO_INDIVIDUAL) {
 839         fd->fp_ind = file_offsets[file_list_count-1]+
 840             file_lengths[file_list_count-1];
 841     }
 842     ADIOI_Free(file_offsets);
 843     ADIOI_Free(file_lengths);
 844 
 845     *error_code = MPI_SUCCESS;
 846 
 847 error_state:
 848     fd->fp_sys_posn = -1;   /* set it to null. */
 849 
 850 #ifdef HAVE_STATUS_SET_BYTES
 851     MPIR_Status_set_bytes(status, datatype, bufsize);
 852 /* This is a temporary way of filling in status. The right way is to 
 853    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
 854 #endif
 855 
 856     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
 857 }
/* [<][>][^][v][top][bottom][index][help] */
root/ompi/mca/io/romio321/romio/adio/ad_zoidfs/ad_zoidfs_write_list.c

DEFINITIONS
