root/ompi/mca/io/romio321/romio/adio/ad_pvfs2/ad_pvfs2_write_list_classic.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. ADIOI_PVFS2_OldWriteStrided

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
   2  * vim: ts=8 sts=4 sw=4 noexpandtab 
   3  * 
   4  *   Copyright (C) 2008 University of Chicago. 
   5  *   See COPYRIGHT notice in top-level directory.
   6  */
   7 
   8 #include "adio.h"
   9 #include "adio_extern.h"
  10 #include "ad_pvfs2.h"
  11 
  12 #include "ad_pvfs2_common.h"
  13 
  14 void ADIOI_PVFS2_OldWriteStrided(ADIO_File fd, const void *buf, int count,
  15                         MPI_Datatype datatype, int file_ptr_type,
  16                         ADIO_Offset offset, ADIO_Status *status,
  17                         int *error_code)
  18 {
  19     /* as with all the other WriteStrided functions, offset is in units of
  20      * etype relative to the filetype */
  21 
  22     /* Since PVFS2 does not support file locking, can't do buffered writes
  23        as on Unix */
  24 
  25     ADIOI_Flatlist_node *flat_buf, *flat_file;
  26     int i, j, k, bwr_size, fwr_size=0, st_index=0;
  27     int sum, n_etypes_in_filetype, size_in_filetype;
  28     MPI_Count bufsize;
  29     int n_filetypes, etype_in_filetype;
  30     ADIO_Offset abs_off_in_filetype=0;
  31     MPI_Count filetype_size, etype_size, buftype_size;
  32     MPI_Aint filetype_extent, buftype_extent, filetype_lb, buftype_lb;
  33     int buf_count, buftype_is_contig, filetype_is_contig;
  34     ADIO_Offset off, disp, start_off, initial_off;
  35     int flag, st_fwr_size, st_n_filetypes;
  36     int err_flag=0;
  37 
  38     int mem_list_count, file_list_count;
  39     PVFS_size * mem_offsets;
  40     int64_t *file_offsets;
  41     int *mem_lengths;
  42     int32_t *file_lengths;
  43     int total_blks_to_write;
  44 
  45     int max_mem_list, max_file_list;
  46 
  47     int b_blks_wrote;
  48     int f_data_wrote;
  49     int size_wrote=0, n_write_lists, extra_blks;
  50 
  51     int end_bwr_size, end_fwr_size;
  52     int start_k, start_j, new_file_write, new_buffer_write;
  53     int start_mem_offset;
  54     PVFS_Request mem_req, file_req;
  55     ADIOI_PVFS2_fs * pvfs_fs;
  56     PVFS_sysresp_io resp_io;
  57     MPI_Offset total_bytes_written=0;
  58     static char myname[] = "ADIOI_PVFS2_WRITESTRIDED";
  59 
  60     /* note: don't increase this: several parts of PVFS2 now 
  61      * assume this limit*/
  62 #define MAX_ARRAY_SIZE 64
  63 
  64     /* --BEGIN ERROR HANDLING-- */
  65     if (fd->atomicity) {
  66         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
  67                                            MPIR_ERR_RECOVERABLE,
  68                                            myname, __LINE__,
  69                                            MPI_ERR_ARG,
  70                                            "Atomic noncontiguous writes are not supported by PVFS2", 0);
  71         return;
  72     }
  73     /* --END ERROR HANDLING-- */
  74 
  75     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
  76     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
  77 
  78     /* the HDF5 tests showed a bug in this list processing code (see many many
  79      * lines down below).  We added a workaround, but common HDF5 file types
  80      * are actually contiguous and do not need the expensive workarond */
  81     if (!filetype_is_contig) {
  82         flat_file = ADIOI_Flatlist;
  83         while (flat_file->type != fd->filetype) flat_file = flat_file->next;
  84         if (flat_file->count == 1 && !buftype_is_contig)
  85             filetype_is_contig = 1;
  86     }
  87 
  88     MPI_Type_size_x(fd->filetype, &filetype_size);
  89     if ( ! filetype_size ) {
  90 #ifdef HAVE_STATUS_SET_BYTES
  91         MPIR_Status_set_bytes(status, datatype, 0);
  92 #endif
  93         *error_code = MPI_SUCCESS; 
  94         return;
  95     }
  96 
  97     MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
  98     MPI_Type_size_x(datatype, &buftype_size);
  99     MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
 100     etype_size = fd->etype_size;
 101     
 102     bufsize = buftype_size * count;
 103 
 104     pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
 105 
 106     if (!buftype_is_contig && filetype_is_contig) {
 107 
 108 /* noncontiguous in memory, contiguous in file.  */
 109        int64_t file_offset;
 110         int32_t file_length;
 111 
 112         flat_buf = ADIOI_Flatten_and_find(datatype);
 113         
 114         if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
 115             off = fd->disp + etype_size * offset;
 116         }
 117         else off = fd->fp_ind;
 118 
 119         file_list_count = 1;
 120         file_offset = off;
 121         file_length = 0;
 122         total_blks_to_write = count*flat_buf->count;
 123         b_blks_wrote = 0;
 124 
 125         /* allocate arrays according to max usage */
 126         if (total_blks_to_write > MAX_ARRAY_SIZE)
 127             mem_list_count = MAX_ARRAY_SIZE;
 128         else mem_list_count = total_blks_to_write;
 129         mem_offsets = (PVFS_size*)ADIOI_Malloc(mem_list_count*sizeof(PVFS_size));
 130         mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
 131 
 132         j = 0;
 133         /* step through each block in memory, filling memory arrays */
 134         while (b_blks_wrote < total_blks_to_write) {
 135             for (i=0; i<flat_buf->count; i++) {
 136                 mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] = 
 137                     /* TODO: fix this warning by casting to an integer that's
 138                      * the same size as a char * and /then/ casting to
 139                      * PVFS_size */
 140                     ((PVFS_size)buf + j*buftype_extent + flat_buf->indices[i]);
 141                 mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] = 
 142                     flat_buf->blocklens[i];
 143                 file_length += flat_buf->blocklens[i];
 144                 b_blks_wrote++;
 145                 if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
 146                     (b_blks_wrote == total_blks_to_write)) {
 147 
 148                     /* in the case of the last write list call,
 149                        adjust mem_list_count */
 150                     if (b_blks_wrote == total_blks_to_write) {
 151                         mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
 152                         /* in case last write list call fills max arrays */
 153                         if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
 154                     }
 155                     err_flag = PVFS_Request_hindexed(mem_list_count, 
 156                                                      mem_lengths, mem_offsets,
 157                                                      PVFS_BYTE, &mem_req);
 158                     /* --BEGIN ERROR HANDLING-- */
 159                     if (err_flag != 0) {
 160                         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 161                                                            MPIR_ERR_RECOVERABLE,
 162                                                            myname, __LINE__,
 163                                                            ADIOI_PVFS2_error_convert(err_flag),
 164                                                            "Error in PVFS_Request_hindexed (memory)", 0);
 165                         break;
 166                     }
 167                     /* --END ERROR HANDLING-- */
 168 
 169                     err_flag = PVFS_Request_contiguous(file_length,
 170                                                        PVFS_BYTE, &file_req);
 171                     /* --BEGIN ERROR HANDLING-- */
 172                     if (err_flag != 0) {
 173                         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 174                                                            MPIR_ERR_RECOVERABLE,
 175                                                            myname, __LINE__,
 176                                                            ADIOI_PVFS2_error_convert(err_flag),
 177                                                            "Error in PVFS_Request_contiguous (file)", 0);
 178                         break;
 179                     }
 180                     /* --END ERROR HANDLING-- */
 181 
 182 #ifdef ADIOI_MPE_LOGGING
 183                     MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 184 #endif
 185                     err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 
 186                                               file_offset, PVFS_BOTTOM,
 187                                               mem_req, 
 188                                               &(pvfs_fs->credentials),
 189                                               &resp_io);
 190 #ifdef ADIOI_MPE_LOGGING
 191                     MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 192 #endif
 193                     total_bytes_written += resp_io.total_completed;
 194                   
 195                     /* in the case of error or the last write list call, 
 196                      * leave here */
 197                     /* --BEGIN ERROR HANDLING-- */
 198                     if (err_flag) {
 199                         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 200                                                            MPIR_ERR_RECOVERABLE,
 201                                                            myname, __LINE__,
 202                                                            ADIOI_PVFS2_error_convert(err_flag),
 203                                                            "Error in PVFS_sys_write", 0);
 204                         break;
 205                     }
 206                     /* --END ERROR HANDLING-- */
 207                     if (b_blks_wrote == total_blks_to_write) break;
 208 
 209                     file_offset += file_length;
 210                     file_length = 0;
 211                     PVFS_Request_free(&mem_req);
 212                     PVFS_Request_free(&file_req);
 213                 } 
 214             } /* for (i=0; i<flat_buf->count; i++) */
 215             j++;
 216         } /* while (b_blks_wrote < total_blks_to_write) */
 217         ADIOI_Free(mem_offsets);
 218         ADIOI_Free(mem_lengths);
 219 
 220         if (file_ptr_type == ADIO_INDIVIDUAL) 
 221             fd->fp_ind += total_bytes_written;
 222 
 223         if (!err_flag)  *error_code = MPI_SUCCESS;
 224 
 225         fd->fp_sys_posn = -1;   /* clear this. */
 226 
 227 #ifdef HAVE_STATUS_SET_BYTES
 228         MPIR_Status_set_bytes(status, datatype, bufsize);
 229 /* This is a temporary way of filling in status. The right way is to 
 230    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
 231 #endif
 232 
 233         ADIOI_Delete_flattened(datatype);
 234         return;
 235     } /* if (!buftype_is_contig && filetype_is_contig) */
 236 
 237     /* already know that file is noncontiguous from above */
 238     /* noncontiguous in file */
 239 
 240 /* filetype already flattened in ADIO_Open */
 241     flat_file = ADIOI_Flatlist;
 242     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
 243 
 244     disp = fd->disp;
 245     initial_off = offset;
 246 
 247     /* for each case - ADIO_Individual pointer or explicit, find offset
 248        (file offset in bytes), n_filetypes (how many filetypes into file 
 249        to start), fwr_size (remaining amount of data in present file
 250        block), and st_index (start point in terms of blocks in starting
 251        filetype) */
 252     if (file_ptr_type == ADIO_INDIVIDUAL) {
 253         offset = fd->fp_ind; /* in bytes */
 254         n_filetypes = -1;
 255         flag = 0;
 256         while (!flag) {
 257             n_filetypes++;
 258             for (i=0; i<flat_file->count; i++) {
 259                 if (disp + flat_file->indices[i] + 
 260                     ((ADIO_Offset) n_filetypes)*filetype_extent +
 261                       flat_file->blocklens[i] >= offset) {
 262                   st_index = i;
 263                   fwr_size = disp + flat_file->indices[i] + 
 264                     ((ADIO_Offset) n_filetypes)*filetype_extent
 265                     + flat_file->blocklens[i] - offset;
 266                   flag = 1;
 267                   break;
 268                 }
 269             }
 270         } /* while (!flag) */
 271     } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
 272     else {
 273         n_etypes_in_filetype = filetype_size/etype_size;
 274         n_filetypes = (int) (offset / n_etypes_in_filetype);
 275         etype_in_filetype = (int) (offset % n_etypes_in_filetype);
 276         size_in_filetype = etype_in_filetype * etype_size;
 277         
 278         sum = 0;
 279         for (i=0; i<flat_file->count; i++) {
 280             sum += flat_file->blocklens[i];
 281             if (sum > size_in_filetype) {
 282                 st_index = i;
 283                 fwr_size = sum - size_in_filetype;
 284                 abs_off_in_filetype = flat_file->indices[i] +
 285                     size_in_filetype - (sum - flat_file->blocklens[i]);
 286                 break;
 287             }
 288         }
 289 
 290         /* abs. offset in bytes in the file */
 291         offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent +
 292             abs_off_in_filetype;
 293     } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
 294 
 295     start_off = offset;
 296     st_fwr_size = fwr_size;
 297     st_n_filetypes = n_filetypes;
 298     
 299     if (buftype_is_contig && !filetype_is_contig) {
 300 
 301 /* contiguous in memory, noncontiguous in file. should be the most
 302    common case. */
 303 
 304        int mem_length;
 305         intptr_t mem_offset;
 306         
 307         i = 0;
 308         j = st_index;
 309         off = offset;
 310         n_filetypes = st_n_filetypes;
 311         
 312         mem_list_count = 1;
 313         
 314         /* determine how many blocks in file to write */
 315         f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
 316         total_blks_to_write = 1;
 317         if (j < (flat_file->count -1)) j++;
 318         else {
 319             j = 0;
 320             n_filetypes++;
 321         }
 322         while (f_data_wrote < bufsize) {
 323             f_data_wrote += flat_file->blocklens[j];
 324             total_blks_to_write++;
 325             if (j<(flat_file->count-1)) j++;
 326             else j = 0; 
 327         }
 328             
 329         j = st_index;
 330         n_filetypes = st_n_filetypes;
 331         n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
 332         extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
 333         
 334         mem_offset = (intptr_t) buf;
 335         mem_length = 0;
 336         
 337         /* if at least one full writelist, allocate file arrays
 338            at max array size and don't free until very end */
 339         if (n_write_lists) {
 340             file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
 341                                                   sizeof(int64_t));
 342             file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
 343                                                   sizeof(int32_t));
 344         }
 345         /* if there's no full writelist allocate file arrays according
 346            to needed size (extra_blks) */
 347         else {
 348             file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
 349                                                   sizeof(int64_t));
 350             file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
 351                                                   sizeof(int32_t));
 352         }
 353         
 354         /* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
 355         for (i=0; i<n_write_lists; i++) {
 356             file_list_count = MAX_ARRAY_SIZE;
 357             if(!i) {
 358                 file_offsets[0] = offset;
 359                 file_lengths[0] = st_fwr_size;
 360                 mem_length = st_fwr_size;
 361             }
 362             for (k=0; k<MAX_ARRAY_SIZE; k++) {
 363                 if (i || k) {
 364                     file_offsets[k] = disp + 
 365                         ((ADIO_Offset)n_filetypes)*filetype_extent
 366                         + flat_file->indices[j];
 367                     file_lengths[k] = flat_file->blocklens[j];
 368                     mem_length += file_lengths[k];
 369                 }
 370                 if (j<(flat_file->count - 1)) j++;
 371                 else {
 372                     j = 0;
 373                     n_filetypes++;
 374                 }
 375             } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
 376 
 377             err_flag = PVFS_Request_contiguous(mem_length,
 378                                                PVFS_BYTE, &mem_req);
 379             /* --BEGIN ERROR HANDLING-- */
 380             if (err_flag != 0) {
 381                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 382                                                    MPIR_ERR_RECOVERABLE,
 383                                                    myname, __LINE__,
 384                                                    ADIOI_PVFS2_error_convert(err_flag),
 385                                                    "Error in PVFS_Request_contiguous (memory)", 0);
 386                 goto error_state;
 387             }
 388             /* --END ERROR HANDLING-- */
 389 
 390             err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
 391                                              file_offsets, PVFS_BYTE,
 392                                              &file_req);
 393             /* --BEGIN ERROR HANDLING-- */
 394             if (err_flag != 0) {
 395                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 396                                                    MPIR_ERR_RECOVERABLE,
 397                                                    myname, __LINE__,
 398                                                    ADIOI_PVFS2_error_convert(err_flag),
 399                                                    "Error in PVFS_Request_hindexed (file)", 0);
 400                 goto error_state;
 401             }
 402             /* --END ERROR HANDLING-- */
 403 
 404             /* PVFS_Request_hindexed already expresses the offsets into the
 405              * file, so we should not pass in an offset if we are using
 406              * hindexed for the file type */
 407 #ifdef ADIOI_MPE_LOGGING
 408             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 409 #endif
 410             err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0, 
 411                                       (void *)mem_offset, mem_req,
 412                                       &(pvfs_fs->credentials), &resp_io);
 413 #ifdef ADIOI_MPE_LOGGING
 414             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 415 #endif
 416             /* --BEGIN ERROR HANDLING-- */
 417             if (err_flag != 0) {
 418                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 419                                                    MPIR_ERR_RECOVERABLE,
 420                                                    myname, __LINE__,
 421                                                    ADIOI_PVFS2_error_convert(err_flag),
 422                                                    "Error in PVFS_sys_write", 0);
 423                 goto error_state;
 424             }
 425             /* --END ERROR HANDLING-- */
 426             total_bytes_written += resp_io.total_completed;
 427 
 428             mem_offset += mem_length;
 429             mem_lengths = 0;
 430             PVFS_Request_free(&file_req);
 431             PVFS_Request_free(&mem_req);
 432 
 433         } /* for (i=0; i<n_write_lists; i++) */
 434 
 435         /* for file arrays smaller than MAX_ARRAY_SIZE (last write_list call) */
 436         if (extra_blks) {
 437             file_list_count = extra_blks;
 438             if(!i) {
 439                 file_offsets[0] = offset;
 440                 file_lengths[0] = ADIOI_MIN(st_fwr_size, bufsize);
 441             }
 442             for (k=0; k<extra_blks; k++) {
 443                 if(i || k) {
 444                     file_offsets[k] = disp + 
 445                         ((ADIO_Offset)n_filetypes)*filetype_extent +
 446                         flat_file->indices[j];
 447                     if (k == (extra_blks - 1)) {
 448                         file_lengths[k] = bufsize - (int32_t) mem_lengths
 449                           - mem_offset + (int32_t)  buf;
 450                     }
 451                     else file_lengths[k] = flat_file->blocklens[j];
 452                 } /* if(i || k) */
 453                 mem_lengths += file_lengths[k];
 454                 if (j<(flat_file->count - 1)) j++;
 455                 else {
 456                     j = 0;
 457                     n_filetypes++;
 458                 }
 459             } /* for (k=0; k<extra_blks; k++) */
 460 
 461             err_flag = PVFS_Request_contiguous(mem_length,
 462                                                PVFS_BYTE, &mem_req);
 463             /* --BEGIN ERROR HANDLING-- */
 464             if (err_flag != 0) {
 465                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 466                                                    MPIR_ERR_RECOVERABLE,
 467                                                    myname, __LINE__,
 468                                                    ADIOI_PVFS2_error_convert(err_flag),
 469                                                    "Error in PVFS_Request_contiguous (memory)", 0);
 470                 goto error_state;
 471             }
 472             /* --END ERROR HANDLING-- */
 473 
 474             err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
 475                                              file_offsets, PVFS_BYTE,
 476                                              &file_req);
 477             /* --BEGIN ERROR HANDLING-- */
 478             if (err_flag != 0) {
 479                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 480                                                    MPIR_ERR_RECOVERABLE,
 481                                                    myname, __LINE__,
 482                                                    ADIOI_PVFS2_error_convert(err_flag),
 483                                                    "Error in PVFS_Request_hindexed(file)", 0);
 484                 goto error_state;
 485             }
 486             /* --END ERROR HANDLING-- */
 487 
 488             /* as above, use 0 for 'offset' when using hindexed file type*/
 489 #ifdef ADIOI_MPE_LOGGING
 490             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 491 #endif
 492             err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0, 
 493                                       (void *)mem_offset, mem_req,
 494                                       &(pvfs_fs->credentials), &resp_io);
 495 #ifdef ADIOI_MPE_LOGGING
 496             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 497 #endif
 498             /* --BEGIN ERROR HANDLING-- */
 499             if (err_flag != 0) {
 500                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 501                                                    MPIR_ERR_RECOVERABLE,
 502                                                    myname, __LINE__,
 503                                                    ADIOI_PVFS2_error_convert(err_flag),
 504                                                    "Error in PVFS_sys_write", 0);
 505                 goto error_state;
 506             }
 507             /* --END ERROR HANDLING-- */
 508             total_bytes_written += resp_io.total_completed;
 509             PVFS_Request_free(&mem_req);
 510             PVFS_Request_free(&file_req);
 511         }
 512     } 
 513     else {
 514         /* noncontiguous in memory as well as in file */
 515 
 516         flat_buf = ADIOI_Flatten_and_find(datatype);
 517 
 518         size_wrote = 0;
 519         n_filetypes = st_n_filetypes;
 520         fwr_size = st_fwr_size;
 521         bwr_size = flat_buf->blocklens[0];
 522         buf_count = 0;
 523         start_mem_offset = 0;
 524         start_k = k = 0;
 525         start_j = st_index;
 526         max_mem_list = 0;
 527         max_file_list = 0;
 528 
 529         /* run through and file max_file_list and max_mem_list so that you 
 530            can allocate the file and memory arrays less than MAX_ARRAY_SIZE
 531            if possible */
 532 
 533         while (size_wrote < bufsize) {
 534             k = start_k;
 535             new_buffer_write = 0;
 536             mem_list_count = 0;
 537             while ((mem_list_count < MAX_ARRAY_SIZE) && 
 538                    (new_buffer_write < bufsize-size_wrote)) {
 539                 /* find mem_list_count and file_list_count such that both are
 540                    less than MAX_ARRAY_SIZE, the sum of their lengths are
 541                    equal, and the sum of all the data written and data to be
 542                    written in the next immediate write list is less than
 543                    bufsize */
 544                 if(mem_list_count) {
 545                     if((new_buffer_write + flat_buf->blocklens[k] + 
 546                         size_wrote) > bufsize) {
 547                         end_bwr_size = new_buffer_write + 
 548                             flat_buf->blocklens[k] - (bufsize - size_wrote);
 549                         new_buffer_write = bufsize - size_wrote;
 550                     }
 551                     else {
 552                         new_buffer_write += flat_buf->blocklens[k];
 553                         end_bwr_size = flat_buf->blocklens[k];
 554                     }
 555                 }
 556                 else {
 557                     if (bwr_size > (bufsize - size_wrote)) {
 558                         new_buffer_write = bufsize - size_wrote;
 559                         bwr_size = new_buffer_write;
 560                     }
 561                     else new_buffer_write = bwr_size;
 562                 }
 563                 mem_list_count++;
 564                 k = (k + 1)%flat_buf->count;
 565              } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
 566                (new_buffer_write < bufsize-size_wrote)) */
 567             j = start_j;
 568             new_file_write = 0;
 569             file_list_count = 0;
 570             while ((file_list_count < MAX_ARRAY_SIZE) && 
 571                    (new_file_write < new_buffer_write)) { 
 572                 if(file_list_count) {
 573                     if((new_file_write + flat_file->blocklens[j]) > 
 574                        new_buffer_write) {
 575                         end_fwr_size = new_buffer_write - new_file_write;
 576                         new_file_write = new_buffer_write;
 577                         j--;
 578                     }
 579                     else {
 580                         new_file_write += flat_file->blocklens[j];
 581                         end_fwr_size = flat_file->blocklens[j];
 582                     }
 583                 }
 584                 else {
 585                     if (fwr_size > new_buffer_write) {
 586                         new_file_write = new_buffer_write;
 587                         fwr_size = new_file_write;
 588                     }
 589                     else new_file_write = fwr_size;
 590                 }
 591                 file_list_count++;
 592                 if (j < (flat_file->count - 1)) j++;
 593                 else j = 0;
 594                 
 595                 k = start_k;
 596                 if ((new_file_write < new_buffer_write) && 
 597                     (file_list_count == MAX_ARRAY_SIZE)) {
 598                     new_buffer_write = 0;
 599                     mem_list_count = 0;
 600                     while (new_buffer_write < new_file_write) {
 601                         if(mem_list_count) {
 602                             if((new_buffer_write + flat_buf->blocklens[k]) >
 603                                new_file_write) {
 604                                 end_bwr_size = new_file_write - 
 605                                     new_buffer_write;
 606                                 new_buffer_write = new_file_write;
 607                                 k--;
 608                             }
 609                             else {
 610                                 new_buffer_write += flat_buf->blocklens[k];
 611                                 end_bwr_size = flat_buf->blocklens[k];
 612                             }
 613                         }
 614                         else {
 615                             new_buffer_write = bwr_size;
 616                             if (bwr_size > (bufsize - size_wrote)) {
 617                                 new_buffer_write = bufsize - size_wrote;
 618                                 bwr_size = new_buffer_write;
 619                             }
 620                         }
 621                         mem_list_count++;
 622                         k = (k + 1)%flat_buf->count;
 623                     } /* while (new_buffer_write < new_file_write) */
 624                 } /* if ((new_file_write < new_buffer_write) &&
 625                      (file_list_count == MAX_ARRAY_SIZE)) */
 626             } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
 627                  (new_buffer_write < bufsize-size_wrote)) */
 628 
 629             /*  fakes filling the writelist arrays of lengths found above  */
 630             k = start_k;
 631             j = start_j;
 632             for (i=0; i<mem_list_count; i++) {       
 633                 if(i) {
 634                     if (i == (mem_list_count - 1)) {
 635                         if (flat_buf->blocklens[k] == end_bwr_size)
 636                             bwr_size = flat_buf->blocklens[(k+1)%
 637                                                           flat_buf->count];
 638                         else {
 639                             bwr_size = flat_buf->blocklens[k] - end_bwr_size;
 640                             k--;
 641                             buf_count--;
 642                         }
 643                     }
 644                 }
 645                 buf_count++;
 646                 k = (k + 1)%flat_buf->count;
 647             } /* for (i=0; i<mem_list_count; i++) */
 648             for (i=0; i<file_list_count; i++) {
 649                 if (i) {
 650                     if (i == (file_list_count - 1)) {
 651                         if (flat_file->blocklens[j] == end_fwr_size)
 652                             fwr_size = flat_file->blocklens[(j+1)%
 653                                                           flat_file->count];   
 654                         else {
 655                             fwr_size = flat_file->blocklens[j] - end_fwr_size;
 656                             j--;
 657                         }
 658                     }
 659                 }
 660                 if (j < flat_file->count - 1) j++;
 661                 else {
 662                     j = 0;
 663                     n_filetypes++;
 664                 }
 665             } /* for (i=0; i<file_list_count; i++) */
 666             size_wrote += new_buffer_write;
 667             start_k = k;
 668             start_j = j;
 669             if (max_mem_list < mem_list_count)
 670                 max_mem_list = mem_list_count;
 671             if (max_file_list < file_list_count)
 672                 max_file_list = file_list_count;
 673         } /* while (size_wrote < bufsize) */
 674 
 675         /* one last check before we actually carry out the operation:
 676          * this code has hard-to-fix bugs when a noncontiguous file type has
 677          * such large pieces that the sum of the lengths of the memory type is
 678          * not larger than one of those pieces (and vice versa for large memory
 679          * types and many pices of file types.  In these cases, give up and
 680          * fall back to naive reads and writes.  The testphdf5 test created a
 681          * type with two very large memory regions and 600 very small file
 682          * regions.  The same test also created a type with one very large file
 683          * region and many (700) very small memory regions.  both cases caused
 684          * problems for this code */
 685 
 686         if ( ( (file_list_count == 1) && 
 687                     (new_file_write < flat_file->blocklens[0] ) ) ||
 688                 ((mem_list_count == 1) && 
 689                     (new_buffer_write < flat_buf->blocklens[0]) ) ||
 690                 ((file_list_count == MAX_ARRAY_SIZE) && 
 691                     (new_file_write < flat_buf->blocklens[0]) ) ||
 692                 ( (mem_list_count == MAX_ARRAY_SIZE) &&
 693                     (new_buffer_write < flat_file->blocklens[0])) )
 694         {
 695             ADIOI_Delete_flattened(datatype);
 696             ADIOI_GEN_WriteStrided_naive(fd, buf, count, datatype,
 697                     file_ptr_type, initial_off, status, error_code);
 698             return;
 699         }
 700 
 701 
 702         mem_offsets = (PVFS_size*)ADIOI_Malloc(max_mem_list*sizeof(PVFS_size));
 703         mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
 704         file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
 705         file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
 706             
 707         size_wrote = 0;
 708         n_filetypes = st_n_filetypes;
 709         fwr_size = st_fwr_size;
 710         bwr_size = flat_buf->blocklens[0];
 711         buf_count = 0;
 712         start_mem_offset = 0;
 713         start_k = k = 0;
 714         start_j = st_index;
 715 
 716         /*  this section calculates mem_list_count and file_list_count
 717             and also finds the possibly odd sized last array elements
 718             in new_fwr_size and new_bwr_size  */
 719         
 720         while (size_wrote < bufsize) {
 721             k = start_k;
 722             new_buffer_write = 0;
 723             mem_list_count = 0;
 724             while ((mem_list_count < MAX_ARRAY_SIZE) && 
 725                    (new_buffer_write < bufsize-size_wrote)) {
 726                 /* find mem_list_count and file_list_count such that both are
 727                    less than MAX_ARRAY_SIZE, the sum of their lengths are
 728                    equal, and the sum of all the data written and data to be
 729                    written in the next immediate write list is less than
 730                    bufsize */
 731                 if(mem_list_count) {
 732                     if((new_buffer_write + flat_buf->blocklens[k] + 
 733                         size_wrote) > bufsize) {
 734                         end_bwr_size = new_buffer_write + 
 735                             flat_buf->blocklens[k] - (bufsize - size_wrote);
 736                         new_buffer_write = bufsize - size_wrote;
 737                     }
 738                     else {
 739                         new_buffer_write += flat_buf->blocklens[k];
 740                         end_bwr_size = flat_buf->blocklens[k];
 741                     }
 742                 }
 743                 else {
 744                     if (bwr_size > (bufsize - size_wrote)) {
 745                         new_buffer_write = bufsize - size_wrote;
 746                         bwr_size = new_buffer_write;
 747                     }
 748                     else new_buffer_write = bwr_size;
 749                 }
 750                 mem_list_count++;
 751                 k = (k + 1)%flat_buf->count;
 752              } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
 753                (new_buffer_write < bufsize-size_wrote)) */
 754             j = start_j;
 755             new_file_write = 0;
 756             file_list_count = 0;
 757             while ((file_list_count < MAX_ARRAY_SIZE) && 
 758                    (new_file_write < new_buffer_write)) {
 759                 if(file_list_count) {
 760                     if((new_file_write + flat_file->blocklens[j]) > 
 761                        new_buffer_write) {
 762                         end_fwr_size = new_buffer_write - new_file_write;
 763                         new_file_write = new_buffer_write;
 764                         j--;
 765                     }
 766                     else {
 767                         new_file_write += flat_file->blocklens[j];
 768                         end_fwr_size = flat_file->blocklens[j];
 769                     }
 770                 }
 771                 else {
 772                     if (fwr_size > new_buffer_write) {
 773                         new_file_write = new_buffer_write;
 774                         fwr_size = new_file_write;
 775                     }
 776                     else new_file_write = fwr_size;
 777                 }
 778                 file_list_count++;
 779                 if (j < (flat_file->count - 1)) j++;
 780                 else j = 0;
 781                 
 782                 k = start_k;
 783                 if ((new_file_write < new_buffer_write) && 
 784                     (file_list_count == MAX_ARRAY_SIZE)) {
 785                     new_buffer_write = 0;
 786                     mem_list_count = 0;
 787                     while (new_buffer_write < new_file_write) {
 788                         if(mem_list_count) {
 789                             if((new_buffer_write + flat_buf->blocklens[k]) >
 790                                new_file_write) {
 791                                 end_bwr_size = new_file_write -
 792                                   new_buffer_write;
 793                                 new_buffer_write = new_file_write;
 794                                 k--;
 795                             }
 796                             else {
 797                                 new_buffer_write += flat_buf->blocklens[k];
 798                                 end_bwr_size = flat_buf->blocklens[k];
 799                             }
 800                         }
 801                         else {
 802                             new_buffer_write = bwr_size;
 803                             if (bwr_size > (bufsize - size_wrote)) {
 804                                 new_buffer_write = bufsize - size_wrote;
 805                                 bwr_size = new_buffer_write;
 806                             }
 807                         }
 808                         mem_list_count++;
 809                         k = (k + 1)%flat_buf->count;
 810                     } /* while (new_buffer_write < new_file_write) */
 811                 } /* if ((new_file_write < new_buffer_write) &&
 812                      (file_list_count == MAX_ARRAY_SIZE)) */
 813             } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
 814                  (new_buffer_write < bufsize-size_wrote)) */
 815 
 816             /*  fills the allocated writelist arrays  */
 817             k = start_k;
 818             j = start_j;
 819             for (i=0; i<mem_list_count; i++) {       
 820                 /* TODO: fix this warning by casting to an integer that's the
 821                  * same size as a char * and /then/ casting to PVFS_size */
 822                 mem_offsets[i] = ((PVFS_size)buf + buftype_extent*
 823                                   (buf_count/flat_buf->count) +
 824                                   (int)flat_buf->indices[k]);
 825                 
 826                 if(!i) {
 827                     mem_lengths[0] = bwr_size;
 828                     mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
 829                 }
 830                 else {
 831                     if (i == (mem_list_count - 1)) {
 832                         mem_lengths[i] = end_bwr_size;
 833                         if (flat_buf->blocklens[k] == end_bwr_size)
 834                             bwr_size = flat_buf->blocklens[(k+1)%
 835                                                           flat_buf->count];
 836                         else {
 837                             bwr_size = flat_buf->blocklens[k] - end_bwr_size;
 838                             k--;
 839                             buf_count--;
 840                         }
 841                     }
 842                     else {
 843                         mem_lengths[i] = flat_buf->blocklens[k];
 844                     }
 845                 }
 846                 buf_count++;
 847                 k = (k + 1)%flat_buf->count;
 848             } /* for (i=0; i<mem_list_count; i++) */
 849             for (i=0; i<file_list_count; i++) {
 850                 file_offsets[i] = disp + flat_file->indices[j] + 
 851                     ((ADIO_Offset)n_filetypes) * filetype_extent;
 852                 if (!i) {
 853                     file_lengths[0] = fwr_size;
 854                     file_offsets[0] += flat_file->blocklens[j] - fwr_size;
 855                 }
 856                 else {
 857                     if (i == (file_list_count - 1)) {
 858                         file_lengths[i] = end_fwr_size;
 859                         if (flat_file->blocklens[j] == end_fwr_size)
 860                             fwr_size = flat_file->blocklens[(j+1)%
 861                                                           flat_file->count];   
 862                         else {
 863                             fwr_size = flat_file->blocklens[j] - end_fwr_size;
 864                             j--;
 865                         }
 866                     }
 867                     else file_lengths[i] = flat_file->blocklens[j];
 868                 }
 869                 if (j < flat_file->count - 1) j++;
 870                 else {
 871                     j = 0;
 872                     n_filetypes++;
 873                 }
 874             } /* for (i=0; i<file_list_count; i++) */
 875 
 876             err_flag = PVFS_Request_hindexed(mem_list_count, mem_lengths, 
 877                                              mem_offsets, PVFS_BYTE, &mem_req);
 878             /* --BEGIN ERROR HANDLING-- */
 879             if (err_flag != 0 ) {
 880                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 881                                                    MPIR_ERR_RECOVERABLE,
 882                                                    myname, __LINE__,
 883                                                    ADIOI_PVFS2_error_convert(err_flag),
 884                                                    "Error in PVFS_Request_hindexed (memory)", 0);
 885                 goto error_state;
 886             }
 887             /* --END ERROR HANDLING-- */
 888 
 889             err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
 890                                              file_offsets, PVFS_BYTE,
 891                                              &file_req);
 892             /* --BEGIN ERROR HANDLING-- */
 893             if (err_flag != 0) {
 894                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 895                                                    MPIR_ERR_RECOVERABLE,
 896                                                    myname, __LINE__,
 897                                                    ADIOI_PVFS2_error_convert(err_flag),
 898                                                    "Error in PVFS_Request_hindexed", 0);
 899                 goto error_state;
 900             }
 901             /* --END ERROR HANDLING-- */
 902 
 903             /* offset will be expressed in memory and file datatypes */
 904 
 905 #ifdef ADIOI_MPE_LOGGING
 906             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 907 #endif
 908             err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0, 
 909                                       PVFS_BOTTOM, mem_req,
 910                                       &(pvfs_fs->credentials), &resp_io);
 911 #ifdef ADIOI_MPE_LOGGING
 912             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 913 #endif
 914             /* --BEGIN ERROR HANDLING-- */
 915             if (err_flag != 0) {
 916                 *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 917                                                    MPIR_ERR_RECOVERABLE,
 918                                                    myname, __LINE__,
 919                                                    ADIOI_PVFS2_error_convert(err_flag),
 920                                                    "Error in PVFS_sys_write", 0);
 921                 goto error_state;
 922             }
 923             /* --END ERROR HANDLING-- */
 924 
 925             size_wrote += new_buffer_write;
 926             total_bytes_written += resp_io.total_completed;
 927             start_k = k;
 928             start_j = j;
 929             PVFS_Request_free(&mem_req);
 930             PVFS_Request_free(&file_req);
 931         } /* while (size_wrote < bufsize) */
 932         ADIOI_Free(mem_offsets);
 933         ADIOI_Free(mem_lengths);
 934     }
 935     /* when incrementing fp_ind, need to also take into account the file type:
 936      * consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
 937      * if we wrote N elements, offset needs to point at beginning of type, not
 938      * at empty region at offset N+1).  
 939      *
 940      * As we discussed on mpich-discuss in may/june 2009, the code below might
 941      * look wierd, but by putting fp_ind at the last byte written, the next
 942      * time we run through the strided code we'll update the fp_ind to the
 943      * right location. */
 944     if (file_ptr_type == ADIO_INDIVIDUAL) {
 945         fd->fp_ind = file_offsets[file_list_count-1]+
 946             file_lengths[file_list_count-1];
 947     }
 948     ADIOI_Free(file_offsets);
 949     ADIOI_Free(file_lengths);
 950 
 951     *error_code = MPI_SUCCESS;
 952 
 953 error_state:
 954     fd->fp_sys_posn = -1;   /* set it to null. */
 955 
 956 #ifdef HAVE_STATUS_SET_BYTES
 957     MPIR_Status_set_bytes(status, datatype, bufsize);
 958 /* This is a temporary way of filling in status. The right way is to 
 959    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
 960 #endif
 961 
 962     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
 963 }

/* [<][>][^][v][top][bottom][index][help] */