This source file includes following definitions.
- ADIOI_PVFS_WriteContig
- ADIOI_PVFS_WriteStrided
- ADIOI_PVFS_WriteStridedListIO
   1 
   2 
   3 
   4 
   5 
   6 
   7 
   8 #include "ad_pvfs.h"
   9 #include "adio_extern.h"
  10 
  11 #ifdef HAVE_PVFS_LISTIO
  12 void ADIOI_PVFS_WriteStridedListIO(ADIO_File fd, void *buf, int count,
  13                        MPI_Datatype datatype, int file_ptr_type,
  14                        ADIO_Offset offset, ADIO_Status *status, int
  15                        *error_code);
  16 #endif
  17 
  18 void ADIOI_PVFS_WriteContig(ADIO_File fd, void *buf, int count, 
  19                             MPI_Datatype datatype, int file_ptr_type,
  20                             ADIO_Offset offset, ADIO_Status *status,
  21                             int *error_code)
  22 {
  23     MPI_Count err=-1, datatype_size, len;
  24     static char myname[] = "ADIOI_PVFS_WRITECONTIG";
  25 
  26     MPI_Type_size_x(datatype, &datatype_size);
  27     len = datatype_size * count;
  28 
  29     if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
  30         if (fd->fp_sys_posn != offset) {
  31 #ifdef ADIOI_MPE_LOGGING
  32             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
  33 #endif
  34             pvfs_lseek64(fd->fd_sys, offset, SEEK_SET);
  35 #ifdef ADIOI_MPE_LOGGING
  36             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
  37 #endif
  38         }
  39 #ifdef ADIOI_MPE_LOGGING
  40         MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
  41 #endif
  42         err = pvfs_write(fd->fd_sys, buf, len);
  43 #ifdef ADIOI_MPE_LOGGING
  44         MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
  45 #endif
  46         if (err > 0)
  47                 fd->fp_sys_posn = offset + err;
  48                 
  49     }
  50     else { 
  51         if (fd->fp_sys_posn != fd->fp_ind) {
  52 #ifdef ADIOI_MPE_LOGGING
  53             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
  54 #endif
  55             pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
  56 #ifdef ADIOI_MPE_LOGGING
  57             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
  58 #endif
  59         }
  60 #ifdef ADIOI_MPE_LOGGING
  61         MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
  62 #endif
  63         err = pvfs_write(fd->fd_sys, buf, len);
  64 #ifdef ADIOI_MPE_LOGGING
  65         MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
  66 #endif
  67         if (err > 0)
  68                 fd->fp_ind += err;
  69         fd->fp_sys_posn = fd->fp_ind;
  70     }
  71 
  72 #ifdef HAVE_STATUS_SET_BYTES
  73     if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
  74 #endif
  75 
  76     if (err == -1) {
  77         *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
  78                                            myname, __LINE__, MPI_ERR_IO,
  79                                            "**io",
  80                                            "**io %s", strerror(errno));
  81     }
  82     else *error_code = MPI_SUCCESS;
  83 }
  84 
  85 
  86 
  87 void ADIOI_PVFS_WriteStrided(ADIO_File fd, void *buf, int count,
  88                              MPI_Datatype datatype, int file_ptr_type,
  89                              ADIO_Offset offset, ADIO_Status *status, int
  90                              *error_code)
  91 {
  92 
  93 
  94 
  95 
  96 
  97     ADIOI_Flatlist_node *flat_buf, *flat_file;
  98     int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;
  99     int num, size, sum, n_etypes_in_filetype, size_in_filetype;
 100     MPI_Count bufsize;
 101     int n_filetypes, etype_in_filetype;
 102     ADIO_Offset abs_off_in_filetype=0;
 103     MPI_Count filetype_size, etype_size, buftype_size;
 104     MPI_Aint filetype_extent, buftype_extent, indx, filetype_lb, buftype_lb;
 105     int buf_count, buftype_is_contig, filetype_is_contig;
 106     ADIO_Offset off, disp;
 107     int flag, new_bwr_size, new_fwr_size, err_flag=0;
 108     static char myname[] = "ADIOI_PVFS_WRITESTRIDED";
 109 
 110 #ifdef HAVE_PVFS_LISTIO
 111     if ( fd->hints->fs_hints.pvfs.listio_write == ADIOI_HINT_ENABLE ) {
 112             ADIOI_PVFS_WriteStridedListIO(fd, buf, count, datatype, 
 113                             file_ptr_type, offset, status, error_code);
 114             return;
 115     }
 116 #endif
 117     
 118 
 119     
 120     if (fd->atomicity) {
 121         *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
 122                                            myname, __LINE__,
 123                                            MPI_ERR_INTERN,
 124                                            "Atomic mode set in PVFS I/O function", 0);
 125         return;
 126     }
 127     
 128 
 129     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
 130     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
 131 
 132     MPI_Type_size_x(fd->filetype, &filetype_size);
 133     if ( ! filetype_size ) {
 134 #ifdef HAVE_STATUS_SET_BYTES
 135         MPIR_Status_set_bytes(status, datatype, 0);
 136 #endif
 137         *error_code = MPI_SUCCESS; 
 138         return;
 139     }
 140 
 141     MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
 142     MPI_Type_size_x(datatype, &buftype_size);
 143     MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
 144     etype_size = fd->etype_size;
 145     
 146     bufsize = buftype_size * count;
 147 
 148     if (!buftype_is_contig && filetype_is_contig) {
 149         char *combine_buf, *combine_buf_ptr;
 150         ADIO_Offset combine_buf_remain;
 151 
 152 
 153         flat_buf = ADIOI_Flatten_and_find(datatype);
 154 
 155         
 156         combine_buf = (char *) ADIOI_Malloc(fd->hints->ind_wr_buffer_size);
 157         combine_buf_ptr = combine_buf;
 158         combine_buf_remain = fd->hints->ind_wr_buffer_size;
 159 
 160         
 161         if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
 162             off = fd->disp + etype_size * offset;
 163 #ifdef ADIOI_MPE_LOGGING
 164             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
 165 #endif
 166             pvfs_lseek64(fd->fd_sys, off, SEEK_SET);
 167 #ifdef ADIOI_MPE_LOGGING
 168             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
 169 #endif
 170         }
 171         else {
 172 #ifdef ADIOI_MPE_LOGGING
 173             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
 174 #endif
 175             off = pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
 176 #ifdef ADIOI_MPE_LOGGING
 177             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
 178 #endif
 179         }
 180 
 181         
 182 
 183 
 184 
 185 
 186 
 187         for (j=0; j<count; j++) {
 188             for (i=0; i<flat_buf->count; i++) {
 189                 if (flat_buf->blocklens[i] > combine_buf_remain && combine_buf != combine_buf_ptr) {
 190                     
 191 #ifdef ADIOI_MPE_LOGGING
 192                     MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 193 #endif
 194                     err = pvfs_write(fd->fd_sys,
 195                                      combine_buf,
 196                                      fd->hints->ind_wr_buffer_size - combine_buf_remain);
 197 #ifdef ADIOI_MPE_LOGGING
 198                     MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 199 #endif
 200                     if (err == -1) err_flag = 1;
 201 
 202                     
 203                     combine_buf_ptr = combine_buf;
 204                     combine_buf_remain = fd->hints->ind_wr_buffer_size;
 205                 }
 206 
 207                 
 208                 if (flat_buf->blocklens[i] >= combine_buf_remain) {
 209                     
 210 
 211 
 212 #ifdef ADIOI_MPE_LOGGING
 213                     MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 214 #endif
 215                     err = pvfs_write(fd->fd_sys,
 216                                      ((char *) buf) + j*buftype_extent + flat_buf->indices[i],
 217                                      flat_buf->blocklens[i]);
 218 #ifdef ADIOI_MPE_LOGGING
 219                     MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 220 #endif
 221                     if (err == -1) err_flag = 1;
 222                     off += flat_buf->blocklens[i]; 
 223                 }
 224                 else {
 225                     
 226                     memcpy(combine_buf_ptr,
 227                            ((char *) buf) + j*buftype_extent + flat_buf->indices[i],
 228                            flat_buf->blocklens[i]);
 229                     combine_buf_ptr += flat_buf->blocklens[i];
 230                     combine_buf_remain -= flat_buf->blocklens[i];
 231                     off += flat_buf->blocklens[i]; 
 232                 }
 233             }
 234         }
 235 
 236         if (combine_buf_ptr != combine_buf) {
 237             
 238 #ifdef ADIOI_MPE_LOGGING
 239             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 240 #endif
 241             err = pvfs_write(fd->fd_sys,
 242                              combine_buf,
 243                              fd->hints->ind_wr_buffer_size - combine_buf_remain);
 244 #ifdef ADIOI_MPE_LOGGING
 245             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 246 #endif
 247             if (err == -1) err_flag = 1;
 248         }
 249 
 250         if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
 251 
 252         ADIOI_Free(combine_buf);
 253 
 254         if (err_flag) {
 255             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 256                                                MPIR_ERR_RECOVERABLE, myname,
 257                                                __LINE__, MPI_ERR_IO, "**io",
 258                                                "**io %s", strerror(errno));
 259         }
 260         else *error_code = MPI_SUCCESS;
 261     } 
 262 
 263     else {  
 264 
 265 
 266 
 267 
 268 
 269 
 270         flat_file = ADIOI_Flatlist;
 271         while (flat_file->type != fd->filetype) flat_file = flat_file->next;
 272         disp = fd->disp;
 273 
 274         if (file_ptr_type == ADIO_INDIVIDUAL) {
 275             offset = fd->fp_ind; 
 276             n_filetypes = -1;
 277             flag = 0;
 278             while (!flag) {
 279                 n_filetypes++;
 280                 for (i=0; i<flat_file->count; i++) {
 281                     if (disp + flat_file->indices[i] + 
 282                         (ADIO_Offset) n_filetypes*filetype_extent + flat_file->blocklens[i] 
 283                             >= offset) {
 284                         st_index = i;
 285                         fwr_size = disp + flat_file->indices[i] + 
 286                                 (ADIO_Offset) n_filetypes*filetype_extent
 287                                  + flat_file->blocklens[i] - offset;
 288                         flag = 1;
 289                         break;
 290                     }
 291                 }
 292             }
 293         }
 294         else {
 295             n_etypes_in_filetype = filetype_size/etype_size;
 296             n_filetypes = (int) (offset / n_etypes_in_filetype);
 297             etype_in_filetype = (int) (offset % n_etypes_in_filetype);
 298             size_in_filetype = etype_in_filetype * etype_size;
 299  
 300             sum = 0;
 301             for (i=0; i<flat_file->count; i++) {
 302                 sum += flat_file->blocklens[i];
 303                 if (sum > size_in_filetype) {
 304                     st_index = i;
 305                     fwr_size = sum - size_in_filetype;
 306                     abs_off_in_filetype = flat_file->indices[i] +
 307                         size_in_filetype - (sum - flat_file->blocklens[i]);
 308                     break;
 309                 }
 310             }
 311 
 312             
 313             offset = disp + (ADIO_Offset) n_filetypes*filetype_extent + abs_off_in_filetype;
 314         }
 315 
 316         if (buftype_is_contig && !filetype_is_contig) {
 317 
 318 
 319 
 320 
 321             i = 0;
 322             j = st_index;
 323             off = offset;
 324             fwr_size = ADIOI_MIN(fwr_size, bufsize);
 325             while (i < bufsize) {
 326                 if (fwr_size) { 
 327                     
 328  
 329 #ifdef ADIOI_MPE_LOGGING
 330                     MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
 331 #endif
 332                     pvfs_lseek64(fd->fd_sys, off, SEEK_SET);
 333 #ifdef ADIOI_MPE_LOGGING
 334                     MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
 335 #endif
 336 #ifdef ADIOI_MPE_LOGGING
 337                     MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 338 #endif
 339                     err = pvfs_write(fd->fd_sys, ((char *) buf) + i, fwr_size);
 340 #ifdef ADIOI_MPE_LOGGING
 341                     MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 342 #endif
 343                     if (err == -1) err_flag = 1;
 344                 }
 345                 i += fwr_size;
 346 
 347                 if (off + fwr_size < disp + flat_file->indices[j] +
 348                    flat_file->blocklens[j] + (ADIO_Offset) n_filetypes*filetype_extent)
 349                        off += fwr_size;
 350                 
 351 
 352                 else {
 353                     if (j < (flat_file->count - 1)) j++;
 354                     else {
 355                         j = 0;
 356                         n_filetypes++;
 357                     }
 358                     off = disp + flat_file->indices[j] + 
 359                                         (ADIO_Offset) n_filetypes*filetype_extent;
 360                     fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
 361                 }
 362             }
 363         }
 364         else {
 365 
 366 
 367             flat_buf = ADIOI_Flatten_and_find(datatype);
 368 
 369             k = num = buf_count = 0;
 370             indx = flat_buf->indices[0];
 371             j = st_index;
 372             off = offset;
 373             bwr_size = flat_buf->blocklens[0];
 374 
 375             while (num < bufsize) {
 376                 size = ADIOI_MIN(fwr_size, bwr_size);
 377                 if (size) {
 378 #ifdef ADIOI_MPE_LOGGING
 379                     MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
 380 #endif
 381                     pvfs_lseek64(fd->fd_sys, off, SEEK_SET);
 382 #ifdef ADIOI_MPE_LOGGING
 383                     MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
 384 #endif
 385 #ifdef ADIOI_MPE_LOGGING
 386                     MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
 387 #endif
 388                     err = pvfs_write(fd->fd_sys, ((char *) buf) + indx, size);
 389 #ifdef ADIOI_MPE_LOGGING
 390                     MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
 391 #endif
 392                     if (err == -1) err_flag = 1;
 393                 }
 394 
 395                 new_fwr_size = fwr_size;
 396                 new_bwr_size = bwr_size;
 397 
 398                 if (size == fwr_size) {
 399 
 400                     if (j < (flat_file->count - 1)) j++;
 401                     else {
 402                         j = 0;
 403                         n_filetypes++;
 404                     }
 405 
 406                     off = disp + flat_file->indices[j] + 
 407                                    (ADIO_Offset) n_filetypes*filetype_extent;
 408 
 409                     new_fwr_size = flat_file->blocklens[j];
 410                     if (size != bwr_size) {
 411                         indx += size;
 412                         new_bwr_size -= size;
 413                     }
 414                 }
 415 
 416                 if (size == bwr_size) {
 417 
 418 
 419                     k = (k + 1)%flat_buf->count;
 420                     buf_count++;
 421                     indx = buftype_extent*(buf_count/flat_buf->count) +
 422                         flat_buf->indices[k]; 
 423                     new_bwr_size = flat_buf->blocklens[k];
 424                     if (size != fwr_size) {
 425                         off += size;
 426                         new_fwr_size -= size;
 427                     }
 428                 }
 429                 num += size;
 430                 fwr_size = new_fwr_size;
 431                 bwr_size = new_bwr_size;
 432             }
 433         }
 434 
 435         if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
 436         if (err_flag) {
 437             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 438                                                MPIR_ERR_RECOVERABLE, myname,
 439                                                __LINE__, MPI_ERR_IO, "**io",
 440                                                "**io %s", strerror(errno));
 441         }
 442         else *error_code = MPI_SUCCESS;
 443     }
 444 
 445     fd->fp_sys_posn = -1;   
 446 
 447 #ifdef HAVE_STATUS_SET_BYTES
 448     MPIR_Status_set_bytes(status, datatype, bufsize);
 449 
 450 
 451 #endif
 452 
 453     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
 454 }
 455 
 456 #ifdef HAVE_PVFS_LISTIO
 457 void ADIOI_PVFS_WriteStridedListIO(ADIO_File fd, void *buf, int count,
 458                        MPI_Datatype datatype, int file_ptr_type,
 459                        ADIO_Offset offset, ADIO_Status *status, int
 460                        *error_code) 
 461 {
 462 
 463 
 464 
 465 
 466 
 467     ADIOI_Flatlist_node *flat_buf, *flat_file;
 468     int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;
 469     int size, sum, n_etypes_in_filetype, size_in_filetype;
 470     MPI_Count bufsize;
 471     int n_filetypes, etype_in_filetype;
 472     ADIO_Offset abs_off_in_filetype=0;
 473     MPI_Count filetype_size, etype_size, buftype_size;
 474     MPI_Aint filetype_extent, buftype_extent, filetype_lb, buftype_lb;
 475     int buf_count, buftype_is_contig, filetype_is_contig;
 476     ADIO_Offset userbuf_off;
 477     ADIO_Offset off, disp, start_off;
 478     int flag, st_fwr_size, st_n_filetypes;
 479     int new_bwr_size, new_fwr_size, err_flag=0;
 480 
 481     int mem_list_count, file_list_count;
 482     char ** mem_offsets;
 483     int64_t *file_offsets;
 484     int *mem_lengths;
 485     int32_t *file_lengths;
 486     int total_blks_to_write;
 487 
 488     int max_mem_list, max_file_list;
 489 
 490     int b_blks_wrote;
 491     int f_data_wrote;
 492     int size_wrote=0, n_write_lists, extra_blks;
 493 
 494     int end_bwr_size, end_fwr_size;
 495     int start_k, start_j, new_file_write, new_buffer_write;
 496     int start_mem_offset;
 497 #define MAX_ARRAY_SIZE 1024
 498     static char myname[] = "ADIOI_PVFS_WRITESTRIDED";
 499 
 500 
 501 
 502 
 503     
 504     if (fd->atomicity) {
 505         *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
 506                                            myname, __LINE__,
 507                                            MPI_ERR_INTERN,
 508                                            "Atomic mode set in PVFS I/O function", 0);
 509         return;
 510     }
 511     
 512 
 513     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
 514     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
 515 
 516     MPI_Type_size_x(fd->filetype, &filetype_size);
 517     if ( ! filetype_size ) {
 518 #ifdef HAVE_STATUS_SET_BYTES
 519         MPIR_Status_set_bytes(status, datatype, 0);
 520 #endif
 521         *error_code = MPI_SUCCESS; 
 522         return;
 523     }
 524 
 525     MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
 526     MPI_Type_size_x(datatype, &buftype_size);
 527     MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
 528     etype_size = fd->etype_size;
 529     
 530     bufsize = buftype_size * count;
 531 
 532     if (!buftype_is_contig && filetype_is_contig) {
 533 
 534 
 535         int64_t file_offsets;
 536         int32_t file_lengths;
 537 
 538         flat_buf = ADIOI_Flatten_and_find(datatype);
 539         
 540         if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
 541             off = fd->disp + etype_size * offset;
 542 #ifdef ADIOI_MPE_LOGGING
 543             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
 544 #endif
 545             pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
 546 #ifdef ADIOI_MPE_LOGGING
 547             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
 548 #endif
 549         }
 550         else {
 551 #ifdef ADIOI_MPE_LOGGING
 552             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
 553 #endif
 554             off = pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
 555 #ifdef ADIOI_MPE_LOGGING
 556             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
 557 #endif
 558         }
 559 
 560         file_list_count = 1;
 561         file_offsets = off;
 562         file_lengths = 0;
 563         total_blks_to_write = count*flat_buf->count;
 564         b_blks_wrote = 0;
 565 
 566         
 567         if (total_blks_to_write > MAX_ARRAY_SIZE)
 568             mem_list_count = MAX_ARRAY_SIZE;
 569         else mem_list_count = total_blks_to_write;
 570         mem_offsets = (char**)ADIOI_Malloc(mem_list_count*sizeof(char*));
 571         mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
 572 
 573         j = 0;
 574         
 575         while (b_blks_wrote < total_blks_to_write) {
 576             for (i=0; i<flat_buf->count; i++) {
 577                 mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] = 
 578                     ((char*)buf + j*buftype_extent + flat_buf->indices[i]);
 579                 mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] = 
 580                     flat_buf->blocklens[i];
 581                 file_lengths += flat_buf->blocklens[i];
 582                 b_blks_wrote++;
 583                 if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
 584                     (b_blks_wrote == total_blks_to_write)) {
 585 
 586                     
 587 
 588                     if (b_blks_wrote == total_blks_to_write) {
 589                         mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
 590                         
 591                         if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
 592                     }
 593 
 594                     pvfs_write_list(fd->fd_sys ,mem_list_count, mem_offsets,
 595                                    mem_lengths, file_list_count,
 596                                    &file_offsets, &file_lengths);
 597                   
 598                     
 599                     if (b_blks_wrote == total_blks_to_write) break;
 600 
 601                     file_offsets += file_lengths;
 602                     file_lengths = 0;
 603                 } 
 604             } 
 605             j++;
 606         } 
 607         ADIOI_Free(mem_offsets);
 608         ADIOI_Free(mem_lengths);
 609 
 610         if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
 611 
 612         if (err_flag) {
 613             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 614                                                MPIR_ERR_RECOVERABLE, myname,
 615                                                __LINE__, MPI_ERR_IO, "**io",
 616                                                "**io %s", strerror(errno));
 617         }
 618         else *error_code = MPI_SUCCESS;
 619 
 620         fd->fp_sys_posn = -1;   
 621 
 622 #ifdef HAVE_STATUS_SET_BYTES
 623         MPIR_Status_set_bytes(status, datatype, bufsize);
 624 
 625 
 626 #endif
 627 
 628         ADIOI_Delete_flattened(datatype);
 629         return;
 630     } 
 631 
 632     
 633     
 634 
 635 
 636     flat_file = ADIOI_Flatlist;
 637     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
 638 
 639     disp = fd->disp;
 640 
 641     
 642 
 643 
 644 
 645 
 646     if (file_ptr_type == ADIO_INDIVIDUAL) {
 647         offset = fd->fp_ind; 
 648         n_filetypes = -1;
 649         flag = 0;
 650         while (!flag) {
 651             n_filetypes++;
 652             for (i=0; i<flat_file->count; i++) {
 653                 if (disp + flat_file->indices[i] + 
 654                     (ADIO_Offset) n_filetypes*filetype_extent +
 655                       flat_file->blocklens[i] >= offset) {
 656                   st_index = i;
 657                   fwr_size = disp + flat_file->indices[i] + 
 658                     (ADIO_Offset) n_filetypes*filetype_extent
 659                     + flat_file->blocklens[i] - offset;
 660                   flag = 1;
 661                   break;
 662                 }
 663             }
 664         } 
 665     } 
 666     else {
 667         n_etypes_in_filetype = filetype_size/etype_size;
 668         n_filetypes = (int) (offset / n_etypes_in_filetype);
 669         etype_in_filetype = (int) (offset % n_etypes_in_filetype);
 670         size_in_filetype = etype_in_filetype * etype_size;
 671         
 672         sum = 0;
 673         for (i=0; i<flat_file->count; i++) {
 674             sum += flat_file->blocklens[i];
 675             if (sum > size_in_filetype) {
 676                 st_index = i;
 677                 fwr_size = sum - size_in_filetype;
 678                 abs_off_in_filetype = flat_file->indices[i] +
 679                     size_in_filetype - (sum - flat_file->blocklens[i]);
 680                 break;
 681             }
 682         }
 683 
 684         
 685         offset = disp + (ADIO_Offset) n_filetypes*filetype_extent +
 686             abs_off_in_filetype;
 687     } 
 688 
 689     start_off = offset;
 690     st_fwr_size = fwr_size;
 691     st_n_filetypes = n_filetypes;
 692     
 693     if (buftype_is_contig && !filetype_is_contig) {
 694 
 695 
 696 
 697 
 698         int mem_lengths;
 699         char *mem_offsets;
 700         
 701         i = 0;
 702         j = st_index;
 703         off = offset;
 704         n_filetypes = st_n_filetypes;
 705         
 706         mem_list_count = 1;
 707         
 708         
 709         f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
 710         total_blks_to_write = 1;
 711         j++;
 712         while (f_data_wrote < bufsize) {
 713             f_data_wrote += flat_file->blocklens[j];
 714             total_blks_to_write++;
 715             if (j<(flat_file->count-1)) j++;
 716             else j = 0; 
 717         }
 718             
 719         j = st_index;
 720         n_filetypes = st_n_filetypes;
 721         n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
 722         extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
 723         
 724         mem_offsets = buf;
 725         mem_lengths = 0;
 726         
 727         
 728 
 729         if (n_write_lists) {
 730             file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
 731                                                   sizeof(int64_t));
 732             file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
 733                                                   sizeof(int32_t));
 734         }
 735         
 736 
 737         else {
 738             file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
 739                                                   sizeof(int64_t));
 740             file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
 741                                                   sizeof(int32_t));
 742         }
 743         
 744         
 745         for (i=0; i<n_write_lists; i++) {
 746             file_list_count = MAX_ARRAY_SIZE;
 747             if(!i) {
 748                 file_offsets[0] = offset;
 749                 file_lengths[0] = st_fwr_size;
 750                 mem_lengths = st_fwr_size;
 751             }
 752             for (k=0; k<MAX_ARRAY_SIZE; k++) {
 753                 if (i || k) {
 754                     file_offsets[k] = disp + n_filetypes*filetype_extent
 755                       + flat_file->indices[j];
 756                     file_lengths[k] = flat_file->blocklens[j];
 757                     mem_lengths += file_lengths[k];
 758                 }
 759                 if (j<(flat_file->count - 1)) j++;
 760                 else {
 761                     j = 0;
 762                     n_filetypes++;
 763                 }
 764             } 
 765             pvfs_write_list(fd->fd_sys, mem_list_count,
 766                            &mem_offsets, &mem_lengths,
 767                            file_list_count, file_offsets,
 768                            file_lengths);
 769             mem_offsets += mem_lengths;
 770             mem_lengths = 0;
 771         } 
 772 
 773         
 774         if (extra_blks) {
 775             file_list_count = extra_blks;
 776             if(!i) {
 777                 file_offsets[0] = offset;
 778                 file_lengths[0] = st_fwr_size;
 779             }
 780             for (k=0; k<extra_blks; k++) {
 781                 if(i || k) {
 782                     file_offsets[k] = disp + n_filetypes*filetype_extent +
 783                       flat_file->indices[j];
 784                     if (k == (extra_blks - 1)) {
 785                         file_lengths[k] = bufsize - (int32_t) mem_lengths
 786                           - (int32_t) mem_offsets + (int32_t)  buf;
 787                     }
 788                     else file_lengths[k] = flat_file->blocklens[j];
 789                 } 
 790                 mem_lengths += file_lengths[k];
 791                 if (j<(flat_file->count - 1)) j++;
 792                 else {
 793                     j = 0;
 794                     n_filetypes++;
 795                 }
 796             } 
 797             pvfs_write_list(fd->fd_sys, mem_list_count, &mem_offsets,
 798                            &mem_lengths, file_list_count, file_offsets,
 799                            file_lengths);
 800         }
 801     } 
 802     else {
 803         
 804 
 805         flat_buf = ADIOI_Flatten_and_find(datatype);
 806 
 807         size_wrote = 0;
 808         n_filetypes = st_n_filetypes;
 809         fwr_size = st_fwr_size;
 810         bwr_size = flat_buf->blocklens[0];
 811         buf_count = 0;
 812         start_mem_offset = 0;
 813         start_k = k = 0;
 814         start_j = st_index;
 815         max_mem_list = 0;
 816         max_file_list = 0;
 817 
 818         
 819 
 820 
 821 
 822         while (size_wrote < bufsize) {
 823             k = start_k;
 824             new_buffer_write = 0;
 825             mem_list_count = 0;
 826             while ((mem_list_count < MAX_ARRAY_SIZE) && 
 827                    (new_buffer_write < bufsize-size_wrote)) {
 828                 
 829 
 830 
 831 
 832 
 833                 if(mem_list_count) {
 834                     if((new_buffer_write + flat_buf->blocklens[k] + 
 835                         size_wrote) > bufsize) {
 836                         end_bwr_size = new_buffer_write + 
 837                             flat_buf->blocklens[k] - (bufsize - size_wrote);
 838                         new_buffer_write = bufsize - size_wrote;
 839                     }
 840                     else {
 841                         new_buffer_write += flat_buf->blocklens[k];
 842                         end_bwr_size = flat_buf->blocklens[k];
 843                     }
 844                 }
 845                 else {
 846                     if (bwr_size > (bufsize - size_wrote)) {
 847                         new_buffer_write = bufsize - size_wrote;
 848                         bwr_size = new_buffer_write;
 849                     }
 850                     else new_buffer_write = bwr_size;
 851                 }
 852                 mem_list_count++;
 853                 k = (k + 1)%flat_buf->count;
 854              } 
 855 
 856             j = start_j;
 857             new_file_write = 0;
 858             file_list_count = 0;
 859             while ((file_list_count < MAX_ARRAY_SIZE) && 
 860                    (new_file_write < new_buffer_write)) {
 861                 if(file_list_count) {
 862                     if((new_file_write + flat_file->blocklens[j]) > 
 863                        new_buffer_write) {
 864                         end_fwr_size = new_buffer_write - new_file_write;
 865                         new_file_write = new_buffer_write;
 866                         j--;
 867                     }
 868                     else {
 869                         new_file_write += flat_file->blocklens[j];
 870                         end_fwr_size = flat_file->blocklens[j];
 871                     }
 872                 }
 873                 else {
 874                     if (fwr_size > new_buffer_write) {
 875                         new_file_write = new_buffer_write;
 876                         fwr_size = new_file_write;
 877                     }
 878                     else new_file_write = fwr_size;
 879                 }
 880                 file_list_count++;
 881                 if (j < (flat_file->count - 1)) j++;
 882                 else j = 0;
 883                 
 884                 k = start_k;
 885                 if ((new_file_write < new_buffer_write) && 
 886                     (file_list_count == MAX_ARRAY_SIZE)) {
 887                     new_buffer_write = 0;
 888                     mem_list_count = 0;
 889                     while (new_buffer_write < new_file_write) {
 890                         if(mem_list_count) {
 891                             if((new_buffer_write + flat_buf->blocklens[k]) >
 892                                new_file_write) {
 893                                 end_bwr_size = new_file_write - 
 894                                     new_buffer_write;
 895                                 new_buffer_write = new_file_write;
 896                                 k--;
 897                             }
 898                             else {
 899                                 new_buffer_write += flat_buf->blocklens[k];
 900                                 end_bwr_size = flat_buf->blocklens[k];
 901                             }
 902                         }
 903                         else {
 904                             new_buffer_write = bwr_size;
 905                             if (bwr_size > (bufsize - size_wrote)) {
 906                                 new_buffer_write = bufsize - size_wrote;
 907                                 bwr_size = new_buffer_write;
 908                             }
 909                         }
 910                         mem_list_count++;
 911                         k = (k + 1)%flat_buf->count;
 912                     } 
 913                 } 
 914 
 915             } 
 916 
 917 
 918             
 919             k = start_k;
 920             j = start_j;
 921             for (i=0; i<mem_list_count; i++) {       
 922                 if(i) {
 923                     if (i == (mem_list_count - 1)) {
 924                         if (flat_buf->blocklens[k] == end_bwr_size)
 925                             bwr_size = flat_buf->blocklens[(k+1)%
 926                                                           flat_buf->count];
 927                         else {
 928                             bwr_size = flat_buf->blocklens[k] - end_bwr_size;
 929                             k--;
 930                             buf_count--;
 931                         }
 932                     }
 933                 }
 934                 buf_count++;
 935                 k = (k + 1)%flat_buf->count;
 936             } 
 937             for (i=0; i<file_list_count; i++) {
 938                 if (i) {
 939                     if (i == (file_list_count - 1)) {
 940                         if (flat_file->blocklens[j] == end_fwr_size)
 941                             fwr_size = flat_file->blocklens[(j+1)%
 942                                                           flat_file->count];   
 943                         else {
 944                             fwr_size = flat_file->blocklens[j] - end_fwr_size;
 945                             j--;
 946                         }
 947                     }
 948                 }
 949                 if (j < flat_file->count - 1) j++;
 950                 else {
 951                     j = 0;
 952                     n_filetypes++;
 953                 }
 954             } 
 955             size_wrote += new_buffer_write;
 956             start_k = k;
 957             start_j = j;
 958             if (max_mem_list < mem_list_count)
 959                 max_mem_list = mem_list_count;
 960             if (max_file_list < file_list_count)
 961                 max_file_list = file_list_count;
 962         } 
 963 
 964         mem_offsets = (char **)ADIOI_Malloc(max_mem_list*sizeof(char *));
 965         mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
 966         file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
 967         file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
 968             
 969         size_wrote = 0;
 970         n_filetypes = st_n_filetypes;
 971         fwr_size = st_fwr_size;
 972         bwr_size = flat_buf->blocklens[0];
 973         buf_count = 0;
 974         start_mem_offset = 0;
 975         start_k = k = 0;
 976         start_j = st_index;
 977 
 978         
 979 
 980 
 981         
 982         while (size_wrote < bufsize) {
 983             k = start_k;
 984             new_buffer_write = 0;
 985             mem_list_count = 0;
 986             while ((mem_list_count < MAX_ARRAY_SIZE) && 
 987                    (new_buffer_write < bufsize-size_wrote)) {
 988                 
 989 
 990 
 991 
 992 
 993                 if(mem_list_count) {
 994                     if((new_buffer_write + flat_buf->blocklens[k] + 
 995                         size_wrote) > bufsize) {
 996                         end_bwr_size = new_buffer_write + 
 997                             flat_buf->blocklens[k] - (bufsize - size_wrote);
 998                         new_buffer_write = bufsize - size_wrote;
 999                     }
1000                     else {
1001                         new_buffer_write += flat_buf->blocklens[k];
1002                         end_bwr_size = flat_buf->blocklens[k];
1003                     }
1004                 }
1005                 else {
1006                     if (bwr_size > (bufsize - size_wrote)) {
1007                         new_buffer_write = bufsize - size_wrote;
1008                         bwr_size = new_buffer_write;
1009                     }
1010                     else new_buffer_write = bwr_size;
1011                 }
1012                 mem_list_count++;
1013                 k = (k + 1)%flat_buf->count;
1014              } 
1015 
1016             j = start_j;
1017             new_file_write = 0;
1018             file_list_count = 0;
1019             while ((file_list_count < MAX_ARRAY_SIZE) && 
1020                    (new_file_write < new_buffer_write)) {
1021                 if(file_list_count) {
1022                     if((new_file_write + flat_file->blocklens[j]) > 
1023                        new_buffer_write) {
1024                         end_fwr_size = new_buffer_write - new_file_write;
1025                         new_file_write = new_buffer_write;
1026                         j--;
1027                     }
1028                     else {
1029                         new_file_write += flat_file->blocklens[j];
1030                         end_fwr_size = flat_file->blocklens[j];
1031                     }
1032                 }
1033                 else {
1034                     if (fwr_size > new_buffer_write) {
1035                         new_file_write = new_buffer_write;
1036                         fwr_size = new_file_write;
1037                     }
1038                     else new_file_write = fwr_size;
1039                 }
1040                 file_list_count++;
1041                 if (j < (flat_file->count - 1)) j++;
1042                 else j = 0;
1043                 
1044                 k = start_k;
1045                 if ((new_file_write < new_buffer_write) && 
1046                     (file_list_count == MAX_ARRAY_SIZE)) {
1047                     new_buffer_write = 0;
1048                     mem_list_count = 0;
1049                     while (new_buffer_write < new_file_write) {
1050                         if(mem_list_count) {
1051                             if((new_buffer_write + flat_buf->blocklens[k]) >
1052                                new_file_write) {
1053                                 end_bwr_size = new_file_write -
1054                                   new_buffer_write;
1055                                 new_buffer_write = new_file_write;
1056                                 k--;
1057                             }
1058                             else {
1059                                 new_buffer_write += flat_buf->blocklens[k];
1060                                 end_bwr_size = flat_buf->blocklens[k];
1061                             }
1062                         }
1063                         else {
1064                             new_buffer_write = bwr_size;
1065                             if (bwr_size > (bufsize - size_wrote)) {
1066                                 new_buffer_write = bufsize - size_wrote;
1067                                 bwr_size = new_buffer_write;
1068                             }
1069                         }
1070                         mem_list_count++;
1071                         k = (k + 1)%flat_buf->count;
1072                     } 
1073                 } 
1074 
1075             } 
1076 
1077 
1078             
1079             k = start_k;
1080             j = start_j;
1081             for (i=0; i<mem_list_count; i++) {       
1082                 mem_offsets[i] = ((char*)buf + buftype_extent*
1083                                          (buf_count/flat_buf->count) +
1084                                          (int)flat_buf->indices[k]);
1085                 
1086                 if(!i) {
1087                     mem_lengths[0] = bwr_size;
1088                     mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
1089                 }
1090                 else {
1091                     if (i == (mem_list_count - 1)) {
1092                         mem_lengths[i] = end_bwr_size;
1093                         if (flat_buf->blocklens[k] == end_bwr_size)
1094                             bwr_size = flat_buf->blocklens[(k+1)%
1095                                                           flat_buf->count];
1096                         else {
1097                             bwr_size = flat_buf->blocklens[k] - end_bwr_size;
1098                             k--;
1099                             buf_count--;
1100                         }
1101                     }
1102                     else {
1103                         mem_lengths[i] = flat_buf->blocklens[k];
1104                     }
1105                 }
1106                 buf_count++;
1107                 k = (k + 1)%flat_buf->count;
1108             } 
1109             for (i=0; i<file_list_count; i++) {
1110                 file_offsets[i] = disp + flat_file->indices[j] + n_filetypes *
1111                     filetype_extent;
1112                 if (!i) {
1113                     file_lengths[0] = fwr_size;
1114                     file_offsets[0] += flat_file->blocklens[j] - fwr_size;
1115                 }
1116                 else {
1117                     if (i == (file_list_count - 1)) {
1118                         file_lengths[i] = end_fwr_size;
1119                         if (flat_file->blocklens[j] == end_fwr_size)
1120                             fwr_size = flat_file->blocklens[(j+1)%
1121                                                           flat_file->count];   
1122                         else {
1123                             fwr_size = flat_file->blocklens[j] - end_fwr_size;
1124                             j--;
1125                         }
1126                     }
1127                     else file_lengths[i] = flat_file->blocklens[j];
1128                 }
1129                 if (j < flat_file->count - 1) j++;
1130                 else {
1131                     j = 0;
1132                     n_filetypes++;
1133                 }
1134             } 
1135             pvfs_write_list(fd->fd_sys,mem_list_count, mem_offsets,
1136                            mem_lengths, file_list_count, file_offsets,
1137                            file_lengths);
1138             size_wrote += new_buffer_write;
1139             start_k = k;
1140             start_j = j;
1141         } 
1142         ADIOI_Free(mem_offsets);
1143         ADIOI_Free(mem_lengths);
1144     }
1145     ADIOI_Free(file_offsets);
1146     ADIOI_Free(file_lengths);
1147 
1148     if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
1149     if (err_flag) {
1150         *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
1151                                            myname, __LINE__, MPI_ERR_IO,
1152                                            "**io",
1153                                            "**io %s", strerror(errno));
1154     }
1155     else *error_code = MPI_SUCCESS;
1156 
1157     fd->fp_sys_posn = -1;   
1158 
1159 #ifdef HAVE_STATUS_SET_BYTES
1160     MPIR_Status_set_bytes(status, datatype, bufsize);
1161 
1162 
1163 #endif
1164 
1165     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
1166 }
1167 #endif