root/ompi/mca/io/romio321/romio/adio/ad_pvfs2/ad_pvfs2_io_list.c

DEFINITIONS

1. ADIOI_PVFS2_StridedListIO
2. gen_listio_arr
3. print_buf_file_ol_pairs

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
   2  *   vim: ts=8 sts=4 sw=4 noexpandtab
   3  *
   4  *   Copyright (C) 2006 Unknown (TODO: fix this)
   5  */
   6 
   7 #include <assert.h>
   8 #include "adio.h"
   9 #include "adio_extern.h"
  10 #include "ad_pvfs2.h"
  11 #include "ad_pvfs2_io.h"
  12 #include "ad_pvfs2_common.h"
  13 
  14 #define COALESCE_REGIONS  /* TODO: would we ever want to *not* coalesce? */
  15 #define MAX_OL_COUNT 64
  16 int ADIOI_PVFS2_StridedListIO(ADIO_File fd, void *buf, int count,
  17                               MPI_Datatype datatype, int file_ptr_type,
  18                               ADIO_Offset offset, ADIO_Status *status,
  19                               int *error_code, int rw_type)
  20 {
  21     /* list I/O parameters */
  22     int i = -1, ret = -1;
  23     int tmp_filetype_size = -1;
  24     int64_t cur_io_size = 0, io_size = 0;
  25     int etype_size = -1;
  26     int num_etypes_in_filetype = -1, num_filetypes = -1;
  27     int etypes_in_filetype = -1, size_in_filetype = -1;
  28     int bytes_into_filetype = 0;
  29     MPI_Offset total_bytes_accessed = 0;
  30     
  31     /* parameters for offset-length pairs arrays */
  32     int64_t buf_off_arr[MAX_OL_COUNT];
  33     int32_t buf_len_arr[MAX_OL_COUNT];
  34     int64_t file_off_arr[MAX_OL_COUNT];
  35     int32_t file_len_arr[MAX_OL_COUNT];
  36     int32_t buf_ol_count = 0;
  37     int32_t file_ol_count = 0;
  38     
  39     /* parameters for flattened memory and file datatypes*/
  40     int flat_buf_index = 0;
  41     int flat_file_index = 0;
  42     int64_t cur_flat_buf_reg_off = 0;
  43     int64_t cur_flat_file_reg_off = 0;
  44     ADIOI_Flatlist_node *flat_buf_p, *flat_file_p;
  45     MPI_Count buftype_size = -1, filetype_size = -1;
  46     MPI_Aint filetype_extent = -1, buftype_extent = -1;;
  47     MPI_Aint filetype_lb = -1, buftype_lb = -1;;
  48     int buftype_is_contig = -1, filetype_is_contig = -1;
  49     
  50     /* PVFS2 specific parameters */
  51     PVFS_Request mem_req, file_req;
  52     ADIOI_PVFS2_fs * pvfs_fs;
  53     PVFS_sysresp_io resp_io;
  54     static char myname[] = "ADIOI_PVFS2_STRIDED_LISTIO";
  55 
  56     if (fd->atomicity) {
  57         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
  58                                            MPIR_ERR_RECOVERABLE,
  59                                            myname, __LINE__,
  60                                            MPI_ERR_ARG,
  61                                            "Atomic noncontiguous writes"
  62                                            " are not supported by PVFS2", 0);
  63         return -1;
  64     }
  65 
  66     MPI_Type_size_x(fd->filetype, &filetype_size);
  67     if (filetype_size == 0) {
  68         *error_code = MPI_SUCCESS;
  69         return -1;
  70     }
  71     MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
  72     MPI_Type_size_x(datatype, &buftype_size);
  73     MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
  74     io_size = buftype_size*count;
  75 
  76     pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
  77     
  78     /* Flatten the memory datatype
  79      * (file datatype has already been flattened in ADIO open
  80      * unless it is contibuous, then we need to flatten it manually)
  81      * and set the correct buffers for flat_buf and flat_file */
  82     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
  83     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
  84     if (buftype_is_contig == 0)
  85     {
  86         flat_buf_p = ADIOI_Flatten_and_find(datatype);
  87     }
  88     else 
  89     {
  90         /* flatten and add to the list */
  91         flat_buf_p = (ADIOI_Flatlist_node *) ADIOI_Malloc
  92             (sizeof(ADIOI_Flatlist_node));
  93         flat_buf_p->blocklens = (ADIO_Offset*)ADIOI_Malloc(sizeof(ADIO_Offset));
  94         flat_buf_p->indices = 
  95                 (ADIO_Offset *) ADIOI_Malloc(sizeof(ADIO_Offset));
  96         /* For the buffer, we can optimize the buftype, this is not 
  97          * possible with the filetype since it is tiled */
  98         buftype_size = buftype_size*count;
  99         buftype_extent = buftype_size*count;
 100         flat_buf_p->blocklens[0] = buftype_size;
 101         flat_buf_p->indices[0] = 0;
 102         flat_buf_p->count = 1;
 103     }
 104     if (filetype_is_contig == 0)
 105     {
 106             /* TODO: why does avery say this should already have been
 107              * flattened in Open, but also says contig types don't get
 108              * flattened */
 109         flat_file_p = ADIOI_Flatten_and_find(fd->filetype);
 110     }
 111     else
 112     {
 113         /* flatten and add to the list */
 114         flat_file_p = (ADIOI_Flatlist_node *) ADIOI_Malloc
 115             (sizeof(ADIOI_Flatlist_node));
 116         flat_file_p->blocklens =(ADIO_Offset*)ADIOI_Malloc(sizeof(ADIO_Offset));
 117         flat_file_p->indices = 
 118                 (ADIO_Offset *) ADIOI_Malloc(sizeof(ADIO_Offset));
 119         flat_file_p->blocklens[0] = filetype_size;
 120         flat_file_p->indices[0] = 0;
 121         flat_file_p->count = 1;
 122     }
 123         
 124     /* Find out where we are in the flattened filetype (the block index, 
 125      * how far into the block, and how many bytes_into_filetype)
 126      * If the file_ptr_type == ADIO_INDIVIDUAL we will use disp, fp_ind 
 127      * to figure this out (offset should always be zero) 
 128      * If file_ptr_type == ADIO_EXPLICIT, we will use disp and offset 
 129      * to figure this out. */
 130 
 131     etype_size = fd->etype_size;
 132     num_etypes_in_filetype = filetype_size / etype_size;
 133     if (file_ptr_type == ADIO_INDIVIDUAL)
 134     {
 135         int flag = 0;
 136         /* Should have already been flattened in ADIO_Open*/
 137         num_filetypes = -1;
 138         while (!flag)
 139         {
 140             num_filetypes++;
 141             for (i = 0; i < flat_file_p->count; i++)
 142             {
 143                 /* Start on a non zero-length region */
 144                 if (flat_file_p->blocklens[i])
 145                 {
 146                     if (fd->disp + flat_file_p->indices[i] +
 147                         (num_filetypes * filetype_extent) +
 148                         flat_file_p->blocklens[i] > fd->fp_ind &&
 149                         fd->disp + flat_file_p->indices[i] <=
 150                         fd->fp_ind)
 151                     {
 152                         flat_file_index = i;
 153                         cur_flat_file_reg_off = fd->fp_ind -
 154                             (fd->disp + flat_file_p->indices[i] +
 155                              (num_filetypes * filetype_extent));
 156                         flag = 1;
 157                         break;
 158                     }
 159                     else
 160                         bytes_into_filetype += flat_file_p->blocklens[i];
 161                 }
 162             }
 163         }
 164         /* Impossible that we don't find it in this datatype */
 165         assert(i != flat_file_p->count);
 166     }
 167     else
 168     { 
 169         num_filetypes = (int) (offset / num_etypes_in_filetype);
 170         etypes_in_filetype = (int) (offset % num_etypes_in_filetype);
 171         size_in_filetype = etypes_in_filetype * etype_size;
 172 
 173         tmp_filetype_size = 0;
 174         for (i=0; i<flat_file_p->count; i++) {
 175             tmp_filetype_size += flat_file_p->blocklens[i];
 176             if (tmp_filetype_size > size_in_filetype) 
 177             {
 178                 flat_file_index = i;
 179                 cur_flat_file_reg_off = flat_file_p->blocklens[i] - 
 180                     (tmp_filetype_size - size_in_filetype);
 181                 bytes_into_filetype = offset * filetype_size -
 182                     flat_file_p->blocklens[i];
 183                 break;
 184             }
 185         }
 186     }
 187 #ifdef DEBUG_LIST
 188     fprintf(stderr, "ADIOI_PVFS2_StridedListIO: (fd->fp_ind=%Ld,fd->disp=%Ld,"
 189             " offset=%Ld)\n(flat_file_index=%d,cur_flat_file_reg_off=%Ld,"
 190             "bytes_into_filetype=%d)\n",
 191             fd->fp_ind, fd->disp, offset, flat_file_index, 
 192             cur_flat_file_reg_off, bytes_into_filetype);
 193 #endif
 194 #ifdef DEBUG_LIST2
 195     fprintf(stderr, "flat_buf:\n");
 196     for (i = 0; i < flat_buf_p->count; i++)
 197         fprintf(stderr, "(offset, length) = (%Ld, %d)\n",
 198                flat_buf_p->indices[i],
 199                flat_buf_p->blocklens[i]);
 200     fprintf(stderr, "flat_file:\n");
 201     for (i = 0; i < flat_file_p->count; i++)
 202         fprintf(stderr, "(offset, length) = (%Ld, %d)\n",
 203                flat_file_p->indices[i],
 204                flat_file_p->blocklens[i]);
 205 #endif    
 206 
 207     /* total data written */
 208     cur_io_size = 0;
 209     while (cur_io_size != io_size)
 210     {
 211         /* Initialize the temporarily unrolling lists and 
 212          * and associated variables */
 213         buf_ol_count = 0;
 214         file_ol_count = 0;
 215         for (i = 0; i < MAX_OL_COUNT; i++)
 216         {
 217             buf_off_arr[i] = 0;
 218             buf_len_arr[i] = 0;
 219             file_off_arr[i] = 0;
 220             file_len_arr[i] = 0;
 221         }
 222 
 223         /* Generate the offset-length pairs for a
 224          * list I/O operation */
 225         gen_listio_arr(flat_buf_p,
 226                        &flat_buf_index,
 227                        &cur_flat_buf_reg_off,
 228                        buftype_size,
 229                        buftype_extent,
 230                        flat_file_p,
 231                        &flat_file_index,
 232                        &cur_flat_file_reg_off,
 233                        filetype_size,
 234                        filetype_extent,
 235                        MAX_OL_COUNT,
 236                        fd->disp,
 237                        bytes_into_filetype,
 238                        &cur_io_size,
 239                        io_size,
 240                        buf_off_arr,
 241                        buf_len_arr,
 242                        &buf_ol_count,
 243                        file_off_arr,
 244                        file_len_arr,
 245                        &file_ol_count);
 246 
 247         assert(buf_ol_count <= MAX_OL_COUNT);
 248         assert(file_ol_count <= MAX_OL_COUNT);
 249 #ifdef DEBUG_LIST2
 250         print_buf_file_ol_pairs(buf_off_arr,
 251                                 buf_len_arr,
 252                                 buf_ol_count,
 253                                 file_off_arr,
 254                                 file_len_arr,
 255                                 file_ol_count,
 256                                 buf,
 257                                 rw_type);
 258 #endif
 259 #ifdef DEBUG_LIST2
 260         do {
 261             int y, z;
 262             fprintf(stderr, "ad_pvfs2_io_list.c::\n");
 263             for (y = 0; y < buf_ol_count; y++)
 264             {
 265                 for (z = 0; z < buf_len_arr[y]; z++)
 266                 {
 267                     fprintf(stderr, "buf[%d][%d]=%c\n",
 268                             y, z, ((char *) buf + buf_off_arr[y])[z]);
 269                 }
 270             }
 271         } while (0);
 272 #endif
 273         
 274         /* Run list I/O operation */
 275         ret = PVFS_Request_hindexed(buf_ol_count, buf_len_arr,
 276                                     buf_off_arr, PVFS_BYTE, &mem_req);
 277 
 278         ret = PVFS_Request_hindexed(file_ol_count, file_len_arr,
 279                                     file_off_arr, PVFS_BYTE, &file_req);
 280         if (rw_type == READ)
 281         {
 282             ret = PVFS_sys_read(pvfs_fs->object_ref, file_req, 0,
 283                                 buf, mem_req, 
 284                                 &(pvfs_fs->credentials), &resp_io);
 285         }
 286         else 
 287         {
 288             ret = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0,
 289                                  buf, mem_req, 
 290                                  &(pvfs_fs->credentials), &resp_io);
 291         }
 292         if (ret != 0) 
 293         {
 294             fprintf(stderr, "ADIOI_PVFS2_StridedListIO: Warning - PVFS_sys_"
 295                     "read/write returned %d and completed %lld bytes.\n",
 296                     ret, (long long)resp_io.total_completed);
 297             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
 298                                                MPIR_ERR_RECOVERABLE,
 299                                                myname, __LINE__,
 300                                                ADIOI_PVFS2_error_convert(ret),
 301                                                "Error in PVFS_sys_io \n", 0);
 302             PVFS_Request_free(&mem_req);
 303             PVFS_Request_free(&file_req);
 304             goto error_state;
 305         }
 306         total_bytes_accessed += resp_io.total_completed;
 307 
 308         PVFS_Request_free(&mem_req);
 309         PVFS_Request_free(&file_req);
 310     }
 311     
 312 #ifdef DEBUG_LIST
 313     fprintf(stderr, "ADIOI_PVFS2_StridedListIO: "
 314             "total_bytes_accessed=%Ld,ret=%d\n", 
 315             total_bytes_accessed, ret);
 316 #endif
 317 
 318     if (file_ptr_type == ADIO_INDIVIDUAL) 
 319         fd->fp_ind += total_bytes_accessed;
 320     *error_code = MPI_SUCCESS;
 321 
 322 error_state:
 323 #ifdef HAVE_STATUS_SET_BYTES
 324     /* TODO: why the cast? */
 325     MPIR_Status_set_bytes(status, datatype, total_bytes_accessed);
 326 /* This is a temporary way of filling in status. The right way is to
 327    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
 328 #endif
 329     if (buftype_is_contig == 0)
 330         ADIOI_Delete_flattened(datatype);
 331     else
 332     {
 333         ADIOI_Free(flat_buf_p->blocklens);
 334         ADIOI_Free(flat_buf_p->indices);
 335         ADIOI_Free(flat_buf_p);
 336     }
 337 
 338     if (filetype_is_contig == 0)
 339         ADIOI_Delete_flattened(fd->filetype);
 340     else
 341     {
 342         ADIOI_Free(flat_file_p->blocklens);
 343         ADIOI_Free(flat_file_p->indices);
 344         ADIOI_Free(flat_file_p);
 345     }
 346 
 347     return 0;
 348 }
 349 
 350 /* To do: Fix the code to coalesce the offset-length pairs for memory
 351  * and file. */
 352 
 353 /* gen_listio_arr - fills in offset-length pairs for memory and file
 354  * for list I/O */
 355 int gen_listio_arr(ADIOI_Flatlist_node *flat_buf_p,
 356                    int *flat_buf_index_p,
 357                    int64_t *cur_flat_buf_reg_off_p,
 358                    int flat_buf_size,
 359                    int flat_buf_extent,
 360                    ADIOI_Flatlist_node *flat_file_p,
 361                    int *flat_file_index_p,
 362                    int64_t *cur_flat_file_reg_off_p,
 363                    int flat_file_size,
 364                    int flat_file_extent,
 365                    int max_ol_count,
 366                    ADIO_Offset disp,
 367                    int bytes_into_filetype,
 368                    int64_t *bytes_completed,
 369                    int64_t total_io_size,
 370                    int64_t buf_off_arr[],
 371                    int32_t buf_len_arr[],
 372                    int32_t *buf_ol_count_p,
 373                    int64_t file_off_arr[],
 374                    int32_t file_len_arr[],
 375                    int32_t *file_ol_count_p)
 376 {
 377     int region_size = -1;
 378     
 379     /* parameters for flattened memory and file datatypes*/
 380     int64_t cur_flat_buf_reg_left = 0;
 381     int64_t cur_flat_file_reg_left = 0;
 382     
 383 #ifdef DEBUG_LIST2
 384     fprintf(stderr, "gen_list_arr:\n");
 385 #endif
 386 
 387     if ((*buf_ol_count_p) != 0 ||(*file_ol_count_p) != 0)
 388     {
 389         fprintf(stderr, "buf_ol_count != 0 || file_ol_count != 0\n");
 390         return -1;
 391     }
 392     
 393     /* Start on a non-zero memory and file region 
 394      * Note this does not affect the bytes_completed 
 395      * since no data is in these regions.  Initialize the 
 396      * first memory and file offsets. */
 397     while (flat_buf_p->blocklens[(*flat_buf_index_p)] == 0)
 398     {
 399         (*flat_buf_index_p) = ((*flat_buf_index_p) + 1) % 
 400             flat_buf_p->count;
 401     }
 402     buf_off_arr[*buf_ol_count_p] =
 403         (*bytes_completed / flat_buf_size) * 
 404         flat_buf_extent + 
 405         flat_buf_p->indices[*flat_buf_index_p] +
 406         *cur_flat_buf_reg_off_p;
 407     buf_len_arr[*buf_ol_count_p] = 0;
 408 
 409     while (flat_file_p->blocklens[(*flat_file_index_p)] == 0)
 410     {
 411         (*flat_file_index_p) = ((*flat_file_index_p) + 1) % 
 412             flat_file_p->count;
 413     }
 414     file_off_arr[*file_ol_count_p] = disp + 
 415         (((bytes_into_filetype + *bytes_completed) / flat_file_size) * 
 416          flat_file_extent) + 
 417         flat_file_p->indices[*flat_file_index_p] +
 418         *cur_flat_file_reg_off_p;
 419     file_len_arr[*file_ol_count_p] = 0;
 420 
 421 #ifdef DEBUG_LIST2
 422     fprintf(stderr, "initial buf_off_arr[%d] = %Ld\n", *buf_ol_count_p,
 423             buf_off_arr[*buf_ol_count_p]);
 424     fprintf(stderr, "initial file_off_arr[%d] = %Ld\n", *file_ol_count_p,
 425             file_off_arr[*file_ol_count_p]);
 426 #endif
 427 
 428     while (*bytes_completed != total_io_size
 429            && (*buf_ol_count_p) < max_ol_count
 430            && (*file_ol_count_p) < max_ol_count)
 431     {
 432         /* How much data is left in the current piece in
 433          * the flattened datatypes */
 434         cur_flat_buf_reg_left = flat_buf_p->blocklens[*flat_buf_index_p]
 435             - *cur_flat_buf_reg_off_p;
 436         cur_flat_file_reg_left = flat_file_p->blocklens[*flat_file_index_p]
 437             - *cur_flat_file_reg_off_p;
 438 
 439 #ifdef DEBUG_LIST2
 440         fprintf(stderr, 
 441                 "flat_buf_index=%d flat_buf->blocklens[%d]=%d\n"
 442                 "cur_flat_buf_reg_left=%Ld "
 443                 "*cur_flat_buf_reg_off_p=%Ld\n" 
 444                 "flat_file_index=%d flat_file->blocklens[%d]=%d\n"
 445                 "cur_flat_file_reg_left=%Ld "
 446                 "*cur_flat_file_reg_off_p=%Ld\n" 
 447                 "bytes_completed=%Ld\n"
 448                 "buf_ol_count=%d file_ol_count=%d\n"
 449                 "buf_len_arr[%d]=%d file_len_arr[%d]=%d\n\n",
 450                 *flat_buf_index_p, *flat_buf_index_p, 
 451                 flat_buf_p->blocklens[*flat_buf_index_p],
 452                 cur_flat_buf_reg_left,
 453                 *cur_flat_buf_reg_off_p,
 454                 *flat_file_index_p, *flat_file_index_p, 
 455                 flat_file_p->blocklens[*flat_file_index_p],
 456                 cur_flat_file_reg_left,
 457                 *cur_flat_file_reg_off_p,
 458                 *bytes_completed,
 459                 *buf_ol_count_p, *file_ol_count_p,
 460                 *buf_ol_count_p,
 461                 buf_len_arr[*buf_ol_count_p],
 462                 *file_ol_count_p,
 463                 file_len_arr[*file_ol_count_p]);
 464 #endif
 465 
 466         /* What is the size of the next contiguous region agreed
 467          * upon by both memory and file regions that does not
 468          * surpass the file size */
 469         if (cur_flat_buf_reg_left > cur_flat_file_reg_left)
 470             region_size = cur_flat_file_reg_left;
 471         else
 472             region_size = cur_flat_buf_reg_left;
 473         
 474         if (region_size > total_io_size - *bytes_completed)
 475             region_size = total_io_size - *bytes_completed;
 476         
 477         /* Add this piece to both the mem and file arrays
 478          * coalescing offset-length pairs if possible and advance 
 479          * the pointers through the flatten mem and file datatypes
 480          * as well Note: no more than a single piece can be done 
 481          * since we take the smallest one possible */
 482         
 483         if (cur_flat_buf_reg_left == region_size)
 484         {
 485 #ifdef DEBUG_LIST2
 486             fprintf(stderr, "reached end of memory block...\n");
 487 #endif
 488             (*flat_buf_index_p) = ((*flat_buf_index_p) + 1) % 
 489                 flat_buf_p->count;
 490             while (flat_buf_p->blocklens[(*flat_buf_index_p)] == 0)
 491             {
 492                 (*flat_buf_index_p) = ((*flat_buf_index_p) + 1) % 
 493                     flat_buf_p->count;
 494             }
 495             *cur_flat_buf_reg_off_p = 0;
 496 
 497 #ifdef COALESCE_REGIONS
 498             if (*buf_ol_count_p != 0)
 499             {
 500                 if (buf_off_arr[(*buf_ol_count_p) - 1] +
 501                     buf_len_arr[(*buf_ol_count_p) - 1] ==
 502                     buf_off_arr[*buf_ol_count_p])
 503                 {
 504                     buf_len_arr[(*buf_ol_count_p) - 1] +=
 505                         region_size;
 506                 }
 507                 else
 508                 {
 509                     buf_len_arr[*buf_ol_count_p] += region_size;
 510                     (*buf_ol_count_p)++;
 511                 }
 512             }
 513             else
 514             {
 515 #endif
 516                 buf_len_arr[*buf_ol_count_p] += region_size;
 517                 (*buf_ol_count_p)++;
 518 #ifdef COALESCE_REGIONS
 519             }
 520 #endif
 521 
 522             /* Don't prepare for the next piece if we have reached 
 523              * the limit or else it will segment fault. */
 524             if ((*buf_ol_count_p) != max_ol_count)
 525             {
 526                 buf_off_arr[*buf_ol_count_p] = 
 527                     ((*bytes_completed + region_size) / flat_buf_size) * 
 528                     flat_buf_extent + 
 529                     flat_buf_p->indices[*flat_buf_index_p] +
 530                     (*cur_flat_buf_reg_off_p);
 531                 buf_len_arr[*buf_ol_count_p] = 0;
 532             }
 533         }
 534         else if (cur_flat_buf_reg_left > region_size)
 535         {
 536 #ifdef DEBUG_LIST2
 537             fprintf(stderr, "advanced %d in memory block...\n",
 538                     region_size);
 539 #endif
 540             (*cur_flat_buf_reg_off_p) += region_size;
 541             buf_len_arr[*buf_ol_count_p] += region_size;
 542         }
 543         else
 544         {
 545             fprintf(stderr, "gen_listio_arr: Error\n");
 546         }
 547         
 548         /* To calculate the absolute file offset we need to 
 549          * add the disp, how many filetypes we have gone through,
 550          * the relative block offset in the filetype and how far
 551          * into the block we have gone. */
 552         if (cur_flat_file_reg_left == region_size)
 553         {
 554 #ifdef DEBUG_LIST2
 555             fprintf(stderr, "reached end of file block...\n");
 556 #endif
 557             (*flat_file_index_p) = ((*flat_file_index_p) + 1) % 
 558                 flat_file_p->count;
 559             while (flat_file_p->blocklens[(*flat_file_index_p)] == 0)
 560             {
 561                 (*flat_file_index_p) = ((*flat_file_index_p) + 1) % 
 562                     flat_file_p->count;
 563             }
 564             (*cur_flat_file_reg_off_p) = 0;
 565 
 566 #ifdef COALESCE_REGIONS
 567             if (*file_ol_count_p != 0)
 568             {
 569                 if (file_off_arr[(*file_ol_count_p) - 1] +
 570                     file_len_arr[(*file_ol_count_p) - 1] ==
 571                     file_off_arr[*file_ol_count_p])
 572                 {
 573                     file_len_arr[(*file_ol_count_p) - 1] +=
 574                         region_size;
 575                 }
 576                 else
 577                 {
 578                     file_len_arr[*file_ol_count_p] += region_size;
 579                     (*file_ol_count_p)++;
 580                 }
 581             }
 582             else
 583             {
 584 #endif
 585                 file_len_arr[*file_ol_count_p] += region_size;
 586                 (*file_ol_count_p)++;
 587 #ifdef COALESCE_REGIONS
 588             }
 589 #endif
 590 
 591             /* Don't prepare for the next piece if we have reached
 592              * the limit or else it will segment fault. */
 593             if ((*file_ol_count_p) != max_ol_count)
 594             {
 595                 file_off_arr[*file_ol_count_p] = disp + 
 596                     (((bytes_into_filetype + *bytes_completed + region_size) 
 597                       / flat_file_size) * 
 598                      flat_file_extent) + 
 599                     flat_file_p->indices[*flat_file_index_p] +
 600                     (*cur_flat_file_reg_off_p);
 601                 file_len_arr[*file_ol_count_p] = 0;
 602             }
 603         }
 604         else if (cur_flat_file_reg_left > region_size)
 605         {
 606 #ifdef DEBUG_LIST2
 607             fprintf(stderr, "advanced %d in file block...\n",
 608                     region_size);
 609 #endif
 610             (*cur_flat_file_reg_off_p) += region_size;
 611             file_len_arr[*file_ol_count_p] += region_size;
 612         }
 613         else
 614         {
 615             fprintf(stderr, "gen_listio_arr: Error\n");
 616         }
 617 #ifdef DEBUG_LIST2
 618         fprintf(stderr, 
 619                 "------------------------------\n\n");
 620 #endif
 621         *bytes_completed += region_size;
 622     }
 623     /* Increment the count if we stopped in the middle of a 
 624      * memory or file region */
 625     if (*cur_flat_buf_reg_off_p != 0)
 626         (*buf_ol_count_p)++;
 627     if (*cur_flat_file_reg_off_p != 0)
 628         (*file_ol_count_p)++;
 629 
 630     return 0;
 631 }
 632 
 633 void print_buf_file_ol_pairs(int64_t buf_off_arr[],
 634                              int32_t buf_len_arr[],
 635                              int32_t buf_ol_count,
 636                              int64_t file_off_arr[],
 637                              int32_t file_len_arr[],
 638                              int32_t file_ol_count,
 639                              void *buf,
 640                              int rw_type)
 641 {
 642     int i = -1;
 643     
 644     fprintf(stderr, "buf_ol_pairs(offset,length) count = %d\n",
 645             buf_ol_count);
 646     for (i = 0; i < buf_ol_count; i++)
 647     {
 648         fprintf(stderr, "(%lld, %d) ", (long long)buf_off_arr[i], buf_len_arr[i]);
 649     }
 650     fprintf(stderr, "\n");
 651 
 652     fprintf(stderr, "file_ol_pairs(offset,length) count = %d\n",
 653             file_ol_count);
 654     for (i = 0; i < file_ol_count; i++)
 655     {
 656         fprintf(stderr, "(%lld, %d) ", (long long)file_off_arr[i], file_len_arr[i]);
 657     }
 658     fprintf(stderr, "\n\n");
 659 
 660 }