root/ompi/mca/fbtl/ime/fbtl_ime_blocking_op.c

DEFINITIONS

1. mca_fbtl_ime_preadv
2. mca_fbtl_ime_pwritev
3. mca_fbtl_ime_blocking_op

   1 /*
   2  * Copyright (c) 2018      DataDirect Networks. All rights reserved.
   3  * $COPYRIGHT$
   4  *
   5  * Additional copyrights may follow
   6  *
   7  * $HEADER$
   8  */
   9 
  10 #include "ompi_config.h"
  11 #include "fbtl_ime.h"
  12 
  13 #include "mpi.h"
  14 #include "ompi/constants.h"
  15 #include "ompi/mca/fbtl/fbtl.h"
  16 
  17 static ssize_t mca_fbtl_ime_blocking_op(ompio_file_t *fh, int io_op);
  18 
  19 ssize_t mca_fbtl_ime_preadv(ompio_file_t *fh)
  20 {
  21     return mca_fbtl_ime_blocking_op(fh, FBTL_IME_READ);
  22 }
  23 
  24 ssize_t  mca_fbtl_ime_pwritev(ompio_file_t *fh)
  25 {
  26     return mca_fbtl_ime_blocking_op(fh, FBTL_IME_WRITE);
  27 }
  28 
  29 static ssize_t mca_fbtl_ime_blocking_op(ompio_file_t *fh, int io_op)
  30 {
  31     int i, block = 1, ret;
  32     struct iovec *iov = NULL;
  33     int iov_count = 0;
  34     OMPI_MPI_OFFSET_TYPE iov_offset = 0;
  35     ssize_t bytes_processed = 0, ret_code = 0;
  36 
  37     if (NULL == fh->f_io_array) {
  38         return OMPI_ERROR;
  39     }
  40 
  41     iov = (struct iovec *) malloc
  42         (OMPIO_IOVEC_INITIAL_SIZE * sizeof (struct iovec));
  43     if (NULL == iov) {
  44         opal_output(1, "OUT OF MEMORY\n");
  45         return OMPI_ERR_OUT_OF_RESOURCE;
  46     }
  47 
  48     /* Go through all IO entries and try to aggregate them. */
  49     for (i = 0 ; i < fh->f_num_of_io_entries; i++) {
  50         iov[iov_count].iov_base = fh->f_io_array[i].memory_address;
  51         iov[iov_count].iov_len = fh->f_io_array[i].length;
  52         iov_count++;
  53 
  54         /* Save the file offset if the current iovec is
  55            the first one in the iovec array. */
  56         if (iov_count == 1) {
  57             iov_offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset;
  58         }
  59 
  60         /* Allocate more memory for the iovecs if necessary */
  61         if (iov_count == OMPIO_IOVEC_INITIAL_SIZE * block) {
  62             block++;
  63             struct iovec *new_iov = (struct iovec *) realloc(iov, 
  64                     OMPIO_IOVEC_INITIAL_SIZE * block * sizeof(struct iovec));
  65             if (new_iov == NULL) {
  66                 free(iov);
  67                 opal_output(1, "OUT OF MEMORY\n");
  68                 return OMPI_ERR_OUT_OF_RESOURCE;
  69             }
  70         }
  71 
  72         /* If:
  73            - There is no next iovec
  74            - OR the next iovec is not "contiguous"
  75            - OR we exceeded the advised number of iovecs for IME
  76            Then: pwritev/preadv shall be called,
  77                  and the iovec array resetted */
  78         if (i+1 == fh->f_num_of_io_entries ||
  79             ((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset +
  80              (ptrdiff_t)fh->f_io_array[i].length) !=
  81               (OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i+1].offset ||
  82             iov_count >= mca_fbtl_ime_iov_max ) {
  83 
  84             switch (io_op) {
  85             case FBTL_IME_READ:
  86                 ret_code = ime_native_preadv(fh->fd, iov, iov_count, iov_offset);
  87                 if (ret_code < 0) {
  88                     opal_output(1, "mca_fbtl_ime_blocking_op: error in "
  89                                    "ime_native_preadv error ret=%zd  %s",
  90                                    ret_code, strerror(errno));
  91                     goto error_exit;
  92                 }
  93                 break;
  94 
  95             case FBTL_IME_WRITE:
  96                 ret_code = ime_native_pwritev(fh->fd, iov, iov_count, iov_offset);
  97                 if (ret_code < 0) {
  98                     opal_output(1, "mca_fbtl_ime_blocking_op: error in "
  99                                    "ime_native_pwritev error ret=%zd  %s",
 100                                    ret_code, strerror(errno));
 101                     goto error_exit;
 102                 }
 103                 break;
 104 
 105             default:
 106                 opal_output(1, "mca_fbtl_ime_blocking_op: an unsupported "
 107                                "IO operation was requested. io_op=%d", io_op);
 108                 goto error_exit;
 109             }
 110 
 111             bytes_processed += ret_code;
 112             iov_count = 0;
 113         }
 114     }
 115 
 116     free (iov);
 117     return bytes_processed;
 118 
 119 error_exit:
 120     free(iov);
 121     return OMPI_ERROR;
 122 }