root/opal/mca/btl/vader/btl_vader_put.c

DEFINITIONS

1. mca_btl_vader_put_xpmem
2. mca_btl_vader_put_cma
3. mca_btl_vader_put_knem
4. mca_btl_vader_sc_emu_put_complete
5. mca_btl_vader_put_sc_emu

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2010-2014 Los Alamos National Security, LLC. All rights
   4  *                         reserved.
   5  * Copyright (c) 2014-2018 Research Organization for Information Science
   6  *                         and Technology (RIST).  All rights reserved.
   7  * $COPYRIGHT$
   8  *
   9  * Additional copyrights may follow
  10  *
  11  * $HEADER$
  12  */
  13 
  14 #include "opal_config.h"
  15 
  16 #include "btl_vader.h"
  17 #include "btl_vader_frag.h"
  18 #include "btl_vader_endpoint.h"
  19 #include "btl_vader_xpmem.h"
  20 
  21 #if OPAL_BTL_VADER_HAVE_CMA
  22 #include <sys/uio.h>
  23 
  24 #if OPAL_CMA_NEED_SYSCALL_DEFS
  25 #include "opal/sys/cma.h"
  26 #endif /* OPAL_CMA_NEED_SYSCALL_DEFS */
  27 
  28 #endif
  29 
  30 /**
  31  * Initiate an synchronous put.
  32  *
  33  * @param btl (IN)         BTL module
  34  * @param endpoint (IN)    BTL addressing information
  35  * @param descriptor (IN)  Description of the data to be transferred
  36  */
  37 #if OPAL_BTL_VADER_HAVE_XPMEM
  38 int mca_btl_vader_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
  39                              uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
  40                              mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
  41                              int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
  42 {
  43     mca_rcache_base_registration_t *reg;
  44     void *rem_ptr;
  45 
  46     reg = vader_get_registation (endpoint, (void *)(intptr_t) remote_address, size, 0, &rem_ptr);
  47     if (OPAL_UNLIKELY(NULL == reg)) {
  48         return OPAL_ERROR;
  49     }
  50 
  51     vader_memmove (rem_ptr, local_address, size);
  52 
  53     vader_return_registration (reg, endpoint);
  54 
  55     /* always call the callback function */
  56     cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
  57 
  58     return OPAL_SUCCESS;
  59 }
  60 #endif
  61 
  62 #if OPAL_BTL_VADER_HAVE_CMA
  63 int mca_btl_vader_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
  64                            uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
  65                            mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
  66                            int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
  67 {
  68     struct iovec src_iov = {.iov_base = local_address, .iov_len = size};
  69     struct iovec dst_iov = {.iov_base = (void *)(intptr_t) remote_address, .iov_len = size};
  70     ssize_t ret;
  71 
  72     /* This should not be needed, see the rationale in mca_btl_vader_get_cma() */
  73     do {
  74         ret = process_vm_writev (endpoint->segment_data.other.seg_ds->seg_cpid, &src_iov, 1, &dst_iov, 1, 0);
  75         if (0 > ret) {
  76             opal_output(0, "Wrote %ld, expected %lu, errno = %d\n", (long)ret, (unsigned long)size, errno);
  77             return OPAL_ERROR;
  78         }
  79         src_iov.iov_base = (void *)((char *)src_iov.iov_base + ret);
  80         src_iov.iov_len -= ret;
  81         dst_iov.iov_base = (void *)((char *)dst_iov.iov_base + ret);
  82         dst_iov.iov_len -= ret;
  83     } while (0 < src_iov.iov_len);
  84 
  85     /* always call the callback function */
  86     cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
  87 
  88     return OPAL_SUCCESS;
  89 }
  90 #endif
  91 
  92 #if OPAL_BTL_VADER_HAVE_KNEM
  93 int mca_btl_vader_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
  94                             uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
  95                             mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
  96                             int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
  97 {
  98     struct knem_cmd_param_iovec send_iovec;
  99     struct knem_cmd_inline_copy icopy;
 100 
 101     /* Fill in the ioctl data fields.  There's no async completion, so
 102        we don't need to worry about getting a slot, etc. */
 103     send_iovec.base = (uintptr_t) local_address;
 104     send_iovec.len = size;
 105     icopy.local_iovec_array = (uintptr_t) &send_iovec;
 106     icopy.local_iovec_nr    = 1;
 107     icopy.remote_cookie     = remote_handle->cookie;
 108     icopy.remote_offset     = remote_address - remote_handle->base_addr;
 109     icopy.write             = 1;
 110     icopy.flags             = 0;
 111 
 112     /* Use the DMA flag if knem supports it *and* the segment length
 113      * is greater than the cutoff. Not that if DMA is not supported
 114      * or the user specified 0 for knem_dma_min the knem_dma_min was
 115      * set to UINT_MAX in mca_btl_vader_knem_init. */
 116     if (mca_btl_vader_component.knem_dma_min <= size) {
 117         icopy.flags = KNEM_FLAG_DMA;
 118     }
 119     /* synchronous flags only, no need to specify icopy.async_status_index */
 120 
 121     /* When the ioctl returns, the transfer is done and we can invoke
 122        the btl callback and return the frag */
 123     if (OPAL_UNLIKELY(0 != ioctl (mca_btl_vader.knem_fd, KNEM_CMD_INLINE_COPY, &icopy))) {
 124         return OPAL_ERROR;
 125     }
 126 
 127     if (KNEM_STATUS_FAILED == icopy.current_status) {
 128         return OPAL_ERROR;
 129     }
 130 
 131     /* always call the callback function */
 132     cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
 133 
 134     return OPAL_SUCCESS;
 135 }
 136 #endif
 137 
 138 static void mca_btl_vader_sc_emu_put_complete (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
 139                                                mca_btl_base_descriptor_t *desc, int status)
 140 {
 141     mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) desc;
 142     void *local_address = frag->rdma.local_address;
 143     void *context = frag->rdma.context;
 144     void *cbdata = frag->rdma.cbdata;
 145     mca_btl_base_rdma_completion_fn_t cbfunc = frag->rdma.cbfunc;
 146 
 147     /* return the fragment first since the callback may call put/get/amo and could use this fragment */
 148     MCA_BTL_VADER_FRAG_RETURN(frag);
 149 
 150     cbfunc (btl, endpoint, local_address, NULL, context, cbdata, status);
 151 }
 152 
 153 /**
 154  * @brief Provides an emulated put path which uses copy-in copy-out with shared memory buffers
 155  */
 156 int mca_btl_vader_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
 157                               uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 158                               mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 159                               int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 160 {
 161     mca_btl_vader_sc_emu_hdr_t *hdr;
 162     mca_btl_vader_frag_t *frag;
 163 
 164     if (size > mca_btl_vader.super.btl_put_limit) {
 165         return OPAL_ERR_NOT_AVAILABLE;
 166     }
 167 
 168     frag = mca_btl_vader_rdma_frag_alloc (btl, endpoint, MCA_BTL_VADER_OP_PUT, 0, 0, 0, order, flags, size,
 169                                           local_address, remote_address, cbfunc, cbcontext, cbdata,
 170                                           mca_btl_vader_sc_emu_put_complete);
 171     if (OPAL_UNLIKELY(NULL == frag)) {
 172         return OPAL_ERR_OUT_OF_RESOURCE;
 173     }
 174 
 175     hdr = (mca_btl_vader_sc_emu_hdr_t *) frag->segments[0].seg_addr.pval;
 176 
 177     memcpy ((void *) (hdr + 1), local_address, size);
 178 
 179     /* send is always successful */
 180     (void) mca_btl_vader_send (btl, endpoint, &frag->base, MCA_BTL_TAG_VADER);
 181 
 182     return OPAL_SUCCESS;
 183 }