root/ompi/mca/osc/rdma/osc_rdma_comm.h

INCLUDED FROM

DEFINITIONS

1. osc_rdma_get_remote_segment

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2014-2015 Los Alamos National Security, LLC.  All rights
   4  *                         reserved.
   5  * Copyright (c) 2017      Research Organization for Information Science
   6  *                         and Technology (RIST). All rights reserved.
   7  * $COPYRIGHT$
   8  *
   9  * Additional copyrights may follow
  10  *
  11  * $HEADER$
  12  */
  13 
  14 #if !defined(OMPI_OSC_RDMA_COMM_H)
  15 #define OMPI_OSC_RDMA_COMM_H
  16 
  17 #include "osc_rdma_dynamic.h"
  18 #include "osc_rdma_request.h"
  19 #include "osc_rdma_sync.h"
  20 #include "osc_rdma_lock.h"
  21 
  22 #define OMPI_OSC_RDMA_DECODE_MAX 64
  23 
  24 #define min(a,b) ((a) < (b) ? (a) : (b))
  25 #define ALIGNMENT_MASK(x) ((x) ? (x) - 1 : 0)
  26 
  27 /**
  28  * @brief find a remote segment associate with the memory region
  29  *
  30  * @param[in]  module         osc rdma module
  31  * @param[in]  peer           peer object for remote peer
  32  * @param[in]  target_disp    displacement in remote region
  33  * @param[in]  length         length of remote region
  34  * @param[out] remote_address remote address
  35  * @param[out] remote_handle  btl handle for remote region (valid over entire region)
  36  *
  37  * @returns OMPI_SUCCESS on success
  38  * @returns OMPI_ERR_RMA_RANGE if the address range is not valid at the remote window
  39  * @returns other OMPI error on error
  40  */
  41 static inline int osc_rdma_get_remote_segment (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer, ptrdiff_t target_disp,
  42                                                size_t length, uint64_t *remote_address, mca_btl_base_registration_handle_t **remote_handle)
  43 {
  44     ompi_osc_rdma_region_t *region;
  45     int ret;
  46 
  47     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "getting remote address for peer %d target_disp %lu. peer flags: 0x%x",
  48                      peer->rank, (unsigned long) target_disp, peer->flags);
  49 
  50     if (MPI_WIN_FLAVOR_DYNAMIC == module->flavor) {
  51         ret = ompi_osc_rdma_find_dynamic_region (module, peer, (uint64_t) target_disp, length, &region);
  52         if (OMPI_SUCCESS != ret) {
  53             OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "could not retrieve region for %" PRIx64 " from window rank %d",
  54                              (uint64_t) target_disp, peer->rank);
  55             return ret;
  56         }
  57 
  58         *remote_address = (uint64_t) target_disp;
  59         *remote_handle = (mca_btl_base_registration_handle_t *) region->btl_handle_data;
  60     } else {
  61         ompi_osc_rdma_peer_extended_t *ex_peer = (ompi_osc_rdma_peer_extended_t *) peer;
  62         int disp_unit = (module->same_disp_unit) ? module->disp_unit : ex_peer->disp_unit;
  63         size_t size = (module->same_size) ? module->size : (size_t) ex_peer->size;
  64 
  65         *remote_address = ex_peer->super.base + disp_unit * target_disp;
  66         if (OPAL_UNLIKELY(*remote_address + length > (ex_peer->super.base + size))) {
  67             OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "remote address range 0x%" PRIx64 " - 0x%" PRIx64
  68                              " is out of range. Valid address range is 0x%" PRIx64 " - 0x%" PRIx64 " (%" PRIu64 " bytes)",
  69                              *remote_address, *remote_address + length, ex_peer->super.base, ex_peer->super.base + size,
  70                              (uint64_t) size);
  71             return OMPI_ERR_RMA_RANGE;
  72         }
  73 
  74         *remote_handle = ex_peer->super.base_handle;
  75     }
  76 
  77     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "remote address: 0x%" PRIx64 ", handle: %p", *remote_address, (void *) *remote_handle);
  78 
  79     return OMPI_SUCCESS;
  80 }
  81 
  82 /* prototypes for implementations of MPI RMA window functions. these will be called from the
  83  * mpi interface (ompi/mpi/c) */
  84 int ompi_osc_rdma_put (const void *origin_addr, int origin_count, ompi_datatype_t *origin_dt,
  85                        int target, ptrdiff_t target_disp, int target_count,
  86                        ompi_datatype_t *target_dt, ompi_win_t *win);
  87 
  88 int ompi_osc_rdma_get (void *origin_addr, int origin_count, ompi_datatype_t *origin_dt,
  89                        int target, ptrdiff_t target_disp, int target_count,
  90                        ompi_datatype_t *target_dt, ompi_win_t *win);
  91 
  92 int ompi_osc_rdma_rput (const void *origin_addr, int origin_count, ompi_datatype_t *origin_dt,
  93                         int target, ptrdiff_t target_disp, int target_count,
  94                         ompi_datatype_t *target_dt, ompi_win_t *win,
  95                         ompi_request_t **request);
  96 
  97 int ompi_osc_rdma_rget (void *origin_addr, int origin_count, ompi_datatype_t *origin_dt,
  98                         int target, ptrdiff_t target_disp, int target_count,
  99                         ompi_datatype_t *target_dt, ompi_win_t *win,
 100                         ompi_request_t **request);
 101 
 102 /**
 103  * @brief read data from a remote memory region (blocking)
 104  *
 105  * @param[in] module          osc rdma module
 106  * @param[in] endpoint        btl endpoint
 107  * @param[in] source_address  remote address to read from
 108  * @param[in] source_handle   btl registration handle for remote region (must be valid for the entire region)
 109  * @param[in] data            local buffer to store to
 110  * @param[in] len             number of bytes to read
 111  *
 112  * This is an internal function for reading data from a remote peer. It is used to read peer and state
 113  * data that is stored on the remote peer. The peer object does not have to be fully initialized to
 114  * work. Only the btl endpoint is needed.
 115  */
 116 int ompi_osc_get_data_blocking (ompi_osc_rdma_module_t *module, struct mca_btl_base_endpoint_t *endpoint,
 117                                 uint64_t source_address, mca_btl_base_registration_handle_t *source_handle,
 118                                 void *data, size_t len);
 119 
 120 int ompi_osc_rdma_put_contig (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_peer_t *peer, uint64_t target_address,
 121                               mca_btl_base_registration_handle_t *target_handle, void *source_buffer, size_t size,
 122                               ompi_osc_rdma_request_t *request);
 123 
 124 #endif /* OMPI_OSC_RDMA_COMM_H */