root/opal/mca/btl/portals4/btl_portals4_rdma.c

DEFINITIONS

1. mca_btl_portals4_put
2. mca_btl_portals4_get

   1 /*
   2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   3  *                         University Research and Technology
   4  *                         Corporation.  All rights reserved.
   5  * Copyright (c) 2004-2005 The University of Tennessee and The University
   6  *                         of Tennessee Research Foundation.  All rights
   7  *                         reserved.
   8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   9  *                         University of Stuttgart.  All rights reserved.
  10  * Copyright (c) 2004-2005 The Regents of the University of California.
  11  *                         All rights reserved.
  12  * Copyright (c) 2008      UT-Battelle, LLC. All rights reserved.
  13  * Copyright (c) 2014      Bull SAS.  All rights reserved.
  14  * $COPYRIGHT$
  15  *
  16  * Additional copyrights may follow
  17  *
  18  * $HEADER$
  19  */
  20 
  21 #include "opal_config.h"
  22 #include "opal/constants.h"
  23 #include "btl_portals4.h"
  24 
  25 int
  26 mca_btl_portals4_put(struct mca_btl_base_module_t* btl_base,
  27                     struct mca_btl_base_endpoint_t* btl_peer,
  28                     struct mca_btl_base_descriptor_t* descriptor)
  29 {
  30     opal_output(opal_btl_base_framework.framework_output, "mca_btl_portals4_put not implemented\n");
  31 
  32     BTL_ERROR(("mca_btl_portals4_put not implemented\n"));
  33     return OPAL_SUCCESS;
  34 }
  35 
  36 
  37 int
  38 mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
  39                     struct mca_btl_base_endpoint_t* btl_peer,
  40                     void *local_address,
  41                     uint64_t remote_address,
  42                     struct mca_btl_base_registration_handle_t *local_handle,
  43                     struct mca_btl_base_registration_handle_t *remote_handle,
  44                     size_t size,
  45                     int flags,
  46                     int order,
  47                     mca_btl_base_rdma_completion_fn_t cbfunc,
  48                     void *cbcontext,
  49                     void *cbdata)
  50 {
  51     mca_btl_portals4_module_t *portals4_btl = (mca_btl_portals4_module_t *) btl_base;
  52     mca_btl_portals4_frag_t   *frag         = NULL;
  53     int ret;
  54 
  55     /* reserve space in the event queue for rdma operations immediately */
  56     while (OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, 1) >
  57            portals4_btl->portals_max_outstanding_ops) {
  58         OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, -1);
  59         OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Call to mca_btl_portals4_component_progress (1)\n"));
  60         mca_btl_portals4_component_progress();
  61     }
  62 
  63     OPAL_BTL_PORTALS4_FRAG_ALLOC_USER(portals4_btl, frag);
  64     if (NULL == frag){
  65         OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, -1);
  66         return OPAL_ERROR;
  67     }
  68     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
  69         "mca_btl_portals4_get: Incrementing portals_outstanding_ops=%d frag=%p",
  70         portals4_btl->portals_outstanding_ops, (void *)frag));
  71 
  72     frag->rdma_cb.func         = cbfunc;
  73     frag->rdma_cb.context      = cbcontext;
  74     frag->rdma_cb.data         = cbdata;
  75     frag->rdma_cb.local_handle = local_handle;
  76 
  77     frag->endpoint = btl_peer;
  78     frag->hdr.tag = MCA_BTL_TAG_MAX;
  79 
  80     frag->match_bits = remote_handle->key;
  81     frag->addr = local_address;
  82     frag->length = size;
  83     frag->peer_proc = btl_peer->ptl_proc;
  84 
  85     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlGet offset=%p length=%ld remote_offset=%p nid=%x pid=%x match_bits=%lx",
  86         local_address, size, (void*)local_handle->remote_offset, btl_peer->ptl_proc.phys.nid, btl_peer->ptl_proc.phys.pid, frag->match_bits));
  87 
  88     ret = PtlGet(portals4_btl->send_md_h,
  89                  (ptl_size_t) local_address,
  90                  size,
  91                  btl_peer->ptl_proc,
  92                  portals4_btl->recv_idx,
  93                  frag->match_bits, /* match bits */
  94                  local_handle->remote_offset,
  95                  frag);
  96     if (OPAL_UNLIKELY(PTL_OK != ret)) {
  97         opal_output_verbose(1, opal_btl_base_framework.framework_output,
  98                             "%s:%d: PtlGet failed: %d",
  99                             __FILE__, __LINE__, ret);
 100         return OPAL_ERROR;
 101     }
 102     local_handle->remote_offset += size;
 103 
 104     return OPAL_SUCCESS;
 105 }