root/opal/mca/btl/ugni/btl_ugni_frag.h

INCLUDED FROM

DEFINITIONS

1. mca_btl_ugni_alloc_rdma_desc
2. mca_btl_ugni_return_rdma_desc
3. mca_btl_ugni_post_desc_complete
4. mca_btl_ugni_frag_alloc
5. mca_btl_ugni_frag_return
6. mca_btl_ugni_frag_del_ref
7. mca_btl_ugni_frag_complete
8. mca_btl_ugni_frag_check_complete
9. mca_btl_ugni_frag_alloc_smsg
10. mca_btl_ugni_frag_alloc_rdma
11. mca_btl_ugni_frag_alloc_rdma_int
12. mca_btl_ugni_frag_alloc_eager_send
13. mca_btl_ugni_frag_alloc_eager_recv

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
   4  *                         reserved.
   5  * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
   6  * Copyright (c) 2013      The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * $COPYRIGHT$
  10  *
  11  * Additional copyrights may follow
  12  *
  13  * $HEADER$
  14  */
  15 
  16 #if !defined(MCA_BTL_UGNI_FRAG_H)
  17 #define MCA_BTL_UGNI_FRAG_H
  18 
  19 #include "btl_ugni.h"
  20 #include "btl_ugni_endpoint.h"
  21 
  22 #include <string.h>
  23 
  24 typedef struct mca_btl_ugni_send_frag_hdr_t {
  25     uint32_t lag;
  26 } mca_btl_ugni_send_frag_hdr_t;
  27 
  28 typedef struct mca_btl_ugni_send_ex_frag_hdr_t {
  29     mca_btl_ugni_send_frag_hdr_t send;
  30     uint8_t                      pml_header[128];
  31 } mca_btl_ugni_send_ex_frag_hdr_t;
  32 
  33 typedef struct mca_btl_ugni_rdma_frag_hdr_t {
  34     void *ctx;
  35 } mca_btl_ugni_rdma_frag_hdr_t;
  36 
  37 typedef struct mca_btl_ugni_eager_frag_hdr_t {
  38     mca_btl_ugni_send_frag_hdr_t send;
  39     uint32_t size;
  40     uint64_t address;
  41     mca_btl_base_registration_handle_t memory_handle;
  42     void *ctx;
  43 } mca_btl_ugni_eager_frag_hdr_t;
  44 
  45 typedef struct mca_btl_ugni_eager_ex_frag_hdr_t {
  46     mca_btl_ugni_eager_frag_hdr_t eager;
  47     uint8_t                       pml_header[128];
  48 } mca_btl_ugni_eager_ex_frag_hdr_t;
  49 
  50 typedef union mca_btl_ugni_frag_hdr_t {
  51     mca_btl_ugni_send_frag_hdr_t     send;
  52     mca_btl_ugni_send_ex_frag_hdr_t  send_ex;
  53     mca_btl_ugni_rdma_frag_hdr_t     rdma;
  54     mca_btl_ugni_eager_frag_hdr_t    eager;
  55     mca_btl_ugni_eager_ex_frag_hdr_t eager_ex;
  56 } mca_btl_ugni_frag_hdr_t;
  57 
  58 enum {
  59     MCA_BTL_UGNI_FRAG_BUFFERED      = 1,  /* frag data is buffered */
  60     MCA_BTL_UGNI_FRAG_COMPLETE      = 2,  /* smsg complete for frag */
  61     MCA_BTL_UGNI_FRAG_EAGER         = 4,  /* eager get frag */
  62     MCA_BTL_UGNI_FRAG_IGNORE        = 8,  /* ignore local smsg completion */
  63     MCA_BTL_UGNI_FRAG_SMSG_COMPLETE = 16, /* SMSG has completed for this message */
  64     MCA_BTL_UGNI_FRAG_RESPONSE      = 32,
  65 };
  66 
  67 struct mca_btl_ugni_base_frag_t;
  68 
  69 typedef struct mca_btl_ugni_base_frag_t {
  70     mca_btl_base_descriptor_t    base;
  71     opal_atomic_int32_t          ref_cnt;
  72     uint32_t                     msg_id;
  73     uint16_t                     hdr_size;
  74     uint16_t                     flags;
  75     mca_btl_ugni_frag_hdr_t      hdr;
  76     mca_btl_base_segment_t       segments[2];
  77     gni_post_descriptor_t        post_desc;
  78     mca_btl_base_endpoint_t     *endpoint;
  79     mca_btl_ugni_reg_t          *registration;
  80     opal_free_list_t            *my_list;
  81     mca_btl_base_registration_handle_t memory_handle;
  82 } mca_btl_ugni_base_frag_t;
  83 
  84 typedef struct mca_btl_ugni_base_frag_t mca_btl_ugni_smsg_frag_t;
  85 typedef struct mca_btl_ugni_base_frag_t mca_btl_ugni_rdma_frag_t;
  86 typedef struct mca_btl_ugni_base_frag_t mca_btl_ugni_eager_frag_t;
  87 
  88 typedef struct mca_btl_ugni_post_descriptor_t {
  89     /** endpoint currently associated with this desctiptor */
  90     mca_btl_base_endpoint_t *endpoint;
  91     /** local memory handle (for callback) */
  92     mca_btl_base_registration_handle_t *local_handle;
  93     /** currently associated completion queue */
  94     mca_btl_ugni_cq_t *cq;
  95     /** user callback function */
  96     mca_btl_base_rdma_completion_fn_t cbfunc;
  97     /** user callback data */
  98     void *cbdata;
  99     /** user callback context */
 100     void *ctx;
 101     /** opal status of this descriptor. filled in by
 102      * mca_btl_ugni_cq_get_completed_desc_device() */
 103     int rc;
 104     /** true if posted with the BTE. false if FMA. this is used as part
 105      * of the BTE throttling code. */
 106     bool use_bte;
 107     /** uGNI library post descriptor. this is last in this structure
 108      * to try to keep it hot in the cache after copying this descriptor
 109      * into the allocated descritor. (post follows almost immediately
 110      * after allocate. */
 111     gni_post_descriptor_t gni_desc;
 112 } mca_btl_ugni_post_descriptor_t;
 113 
 114 OBJ_CLASS_DECLARATION(mca_btl_ugni_post_descriptor_t);
 115 
 116 typedef struct mca_btl_ugni_rdma_desc_t {
 117     opal_free_list_item_t super;
 118     mca_btl_ugni_post_descriptor_t btl_ugni_desc;
 119     mca_btl_ugni_device_t *device;
 120     gni_ep_handle_t gni_handle;
 121     int tries;
 122 } mca_btl_ugni_rdma_desc_t;
 123 
 124 OBJ_CLASS_DECLARATION(mca_btl_ugni_rdma_desc_t);
 125 
 126 #define MCA_BTL_UGNI_GNI_DESC_TO_RDMA_DESC(desc) \
 127     ((mca_btl_ugni_rdma_desc_t *) ((uintptr_t)(desc) - offsetof (mca_btl_ugni_rdma_desc_t, btl_ugni_desc) - offsetof (mca_btl_ugni_post_descriptor_t, gni_desc)))
 128 
 129 /**
 130  * Initialize a RDMA descriptor
 131  *
 132  * @param[in]    item    free list item (must be of class mca_btl_ugni_rdma_desc_t)
 133  * @param[in]    ctx     pointer to ugni device context
 134  *
 135  * This function initializes a mca_btl_ugni_rdma_desc_t for use. It allocates
 136  * resources from the ugni library. This must be called before a RDMA
 137  * descriptor can be used. Usually this is passed as an argument to
 138  * opal_free_list_init().
 139  */
 140 int mca_btl_ugni_rdma_desc_init (opal_free_list_item_t *item, void *ctx);
 141 
 142 /**
 143  * @brief get an endpoint handle from a device's free list
 144  *
 145  * @param[in] ep      btl endpoint
 146  * @param[in] device  btl device to use
 147  * @param[in] use_bte whether this descriptor will be used with the BTE
 148  *
 149  * This function MUST be called with the device lock held. This was done over using
 150  * the atomic free list to avoid unnecessary atomics in the critical path.
 151  */
 152 static inline mca_btl_ugni_rdma_desc_t *
 153 mca_btl_ugni_alloc_rdma_desc (mca_btl_ugni_device_t *device, mca_btl_ugni_post_descriptor_t *ugni_desc, const bool use_bte)
 154 {
 155     mca_btl_ugni_rdma_desc_t *desc = (mca_btl_ugni_rdma_desc_t *) opal_free_list_get_st (&device->rdma_descs);
 156     mca_btl_ugni_endpoint_t *ep = ugni_desc->endpoint;
 157     gni_return_t grc;
 158 
 159     if (OPAL_LIKELY(NULL != desc)) {
 160         grc = GNI_EpBind (desc->gni_handle, ep->ep_rem_addr, ep->ep_rem_id | device->dev_index);
 161         if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc)) {
 162             opal_free_list_return_st (&device->rdma_descs, &desc->super);
 163             return NULL;
 164         }
 165 
 166         desc->device = device;
 167         desc->tries = 0;
 168         desc->btl_ugni_desc = *ugni_desc;
 169         desc->btl_ugni_desc.use_bte = use_bte;
 170     }
 171 
 172     return desc;
 173 }
 174 
 175 static inline void mca_btl_ugni_return_rdma_desc (mca_btl_ugni_rdma_desc_t *desc)
 176 {
 177     (void) GNI_EpUnbind (desc->gni_handle);
 178     opal_free_list_return_st (&desc->device->rdma_descs, &desc->super);
 179 }
 180 
 181 static inline void mca_btl_ugni_post_desc_complete (mca_btl_ugni_module_t *module, mca_btl_ugni_post_descriptor_t *desc, int rc)
 182 {
 183     BTL_VERBOSE(("RDMA/FMA/ATOMIC operation complete for post descriptor %p. rc = %d", (void *) desc, rc));
 184 
 185     if (NULL != desc->cbfunc) {
 186         /* call the user's callback function */
 187         desc->cbfunc (&module->super, desc->endpoint, (void *)(intptr_t) desc->gni_desc.local_addr,
 188                       desc->local_handle, desc->ctx, desc->cbdata, rc);
 189     }
 190 }
 191 
 192 OBJ_CLASS_DECLARATION(mca_btl_ugni_smsg_frag_t);
 193 OBJ_CLASS_DECLARATION(mca_btl_ugni_rdma_frag_t);
 194 OBJ_CLASS_DECLARATION(mca_btl_ugni_eager_frag_t);
 195 
 196 int mca_btl_ugni_frag_init (mca_btl_ugni_base_frag_t *frag, void *id);
 197 
 198 static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc (mca_btl_base_endpoint_t *ep,
 199                                                                  opal_free_list_t *list)
 200 {
 201     mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) opal_free_list_get (list);
 202     if (OPAL_LIKELY(NULL != frag)) {
 203         frag->endpoint = ep;
 204         frag->ref_cnt = 1;
 205     }
 206 
 207     return frag;
 208 }
 209 
 210 static inline int mca_btl_ugni_frag_return (mca_btl_ugni_base_frag_t *frag)
 211 {
 212     mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (frag->endpoint);
 213     if (frag->registration) {
 214         ugni_module->rcache->rcache_deregister (ugni_module->rcache,
 215                                                 (mca_rcache_base_registration_t *) frag->registration);
 216         frag->registration = NULL;
 217     }
 218 
 219     frag->flags = 0;
 220 
 221     opal_free_list_return (frag->my_list, (opal_free_list_item_t *) frag);
 222 
 223     return OPAL_SUCCESS;
 224 }
 225 
 226 static inline bool mca_btl_ugni_frag_del_ref (mca_btl_ugni_base_frag_t *frag, int rc) {
 227     mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (frag->endpoint);
 228     /* save the descriptor flags since the callback is allowed to free the frag */
 229     int des_flags = frag->base.des_flags;
 230     int32_t ref_cnt;
 231 
 232     opal_atomic_mb ();
 233 
 234     ref_cnt = OPAL_THREAD_ADD_FETCH32(&frag->ref_cnt, -1);
 235     if (ref_cnt) {
 236         assert (ref_cnt > 0);
 237         return false;
 238     }
 239 
 240     /* call callback if specified */
 241     if (des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) {
 242         frag->base.des_cbfunc(&ugni_module->super, frag->endpoint, &frag->base, rc);
 243     }
 244 
 245     if (des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP) {
 246         mca_btl_ugni_frag_return (frag);
 247     }
 248 
 249     return true;
 250 }
 251 
 252 static inline void mca_btl_ugni_frag_complete (mca_btl_ugni_base_frag_t *frag, int rc) {
 253     BTL_VERBOSE(("frag complete. flags = %d", frag->base.des_flags));
 254 
 255     frag->flags |= MCA_BTL_UGNI_FRAG_COMPLETE;
 256 
 257     mca_btl_ugni_frag_del_ref (frag, rc);
 258 }
 259 
 260 static inline bool mca_btl_ugni_frag_check_complete (mca_btl_ugni_base_frag_t *frag) {
 261     return !!(MCA_BTL_UGNI_FRAG_COMPLETE & frag->flags);
 262 }
 263 
 264 
 265 void mca_btl_ugni_wait_list_append (mca_btl_ugni_module_t *ugni_module, mca_btl_base_endpoint_t *endpoint,
 266                                     mca_btl_ugni_base_frag_t *frag);
 267 
 268 static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc_smsg (mca_btl_base_endpoint_t *ep)
 269 {
 270     mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
 271     return mca_btl_ugni_frag_alloc (ep, ugni_module->frags_lists + MCA_BTL_UGNI_LIST_SMSG);
 272 }
 273 
 274 static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc_rdma (mca_btl_base_endpoint_t *ep)
 275 {
 276     mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
 277     return mca_btl_ugni_frag_alloc (ep, ugni_module->frags_lists + MCA_BTL_UGNI_LIST_RDMA);
 278 }
 279 
 280 static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc_rdma_int (mca_btl_base_endpoint_t *ep)
 281 {
 282     mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
 283     return mca_btl_ugni_frag_alloc (ep, ugni_module->frags_lists + MCA_BTL_UGNI_LIST_RDMA_INT);
 284 }
 285 
 286 static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc_eager_send (mca_btl_base_endpoint_t *ep)
 287 {
 288     mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
 289     return mca_btl_ugni_frag_alloc (ep, ugni_module->frags_lists + MCA_BTL_UGNI_LIST_EAGER_SEND);
 290 }
 291 
 292 static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc_eager_recv (mca_btl_base_endpoint_t *ep)
 293 {
 294     mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
 295     return mca_btl_ugni_frag_alloc (ep, ugni_module->frags_lists + MCA_BTL_UGNI_LIST_EAGER_RECV);
 296 }
 297 
 298 #endif /* MCA_BTL_UGNI_FRAG_H */