root/opal/mca/btl/ugni/btl_ugni_endpoint.h

INCLUDED FROM

DEFINITIONS

1. mca_btl_ugni_check_endpoint_state
2. mca_btl_ugni_ep_btl

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
   4  *                         reserved.
   5  * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
   6  * $COPYRIGHT$
   7  *
   8  * Additional copyrights may follow
   9  *
  10  * $HEADER$
  11  */
  12 
  13 #ifndef MCA_BTL_UGNI_ENDPOINT_H
  14 #define MCA_BTL_UGNI_ENDPOINT_H
  15 
  16 #include "btl_ugni.h"
  17 
  18 enum mca_btl_ugni_endpoint_state_t {
  19     MCA_BTL_UGNI_EP_STATE_INIT = 0,
  20     MCA_BTL_UGNI_EP_STATE_CONNECTING,
  21     MCA_BTL_UGNI_EP_STATE_CONNECTED,
  22 };
  23 typedef enum mca_btl_ugni_endpoint_state_t mca_btl_ugni_endpoint_state_t;
  24 
  25 struct mca_btl_ugni_smsg_mbox_t;
  26 
  27 struct mca_btl_ugni_endpoint_handle_t {
  28     mca_btl_ugni_device_t *device;
  29     gni_ep_handle_t gni_handle;
  30 };
  31 
  32 typedef struct mca_btl_ugni_endpoint_handle_t mca_btl_ugni_endpoint_handle_t;
  33 
  34 typedef struct mca_btl_base_endpoint_t {
  35     opal_list_item_t super;
  36 
  37     opal_proc_t *peer_proc;
  38 
  39     /** may need to lock recursively as the modex lookup could call opal_progress
  40      * and hence our progress function. if this changes modify this mutex to not
  41      * be recursive. also need to update the constructor function. */
  42     opal_recursive_mutex_t lock;
  43     mca_btl_ugni_endpoint_state_t state;
  44 
  45     /** Remote NIC address */
  46     uint32_t ep_rem_addr;
  47 
  48     /** Remote CDM identifier (base) */
  49     uint32_t ep_rem_id;
  50 
  51     /** endpoint to use for SMSG messages */
  52     mca_btl_ugni_endpoint_handle_t smsg_ep_handle;
  53 
  54     /** temporary space to store the remote SMSG attributes */
  55     mca_btl_ugni_endpoint_attr_t *remote_attr;
  56 
  57     /** SMSG mailbox assigned to this endpoint */
  58     struct mca_btl_ugni_smsg_mbox_t *mailbox;
  59 
  60     /** Remote IRQ handle (for async completion) */
  61     gni_mem_handle_t rmt_irq_mem_hndl;
  62 
  63     /** frags waiting for SMSG credits */
  64     opal_list_t frag_wait_list;
  65 
  66     /** endpoint is currently wait-listed for SMSG progress */
  67     bool wait_listed;
  68 
  69     /** protect against race on connection */
  70     bool dg_posted;
  71 
  72     /** protect against re-entry to SMSG */
  73     opal_atomic_int32_t smsg_progressing;
  74 
  75     int index;
  76 } mca_btl_base_endpoint_t;
  77 
  78 typedef mca_btl_base_endpoint_t  mca_btl_ugni_endpoint_t;
  79 OBJ_CLASS_DECLARATION(mca_btl_ugni_endpoint_t);
  80 
  81 int mca_btl_ugni_ep_connect_progress (mca_btl_ugni_endpoint_t *ep);
  82 int mca_btl_ugni_ep_disconnect (mca_btl_ugni_endpoint_t *ep, bool send_disconnect);
  83 int mca_btl_ugni_wildcard_ep_post (mca_btl_ugni_module_t *ugni_module);
  84 void mca_btl_ugni_release_ep (mca_btl_ugni_endpoint_t *ep);
  85 int mca_btl_ugni_init_ep (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_endpoint_t **ep,
  86                           mca_btl_ugni_module_t *btl, opal_proc_t *peer_proc);
  87 
  88 static inline int mca_btl_ugni_check_endpoint_state (mca_btl_ugni_endpoint_t *ep) {
  89     int rc;
  90 
  91     if (OPAL_LIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state)) {
  92         return OPAL_SUCCESS;
  93     }
  94 
  95     opal_mutex_lock (&ep->lock);
  96 
  97     switch (ep->state) {
  98     case MCA_BTL_UGNI_EP_STATE_INIT:
  99         rc = mca_btl_ugni_ep_connect_progress (ep);
 100         if (OPAL_SUCCESS != rc) {
 101             break;
 102         }
 103     case MCA_BTL_UGNI_EP_STATE_CONNECTING:
 104         rc = OPAL_ERR_RESOURCE_BUSY;
 105         break;
 106     default:
 107         rc = OPAL_SUCCESS;
 108     }
 109 
 110     opal_mutex_unlock (&ep->lock);
 111 
 112     return rc;
 113 }
 114 
 115 /**
 116  * Accessor function for endpoint btl
 117  *
 118  * @param[in] ep   endpoint to query
 119  *
 120  * This helper function exists to make it easy to switch between using a single
 121  * and multiple ugni modules. Currently there is only one so we just use the
 122  * pointer in the component structure. This saves 4-8 bytes in the endpoint
 123  * structure.
 124  */
 125 static inline mca_btl_ugni_module_t *mca_btl_ugni_ep_btl (mca_btl_ugni_endpoint_t *ep)
 126 {
 127     /* there is only one ugni module at this time. if that changes add a btl pointer back
 128      * to the endpoint structure. */
 129     return mca_btl_ugni_component.modules;
 130 }
 131 
 132 /**
 133  * Initialize and bind an endpoint handle
 134  *
 135  * @param[in]  ep                 BTL endpoint
 136  * @param[in]  cq                 completion queue
 137  * @param[in]  device             device to bind with
 138  * @param[in]  ep_handle          endpoint handle to initialize and bind
 139  */
 140 int mca_btl_ugni_ep_handle_init (mca_btl_ugni_endpoint_t *ep, gni_cq_handle_t cq,
 141                                  mca_btl_ugni_device_t *device, mca_btl_ugni_endpoint_handle_t *ep_handle);
 142 
 143 int mca_btl_ugni_ep_handle_cleanup (mca_btl_ugni_endpoint_handle_t *ep_handle);
 144 
 145 #endif /* MCA_BTL_UGNI_ENDPOINT_H */