root/opal/mca/btl/portals4/btl_portals4.h

INCLUDED FROM

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2007 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2010-2012 Sandia National Laboratories.  All rights reserved.
  14  * Copyright (c) 2014      Bull SAS.  All rights reserved.
  15  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  16  *                         reserved.
  17  * $COPYRIGHT$
  18  *
  19  * Additional copyrights may follow
  20  *
  21  * $HEADER$
  22  */
  23 
  24 #ifndef BTL_PORTALS_H_HAS_BEEN_INCLUDED
  25 #define BTL_PORTALS_H_HAS_BEEN_INCLUDED
  26 
  27 #include <portals4.h>
  28 #include <btl_portals4_frag.h>
  29 
  30 #include "opal/class/opal_free_list.h"
  31 #include "opal/class/opal_list.h"
  32 #include "opal/datatype/opal_convertor.h"
  33 #include "opal/mca/btl/btl.h"
  34 #include "opal/mca/btl/base/base.h"
  35 #include "opal/mca/btl/base/btl_base_error.h"
  36 
  37 BEGIN_C_DECLS
  38 
  39 /*
  40  * Portals BTL component.
  41  */
  42 struct mca_btl_portals4_component_t {
  43     /* base BTL component */
  44     mca_btl_base_component_2_0_0_t super;
  45 
  46     unsigned int num_btls;
  47     unsigned int max_btls; /* Maximum number of accepted Portals4 cards */
  48 
  49     struct mca_btl_portals4_module_t** btls; /* array of available BTL modules */
  50 
  51     /* add_procs() can get called multiple times.  this prevents multiple calls to portals4_init_interface(). */
  52     int need_init;
  53 
  54     /* Use the logical to physical table to accelerate portals4 adressing: 1 (true) : 0 (false) */
  55     int use_logical;
  56 
  57     /* initial size of free lists */
  58     int portals_free_list_init_num;
  59     /* max size of free lists */
  60     int portals_free_list_max_num;
  61     /* numer of elements to grow free lists */
  62     int portals_free_list_inc_num;
  63 
  64     /* number of eager fragments */
  65     int portals_free_list_eager_max_num;
  66 
  67     /* do I need a portals ACK? */
  68     int portals_need_ack;
  69 
  70     /** Length of the receive event queues */
  71     int recv_queue_size;
  72 
  73     /* number outstanding sends and local rdma */
  74     int32_t portals_max_outstanding_ops;
  75 
  76     /* incoming send message receive memory descriptors */
  77     int portals_recv_mds_num;
  78     int portals_recv_mds_size;
  79 
  80     /** Event queue handles table used in PtlEQPoll */
  81     ptl_handle_eq_t *eqs_h;
  82 
  83     /** Upper limit for message sizes */
  84     unsigned long portals_max_msg_size;
  85 };
  86 
  87 typedef struct mca_btl_portals4_component_t mca_btl_portals4_component_t;
  88 
  89 struct mca_btl_portals4_module_t {
  90     /* base BTL module interface */
  91     mca_btl_base_module_t super;
  92 
  93     /* number of processes we're actively connected to.  Needed to
  94        know when to do activation / shutdown */
  95     opal_atomic_int32_t portals_num_procs;
  96 
  97     /* number of the interface (btl) */
  98     uint32_t interface_num;
  99 
 100     /* fragment free lists */
 101     opal_free_list_t portals_frag_eager;
 102     opal_free_list_t portals_frag_max;
 103     opal_free_list_t portals_frag_user;
 104 
 105     opal_list_t portals_recv_blocks;
 106 
 107     /** Length of the receive event queues */
 108     int recv_queue_size;
 109 
 110     /** Event queue handle */
 111     ptl_handle_eq_t recv_eq_h;
 112 
 113     /* number outstanding sends and local rdma */
 114     opal_atomic_int32_t portals_outstanding_ops;
 115     int32_t portals_max_outstanding_ops;
 116 
 117     /* key to use for next rdma operation */
 118     opal_atomic_int64_t portals_rdma_key;
 119 
 120     /* our portals network interface */
 121     ptl_handle_ni_t portals_ni_h;
 122 
 123     /** portals index */
 124     ptl_pt_index_t recv_idx;
 125 
 126     /** MD handle for sending ACKS */
 127     ptl_handle_md_t zero_md_h;
 128 
 129     /** Send MD handle */
 130     ptl_handle_md_t send_md_h;
 131 
 132     /** long message receive overflow ME.  Persistent ME, first in
 133         overflow list on the recv_idx portal table. */
 134     ptl_handle_me_t long_overflow_me_h;
 135 };
 136 
 137 typedef struct mca_btl_portals4_module_t mca_btl_portals4_module_t;
 138 
 139 /* match/ignore bit manipulation
 140  *
 141  * 0123 4567 01234567 01234567 01234567 01234567 01234567 01234567 01234567
 142  *     |             |                 |
 143  * ^   | context id  |      source     |            message tag
 144  * |   |             |                 |
 145  * +---- protocol
 146  */
 147 
 148 #define BTL_PORTALS4_PROTOCOL_MASK  0xF000000000000000ULL
 149 #define BTL_PORTALS4_CONTEXT_MASK   0x0FFF000000000000ULL
 150 #define BTL_PORTALS4_SOURCE_MASK    0x0000FFFF00000000ULL
 151 #define BTL_PORTALS4_TAG_MASK       0x00000000FFFFFFFFULL
 152 
 153 #define BTL_PORTALS4_PROTOCOL_IGNR  BTL_PORTALS4_PROTOCOL_MASK
 154 #define BTL_PORTALS4_CONTEXT_IGNR   BTL_PORTALS4_CONTEXT_MASK
 155 #define BTL_PORTALS4_SOURCE_IGNR    BTL_PORTALS4_SOURCE_MASK
 156 #define BTL_PORTALS4_TAG_IGNR       0x000000007FFFFFFFULL
 157 
 158 #define BTL_PORTALS4_SHORT_MSG      0x1000000000000000ULL
 159 #define BTL_PORTALS4_LONG_MSG       0x2000000000000000ULL
 160 
 161 /* send posting */
 162 #define BTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, source, tag, type) \
 163     {                                                                   \
 164         match_bits = contextid;                                         \
 165         match_bits = (match_bits << 16);                                \
 166         match_bits |= source;                                           \
 167         match_bits = (match_bits << 32);                                \
 168         match_bits |= (BTL_PORTALS4_TAG_MASK & tag) | type;             \
 169     }
 170 
 171 #define BTL_PORTALS4_SET_HDR_DATA(hdr_data, opcount, length, sync)   \
 172     {                                                                \
 173         hdr_data = (sync) ? 1 : 0;                                   \
 174         hdr_data = (hdr_data << 15);                                 \
 175         hdr_data |= opcount & 0x7FFFULL;                             \
 176         hdr_data = (hdr_data << 48);                                 \
 177         hdr_data |= (length & 0xFFFFFFFFFFFFULL);                    \
 178     }
 179 
 180 #define REQ_BTL_TABLE_ID        2
 181 
 182 int mca_btl_portals4_component_progress(void);
 183 void mca_btl_portals4_free_module(mca_btl_portals4_module_t *portals4_btl);
 184 
 185 /* BTL interface functions */
 186 int mca_btl_portals4_finalize(struct mca_btl_base_module_t* btl_base);
 187 
 188 
 189 int mca_btl_portals4_add_procs(struct mca_btl_base_module_t* btl_base,
 190                               size_t nprocs,
 191                               struct opal_proc_t **procs,
 192                               struct mca_btl_base_endpoint_t** peers,
 193                               opal_bitmap_t* reachable);
 194 
 195 int mca_btl_portals4_del_procs(struct mca_btl_base_module_t* btl_base,
 196                               size_t nprocs,
 197                               struct opal_proc_t **procs,
 198                               struct mca_btl_base_endpoint_t** peers);
 199 
 200 mca_btl_base_descriptor_t*
 201 mca_btl_portals4_alloc(struct mca_btl_base_module_t* btl_base,
 202                       struct mca_btl_base_endpoint_t* endpoint,
 203                       uint8_t order,
 204                       size_t size,
 205                       uint32_t flags);
 206 
 207 int mca_btl_portals4_free(struct mca_btl_base_module_t* btl_base,
 208                          mca_btl_base_descriptor_t* des);
 209 
 210 mca_btl_base_descriptor_t*
 211 mca_btl_portals4_prepare_src(struct mca_btl_base_module_t* btl_base,
 212                             struct mca_btl_base_endpoint_t* peer,
 213                             struct opal_convertor_t* convertor,
 214                             uint8_t order,
 215                             size_t reserve,
 216                             size_t* size,
 217                             uint32_t flags);
 218 
 219 int mca_btl_portals4_send(struct mca_btl_base_module_t* btl_base,
 220                          struct mca_btl_base_endpoint_t* btl_peer,
 221                          struct mca_btl_base_descriptor_t* descriptor,
 222                          mca_btl_base_tag_t tag);
 223 
 224 
 225 int mca_btl_portals4_sendi(struct mca_btl_base_module_t* btl_base,
 226                           struct mca_btl_base_endpoint_t* endpoint,
 227                           struct opal_convertor_t* convertor,
 228                           void* header,
 229                           size_t header_size,
 230                           size_t payload_size,
 231                           uint8_t order,
 232                           uint32_t flags,
 233                           mca_btl_base_tag_t tag,
 234                           mca_btl_base_descriptor_t** des);
 235 
 236 int mca_btl_portals4_put(struct mca_btl_base_module_t* btl_base,
 237                         struct mca_btl_base_endpoint_t* btl_peer,
 238                         struct mca_btl_base_descriptor_t* decriptor);
 239 
 240 
 241 int mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
 242                         struct mca_btl_base_endpoint_t* btl_peer,
 243                         void *local_address,
 244                         uint64_t remote_address,
 245                         struct mca_btl_base_registration_handle_t *local_handle,
 246                         struct mca_btl_base_registration_handle_t *remote_handle,
 247                         size_t size,
 248                         int flags,
 249                         int order,
 250                         mca_btl_base_rdma_completion_fn_t cbfunc,
 251                         void *cbcontext,
 252                         void *cbdata);
 253 
 254 int mca_btl_portals4_get_error(int ptl_error);
 255 
 256 struct mca_btl_base_registration_handle_t {
 257     /** Portals4 match bits */
 258     ptl_match_bits_t key;
 259     /** Portals4 me_h */
 260     ptl_handle_me_t me_h;
 261     /** Remote offset */
 262     ptl_size_t remote_offset;
 263 };
 264 
 265 /*
 266  * global structures
 267  */
 268 OPAL_MODULE_DECLSPEC extern mca_btl_portals4_component_t mca_btl_portals4_component;
 269 extern mca_btl_portals4_module_t mca_btl_portals4_module;
 270 
 271 /**
 272  * An abstraction that represents a connection to a endpoint process.
 273  * An instance of mca_btl_base_endpoint_t is associated w/ each process
 274  * and BTL pair at startup. However, connections to the endpoint
 275  * are established dynamically on an as-needed basis:
 276  */
 277 struct mca_btl_base_endpoint_t {
 278     ptl_process_t ptl_proc;
 279 };
 280 typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
 281 
 282 END_C_DECLS
 283 
 284 #endif  /* BTL_PORTALS_H_HAS_BEEN_INCLUDED */