ompi/mca/mtl/portals4/mtl

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
ompi_mtl_portals4_get_peer_group
ompi_mtl_portals4_get_proc
ompi_mtl_portals4_get_peer
   1 /*
   2  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
   3  *                         University Research and Technology
   4  *                         Corporation.  All rights reserved.
   5  * Copyright (c) 2004-2007 The University of Tennessee and The University
   6  *                         of Tennessee Research Foundation.  All rights
   7  *                         reserved.
   8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   9  *                         University of Stuttgart.  All rights reserved.
  10  * Copyright (c) 2004-2005 The Regents of the University of California.
  11  *                         All rights reserved.
  12  * Copyright (c) 2010-2012 Sandia National Laboratories.  All rights reserved.
  13  * $COPYRIGHT$
  14  *
  15  * Additional copyrights may follow
  16  *
  17  * $HEADER$
  18  */
  19 
  20 #ifndef MTL_PORTALS_H_HAS_BEEN_INCLUDED
  21 #define MTL_PORTALS_H_HAS_BEEN_INCLUDED
  22 
  23 #include <portals4.h>
  24 
  25 #include "opal/include/opal_config.h"
  26 #include "opal/class/opal_free_list.h"
  27 #include "opal/class/opal_list.h"
  28 #include "opal/datatype/opal_convertor.h"
  29 #include "ompi/proc/proc.h"
  30 #include "ompi/mca/mtl/mtl.h"
  31 #include "ompi/mca/mtl/base/base.h"
  32 
  33 #include "ompi/communicator/communicator.h"
  34 
  35 #include "mtl_portals4_flowctl.h"
  36 
  37 BEGIN_C_DECLS
  38 
  39 struct mca_mtl_portals4_send_request_t;
  40 
  41 struct mca_mtl_portals4_module_t {
  42     mca_mtl_base_module_t base;
  43 
  44     /* add_procs() can get called multiple times.  this prevents multiple calls to portals4_init_interface(). */
  45     int32_t need_init;
  46 
  47     /* Use the logical to physical table to accelerate portals4 adressing: 1 (true) : 0 (false) */
  48     int32_t use_logical;
  49 
  50     /* Process_id */
  51     ptl_process_t ptl_process_id;
  52 
  53     /* Use flow control: 1 (true) : 0 (false) */
  54     int32_t use_flowctl;
  55 
  56     /** Short limit; Size limit for short messages */
  57     uint64_t short_limit;
  58     /** Eager limit; messages greater than this use a rendezvous protocol */
  59     uint64_t eager_limit;
  60     /** Size of short message blocks */
  61     uint64_t recv_short_size;
  62     /** Number of short message blocks which should be created during startup */
  63     uint32_t recv_short_num;
  64     /** Length of the send event queues */
  65     uint32_t send_queue_size;
  66     /** Length of the receive event queues */
  67     uint32_t recv_queue_size;
  68     /** Protocol for long message transfer */
  69     enum { eager, rndv } protocol;
  70 
  71     /* free list of message for matched probe */
  72     opal_free_list_t fl_message;
  73 
  74     /* free list of rendezvous get fragments */
  75     opal_free_list_t fl_rndv_get_frag;
  76     int get_retransmit_timeout;
  77 
  78     /** Network interface handle for matched interface */
  79     ptl_handle_ni_t ni_h;
  80     /** Limit given by portals after NIInit */
  81     uint64_t max_msg_size_mtl;
  82     /** Uid for current user */
  83     ptl_uid_t uid;
  84 
  85     /** portals index for message matching */
  86     ptl_pt_index_t recv_idx;
  87     /** portals index for long message rendezvous */
  88     ptl_pt_index_t read_idx;
  89     /** portals index for flow control recovery */
  90     ptl_pt_index_t flowctl_idx;
  91 
  92     /** Event queue handles.  See send_eq_h and recv_eq_h defines for
  93         usage.  Array for PtlEQPoll */
  94     ptl_handle_eq_t eqs_h[2];
  95 
  96     /** MD handle for sending ACKS */
  97     ptl_handle_md_t zero_md_h;
  98 
  99     /** Send MD handle */
 100     ptl_handle_md_t send_md_h;
 101 
 102     /** long message receive overflow ME.  Persistent ME, first in
 103         overflow list on the recv_idx portal table. */
 104     ptl_handle_me_t long_overflow_me_h;
 105 
 106     /** List of short receive blocks. */
 107     opal_list_t recv_short_blocks;
 108 
 109     /** Number of active short receive blocks. Active means that the ME
 110         was posted to the overflow list, the LINK event has been received but the UNLINK or the FREE event has not
 111         yet been received. */
 112     uint32_t active_recv_short_blocks;
 113 
 114     /** Mutex to protect opal_list */
 115     opal_mutex_t short_block_mutex;
 116 
 117     /** number of send-side operations started */
 118     opal_atomic_uint64_t opcount;
 119 
 120 #if OPAL_ENABLE_DEBUG
 121     /** number of receive-side operations started.  Used only for
 122         debugging */
 123     opal_atomic_uint64_t recv_opcount;
 124 #endif
 125 
 126 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
 127     ompi_mtl_portals4_flowctl_t flowctl;
 128 #endif
 129 };
 130 typedef struct mca_mtl_portals4_module_t mca_mtl_portals4_module_t;
 131 
 132 #define send_eq_h eqs_h[0]
 133 #define recv_eq_h eqs_h[1]
 134 
 135 extern mca_mtl_portals4_module_t ompi_mtl_portals4;
 136 
 137 #define REQ_RECV_TABLE_ID    12
 138 #define REQ_READ_TABLE_ID    13
 139 #define REQ_FLOWCTL_TABLE_ID 14
 140 
 141 #define MTL_PORTALS4_FLOWCTL_TRIGGER 0x01
 142 #define MTL_PORTALS4_FLOWCTL_ALERT   0x02
 143 #define MTL_PORTALS4_FLOWCTL_FANIN   0x03
 144 #define MTL_PORTALS4_FLOWCTL_FANOUT  0x04
 145 
 146 /* match/ignore bit manipulation
 147  *
 148  * 0123 4567 01234567 01234567 01234567 01234567 01234567 01234567 01234567
 149  *     |             |                          |
 150  * ^   | context id  |          source          |        message tag
 151  * |   |             |                          |
 152  * +---- protocol
 153  */
 154 
 155 #define MTL_PORTALS4_MAX_TAG       ((1UL << 24) -1)
 156 
 157 #define MTL_PORTALS4_PROTOCOL_MASK 0xF000000000000000ULL
 158 #define MTL_PORTALS4_CONTEXT_MASK  0x0FFF000000000000ULL
 159 #define MTL_PORTALS4_SOURCE_MASK   0x0000FFFFFF000000ULL
 160 #define MTL_PORTALS4_TAG_MASK      0x0000000000FFFFFFULL
 161 
 162 #define MTL_PORTALS4_PROTOCOL_IGNR MTL_PORTALS4_PROTOCOL_MASK
 163 #define MTL_PORTALS4_CONTEXT_IGNR  MTL_PORTALS4_CONTEXT_MASK
 164 #define MTL_PORTALS4_SOURCE_IGNR   MTL_PORTALS4_SOURCE_MASK
 165 #define MTL_PORTALS4_TAG_IGNR      0x00000000007FFFFFULL
 166 
 167 #define MTL_PORTALS4_SHORT_MSG      0x1000000000000000ULL
 168 #define MTL_PORTALS4_LONG_MSG       0x2000000000000000ULL
 169 
 170 /* send posting */
 171 #define MTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, source, tag, type) \
 172     {                                                                   \
 173         match_bits = contextid;                                         \
 174         match_bits = (match_bits << 24);                                \
 175         match_bits |= source;                                           \
 176         match_bits = (match_bits << 24);                                \
 177         match_bits |= (MTL_PORTALS4_TAG_MASK & tag) | type;             \
 178     }
 179 
 180 /* receive posting */
 181 #define MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, contextid, source, tag) \
 182     {                                                                   \
 183         match_bits = 0;                                                 \
 184         ignore_bits = MTL_PORTALS4_PROTOCOL_IGNR;                       \
 185                                                                         \
 186         match_bits = contextid;                                         \
 187         match_bits = (match_bits << 24);                                \
 188                                                                         \
 189         if (MPI_ANY_SOURCE == source) {                                 \
 190             match_bits = (match_bits << 24);                            \
 191             ignore_bits |= MTL_PORTALS4_SOURCE_IGNR;                    \
 192         } else {                                                        \
 193             match_bits |= source;                                       \
 194             match_bits = (match_bits << 24);                            \
 195         }                                                               \
 196                                                                         \
 197         if (MPI_ANY_TAG == tag) {                                       \
 198             ignore_bits |= MTL_PORTALS4_TAG_IGNR;                       \
 199         } else {                                                        \
 200             match_bits |= (MTL_PORTALS4_TAG_MASK & tag);                \
 201         }                                                               \
 202     }
 203 
 204 #define MTL_PORTALS4_IS_SHORT_MSG(match_bits)           \
 205     (0 != (MTL_PORTALS4_SHORT_MSG & match_bits))
 206 #define MTL_PORTALS4_IS_LONG_MSG(match_bits)            \
 207     (0 != (MTL_PORTALS4_LONG_MSG & match_bits))
 208 #define MTL_PORTALS4_IS_READY_MSG(match_bits)           \
 209     (0 != (MTL_PORTALS4_READY_MSG & match_bits))
 210 
 211 #define MTL_PORTALS4_GET_TAG(match_bits)                \
 212     ((int)(match_bits & MTL_PORTALS4_TAG_MASK))
 213 #define MTL_PORTALS4_GET_SOURCE(match_bits)             \
 214     ((int)((match_bits & MTL_PORTALS4_SOURCE_MASK) >> 24))
 215 
 216 
 217 #define MTL_PORTALS4_SYNC_MSG       0x8000000000000000ULL
 218 
 219 #define MTL_PORTALS4_SET_HDR_DATA(hdr_data, opcount, length, sync)   \
 220     {                                                                \
 221         hdr_data = (sync) ? 1 : 0;                                   \
 222         hdr_data = (hdr_data << 15);                                 \
 223         hdr_data |= opcount & 0x7FFFULL;                             \
 224         hdr_data = (hdr_data << 48);                                 \
 225         hdr_data |= (length & 0xFFFFFFFFFFFFULL);                    \
 226     }
 227 
 228 #define MTL_PORTALS4_GET_LENGTH(hdr_data) ((size_t)(hdr_data & 0xFFFFFFFFFFFFULL))
 229 #define MTL_PORTALS4_IS_SYNC_MSG(hdr_data)            \
 230     (0 != (MTL_PORTALS4_SYNC_MSG & hdr_data))
 231 
 232 /* mtl-portals4 helpers */
 233 OMPI_DECLSPEC ompi_proc_t *
 234 ompi_mtl_portals4_get_proc_group(struct ompi_group_t *group, int rank);
 235 
 236 static inline ptl_process_t
 237 ompi_mtl_portals4_get_peer_group(struct ompi_group_t *group, int rank)
 238 {
 239     return *((ptl_process_t*)(ompi_mtl_portals4_get_proc_group(group, rank)->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]));
 240 }
 241 
 242 static inline ompi_proc_t *
 243 ompi_mtl_portals4_get_proc(struct ompi_communicator_t *comm, int rank)
 244 {
 245     return ompi_mtl_portals4_get_proc_group(comm->c_remote_group, rank);
 246 }
 247 
 248 static inline ptl_process_t
 249 ompi_mtl_portals4_get_peer(struct ompi_communicator_t *comm, int rank)
 250 {
 251     return *((ptl_process_t*)(ompi_mtl_portals4_get_proc(comm, rank)->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PORTALS4]));
 252 }
 253 
 254 
 255 /* MTL interface functions */
 256 extern int ompi_mtl_portals4_finalize(struct mca_mtl_base_module_t *mtl);
 257 
 258 extern int ompi_mtl_portals4_add_procs(struct mca_mtl_base_module_t* mtl,
 259                                        size_t nprocs,
 260                                        struct ompi_proc_t** procs);
 261 
 262 extern int ompi_mtl_portals4_del_procs(struct mca_mtl_base_module_t* mtl,
 263                                        size_t nprocs,
 264                                        struct ompi_proc_t** procs);
 265 
 266 extern int ompi_mtl_portals4_send(struct mca_mtl_base_module_t* mtl,
 267                                   struct ompi_communicator_t* comm,
 268                                   int dest,
 269                                   int tag,
 270                                   struct opal_convertor_t *convertor,
 271                                   mca_pml_base_send_mode_t mode);
 272 
 273 extern int ompi_mtl_portals4_isend(struct mca_mtl_base_module_t* mtl,
 274                                    struct ompi_communicator_t* comm,
 275                                    int dest,
 276                                    int tag,
 277                                    struct opal_convertor_t *convertor,
 278                                    mca_pml_base_send_mode_t mode,
 279                                    bool blocking,
 280                                    mca_mtl_request_t *mtl_request);
 281 
 282 extern int ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl,
 283                                    struct ompi_communicator_t *comm,
 284                                    int src,
 285                                    int tag,
 286                                    struct opal_convertor_t *convertor,
 287                                    mca_mtl_request_t *mtl_request);
 288 
 289 extern int ompi_mtl_portals4_iprobe(struct mca_mtl_base_module_t* mtl,
 290                                     struct ompi_communicator_t *comm,
 291                                     int src,
 292                                     int tag,
 293                                     int *flag,
 294                                     struct ompi_status_public_t *status);
 295 
 296 extern int ompi_mtl_portals4_imrecv(struct mca_mtl_base_module_t* mtl,
 297                                     struct opal_convertor_t *convertor,
 298                                     struct ompi_message_t **message,
 299                                     struct mca_mtl_request_t *mtl_request);
 300 
 301 extern int ompi_mtl_portals4_improbe(struct mca_mtl_base_module_t *mtl,
 302                                      struct ompi_communicator_t *comm,
 303                                      int src,
 304                                      int tag,
 305                                      int *matched,
 306                                      struct ompi_message_t **message,
 307                                      struct ompi_status_public_t *status);
 308 
 309 extern int ompi_mtl_portals4_cancel(struct mca_mtl_base_module_t* mtl,
 310                                     mca_mtl_request_t *mtl_request,
 311                                     int flag);
 312 
 313 extern int ompi_mtl_portals4_add_comm(struct mca_mtl_base_module_t *mtl,
 314                                       struct ompi_communicator_t *comm);
 315 
 316 extern int ompi_mtl_portals4_del_comm(struct mca_mtl_base_module_t *mtl,
 317                                       struct ompi_communicator_t *comm);
 318 
 319 extern int ompi_mtl_portals4_progress(void);
 320 
 321 extern int ompi_mtl_portals4_get_error(int ptl_error);
 322 
 323 END_C_DECLS
 324 
 325 #endif  /* MTL_PORTALS_H_HAS_BEEN_INCLUDED */
/* [<][>][^][v][top][bottom][index][help] */
root/ompi/mca/mtl/portals4/mtl_portals4.h

INCLUDED FROM

DEFINITIONS
