root/ompi/mca/osc/pt2pt/osc_pt2pt_sync.h

INCLUDED FROM

DEFINITIONS

1. ompi_osc_pt2pt_sync_wait_nolock
2. ompi_osc_pt2pt_sync_wait
3. ompi_osc_pt2pt_sync_wait_expected
4. ompi_osc_pt2pt_sync_expected
5. ompi_osc_pt2pt_sync_reset

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2015-2018 Los Alamos National Security, LLC.  All rights
   4  *                         reserved.
   5  * $COPYRIGHT$
   6  *
   7  * Additional copyrights may follow
   8  *
   9  * $HEADER$
  10  */
  11 
  12 #ifndef OMPI_OSC_PT2PT_SYNC_H
  13 #define OMPI_OSC_PT2PT_SYNC_H
  14 
  15 #include "ompi_config.h"
  16 #include "opal/class/opal_free_list.h"
  17 #include "opal/threads/threads.h"
  18 
  19 enum ompi_osc_pt2pt_sync_type_t {
  20     /** default value */
  21     OMPI_OSC_PT2PT_SYNC_TYPE_NONE,
  22     /** lock access epoch */
  23     OMPI_OSC_PT2PT_SYNC_TYPE_LOCK,
  24     /** fence access epoch */
  25     OMPI_OSC_PT2PT_SYNC_TYPE_FENCE,
  26     /* post-start-complete-wait access epoch */
  27     OMPI_OSC_PT2PT_SYNC_TYPE_PSCW,
  28 };
  29 typedef enum ompi_osc_pt2pt_sync_type_t ompi_osc_pt2pt_sync_type_t;
  30 
  31 struct ompi_osc_pt2pt_module_t;
  32 struct ompi_osc_pt2pt_peer_t;
  33 
  34 /**
  35  * @brief synchronization object
  36  *
  37  * This structure holds information about an access epoch.
  38  */
  39 struct ompi_osc_pt2pt_sync_t {
  40     opal_free_list_item_t super;
  41 
  42     struct ompi_osc_pt2pt_module_t *module;
  43 
  44     /** synchronization type */
  45     ompi_osc_pt2pt_sync_type_t type;
  46 
  47     /** synchronization data */
  48     union {
  49         /** lock specific synchronization data */
  50         struct {
  51             /** lock target rank (-1 for all) */
  52             int target;
  53             /** lock type: MPI_LOCK_SHARED, MPI_LOCK_EXCLUSIVE */
  54             int type;
  55             /** assert specified at lock acquire time */
  56             int assert;
  57         } lock;
  58         /** post/start/complete/wait specific synchronization data */
  59         struct {
  60             /** group passed to ompi_osc_pt2pt_start */
  61             ompi_group_t *group;
  62         } pscw;
  63     } sync;
  64 
  65     /** array of peers for this sync */
  66     union {
  67         /** multiple peers (lock all, pscw, fence) */
  68         struct ompi_osc_pt2pt_peer_t **peers;
  69         /** single peer (targeted lock) */
  70         struct ompi_osc_pt2pt_peer_t *peer;
  71     } peer_list;
  72 
  73     /** number of peers */
  74     int num_peers;
  75 
  76     /** number of synchronization messages expected */
  77     opal_atomic_int32_t sync_expected;
  78 
  79     /** eager sends are active to all peers in this access epoch */
  80     volatile bool eager_send_active;
  81 
  82     /** communication has started on this epoch */
  83     bool epoch_active;
  84 
  85     /** lock to protect sync structure members */
  86     opal_mutex_t lock;
  87 
  88     /** condition variable for changes in the sync object */
  89     opal_condition_t cond;
  90 };
  91 typedef struct ompi_osc_pt2pt_sync_t ompi_osc_pt2pt_sync_t;
  92 
  93 OBJ_CLASS_DECLARATION(ompi_osc_pt2pt_sync_t);
  94 
  95 /**
  96  * @brief allocate a new synchronization object
  97  *
  98  * @param[in] module   osc pt2pt module
  99  *
 100  * @returns NULL on failure
 101  * @returns a new synchronization object on success
 102  */
 103 ompi_osc_pt2pt_sync_t *ompi_osc_pt2pt_sync_allocate (struct ompi_osc_pt2pt_module_t *module);
 104 
 105 /**
 106  * @brief release a synchronization object
 107  *
 108  * @param[in] pt2pt_sync   synchronization object allocated by ompi_osc_pt2pt_sync_allocate()
 109  */
 110 void ompi_osc_pt2pt_sync_return (ompi_osc_pt2pt_sync_t *pt2pt_sync);
 111 
 112 /**
 113  * Check if the target is part of a PSCW access epoch
 114  *
 115  * @param[in] module   osc pt2pt module
 116  * @param[in] target   target rank
 117  * @param[out] peer    peer object
 118  *
 119  * @returns false if the window is not in a PSCW access epoch or the peer is not
 120  *          in the group passed to MPI_Win_start
 121  * @returns true otherwise
 122  *
 123  * This functions verifies the target is part of an active PSCW access epoch.
 124  */
 125 bool ompi_osc_pt2pt_sync_pscw_peer (struct ompi_osc_pt2pt_module_t *module, int target, struct ompi_osc_pt2pt_peer_t **peer);
 126 
 127 /**
 128  * Wait for all remote peers in the synchronization to respond
 129  */
 130 static inline void ompi_osc_pt2pt_sync_wait_nolock (ompi_osc_pt2pt_sync_t *sync)
 131 {
 132     while (!sync->eager_send_active) {
 133         OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
 134                              "waiting for access epoch to start"));
 135         opal_condition_wait(&sync->cond, &sync->lock);
 136     }
 137 
 138     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
 139                          "access epoch ready"));
 140 }
 141 
 142 static inline void ompi_osc_pt2pt_sync_wait (ompi_osc_pt2pt_sync_t *sync)
 143 {
 144     OPAL_THREAD_LOCK(&sync->lock);
 145     ompi_osc_pt2pt_sync_wait_nolock (sync);
 146     OPAL_THREAD_UNLOCK(&sync->lock);
 147 }
 148 
 149 /**
 150  * Wait for all remote peers in the synchronization to respond
 151  */
 152 static inline void ompi_osc_pt2pt_sync_wait_expected (ompi_osc_pt2pt_sync_t *sync)
 153 {
 154     OPAL_THREAD_LOCK(&sync->lock);
 155     while (sync->sync_expected) {
 156         OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
 157                              "waiting for %d syncronization messages",
 158                              sync->sync_expected));
 159         opal_condition_wait(&sync->cond, &sync->lock);
 160     }
 161     OPAL_THREAD_UNLOCK(&sync->lock);
 162 
 163     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
 164                          "all synchronization messages received"));
 165 }
 166 
 167 static inline void ompi_osc_pt2pt_sync_expected (ompi_osc_pt2pt_sync_t *sync)
 168 {
 169     int32_t new_value = OPAL_THREAD_ADD_FETCH32 (&sync->sync_expected, -1);
 170     if (0 == new_value) {
 171         OPAL_THREAD_LOCK(&sync->lock);
 172         if (!(sync->type == OMPI_OSC_PT2PT_SYNC_TYPE_LOCK && sync->num_peers > 1)) {
 173             sync->eager_send_active = true;
 174         }
 175         opal_condition_broadcast (&sync->cond);
 176         OPAL_THREAD_UNLOCK(&sync->lock);
 177     }
 178 }
 179 
 180 static inline void ompi_osc_pt2pt_sync_reset (ompi_osc_pt2pt_sync_t *sync)
 181 {
 182     sync->type = OMPI_OSC_PT2PT_SYNC_TYPE_NONE;
 183     sync->eager_send_active = false;
 184     sync->epoch_active = 0;
 185     sync->peer_list.peers = NULL;
 186     sync->sync.pscw.group = NULL;
 187 }
 188 
 189 #endif /* OMPI_OSC_PT2PT_SYNC_H */