root/oshmem/mca/scoll/basic/scoll_basic_broadcast.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. mca_scoll_basic_broadcast
  2. _algorithm_central_counter
  3. _algorithm_binomial_tree

   1 /*
   2  * Copyright (c) 2013      Mellanox Technologies, Inc.
   3  *                         All rights reserved.
   4  * Copyright (c) 2019      Research Organization for Information Science
   5  *                         and Technology (RIST).  All rights reserved.
   6  * $COPYRIGHT$
   7  *
   8  * Additional copyrights may follow
   9  *
  10  * $HEADER$
  11  */
  12 
  13 #include "oshmem_config.h"
  14 #include <stdio.h>
  15 #include <stdlib.h>
  16 
  17 #include "opal/util/bit_ops.h"
  18 
  19 #include "oshmem/constants.h"
  20 #include "oshmem/mca/spml/spml.h"
  21 #include "oshmem/mca/scoll/scoll.h"
  22 #include "oshmem/mca/scoll/base/base.h"
  23 #include "scoll_basic.h"
  24 
  25 static int _algorithm_central_counter(struct oshmem_group_t *group,
  26                                        int PE_root,
  27                                        void *target,
  28                                        const void *source,
  29                                        size_t nlong,
  30                                        long *pSync);
  31 static int _algorithm_binomial_tree(struct oshmem_group_t *group,
  32                                      int PE_root,
  33                                      void *target,
  34                                      const void *source,
  35                                      size_t nlong,
  36                                      long *pSync);
  37 
  38 int mca_scoll_basic_broadcast(struct oshmem_group_t *group,
  39                               int PE_root,
  40                               void *target,
  41                               const void *source,
  42                               size_t nlong,
  43                               long *pSync,
  44                               bool nlong_type,
  45                               int alg)
  46 {
  47     int rc = OSHMEM_SUCCESS;
  48 
  49     /* Arguments validation */
  50     if (!group) {
  51         SCOLL_ERROR("Active set (group) of PE is not defined");
  52         rc = OSHMEM_ERR_BAD_PARAM;
  53     }
  54 
  55     /* Check if this PE is part of the group */
  56     if ((rc == OSHMEM_SUCCESS) && oshmem_proc_group_is_member(group)) {
  57         int i = 0;
  58 
  59         /* Do nothing on zero-length request */
  60         if (OPAL_UNLIKELY(nlong_type && !nlong)) {
  61             return OSHMEM_SUCCESS;
  62         }
  63 
  64         if (pSync) {
  65             alg = (alg == SCOLL_DEFAULT_ALG ?
  66                     mca_scoll_basic_param_broadcast_algorithm : alg);
  67             switch (alg) {
  68             case SCOLL_ALG_BROADCAST_CENTRAL_COUNTER:
  69                 {
  70                     rc = _algorithm_central_counter(group,
  71                                                      PE_root,
  72                                                      target,
  73                                                      source,
  74                                                      nlong,
  75                                                      pSync);
  76                     break;
  77                 }
  78             case SCOLL_ALG_BROADCAST_BINOMIAL:
  79                 {
  80                     rc = _algorithm_binomial_tree(group,
  81                                                    PE_root,
  82                                                    target,
  83                                                    source,
  84                                                    nlong,
  85                                                    pSync);
  86                     break;
  87                 }
  88             default:
  89                 {
  90                     rc = _algorithm_binomial_tree(group,
  91                                                    PE_root,
  92                                                    target,
  93                                                    source,
  94                                                    nlong,
  95                                                    pSync);
  96                 }
  97             }
  98         } else {
  99             SCOLL_ERROR("Incorrect argument pSync");
 100             rc = OSHMEM_ERR_BAD_PARAM;
 101         }
 102 
 103         /* Restore initial values */
 104         SCOLL_VERBOSE(12,
 105                       "[#%d] Restore special synchronization array",
 106                       group->my_pe);
 107         for (i = 0; pSync && (i < _SHMEM_BCAST_SYNC_SIZE); i++) {
 108             pSync[i] = _SHMEM_SYNC_VALUE;
 109         }
 110     }
 111 
 112     return rc;
 113 }
 114 
 115 /*
 116  This algorithm is quite simple and straightforward. But because of it�s obvious simplicity and
 117  the naive prove for correctness it is implemented quite often. The root send data to all.
 118  Outlay:
 119  NP-1 competing network transfers are needed to implement the counter
 120  The memory usage is constant (1 byte) per node.
 121  */
 122 static int _algorithm_central_counter(struct oshmem_group_t *group,
 123                                        int PE_root,
 124                                        void *target,
 125                                        const void *source,
 126                                        size_t nlong,
 127                                        long *pSync)
 128 {
 129     int rc = OSHMEM_SUCCESS;
 130     int i = 0;
 131 
 132     SCOLL_VERBOSE(12,
 133                   "[#%d] Broadcast algorithm: Central Counter",
 134                   group->my_pe);
 135     SCOLL_VERBOSE(15,
 136                   "[#%d] pSync[0] = %ld root = #%d",
 137                   group->my_pe, pSync[0], PE_root);
 138 
 139     /* Check if this PE is the root */
 140     if (PE_root == group->my_pe) {
 141         int pe_cur = 0;
 142 
 143         SCOLL_VERBOSE(14,
 144                       "[#%d] send data to all PE in the group",
 145                       group->my_pe);
 146         for (i = 0; (i < group->proc_count) && (rc == OSHMEM_SUCCESS); i++) {
 147             pe_cur = oshmem_proc_pe(group->proc_array[i]);
 148             if (pe_cur != PE_root) {
 149                 SCOLL_VERBOSE(15,
 150                               "[#%d] send data to #%d",
 151                               group->my_pe, pe_cur);
 152                 rc = MCA_SPML_CALL(put(oshmem_ctx_default, target, nlong, (void *)source, pe_cur));
 153             }
 154         }
 155         /* quiet is needed because scoll level barrier does not
 156          * guarantee put completion
 157          */
 158         MCA_SPML_CALL(quiet(oshmem_ctx_default));
 159     }
 160 
 161     if (rc == OSHMEM_SUCCESS) {
 162         SCOLL_VERBOSE(14, "[#%d] Wait for operation completion", group->my_pe);
 163         /* wait until root finishes sending data  */
 164         rc = BARRIER_FUNC(group,
 165                 (pSync + 1),
 166                 SCOLL_DEFAULT_ALG);
 167     }
 168 
 169     return rc;
 170 }
 171 
 172 /*
 173  The Binomial Spanning Tree algorithm.
 174  Outlay:
 175  The game scales with log2(NP) and uses 1 byte of memory.
 176  */
 177 static int _algorithm_binomial_tree(struct oshmem_group_t *group,
 178                                      int PE_root,
 179                                      void *target,
 180                                      const void *source,
 181                                      size_t nlong,
 182                                      long *pSync)
 183 {
 184     int rc = OSHMEM_SUCCESS;
 185     long value = SHMEM_SYNC_INIT;
 186     int root_id = oshmem_proc_group_find_id(group, PE_root);
 187     int my_id = oshmem_proc_group_find_id(group, group->my_pe);
 188     int peer_id = 0;
 189     int peer_pe = 0;
 190     int vrank;
 191     int dim = opal_cube_dim(group->proc_count);
 192     int hibit;
 193     int mask;
 194     int i = 0;
 195 
 196     SCOLL_VERBOSE(12, "[#%d] Broadcast algorithm: Tree", group->my_pe);
 197     SCOLL_VERBOSE(15,
 198                   "[#%d] pSync[0] = %ld root = #%d",
 199                   group->my_pe, pSync[0], PE_root);
 200 
 201     vrank = (my_id + group->proc_count - root_id) % group->proc_count;
 202     hibit = opal_hibit(vrank, dim);
 203 
 204     SCOLL_VERBOSE(15,
 205                   "[#%d] dim = %d vrank = %d hibit = %d",
 206                   group->my_pe, dim, vrank, hibit);
 207 
 208     dim--;
 209 
 210     pSync[0] = SHMEM_SYNC_READY;
 211     /* Receive data from parent in the tree. */
 212     if (vrank > 0) {
 213         value = SHMEM_SYNC_READY;
 214 
 215         SCOLL_VERBOSE(14, "[#%d] wait", group->my_pe);
 216         rc = MCA_SPML_CALL(wait((void*)pSync, SHMEM_CMP_NE, (void*)&value, SHMEM_LONG));
 217         while ((value = pSync[0]) < 0) {
 218             SCOLL_VERBOSE(14,
 219                           "[#%d] Broadcast size is a negative value (%li)\n",
 220                           group->my_pe, pSync[0]);
 221             MCA_SPML_CALL(wait((void*)pSync, SHMEM_CMP_NE, (void*)&value, SHMEM_LONG));
 222         }
 223         if (OSHMEM_SUCCESS != rc) {
 224             return rc;
 225         }
 226         nlong = (size_t) pSync[0];
 227     }
 228 
 229     /* Send data to the children. */
 230     for (i = hibit + 1, mask = 1 << i; i <= dim; ++i, mask <<= 1) {
 231         peer_id = vrank | mask;
 232 
 233         if (peer_id < group->proc_count) {
 234             /* Wait for the child to be ready to receive (pSync must have the initial value) */
 235             peer_id = (peer_id + root_id) % group->proc_count;
 236             peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
 237 
 238             SCOLL_VERBOSE(14,
 239                           "[#%d] check remote pe is ready to receive #%d",
 240                           group->my_pe, peer_pe);
 241             do {
 242                 rc = MCA_SPML_CALL(get(oshmem_ctx_default, (void*)pSync, sizeof(long), (void*)pSync, peer_pe));
 243             } while ((OSHMEM_SUCCESS == rc) && (pSync[0] != SHMEM_SYNC_READY));
 244 
 245             SCOLL_VERBOSE(14, "[#%d] send data to #%d", group->my_pe, peer_pe);
 246             rc = MCA_SPML_CALL(put(oshmem_ctx_default, target, nlong, (my_id == root_id ? (void *)source : target), peer_pe));
 247 
 248             MCA_SPML_CALL(fence(oshmem_ctx_default));
 249 
 250             SCOLL_VERBOSE(14, "[#%d] signals to #%d", group->my_pe, peer_pe);
 251             value = nlong;
 252             rc = MCA_SPML_CALL(put(oshmem_ctx_default, (void*)pSync, sizeof(value), (void*)&value, peer_pe));
 253             if (OSHMEM_SUCCESS != rc) {
 254                 break;
 255             }
 256         }
 257     }
 258 
 259     return rc;
 260 }

/* [<][>][^][v][top][bottom][index][help] */