root/oshmem/proc/proc.h

INCLUDED FROM

DEFINITIONS

1. oshmem_proc_local
2. oshmem_proc_for_find
3. oshmem_proc_find
4. oshmem_proc_pe
5. oshmem_proc_group_create_nofail
6. oshmem_proc_group_all
7. oshmem_proc_group_find
8. oshmem_proc_group_find_id
9. oshmem_proc_group_is_member
10. oshmem_num_procs
11. oshmem_my_proc_id
12. oshmem_get_transport_id
13. oshmem_get_transport_count

   1 /*
   2  * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
   3  *                         All rights reserved.
   4  * Copyright (c) 2016-2019 Research Organization for Information Science
   5  *                         and Technology (RIST). All rights reserved.
   6  * Copyright (c) 2017      Cisco Systems, Inc.  All rights reserved
   7  * $COPYRIGHT$
   8  *
   9  * Additional copyrights may follow
  10  *
  11  * $HEADER$
  12  */
  13 #ifndef OSHMEM_PROC_PROC_H
  14 #define OSHMEM_PROC_PROC_H
  15 
  16 #include "oshmem_config.h"
  17 #include "oshmem/types.h"
  18 #include "oshmem/constants.h"
  19 
  20 #include "opal/class/opal_list.h"
  21 #include "opal/util/proc.h"
  22 #include "opal/dss/dss_types.h"
  23 #include "opal/mca/hwloc/hwloc-internal.h"
  24 
  25 #include "ompi/proc/proc.h"
  26 #include "ompi/communicator/communicator.h"
  27 
  28 #include "oshmem/mca/scoll/scoll.h"
  29 #include "oshmem/runtime/runtime.h"
  30 #include "oshmem/shmem/shmem_api_logger.h"
  31 
  32 BEGIN_C_DECLS
  33 
  34 /* ******************************************************************** */
  35 
  36 struct oshmem_group_t;
  37 
  38 #define OSHMEM_PE_INVALID   (-1)
  39 
  40 /* This struct will be copied into the padding field of an ompi_proc_t
  41  * so the size of oshmem_proc_data_t must be less or equal than
  42  * OMPI_PROC_PADDING_SIZE */
  43 struct oshmem_proc_data_t {
  44     char * transport_ids;
  45     int num_transports;
  46 };
  47 
  48 typedef struct oshmem_proc_data_t oshmem_proc_data_t;
  49 
  50 #define OSHMEM_PROC_DATA(proc) \
  51     ((oshmem_proc_data_t *)(proc)->padding)
  52 
  53 /**
  54  * Group of Open SHMEM processes structure
  55  *
  56  * Set of processes used in collective operations.
  57  */
  58 struct oshmem_group_t {
  59     opal_object_t               base;
  60     int                         id;             /**< index in global array */
  61     int                         my_pe;
  62     int                         proc_count;     /**< number of processes in group */
  63     int                         is_member;   /* true if my_pe is part of the group, participate in collectives */
  64     struct ompi_proc_t          **proc_array; /**< list of pointers to ompi_proc_t structures
  65                                                    for each process in the group */
  66     opal_list_t                 peer_list;
  67 
  68     /* Collectives module interface and data */
  69     mca_scoll_base_group_scoll_t g_scoll;
  70     ompi_communicator_t*         ompi_comm;
  71 };
  72 typedef struct oshmem_group_t oshmem_group_t;
  73 OSHMEM_DECLSPEC OBJ_CLASS_DECLARATION(oshmem_group_t);
  74 
  75 OSHMEM_DECLSPEC extern oshmem_group_t* oshmem_group_all;
  76 OSHMEM_DECLSPEC extern oshmem_group_t* oshmem_group_self;
  77 OSHMEM_DECLSPEC extern oshmem_group_t* oshmem_group_null;
  78 
  79 
  80 /* ******************************************************************** */
  81 
  82 /**
  83  * Initialize the OSHMEM process subsystem
  84  *
  85  * Initialize the Open SHMEM process subsystem.  This function will
  86  * query the run-time environment and build a list of the proc
  87  * instances in the current pe set.  The local information not
  88  * easily determined by the run-time ahead of time (architecture and
  89  * hostname) will be published during this call.
  90  *
  91  * @note While an ompi_proc_t will exist with mostly valid information
  92  * for each process in the pe set at the conclusion of this
  93  * call, some information will not be immediately available.  This
  94  * includes the architecture and hostname, which will be available by
  95  * the conclusion of the stage gate.
  96  *
  97  * @retval OSHMEM_SUCESS  System successfully initialized
  98  * @retval OSHMEM_ERROR   Initialization failed due to unspecified error
  99  */
 100 OSHMEM_DECLSPEC int oshmem_proc_init(void);
 101 
 102 /**
 103  * Finalize the OSHMEM Process subsystem
 104  *
 105  * Finalize the Open SHMEM process subsystem.  This function will
 106  * release all memory created during the life of the application,
 107  * including all ompi_proc_t structures.
 108  *
 109  * @retval OSHMEM_SUCCESS  System successfully finalized
 110  */
 111 OSHMEM_DECLSPEC int oshmem_proc_finalize(void);
 112 
 113 /**
 114  * Returns a pointer to the local process
 115  *
 116  * Returns a pointer to the local process.  Unlike oshmem_proc_self(),
 117  * the reference count on the local proc instance is not modified by
 118  * this function.
 119  *
 120  * @return Pointer to the local process structure
 121  */
 122 static inline ompi_proc_t *oshmem_proc_local(void)
 123 {
 124     return (ompi_proc_t *)ompi_proc_local_proc;
 125 }
 126 
 127 /**
 128  * Returns the proc instance for a given name
 129  *
 130  * Returns the proc instance for the specified process name.  The
 131  * reference count for the proc instance is not incremented by this
 132  * function.
 133  *
 134  * @param[in] name     The process name to look for
 135  *
 136  * @return Pointer to the process instance for \c name
 137  */
 138 static inline ompi_proc_t *oshmem_proc_for_find(const ompi_process_name_t name)
 139 {
 140     return (ompi_proc_t *)ompi_proc_for_name(name);
 141 }
 142 
 143 static inline ompi_proc_t *oshmem_proc_find(int pe)
 144 {
 145     ompi_process_name_t name;
 146 
 147     name.jobid = OMPI_PROC_MY_NAME->jobid;
 148     name.vpid = pe;
 149     return oshmem_proc_for_find(name);
 150 }
 151 
 152 static inline int oshmem_proc_pe(ompi_proc_t *proc)
 153 {
 154     return (proc ? (int) ((ompi_process_name_t*)&proc->super.proc_name)->vpid : -1);
 155 }
 156 
 157 #define OSHMEM_PROC_JOBID(PROC)    (((ompi_process_name_t*)&((PROC)->super.proc_name))->jobid)
 158 #define OSHMEM_PROC_VPID(PROC)     (((ompi_process_name_t*)&((PROC)->super.proc_name))->vpid)
 159 
 160 /**
 161  * Initialize the OSHMEM process predefined groups
 162  *
 163  * Initialize the Open SHMEM process predefined groups.  This function will
 164  * query the run-time environment and build a list of the proc
 165  * instances in the current pe set.  The local information not
 166  * easily determined by the run-time ahead of time (architecture and
 167  * hostname) will be published during this call.
 168  *
 169  * @note This is primarily used once during SHMEM setup.
 170  *
 171  * @retval OSHMEM_SUCESS  System successfully initialized
 172  * @retval OSHMEM_ERROR   Initialization failed due to unspecified error
 173  */
 174 OSHMEM_DECLSPEC int oshmem_proc_group_init(void);
 175 
 176 /**
 177  * Finalize the OSHMEM process predefined groups
 178  *
 179  * @retval OSHMEM_SUCESS  System successfully initialized
 180  * @retval OSHMEM_ERROR   Initialization failed due to unspecified error
 181  */
 182 OSHMEM_DECLSPEC int oshmem_proc_group_finalize(void);
 183 
 184 /**
 185  * Release collectives used by the groups. The function
 186  * must be called prior to the oshmem_proc_group_finalize()
 187  */
 188 OSHMEM_DECLSPEC void oshmem_proc_group_finalize_scoll(void);
 189 
 190 /**
 191  * Create processes group.
 192  *
 193  * Returns the list of known proc instances located in this group.
 194  *
 195  * @param[in] pe_start     The lowest PE in the active set.
 196  * @param[in] pe_stride    The log (base 2) of the stride between consecutive
 197  *                         PEs in the active set.
 198  * @param[in] pe_size      The number of PEs in the active set.
 199  *
 200  * @return Array of pointers to proc instances in the current
 201  * known universe, or NULL if there is an internal failure.
 202  */
 203 OSHMEM_DECLSPEC oshmem_group_t *oshmem_proc_group_create(int pe_start,
 204                                                          int pe_stride,
 205                                                          int pe_size);
 206 
 207 /**
 208  * same as above but abort on failure
 209  */
 210 static inline oshmem_group_t *
 211 oshmem_proc_group_create_nofail(int pe_start, int pe_stride, int pe_size)
 212 {
 213     oshmem_group_t *group;
 214 
 215     group = oshmem_proc_group_create(pe_start, pe_stride, pe_size);
 216     if (NULL == group) {
 217         goto fatal;
 218     }
 219     return group;
 220 
 221 fatal:
 222     SHMEM_API_ERROR("Failed to create group (%d,%d,%d)",
 223                     pe_start, pe_stride, pe_size);
 224     oshmem_shmem_abort(-1);
 225     return NULL;
 226 }
 227 
 228 
 229 /**
 230  * Destroy processes group.
 231  *
 232  */
 233 OSHMEM_DECLSPEC void oshmem_proc_group_destroy(oshmem_group_t* group);
 234 
 235 static inline ompi_proc_t *oshmem_proc_group_all(int pe)
 236 {
 237     return oshmem_group_all->proc_array[pe];
 238 }
 239 
 240 static inline ompi_proc_t *oshmem_proc_group_find(oshmem_group_t* group,
 241                                                     int pe)
 242 {
 243     int i = 0;
 244     ompi_proc_t* proc = NULL;
 245 
 246     if (OPAL_LIKELY(group)) {
 247         if (OPAL_LIKELY(group == oshmem_group_all)) {
 248             /* To improve performance use direct index. It is feature of oshmem_group_all */
 249             proc = group->proc_array[pe];
 250         } else {
 251             for (i = 0; i < group->proc_count; i++) {
 252                 if (pe == oshmem_proc_pe(group->proc_array[i])) {
 253                     proc = group->proc_array[i];
 254                     break;
 255                 }
 256             }
 257         }
 258     } else {
 259         ompi_process_name_t name;
 260 
 261         name.jobid = OMPI_PROC_MY_NAME->jobid;
 262         name.vpid = pe;
 263         proc = oshmem_proc_for_find(name);
 264     }
 265 
 266     return proc;
 267 }
 268 
 269 static inline int oshmem_proc_group_find_id(oshmem_group_t* group, int pe)
 270 {
 271     int i = 0;
 272     int id = -1;
 273 
 274     if (group) {
 275         for (i = 0; i < group->proc_count; i++) {
 276             if (pe == oshmem_proc_pe(group->proc_array[i])) {
 277                 id = i;
 278                 break;
 279             }
 280         }
 281     }
 282 
 283     return id;
 284 }
 285 
 286 static inline int oshmem_proc_group_is_member(oshmem_group_t *group)
 287 {
 288     return group->is_member;
 289 }
 290 
 291 static inline int oshmem_num_procs(void)
 292 {
 293     return (oshmem_group_all ?
 294         oshmem_group_all->proc_count : (int)opal_list_get_size(&ompi_proc_list));
 295 }
 296 
 297 static inline int oshmem_my_proc_id(void)
 298 {
 299     return oshmem_group_self->my_pe;
 300 }
 301 
 302 static inline int oshmem_get_transport_id(int pe)
 303 {
 304     ompi_proc_t *proc;
 305 
 306     proc = oshmem_proc_group_find(oshmem_group_all, pe);
 307 
 308     return (int) OSHMEM_PROC_DATA(proc)->transport_ids[0];
 309 }
 310 
 311 static inline int oshmem_get_transport_count(int pe)
 312 {
 313     ompi_proc_t *proc;
 314     proc = oshmem_proc_group_find(oshmem_group_all, pe);
 315     return OSHMEM_PROC_DATA(proc)->num_transports;
 316 }
 317 
 318 END_C_DECLS
 319 
 320 #endif /* OSHMEM_PROC_PROC_H */