oshmem/proc/proc.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
oshmem_proc_init
oshmem_proc_finalize
oshmem_proc_group_init
oshmem_proc_group_finalize_scoll
oshmem_proc_group_finalize
oshmem_proc_group_create
oshmem_proc_group_destroy_internal
oshmem_proc_group_destroy
   1 /*
   2  * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
   3  *                         All rights reserved.
   4  * Copyright (c) 2014-2019 Research Organization for Information Science
   5  *                         and Technology (RIST).  All rights reserved.
   6  * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
   7  * Copyright (c) 2016      ARM, Inc. All rights reserved.
   8  * $COPYRIGHT$
   9  *
  10  * Additional copyrights may follow
  11  *
  12  * $HEADER$
  13  */
  14 
  15 #include "oshmem_config.h"
  16 #include "oshmem/proc/proc.h"
  17 #include "oshmem/constants.h"
  18 #include "oshmem/runtime/runtime.h"
  19 #include "oshmem/mca/scoll/base/base.h"
  20 #include "oshmem/proc/proc_group_cache.h"
  21 
  22 #ifdef HAVE_STRINGS_H
  23 #include <strings.h>
  24 #endif
  25 
  26 #include "opal/datatype/opal_convertor.h"
  27 #include "opal/threads/mutex.h"
  28 #include "opal/dss/dss.h"
  29 #include "opal/util/arch.h"
  30 #include "opal/class/opal_list.h"
  31 
  32 
  33 static opal_mutex_t oshmem_proc_lock;
  34 
  35 int oshmem_proc_init(void)
  36 {
  37     OBJ_CONSTRUCT(&oshmem_proc_lock, opal_mutex_t);
  38 
  39     /* check oshmem_proc_data_t can fit within ompi_proc_t padding */
  40     assert(sizeof(oshmem_proc_data_t) <= OMPI_PROC_PADDING_SIZE);
  41     /* check ompi_proc_t padding is aligned on a pointer */
  42     assert(0 == (offsetof(ompi_proc_t, padding) & (sizeof(char *)-1)));
  43 
  44     return OSHMEM_SUCCESS;
  45 }
  46 
  47 int oshmem_proc_finalize(void)
  48 {
  49     OBJ_DESTRUCT(&oshmem_proc_lock);
  50 
  51     return OSHMEM_SUCCESS;
  52 }
  53 
  54 opal_pointer_array_t oshmem_group_array = {{0}};
  55 
  56 oshmem_group_t* oshmem_group_all = NULL;
  57 oshmem_group_t* oshmem_group_self = NULL;
  58 oshmem_group_t* oshmem_group_null = NULL;
  59 
  60 OBJ_CLASS_INSTANCE(oshmem_group_t, opal_object_t, NULL, NULL);
  61 
  62 static void oshmem_proc_group_destroy_internal(oshmem_group_t* group,
  63                                                int scoll_unselect);
  64 
  65 int oshmem_proc_group_init(void)
  66 {
  67     int rc;
  68 
  69     rc = oshmem_group_cache_init();
  70     if (OSHMEM_SUCCESS != rc) {
  71         return rc;
  72     }
  73 
  74     /* Setup communicator array */
  75     OBJ_CONSTRUCT(&oshmem_group_array, opal_pointer_array_t);
  76 
  77     rc = opal_pointer_array_init(&oshmem_group_array, 0,
  78                                  INT_MAX, 1);
  79     if (OPAL_SUCCESS != rc) {
  80         goto err1;
  81     }
  82 
  83     /* Setup SHMEM_GROUP_ALL */
  84     oshmem_group_all = oshmem_proc_group_create(0, 1, ompi_comm_size(oshmem_comm_world));
  85     if (NULL == oshmem_group_all) {
  86         goto err2;
  87     }
  88 
  89     /* Setup SHMEM_GROUP_SELF */
  90     oshmem_group_self = oshmem_proc_group_create(oshmem_proc_pe(oshmem_proc_local()), 0, 1);
  91     if (NULL == oshmem_group_self) {
  92         goto err3;
  93     }
  94 
  95     /* Setup SHMEM_GROUP_NULL */
  96     oshmem_group_null = NULL;
  97 
  98     return OSHMEM_SUCCESS;
  99 
 100 err3:
 101     oshmem_proc_group_destroy_internal(oshmem_group_all, 1);
 102 err2:
 103     OBJ_DESTRUCT(&oshmem_group_array);
 104 err1:
 105     oshmem_group_cache_destroy();
 106     return OSHMEM_ERROR;
 107 }
 108 
 109 void oshmem_proc_group_finalize_scoll(void)
 110 {
 111     int max, i;
 112     oshmem_group_t *group;
 113 
 114     /* Check whether we have some left */
 115     max = opal_pointer_array_get_size(&oshmem_group_array);
 116     for (i = 0; i < max; i++) {
 117         group = (oshmem_group_t *) opal_pointer_array_get_item(&oshmem_group_array,
 118                                                                i);
 119         if (NULL != group) {
 120             mca_scoll_base_group_unselect(group);
 121         }
 122     }
 123 }
 124 
 125 int oshmem_proc_group_finalize(void)
 126 {
 127     int max, i;
 128     oshmem_group_t *group;
 129 
 130     /* Check whether we have some left */
 131     max = opal_pointer_array_get_size(&oshmem_group_array);
 132     for (i = 0; i < max; i++) {
 133         group =
 134                 (oshmem_group_t *) opal_pointer_array_get_item(&oshmem_group_array,
 135                                                                i);
 136         if (NULL != group) {
 137             /* Group has not been freed before finalize */
 138             oshmem_proc_group_destroy_internal(group, 0);
 139         }
 140     }
 141 
 142     OBJ_DESTRUCT(&oshmem_group_array);
 143 
 144     oshmem_group_cache_destroy();
 145     return OSHMEM_SUCCESS;
 146 }
 147 
 148 oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_size)
 149 {
 150     int cur_pe, count_pe;
 151     int i;
 152     oshmem_group_t* group = NULL;
 153     ompi_proc_t** proc_array = NULL;
 154     ompi_proc_t* proc = NULL;
 155 
 156     assert(oshmem_proc_local());
 157 
 158     group = oshmem_group_cache_find(pe_start, pe_stride, pe_size);
 159     if (NULL != group) {
 160         return group;
 161     }
 162 
 163     group = OBJ_NEW(oshmem_group_t);
 164     if (NULL == group) {
 165         return NULL;
 166     }
 167 
 168     cur_pe = 0;
 169     count_pe = 0;
 170 
 171     OPAL_THREAD_LOCK(&oshmem_proc_lock);
 172 
 173     /* allocate an array */
 174     proc_array = (ompi_proc_t**) malloc(pe_size * sizeof(ompi_proc_t*));
 175     if (NULL == proc_array) {
 176         OBJ_RELEASE(group);
 177         OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
 178         return NULL ;
 179     }
 180 
 181     group->my_pe = oshmem_proc_pe(oshmem_proc_local());
 182     group->is_member = 0;
 183     for (i = 0 ; i < ompi_comm_size(oshmem_comm_world) ; i++) {
 184         proc = oshmem_proc_find(i);
 185         if (NULL == proc) {
 186             opal_output(0,
 187                     "Error: Can not find proc object for pe = %d", i);
 188             free(proc_array);
 189             OBJ_RELEASE(group);
 190             OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
 191             return NULL;
 192         }
 193         if (count_pe >= (int) pe_size) {
 194             break;
 195         } else if ((cur_pe >= pe_start)
 196                 && ((pe_stride == 0)
 197                     || (((cur_pe - pe_start) % pe_stride) == 0))) {
 198             proc_array[count_pe++] = proc;
 199             if (oshmem_proc_pe(proc) == group->my_pe)
 200                 group->is_member = 1;
 201         }
 202         cur_pe++;
 203     }
 204     group->proc_array = proc_array;
 205     group->proc_count = (int) count_pe;
 206     group->ompi_comm = NULL;
 207 
 208     /* Prepare peers list */
 209     OBJ_CONSTRUCT(&(group->peer_list), opal_list_t);
 210     {
 211         ompi_namelist_t *peer = NULL;
 212 
 213         for (i = 0; i < group->proc_count; i++) {
 214             peer = OBJ_NEW(ompi_namelist_t);
 215             peer->name.jobid = OSHMEM_PROC_JOBID(group->proc_array[i]);
 216             peer->name.vpid = OSHMEM_PROC_VPID(group->proc_array[i]);
 217             opal_list_append(&(group->peer_list), &peer->super);
 218         }
 219     }
 220     group->id = opal_pointer_array_add(&oshmem_group_array, group);
 221 
 222     memset(&group->g_scoll, 0, sizeof(mca_scoll_base_group_scoll_t));
 223 
 224     if (OSHMEM_SUCCESS != mca_scoll_base_select(group)) {
 225         opal_output(0,
 226                 "Error: No collective modules are available: group is not created, returning NULL");
 227         oshmem_proc_group_destroy_internal(group, 0);
 228         OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
 229         return NULL;
 230     }
 231 
 232     if (OSHMEM_SUCCESS != oshmem_group_cache_insert(group, pe_start,
 233                                                     pe_stride, pe_size)) {
 234         oshmem_proc_group_destroy_internal(group, 1);
 235         OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
 236         return NULL;
 237     }
 238 
 239     OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
 240     return group;
 241 }
 242 
 243 static void
 244 oshmem_proc_group_destroy_internal(oshmem_group_t* group, int scoll_unselect)
 245 {
 246     if (NULL == group) {
 247         return;
 248     }
 249 
 250     if (scoll_unselect) {
 251         mca_scoll_base_group_unselect(group);
 252     }
 253 
 254     /* Destroy proc array */
 255     if (group->proc_array) {
 256         free(group->proc_array);
 257     }
 258 
 259     /* Destroy peer list */
 260     {
 261         opal_list_item_t *item;
 262 
 263         while (NULL != (item = opal_list_remove_first(&(group->peer_list)))) {
 264             /* destruct the item (we constructed it), then free the memory chunk */
 265             OBJ_RELEASE(item);
 266         }
 267         OBJ_DESTRUCT(&(group->peer_list));
 268     }
 269 
 270     /* reset the oshmem_group_array entry - make sure that the
 271      * entry is in the table */
 272     if (NULL
 273             != opal_pointer_array_get_item(&oshmem_group_array,
 274                 group->id)) {
 275         opal_pointer_array_set_item(&oshmem_group_array, group->id, NULL );
 276     }
 277 
 278     OBJ_RELEASE(group);
 279 }
 280 
 281 void oshmem_proc_group_destroy(oshmem_group_t* group)
 282 {
 283     if (oshmem_group_cache_enabled()) {
 284         return;
 285     }
 286     oshmem_proc_group_destroy_internal(group, 1);
 287 }
/* [<][>][^][v][top][bottom][index][help] */
root/oshmem/proc/proc.c

DEFINITIONS
