oshmem/runtime/oshmem_shmem

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
shmem_opal_thread
sighandler__SIGUSR1
sighandler__SIGTERM
oshmem_shmem_init
oshmem_shmem_preconnect_all
oshmem_shmem_preconnect_all_finalize
_shmem_init
   1 /*
   2  * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
   3  *                         All rights reserved.
   4  * Copyright (c) 2015-2019 Research Organization for Information Science
   5  *                         and Technology (RIST).  All rights reserved.
   6  * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
   7  * $COPYRIGHT$
   8  *
   9  * Additional copyrights may follow
  10  *
  11  * $HEADER$
  12  */
  13 
  14 #include "oshmem_config.h"
  15 
  16 #ifdef HAVE_SYS_TIME_H
  17 #include <sys/time.h>
  18 #endif  /* HAVE_SYS_TIME_H */
  19 #include <pthread.h>
  20 #ifdef HAVE_UNISTD_H
  21 #include <unistd.h>
  22 #endif
  23 
  24 #include <float.h>
  25 
  26 #include "math.h"
  27 #include "opal/class/opal_list.h"
  28 #include "opal/mca/base/base.h"
  29 #include "opal/runtime/opal_progress.h"
  30 #include "opal/threads/threads.h"
  31 #include "opal/util/argv.h"
  32 #include "opal/util/output.h"
  33 #include "opal/util/error.h"
  34 #include "opal/util/stacktrace.h"
  35 #include "opal/util/show_help.h"
  36 #include "opal/runtime/opal.h"
  37 
  38 #include "ompi/datatype/ompi_datatype.h"
  39 #include "opal/mca/rcache/base/base.h"
  40 #include "opal/mca/mpool/base/base.h"
  41 #include "opal/mca/allocator/base/base.h"
  42 #include "ompi/proc/proc.h"
  43 #include "ompi/runtime/mpiruntime.h"
  44 #include "ompi/util/timings.h"
  45 
  46 #include "oshmem/constants.h"
  47 #include "oshmem/runtime/runtime.h"
  48 #include "oshmem/runtime/params.h"
  49 #include "oshmem/runtime/oshmem_shmem_preconnect.h"
  50 #include "oshmem/mca/spml/base/base.h"
  51 #include "oshmem/mca/scoll/base/base.h"
  52 #include "oshmem/mca/atomic/base/base.h"
  53 #include "oshmem/mca/memheap/base/base.h"
  54 #include "oshmem/mca/sshmem/base/base.h"
  55 #include "oshmem/info/info.h"
  56 #include "oshmem/proc/proc.h"
  57 #include "oshmem/proc/proc_group_cache.h"
  58 #include "oshmem/op/op.h"
  59 #include "oshmem/request/request.h"
  60 #include "oshmem/shmem/shmem_api_logger.h"
  61 
  62 #include "oshmem/shmem/shmem_lock.h"
  63 
  64 #ifdef HAVE_SYS_MMAN_H
  65 #include <sys/mman.h>
  66 #endif
  67 
  68 #if OPAL_CC_USE_PRAGMA_IDENT
  69 #pragma ident OMPI_IDENT_STRING
  70 #elif OPAL_CC_USE_IDENT
  71 #ident OSHMEM_IDENT_STRING
  72 #endif
  73 
  74 /*
  75  * WHAT: add thread for invoking opal_progress() function
  76  * WHY:  SHMEM based on current ompi/trunk (by the time of integrating into Open MPI)
  77  *       has put/get implementation via send and needs opal_progress() invocation
  78  *       on the remote side (i.e. not true one-sided operations).
  79  */
  80 #define OSHMEM_OPAL_THREAD_ENABLE 0
  81 
  82 const char oshmem_version_string[] = OSHMEM_IDENT_STRING;
  83 
  84 /*
  85  * Global variables and symbols for the MPI layer
  86  */
  87 
  88 bool oshmem_shmem_initialized = false;
  89 bool oshmem_shmem_aborted = false;
  90 bool oshmem_mpi_thread_multiple = false;
  91 int oshmem_mpi_thread_requested = SHMEM_THREAD_SINGLE;
  92 int oshmem_mpi_thread_provided = SHMEM_THREAD_SINGLE;
  93 long *preconnect_value = 0;
  94 int shmem_api_logger_output = -1;
  95 
  96 MPI_Comm oshmem_comm_world = {0};
  97 
  98 opal_thread_t *oshmem_mpi_main_thread = NULL;
  99 
 100 shmem_internal_mutex_t shmem_internal_mutex_alloc = {{0}};
 101 
 102 shmem_ctx_t oshmem_ctx_default = NULL;
 103 
 104 static int _shmem_init(int argc, char **argv, int requested, int *provided);
 105 
 106 #if OSHMEM_OPAL_THREAD_ENABLE
 107 static void* shmem_opal_thread(void* argc)
 108 {
 109 /*
 110  * WHAT: sleep() invocation
 111  * WHY:  there occures a segfault sometimes and sleep()
 112  *       reduces it's possibility
 113  */
 114     sleep(1);
 115     while(oshmem_shmem_initialized)
 116         opal_progress();
 117     return NULL;
 118 }
 119 #endif
 120 
 121 int oshmem_shmem_inglobalexit = 0;
 122 int oshmem_shmem_globalexit_status = -1;
 123 
 124 static void sighandler__SIGUSR1(int signum)
 125 {
 126     if (0 != oshmem_shmem_inglobalexit)
 127     {
 128         return;
 129     }
 130     _exit(0);
 131 }
 132 static void sighandler__SIGTERM(int signum)
 133 {
 134     /* Do nothing. Just replace other unpredictalbe handlers with this one (e.g. mxm handler). */
 135 }
 136 
 137 int oshmem_shmem_init(int argc, char **argv, int requested, int *provided)
 138 {
 139     int ret = OSHMEM_SUCCESS;
 140 
 141     OMPI_TIMING_INIT(32);
 142 
 143     if (!oshmem_shmem_initialized) {
 144         ret = ompi_mpi_init(argc, argv, requested, provided, true);
 145         OMPI_TIMING_NEXT("ompi_mpi_init");
 146 
 147         if (OSHMEM_SUCCESS != ret) {
 148             return ret;
 149         }
 150 
 151         PMPI_Comm_dup(MPI_COMM_WORLD, &oshmem_comm_world);
 152         OMPI_TIMING_NEXT("PMPI_Comm_dup");
 153 
 154         SHMEM_MUTEX_INIT(shmem_internal_mutex_alloc);
 155 
 156         ret = _shmem_init(argc, argv, requested, provided);
 157         OMPI_TIMING_NEXT("_shmem_init");
 158         OMPI_TIMING_IMPORT_OPAL("mca_scoll_mpi_comm_query");
 159         OMPI_TIMING_IMPORT_OPAL("mca_scoll_enable");
 160         OMPI_TIMING_IMPORT_OPAL("mca_scoll_base_select");
 161 
 162         if (OSHMEM_SUCCESS != ret) {
 163             return ret;
 164         }
 165         oshmem_shmem_initialized = true;
 166 
 167         if (OSHMEM_SUCCESS != shmem_lock_init()) {
 168             SHMEM_API_ERROR( "shmem_lock_init() failed");
 169             return OSHMEM_ERROR;
 170         }
 171         OMPI_TIMING_NEXT("shmem_lock_init");
 172 
 173         /* this is a collective op, implies barrier */
 174         MCA_MEMHEAP_CALL(get_all_mkeys());
 175         OMPI_TIMING_NEXT("get_all_mkeys()");
 176 
 177         oshmem_shmem_preconnect_all();
 178         OMPI_TIMING_NEXT("shmem_preconnect_all");
 179 
 180 #if OSHMEM_OPAL_THREAD_ENABLE
 181         pthread_t thread_id;
 182         int perr;
 183         perr = pthread_create(&thread_id, NULL, &shmem_opal_thread, NULL);
 184         if (0 != perr) {
 185             SHMEM_API_ERROR("cannot create opal thread for SHMEM");
 186             return OSHMEM_ERROR;
 187         }
 188 #endif
 189         OMPI_TIMING_NEXT("THREAD_ENABLE");
 190     }
 191 #ifdef SIGUSR1
 192     signal(SIGUSR1,sighandler__SIGUSR1);
 193     signal(SIGTERM,sighandler__SIGTERM);
 194 #endif
 195     OMPI_TIMING_OUT;
 196     OMPI_TIMING_FINALIZE;
 197     return ret;
 198 }
 199 
 200 int oshmem_shmem_preconnect_all(void)
 201 {
 202     int rc = OSHMEM_SUCCESS;
 203 
 204     /* force qp creation and rkey exchange for memheap. Does not force exchange of static vars */
 205     if (oshmem_preconnect_all) {
 206         long val;
 207         int nproc;
 208         int my_pe;
 209         int i;
 210 
 211         val = 0xdeadbeaf;
 212 
 213         if (!preconnect_value) {
 214             rc =
 215                     MCA_MEMHEAP_CALL(private_alloc(sizeof(long), (void **)&preconnect_value));
 216         }
 217         if (!preconnect_value || (rc != OSHMEM_SUCCESS)) {
 218             SHMEM_API_ERROR("shmem_preconnect_all failed");
 219             return OSHMEM_ERR_OUT_OF_RESOURCE;
 220         }
 221 
 222         nproc = oshmem_num_procs();
 223         my_pe = oshmem_my_proc_id();
 224         for (i = 0; i < nproc; i++) {
 225             shmem_long_p(preconnect_value, val, (my_pe + i) % nproc);
 226         }
 227         shmem_barrier_all();
 228         SHMEM_API_VERBOSE(5, "Preconnected all PEs");
 229     }
 230 
 231     return OSHMEM_SUCCESS;
 232 }
 233 
 234 int oshmem_shmem_preconnect_all_finalize(void)
 235 {
 236     if (preconnect_value) {
 237         MCA_MEMHEAP_CALL(private_free(preconnect_value));
 238         preconnect_value = 0;
 239     }
 240 
 241     return OSHMEM_SUCCESS;
 242 }
 243 
 244 static int _shmem_init(int argc, char **argv, int requested, int *provided)
 245 {
 246     int ret = OSHMEM_SUCCESS;
 247     char *error = NULL;
 248 
 249     oshmem_mpi_thread_requested = requested;
 250     oshmem_mpi_thread_provided = requested;
 251 
 252     /* Register the OSHMEM layer's MCA parameters */
 253     if (OSHMEM_SUCCESS != (ret = oshmem_shmem_register_params())) {
 254         error = "oshmem_info_register: oshmem_register_params failed";
 255         goto error;
 256     }
 257     /* Setting verbosity for macros like SHMEM_API_VERBOSE, SHMEM_API_ERROR.
 258      * We need to set it right after registering mca verbosity variables
 259      */
 260     shmem_api_logger_output = opal_output_open(NULL);
 261     opal_output_set_verbosity(shmem_api_logger_output,
 262                               oshmem_shmem_api_verbose);
 263 
 264     /* initialize info */
 265     if (OSHMEM_SUCCESS != (ret = oshmem_info_init())) {
 266         error = "oshmem_info_init() failed";
 267         goto error;
 268     }
 269 
 270     /* initialize proc */
 271     if (OSHMEM_SUCCESS != (ret = oshmem_proc_init())) {
 272         error = "oshmem_proc_init() failed";
 273         goto error;
 274     }
 275 
 276     if (OSHMEM_SUCCESS != (ret = oshmem_op_init())) {
 277         error = "oshmem_op_init() failed";
 278         goto error;
 279     }
 280 
 281     if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_spml_base_framework, MCA_BASE_OPEN_DEFAULT))) {
 282         error = "mca_spml_base_open() failed";
 283         goto error;
 284     }
 285 
 286     if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_scoll_base_framework, MCA_BASE_OPEN_DEFAULT))) {
 287         error = "mca_scoll_base_open() failed";
 288         goto error;
 289     }
 290 
 291     if (OSHMEM_SUCCESS != (ret = mca_spml_base_select(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
 292         error = "mca_spml_base_select() failed";
 293         goto error;
 294     }
 295 
 296     if (OSHMEM_SUCCESS != (ret = mca_scoll_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
 297         error = "mca_scoll_base_find_available() failed";
 298         goto error;
 299     }
 300 
 301     /* Initialize each SHMEM handle subsystem */
 302     /* Initialize requests */
 303     if (OSHMEM_SUCCESS != (ret = oshmem_request_init())) {
 304         error = "oshmem_request_init() failed";
 305         goto error;
 306     }
 307 
 308     if (OSHMEM_SUCCESS != (ret = oshmem_proc_group_init())) {
 309         error = "oshmem_proc_group_init() failed";
 310         goto error;
 311     }
 312 
 313     /* start SPML/BTL's */
 314     ret = MCA_SPML_CALL(enable(true));
 315     if (OSHMEM_SUCCESS != ret) {
 316         error = "SPML control failed";
 317         goto error;
 318     }
 319 
 320     ret =
 321             MCA_SPML_CALL(add_procs(oshmem_group_all->proc_array, oshmem_group_all->proc_count));
 322     if (OSHMEM_SUCCESS != ret) {
 323         error = "SPML add procs failed";
 324         goto error;
 325     }
 326 
 327     if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_sshmem_base_framework, MCA_BASE_OPEN_DEFAULT))) {
 328         error = "mca_sshmem_base_open() failed";
 329         goto error;
 330     }
 331 
 332     if (OSHMEM_SUCCESS != (ret = mca_sshmem_base_select())) {
 333         error = "mca_sshmem_base_select() failed";
 334         goto error;
 335     }
 336 
 337     if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_memheap_base_framework, MCA_BASE_OPEN_DEFAULT))) {
 338         error = "mca_memheap_base_open() failed";
 339         goto error;
 340     }
 341 
 342     if (OSHMEM_SUCCESS != (ret = mca_memheap_base_select())) {
 343         error = "mca_memheap_base_select() failed";
 344         goto error;
 345     }
 346 
 347     if (OSHMEM_SUCCESS != (ret = mca_base_framework_open(&oshmem_atomic_base_framework, MCA_BASE_OPEN_DEFAULT))) {
 348         error = "mca_atomic_base_open() failed";
 349         goto error;
 350     }
 351 
 352     if (OSHMEM_SUCCESS != (ret = mca_atomic_base_find_available(OPAL_ENABLE_PROGRESS_THREADS, 1))) {
 353         error = "mca_atomic_base_find_available() failed";
 354         goto error;
 355     }
 356 
 357     /* This call should be done after memheap initialization */
 358     if (OSHMEM_SUCCESS != (ret = mca_scoll_enable())) {
 359         error = "mca_scoll_enable() failed";
 360         goto error;
 361     }
 362 
 363     (*provided) = oshmem_mpi_thread_provided;
 364 
 365     oshmem_mpi_thread_multiple = (oshmem_mpi_thread_provided == SHMEM_THREAD_MULTIPLE) ? true : false;
 366 
 367     error: if (ret != OSHMEM_SUCCESS) {
 368         const char *err_msg = opal_strerror(ret);
 369         opal_show_help("help-shmem-runtime.txt",
 370                        "shmem_init:startup:internal-failure",
 371                        true,
 372                        "SHMEM_INIT",
 373                        "SHMEM_INIT",
 374                        error,
 375                        err_msg,
 376                        ret);
 377         return ret;
 378     }
 379 
 380     return ret;
 381 }
 382
/* [<][>][^][v][top][bottom][index][help] */
root/oshmem/runtime/oshmem_shmem_init.c

DEFINITIONS
