orte/mca/rmaps/base/rmaps_base_map

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
orte_rmaps_base_map_job
orte_rmaps_base_display_map
   1 /*
   2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   3  *                         University Research and Technology
   4  *                         Corporation.  All rights reserved.
   5  * Copyright (c) 2004-2018 The University of Tennessee and The University
   6  *                         of Tennessee Research Foundation.  All rights
   7  *                         reserved.
   8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   9  *                         University of Stuttgart.  All rights reserved.
  10  * Copyright (c) 2004-2005 The Regents of the University of California.
  11  *                         All rights reserved.
  12  * Copyright (c) 2011-2018 Cisco Systems, Inc.  All rights reserved
  13  * Copyright (c) 2011-2012 Los Alamos National Security, LLC.
  14  *                         All rights reserved.
  15  * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  16  * Copyright (c) 2016      Research Organization for Information Science
  17  *                         and Technology (RIST). All rights reserved.
  18  * $COPYRIGHT$
  19  *
  20  * Additional copyrights may follow
  21  *
  22  * $HEADER$
  23  */
  24 
  25 #include "orte_config.h"
  26 #include "orte/constants.h"
  27 
  28 #include <string.h>
  29 
  30 #include "orte/mca/mca.h"
  31 #include "opal/util/output.h"
  32 #include "opal/util/string_copy.h"
  33 #include "opal/mca/base/base.h"
  34 #include "opal/mca/hwloc/base/base.h"
  35 #include "opal/dss/dss.h"
  36 
  37 #include "orte/mca/errmgr/errmgr.h"
  38 #include "orte/runtime/orte_globals.h"
  39 #include "orte/util/show_help.h"
  40 #include "orte/util/threads.h"
  41 #include "orte/mca/state/state.h"
  42 
  43 #include "orte/mca/rmaps/base/base.h"
  44 #include "orte/mca/rmaps/base/rmaps_private.h"
  45 
  46 
  47 void orte_rmaps_base_map_job(int fd, short args, void *cbdata)
  48 {
  49     orte_state_caddy_t *caddy = (orte_state_caddy_t*)cbdata;
  50     orte_job_t *jdata;
  51     orte_node_t *node;
  52     int rc, i, ppx = 0;
  53     bool did_map, pernode = false, persocket = false;
  54     orte_rmaps_base_selected_module_t *mod;
  55     orte_job_t *parent;
  56     orte_vpid_t nprocs;
  57     orte_app_context_t *app;
  58     bool inherit = false;
  59 
  60     ORTE_ACQUIRE_OBJECT(caddy);
  61     jdata = caddy->jdata;
  62 
  63     jdata->state = ORTE_JOB_STATE_MAP;
  64 
  65     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
  66                         "mca:rmaps: mapping job %s",
  67                         ORTE_JOBID_PRINT(jdata->jobid));
  68 
  69     /* if this is a dynamic job launch and they didn't explicitly
  70      * request inheritance, then don't inherit the launch directives */
  71     if (orte_get_attribute(&jdata->attributes, ORTE_JOB_LAUNCH_PROXY, NULL, OPAL_NAME)) {
  72         inherit = orte_rmaps_base.inherit;
  73         opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
  74                             "mca:rmaps: dynamic job %s %s inherit launch directives",
  75                             ORTE_JOBID_PRINT(jdata->jobid),
  76                             inherit ? "will" : "will not");
  77     } else {
  78         /* initial launch always takes on MCA params */
  79         inherit = true;
  80     }
  81 
  82     if (inherit) {
  83         if (NULL == jdata->map->ppr && NULL != orte_rmaps_base.ppr) {
  84             jdata->map->ppr = strdup(orte_rmaps_base.ppr);
  85         }
  86         if (0 == jdata->map->cpus_per_rank) {
  87             jdata->map->cpus_per_rank = orte_rmaps_base.cpus_per_rank;
  88         }
  89     }
  90     if (NULL != jdata->map->ppr) {
  91         /* get the procs/object */
  92         ppx = strtoul(jdata->map->ppr, NULL, 10);
  93         if (NULL != strstr(jdata->map->ppr, "node")) {
  94             pernode = true;
  95         } else if (NULL != strstr(jdata->map->ppr, "socket")) {
  96             persocket = true;
  97         }
  98     }
  99 
 100     /* compute the number of procs and check validity */
 101     nprocs = 0;
 102     for (i=0; i < jdata->apps->size; i++) {
 103         if (NULL != (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
 104             if (0 == app->num_procs) {
 105                 opal_list_t nodes;
 106                 orte_std_cntr_t slots;
 107                 OBJ_CONSTRUCT(&nodes, opal_list_t);
 108                 orte_rmaps_base_get_target_nodes(&nodes, &slots, app, ORTE_MAPPING_BYNODE, true, true);
 109                 if (pernode) {
 110                     slots = ppx * opal_list_get_size(&nodes);
 111                 } else if (persocket) {
 112                     /* add in #sockets for each node */
 113                     OPAL_LIST_FOREACH(node, &nodes, orte_node_t) {
 114                         slots += ppx * opal_hwloc_base_get_nbobjs_by_type(node->topology->topo,
 115                                                                           HWLOC_OBJ_SOCKET, 0,
 116                                                                           OPAL_HWLOC_AVAILABLE);
 117                     }
 118                 }
 119                 app->num_procs = slots;
 120                 OPAL_LIST_DESTRUCT(&nodes);
 121             }
 122             nprocs += app->num_procs;
 123         }
 124     }
 125 
 126 
 127     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 128                         "mca:rmaps: setting mapping policies for job %s nprocs %d",
 129                         ORTE_JOBID_PRINT(jdata->jobid), (int)nprocs);
 130 
 131     if (inherit && !jdata->map->display_map) {
 132         jdata->map->display_map = orte_rmaps_base.display_map;
 133     }
 134 
 135     /* set the default mapping policy IFF it wasn't provided */
 136     if (!ORTE_MAPPING_POLICY_IS_SET(jdata->map->mapping)) {
 137         if (inherit && (ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
 138             opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 139                                 "mca:rmaps mapping given by MCA param");
 140             jdata->map->mapping = orte_rmaps_base.mapping;
 141         } else {
 142             /* default based on number of procs */
 143             if (nprocs <= 2) {
 144                 if (1 < orte_rmaps_base.cpus_per_rank) {
 145                     /* assigning multiple cpus to a rank requires that we map to
 146                      * objects that have multiple cpus in them, so default
 147                      * to byslot if nothing else was specified by the user.
 148                      */
 149                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 150                                         "mca:rmaps[%d] mapping not given - using byslot", __LINE__);
 151                     ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
 152                 } else if (opal_hwloc_use_hwthreads_as_cpus) {
 153                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 154                                         "mca:rmaps[%d] mapping not given - using byhwthread", __LINE__);
 155                     ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYHWTHREAD);
 156                 } else {
 157                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 158                                         "mca:rmaps[%d] mapping not given - using bycore", __LINE__);
 159                     ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYCORE);
 160                 }
 161             } else {
 162                 /* if NUMA is available, map by that */
 163                 if (NULL != hwloc_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE, 0)) {
 164                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 165                                         "mca:rmaps[%d] mapping not set by user - using bynuma", __LINE__);
 166                     ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYNUMA);
 167                 } else if (NULL != hwloc_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_SOCKET, 0)) {
 168                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 169                                         "mca:rmaps[%d] mapping not set by user and no NUMA - using bysocket", __LINE__);
 170                     ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSOCKET);
 171                 } else {
 172                     /* if we have neither, then just do by slot */
 173                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 174                                         "mca:rmaps[%d] mapping not given and no NUMA or sockets - using byslot", __LINE__);
 175                     ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
 176                 }
 177             }
 178         }
 179     }
 180 
 181     /* check for oversubscribe directives */
 182     if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping))) {
 183         if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
 184             ORTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
 185         } else if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) {
 186             ORTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
 187         } else {
 188             ORTE_UNSET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
 189             ORTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_MAPPING_SUBSCRIBE_GIVEN);
 190         }
 191     }
 192 
 193     /* check for no-use-local directive */
 194     if (!(ORTE_MAPPING_LOCAL_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping))) {
 195         if (inherit && (ORTE_MAPPING_NO_USE_LOCAL & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
 196             ORTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_MAPPING_NO_USE_LOCAL);
 197         }
 198     }
 199 
 200     /* we don't have logic to determine default rank policy, so
 201      * just inherit it if they didn't give us one */
 202     if (!ORTE_RANKING_POLICY_IS_SET(jdata->map->ranking)) {
 203         jdata->map->ranking = orte_rmaps_base.ranking;
 204     }
 205 
 206     /* define the binding policy for this job - if the user specified one
 207      * already (e.g., during the call to comm_spawn), then we don't
 208      * override it */
 209     if (!OPAL_BINDING_POLICY_IS_SET(jdata->map->binding)) {
 210         if (inherit && OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy)) {
 211             /* if the user specified a default binding policy via
 212              * MCA param, then we use it - this can include a directive
 213              * to overload */
 214             opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 215                             "mca:rmaps[%d] binding policy given", __LINE__);
 216             jdata->map->binding = opal_hwloc_binding_policy;
 217         } else if (0 < jdata->map->cpus_per_rank) {
 218             /* bind to cpus */
 219             if (opal_hwloc_use_hwthreads_as_cpus) {
 220                 /* if we are using hwthread cpus, then bind to those */
 221                 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 222                                 "mca:rmaps[%d] binding not given - using byhwthread", __LINE__);
 223                 OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_HWTHREAD);
 224             } else {
 225                 /* bind to core */
 226                 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 227                                 "mca:rmaps[%d] binding not given - using bycore", __LINE__);
 228                 OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_CORE);
 229             }
 230         } else {
 231             /* if the user explicitly mapped-by some object, then we default
 232              * to binding to that object */
 233             orte_mapping_policy_t mpol;
 234             mpol = ORTE_GET_MAPPING_POLICY(jdata->map->mapping);
 235             if (ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
 236                 if (ORTE_MAPPING_BYHWTHREAD == mpol) {
 237                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 238                                     "mca:rmaps[%d] binding not given - using byhwthread", __LINE__);
 239                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_HWTHREAD);
 240                 } else if (ORTE_MAPPING_BYCORE == mpol) {
 241                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 242                                     "mca:rmaps[%d] binding not given - using bycore", __LINE__);
 243                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_CORE);
 244                 } else if (ORTE_MAPPING_BYL1CACHE == mpol) {
 245                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 246                                     "mca:rmaps[%d] binding not given - using byL1", __LINE__);
 247                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_L1CACHE);
 248                 } else if (ORTE_MAPPING_BYL2CACHE == mpol) {
 249                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 250                                     "mca:rmaps[%d] binding not given - using byL2", __LINE__);
 251                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_L2CACHE);
 252                 } else if (ORTE_MAPPING_BYL3CACHE == mpol) {
 253                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 254                                     "mca:rmaps[%d] binding not given - using byL3", __LINE__);
 255                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_L3CACHE);
 256                 } else if (ORTE_MAPPING_BYSOCKET == mpol) {
 257                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 258                                     "mca:rmaps[%d] binding not given - using bysocket", __LINE__);
 259                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_SOCKET);
 260                 } else if (ORTE_MAPPING_BYNUMA == mpol) {
 261                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 262                                     "mca:rmaps[%d] binding not given - using bynuma", __LINE__);
 263                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NUMA);
 264                 } else {
 265                     /* we are mapping by node or some other non-object method */
 266                     if (nprocs <= 2) {
 267                         if (opal_hwloc_use_hwthreads_as_cpus) {
 268                             /* if we are using hwthread cpus, then bind to those */
 269                             opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 270                                             "mca:rmaps[%d] binding not given - using byhwthread", __LINE__);
 271                             OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_HWTHREAD);
 272                         } else {
 273                             /* for performance, bind to core */
 274                             opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 275                                             "mca:rmaps[%d] binding not given - using bycore", __LINE__);
 276                             OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_CORE);
 277                         }
 278                     } else {
 279                         if (NULL != hwloc_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE, 0)) {
 280                             opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 281                                                 "mca:rmaps[%d] binding not given - using bynuma", __LINE__);
 282                             OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NUMA);
 283                         } else if (NULL != hwloc_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_SOCKET, 0)) {
 284                             opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 285                                                 "mca:rmaps[%d] binding not given and no NUMA - using bysocket", __LINE__);
 286                             OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_SOCKET);
 287                         } else {
 288                             /* if we have neither, then just don't bind */
 289                             opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 290                                                 "mca:rmaps[%d] binding not given and no NUMA or sockets - not binding", __LINE__);
 291                             OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE);
 292                         }
 293                     }
 294                 }
 295             } else if (nprocs <= 2) {
 296                 if (opal_hwloc_use_hwthreads_as_cpus) {
 297                     /* if we are using hwthread cpus, then bind to those */
 298                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 299                                     "mca:rmaps[%d] binding not given - using byhwthread", __LINE__);
 300                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_HWTHREAD);
 301                 } else {
 302                     /* for performance, bind to core */
 303                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 304                                     "mca:rmaps[%d] binding not given - using bycore", __LINE__);
 305                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_CORE);
 306                 }
 307             } else {
 308                 /* for performance, bind to NUMA, if available */
 309                 if (NULL != hwloc_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE, 0)) {
 310                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 311                                         "mca:rmaps[%d] binding not given - using bynuma", __LINE__);
 312                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NUMA);
 313                 } else if (NULL != hwloc_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_SOCKET, 0)) {
 314                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 315                                         "mca:rmaps[%d] binding not given and no NUMA - using bysocket", __LINE__);
 316                     OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_SOCKET);
 317                 } else {
 318                     /* if we have neither, then just don't bind */
 319                     opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
 320                                         "mca:rmaps[%d] binding not given and no NUMA or sockets - not binding", __LINE__);
 321                     OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE);
 322                 }
 323             }
 324             if (OPAL_BIND_OVERLOAD_ALLOWED(opal_hwloc_binding_policy)) {
 325                 jdata->map->binding |= OPAL_BIND_ALLOW_OVERLOAD;
 326             }
 327         }
 328     }
 329 
 330     /* if we are not going to launch, then we need to set any
 331      * undefined topologies to match our own so the mapper
 332      * can operate
 333      */
 334     if (orte_do_not_launch) {
 335         orte_node_t *node;
 336         orte_topology_t *t0;
 337         int i;
 338         if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0))) {
 339             ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
 340             OBJ_RELEASE(caddy);
 341             ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 342             return;
 343         }
 344         t0 = node->topology;
 345         for (i=1; i < orte_node_pool->size; i++) {
 346             if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, i))) {
 347                 continue;
 348             }
 349             if (NULL == node->topology) {
 350                 node->topology = t0;
 351             }
 352         }
 353     }
 354 
 355     /* cycle thru the available mappers until one agrees to map
 356      * the job
 357      */
 358     did_map = false;
 359     if (1 == opal_list_get_size(&orte_rmaps_base.selected_modules)) {
 360         /* forced selection */
 361         mod = (orte_rmaps_base_selected_module_t*)opal_list_get_first(&orte_rmaps_base.selected_modules);
 362         jdata->map->req_mapper = strdup(mod->component->mca_component_name);
 363     }
 364     OPAL_LIST_FOREACH(mod, &orte_rmaps_base.selected_modules, orte_rmaps_base_selected_module_t) {
 365         if (ORTE_SUCCESS == (rc = mod->module->map_job(jdata)) ||
 366             ORTE_ERR_RESOURCE_BUSY == rc) {
 367             did_map = true;
 368             break;
 369         }
 370         /* mappers return "next option" if they didn't attempt to
 371          * map the job. anything else is a true error.
 372          */
 373         if (ORTE_ERR_TAKE_NEXT_OPTION != rc) {
 374             ORTE_ERROR_LOG(rc);
 375             ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 376             goto cleanup;
 377         }
 378     }
 379 
 380     if (did_map && ORTE_ERR_RESOURCE_BUSY == rc) {
 381         /* the map was done but nothing could be mapped
 382          * for launch as all the resources were busy
 383          */
 384         orte_show_help("help-orte-rmaps-base.txt", "cannot-launch", true);
 385         ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 386         goto cleanup;
 387     }
 388 
 389     /* if we get here without doing the map, or with zero procs in
 390      * the map, then that's an error
 391      */
 392     if (!did_map || 0 == jdata->num_procs || 0 == jdata->map->num_nodes) {
 393         orte_show_help("help-orte-rmaps-base.txt", "failed-map", true,
 394                        did_map ? "mapped" : "unmapped",
 395                        jdata->num_procs, jdata->map->num_nodes);
 396         ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 397         goto cleanup;
 398     }
 399 
 400     /* if any node is oversubscribed, then check to see if a binding
 401      * directive was given - if not, then we want to clear the default
 402      * binding policy so we don't attempt to bind */
 403     if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_OVERSUBSCRIBED)) {
 404         if (!OPAL_BINDING_POLICY_IS_SET(jdata->map->binding)) {
 405             /* clear any default binding policy we might have set */
 406             OPAL_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE);
 407         }
 408     }
 409 
 410     if (orte_do_not_launch) {
 411         /* compute the ranks and add the proc objects
 412          * to the jdata->procs array */
 413         if (ORTE_SUCCESS != (rc = orte_rmaps_base_compute_vpids(jdata))) {
 414             ORTE_ERROR_LOG(rc);
 415             ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 416             goto cleanup;
 417         }
 418         /* compute and save local ranks */
 419         if (ORTE_SUCCESS != (rc = orte_rmaps_base_compute_local_ranks(jdata))) {
 420             ORTE_ERROR_LOG(rc);
 421             ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 422             goto cleanup;
 423         }
 424         /* compute and save location assignments */
 425         if (ORTE_SUCCESS != (rc = orte_rmaps_base_assign_locations(jdata))) {
 426             ORTE_ERROR_LOG(rc);
 427             ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 428             goto cleanup;
 429         }
 430         /* compute and save bindings */
 431         if (ORTE_SUCCESS != (rc = orte_rmaps_base_compute_bindings(jdata))) {
 432             ORTE_ERROR_LOG(rc);
 433             ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 434             goto cleanup;
 435         }
 436     } else if (!orte_get_attribute(&jdata->attributes, ORTE_JOB_FULLY_DESCRIBED, NULL, OPAL_BOOL)) {
 437         /* compute and save location assignments */
 438         if (ORTE_SUCCESS != (rc = orte_rmaps_base_assign_locations(jdata))) {
 439             ORTE_ERROR_LOG(rc);
 440             ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 441             goto cleanup;
 442         }
 443     } else {
 444         /* compute and save local ranks */
 445         if (ORTE_SUCCESS != (rc = orte_rmaps_base_compute_local_ranks(jdata))) {
 446             ORTE_ERROR_LOG(rc);
 447             ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 448             goto cleanup;
 449         }
 450 
 451         /* compute and save bindings */
 452         if (ORTE_SUCCESS != (rc = orte_rmaps_base_compute_bindings(jdata))) {
 453             ORTE_ERROR_LOG(rc);
 454             ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_FAILED);
 455             goto cleanup;
 456         }
 457     }
 458 
 459     /* set the offset so shared memory components can potentially
 460      * connect to any spawned jobs
 461      */
 462     jdata->offset = orte_total_procs;
 463     /* track the total number of procs launched by us */
 464     orte_total_procs += jdata->num_procs;
 465 
 466     /* if it is a dynamic spawn, save the bookmark on the parent's job too */
 467     if (ORTE_JOBID_INVALID != jdata->originator.jobid) {
 468         if (NULL != (parent = orte_get_job_data_object(jdata->originator.jobid))) {
 469             parent->bookmark = jdata->bookmark;
 470         }
 471     }
 472 
 473     if (orte_do_not_launch) {
 474         /* display the devel map */
 475         orte_rmaps_base_display_map(jdata);
 476     }
 477 
 478     /* set the job state to the next position */
 479     ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_COMPLETE);
 480 
 481   cleanup:
 482       /* reset any node map flags we used so the next job will start clean */
 483        for (i=0; i < jdata->map->nodes->size; i++) {
 484            if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, i))) {
 485                ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED);
 486            }
 487        }
 488 
 489     /* cleanup */
 490     OBJ_RELEASE(caddy);
 491 }
 492 
 493 void orte_rmaps_base_display_map(orte_job_t *jdata)
 494 {
 495     /* ignore daemon job */
 496     char *output=NULL;
 497     int i, j, cnt;
 498     orte_node_t *node;
 499     orte_proc_t *proc;
 500     char tmp1[1024];
 501     hwloc_obj_t bd=NULL;;
 502     opal_hwloc_locality_t locality;
 503     orte_proc_t *p0;
 504     char *p0bitmap, *procbitmap;
 505 
 506     if (orte_display_diffable_output) {
 507         /* intended solely to test mapping methods, this output
 508          * can become quite long when testing at scale. Rather
 509          * than enduring all the malloc/free's required to
 510          * create an arbitrary-length string, custom-generate
 511          * the output a line at a time here
 512          */
 513         /* display just the procs in a diffable format */
 514         opal_output(orte_clean_output, "<map>\n");
 515         fflush(stderr);
 516         /* loop through nodes */
 517         cnt = 0;
 518         for (i=0; i < jdata->map->nodes->size; i++) {
 519             if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, i))) {
 520                 continue;
 521             }
 522             opal_output(orte_clean_output, "\t<host num=%d>", cnt);
 523             fflush(stderr);
 524             cnt++;
 525             for (j=0; j < node->procs->size; j++) {
 526                 if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
 527                     continue;
 528                 }
 529                 memset(tmp1, 0, sizeof(tmp1));
 530                 if (orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_BOUND, (void**)&bd, OPAL_PTR)) {
 531                     if (NULL == bd) {
 532                         (void)opal_string_copy(tmp1, "UNBOUND", sizeof(tmp1));
 533                     } else {
 534                         if (OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2mapstr(tmp1, sizeof(tmp1), node->topology->topo, bd->cpuset)) {
 535                             (void)opal_string_copy(tmp1, "UNBOUND", sizeof(tmp1));
 536                         }
 537                     }
 538                 } else {
 539                     (void)opal_string_copy(tmp1, "UNBOUND", sizeof(tmp1));
 540                 }
 541                 opal_output(orte_clean_output, "\t\t<process rank=%s app_idx=%ld local_rank=%lu node_rank=%lu binding=%s>",
 542                             ORTE_VPID_PRINT(proc->name.vpid),  (long)proc->app_idx,
 543                             (unsigned long)proc->local_rank,
 544                             (unsigned long)proc->node_rank, tmp1);
 545             }
 546             opal_output(orte_clean_output, "\t</host>");
 547             fflush(stderr);
 548         }
 549 
 550          /* test locality - for the first node, print the locality of each proc relative to the first one */
 551         node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, 0);
 552         p0 = (orte_proc_t*)opal_pointer_array_get_item(node->procs, 0);
 553         p0bitmap = NULL;
 554         if (orte_get_attribute(&p0->attributes, ORTE_PROC_CPU_BITMAP, (void**)&p0bitmap, OPAL_STRING) &&
 555             NULL != p0bitmap) {
 556             opal_output(orte_clean_output, "\t<locality>");
 557             for (j=1; j < node->procs->size; j++) {
 558                 if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
 559                     continue;
 560                 }
 561                 procbitmap = NULL;
 562                 if (orte_get_attribute(&proc->attributes, ORTE_PROC_CPU_BITMAP, (void**)&procbitmap, OPAL_STRING) &&
 563                     NULL != procbitmap) {
 564                     locality = opal_hwloc_base_get_relative_locality(node->topology->topo,
 565                                                                      p0bitmap,
 566                                                                      procbitmap);
 567                     opal_output(orte_clean_output, "\t\t<rank=%s rank=%s locality=%s>",
 568                                 ORTE_VPID_PRINT(p0->name.vpid),
 569                                 ORTE_VPID_PRINT(proc->name.vpid),
 570                                 opal_hwloc_base_print_locality(locality));
 571                 }
 572             }
 573             opal_output(orte_clean_output, "\t</locality>\n</map>");
 574             fflush(stderr);
 575             if (NULL != p0bitmap) {
 576                 free(p0bitmap);
 577             }
 578             if (NULL != procbitmap) {
 579                 free(procbitmap);
 580             }
 581         }
 582     } else {
 583         opal_output(orte_clean_output, " Data for JOB %s offset %s Total slots allocated %lu",
 584                     ORTE_JOBID_PRINT(jdata->jobid), ORTE_VPID_PRINT(jdata->offset),
 585                     (long unsigned)jdata->total_slots_alloc);
 586         opal_dss.print(&output, NULL, jdata->map, ORTE_JOB_MAP);
 587         if (orte_xml_output) {
 588             fprintf(orte_xml_fp, "%s\n", output);
 589             fflush(orte_xml_fp);
 590         } else {
 591             opal_output(orte_clean_output, "%s", output);
 592         }
 593         free(output);
 594     }
 595 }
/* [<][>][^][v][top][bottom][index][help] */
root/orte/mca/rmaps/base/rmaps_base_map_job.c

DEFINITIONS
