root/orte/mca/rmaps/base/rmaps_base_support_fns.c

DEFINITIONS

1. orte_rmaps_base_filter_nodes
2. orte_rmaps_base_get_target_nodes
3. orte_rmaps_base_setup_proc
4. orte_rmaps_base_get_starting_point

   1 /*
   2  * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
   3  *                         University Research and Technology
   4  *                         Corporation.  All rights reserved.
   5  * Copyright (c) 2004-2011 The University of Tennessee and The University
   6  *                         of Tennessee Research Foundation.  All rights
   7  *                         reserved.
   8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   9  *                         University of Stuttgart.  All rights reserved.
  10  * Copyright (c) 2004-2005 The Regents of the University of California.
  11  *                         All rights reserved.
  12  * Copyright (c) 2011      Cisco Systems, Inc.  All rights reserved.
  13  * Copyright (c) 2011-2012 Los Alamos National Security, LLC.
  14  *                         All rights reserved.
  15  * Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
  16  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
  17  * $COPYRIGHT$
  18  *
  19  * Additional copyrights may follow
  20  *
  21  * $HEADER$
  22  */
  23 
  24 #include "orte_config.h"
  25 #include "orte/constants.h"
  26 
  27 #include <sys/types.h>
  28 #ifdef HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif  /* HAVE_UNISTD_H */
  31 #include <string.h>
  32 
  33 #include "opal/util/argv.h"
  34 #include "opal/util/if.h"
  35 #include "opal/util/output.h"
  36 #include "orte/mca/mca.h"
  37 #include "opal/mca/base/base.h"
  38 #include "opal/mca/hwloc/base/base.h"
  39 #include "opal/threads/tsd.h"
  40 
  41 #include "orte/types.h"
  42 #include "orte/util/show_help.h"
  43 #include "orte/util/name_fns.h"
  44 #include "orte/runtime/orte_globals.h"
  45 #include "orte/util/hostfile/hostfile.h"
  46 #include "orte/util/dash_host/dash_host.h"
  47 #include "orte/mca/errmgr/errmgr.h"
  48 #include "orte/mca/ess/ess.h"
  49 #include "orte/runtime/data_type_support/orte_dt_support.h"
  50 
  51 #include "orte/mca/rmaps/base/rmaps_private.h"
  52 #include "orte/mca/rmaps/base/base.h"
  53 
  54 int orte_rmaps_base_filter_nodes(orte_app_context_t *app,
  55                                  opal_list_t *nodes, bool remove)
  56 {
  57     int rc=ORTE_ERR_TAKE_NEXT_OPTION;
  58     char *hosts;
  59 
  60     /* did the app_context contain a hostfile? */
  61     if (orte_get_attribute(&app->attributes, ORTE_APP_HOSTFILE, (void**)&hosts, OPAL_STRING)) {
  62         /* yes - filter the node list through the file, removing
  63          * any nodes not found in the file
  64          */
  65         if (ORTE_SUCCESS != (rc = orte_util_filter_hostfile_nodes(nodes, hosts, remove))) {
  66             ORTE_ERROR_LOG(rc);
  67             free(hosts);
  68             return rc;
  69         }
  70         /** check that anything is here */
  71         if (0 == opal_list_get_size(nodes)) {
  72             orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:no-mapped-node",
  73                            true, app->app, "-hostfile", hosts);
  74             free(hosts);
  75             return ORTE_ERR_SILENT;
  76         }
  77         free(hosts);
  78     }
  79     /* did the app_context contain an add-hostfile? */
  80     if (orte_get_attribute(&app->attributes, ORTE_APP_ADD_HOSTFILE, (void**)&hosts, OPAL_STRING)) {
  81         /* yes - filter the node list through the file, removing
  82          * any nodes not found in the file
  83          */
  84         if (ORTE_SUCCESS != (rc = orte_util_filter_hostfile_nodes(nodes, hosts, remove))) {
  85             free(hosts);
  86             ORTE_ERROR_LOG(rc);
  87             return rc;
  88         }
  89         /** check that anything is here */
  90         if (0 == opal_list_get_size(nodes)) {
  91             orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:no-mapped-node",
  92                            true, app->app, "-add-hostfile", hosts);
  93             free(hosts);
  94             return ORTE_ERR_SILENT;
  95         }
  96         free(hosts);
  97     }
  98     /* now filter the list through any -host specification */
  99     if (!orte_soft_locations &&
 100         orte_get_attribute(&app->attributes, ORTE_APP_DASH_HOST, (void**)&hosts, OPAL_STRING)) {
 101         if (ORTE_SUCCESS != (rc = orte_util_filter_dash_host_nodes(nodes, hosts, remove))) {
 102             ORTE_ERROR_LOG(rc);
 103             free(hosts);
 104             return rc;
 105         }
 106         /** check that anything is left! */
 107         if (0 == opal_list_get_size(nodes)) {
 108             orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:no-mapped-node",
 109                            true, app->app, "-host", hosts);
 110             free(hosts);
 111             return ORTE_ERR_SILENT;
 112         }
 113         free(hosts);
 114     }
 115     /* now filter the list through any add-host specification */
 116     if (orte_get_attribute(&app->attributes, ORTE_APP_ADD_HOST, (void**)&hosts, OPAL_STRING)) {
 117         if (ORTE_SUCCESS != (rc = orte_util_filter_dash_host_nodes(nodes, hosts, remove))) {
 118             ORTE_ERROR_LOG(rc);
 119             free(hosts);
 120             return rc;
 121         }
 122         /** check that anything is left! */
 123         if (0 == opal_list_get_size(nodes)) {
 124             orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:no-mapped-node",
 125                            true, app->app, "-add-host", hosts);
 126             free(hosts);
 127             return ORTE_ERR_SILENT;
 128         }
 129         free(hosts);
 130     }
 131 
 132     return rc;
 133 }
 134 
 135 
 136 /*
 137  * Query the registry for all nodes allocated to a specified app_context
 138  */
 139 int orte_rmaps_base_get_target_nodes(opal_list_t *allocated_nodes, orte_std_cntr_t *total_num_slots,
 140                                      orte_app_context_t *app, orte_mapping_policy_t policy,
 141                                      bool initial_map, bool silent)
 142 {
 143     opal_list_item_t *item;
 144     orte_node_t *node, *nd, *nptr, *next;
 145     orte_std_cntr_t num_slots;
 146     orte_std_cntr_t i;
 147     int rc;
 148     orte_job_t *daemons;
 149     bool novm;
 150     opal_list_t nodes;
 151     char *hosts = NULL;
 152 
 153     /** set default answer */
 154     *total_num_slots = 0;
 155 
 156     /* get the daemon job object */
 157     daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid);
 158     /* see if we have a vm or not */
 159     novm = orte_get_attribute(&daemons->attributes, ORTE_JOB_NO_VM, NULL, OPAL_BOOL);
 160 
 161     /* if this is NOT a managed allocation, then we use the nodes
 162      * that were specified for this app - there is no need to collect
 163      * all available nodes and "filter" them
 164      */
 165     if (!orte_managed_allocation) {
 166         OBJ_CONSTRUCT(&nodes, opal_list_t);
 167         /* if the app provided a dash-host, and we are not treating
 168          * them as requested or "soft" locations, then use those nodes
 169          */
 170         hosts = NULL;
 171         if (!orte_soft_locations &&
 172             orte_get_attribute(&app->attributes, ORTE_APP_DASH_HOST, (void**)&hosts, OPAL_STRING)) {
 173             OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 174                                  "%s using dash_host %s",
 175                                  ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), hosts));
 176             if (ORTE_SUCCESS != (rc = orte_util_add_dash_host_nodes(&nodes, hosts, false))) {
 177                 ORTE_ERROR_LOG(rc);
 178                 free(hosts);
 179                 return rc;
 180             }
 181             free(hosts);
 182         } else if (orte_get_attribute(&app->attributes, ORTE_APP_HOSTFILE, (void**)&hosts, OPAL_STRING)) {
 183             /* otherwise, if the app provided a hostfile, then use that */
 184             OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 185                                  "%s using hostfile %s",
 186                                  ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), hosts));
 187             if (ORTE_SUCCESS != (rc = orte_util_add_hostfile_nodes(&nodes, hosts))) {
 188                 free(hosts);
 189                 ORTE_ERROR_LOG(rc);
 190                 return rc;
 191             }
 192             free(hosts);
 193         } else {
 194             /* if nothing else was specified by the app, then use all known nodes, which
 195              * will include ourselves
 196              */
 197             OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 198                                  "%s using known nodes",
 199                                  ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
 200             goto addknown;
 201         }
 202         /** if we still don't have anything */
 203         if (0 == opal_list_get_size(&nodes)) {
 204             if (!silent) {
 205                 orte_show_help("help-orte-rmaps-base.txt",
 206                                "orte-rmaps-base:no-available-resources",
 207                                true);
 208             }
 209             OBJ_DESTRUCT(&nodes);
 210             return ORTE_ERR_SILENT;
 211         }
 212         /* find the nodes in our node array and assemble them
 213          * in list order as that is what the user specified
 214          */
 215         OPAL_LIST_FOREACH_SAFE(nptr, next, &nodes, orte_node_t) {
 216             for (i=0; i < orte_node_pool->size; i++) {
 217                 if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, i))) {
 218                     continue;
 219                 }
 220                 /* ignore nodes that are non-usable */
 221                 if (ORTE_FLAG_TEST(node, ORTE_NODE_NON_USABLE)) {
 222                     continue;
 223                 }
 224                 if (0 != strcmp(node->name, nptr->name)) {
 225                     OPAL_OUTPUT_VERBOSE((10, orte_rmaps_base_framework.framework_output,
 226                                          "NODE %s DOESNT MATCH NODE %s",
 227                                          node->name, nptr->name));
 228                     continue;
 229                 }
 230                 /* ignore nodes that are marked as do-not-use for this mapping */
 231                 if (ORTE_NODE_STATE_DO_NOT_USE == node->state) {
 232                     OPAL_OUTPUT_VERBOSE((10, orte_rmaps_base_framework.framework_output,
 233                                          "NODE %s IS MARKED NO_USE", node->name));
 234                     /* reset the state so it can be used another time */
 235                     node->state = ORTE_NODE_STATE_UP;
 236                     continue;
 237                 }
 238                 if (ORTE_NODE_STATE_DOWN == node->state) {
 239                     OPAL_OUTPUT_VERBOSE((10, orte_rmaps_base_framework.framework_output,
 240                                          "NODE %s IS DOWN", node->name));
 241                     continue;
 242                 }
 243                 if (ORTE_NODE_STATE_NOT_INCLUDED == node->state) {
 244                     OPAL_OUTPUT_VERBOSE((10, orte_rmaps_base_framework.framework_output,
 245                                          "NODE %s IS MARKED NO_INCLUDE", node->name));
 246                     /* not to be used */
 247                     continue;
 248                 }
 249                 /* if this node wasn't included in the vm (e.g., by -host), ignore it,
 250                  * unless we are mapping prior to launching the vm
 251                  */
 252                 if (NULL == node->daemon && !novm) {
 253                     OPAL_OUTPUT_VERBOSE((10, orte_rmaps_base_framework.framework_output,
 254                                          "NODE %s HAS NO DAEMON", node->name));
 255                     continue;
 256                 }
 257                 /* retain a copy for our use in case the item gets
 258                  * destructed along the way
 259                  */
 260                 OBJ_RETAIN(node);
 261                 if (initial_map) {
 262                     /* if this is the first app_context we
 263                      * are getting for an initial map of a job,
 264                      * then mark all nodes as unmapped
 265                      */
 266                     ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED);
 267                 }
 268                 /* the list is ordered as per user direction using -host
 269                  * or the listing in -hostfile - preserve that ordering */
 270                 opal_list_append(allocated_nodes, &node->super);
 271                 break;
 272             }
 273             /* remove the item from the list as we have allocated it */
 274             opal_list_remove_item(&nodes, (opal_list_item_t*)nptr);
 275             OBJ_RELEASE(nptr);
 276         }
 277         OBJ_DESTRUCT(&nodes);
 278         /* now prune for usage and compute total slots */
 279         goto complete;
 280     }
 281 
 282   addknown:
 283     /* add everything in the node pool that can be used - add them
 284      * in daemon order, which may be different than the order in the
 285      * node pool. Since an empty list is passed into us, the list at
 286      * this point either has the HNP node or nothing, and the HNP
 287      * node obviously has a daemon on it (us!)
 288      */
 289     if (0 == opal_list_get_size(allocated_nodes)) {
 290         /* the list is empty - if the HNP is allocated, then add it */
 291         if (orte_hnp_is_allocated) {
 292             nd = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
 293             OBJ_RETAIN(nd);
 294             opal_list_append(allocated_nodes, &nd->super);
 295         } else {
 296             nd = NULL;
 297         }
 298     } else {
 299         nd = (orte_node_t*)opal_list_get_last(allocated_nodes);
 300     }
 301     for (i=1; i < orte_node_pool->size; i++) {
 302         if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, i))) {
 303             /* ignore nodes that are non-usable */
 304             if (ORTE_FLAG_TEST(node, ORTE_NODE_NON_USABLE)) {
 305                 continue;
 306             }
 307             /* ignore nodes that are marked as do-not-use for this mapping */
 308             if (ORTE_NODE_STATE_DO_NOT_USE == node->state) {
 309                 OPAL_OUTPUT_VERBOSE((10, orte_rmaps_base_framework.framework_output,
 310                                      "NODE %s IS MARKED NO_USE", node->name));
 311                 /* reset the state so it can be used another time */
 312                 node->state = ORTE_NODE_STATE_UP;
 313                 continue;
 314             }
 315             if (ORTE_NODE_STATE_DOWN == node->state) {
 316                 OPAL_OUTPUT_VERBOSE((10, orte_rmaps_base_framework.framework_output,
 317                                      "NODE %s IS MARKED DOWN", node->name));
 318                 continue;
 319             }
 320             if (ORTE_NODE_STATE_NOT_INCLUDED == node->state) {
 321                 OPAL_OUTPUT_VERBOSE((10, orte_rmaps_base_framework.framework_output,
 322                                      "NODE %s IS MARKED NO_INCLUDE", node->name));
 323                 /* not to be used */
 324                 continue;
 325             }
 326             /* if this node wasn't included in the vm (e.g., by -host), ignore it,
 327              * unless we are mapping prior to launching the vm
 328              */
 329             if (NULL == node->daemon && !novm) {
 330                 OPAL_OUTPUT_VERBOSE((10, orte_rmaps_base_framework.framework_output,
 331                                      "NODE %s HAS NO DAEMON", node->name));
 332                 continue;
 333             }
 334             /* retain a copy for our use in case the item gets
 335              * destructed along the way
 336              */
 337             OBJ_RETAIN(node);
 338             if (initial_map) {
 339                 /* if this is the first app_context we
 340                  * are getting for an initial map of a job,
 341                  * then mark all nodes as unmapped
 342                  */
 343                 ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED);
 344             }
 345             if (NULL == nd || NULL == nd->daemon ||
 346                 NULL == node->daemon ||
 347                 nd->daemon->name.vpid < node->daemon->name.vpid) {
 348                 /* just append to end */
 349                 opal_list_append(allocated_nodes, &node->super);
 350                 nd = node;
 351             } else {
 352                 /* starting from end, put this node in daemon-vpid order */
 353                 while (node->daemon->name.vpid < nd->daemon->name.vpid) {
 354                     if (opal_list_get_begin(allocated_nodes) == opal_list_get_prev(&nd->super)) {
 355                         /* insert at beginning */
 356                         opal_list_prepend(allocated_nodes, &node->super);
 357                         goto moveon;
 358                     }
 359                     nd = (orte_node_t*)opal_list_get_prev(&nd->super);
 360                 }
 361                 item = opal_list_get_next(&nd->super);
 362                 if (item == opal_list_get_end(allocated_nodes)) {
 363                     /* we are at the end - just append */
 364                     opal_list_append(allocated_nodes, &node->super);
 365                 } else {
 366                     nd = (orte_node_t*)item;
 367                     opal_list_insert_pos(allocated_nodes, item, &node->super);
 368                 }
 369             moveon:
 370                 /* reset us back to the end for the next node */
 371                 nd = (orte_node_t*)opal_list_get_last(allocated_nodes);
 372             }
 373         }
 374     }
 375 
 376     OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 377                          "%s Starting with %d nodes in list",
 378                          ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
 379                          (int)opal_list_get_size(allocated_nodes)));
 380 
 381     /** check that anything is here */
 382     if (0 == opal_list_get_size(allocated_nodes)) {
 383         if (!silent) {
 384             orte_show_help("help-orte-rmaps-base.txt",
 385                            "orte-rmaps-base:no-available-resources",
 386                            true);
 387         }
 388         return ORTE_ERR_SILENT;
 389     }
 390 
 391     /* filter the nodes thru any hostfile and dash-host options */
 392     OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 393                          "%s Filtering thru apps",
 394                          ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
 395 
 396     if (ORTE_SUCCESS != (rc = orte_rmaps_base_filter_nodes(app, allocated_nodes, true))
 397         && ORTE_ERR_TAKE_NEXT_OPTION != rc) {
 398         ORTE_ERROR_LOG(rc);
 399         return rc;
 400     }
 401     OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 402                          "%s Retained %d nodes in list",
 403                          ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
 404                          (int)opal_list_get_size(allocated_nodes)));
 405 
 406   complete:
 407     num_slots = 0;
 408     /* remove all nodes that are already at max usage, and
 409      * compute the total number of allocated slots while
 410      * we do so - can ignore this if we are mapping debugger
 411      * daemons as they do not count against the allocation */
 412     if (ORTE_MAPPING_DEBUGGER & ORTE_GET_MAPPING_DIRECTIVE(policy)) {
 413         num_slots = opal_list_get_size(allocated_nodes);    // tell the mapper there is one slot/node for debuggers
 414     } else {
 415         item  = opal_list_get_first(allocated_nodes);
 416         OPAL_LIST_FOREACH_SAFE(node, next, allocated_nodes, orte_node_t) {
 417             /* if the hnp was not allocated, or flagged not to be used,
 418              * then remove it here */
 419             if (!orte_hnp_is_allocated || (ORTE_GET_MAPPING_DIRECTIVE(policy) & ORTE_MAPPING_NO_USE_LOCAL)) {
 420                 if (0 == node->index) {
 421                     opal_list_remove_item(allocated_nodes, &node->super);
 422                     OBJ_RELEASE(node);  /* "un-retain" it */
 423                     continue;
 424                 }
 425             }
 426             /** check to see if this node is fully used - remove if so */
 427             if (0 != node->slots_max && node->slots_inuse > node->slots_max) {
 428                 OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 429                                      "%s Removing node %s: max %d inuse %d",
 430                                      ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
 431                                      node->name, node->slots_max, node->slots_inuse));
 432                 opal_list_remove_item(allocated_nodes, &node->super);
 433                 OBJ_RELEASE(node);  /* "un-retain" it */
 434                 continue;
 435             }
 436             if (node->slots <= node->slots_inuse &&
 437                        (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(policy))) {
 438                 /* remove the node as fully used */
 439                 OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 440                                      "%s Removing node %s slots %d inuse %d",
 441                                      ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
 442                                      node->name, node->slots, node->slots_inuse));
 443                 opal_list_remove_item(allocated_nodes, &node->super);
 444                 OBJ_RELEASE(node);  /* "un-retain" it */
 445                 continue;
 446             }
 447             if (node->slots > node->slots_inuse) {
 448                 orte_std_cntr_t s;
 449                 /* check for any -host allocations */
 450                 if (orte_get_attribute(&app->attributes, ORTE_APP_DASH_HOST, (void**)&hosts, OPAL_STRING)) {
 451                     s = orte_util_dash_host_compute_slots(node, hosts);
 452                 } else {
 453                     s = node->slots - node->slots_inuse;
 454                 }
 455                 /* add the available slots */
 456                 OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 457                                      "%s node %s has %d slots available",
 458                                      ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
 459                                      node->name, s));
 460                 num_slots += s;
 461                 continue;
 462             }
 463             if (!(ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(policy))) {
 464                 /* nothing needed to do here - we don't add slots to the
 465                  * count as we don't have any available. Just let the mapper
 466                  * do what it needs to do to meet the request
 467                  */
 468                 OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 469                                      "%s node %s is fully used, but available for oversubscription",
 470                                      ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
 471                                      node->name));
 472             } else {
 473                 /* if we cannot use it, remove it from list */
 474                 opal_list_remove_item(allocated_nodes, &node->super);
 475                 OBJ_RELEASE(node);  /* "un-retain" it */
 476             }
 477         }
 478     }
 479 
 480     /* Sanity check to make sure we have resources available */
 481     if (0 == opal_list_get_size(allocated_nodes)) {
 482         if (silent) {
 483             /* let the caller know that the resources exist,
 484              * but are currently busy
 485              */
 486             return ORTE_ERR_RESOURCE_BUSY;
 487         } else {
 488             orte_show_help("help-orte-rmaps-base.txt",
 489                            "orte-rmaps-base:all-available-resources-used", true);
 490             return ORTE_ERR_SILENT;
 491         }
 492     }
 493 
 494     /* pass back the total number of available slots */
 495     *total_num_slots = num_slots;
 496 
 497     if (4 < opal_output_get_verbosity(orte_rmaps_base_framework.framework_output)) {
 498         opal_output(0, "AVAILABLE NODES FOR MAPPING:");
 499         for (item = opal_list_get_first(allocated_nodes);
 500              item != opal_list_get_end(allocated_nodes);
 501              item = opal_list_get_next(item)) {
 502             node = (orte_node_t*)item;
 503             opal_output(0, "    node: %s daemon: %s", node->name,
 504                         (NULL == node->daemon) ? "NULL" : ORTE_VPID_PRINT(node->daemon->name.vpid));
 505         }
 506     }
 507 
 508     return ORTE_SUCCESS;
 509 }
 510 
 511 orte_proc_t* orte_rmaps_base_setup_proc(orte_job_t *jdata,
 512                                         orte_node_t *node,
 513                                         orte_app_idx_t idx)
 514 {
 515     orte_proc_t *proc;
 516     int rc;
 517 
 518     proc = OBJ_NEW(orte_proc_t);
 519     /* set the jobid */
 520     proc->name.jobid = jdata->jobid;
 521     /* flag the proc as ready for launch */
 522     proc->state = ORTE_PROC_STATE_INIT;
 523     proc->app_idx = idx;
 524     /* mark the proc as UPDATED so it will be included in the launch */
 525     ORTE_FLAG_SET(proc, ORTE_PROC_FLAG_UPDATED);
 526     if (NULL == node->daemon) {
 527         proc->parent = ORTE_VPID_INVALID;
 528     } else {
 529         proc->parent = node->daemon->name.vpid;
 530     }
 531 
 532     OBJ_RETAIN(node);  /* maintain accounting on object */
 533     proc->node = node;
 534     /* if this is a debugger job, then it doesn't count against
 535      * available slots - otherwise, it does */
 536     if (!ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_DEBUGGER_DAEMON)) {
 537         node->num_procs++;
 538         ++node->slots_inuse;
 539     }
 540     if (0 > (rc = opal_pointer_array_add(node->procs, (void*)proc))) {
 541         ORTE_ERROR_LOG(rc);
 542         OBJ_RELEASE(proc);
 543         return NULL;
 544     }
 545     /* retain the proc struct so that we correctly track its release */
 546     OBJ_RETAIN(proc);
 547 
 548     return proc;
 549 }
 550 
 551 /*
 552  * determine the proper starting point for the next mapping operation
 553  */
 554 orte_node_t* orte_rmaps_base_get_starting_point(opal_list_t *node_list,
 555                                                 orte_job_t *jdata)
 556 {
 557     opal_list_item_t *item, *cur_node_item;
 558     orte_node_t *node, *nd1, *ndmin;
 559     int overload;
 560 
 561     /* if a bookmark exists from some prior mapping, set us to start there */
 562     if (NULL != jdata->bookmark) {
 563         cur_node_item = NULL;
 564         /* find this node on the list */
 565         for (item = opal_list_get_first(node_list);
 566              item != opal_list_get_end(node_list);
 567              item = opal_list_get_next(item)) {
 568             node = (orte_node_t*)item;
 569 
 570             if (node->index == jdata->bookmark->index) {
 571                 cur_node_item = item;
 572                 break;
 573             }
 574         }
 575         /* see if we found it - if not, just start at the beginning */
 576         if (NULL == cur_node_item) {
 577             cur_node_item = opal_list_get_first(node_list);
 578         }
 579     } else {
 580         /* if no bookmark, then just start at the beginning of the list */
 581         cur_node_item = opal_list_get_first(node_list);
 582     }
 583 
 584     OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 585                          "%s Starting bookmark at node %s",
 586                          ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
 587                          ((orte_node_t*)cur_node_item)->name));
 588 
 589     /* is this node fully subscribed? If so, then the first
 590      * proc we assign will oversubscribe it, so let's look
 591      * for another candidate
 592      */
 593     node = (orte_node_t*)cur_node_item;
 594     ndmin = node;
 595     overload = ndmin->slots_inuse - ndmin->slots;
 596     if (node->slots_inuse >= node->slots) {
 597         /* work down the list - is there another node that
 598          * would not be oversubscribed?
 599          */
 600         if (cur_node_item != opal_list_get_last(node_list)) {
 601             item = opal_list_get_next(cur_node_item);
 602         } else {
 603             item = opal_list_get_first(node_list);
 604         }
 605         nd1 = NULL;
 606         while (item != cur_node_item) {
 607             nd1 = (orte_node_t*)item;
 608             if (nd1->slots_inuse < nd1->slots) {
 609                 /* this node is not oversubscribed! use it! */
 610                 cur_node_item = item;
 611                 goto process;
 612             }
 613             /* this one was also oversubscribed, keep track of the
 614              * node that has the least usage - if we can't
 615              * find anyone who isn't fully utilized, we will
 616              * start with the least used node
 617              */
 618             if (overload >= (nd1->slots_inuse - nd1->slots)) {
 619                 ndmin = nd1;
 620                 overload = ndmin->slots_inuse - ndmin->slots;
 621             }
 622             if (item == opal_list_get_last(node_list)) {
 623                 item = opal_list_get_first(node_list);
 624             } else {
 625                 item= opal_list_get_next(item);
 626             }
 627         }
 628         /* if we get here, then we cycled all the way around the
 629          * list without finding a better answer - just use the node
 630          * that is minimally overloaded if it is better than
 631          * what we already have
 632          */
 633         if (NULL != nd1 &&
 634             (nd1->slots_inuse - nd1->slots) < (node->slots_inuse - node->slots)) {
 635             cur_node_item = (opal_list_item_t*)ndmin;
 636         }
 637     }
 638 
 639   process:
 640     OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
 641                          "%s Starting at node %s",
 642                          ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
 643                          ((orte_node_t*)cur_node_item)->name));
 644 
 645     return (orte_node_t*)cur_node_item;
 646 }