opal/mca/hwloc/base/hwloc_base

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
opal_hwloc_base_get_pu
opal_hwloc_base_filter_cpus
fill_cache_line_size
opal_hwloc_base_get_topology
opal_hwloc_base_set_topology
free_object
opal_hwloc_base_free_topology
opal_hwloc_base_get_local_cpuset
opal_hwloc_base_report_bind_failure
opal_hwloc_base_single_cpu
opal_hwloc_base_get_npus
opal_hwloc_base_get_obj_idx
df_search
opal_hwloc_base_get_nbobjs_by_type
opal_hwloc_base_get_obj_by_type
df_clear
opal_hwloc_base_clear_usage
socket_to_cpu_set
socket_core_to_cpu_set
opal_hwloc_base_cpu_list_parse
opal_hwloc_base_get_relative_locality
opal_hwloc_base_find_coprocessors
hwloc_getline
opal_hwloc_base_check_on_coprocessor
opal_hwloc_base_print_binding
bitmap2rangestr
build_map
opal_hwloc_base_cset2str
opal_hwloc_base_cset2mapstr
dist_cmp_fn
sort_by_dist
find_devices
opal_hwloc_get_sorted_numa_list
opal_hwloc_base_get_topo_signature
opal_hwloc_base_get_locality_string
opal_hwloc_base_get_location
opal_hwloc_compute_relative_locality
opal_hwloc_base_topology_export_xmlbuffer
opal_hwloc_base_topology_set_flags
   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2006 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2011-2018 Cisco Systems, Inc.  All rights reserved
  14  * Copyright (c) 2012-2017 Los Alamos National Security, LLC.
  15  *                         All rights reserved.
  16  * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
  17  * Copyright (c) 2015-2017 Research Organization for Information Science
  18  *                         and Technology (RIST). All rights reserved.
  19  * Copyright (C) 2018      Mellanox Technologies, Ltd.
  20  *                         All rights reserved.
  21  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  22  * Copyright (c) 2019 IBM Corporation. All rights reserved.
  23  * $COPYRIGHT$
  24  *
  25  * Additional copyrights may follow
  26  *
  27  * $HEADER$
  28  */
  29 
  30 #define OPAL_HWLOC_WANT_SHMEM 1
  31 
  32 #include "opal_config.h"
  33 
  34 #ifdef HAVE_SYS_TYPES_H
  35 #include <sys/types.h>
  36 #endif
  37 #ifdef HAVE_UNISTD_H
  38 #include <unistd.h>
  39 #endif
  40 #ifdef HAVE_ENDIAN_H
  41 #include <endian.h>
  42 #endif
  43 #ifdef HAVE_SYS_STAT_H
  44 #include <sys/stat.h>
  45 #endif
  46 #if HAVE_FCNTL_H
  47 #include <fcntl.h>
  48 #endif
  49 
  50 #include "opal/runtime/opal.h"
  51 #include "opal/constants.h"
  52 #include "opal/util/argv.h"
  53 #include "opal/util/output.h"
  54 #include "opal/util/os_dirpath.h"
  55 #include "opal/util/show_help.h"
  56 #include "opal/util/printf.h"
  57 #include "opal/threads/tsd.h"
  58 #include "opal/mca/pmix/pmix.h"
  59 
  60 #include "opal/mca/hwloc/hwloc-internal.h"
  61 #include "opal/mca/hwloc/base/base.h"
  62 
  63 static bool topo_in_shmem = false;
  64 
  65 /*
  66  * Provide the hwloc object that corresponds to the given
  67  * processor id of the given type.  Remember: "processor" here [usually] means "core" --
  68  * except that on some platforms, hwloc won't find any cores; it'll
  69  * only find PUs (!).  On such platforms, then do the same calculation
  70  * but with PUs instead of COREs.
  71  */
  72 hwloc_obj_t opal_hwloc_base_get_pu(hwloc_topology_t topo,
  73                                    int lid,
  74                                    opal_hwloc_resource_type_t rtype)
  75 {
  76     hwloc_obj_type_t obj_type = HWLOC_OBJ_CORE;
  77     hwloc_obj_t obj;
  78 
  79     /* hwloc isn't able to find cores on all platforms.  Example:
  80        PPC64 running RHEL 5.4 (linux kernel 2.6.18) only reports NUMA
  81        nodes and PU's.  Fine.
  82 
  83        However, note that hwloc_get_obj_by_type() will return NULL in
  84        2 (effectively) different cases:
  85 
  86        - no objects of the requested type were found
  87        - the Nth object of the requested type was not found
  88 
  89        So first we have to see if we can find *any* cores by looking
  90        for the 0th core.  If we find it, then try to find the Nth
  91        core.  Otherwise, try to find the Nth PU. */
  92     if (opal_hwloc_use_hwthreads_as_cpus || (NULL == hwloc_get_obj_by_type(topo, HWLOC_OBJ_CORE, 0))) {
  93         obj_type = HWLOC_OBJ_PU;
  94     }
  95 
  96     if (OPAL_HWLOC_PHYSICAL == rtype) {
  97         /* find the pu - note that we can only find physical PUs
  98          * as cores do not have unique physical numbers (they are
  99          * numbered within their sockets instead). So we find the
 100          * specified PU, and then return the core object that contains it */
 101         obj = hwloc_get_pu_obj_by_os_index(topo, lid);
 102         OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 103                              "physical cpu %d %s found in cpuset %s",
 104                              lid, (NULL == obj) ? "not" : "is",
 105                              (NULL == opal_hwloc_base_cpu_list) ? "None" : opal_hwloc_base_cpu_list));
 106         /* we now need to shift upward to the core including this PU */
 107         if (NULL != obj && HWLOC_OBJ_CORE == obj_type) {
 108             obj = obj->parent;
 109         }
 110         return obj;
 111     }
 112 
 113     opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
 114                         "Searching for %d LOGICAL PU", lid);
 115 
 116     /* Now do the actual lookup. */
 117     obj = hwloc_get_obj_by_type(topo, obj_type, lid);
 118     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 119                          "logical cpu %d %s found in cpuset %s",
 120                          lid, (NULL == obj) ? "not" : "is",
 121                          (NULL == opal_hwloc_base_cpu_list) ? "None" : opal_hwloc_base_cpu_list));
 122 
 123     /* Found the right core (or PU). Return the object */
 124     return obj;
 125 }
 126 
 127 /* determine the node-level available cpuset based on
 128  * online vs allowed vs user-specified cpus
 129  */
 130 int opal_hwloc_base_filter_cpus(hwloc_topology_t topo)
 131 {
 132     hwloc_obj_t root, pu;
 133     hwloc_cpuset_t avail = NULL, pucpus, res;
 134     opal_hwloc_topo_data_t *sum;
 135     opal_hwloc_obj_data_t *data;
 136     char **ranges=NULL, **range=NULL;
 137     int idx, cpu, start, end;
 138 
 139     root = hwloc_get_root_obj(topo);
 140 
 141     if (NULL == root->userdata) {
 142         root->userdata = (void*)OBJ_NEW(opal_hwloc_topo_data_t);
 143     }
 144     sum = (opal_hwloc_topo_data_t*)root->userdata;
 145 
 146     /* should only ever enter here once, but check anyway */
 147     if (NULL != sum->available) {
 148         return OPAL_SUCCESS;
 149     }
 150 
 151     /* process any specified default cpu set against this topology */
 152     if (NULL == opal_hwloc_base_cpu_list) {
 153         /* get the root available cpuset */
 154         #if HWLOC_API_VERSION < 0x20000
 155             avail = hwloc_bitmap_alloc();
 156             hwloc_bitmap_and(avail, root->online_cpuset, root->allowed_cpuset);
 157         #else
 158             avail = hwloc_bitmap_dup(root->cpuset);
 159         #endif
 160         OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 161                              "hwloc:base: no cpus specified - using root available cpuset"));
 162     } else {
 163         OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 164                              "hwloc:base: filtering cpuset"));
 165         /* find the specified logical cpus */
 166         ranges = opal_argv_split(opal_hwloc_base_cpu_list, ',');
 167         avail = hwloc_bitmap_alloc();
 168         hwloc_bitmap_zero(avail);
 169         res = hwloc_bitmap_alloc();
 170         pucpus = hwloc_bitmap_alloc();
 171         for (idx=0; idx < opal_argv_count(ranges); idx++) {
 172             range = opal_argv_split(ranges[idx], '-');
 173             switch (opal_argv_count(range)) {
 174             case 1:
 175                 /* only one cpu given - get that object */
 176                 cpu = strtoul(range[0], NULL, 10);
 177                 if (NULL != (pu = opal_hwloc_base_get_pu(topo, cpu, OPAL_HWLOC_LOGICAL))) {
 178                     #if HWLOC_API_VERSION < 0x20000
 179                         hwloc_bitmap_and(pucpus, pu->online_cpuset, pu->allowed_cpuset);
 180                     #else
 181                         hwloc_bitmap_free(pucpus);
 182                         pucpus = hwloc_bitmap_dup(pu->cpuset);
 183                     #endif
 184                     hwloc_bitmap_or(res, avail, pucpus);
 185                     hwloc_bitmap_copy(avail, res);
 186                     data = (opal_hwloc_obj_data_t*)pu->userdata;
 187                     if (NULL == data) {
 188                         pu->userdata = (void*)OBJ_NEW(opal_hwloc_obj_data_t);
 189                         data = (opal_hwloc_obj_data_t*)pu->userdata;
 190                     }
 191                     data->npus++;
 192                 }
 193                 break;
 194             case 2:
 195                 /* range given */
 196                 start = strtoul(range[0], NULL, 10);
 197                 end = strtoul(range[1], NULL, 10);
 198                 for (cpu=start; cpu <= end; cpu++) {
 199                     if (NULL != (pu = opal_hwloc_base_get_pu(topo, cpu, OPAL_HWLOC_LOGICAL))) {
 200                         #if HWLOC_API_VERSION < 0x20000
 201                             hwloc_bitmap_and(pucpus, pu->online_cpuset, pu->allowed_cpuset);
 202                         #else
 203                             hwloc_bitmap_free(pucpus);
 204                             pucpus = hwloc_bitmap_dup(pu->cpuset);
 205                         #endif
 206                         hwloc_bitmap_or(res, avail, pucpus);
 207                         hwloc_bitmap_copy(avail, res);
 208                         data = (opal_hwloc_obj_data_t*)pu->userdata;
 209                         if (NULL == data) {
 210                             pu->userdata = (void*)OBJ_NEW(opal_hwloc_obj_data_t);
 211                             data = (opal_hwloc_obj_data_t*)pu->userdata;
 212                         }
 213                         data->npus++;
 214                     }
 215                 }
 216                 break;
 217             default:
 218                 break;
 219             }
 220             opal_argv_free(range);
 221         }
 222         if (NULL != ranges) {
 223             opal_argv_free(ranges);
 224         }
 225         hwloc_bitmap_free(res);
 226         hwloc_bitmap_free(pucpus);
 227     }
 228 
 229     /* cache this info */
 230     sum->available = avail;
 231 
 232     return OPAL_SUCCESS;
 233 }
 234 
 235 static void fill_cache_line_size(void)
 236 {
 237     int i = 0, cache_level = 2;
 238     unsigned size;
 239     unsigned int cache_object = HWLOC_OBJ_L2CACHE;
 240     hwloc_obj_t obj;
 241     bool found = false;
 242 
 243     /* Look for the smallest L2 cache size */
 244     size = 4096;
 245     while (cache_level > 0 && !found) {
 246         i=0;
 247         while (1) {
 248             obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology,
 249                                                   cache_object, cache_level,
 250                                                   i, OPAL_HWLOC_LOGICAL);
 251             if (NULL == obj) {
 252                 --cache_level;
 253                 cache_object = HWLOC_OBJ_L1CACHE;
 254                 break;
 255             } else {
 256                 if (NULL != obj->attr &&
 257                     obj->attr->cache.linesize > 0 &&
 258                     size > obj->attr->cache.linesize) {
 259                     size = obj->attr->cache.linesize;
 260                     found = true;
 261                 }
 262             }
 263             ++i;
 264         }
 265     }
 266 
 267     /* If we found an L2 cache size in the hwloc data, save it in
 268        opal_cache_line_size.  Otherwise, we'll leave whatever default
 269        was set in opal_init.c */
 270     if (found) {
 271         opal_cache_line_size = (int) size;
 272     }
 273 }
 274 
 275 int opal_hwloc_base_get_topology(void)
 276 {
 277     int rc;
 278     opal_process_name_t wildcard_rank;
 279     char *val = NULL;
 280 #if HWLOC_API_VERSION >= 0x20000
 281     int rc2, rc3, fd;
 282     uint64_t addr, *aptr, size, *sptr;
 283     char *shmemfile;
 284 #endif
 285 
 286     opal_output_verbose(2, opal_hwloc_base_framework.framework_output,
 287                          "hwloc:base:get_topology");
 288 
 289     /* see if we already have it */
 290     if (NULL != opal_hwloc_topology) {
 291         return OPAL_SUCCESS;
 292     }
 293     wildcard_rank.jobid = OPAL_PROC_MY_NAME.jobid;
 294     wildcard_rank.vpid = OPAL_VPID_WILDCARD;
 295 
 296     if (NULL != opal_pmix.get) {
 297 #if HWLOC_API_VERSION >= 0x20000
 298         opal_output_verbose(2, opal_hwloc_base_framework.framework_output,
 299                              "hwloc:base: looking for topology in shared memory");
 300 
 301         /* first try to get the shmem link, if available */
 302         aptr = &addr;
 303         sptr = &size;
 304         OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, OPAL_PMIX_HWLOC_SHMEM_FILE,
 305                                        &wildcard_rank, (void**)&shmemfile, OPAL_STRING);
 306         OPAL_MODEX_RECV_VALUE_OPTIONAL(rc2, OPAL_PMIX_HWLOC_SHMEM_ADDR,
 307                                        &wildcard_rank, (void**)&aptr, OPAL_SIZE);
 308         OPAL_MODEX_RECV_VALUE_OPTIONAL(rc3, OPAL_PMIX_HWLOC_SHMEM_SIZE,
 309                                        &wildcard_rank, (void**)&sptr, OPAL_SIZE);
 310         if (OPAL_SUCCESS == rc && OPAL_SUCCESS == rc2 && OPAL_SUCCESS == rc3) {
 311             if (0 > (fd = open(shmemfile, O_RDONLY))) {
 312                 free(shmemfile);
 313                 OPAL_ERROR_LOG(OPAL_ERR_FILE_OPEN_FAILURE)
 314                 return OPAL_ERR_FILE_OPEN_FAILURE;
 315             }
 316             free(shmemfile);
 317             if (0 != hwloc_shmem_topology_adopt(&opal_hwloc_topology, fd,
 318                                                 0, (void*)addr, size, 0)) {
 319                 if (4 < opal_output_get_verbosity(opal_hwloc_base_framework.framework_output)) {
 320                     FILE *file = fopen("/proc/self/maps", "r");
 321                     if (file) {
 322                         char line[256];
 323                         opal_output(0, "Dumping /proc/self/maps");
 324 
 325                         while (fgets(line, sizeof(line), file) != NULL) {
 326                             char *end = strchr(line, '\n');
 327                             if (end) {
 328                                 *end = '\0';
 329                             }
 330                             opal_output(0, "%s", line);
 331                         }
 332                         fclose(file);
 333                     }
 334                 }
 335                 /* failed to adopt from shmem, fallback to other ways to get the topology */
 336             } else {
 337                 opal_output_verbose(2, opal_hwloc_base_framework.framework_output,
 338                                     "hwloc:base: topology in shared memory");
 339                 topo_in_shmem = true;
 340                 return OPAL_SUCCESS;
 341             }
 342         }
 343 #endif
 344         /* if that isn't available, then try to retrieve
 345          * the xml representation from the PMIx data store */
 346         opal_output_verbose(1, opal_hwloc_base_framework.framework_output,
 347                             "hwloc:base[%s:%d] getting topology XML string",
 348                             __FILE__, __LINE__);
 349 #if HWLOC_API_VERSION >= 0x20000
 350         OPAL_MODEX_RECV_VALUE_IMMEDIATE(rc, OPAL_PMIX_HWLOC_XML_V2,
 351                                         &wildcard_rank, &val, OPAL_STRING);
 352 #else
 353         OPAL_MODEX_RECV_VALUE_IMMEDIATE(rc, OPAL_PMIX_HWLOC_XML_V1,
 354                                         &wildcard_rank, &val, OPAL_STRING);
 355 #endif
 356         if (rc != OPAL_SUCCESS) {
 357             /* check the old topo key to keep compatibility with older RMs */
 358             OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, OPAL_PMIX_LOCAL_TOPO,
 359                                            &wildcard_rank, &val, OPAL_STRING);
 360         }
 361     } else {
 362         opal_output_verbose(1, opal_hwloc_base_framework.framework_output,
 363                             "hwloc:base PMIx not available");
 364         rc = OPAL_ERR_NOT_SUPPORTED;
 365     }
 366 
 367     if (OPAL_SUCCESS == rc && NULL != val) {
 368         opal_output_verbose(1, opal_hwloc_base_framework.framework_output,
 369                             "hwloc:base loading topology from XML");
 370         /* load the topology */
 371         if (0 != hwloc_topology_init(&opal_hwloc_topology)) {
 372             free(val);
 373             return OPAL_ERROR;
 374         }
 375         if (0 != hwloc_topology_set_xmlbuffer(opal_hwloc_topology, val, strlen(val)+1)) {
 376             free(val);
 377             hwloc_topology_destroy(opal_hwloc_topology);
 378             return OPAL_ERROR;
 379         }
 380         /* since we are loading this from an external source, we have to
 381          * explicitly set a flag so hwloc sets things up correctly
 382          */
 383         if (0 != opal_hwloc_base_topology_set_flags(opal_hwloc_topology,
 384                                                     HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM,
 385                                                     true)) {
 386             hwloc_topology_destroy(opal_hwloc_topology);
 387             free(val);
 388             return OPAL_ERROR;
 389         }
 390         /* now load the topology */
 391         if (0 != hwloc_topology_load(opal_hwloc_topology)) {
 392             hwloc_topology_destroy(opal_hwloc_topology);
 393             free(val);
 394             return OPAL_ERROR;
 395         }
 396         free(val);
 397         /* filter the cpus thru any default cpu set */
 398         if (OPAL_SUCCESS != (rc = opal_hwloc_base_filter_cpus(opal_hwloc_topology))) {
 399             hwloc_topology_destroy(opal_hwloc_topology);
 400             return rc;
 401         }
 402     } else if (NULL == opal_hwloc_base_topo_file) {
 403         opal_output_verbose(1, opal_hwloc_base_framework.framework_output,
 404                             "hwloc:base discovering topology");
 405         if (0 != hwloc_topology_init(&opal_hwloc_topology) ||
 406             0 != opal_hwloc_base_topology_set_flags(opal_hwloc_topology, 0, true) ||
 407             0 != hwloc_topology_load(opal_hwloc_topology)) {
 408             OPAL_ERROR_LOG(OPAL_ERR_NOT_SUPPORTED);
 409             return OPAL_ERR_NOT_SUPPORTED;
 410         }
 411         /* filter the cpus thru any default cpu set */
 412         if (OPAL_SUCCESS != (rc = opal_hwloc_base_filter_cpus(opal_hwloc_topology))) {
 413             hwloc_topology_destroy(opal_hwloc_topology);
 414             return rc;
 415         }
 416     } else {
 417         opal_output_verbose(1, opal_hwloc_base_framework.framework_output,
 418                             "hwloc:base loading topology from file %s",
 419                             opal_hwloc_base_topo_file);
 420         if (OPAL_SUCCESS != (rc = opal_hwloc_base_set_topology(opal_hwloc_base_topo_file))) {
 421             return rc;
 422         }
 423     }
 424 
 425     /* fill opal_cache_line_size global with the smallest L1 cache
 426        line size */
 427     fill_cache_line_size();
 428 
 429     /* get or update our local cpuset - it will get used multiple
 430      * times, so it's more efficient to keep a global copy
 431      */
 432     opal_hwloc_base_get_local_cpuset();
 433 
 434     return OPAL_SUCCESS;
 435 }
 436 
 437 int opal_hwloc_base_set_topology(char *topofile)
 438 {
 439     struct hwloc_topology_support *support;
 440 
 441      OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 442                           "hwloc:base:set_topology %s", topofile));
 443 
 444    if (NULL != opal_hwloc_topology) {
 445         hwloc_topology_destroy(opal_hwloc_topology);
 446     }
 447     if (0 != hwloc_topology_init(&opal_hwloc_topology)) {
 448         return OPAL_ERR_NOT_SUPPORTED;
 449     }
 450     if (0 != hwloc_topology_set_xml(opal_hwloc_topology, topofile)) {
 451         hwloc_topology_destroy(opal_hwloc_topology);
 452         OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 453                              "hwloc:base:set_topology bad topo file"));
 454         return OPAL_ERR_NOT_SUPPORTED;
 455     }
 456     /* since we are loading this from an external source, we have to
 457      * explicitly set a flag so hwloc sets things up correctly
 458      */
 459     if (0 != opal_hwloc_base_topology_set_flags(opal_hwloc_topology,
 460                                                 HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM,
 461                                                 true)) {
 462         hwloc_topology_destroy(opal_hwloc_topology);
 463         return OPAL_ERR_NOT_SUPPORTED;
 464     }
 465     if (0 != hwloc_topology_load(opal_hwloc_topology)) {
 466         hwloc_topology_destroy(opal_hwloc_topology);
 467         OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 468                              "hwloc:base:set_topology failed to load"));
 469         return OPAL_ERR_NOT_SUPPORTED;
 470     }
 471 
 472     /* unfortunately, hwloc does not include support info in its
 473      * xml output :-(( We default to assuming it is present as
 474      * systems that use this option are likely to provide
 475      * binding support
 476      */
 477     support = (struct hwloc_topology_support*)hwloc_topology_get_support(opal_hwloc_topology);
 478     support->cpubind->set_thisproc_cpubind = true;
 479     support->membind->set_thisproc_membind = true;
 480 
 481     /* fill opal_cache_line_size global with the smallest L1 cache
 482        line size */
 483     fill_cache_line_size();
 484 
 485     /* all done */
 486     return OPAL_SUCCESS;
 487 }
 488 
 489 static void free_object(hwloc_obj_t obj)
 490 {
 491     opal_hwloc_obj_data_t *data;
 492     unsigned k;
 493 
 494     /* free any data hanging on this object */
 495     if (NULL != obj->userdata) {
 496         data = (opal_hwloc_obj_data_t*)obj->userdata;
 497         OBJ_RELEASE(data);
 498         obj->userdata = NULL;
 499     }
 500 
 501     /* loop thru our children */
 502     for (k=0; k < obj->arity; k++) {
 503         free_object(obj->children[k]);
 504     }
 505 }
 506 
 507 void opal_hwloc_base_free_topology(hwloc_topology_t topo)
 508 {
 509     hwloc_obj_t obj;
 510     opal_hwloc_topo_data_t *rdata;
 511     unsigned k;
 512 
 513     if (!topo_in_shmem) {
 514         obj = hwloc_get_root_obj(topo);
 515         /* release the root-level userdata */
 516         if (NULL != obj->userdata) {
 517             rdata = (opal_hwloc_topo_data_t*)obj->userdata;
 518             OBJ_RELEASE(rdata);
 519             obj->userdata = NULL;
 520         }
 521         /* now recursively descend and release userdata
 522          * in the rest of the objects
 523          */
 524         for (k=0; k < obj->arity; k++) {
 525             free_object(obj->children[k]);
 526         }
 527     }
 528     hwloc_topology_destroy(topo);
 529 }
 530 
 531 void opal_hwloc_base_get_local_cpuset(void)
 532 {
 533     hwloc_obj_t root;
 534 
 535     if (NULL != opal_hwloc_topology) {
 536         if (NULL == opal_hwloc_my_cpuset) {
 537             opal_hwloc_my_cpuset = hwloc_bitmap_alloc();
 538         }
 539 
 540         /* get the cpus we are bound to */
 541         if (hwloc_get_cpubind(opal_hwloc_topology,
 542                               opal_hwloc_my_cpuset,
 543                               HWLOC_CPUBIND_PROCESS) < 0) {
 544             /* we are not bound - use the root's available cpuset */
 545             root = hwloc_get_root_obj(opal_hwloc_topology);
 546             hwloc_bitmap_copy(opal_hwloc_my_cpuset, root->cpuset);
 547         }
 548     }
 549 }
 550 
 551 int opal_hwloc_base_report_bind_failure(const char *file,
 552                                         int line,
 553                                         const char *msg, int rc)
 554 {
 555     static int already_reported = 0;
 556 
 557     if (!already_reported &&
 558         OPAL_HWLOC_BASE_MBFA_SILENT != opal_hwloc_base_mbfa) {
 559         char hostname[OPAL_MAXHOSTNAMELEN];
 560         gethostname(hostname, sizeof(hostname));
 561 
 562         opal_show_help("help-opal-hwloc-base.txt", "mbind failure", true,
 563                        hostname, getpid(), file, line, msg,
 564                        (OPAL_HWLOC_BASE_MBFA_WARN == opal_hwloc_base_mbfa) ?
 565                        "Warning -- your job will continue, but possibly with degraded performance" :
 566                        "ERROR -- your job may abort or behave erraticly");
 567         already_reported = 1;
 568         return rc;
 569     }
 570 
 571     return OPAL_SUCCESS;
 572 }
 573 
 574 /* determine if there is a single cpu in a bitmap */
 575 bool opal_hwloc_base_single_cpu(hwloc_cpuset_t cpuset)
 576 {
 577     int i;
 578     bool one=false;
 579 
 580     /* count the number of bits that are set - there is
 581      * one bit for each available pu. We could just
 582      * subtract the first and last indices, but there
 583      * may be "holes" in the bitmap corresponding to
 584      * offline or unallowed cpus - so we have to
 585      * search for them. Return false if we anything
 586      * other than one
 587      */
 588     for (i=hwloc_bitmap_first(cpuset);
 589          i <= hwloc_bitmap_last(cpuset);
 590          i++) {
 591         if (hwloc_bitmap_isset(cpuset, i)) {
 592             if (one) {
 593                 return false;
 594             }
 595             one = true;
 596         }
 597     }
 598 
 599     return one;
 600 }
 601 
 602 /* get the number of pu's under a given hwloc object */
 603 unsigned int opal_hwloc_base_get_npus(hwloc_topology_t topo,
 604                                       hwloc_obj_t obj)
 605 {
 606     opal_hwloc_obj_data_t *data;
 607     unsigned int cnt = 0;
 608 
 609     data = (opal_hwloc_obj_data_t*)obj->userdata;
 610     if (NULL == data || !data->npus_calculated) {
 611         if (!opal_hwloc_use_hwthreads_as_cpus) {
 612             /* if we are treating cores as cpus, then we really
 613              * want to know how many cores are in this object.
 614              * hwloc sets a bit for each "pu", so we can't just
 615              * count bits in this case as there may be more than
 616              * one hwthread/core. Instead, find the number of cores
 617              * in the system
 618              */
 619             cnt = hwloc_get_nbobjs_inside_cpuset_by_type(topo, obj->cpuset, HWLOC_OBJ_CORE);
 620         } else {
 621             hwloc_cpuset_t cpuset;
 622 
 623             /* if we are treating cores as cpus, or the system can't detect
 624              * "cores", then get the available cpuset for this object - this will
 625              * create and store the data
 626              */
 627             if (NULL == (cpuset = obj->cpuset)) {
 628                 return 0;
 629             }
 630             /* count the number of bits that are set - there is
 631              * one bit for each available pu. We could just
 632              * subtract the first and last indices, but there
 633              * may be "holes" in the bitmap corresponding to
 634              * offline or unallowed cpus - so we count them with
 635              * the bitmap "weight" (a.k.a. population count) function
 636              */
 637             cnt = hwloc_bitmap_weight(cpuset);
 638         }
 639         /* cache the info */
 640         data = (opal_hwloc_obj_data_t*)obj->userdata;  // in case it was added
 641         if (NULL == data) {
 642             data = OBJ_NEW(opal_hwloc_obj_data_t);
 643             obj->userdata = (void*)data;
 644         }
 645         data->npus = cnt;
 646         data->npus_calculated = true;
 647     }
 648 
 649     return data->npus;
 650 }
 651 
 652 unsigned int opal_hwloc_base_get_obj_idx(hwloc_topology_t topo,
 653                                          hwloc_obj_t obj,
 654                                          opal_hwloc_resource_type_t rtype)
 655 {
 656     unsigned cache_level=0;
 657     opal_hwloc_obj_data_t *data;
 658     hwloc_obj_t ptr;
 659     unsigned int nobjs, i;
 660 
 661     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 662                          "hwloc:base:get_idx"));
 663 
 664     /* see if we already have the info */
 665     data = (opal_hwloc_obj_data_t*)obj->userdata;
 666 
 667     if (NULL == data) {
 668         data = OBJ_NEW(opal_hwloc_obj_data_t);
 669         obj->userdata = (void*)data;
 670     }
 671 
 672     if (data->idx < UINT_MAX) {
 673         OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 674                              "hwloc:base:get_idx already have data: %u",
 675                              data->idx));
 676         return data->idx;
 677     }
 678 
 679 #if HWLOC_API_VERSION < 0x20000
 680     /* determine the number of objects of this type */
 681     if (HWLOC_OBJ_CACHE == obj->type) {
 682         cache_level = obj->attr->cache.depth;
 683     }
 684 #endif
 685 
 686     nobjs = opal_hwloc_base_get_nbobjs_by_type(topo, obj->type, cache_level, rtype);
 687 
 688     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 689                          "hwloc:base:get_idx found %u objects of type %s:%u",
 690                          nobjs, hwloc_obj_type_string(obj->type), cache_level));
 691 
 692     /* find this object */
 693     for (i=0; i < nobjs; i++) {
 694         ptr = opal_hwloc_base_get_obj_by_type(topo, obj->type, cache_level, i, rtype);
 695         if (ptr == obj) {
 696             data->idx = i;
 697             return i;
 698         }
 699     }
 700     /* if we get here, it wasn't found */
 701     opal_show_help("help-opal-hwloc-base.txt",
 702                    "obj-idx-failed", true,
 703                    hwloc_obj_type_string(obj->type), cache_level);
 704     return UINT_MAX;
 705 }
 706 
 707 /* hwloc treats cache objects as special
 708  * cases. Instead of having a unique type for each cache level,
 709  * there is a single cache object type, and the level is encoded
 710  * in an attribute union. So looking for cache objects involves
 711  * a multi-step test :-(
 712  */
 713 static hwloc_obj_t df_search(hwloc_topology_t topo,
 714                              hwloc_obj_t start,
 715                              hwloc_obj_type_t target,
 716                              unsigned cache_level,
 717                              unsigned int nobj,
 718                              opal_hwloc_resource_type_t rtype,
 719                              unsigned int *num_objs)
 720 {
 721     hwloc_obj_t obj;
 722     int search_depth;
 723 
 724     search_depth = hwloc_get_type_depth(topo, target);
 725     if (HWLOC_TYPE_DEPTH_MULTIPLE == search_depth) {
 726         /* either v1.x Cache, or Groups */
 727 #if HWLOC_API_VERSION >= 0x20000
 728         return NULL;
 729 #else
 730         if (cache_level != HWLOC_OBJ_CACHE)
 731             return NULL;
 732         search_depth = hwloc_get_cache_type_depth(topo, cache_level, (hwloc_obj_cache_type_t) -1);
 733 #endif
 734     }
 735     if (HWLOC_TYPE_DEPTH_UNKNOWN == search_depth)
 736         return NULL;
 737 
 738     if (OPAL_HWLOC_LOGICAL == rtype) {
 739         if (num_objs)
 740             *num_objs = hwloc_get_nbobjs_by_depth(topo, search_depth);
 741         return hwloc_get_obj_by_depth(topo, search_depth, nobj);
 742     }
 743     if (OPAL_HWLOC_PHYSICAL == rtype) {
 744         /* the PHYSICAL object number is stored as the os_index. When
 745          * counting physical objects, we can't just count the number
 746          * that are in the hwloc tree as the only entries in the tree
 747          * are LOGICAL objects - i.e., any physical gaps won't show. So
 748          * we instead return the MAX os_index, as this is the best we
 749          * can do to tell you how many PHYSICAL objects are in the system.
 750          *
 751          * NOTE: if the last PHYSICAL object is not present (e.g., the last
 752          * socket on the node is empty), then the count we return will
 753          * be wrong!
 754          */
 755         hwloc_obj_t found = NULL;
 756         obj = NULL;
 757         if (num_objs)
 758             *num_objs = 0;
 759         while ((obj = hwloc_get_next_obj_by_depth(topo, search_depth, obj)) != NULL) {
 760             if (num_objs && obj->os_index > *num_objs)
 761                 *num_objs = obj->os_index;
 762             if (obj->os_index == nobj)
 763                 found = obj;
 764         }
 765         return found;
 766     }
 767     if (OPAL_HWLOC_AVAILABLE == rtype) {
 768 // The previous (3.x) code included a check for
 769 // available = opal_hwloc_base_get_available_cpus(topo, start)
 770 // and skipped objs that had hwloc_bitmap_iszero(available)
 771         hwloc_obj_t root;
 772         opal_hwloc_topo_data_t *rdata;
 773         root = hwloc_get_root_obj(topo);
 774         rdata = (opal_hwloc_topo_data_t*)root->userdata;
 775         hwloc_cpuset_t constrained_cpuset;
 776 
 777         constrained_cpuset = hwloc_bitmap_alloc();
 778         if (rdata && rdata->available) {
 779             hwloc_bitmap_and(constrained_cpuset, start->cpuset, rdata->available);
 780         } else {
 781             hwloc_bitmap_copy(constrained_cpuset, start->cpuset);
 782         }
 783 
 784         unsigned idx = 0;
 785         if (num_objs)
 786             *num_objs = hwloc_get_nbobjs_inside_cpuset_by_depth(topo, constrained_cpuset, search_depth);
 787         obj = NULL;
 788         while ((obj = hwloc_get_next_obj_inside_cpuset_by_depth(topo, constrained_cpuset, search_depth, obj)) != NULL) {
 789             if (idx == nobj) {
 790                 hwloc_bitmap_free(constrained_cpuset);
 791                 return obj;
 792             }
 793             idx++;
 794         }
 795         hwloc_bitmap_free(constrained_cpuset);
 796         return NULL;
 797     }
 798     return NULL;
 799 }
 800 
 801 unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
 802                                                 hwloc_obj_type_t target,
 803                                                 unsigned cache_level,
 804                                                 opal_hwloc_resource_type_t rtype)
 805 {
 806     unsigned int num_objs;
 807     hwloc_obj_t obj;
 808     opal_hwloc_summary_t *sum;
 809     opal_hwloc_topo_data_t *data;
 810     int rc;
 811 
 812     /* bozo check */
 813     if (NULL == topo) {
 814         OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 815                              "hwloc:base:get_nbobjs NULL topology"));
 816         return 0;
 817     }
 818 
 819     /* if we want the number of LOGICAL objects, we can just
 820      * use the hwloc accessor to get it, unless it is a CACHE
 821      * as these are treated as special cases
 822      */
 823     if (OPAL_HWLOC_LOGICAL == rtype
 824 #if HWLOC_API_VERSION < 0x20000
 825         && HWLOC_OBJ_CACHE != target
 826 #endif
 827        ) {
 828         /* we should not get an error back, but just in case... */
 829         if (0 > (rc = hwloc_get_nbobjs_by_type(topo, target))) {
 830             opal_output(0, "UNKNOWN HWLOC ERROR");
 831             return 0;
 832         }
 833         return rc;
 834     }
 835 
 836     /* for everything else, we have to do some work */
 837     num_objs = 0;
 838     obj = hwloc_get_root_obj(topo);
 839 
 840     /* first see if the topology already has this summary */
 841     data = (opal_hwloc_topo_data_t*)obj->userdata;
 842     if (NULL == data) {
 843         data = OBJ_NEW(opal_hwloc_topo_data_t);
 844         obj->userdata = (void*)data;
 845     } else {
 846         OPAL_LIST_FOREACH(sum, &data->summaries, opal_hwloc_summary_t) {
 847             if (target == sum->type &&
 848                 cache_level == sum->cache_level &&
 849                 rtype == sum->rtype) {
 850                 /* yep - return the value */
 851                 OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 852                                      "hwloc:base:get_nbojbs pre-existing data %u of %s:%u",
 853                                      sum->num_objs, hwloc_obj_type_string(target), cache_level));
 854                 return sum->num_objs;
 855             }
 856         }
 857     }
 858 
 859     /* don't already know it - go get it */
 860     df_search(topo, obj, target, cache_level, 0, rtype, &num_objs);
 861 
 862     /* cache the results for later */
 863     sum = OBJ_NEW(opal_hwloc_summary_t);
 864     sum->type = target;
 865     sum->cache_level = cache_level;
 866     sum->num_objs = num_objs;
 867     sum->rtype = rtype;
 868     opal_list_append(&data->summaries, &sum->super);
 869 
 870     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 871                          "hwloc:base:get_nbojbs computed data %u of %s:%u",
 872                          num_objs, hwloc_obj_type_string(target), cache_level));
 873 
 874     return num_objs;
 875 }
 876 
 877 /* as above, only return the Nth instance of the specified object
 878  * type from inside the topology
 879  */
 880 hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo,
 881                                             hwloc_obj_type_t target,
 882                                             unsigned cache_level,
 883                                             unsigned int instance,
 884                                             opal_hwloc_resource_type_t rtype)
 885 {
 886     hwloc_obj_t obj;
 887 
 888     /* bozo check */
 889     if (NULL == topo) {
 890         return NULL;
 891     }
 892 
 893     /* if we want the nth LOGICAL object, we can just
 894      * use the hwloc accessor to get it, unless it is a CACHE
 895      * as these are treated as special cases
 896      */
 897     if (OPAL_HWLOC_LOGICAL == rtype
 898 #if HWLOC_API_VERSION < 0x20000
 899         && HWLOC_OBJ_CACHE != target
 900 #endif
 901        ) {
 902         return hwloc_get_obj_by_type(topo, target, instance);
 903     }
 904 
 905     /* for everything else, we have to do some work */
 906     obj = hwloc_get_root_obj(topo);
 907     return df_search(topo, obj, target, cache_level, instance, rtype, NULL);
 908 }
 909 
 910 static void df_clear(hwloc_topology_t topo,
 911                      hwloc_obj_t start)
 912 {
 913     unsigned k;
 914     opal_hwloc_obj_data_t *data;
 915 
 916     /* see how many procs are bound to us */
 917     data = (opal_hwloc_obj_data_t*)start->userdata;
 918     if (NULL != data) {
 919         data->num_bound = 0;
 920     }
 921 
 922     for (k=0; k < start->arity; k++) {
 923         df_clear(topo, start->children[k]);
 924     }
 925 }
 926 
 927 void opal_hwloc_base_clear_usage(hwloc_topology_t topo)
 928 {
 929     hwloc_obj_t root;
 930     unsigned k;
 931 
 932     /* bozo check */
 933     if (NULL == topo) {
 934         OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
 935                              "hwloc:base:clear_usage: NULL topology"));
 936         return;
 937     }
 938 
 939     root = hwloc_get_root_obj(topo);
 940     /* must not start at root as the root object has
 941      * a different userdata attached to it
 942      */
 943     for (k=0; k < root->arity; k++) {
 944         df_clear(topo, root->children[k]);
 945     }
 946 }
 947 
 948 /* The current slot_list notation only goes to the core level - i.e., the location
 949  * is specified as socket:core. Thus, the code below assumes that all locations
 950  * are to be parsed under that notation.
 951  */
 952 
 953 static int socket_to_cpu_set(char *cpus,
 954                              hwloc_topology_t topo,
 955                              opal_hwloc_resource_type_t rtype,
 956                              hwloc_bitmap_t cpumask)
 957 {
 958     char **range;
 959     int range_cnt;
 960     int lower_range, upper_range;
 961     int socket_id;
 962     hwloc_obj_t obj;
 963 
 964     if ('*' == cpus[0]) {
 965         /* requesting cpumask for ALL sockets */
 966         obj = hwloc_get_root_obj(topo);
 967         /* set to all available processors - essentially,
 968          * this specification equates to unbound
 969          */
 970         hwloc_bitmap_or(cpumask, cpumask, obj->cpuset);
 971         return OPAL_SUCCESS;
 972     }
 973 
 974     range = opal_argv_split(cpus,'-');
 975     range_cnt = opal_argv_count(range);
 976     switch (range_cnt) {
 977     case 1:  /* no range was present, so just one socket given */
 978         socket_id = atoi(range[0]);
 979         obj = opal_hwloc_base_get_obj_by_type(topo, HWLOC_OBJ_SOCKET, 0, socket_id, rtype);
 980         /* get the available cpus for this socket */
 981         hwloc_bitmap_or(cpumask, cpumask, obj->cpuset);
 982         break;
 983 
 984     case 2:  /* range of sockets was given */
 985         lower_range = atoi(range[0]);
 986         upper_range = atoi(range[1]);
 987         /* cycle across the range of sockets */
 988         for (socket_id=lower_range; socket_id<=upper_range; socket_id++) {
 989             obj = opal_hwloc_base_get_obj_by_type(topo, HWLOC_OBJ_SOCKET, 0, socket_id, rtype);
 990             /* set the available cpus for this socket bits in the bitmask */
 991             hwloc_bitmap_or(cpumask, cpumask, obj->cpuset);
 992         }
 993         break;
 994     default:
 995         opal_argv_free(range);
 996         return OPAL_ERROR;
 997     }
 998     opal_argv_free(range);
 999 
1000     return OPAL_SUCCESS;
1001 }
1002 
1003 static int socket_core_to_cpu_set(char *socket_core_list,
1004                                   hwloc_topology_t topo,
1005                                   opal_hwloc_resource_type_t rtype,
1006                                   hwloc_bitmap_t cpumask)
1007 {
1008     int rc=OPAL_SUCCESS, i, j;
1009     char **socket_core, *corestr;
1010     char **range, **list;
1011     int range_cnt;
1012     int lower_range, upper_range;
1013     int socket_id, core_id;
1014     hwloc_obj_t socket, core;
1015     hwloc_obj_type_t obj_type = HWLOC_OBJ_CORE;
1016 
1017     socket_core = opal_argv_split(socket_core_list, ':');
1018     socket_id = atoi(socket_core[0]);
1019 
1020     /* get the object for this socket id */
1021     if (NULL == (socket = opal_hwloc_base_get_obj_by_type(topo, HWLOC_OBJ_SOCKET, 0,
1022                                                           socket_id, rtype))) {
1023         opal_argv_free(socket_core);
1024         return OPAL_ERR_NOT_FOUND;
1025     }
1026 
1027     /* as described in comment near top of file, hwloc isn't able
1028      * to find cores on all platforms. Adjust the type here if
1029      * required
1030      */
1031     if (NULL == hwloc_get_obj_by_type(topo, HWLOC_OBJ_CORE, 0)) {
1032         obj_type = HWLOC_OBJ_PU;
1033     }
1034 
1035     for (i=1; NULL != socket_core[i]; i++) {
1036         if ('C' == socket_core[i][0] ||
1037             'c' == socket_core[i][0]) {
1038             corestr = &socket_core[i][1];
1039         } else {
1040             corestr = socket_core[i];
1041         }
1042         if ('*' == corestr[0]) {
1043             /* set to all cpus on this socket */
1044             hwloc_bitmap_or(cpumask, cpumask, socket->cpuset);
1045             /* we are done - already assigned all cores! */
1046             rc = OPAL_SUCCESS;
1047             break;
1048         } else {
1049             range = opal_argv_split(corestr, '-');
1050             range_cnt = opal_argv_count(range);
1051             /* see if a range was set or not */
1052             switch (range_cnt) {
1053             case 1:  /* only one core, or a list of cores, specified */
1054                 list = opal_argv_split(range[0], ',');
1055                 for (j=0; NULL != list[j]; j++) {
1056                     core_id = atoi(list[j]);
1057                     /* get that object */
1058                     if (NULL == (core = df_search(topo, socket, obj_type, 0,
1059                                                   core_id, OPAL_HWLOC_AVAILABLE,
1060                                                   NULL))) {
1061                         opal_argv_free(list);
1062                         opal_argv_free(range);
1063                         opal_argv_free(socket_core);
1064                         return OPAL_ERR_NOT_FOUND;
1065                     }
1066                     /* get the cpus */
1067                     hwloc_bitmap_or(cpumask, cpumask, core->cpuset);
1068                 }
1069                 opal_argv_free(list);
1070                 break;
1071 
1072             case 2:  /* range of core id's was given */
1073                 opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
1074                                     "range of cores given: start %s stop %s",
1075                                     range[0], range[1]);
1076                 lower_range = atoi(range[0]);
1077                 upper_range = atoi(range[1]);
1078                 for (core_id=lower_range; core_id <= upper_range; core_id++) {
1079                     /* get that object */
1080                     if (NULL == (core = df_search(topo, socket, obj_type, 0,
1081                                                   core_id, OPAL_HWLOC_AVAILABLE,
1082                                                   NULL))) {
1083                         opal_argv_free(range);
1084                         opal_argv_free(socket_core);
1085                         return OPAL_ERR_NOT_FOUND;
1086                     }
1087                     /* get the cpus add them into the result */
1088                     hwloc_bitmap_or(cpumask, cpumask, core->cpuset);
1089                 }
1090                 break;
1091 
1092             default:
1093                 opal_argv_free(range);
1094                 opal_argv_free(socket_core);
1095                 return OPAL_ERROR;
1096             }
1097             opal_argv_free(range);
1098         }
1099     }
1100     opal_argv_free(socket_core);
1101 
1102     return rc;
1103 }
1104 
1105 int opal_hwloc_base_cpu_list_parse(const char *slot_str,
1106                                     hwloc_topology_t topo,
1107                                     opal_hwloc_resource_type_t rtype,
1108                                     hwloc_cpuset_t cpumask)
1109 {
1110     char **item, **rngs;
1111     int rc, i, j, k;
1112     hwloc_obj_t pu;
1113     char **range, **list;
1114     size_t range_cnt;
1115     int core_id, lower_range, upper_range;
1116 
1117     /* bozo checks */
1118     if (NULL == opal_hwloc_topology) {
1119         return OPAL_ERR_NOT_SUPPORTED;
1120     }
1121     if (NULL == slot_str || 0 == strlen(slot_str)) {
1122         return OPAL_ERR_BAD_PARAM;
1123     }
1124 
1125     opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
1126                         "slot assignment: slot_list == %s",
1127                         slot_str);
1128 
1129     /* split at ';' */
1130     item = opal_argv_split(slot_str, ';');
1131 
1132     /* start with a clean mask */
1133     hwloc_bitmap_zero(cpumask);
1134     /* loop across the items and accumulate the mask */
1135     for (i=0; NULL != item[i]; i++) {
1136         opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
1137                             "working assignment %s",
1138                             item[i]);
1139         /* if they specified "socket" by starting with an S/s,
1140          * or if they use socket:core notation, then parse the
1141          * socket/core info
1142          */
1143         if ('S' == item[i][0] ||
1144             's' == item[i][0] ||
1145             NULL != strchr(item[i], ':')) {
1146             /* specified a socket */
1147             if (NULL == strchr(item[i], ':')) {
1148                 /* binding just to the socket level, though
1149                  * it could specify multiple sockets
1150                  * Skip the S and look for a ranges
1151                  */
1152                 rngs = opal_argv_split(&item[i][1], ',');
1153                 for (j=0; NULL != rngs[j]; j++) {
1154                     if (OPAL_SUCCESS != (rc = socket_to_cpu_set(rngs[j], topo, rtype, cpumask))) {
1155                         opal_argv_free(rngs);
1156                         opal_argv_free(item);
1157                         return rc;
1158                     }
1159                 }
1160                 opal_argv_free(rngs);
1161             } else {
1162                 /* binding to a socket/whatever specification */
1163                 if ('S' == item[i][0] ||
1164                     's' == item[i][0]) {
1165                     rngs = opal_argv_split(&item[i][1], ',');
1166                     for (j=0; NULL != rngs[j]; j++) {
1167                         if (OPAL_SUCCESS != (rc = socket_core_to_cpu_set(rngs[j], topo, rtype, cpumask))) {
1168                             opal_argv_free(rngs);
1169                             opal_argv_free(item);
1170                             return rc;
1171                         }
1172                     }
1173                     opal_argv_free(rngs);
1174                 } else {
1175                     rngs = opal_argv_split(item[i], ',');
1176                     for (j=0; NULL != rngs[j]; j++) {
1177                         if (OPAL_SUCCESS != (rc = socket_core_to_cpu_set(rngs[j], topo, rtype, cpumask))) {
1178                             opal_argv_free(rngs);
1179                             opal_argv_free(item);
1180                             return rc;
1181                         }
1182                     }
1183                     opal_argv_free(rngs);
1184                 }
1185             }
1186         } else {
1187             rngs = opal_argv_split(item[i], ',');
1188             for (k=0; NULL != rngs[k]; k++) {
1189                 /* just a core specification - see if one or a range was given */
1190                 range = opal_argv_split(rngs[k], '-');
1191                 range_cnt = opal_argv_count(range);
1192                 /* see if a range was set or not */
1193                 switch (range_cnt) {
1194                 case 1:  /* only one core, or a list of cores, specified */
1195                     list = opal_argv_split(range[0], ',');
1196                     for (j=0; NULL != list[j]; j++) {
1197                         core_id = atoi(list[j]);
1198                         /* find the specified available cpu */
1199                         if (NULL == (pu = opal_hwloc_base_get_pu(topo, core_id, rtype))) {
1200                             opal_argv_free(range);
1201                             opal_argv_free(item);
1202                             opal_argv_free(rngs);
1203                             opal_argv_free(list);
1204                             return OPAL_ERR_SILENT;
1205                         }
1206                         /* get the cpus for that object and set them in the massk*/
1207                         hwloc_bitmap_or(cpumask, cpumask, pu->cpuset);
1208                     }
1209                     opal_argv_free(list);
1210                     break;
1211 
1212                 case 2:  /* range of core id's was given */
1213                     lower_range = atoi(range[0]);
1214                     upper_range = atoi(range[1]);
1215                     for (core_id=lower_range; core_id <= upper_range; core_id++) {
1216                         /* find the specified logical available cpu */
1217                         if (NULL == (pu = opal_hwloc_base_get_pu(topo, core_id, rtype))) {
1218                             opal_argv_free(range);
1219                             opal_argv_free(item);
1220                             opal_argv_free(rngs);
1221                             return OPAL_ERR_SILENT;
1222                         }
1223                         /* get the cpus for that object and set them in the mask*/
1224                         hwloc_bitmap_or(cpumask, cpumask, pu->cpuset);
1225                     }
1226                     break;
1227 
1228                 default:
1229                     opal_argv_free(range);
1230                     opal_argv_free(item);
1231                     opal_argv_free(rngs);
1232                     return OPAL_ERROR;
1233                 }
1234                 opal_argv_free(range);
1235             }
1236             opal_argv_free(rngs);
1237         }
1238     }
1239     opal_argv_free(item);
1240     return OPAL_SUCCESS;
1241 }
1242 
1243 opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
1244                                                             char *cpuset1, char *cpuset2)
1245 {
1246     opal_hwloc_locality_t locality;
1247     hwloc_obj_t obj;
1248     unsigned depth, d, width, w;
1249     bool shared;
1250     hwloc_obj_type_t type;
1251     int sect1, sect2;
1252     hwloc_cpuset_t loc1, loc2;
1253 
1254     /* start with what we know - they share a node on a cluster
1255      * NOTE: we may alter that latter part as hwloc's ability to
1256      * sense multi-cu, multi-cluster systems grows
1257      */
1258     locality = OPAL_PROC_ON_NODE | OPAL_PROC_ON_HOST | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER;
1259 
1260     /* if either cpuset is NULL, then that isn't bound */
1261     if (NULL == cpuset1 || NULL == cpuset2) {
1262         return locality;
1263     }
1264 
1265     /* get the max depth of the topology */
1266     depth = hwloc_topology_get_depth(topo);
1267 
1268     /* convert the strings to cpusets */
1269     loc1 = hwloc_bitmap_alloc();
1270     hwloc_bitmap_list_sscanf(loc1, cpuset1);
1271     loc2 = hwloc_bitmap_alloc();
1272     hwloc_bitmap_list_sscanf(loc2, cpuset2);
1273 
1274     /* start at the first depth below the top machine level */
1275     for (d=1; d < depth; d++) {
1276         shared = false;
1277         /* get the object type at this depth */
1278         type = hwloc_get_depth_type(topo, d);
1279         /* if it isn't one of interest, then ignore it */
1280         if (HWLOC_OBJ_NODE != type &&
1281             HWLOC_OBJ_SOCKET != type &&
1282 #if HWLOC_API_VERSION < 0x20000
1283             HWLOC_OBJ_CACHE != type &&
1284 #else
1285             HWLOC_OBJ_L3CACHE != type &&
1286             HWLOC_OBJ_L2CACHE != type &&
1287             HWLOC_OBJ_L1CACHE != type &&
1288 #endif
1289             HWLOC_OBJ_CORE != type &&
1290             HWLOC_OBJ_PU != type) {
1291             continue;
1292         }
1293         /* get the width of the topology at this depth */
1294         width = hwloc_get_nbobjs_by_depth(topo, d);
1295 
1296         /* scan all objects at this depth to see if
1297          * our locations overlap with them
1298          */
1299         for (w=0; w < width; w++) {
1300             /* get the object at this depth/index */
1301             obj = hwloc_get_obj_by_depth(topo, d, w);
1302             /* see if our locations intersect with the cpuset for this obj */
1303             sect1 = hwloc_bitmap_intersects(obj->cpuset, loc1);
1304             sect2 = hwloc_bitmap_intersects(obj->cpuset, loc2);
1305             /* if both intersect, then we share this level */
1306             if (sect1 && sect2) {
1307                 shared = true;
1308                 switch(obj->type) {
1309                 case HWLOC_OBJ_NODE:
1310                     locality |= OPAL_PROC_ON_NUMA;
1311                     break;
1312                 case HWLOC_OBJ_SOCKET:
1313                     locality |= OPAL_PROC_ON_SOCKET;
1314                     break;
1315 #if HWLOC_API_VERSION < 0x20000
1316                 case HWLOC_OBJ_CACHE:
1317                     if (3 == obj->attr->cache.depth) {
1318                         locality |= OPAL_PROC_ON_L3CACHE;
1319                     } else if (2 == obj->attr->cache.depth) {
1320                         locality |= OPAL_PROC_ON_L2CACHE;
1321                     } else {
1322                         locality |= OPAL_PROC_ON_L1CACHE;
1323                     }
1324                     break;
1325 #else
1326                 case HWLOC_OBJ_L3CACHE:
1327                     locality |= OPAL_PROC_ON_L3CACHE;
1328                     break;
1329                 case HWLOC_OBJ_L2CACHE:
1330                     locality |= OPAL_PROC_ON_L2CACHE;
1331                     break;
1332                 case HWLOC_OBJ_L1CACHE:
1333                     locality |= OPAL_PROC_ON_L1CACHE;
1334                     break;
1335 #endif
1336                 case HWLOC_OBJ_CORE:
1337                     locality |= OPAL_PROC_ON_CORE;
1338                     break;
1339                 case HWLOC_OBJ_PU:
1340                     locality |= OPAL_PROC_ON_HWTHREAD;
1341                     break;
1342                 default:
1343                     /* just ignore it */
1344                     break;
1345                 }
1346                 break;
1347             }
1348             /* otherwise, we don't share this
1349              * object - but we still might share another object
1350              * on this level, so we have to keep searching
1351              */
1352         }
1353         /* if we spanned the entire width without finding
1354          * a point of intersection, then no need to go
1355          * deeper
1356          */
1357         if (!shared) {
1358             break;
1359         }
1360     }
1361 
1362     opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
1363                         "locality: %s",
1364                         opal_hwloc_base_print_locality(locality));
1365     hwloc_bitmap_free(loc1);
1366     hwloc_bitmap_free(loc2);
1367 
1368     return locality;
1369 }
1370 
1371 /* searches the given topology for coprocessor objects and returns
1372  * their serial numbers as a comma-delimited string, or NULL
1373  * if no coprocessors are found
1374  */
1375 char* opal_hwloc_base_find_coprocessors(hwloc_topology_t topo)
1376 {
1377     hwloc_obj_t osdev;
1378     unsigned i;
1379     char **cps = NULL;
1380     char *cpstring = NULL;
1381     int depth;
1382 
1383     /* coprocessors are recorded under OS_DEVICEs, so first
1384      * see if we have any of those
1385      */
1386     if (HWLOC_TYPE_DEPTH_UNKNOWN == (depth = hwloc_get_type_depth(topo, HWLOC_OBJ_OS_DEVICE))) {
1387         OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
1388                              "hwloc:base:find_coprocessors: NONE FOUND IN TOPO"));
1389         return NULL;
1390     }
1391 #if HAVE_DECL_HWLOC_OBJ_OSDEV_COPROC
1392     /* check the device objects for coprocessors */
1393     osdev = hwloc_get_obj_by_depth(topo, depth, 0);
1394     while (NULL != osdev) {
1395         if (HWLOC_OBJ_OSDEV_COPROC == osdev->attr->osdev.type) {
1396             /* got one! find and save its serial number */
1397             for (i=0; i < osdev->infos_count; i++) {
1398                 if (0 == strncmp(osdev->infos[i].name, "MICSerialNumber", strlen("MICSerialNumber"))) {
1399                     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
1400                                          "hwloc:base:find_coprocessors: coprocessor %s found",
1401                                          osdev->infos[i].value));
1402                     opal_argv_append_nosize(&cps, osdev->infos[i].value);
1403                 }
1404             }
1405         }
1406         osdev = osdev->next_cousin;
1407     }
1408     if (NULL != cps) {
1409         cpstring = opal_argv_join(cps, ',');
1410         opal_argv_free(cps);
1411     }
1412     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
1413                          "hwloc:base:find_coprocessors: hosting coprocessors %s",
1414                          (NULL == cpstring) ? "NONE" : cpstring));
1415 #else
1416     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
1417                          "hwloc:base:find_coprocessors: the version of hwloc that Open MPI was built against (v%d.%d.%d) does not support detecting coprocessors",
1418                          (HWLOC_API_VERSION>>16)&&0xFF, (HWLOC_API_VERSION>>8)&0xFF, HWLOC_API_VERSION&&0xFF));
1419 #endif
1420     return cpstring;
1421 }
1422 
1423 #define OPAL_HWLOC_MAX_ELOG_LINE 1024
1424 
1425 static char *hwloc_getline(FILE *fp)
1426 {
1427     char *ret, *buff;
1428     char input[OPAL_HWLOC_MAX_ELOG_LINE];
1429 
1430     ret = fgets(input, OPAL_HWLOC_MAX_ELOG_LINE, fp);
1431     if (NULL != ret) {
1432            input[strlen(input)-1] = '\0';  /* remove newline */
1433            buff = strdup(input);
1434            return buff;
1435     }
1436 
1437     return NULL;
1438 }
1439 
1440 /* checks local environment to determine if this process
1441  * is on a coprocessor - if so, it returns the serial number
1442  * as a string, or NULL if it isn't on a coprocessor
1443  */
1444 char* opal_hwloc_base_check_on_coprocessor(void)
1445 {
1446     /* this support currently is limited to Intel Phi processors
1447      * but will hopefully be extended as we get better, more
1448      * generalized ways of identifying coprocessors
1449      */
1450     FILE *fp;
1451     char *t, *cptr, *e, *cp=NULL;
1452 
1453     if (OPAL_SUCCESS != opal_os_dirpath_access("/proc/elog", S_IRUSR)) {
1454         /* if the file isn't there, or we don't have permission
1455          * to read it, then we are not on a coprocessor so far
1456          * as we can tell
1457          */
1458         return NULL;
1459     }
1460     if (NULL == (fp = fopen("/proc/elog", "r"))) {
1461         /* nothing we can do */
1462         return NULL;
1463     }
1464     /* look for the line containing the serial number of this
1465      * card - usually the first line in the file
1466      */
1467     while (NULL != (cptr = hwloc_getline(fp))) {
1468         if (NULL != (t = strstr(cptr, "Card"))) {
1469             /* we want the string right after this - delimited by
1470              * a colon at the end
1471              */
1472             t += 5;  // move past "Card "
1473             if (NULL == (e = strchr(t, ':'))) {
1474                 /* not what we were expecting */
1475                 free(cptr);
1476                 continue;
1477             }
1478             *e = '\0';
1479             cp = strdup(t);
1480             free(cptr);
1481             break;
1482         }
1483         free(cptr);
1484     }
1485     fclose(fp);
1486     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
1487                          "hwloc:base:check_coprocessor: on coprocessor %s",
1488                          (NULL == cp) ? "NONE" : cp));
1489     return cp;
1490 }
1491 
1492 char* opal_hwloc_base_print_binding(opal_binding_policy_t binding)
1493 {
1494     char *ret, *bind;
1495     opal_hwloc_print_buffers_t *ptr;
1496 
1497     switch(OPAL_GET_BINDING_POLICY(binding)) {
1498     case OPAL_BIND_TO_NONE:
1499         bind = "NONE";
1500         break;
1501     case OPAL_BIND_TO_BOARD:
1502         bind = "BOARD";
1503         break;
1504     case OPAL_BIND_TO_NUMA:
1505         bind = "NUMA";
1506         break;
1507     case OPAL_BIND_TO_SOCKET:
1508         bind = "SOCKET";
1509         break;
1510     case OPAL_BIND_TO_L3CACHE:
1511         bind = "L3CACHE";
1512         break;
1513     case OPAL_BIND_TO_L2CACHE:
1514         bind = "L2CACHE";
1515         break;
1516     case OPAL_BIND_TO_L1CACHE:
1517         bind = "L1CACHE";
1518         break;
1519     case OPAL_BIND_TO_CORE:
1520         bind = "CORE";
1521         break;
1522     case OPAL_BIND_TO_HWTHREAD:
1523         bind = "HWTHREAD";
1524         break;
1525     case OPAL_BIND_TO_CPUSET:
1526         bind = "CPUSET";
1527         break;
1528     default:
1529         bind = "UNKNOWN";
1530     }
1531     ptr = opal_hwloc_get_print_buffer();
1532     if (NULL == ptr) {
1533         return opal_hwloc_print_null;
1534     }
1535     /* cycle around the ring */
1536     if (OPAL_HWLOC_PRINT_NUM_BUFS == ptr->cntr) {
1537         ptr->cntr = 0;
1538     }
1539     if (!OPAL_BINDING_REQUIRED(binding) &&
1540         OPAL_BIND_OVERLOAD_ALLOWED(binding)) {
1541         snprintf(ptr->buffers[ptr->cntr], OPAL_HWLOC_PRINT_MAX_SIZE,
1542                  "%s:IF-SUPPORTED:OVERLOAD-ALLOWED", bind);
1543     } else if (OPAL_BIND_OVERLOAD_ALLOWED(binding)) {
1544         snprintf(ptr->buffers[ptr->cntr], OPAL_HWLOC_PRINT_MAX_SIZE,
1545                  "%s:OVERLOAD-ALLOWED", bind);
1546     } else if (!OPAL_BINDING_REQUIRED(binding)) {
1547         snprintf(ptr->buffers[ptr->cntr], OPAL_HWLOC_PRINT_MAX_SIZE,
1548                  "%s:IF-SUPPORTED", bind);
1549     } else {
1550         snprintf(ptr->buffers[ptr->cntr], OPAL_HWLOC_PRINT_MAX_SIZE, "%s", bind);
1551     }
1552     ret = ptr->buffers[ptr->cntr];
1553     ptr->cntr++;
1554 
1555     return ret;
1556 }
1557 
1558 /*
1559  * Turn an int bitmap to a "a-b,c" range kind of string
1560  */
1561 static char *bitmap2rangestr(int bitmap)
1562 {
1563     size_t i;
1564     int range_start, range_end;
1565     bool first, isset;
1566     char tmp[BUFSIZ];
1567     const int stmp = sizeof(tmp) - 1;
1568     static char ret[BUFSIZ];
1569 
1570     memset(ret, 0, sizeof(ret));
1571 
1572     first = true;
1573     range_start = -999;
1574     for (i = 0; i < sizeof(int) * 8; ++i) {
1575         isset = (bitmap & (1 << i));
1576 
1577         /* Do we have a running range? */
1578         if (range_start >= 0) {
1579             if (isset) {
1580                 continue;
1581             } else {
1582                 /* A range just ended; output it */
1583                 if (!first) {
1584                     strncat(ret, ",", sizeof(ret) - strlen(ret) - 1);
1585                 } else {
1586                     first = false;
1587                 }
1588 
1589                 range_end = i - 1;
1590                 if (range_start == range_end) {
1591                     snprintf(tmp, stmp, "%d", range_start);
1592                 } else {
1593                     snprintf(tmp, stmp, "%d-%d", range_start, range_end);
1594                 }
1595                 strncat(ret, tmp, sizeof(ret) - strlen(ret) - 1);
1596 
1597                 range_start = -999;
1598             }
1599         }
1600 
1601         /* No running range */
1602         else {
1603             if (isset) {
1604                 range_start = i;
1605             }
1606         }
1607     }
1608 
1609     /* If we ended the bitmap with a range open, output it */
1610     if (range_start >= 0) {
1611         if (!first) {
1612             strncat(ret, ",", sizeof(ret) - strlen(ret) - 1);
1613             first = false;
1614         }
1615 
1616         range_end = i - 1;
1617         if (range_start == range_end) {
1618             snprintf(tmp, stmp, "%d", range_start);
1619         } else {
1620             snprintf(tmp, stmp, "%d-%d", range_start, range_end);
1621         }
1622         strncat(ret, tmp, sizeof(ret) - strlen(ret) - 1);
1623     }
1624 
1625     return ret;
1626 }
1627 
1628 /*
1629  * Make a map of socket/core/hwthread tuples
1630  */
1631 static int build_map(int *num_sockets_arg, int *num_cores_arg,
1632                      hwloc_cpuset_t cpuset, int ***map, hwloc_topology_t topo)
1633 {
1634     int num_sockets, num_cores;
1635     int socket_index, core_index, pu_index;
1636     hwloc_obj_t socket, core, pu;
1637     int **data;
1638 
1639     /* Find out how many sockets we have */
1640     num_sockets = hwloc_get_nbobjs_by_type(topo, HWLOC_OBJ_SOCKET);
1641     /* some systems (like the iMac) only have one
1642      * socket and so don't report a socket
1643      */
1644     if (0 == num_sockets) {
1645         num_sockets = 1;
1646     }
1647     /* Lazy: take the total number of cores that we have in the
1648        topology; that'll be more than the max number of cores
1649        under any given socket */
1650     num_cores = hwloc_get_nbobjs_by_type(topo, HWLOC_OBJ_CORE);
1651     *num_sockets_arg = num_sockets;
1652     *num_cores_arg = num_cores;
1653 
1654     /* Alloc a 2D array: sockets x cores. */
1655     data = malloc(num_sockets * sizeof(int *));
1656     if (NULL == data) {
1657         return OPAL_ERR_OUT_OF_RESOURCE;
1658     }
1659     data[0] = calloc(num_sockets * num_cores, sizeof(int));
1660     if (NULL == data[0]) {
1661         free(data);
1662         return OPAL_ERR_OUT_OF_RESOURCE;
1663     }
1664     for (socket_index = 1; socket_index < num_sockets; ++socket_index) {
1665         data[socket_index] = data[socket_index - 1] + num_cores;
1666     }
1667 
1668     /* Iterate the PUs in this cpuset; fill in the data[][] array with
1669        the socket/core/pu triples */
1670     for (pu_index = 0,
1671              pu = hwloc_get_obj_inside_cpuset_by_type(topo,
1672                                                       cpuset, HWLOC_OBJ_PU,
1673                                                       pu_index);
1674          NULL != pu;
1675          pu = hwloc_get_obj_inside_cpuset_by_type(topo,
1676                                                   cpuset, HWLOC_OBJ_PU,
1677                                                   ++pu_index)) {
1678         /* Go upward and find the core this PU belongs to */
1679         core = pu;
1680         while (NULL != core && core->type != HWLOC_OBJ_CORE) {
1681             core = core->parent;
1682         }
1683         core_index = 0;
1684         if (NULL != core) {
1685             core_index = core->logical_index;
1686         }
1687 
1688         /* Go upward and find the socket this PU belongs to */
1689         socket = pu;
1690         while (NULL != socket && socket->type != HWLOC_OBJ_SOCKET) {
1691             socket = socket->parent;
1692         }
1693         socket_index = 0;
1694         if (NULL != socket) {
1695             socket_index = socket->logical_index;
1696         }
1697 
1698         /* Save this socket/core/pu combo.  LAZY: Assuming that we
1699            won't have more PU's per core than (sizeof(int)*8). */
1700         data[socket_index][core_index] |= (1 << pu->sibling_rank);
1701     }
1702 
1703     *map = data;
1704     return OPAL_SUCCESS;
1705 }
1706 
1707 /*
1708  * Make a prettyprint string for a hwloc_cpuset_t
1709  */
1710 int opal_hwloc_base_cset2str(char *str, int len,
1711                              hwloc_topology_t topo,
1712                              hwloc_cpuset_t cpuset)
1713 {
1714     bool first;
1715     int num_sockets, num_cores;
1716     int ret, socket_index, core_index;
1717     char tmp[BUFSIZ];
1718     const int stmp = sizeof(tmp) - 1;
1719     int **map=NULL;
1720     hwloc_obj_t root;
1721     opal_hwloc_topo_data_t *sum;
1722 
1723     str[0] = tmp[stmp] = '\0';
1724 
1725     /* if the cpuset is all zero, then not bound */
1726     if (hwloc_bitmap_iszero(cpuset)) {
1727         return OPAL_ERR_NOT_BOUND;
1728     }
1729 
1730     /* if the cpuset includes all available cpus, then we are unbound */
1731     root = hwloc_get_root_obj(topo);
1732     if (NULL != root->userdata) {
1733         sum = (opal_hwloc_topo_data_t*)root->userdata;
1734         if (NULL == sum->available) {
1735            return OPAL_ERROR;
1736         }
1737         if (0 != hwloc_bitmap_isincluded(sum->available, cpuset)) {
1738             return OPAL_ERR_NOT_BOUND;
1739         }
1740     }
1741 
1742     if (OPAL_SUCCESS != (ret = build_map(&num_sockets, &num_cores, cpuset, &map, topo))) {
1743         return ret;
1744     }
1745     /* Iterate over the data matrix and build up the string */
1746     first = true;
1747     for (socket_index = 0; socket_index < num_sockets; ++socket_index) {
1748         for (core_index = 0; core_index < num_cores; ++core_index) {
1749             if (map[socket_index][core_index] > 0) {
1750                 if (!first) {
1751                     strncat(str, ", ", len - strlen(str) - 1);
1752                 }
1753                 first = false;
1754 
1755                 snprintf(tmp, stmp, "socket %d[core %d[hwt %s]]",
1756                          socket_index, core_index,
1757                          bitmap2rangestr(map[socket_index][core_index]));
1758                 strncat(str, tmp, len - strlen(str) - 1);
1759             }
1760         }
1761     }
1762     if (NULL != map) {
1763         if (NULL != map[0]) {
1764             free(map[0]);
1765         }
1766         free(map);
1767     }
1768 
1769     return OPAL_SUCCESS;
1770 }
1771 
1772 /*
1773  * Make a prettyprint string for a cset in a map format.
1774  * Example: [B./..]
1775  * Key:  [] - signifies socket
1776  *        / - divider between cores
1777  *        . - signifies PU a process not bound to
1778  *        B - signifies PU a process is bound to
1779  */
1780 int opal_hwloc_base_cset2mapstr(char *str, int len,
1781                                 hwloc_topology_t topo,
1782                                 hwloc_cpuset_t cpuset)
1783 {
1784     char tmp[BUFSIZ];
1785     int core_index, pu_index;
1786     const int stmp = sizeof(tmp) - 1;
1787     hwloc_obj_t socket, core, pu;
1788     hwloc_obj_t root;
1789     opal_hwloc_topo_data_t *sum;
1790 
1791     str[0] = tmp[stmp] = '\0';
1792 
1793     /* if the cpuset is all zero, then not bound */
1794     if (hwloc_bitmap_iszero(cpuset)) {
1795         return OPAL_ERR_NOT_BOUND;
1796     }
1797 
1798     /* if the cpuset includes all available cpus, then we are unbound */
1799     root = hwloc_get_root_obj(topo);
1800     if (NULL != root->userdata) {
1801         sum = (opal_hwloc_topo_data_t*)root->userdata;
1802         if (NULL == sum->available) {
1803            return OPAL_ERROR;
1804         }
1805         if (0 != hwloc_bitmap_isincluded(sum->available, cpuset)) {
1806             return OPAL_ERR_NOT_BOUND;
1807         }
1808     }
1809 
1810     /* Iterate over all existing sockets */
1811     for (socket = hwloc_get_obj_by_type(topo, HWLOC_OBJ_SOCKET, 0);
1812          NULL != socket;
1813          socket = socket->next_cousin) {
1814         strncat(str, "[", len - strlen(str) - 1);
1815 
1816         /* Iterate over all existing cores in this socket */
1817         core_index = 0;
1818         for (core = hwloc_get_obj_inside_cpuset_by_type(topo,
1819                                                         socket->cpuset,
1820                                                         HWLOC_OBJ_CORE, core_index);
1821              NULL != core;
1822              core = hwloc_get_obj_inside_cpuset_by_type(topo,
1823                                                         socket->cpuset,
1824                                                         HWLOC_OBJ_CORE, ++core_index)) {
1825             if (core_index > 0) {
1826                 strncat(str, "/", len - strlen(str) - 1);
1827             }
1828 
1829             /* Iterate over all existing PUs in this core */
1830             pu_index = 0;
1831             for (pu = hwloc_get_obj_inside_cpuset_by_type(topo,
1832                                                           core->cpuset,
1833                                                           HWLOC_OBJ_PU, pu_index);
1834                  NULL != pu;
1835                  pu = hwloc_get_obj_inside_cpuset_by_type(topo,
1836                                                           core->cpuset,
1837                                                           HWLOC_OBJ_PU, ++pu_index)) {
1838 
1839                 /* Is this PU in the cpuset? */
1840                 if (hwloc_bitmap_isset(cpuset, pu->os_index)) {
1841                     strncat(str, "B", len - strlen(str) - 1);
1842                 } else {
1843                     strncat(str, ".", len - strlen(str) - 1);
1844                 }
1845             }
1846         }
1847         strncat(str, "]", len - strlen(str) - 1);
1848     }
1849 
1850     return OPAL_SUCCESS;
1851 }
1852 
1853 static int dist_cmp_fn (opal_list_item_t **a, opal_list_item_t **b)
1854 {
1855     opal_rmaps_numa_node_t *aitem = *((opal_rmaps_numa_node_t **) a);
1856     opal_rmaps_numa_node_t *bitem = *((opal_rmaps_numa_node_t **) b);
1857 
1858     if (aitem->dist_from_closed > bitem->dist_from_closed) {
1859         return 1;
1860     } else if( aitem->dist_from_closed == bitem->dist_from_closed ) {
1861         return 0;
1862     } else {
1863         return -1;
1864     }
1865 }
1866 
1867 static void sort_by_dist(hwloc_topology_t topo, char* device_name, opal_list_t *sorted_list)
1868 {
1869     hwloc_obj_t device_obj = NULL;
1870     hwloc_obj_t obj = NULL;
1871     struct hwloc_distances_s* distances;
1872     opal_rmaps_numa_node_t *numa_node;
1873     int close_node_index;
1874     float latency;
1875     unsigned int j;
1876 #if HWLOC_API_VERSION < 0x20000
1877     hwloc_obj_t root = NULL;
1878     int depth;
1879     unsigned i;
1880 #else
1881     unsigned distances_nr = 0;
1882 #endif
1883 
1884     for (device_obj = hwloc_get_obj_by_type(topo, HWLOC_OBJ_OS_DEVICE, 0); device_obj; device_obj = hwloc_get_next_osdev(topo, device_obj)) {
1885         if (device_obj->attr->osdev.type == HWLOC_OBJ_OSDEV_OPENFABRICS
1886                 || device_obj->attr->osdev.type == HWLOC_OBJ_OSDEV_NETWORK) {
1887             if (!strcmp(device_obj->name, device_name)) {
1888                 /* find numa node containing this device */
1889                 obj = device_obj->parent;
1890 #if HWLOC_API_VERSION < 0x20000
1891                 while ((obj != NULL) && (obj->type != HWLOC_OBJ_NODE)) {
1892                     obj = obj->parent;
1893                 }
1894 #else
1895                 while (obj && !obj->memory_arity) {
1896                     obj = obj->parent; /* no memory child, walk up */
1897                 }
1898                 if (obj != NULL) {
1899                     obj = obj->memory_first_child;
1900                 }
1901 #endif
1902                 if (obj == NULL) {
1903                     opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
1904                             "hwloc:base:get_sorted_numa_list: NUMA node closest to %s wasn't found.",
1905                             device_name);
1906                     return;
1907                 } else {
1908                     close_node_index = obj->logical_index;
1909                 }
1910 
1911                 /* find distance matrix for all numa nodes */
1912 #if HWLOC_API_VERSION < 0x20000
1913                 distances = (struct hwloc_distances_s*)hwloc_get_whole_distance_matrix_by_type(topo, HWLOC_OBJ_NODE);
1914                 if (NULL ==  distances) {
1915                     /* we can try to find distances under group object. This info can be there. */
1916                     depth = hwloc_get_type_depth(topo, HWLOC_OBJ_NODE);
1917                     if (HWLOC_TYPE_DEPTH_UNKNOWN == depth) {
1918                         opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
1919                                 "hwloc:base:get_sorted_numa_list: There is no information about distances on the node.");
1920                         return;
1921                     }
1922                     root = hwloc_get_root_obj(topo);
1923                     for (i = 0; i < root->arity; i++) {
1924                         obj = root->children[i];
1925                         if (obj->distances_count > 0) {
1926                             for(j = 0; j < obj->distances_count; j++) {
1927                                 if (obj->distances[j]->relative_depth + 1 == (unsigned) depth) {
1928                                     distances = obj->distances[j];
1929                                     break;
1930                                 }
1931                             }
1932                         }
1933                     }
1934                 }
1935                 /* find all distances for our close node with logical index = close_node_index as close_node_index + nbobjs*j */
1936                 if ((NULL == distances) || (0 == distances->nbobjs)) {
1937                     opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
1938                             "hwloc:base:get_sorted_numa_list: There is no information about distances on the node.");
1939                     return;
1940                 }
1941                 /* fill list of numa nodes */
1942                 for (j = 0; j < distances->nbobjs; j++) {
1943                     latency = distances->latency[close_node_index + distances->nbobjs * j];
1944                     numa_node = OBJ_NEW(opal_rmaps_numa_node_t);
1945                     numa_node->index = j;
1946                     numa_node->dist_from_closed = latency;
1947                     opal_list_append(sorted_list, &numa_node->super);
1948                 }
1949 #else
1950                 distances_nr = 1;
1951                 if (0 != hwloc_distances_get_by_type(topo, HWLOC_OBJ_NODE, &distances_nr, &distances,
1952                                                      HWLOC_DISTANCES_KIND_MEANS_LATENCY, 0) || 0 == distances_nr) {
1953                     opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
1954                             "hwloc:base:get_sorted_numa_list: There is no information about distances on the node.");
1955                     return;
1956                 }
1957                 /* fill list of numa nodes */
1958                 for (j = 0; j < distances->nbobjs; j++) {
1959                     latency = distances->values[close_node_index + distances->nbobjs * j];
1960                     numa_node = OBJ_NEW(opal_rmaps_numa_node_t);
1961                     numa_node->index = j;
1962                     numa_node->dist_from_closed = latency;
1963                     opal_list_append(sorted_list, &numa_node->super);
1964                 }
1965                 hwloc_distances_release(topo, distances);
1966 #endif
1967                 /* sort numa nodes by distance from the closest one to PCI */
1968                 opal_list_sort(sorted_list, dist_cmp_fn);
1969                 return;
1970             }
1971         }
1972     }
1973 }
1974 
1975 static int find_devices(hwloc_topology_t topo, char** device_name)
1976 {
1977     hwloc_obj_t device_obj = NULL;
1978     int count = 0;
1979     for (device_obj = hwloc_get_obj_by_type(topo, HWLOC_OBJ_OS_DEVICE, 0); device_obj; device_obj = hwloc_get_next_osdev(topo, device_obj)) {
1980         if (device_obj->attr->osdev.type == HWLOC_OBJ_OSDEV_OPENFABRICS) {
1981             count++;
1982             free(*device_name);
1983             *device_name = strdup(device_obj->name);
1984         }
1985     }
1986     return count;
1987 }
1988 
1989 int opal_hwloc_get_sorted_numa_list(hwloc_topology_t topo, char* device_name, opal_list_t *sorted_list)
1990 {
1991     hwloc_obj_t obj;
1992     opal_hwloc_summary_t *sum;
1993     opal_hwloc_topo_data_t *data;
1994     opal_rmaps_numa_node_t *numa, *copy_numa;
1995     int count;
1996 
1997     obj = hwloc_get_root_obj(topo);
1998 
1999     /* first see if the topology already has this info */
2000     /* we call opal_hwloc_base_get_nbobjs_by_type() before it to fill summary object so it should exist*/
2001     data = (opal_hwloc_topo_data_t*)obj->userdata;
2002     if (NULL != data) {
2003         OPAL_LIST_FOREACH(sum, &data->summaries, opal_hwloc_summary_t) {
2004             if (HWLOC_OBJ_NODE == sum->type) {
2005                 if (opal_list_get_size(&sum->sorted_by_dist_list) > 0) {
2006                     OPAL_LIST_FOREACH(numa, &(sum->sorted_by_dist_list), opal_rmaps_numa_node_t) {
2007                         copy_numa = OBJ_NEW(opal_rmaps_numa_node_t);
2008                         copy_numa->index = numa->index;
2009                         copy_numa->dist_from_closed = numa->dist_from_closed;
2010                         opal_list_append(sorted_list, &copy_numa->super);
2011                     }
2012                     return OPAL_SUCCESS;
2013                 }else {
2014                     /* don't already know it - go get it */
2015                     /* firstly we check if we need to autodetect OpenFabrics  devices or we have the specified one */
2016                     bool free_device_name = false;
2017                     if (!strcmp(device_name, "auto")) {
2018                         count = find_devices(topo, &device_name);
2019                         if (count > 1) {
2020                             free(device_name);
2021                             return count;
2022                         }
2023                         free_device_name = true;
2024                     }
2025                     if (!device_name) {
2026                         return OPAL_ERR_NOT_FOUND;
2027                     } else if (free_device_name && (0 == strlen(device_name))) {
2028                         free(device_name);
2029                         return OPAL_ERR_NOT_FOUND;
2030                     }
2031                     sort_by_dist(topo, device_name, sorted_list);
2032                     if (free_device_name) {
2033                         free(device_name);
2034                     }
2035                     /* store this info in summary object for later usage */
2036                     OPAL_LIST_FOREACH(numa, sorted_list, opal_rmaps_numa_node_t) {
2037                         copy_numa = OBJ_NEW(opal_rmaps_numa_node_t);
2038                         copy_numa->index = numa->index;
2039                         copy_numa->dist_from_closed = numa->dist_from_closed;
2040                         opal_list_append(&(sum->sorted_by_dist_list), &copy_numa->super);
2041                     }
2042                     return OPAL_SUCCESS;
2043                 }
2044             }
2045         }
2046     }
2047     return OPAL_ERR_NOT_FOUND;
2048 }
2049 
2050 char* opal_hwloc_base_get_topo_signature(hwloc_topology_t topo)
2051 {
2052     int nnuma, nsocket, nl3, nl2, nl1, ncore, nhwt;
2053     char *sig=NULL, *arch = NULL, *endian;
2054     hwloc_obj_t obj;
2055     unsigned i;
2056 
2057     nnuma = opal_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_NODE, 0, OPAL_HWLOC_AVAILABLE);
2058     nsocket = opal_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_SOCKET, 0, OPAL_HWLOC_AVAILABLE);
2059     nl3 = opal_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_L3CACHE, 3, OPAL_HWLOC_AVAILABLE);
2060     nl2 = opal_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_L2CACHE, 2, OPAL_HWLOC_AVAILABLE);
2061     nl1 = opal_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_L1CACHE, 1, OPAL_HWLOC_AVAILABLE);
2062     ncore = opal_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_CORE, 0, OPAL_HWLOC_AVAILABLE);
2063     nhwt = opal_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_PU, 0, OPAL_HWLOC_AVAILABLE);
2064 
2065     /* get the root object so we can add the processor architecture */
2066     obj = hwloc_get_root_obj(topo);
2067     for (i=0; i < obj->infos_count; i++) {
2068         if (0 == strcmp(obj->infos[i].name, "Architecture")) {
2069             arch = obj->infos[i].value;
2070             break;
2071         }
2072     }
2073     if (NULL == arch) {
2074         arch = "unknown";
2075     }
2076 
2077 #ifdef __BYTE_ORDER
2078 #if __BYTE_ORDER == __LITTLE_ENDIAN
2079     endian = "le";
2080 #else
2081     endian = "be";
2082 #endif
2083 #else
2084     endian = "unknown";
2085 #endif
2086 
2087     opal_asprintf(&sig, "%dN:%dS:%dL3:%dL2:%dL1:%dC:%dH:%s:%s",
2088              nnuma, nsocket, nl3, nl2, nl1, ncore, nhwt, arch, endian);
2089     return sig;
2090 }
2091 
2092 char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
2093                                           char *bitmap)
2094 {
2095     hwloc_obj_t obj;
2096     char *locality=NULL, *tmp, *t2;
2097     unsigned depth, d, width, w;
2098     hwloc_cpuset_t cpuset, result;
2099     hwloc_obj_type_t type;
2100 
2101     /* if this proc is not bound, then there is no locality. We
2102      * know it isn't bound if the cpuset is NULL, or if it is
2103      * all 1's */
2104     if (NULL == bitmap) {
2105         return NULL;
2106     }
2107     cpuset = hwloc_bitmap_alloc();
2108     hwloc_bitmap_list_sscanf(cpuset, bitmap);
2109     if (hwloc_bitmap_isfull(cpuset)) {
2110         hwloc_bitmap_free(cpuset);
2111         return NULL;
2112     }
2113 
2114     /* we are going to use a bitmap to save the results so
2115      * that we can use a hwloc utility to print them */
2116     result = hwloc_bitmap_alloc();
2117 
2118     /* get the max depth of the topology */
2119     depth = hwloc_topology_get_depth(topo);
2120 
2121     /* start at the first depth below the top machine level */
2122     for (d=1; d < depth; d++) {
2123         /* get the object type at this depth */
2124         type = hwloc_get_depth_type(topo, d);
2125         /* if it isn't one of interest, then ignore it */
2126         if (HWLOC_OBJ_NODE != type &&
2127             HWLOC_OBJ_SOCKET != type &&
2128 #if HWLOC_API_VERSION < 0x20000
2129             HWLOC_OBJ_CACHE != type &&
2130 #else
2131             HWLOC_OBJ_L1CACHE != type &&
2132             HWLOC_OBJ_L2CACHE != type &&
2133             HWLOC_OBJ_L3CACHE != type &&
2134 #endif
2135             HWLOC_OBJ_CORE != type &&
2136             HWLOC_OBJ_PU != type) {
2137             continue;
2138         }
2139 
2140         /* get the width of the topology at this depth */
2141         width = hwloc_get_nbobjs_by_depth(topo, d);
2142         if (0 == width) {
2143             continue;
2144         }
2145 
2146         /* scan all objects at this depth to see if
2147          * the location overlaps with them
2148          */
2149         for (w=0; w < width; w++) {
2150             /* get the object at this depth/index */
2151             obj = hwloc_get_obj_by_depth(topo, d, w);
2152             /* see if the location intersects with it */
2153             if (hwloc_bitmap_intersects(obj->cpuset, cpuset)) {
2154                 hwloc_bitmap_set(result, w);
2155             }
2156         }
2157         /* it should be impossible, but allow for the possibility
2158          * that we came up empty at this depth */
2159         if (!hwloc_bitmap_iszero(result)) {
2160             hwloc_bitmap_list_asprintf(&tmp, result);
2161             switch(obj->type) {
2162                 case HWLOC_OBJ_NODE:
2163                     opal_asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp);
2164                     if (NULL != locality) {
2165                         free(locality);
2166                     }
2167                     locality = t2;
2168                     break;
2169                 case HWLOC_OBJ_SOCKET:
2170                     opal_asprintf(&t2, "%sSK%s:", (NULL == locality) ? "" : locality, tmp);
2171                     if (NULL != locality) {
2172                         free(locality);
2173                     }
2174                     locality = t2;
2175                     break;
2176 #if HWLOC_API_VERSION < 0x20000
2177                 case HWLOC_OBJ_CACHE:
2178                     if (3 == obj->attr->cache.depth) {
2179                         opal_asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp);
2180                         if (NULL != locality) {
2181                             free(locality);
2182                         }
2183                         locality = t2;
2184                         break;
2185                     } else if (2 == obj->attr->cache.depth) {
2186                         opal_asprintf(&t2, "%sL2%s:", (NULL == locality) ? "" : locality, tmp);
2187                         if (NULL != locality) {
2188                             free(locality);
2189                         }
2190                         locality = t2;
2191                         break;
2192                     } else {
2193                         opal_asprintf(&t2, "%sL1%s:", (NULL == locality) ? "" : locality, tmp);
2194                         if (NULL != locality) {
2195                             free(locality);
2196                         }
2197                         locality = t2;
2198                         break;
2199                     }
2200                     break;
2201 #else
2202                 case HWLOC_OBJ_L3CACHE:
2203                     opal_asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp);
2204                     if (NULL != locality) {
2205                         free(locality);
2206                     }
2207                     locality = t2;
2208                     break;
2209                 case HWLOC_OBJ_L2CACHE:
2210                     opal_asprintf(&t2, "%sL2%s:", (NULL == locality) ? "" : locality, tmp);
2211                     if (NULL != locality) {
2212                         free(locality);
2213                     }
2214                     locality = t2;
2215                     break;
2216                 case HWLOC_OBJ_L1CACHE:
2217                     opal_asprintf(&t2, "%sL1%s:", (NULL == locality) ? "" : locality, tmp);
2218                     if (NULL != locality) {
2219                         free(locality);
2220                     }
2221                     locality = t2;
2222                     break;
2223 #endif
2224                 case HWLOC_OBJ_CORE:
2225                     opal_asprintf(&t2, "%sCR%s:", (NULL == locality) ? "" : locality, tmp);
2226                     if (NULL != locality) {
2227                         free(locality);
2228                     }
2229                     locality = t2;
2230                     break;
2231                 case HWLOC_OBJ_PU:
2232                     opal_asprintf(&t2, "%sHT%s:", (NULL == locality) ? "" : locality, tmp);
2233                     if (NULL != locality) {
2234                         free(locality);
2235                     }
2236                     locality = t2;
2237                     break;
2238                 default:
2239                     /* just ignore it */
2240                     break;
2241             }
2242             free(tmp);
2243         }
2244         hwloc_bitmap_zero(result);
2245     }
2246     hwloc_bitmap_free(result);
2247     hwloc_bitmap_free(cpuset);
2248 
2249     /* remove the trailing colon */
2250     if (NULL != locality) {
2251         locality[strlen(locality)-1] = '\0';
2252     }
2253     return locality;
2254 }
2255 
2256 char* opal_hwloc_base_get_location(char *locality,
2257                                    hwloc_obj_type_t type,
2258                                    unsigned index)
2259 {
2260     char **loc;
2261     char *srch, *ans = NULL;
2262     size_t n;
2263 
2264     if (NULL == locality) {
2265         return NULL;
2266     }
2267     switch(type) {
2268         case HWLOC_OBJ_NODE:
2269             srch = "NM";
2270             break;
2271         case HWLOC_OBJ_SOCKET:
2272             srch = "SK";
2273             break;
2274 #if HWLOC_API_VERSION < 0x20000
2275         case HWLOC_OBJ_CACHE:
2276             if (3 == index) {
2277                 srch = "L3";
2278             } else if (2 == index) {
2279                 srch = "L2";
2280             } else {
2281                 srch = "L0";
2282             }
2283             break;
2284 #else
2285         case HWLOC_OBJ_L3CACHE:
2286             srch = "L3";
2287             break;
2288         case HWLOC_OBJ_L2CACHE:
2289             srch = "L2";
2290             break;
2291         case HWLOC_OBJ_L1CACHE:
2292             srch = "L0";
2293             break;
2294 #endif
2295         case HWLOC_OBJ_CORE:
2296             srch = "CR";
2297             break;
2298         case HWLOC_OBJ_PU:
2299             srch = "HT";
2300             break;
2301         default:
2302             return NULL;
2303     }
2304     loc = opal_argv_split(locality, ':');
2305     for (n=0; NULL != loc[n]; n++) {
2306         if (0 == strncmp(loc[n], srch, 2)) {
2307             ans = strdup(&loc[n][2]);
2308             break;
2309         }
2310     }
2311     opal_argv_free(loc);
2312 
2313     return ans;
2314 }
2315 
2316 opal_hwloc_locality_t opal_hwloc_compute_relative_locality(char *loc1, char *loc2)
2317 {
2318     opal_hwloc_locality_t locality;
2319     char **set1, **set2;
2320     hwloc_bitmap_t bit1, bit2;
2321     size_t n1, n2;
2322 
2323     /* start with what we know - they share a node on a cluster
2324      * NOTE: we may alter that latter part as hwloc's ability to
2325      * sense multi-cu, multi-cluster systems grows
2326      */
2327     locality = OPAL_PROC_ON_NODE | OPAL_PROC_ON_HOST | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER;
2328 
2329     /* if either location is NULL, then that isn't bound */
2330     if (NULL == loc1 || NULL == loc2) {
2331         return locality;
2332     }
2333 
2334     set1 = opal_argv_split(loc1, ':');
2335     set2 = opal_argv_split(loc2, ':');
2336     bit1 = hwloc_bitmap_alloc();
2337     bit2 = hwloc_bitmap_alloc();
2338 
2339     /* check each matching type */
2340     for (n1=0; NULL != set1[n1]; n1++) {
2341         /* convert the location into bitmap */
2342         hwloc_bitmap_list_sscanf(bit1, &set1[n1][2]);
2343         /* find the matching type in set2 */
2344         for (n2=0; NULL != set2[n2]; n2++) {
2345             if (0 == strncmp(set1[n1], set2[n2], 2)) {
2346                 /* convert the location into bitmap */
2347                 hwloc_bitmap_list_sscanf(bit2, &set2[n2][2]);
2348                 /* see if they intersect */
2349                 if (hwloc_bitmap_intersects(bit1, bit2)) {
2350                     /* set the corresponding locality bit */
2351                     if (0 == strncmp(set1[n1], "NM", 2)) {
2352                         locality |= OPAL_PROC_ON_NUMA;
2353                     } else if (0 == strncmp(set1[n1], "SK", 2)) {
2354                         locality |= OPAL_PROC_ON_SOCKET;
2355                     } else if (0 == strncmp(set1[n1], "L3", 2)) {
2356                         locality |= OPAL_PROC_ON_L3CACHE;
2357                     } else if (0 == strncmp(set1[n1], "L2", 2)) {
2358                         locality |= OPAL_PROC_ON_L2CACHE;
2359                     } else if (0 == strncmp(set1[n1], "L1", 2)) {
2360                         locality |= OPAL_PROC_ON_L1CACHE;
2361                     } else if (0 == strncmp(set1[n1], "CR", 2)) {
2362                         locality |= OPAL_PROC_ON_CORE;
2363                     } else if (0 == strncmp(set1[n1], "HT", 2)) {
2364                         locality |= OPAL_PROC_ON_HWTHREAD;
2365                     } else {
2366                         /* should never happen */
2367                         opal_output(0, "UNRECOGNIZED LOCALITY %s", set1[n1]);
2368                     }
2369                 }
2370                 break;
2371             }
2372         }
2373     }
2374     opal_argv_free(set1);
2375     opal_argv_free(set2);
2376     hwloc_bitmap_free(bit1);
2377     hwloc_bitmap_free(bit2);
2378     return locality;
2379 }
2380 
2381 int opal_hwloc_base_topology_export_xmlbuffer(hwloc_topology_t topology, char **xmlpath, int *buflen) {
2382 #if HWLOC_API_VERSION < 0x20000
2383     return hwloc_topology_export_xmlbuffer(topology, xmlpath, buflen);
2384 #else
2385     return hwloc_topology_export_xmlbuffer(topology, xmlpath, buflen, 0);
2386 #endif
2387 }
2388 
2389 int opal_hwloc_base_topology_set_flags (hwloc_topology_t topology, unsigned long flags, bool io) {
2390     if (io) {
2391 #if HWLOC_API_VERSION < 0x20000
2392         flags |= HWLOC_TOPOLOGY_FLAG_IO_DEVICES;
2393 #else
2394         int ret = hwloc_topology_set_io_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
2395         if (0 != ret) return ret;
2396 #endif
2397     }
2398     return hwloc_topology_set_flags(topology, flags);
2399 }
/* [<][>][^][v][top][bottom][index][help] */
root/opal/mca/hwloc/base/hwloc_base_util.c

DEFINITIONS
