opal/mca/hwloc/hwloc201/hwloc/hwloc/topology-bgq.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
hwloc_bgq__get_allowed_resources
hwloc_look_bgq
hwloc_bgq_get_thread_cpubind
hwloc_bgq_get_thisthread_cpubind
hwloc_bgq_set_thread_cpubind
hwloc_bgq_set_thisthread_cpubind
hwloc_bgq_get_allowed_resources
hwloc_set_bgq_hooks
hwloc_bgq_component_instantiate
   1 /*
   2  * Copyright © 2013-2018 Inria.  All rights reserved.
   3  * See COPYING in top-level directory.
   4  */
   5 
   6 #include <private/autogen/config.h>
   7 
   8 #include <hwloc.h>
   9 #include <private/private.h>
  10 #include <private/debug.h>
  11 
  12 #include <stdlib.h>
  13 #include <pthread.h>
  14 #include <sys/utsname.h>
  15 #include <spi/include/kernel/location.h>
  16 #include <spi/include/kernel/process.h>
  17 
  18 #ifndef HWLOC_DISABLE_BGQ_PORT_TEST
  19 
  20 #define HWLOC_BGQ_CORES 17 /* spare core ignored for now */
  21 
  22 static int
  23 hwloc_bgq__get_allowed_resources(struct hwloc_topology *topology)
  24 {
  25   const char *env;
  26   unsigned i;
  27 
  28   /* mark the 17th core (OS-reserved) as disallowed */
  29   hwloc_bitmap_clr_range(topology->allowed_cpuset, (HWLOC_BGQ_CORES-1)*4, HWLOC_BGQ_CORES*4-1);
  30 
  31   if (topology->is_thissystem) { /* don't call CNK unless thissystem */
  32     env = getenv("BG_THREADMODEL");
  33     if (!env || atoi(env) != 2) {
  34       /* process cannot use cores/threads outside of its Kernel_ThreadMask() unless BG_THREADMODEL=2 */
  35       uint64_t bgmask = Kernel_ThreadMask(Kernel_MyTcoord());
  36       /* the mask is reversed, manually reverse it */
  37         for(i=0; i<64; i++)
  38         if (((bgmask >> i) & 1) == 0)
  39           hwloc_bitmap_clr(topology->allowed_cpuset, 63-i);
  40     }
  41   }
  42   return 0;
  43 }
  44 
  45 static int
  46 hwloc_look_bgq(struct hwloc_backend *backend)
  47 {
  48   struct hwloc_topology *topology = backend->topology;
  49   hwloc_bitmap_t set;
  50   hwloc_obj_t obj;
  51   unsigned i;
  52 
  53   if (topology->levels[0][0]->cpuset)
  54     /* somebody discovered things */
  55     return -1;
  56 
  57   hwloc_alloc_root_sets(topology->levels[0][0]);
  58 
  59   hwloc_bgq__get_allowed_resources(topology);
  60 
  61   /* a single memory bank */
  62   obj = hwloc_alloc_setup_object(topology, HWLOC_OBJ_NUMANODE, 0);
  63   set = hwloc_bitmap_alloc();
  64   hwloc_bitmap_set_range(set, 0, HWLOC_BGQ_CORES*4-1);
  65   obj->cpuset = set;
  66   set = hwloc_bitmap_alloc();
  67   hwloc_bitmap_set(set, 0);
  68   obj->nodeset = set;
  69   obj->attr->numanode.local_memory = 16ULL*1024*1024*1024ULL;
  70   hwloc_insert_object_by_cpuset(topology, obj);
  71   topology->support.discovery->numa = 1;
  72   topology->support.discovery->numa_memory = 1;
  73 
  74   set = hwloc_bitmap_alloc();
  75   hwloc_bitmap_set_range(set, 0, HWLOC_BGQ_CORES*4-1);
  76 
  77   /* shared L2 */
  78   if (hwloc_filter_check_keep_object_type(topology, HWLOC_OBJ_L2CACHE)) {
  79     obj = hwloc_alloc_setup_object(topology, HWLOC_OBJ_L2CACHE, HWLOC_UNKNOWN_INDEX);
  80     obj->cpuset = hwloc_bitmap_dup(set);
  81     obj->attr->cache.type = HWLOC_OBJ_CACHE_UNIFIED;
  82     obj->attr->cache.depth = 2;
  83     obj->attr->cache.size = 32*1024*1024;
  84     obj->attr->cache.linesize = 128;
  85     obj->attr->cache.associativity = 16;
  86     hwloc_insert_object_by_cpuset(topology, obj);
  87   }
  88 
  89   /* package */
  90   if (hwloc_filter_check_keep_object_type(topology, HWLOC_OBJ_PACKAGE)) {
  91     obj = hwloc_alloc_setup_object(topology, HWLOC_OBJ_PACKAGE, 0);
  92     obj->cpuset = set;
  93     hwloc_obj_add_info(obj, "CPUModel", "IBM PowerPC A2");
  94     hwloc_insert_object_by_cpuset(topology, obj);
  95   } else
  96     hwloc_bitmap_free(set);
  97 
  98   /* Cores */
  99   for(i=0; i<HWLOC_BGQ_CORES; i++) {
 100     set = hwloc_bitmap_alloc();
 101     hwloc_bitmap_set_range(set, i*4, i*4+3);
 102 
 103     /* L1d */
 104     if (hwloc_filter_check_keep_object_type(topology, HWLOC_OBJ_L1CACHE)) {
 105       obj = hwloc_alloc_setup_object(topology, HWLOC_OBJ_L1CACHE, HWLOC_UNKNOWN_INDEX);
 106       obj->cpuset = hwloc_bitmap_dup(set);
 107       obj->attr->cache.type = HWLOC_OBJ_CACHE_DATA;
 108       obj->attr->cache.depth = 1;
 109       obj->attr->cache.size = 16*1024;
 110       obj->attr->cache.linesize = 64;
 111       obj->attr->cache.associativity = 8;
 112       hwloc_insert_object_by_cpuset(topology, obj);
 113     }
 114     /* L1i */
 115     if (hwloc_filter_check_keep_object_type(topology, HWLOC_OBJ_L1ICACHE)) {
 116       obj = hwloc_alloc_setup_object(topology, HWLOC_OBJ_L1ICACHE, HWLOC_UNKNOWN_INDEX);
 117       obj->cpuset = hwloc_bitmap_dup(set);
 118       obj->attr->cache.type = HWLOC_OBJ_CACHE_INSTRUCTION;
 119       obj->attr->cache.depth = 1;
 120       obj->attr->cache.size = 16*1024;
 121       obj->attr->cache.linesize = 64;
 122       obj->attr->cache.associativity = 4;
 123       hwloc_insert_object_by_cpuset(topology, obj);
 124     }
 125     /* there's also a L1p "prefetch cache" of 4kB with 128B lines */
 126 
 127     /* Core */
 128     if (hwloc_filter_check_keep_object_type(topology, HWLOC_OBJ_CORE)) {
 129       obj = hwloc_alloc_setup_object(topology, HWLOC_OBJ_CORE, i);
 130       obj->cpuset = set;
 131       hwloc_insert_object_by_cpuset(topology, obj);
 132     } else
 133       hwloc_bitmap_free(set);
 134   }
 135 
 136   /* PUs */
 137   topology->support.discovery->pu = 1;
 138   hwloc_setup_pu_level(topology, HWLOC_BGQ_CORES*4);
 139 
 140   /* Add BGQ specific information */
 141 
 142   hwloc_obj_add_info(topology->levels[0][0], "Backend", "BGQ");
 143   hwloc_add_uname_info(topology, NULL);
 144   return 0;
 145 }
 146 
 147 static int
 148 hwloc_bgq_get_thread_cpubind(hwloc_topology_t topology, pthread_t thread, hwloc_bitmap_t hwloc_set, int flags __hwloc_attribute_unused)
 149 {
 150   unsigned pu;
 151   cpu_set_t bg_set;
 152   int err;
 153 
 154   if (topology->pid) {
 155     errno = ENOSYS;
 156     return -1;
 157   }
 158   err = pthread_getaffinity_np(thread, sizeof(bg_set), &bg_set);
 159   if (err) {
 160     errno = err;
 161     return -1;
 162   }
 163   for(pu=0; pu<64; pu++)
 164     if (CPU_ISSET(pu, &bg_set)) {
 165       /* the binding cannot contain multiple PUs */
 166       hwloc_bitmap_only(hwloc_set, pu);
 167       break;
 168     }
 169   return 0;
 170 }
 171 
 172 static int
 173 hwloc_bgq_get_thisthread_cpubind(hwloc_topology_t topology, hwloc_bitmap_t hwloc_set, int flags __hwloc_attribute_unused)
 174 {
 175   if (topology->pid) {
 176     errno = ENOSYS;
 177     return -1;
 178   }
 179   hwloc_bitmap_only(hwloc_set, Kernel_ProcessorID());
 180   return 0;
 181 }
 182 
 183 static int
 184 hwloc_bgq_set_thread_cpubind(hwloc_topology_t topology, pthread_t thread, hwloc_const_bitmap_t hwloc_set, int flags)
 185 {
 186   unsigned pu;
 187   cpu_set_t bg_set;
 188   int err;
 189 
 190   if (topology->pid) {
 191     errno = ENOSYS;
 192     return -1;
 193   }
 194   /* the binding cannot contain multiple PUs.
 195    * keep the first PU only, and error out if STRICT.
 196    */
 197   if (hwloc_bitmap_weight(hwloc_set) != 1) {
 198     if ((flags & HWLOC_CPUBIND_STRICT)) {
 199       errno = ENOSYS;
 200       return -1;
 201     }
 202   }
 203   pu = hwloc_bitmap_first(hwloc_set);
 204   CPU_ZERO(&bg_set);
 205   CPU_SET(pu, &bg_set);
 206   err = pthread_setaffinity_np(thread, sizeof(bg_set), &bg_set);
 207   if (err) {
 208     errno = err;
 209     return -1;
 210   }
 211   return 0;
 212 }
 213 
 214 static int
 215 hwloc_bgq_set_thisthread_cpubind(hwloc_topology_t topology, hwloc_const_bitmap_t hwloc_set, int flags)
 216 {
 217   return hwloc_bgq_set_thread_cpubind(topology, pthread_self(), hwloc_set, flags);
 218 }
 219 
 220 static int
 221 hwloc_bgq_get_allowed_resources(struct hwloc_topology *topology)
 222 {
 223   /* Loading BGQ from XML isn't much useful since everything is hardwired anyway.
 224    * But still implement XML + this callback in case portable applications want to always use XMLs.
 225    */
 226 
 227   /* In theory, when applying local restrictions to a XML-loaded topology,
 228    * we should check that the current topology contains 1 NUMA nodes and 17*4 PUs.
 229    *
 230    * Just trust the user when he sets THISSYSTEM=1.
 231    */
 232   return hwloc_bgq__get_allowed_resources(topology);
 233 }
 234 
 235 void
 236 hwloc_set_bgq_hooks(struct hwloc_binding_hooks *hooks __hwloc_attribute_unused,
 237                     struct hwloc_topology_support *support __hwloc_attribute_unused)
 238 {
 239   hooks->set_thisthread_cpubind = hwloc_bgq_set_thisthread_cpubind;
 240   hooks->set_thread_cpubind = hwloc_bgq_set_thread_cpubind;
 241   hooks->get_thisthread_cpubind = hwloc_bgq_get_thisthread_cpubind;
 242   hooks->get_thread_cpubind = hwloc_bgq_get_thread_cpubind;
 243   /* threads cannot be bound to more than one PU, so get_last_cpu_location == get_cpubind */
 244   hooks->get_thisthread_last_cpu_location = hwloc_bgq_get_thisthread_cpubind;
 245   /* hooks->get_thread_last_cpu_location = hwloc_bgq_get_thread_cpubind; */
 246 
 247   hooks->get_allowed_resources = hwloc_bgq_get_allowed_resources;
 248 }
 249 
 250 static struct hwloc_backend *
 251 hwloc_bgq_component_instantiate(struct hwloc_disc_component *component,
 252                                 const void *_data1 __hwloc_attribute_unused,
 253                                 const void *_data2 __hwloc_attribute_unused,
 254                                 const void *_data3 __hwloc_attribute_unused)
 255 {
 256   struct utsname utsname;
 257   struct hwloc_backend *backend;
 258   int forced_nonbgq = 0;
 259   int err;
 260 
 261   err = uname(&utsname);
 262   if (err || strcmp(utsname.sysname, "CNK") || strcmp(utsname.machine, "BGQ")) {
 263     const char *env = getenv("HWLOC_FORCE_BGQ");
 264     if (!env || !atoi(env)) {
 265       fprintf(stderr, "*** Found unexpected uname sysname `%s' machine `%s'.\n", utsname.sysname, utsname.machine);
 266       fprintf(stderr, "*** The BlueGene/Q backend (bgq) is only enabled by default on compute nodes\n"
 267                       "*** (where uname returns sysname=CNK and machine=BGQ).\n"
 268                       "*** If you know you *really* want to run the bgq backend on this non-compute node,\n"
 269                       "*** set HWLOC_FORCE_BGQ=1 in the environment.\n"
 270                       "*** If you just want to discover the native topology of this non-compute node,\n"
 271                       "*** do not pass any BlueGene/Q-specific options on the configure command-line.\n");
 272       return NULL;
 273     } else {
 274       forced_nonbgq = 1;
 275     }
 276   }
 277 
 278   backend = hwloc_backend_alloc(component);
 279   if (!backend)
 280     return NULL;
 281   backend->discover = hwloc_look_bgq;
 282   if (forced_nonbgq)
 283     backend->is_thissystem = 0;
 284   return backend;
 285 }
 286 
 287 static struct hwloc_disc_component hwloc_bgq_disc_component = {
 288   HWLOC_DISC_COMPONENT_TYPE_GLOBAL,
 289   "bgq",
 290   ~0,
 291   hwloc_bgq_component_instantiate,
 292   50,
 293   1,
 294   NULL
 295 };
 296 
 297 const struct hwloc_component hwloc_bgq_component = {
 298   HWLOC_COMPONENT_ABI,
 299   NULL, NULL,
 300   HWLOC_COMPONENT_TYPE_DISC,
 301   0,
 302   &hwloc_bgq_disc_component
 303 };
 304 
 305 #endif /* !HWLOC_DISABLE_BGQ_PORT_TEST */
/* [<][>][^][v][top][bottom][index][help] */
root/opal/mca/hwloc/hwloc201/hwloc/hwloc/topology-bgq.c

DEFINITIONS
