This source file includes following definitions.
- ppr_mapper
- find_split
- prune
- assign_locations
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 #include "orte_config.h"
16 #include "orte/constants.h"
17 #include "orte/types.h"
18
19 #include <errno.h>
20 #ifdef HAVE_UNISTD_H
21 #include <unistd.h>
22 #endif
23 #include <string.h>
24
25 #include "opal/mca/hwloc/base/base.h"
26 #include "opal/util/argv.h"
27
28 #include "orte/util/show_help.h"
29 #include "orte/mca/errmgr/errmgr.h"
30
31 #include "orte/mca/rmaps/base/rmaps_private.h"
32 #include "orte/mca/rmaps/base/base.h"
33 #include "rmaps_ppr.h"
34
35 static int ppr_mapper(orte_job_t *jdata);
36 static int assign_locations(orte_job_t *jdata);
37
38 orte_rmaps_base_module_t orte_rmaps_ppr_module = {
39 .map_job = ppr_mapper,
40 .assign_locations = assign_locations
41 };
42
43
44
45
46
47
48
49
50
51
52 typedef enum {
53 OPAL_HWLOC_NODE_LEVEL=0,
54 OPAL_HWLOC_NUMA_LEVEL,
55 OPAL_HWLOC_SOCKET_LEVEL,
56 OPAL_HWLOC_L3CACHE_LEVEL,
57 OPAL_HWLOC_L2CACHE_LEVEL,
58 OPAL_HWLOC_L1CACHE_LEVEL,
59 OPAL_HWLOC_CORE_LEVEL,
60 OPAL_HWLOC_HWTHREAD_LEVEL
61 } opal_hwloc_level_t;
62
63 static void prune(orte_jobid_t jobid,
64 orte_app_idx_t app_idx,
65 orte_node_t *node,
66 opal_hwloc_level_t *level,
67 orte_vpid_t *nmapped);
68
69 static int ppr[OPAL_HWLOC_HWTHREAD_LEVEL+1];
70
71 static int ppr_mapper(orte_job_t *jdata)
72 {
73 int rc = ORTE_SUCCESS, j, n;
74 mca_base_component_t *c=&mca_rmaps_ppr_component.base_version;
75 orte_node_t *node;
76 orte_proc_t *proc;
77 orte_app_context_t *app;
78 orte_vpid_t total_procs, nprocs_mapped;
79 opal_hwloc_level_t start=OPAL_HWLOC_NODE_LEVEL;
80 hwloc_obj_t obj;
81 hwloc_obj_type_t lowest;
82 unsigned cache_level=0;
83 unsigned int nobjs, i;
84 bool pruning_reqd = false;
85 opal_hwloc_level_t level;
86 opal_list_t node_list;
87 opal_list_item_t *item;
88 orte_std_cntr_t num_slots;
89 orte_app_idx_t idx;
90 char **ppr_req, **ck;
91 size_t len;
92 bool initial_map=true;
93
94
95
96
97 if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_RESTART)) {
98 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
99 "mca:rmaps:ppr: job %s being restarted - ppr cannot map",
100 ORTE_JOBID_PRINT(jdata->jobid));
101 return ORTE_ERR_TAKE_NEXT_OPTION;
102 }
103 if (NULL != jdata->map->req_mapper &&
104 0 != strcasecmp(jdata->map->req_mapper, c->mca_component_name)) {
105
106 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
107 "mca:rmaps:ppr: job %s not using ppr mapper",
108 ORTE_JOBID_PRINT(jdata->jobid));
109 return ORTE_ERR_TAKE_NEXT_OPTION;
110 }
111 if (NULL == jdata->map->ppr ||
112 ORTE_MAPPING_PPR != ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
113
114 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
115 "mca:rmaps:ppr: job %s not using ppr mapper PPR %s policy %s",
116 ORTE_JOBID_PRINT(jdata->jobid),
117 (NULL == jdata->map->ppr) ? "NULL" : jdata->map->ppr,
118 (ORTE_MAPPING_PPR == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) ? "PPRSET" : "PPR NOTSET");
119 return ORTE_ERR_TAKE_NEXT_OPTION;
120 }
121
122 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
123 "mca:rmaps:ppr: mapping job %s with ppr %s",
124 ORTE_JOBID_PRINT(jdata->jobid), jdata->map->ppr);
125
126
127 if (NULL != jdata->map->last_mapper) {
128 free(jdata->map->last_mapper);
129 }
130 jdata->map->last_mapper = strdup(c->mca_component_name);
131
132
133 memset(ppr, 0, OPAL_HWLOC_HWTHREAD_LEVEL * sizeof(opal_hwloc_level_t));
134
135
136 n=0;
137 ppr_req = opal_argv_split(jdata->map->ppr, ',');
138 for (j=0; NULL != ppr_req[j]; j++) {
139
140 ck = opal_argv_split(ppr_req[j], ':');
141 if (2 != opal_argv_count(ck)) {
142
143 orte_show_help("help-orte-rmaps-ppr.txt", "invalid-ppr", true, jdata->map->ppr);
144 opal_argv_free(ppr_req);
145 opal_argv_free(ck);
146 return ORTE_ERR_SILENT;
147 }
148 len = strlen(ck[1]);
149 if (0 == strncasecmp(ck[1], "node", len)) {
150 ppr[OPAL_HWLOC_NODE_LEVEL] = strtol(ck[0], NULL, 10);
151 ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYNODE);
152 start = OPAL_HWLOC_NODE_LEVEL;
153 n++;
154 } else if (0 == strncasecmp(ck[1], "hwthread", len) ||
155 0 == strncasecmp(ck[1], "thread", len)) {
156 ppr[OPAL_HWLOC_HWTHREAD_LEVEL] = strtol(ck[0], NULL, 10);
157 start = OPAL_HWLOC_HWTHREAD_LEVEL;
158 ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYHWTHREAD);
159 n++;
160 } else if (0 == strncasecmp(ck[1], "core", len)) {
161 ppr[OPAL_HWLOC_CORE_LEVEL] = strtol(ck[0], NULL, 10);
162 if (start < OPAL_HWLOC_CORE_LEVEL) {
163 start = OPAL_HWLOC_CORE_LEVEL;
164 ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYCORE);
165 }
166 n++;
167 } else if (0 == strncasecmp(ck[1], "socket", len) ||
168 0 == strncasecmp(ck[1], "skt", len)) {
169 ppr[OPAL_HWLOC_SOCKET_LEVEL] = strtol(ck[0], NULL, 10);
170 if (start < OPAL_HWLOC_SOCKET_LEVEL) {
171 start = OPAL_HWLOC_SOCKET_LEVEL;
172 ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSOCKET);
173 }
174 n++;
175 } else if (0 == strncasecmp(ck[1], "l1cache", len)) {
176 ppr[OPAL_HWLOC_L1CACHE_LEVEL] = strtol(ck[0], NULL, 10);
177 if (start < OPAL_HWLOC_L1CACHE_LEVEL) {
178 start = OPAL_HWLOC_L1CACHE_LEVEL;
179 cache_level = 1;
180 ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYL1CACHE);
181 }
182 n++;
183 } else if (0 == strncasecmp(ck[1], "l2cache", len)) {
184 ppr[OPAL_HWLOC_L2CACHE_LEVEL] = strtol(ck[0], NULL, 10);
185 if (start < OPAL_HWLOC_L2CACHE_LEVEL) {
186 start = OPAL_HWLOC_L2CACHE_LEVEL;
187 cache_level = 2;
188 ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYL2CACHE);
189 }
190 n++;
191 } else if (0 == strncasecmp(ck[1], "l3cache", len)) {
192 ppr[OPAL_HWLOC_L3CACHE_LEVEL] = strtol(ck[0], NULL, 10);
193 if (start < OPAL_HWLOC_L3CACHE_LEVEL) {
194 start = OPAL_HWLOC_L3CACHE_LEVEL;
195 cache_level = 3;
196 ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYL3CACHE);
197 }
198 n++;
199 } else if (0 == strncasecmp(ck[1], "numa", len)) {
200 ppr[OPAL_HWLOC_NUMA_LEVEL] = strtol(ck[0], NULL, 10);
201 if (start < OPAL_HWLOC_NUMA_LEVEL) {
202 start = OPAL_HWLOC_NUMA_LEVEL;
203 ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYNUMA);
204 }
205 n++;
206 } else {
207
208 orte_show_help("help-orte-rmaps-ppr.txt", "unrecognized-ppr-option", true, ck[1], jdata->map->ppr);
209 opal_argv_free(ppr_req);
210 opal_argv_free(ck);
211 return ORTE_ERR_SILENT;
212 }
213 opal_argv_free(ck);
214 }
215 opal_argv_free(ppr_req);
216
217 if (0 == n) {
218 opal_output(0, "NOTHING GIVEN");
219 return ORTE_ERR_SILENT;
220 }
221
222 if (1 < n) {
223 pruning_reqd = true;
224 }
225
226 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
227 "mca:rmaps:ppr: job %s assigned policy %s",
228 ORTE_JOBID_PRINT(jdata->jobid),
229 orte_rmaps_base_print_mapping(jdata->map->mapping));
230
231
232 level = start;
233 lowest = opal_hwloc_levels[start];
234
235 for (idx=0; idx < (orte_app_idx_t)jdata->apps->size; idx++) {
236 if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, idx))) {
237 continue;
238 }
239
240
241
242
243
244 if (0 < app->num_procs) {
245 total_procs = app->num_procs;
246 } else {
247 total_procs = ORTE_VPID_MAX;
248 }
249
250
251 OBJ_CONSTRUCT(&node_list, opal_list_t);
252 if(ORTE_SUCCESS != (rc = orte_rmaps_base_get_target_nodes(&node_list, &num_slots, app,
253 jdata->map->mapping, initial_map, false))) {
254 ORTE_ERROR_LOG(rc);
255 goto error;
256 }
257
258 initial_map = false;
259
260
261 jdata->bookmark = orte_rmaps_base_get_starting_point(&node_list, jdata);
262
263
264 nprocs_mapped = 0;
265 for (item = opal_list_get_first(&node_list);
266 item != opal_list_get_end(&node_list);
267 item = opal_list_get_next(item)) {
268 node = (orte_node_t*)item;
269
270 if (NULL == node->topology || NULL == node->topology->topo) {
271 orte_show_help("help-orte-rmaps-ppr.txt", "ppr-topo-missing",
272 true, node->name);
273 rc = ORTE_ERR_SILENT;
274 goto error;
275 }
276
277 if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
278 ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
279 OBJ_RETAIN(node);
280 opal_pointer_array_add(jdata->map->nodes, node);
281 jdata->map->num_nodes++;
282 }
283
284
285
286 if (OPAL_HWLOC_NODE_LEVEL == start) {
287 obj = hwloc_get_root_obj(node->topology->topo);
288 for (j=0; j < ppr[start] && nprocs_mapped < total_procs; j++) {
289 if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, idx))) {
290 rc = ORTE_ERR_OUT_OF_RESOURCE;
291 goto error;
292 }
293 nprocs_mapped++;
294 orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
295 }
296 } else {
297
298 nobjs = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo,
299 lowest, cache_level,
300 OPAL_HWLOC_AVAILABLE);
301
302
303
304
305 for (i=0; i < nobjs; i++) {
306 obj = opal_hwloc_base_get_obj_by_type(node->topology->topo,
307 lowest, cache_level,
308 i, OPAL_HWLOC_AVAILABLE);
309 for (j=0; j < ppr[start] && nprocs_mapped < total_procs; j++) {
310 if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, idx))) {
311 rc = ORTE_ERR_OUT_OF_RESOURCE;
312 goto error;
313 }
314 nprocs_mapped++;
315 orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
316 }
317 }
318
319 if (pruning_reqd) {
320
321
322
323
324 level--;
325 prune(jdata->jobid, idx, node, &level, &nprocs_mapped);
326 }
327 }
328
329 if (!(ORTE_MAPPING_DEBUGGER & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping))) {
330
331 if ((int)node->num_procs <= node->slots) {
332 node->slots_inuse = (int)node->num_procs;
333 } else {
334 node->slots_inuse = node->slots;
335 }
336
337
338
339
340
341 if ((node->slots < (int)node->num_procs) ||
342 (0 < node->slots_max && node->slots_max < (int)node->num_procs)) {
343 if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
344 orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
345 true, node->num_procs, app->app);
346 ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
347 rc = ORTE_ERR_SILENT;
348 goto error;
349 }
350
351
352
353 ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
354 ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_OVERSUBSCRIBED);
355
356 if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_SLOTS_GIVEN)) {
357
358
359
360 if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping))) {
361 orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
362 true, app->num_procs, app->app);
363 ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
364 return ORTE_ERR_SILENT;
365 } else if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
366
367 orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
368 true, app->num_procs, app->app);
369 ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
370 return ORTE_ERR_SILENT;
371 }
372 }
373 }
374 }
375
376
377
378
379 if (total_procs == nprocs_mapped) {
380 break;
381 }
382 }
383 if (0 == app->num_procs) {
384 app->num_procs = nprocs_mapped;
385 }
386 if (ORTE_VPID_MAX != total_procs && nprocs_mapped < total_procs) {
387
388 orte_show_help("help-orte-rmaps-ppr.txt", "ppr-too-many-procs",
389 true, app->app, app->num_procs, jdata->map->ppr);
390 rc = ORTE_ERR_SILENT;
391 goto error;
392 }
393
394
395
396
397
398 jdata->num_procs += app->num_procs;
399
400 while (NULL != (item = opal_list_remove_first(&node_list))) {
401 OBJ_RELEASE(item);
402 }
403 OBJ_DESTRUCT(&node_list);
404 }
405 return ORTE_SUCCESS;
406
407 error:
408 while (NULL != (item = opal_list_remove_first(&node_list))) {
409 OBJ_RELEASE(item);
410 }
411 OBJ_DESTRUCT(&node_list);
412 return rc;
413 }
414
415 static hwloc_obj_t find_split(hwloc_topology_t topo, hwloc_obj_t obj)
416 {
417 unsigned k;
418 hwloc_obj_t nxt;
419
420 if (1 < obj->arity) {
421 return obj;
422 }
423 for (k=0; k < obj->arity; k++) {
424 nxt = find_split(topo, obj->children[k]);
425 if (NULL != nxt) {
426 return nxt;
427 }
428 }
429 return NULL;
430 }
431
432
433
434
435 static void prune(orte_jobid_t jobid,
436 orte_app_idx_t app_idx,
437 orte_node_t *node,
438 opal_hwloc_level_t *level,
439 orte_vpid_t *nmapped)
440 {
441 hwloc_obj_t obj, top;
442 unsigned int i, nobjs;
443 hwloc_obj_type_t lvl;
444 unsigned cache_level = 0, k;
445 int nprocs;
446 hwloc_cpuset_t avail;
447 int n, limit, nmax, nunder, idx, idxmax = 0;
448 orte_proc_t *proc, *pptr, *procmax;
449 opal_hwloc_level_t ll;
450 char dang[64];
451 hwloc_obj_t locale;
452
453 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
454 "mca:rmaps:ppr: pruning level %d",
455 *level);
456
457
458 ll = *level;
459
460
461 lvl = opal_hwloc_levels[ll];
462 limit = ppr[ll];
463
464 if (0 == limit) {
465
466 if (0 == ll) {
467
468 return;
469 }
470 --(*level);
471 prune(jobid, app_idx, node, level, nmapped);
472 return;
473 }
474
475
476 if (OPAL_HWLOC_L3CACHE_LEVEL == ll) {
477 cache_level = 3;
478 } else if (OPAL_HWLOC_L2CACHE_LEVEL == ll) {
479 cache_level = 2;
480 } else if (OPAL_HWLOC_L1CACHE_LEVEL == ll) {
481 cache_level = 1;
482 }
483
484
485 nobjs = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo,
486 lvl, cache_level,
487 OPAL_HWLOC_AVAILABLE);
488
489
490
491
492 for (i=0; i < nobjs; i++) {
493 obj = opal_hwloc_base_get_obj_by_type(node->topology->topo,
494 lvl, cache_level,
495 i, OPAL_HWLOC_AVAILABLE);
496
497 avail = obj->cpuset;
498
499
500
501
502
503 nprocs = 0;
504 for (n=0; n < node->procs->size; n++) {
505 if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, n))) {
506 continue;
507 }
508 if (proc->name.jobid != jobid ||
509 proc->app_idx != app_idx) {
510 continue;
511 }
512 locale = NULL;
513 if (orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, (void**)&locale, OPAL_PTR)) {
514 ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
515 return;
516 }
517 if (hwloc_bitmap_intersects(avail, locale->cpuset)) {
518 nprocs++;
519 }
520 }
521 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
522 "mca:rmaps:ppr: found %d procs limit %d",
523 nprocs, limit);
524
525
526 while (limit < nprocs) {
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542 top = find_split(node->topology->topo, obj);
543 hwloc_obj_type_snprintf(dang, 64, top, 1);
544 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
545 "mca:rmaps:ppr: SPLIT AT LEVEL %s", dang);
546
547
548 nmax = 0;
549 procmax = NULL;
550 idx = 0;
551
552 for (k=0; k < top->arity && limit < nprocs; k++) {
553
554 nunder = 0;
555 pptr = NULL;
556 for (n=0; n < node->procs->size; n++) {
557 if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, n))) {
558 continue;
559 }
560 if (proc->name.jobid != jobid ||
561 proc->app_idx != app_idx) {
562 continue;
563 }
564 locale = NULL;
565 if (orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, (void**)&locale, OPAL_PTR)) {
566 ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
567 return;
568 }
569 if (hwloc_bitmap_intersects(top->children[k]->cpuset, locale->cpuset)) {
570 nunder++;
571 if (NULL == pptr) {
572
573 pptr = proc;
574 idx = n;
575 }
576 }
577 }
578 if (nmax < nunder) {
579 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
580 "mca:rmaps:ppr: PROCS UNDER CHILD %d %d MAX %d",
581 k, nunder, nmax);
582 nmax = nunder;
583 procmax = pptr;
584 idxmax = idx;
585 }
586 }
587 if (NULL == procmax) {
588
589 goto error;
590 }
591
592 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
593 "mca:rmaps:ppr: removing proc at posn %d",
594 idxmax);
595 opal_pointer_array_set_item(node->procs, idxmax, NULL);
596 node->num_procs--;
597 node->slots_inuse--;
598 if (node->slots_inuse < 0) {
599 node->slots_inuse = 0;
600 }
601 nprocs--;
602 *nmapped -= 1;
603 OBJ_RELEASE(procmax);
604 }
605 }
606
607 if (0 == ll) {
608 return;
609 }
610 --(*level);
611 prune(jobid, app_idx, node, level, nmapped);
612 return;
613
614 error:
615 opal_output(0, "INFINITE LOOP");
616 }
617
618 static int assign_locations(orte_job_t *jdata)
619 {
620 int i, j, m, n;
621 mca_base_component_t *c=&mca_rmaps_ppr_component.base_version;
622 orte_node_t *node;
623 orte_proc_t *proc;
624 orte_app_context_t *app;
625 hwloc_obj_type_t level;
626 hwloc_obj_t obj;
627 unsigned int cache_level=0;
628 int ppr, cnt, nobjs, nprocs_mapped;
629 char **ppr_req, **ck;
630
631 if (NULL == jdata->map->last_mapper ||
632 0 != strcasecmp(jdata->map->last_mapper, c->mca_component_name)) {
633
634 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
635 "mca:rmaps:ppr: job %s not using ppr assign: %s",
636 ORTE_JOBID_PRINT(jdata->jobid),
637 (NULL == jdata->map->last_mapper) ? "NULL" : jdata->map->last_mapper);
638 return ORTE_ERR_TAKE_NEXT_OPTION;
639 }
640
641 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
642 "mca:rmaps:ppr: assigning locations for job %s with ppr %s policy %s",
643 ORTE_JOBID_PRINT(jdata->jobid), jdata->map->ppr,
644 orte_rmaps_base_print_mapping(jdata->map->mapping));
645
646
647 if (ORTE_MAPPING_BYNODE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
648 level = HWLOC_OBJ_MACHINE;
649 } else if (ORTE_MAPPING_BYHWTHREAD == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
650 level = HWLOC_OBJ_PU;
651 } else if (ORTE_MAPPING_BYCORE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
652 level = HWLOC_OBJ_CORE;
653 } else if (ORTE_MAPPING_BYSOCKET == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
654 level = HWLOC_OBJ_SOCKET;
655 } else if (ORTE_MAPPING_BYL1CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
656 level = HWLOC_OBJ_L1CACHE;
657 cache_level = 1;
658 } else if (ORTE_MAPPING_BYL2CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
659 level = HWLOC_OBJ_L2CACHE;
660 cache_level = 2;
661 } else if (ORTE_MAPPING_BYL3CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
662 level = HWLOC_OBJ_L3CACHE;
663 cache_level = 3;
664 } else if (ORTE_MAPPING_BYNUMA == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
665 level = HWLOC_OBJ_NUMANODE;
666 } else {
667 ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
668 return ORTE_ERR_TAKE_NEXT_OPTION;
669 }
670
671
672 ppr_req = opal_argv_split(jdata->map->ppr, ',');
673 ck = opal_argv_split(ppr_req[0], ':');
674 ppr = strtol(ck[0], NULL, 10);
675 opal_argv_free(ck);
676 opal_argv_free(ppr_req);
677
678
679
680 for (n=0; n < jdata->apps->size; n++) {
681 if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, n))) {
682 continue;
683 }
684 nprocs_mapped = 0;
685 for (m=0; m < jdata->map->nodes->size; m++) {
686 if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, m))) {
687 continue;
688 }
689 if (NULL == node->topology || NULL == node->topology->topo) {
690 orte_show_help("help-orte-rmaps-ppr.txt", "ppr-topo-missing",
691 true, node->name);
692 return ORTE_ERR_SILENT;
693 }
694 if (HWLOC_OBJ_MACHINE == level) {
695 obj = hwloc_get_root_obj(node->topology->topo);
696 for (j=0; j < node->procs->size; j++) {
697 if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
698 continue;
699 }
700 if (proc->name.jobid != jdata->jobid) {
701 continue;
702 }
703 orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
704 }
705 } else {
706
707 nobjs = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo,
708 level, cache_level,
709 OPAL_HWLOC_AVAILABLE);
710
711
712
713
714 for (i=0; i < nobjs; i++) {
715 cnt = 0;
716 obj = opal_hwloc_base_get_obj_by_type(node->topology->topo,
717 level, cache_level,
718 i, OPAL_HWLOC_AVAILABLE);
719 for (j=0; j < node->procs->size && cnt < ppr && nprocs_mapped < app->num_procs; j++) {
720 if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
721 continue;
722 }
723 if (proc->name.jobid != jdata->jobid) {
724 continue;
725 }
726
727 if (orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, NULL, OPAL_PTR)) {
728 continue;
729 }
730 nprocs_mapped++;
731 cnt++;
732 orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
733 }
734 }
735 }
736 }
737 }
738 return ORTE_SUCCESS;
739 }