This source file includes following definitions.
- sn_con
- sn_des
- orte_rmaps_seq_map
- orte_getline
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26 #include "orte_config.h"
27 #include "orte/constants.h"
28 #include "orte/types.h"
29
30 #include <errno.h>
31 #ifdef HAVE_UNISTD_H
32 #include <unistd.h>
33 #endif
34 #include <string.h>
35 #include <ctype.h>
36
37 #include "opal/util/if.h"
38 #include "opal/util/net.h"
39 #include "opal/mca/hwloc/hwloc-internal.h"
40
41 #include "orte/util/show_help.h"
42 #include "orte/mca/errmgr/errmgr.h"
43 #include "orte/mca/ess/ess.h"
44 #include "orte/util/hostfile/hostfile.h"
45 #include "orte/util/dash_host/dash_host.h"
46 #include "orte/util/name_fns.h"
47 #include "orte/runtime/orte_globals.h"
48
49 #include "orte/mca/rmaps/base/rmaps_private.h"
50 #include "orte/mca/rmaps/base/base.h"
51 #include "rmaps_seq.h"
52
53 static int orte_rmaps_seq_map(orte_job_t *jdata);
54
55
56 orte_rmaps_base_module_t orte_rmaps_seq_module = {
57 .map_job = orte_rmaps_seq_map
58 };
59
60
61 typedef struct {
62 opal_list_item_t super;
63 char *hostname;
64 char *cpuset;
65 } seq_node_t;
66 static void sn_con(seq_node_t *p)
67 {
68 p->hostname = NULL;
69 p->cpuset = NULL;
70 }
71 static void sn_des(seq_node_t *p)
72 {
73 if (NULL != p->hostname) {
74 free(p->hostname);
75 p->hostname = NULL;
76 }
77 if (NULL != p->cpuset) {
78 free(p->cpuset);
79 p->cpuset = NULL;
80 }
81 }
82 OBJ_CLASS_INSTANCE(seq_node_t,
83 opal_list_item_t,
84 sn_con, sn_des);
85
86 static char *orte_getline(FILE *fp);
87
88
89
90
91
92 static int orte_rmaps_seq_map(orte_job_t *jdata)
93 {
94 orte_job_map_t *map;
95 orte_app_context_t *app;
96 int i, n;
97 orte_std_cntr_t j;
98 opal_list_item_t *item;
99 orte_node_t *node, *nd;
100 seq_node_t *sq, *save=NULL, *seq;;
101 orte_vpid_t vpid;
102 orte_std_cntr_t num_nodes;
103 int rc;
104 opal_list_t default_seq_list;
105 opal_list_t node_list, *seq_list, sq_list;
106 orte_proc_t *proc;
107 mca_base_component_t *c = &mca_rmaps_seq_component.base_version;
108 char *hosts = NULL, *sep, *eptr;
109 FILE *fp;
110 opal_hwloc_resource_type_t rtype;
111
112 OPAL_OUTPUT_VERBOSE((1, orte_rmaps_base_framework.framework_output,
113 "%s rmaps:seq called on job %s",
114 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
115 ORTE_JOBID_PRINT(jdata->jobid)));
116
117
118
119
120
121 if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_RESTART)) {
122 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
123 "mca:rmaps:seq: job %s is being restarted - seq cannot map",
124 ORTE_JOBID_PRINT(jdata->jobid));
125 return ORTE_ERR_TAKE_NEXT_OPTION;
126 }
127 if (NULL != jdata->map->req_mapper) {
128 if (0 != strcasecmp(jdata->map->req_mapper, c->mca_component_name)) {
129
130 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
131 "mca:rmaps:seq: job %s not using sequential mapper",
132 ORTE_JOBID_PRINT(jdata->jobid));
133 return ORTE_ERR_TAKE_NEXT_OPTION;
134 }
135
136 goto process;
137 }
138 if (ORTE_MAPPING_SEQ != ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
139
140 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
141 "mca:rmaps:seq: job %s not using seq mapper",
142 ORTE_JOBID_PRINT(jdata->jobid));
143 return ORTE_ERR_TAKE_NEXT_OPTION;
144 }
145
146 process:
147 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
148 "mca:rmaps:seq: mapping job %s",
149 ORTE_JOBID_PRINT(jdata->jobid));
150
151
152 if (NULL != jdata->map->last_mapper) {
153 free(jdata->map->last_mapper);
154 }
155 jdata->map->last_mapper = strdup(c->mca_component_name);
156
157
158 map = jdata->map;
159
160
161 OBJ_CONSTRUCT(&default_seq_list, opal_list_t);
162 if (NULL != orte_default_hostfile) {
163 char *hstname = NULL;
164
165 fp = fopen(orte_default_hostfile, "r");
166 if (NULL == fp) {
167 ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
168 rc = ORTE_ERR_NOT_FOUND;
169 goto error;
170 }
171 while (NULL != (hstname = orte_getline(fp))) {
172 if (0 == strlen(hstname)) {
173 free(hstname);
174
175 continue;
176 }
177 if( '#' == hstname[0] ) {
178 free(hstname);
179
180 continue;
181 }
182 sq = OBJ_NEW(seq_node_t);
183 if (NULL != (sep = strchr(hstname, ' '))) {
184 *sep = '\0';
185 sep++;
186
187 eptr = sep + strlen(sep) - 1;
188 while (eptr > sep && isspace(*eptr)) {
189 eptr--;
190 }
191 *(eptr+1) = 0;
192 sq->cpuset = strdup(sep);
193 }
194
195
196 if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hstname) ) {
197 char *ptr;
198 if (NULL != (ptr = strchr(hstname, '.'))) {
199 *ptr = '\0';
200 }
201 }
202
203 sq->hostname = hstname;
204 opal_list_append(&default_seq_list, &sq->super);
205 }
206 fclose(fp);
207 }
208
209
210 vpid = 0;
211 jdata->num_procs = 0;
212 if (0 < opal_list_get_size(&default_seq_list)) {
213 save = (seq_node_t*)opal_list_get_first(&default_seq_list);
214 }
215
216
217 if (orte_get_attribute(&jdata->attributes, ORTE_JOB_PHYSICAL_CPUIDS, NULL, OPAL_BOOL)) {
218 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
219 "mca:rmaps:seq: using PHYSICAL processors");
220 rtype = OPAL_HWLOC_PHYSICAL;
221 } else {
222 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
223 "mca:rmaps:seq: using LOGICAL processors");
224 rtype = OPAL_HWLOC_LOGICAL;
225 }
226
227
228 for (j=0; j < orte_node_pool->size; j++) {
229 if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, j))) {
230 ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED);
231 }
232 }
233
234
235 for(i=0; i < jdata->apps->size; i++) {
236 if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
237 continue;
238 }
239
240
241 if (orte_get_attribute(&app->attributes, ORTE_APP_DASH_HOST, (void**)&hosts, OPAL_STRING)) {
242 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
243 "mca:rmaps:seq: using dash-host nodes on app %s", app->app);
244 OBJ_CONSTRUCT(&node_list, opal_list_t);
245
246 if (ORTE_SUCCESS != (rc = orte_util_get_ordered_dash_host_list(&node_list, hosts))) {
247 ORTE_ERROR_LOG(rc);
248 free(hosts);
249 goto error;
250 }
251 free(hosts);
252
253 OBJ_CONSTRUCT(&sq_list, opal_list_t);
254 while (NULL != (nd = (orte_node_t*)opal_list_remove_first(&node_list))) {
255 sq = OBJ_NEW(seq_node_t);
256 sq->hostname = strdup(nd->name);
257 opal_list_append(&sq_list, &sq->super);
258 OBJ_RELEASE(nd);
259 }
260 OBJ_DESTRUCT(&node_list);
261 seq_list = &sq_list;
262 } else if (orte_get_attribute(&app->attributes, ORTE_APP_HOSTFILE, (void**)&hosts, OPAL_STRING)) {
263 char *hstname;
264 if (NULL == hosts) {
265 rc = ORTE_ERR_NOT_FOUND;
266 goto error;
267 }
268 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
269 "mca:rmaps:seq: using hostfile %s nodes on app %s", hosts, app->app);
270 OBJ_CONSTRUCT(&sq_list, opal_list_t);
271
272 fp = fopen(hosts, "r");
273 if (NULL == fp) {
274 ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
275 rc = ORTE_ERR_NOT_FOUND;
276 OBJ_DESTRUCT(&sq_list);
277 goto error;
278 }
279 while (NULL != (hstname = orte_getline(fp))) {
280 if (0 == strlen(hstname)) {
281 free(hstname);
282
283 continue;
284 }
285 if( '#' == hstname[0] ) {
286 free(hstname);
287
288 continue;
289 }
290 sq = OBJ_NEW(seq_node_t);
291 if (NULL != (sep = strchr(hstname, ' '))) {
292 *sep = '\0';
293 sep++;
294
295 eptr = sep + strlen(sep) - 1;
296 while (eptr > sep && isspace(*eptr)) {
297 eptr--;
298 }
299 *(eptr+1) = 0;
300 sq->cpuset = strdup(sep);
301 }
302
303
304 if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hstname) ) {
305 char *ptr;
306 if (NULL != (ptr = strchr(hstname, '.'))) {
307 (*ptr) = '\0';
308 }
309 }
310
311 sq->hostname = hstname;
312 opal_list_append(&sq_list, &sq->super);
313 }
314 fclose(fp);
315 free(hosts);
316 seq_list = &sq_list;
317 } else if (0 < opal_list_get_size(&default_seq_list)) {
318 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
319 "mca:rmaps:seq: using default hostfile nodes on app %s", app->app);
320 seq_list = &default_seq_list;
321 } else {
322
323 orte_show_help("help-orte-rmaps-base.txt",
324 "orte-rmaps-base:no-available-resources",
325 true);
326 return ORTE_ERR_SILENT;
327 }
328
329
330 if (map->mapping & ORTE_MAPPING_NO_USE_LOCAL) {
331 for (item = opal_list_get_first(seq_list);
332 item != opal_list_get_end(seq_list);
333 item = opal_list_get_next(item) ) {
334 seq = (seq_node_t*)item;
335
336
337
338
339 if (orte_ifislocal(seq->hostname)) {
340 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
341 "mca:rmaps:seq: removing head node %s", seq->hostname);
342 opal_list_remove_item(seq_list, item);
343 OBJ_RELEASE(item);
344 }
345 }
346 }
347
348 if (NULL == seq_list || 0 == (num_nodes = (orte_std_cntr_t)opal_list_get_size(seq_list))) {
349 orte_show_help("help-orte-rmaps-base.txt",
350 "orte-rmaps-base:no-available-resources",
351 true);
352 return ORTE_ERR_SILENT;
353 }
354
355
356 if (0 == app->num_procs) {
357 app->num_procs = num_nodes;
358 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
359 "mca:rmaps:seq: setting num procs to %s for app %s",
360 ORTE_VPID_PRINT(app->num_procs), app->app);
361 } else if (num_nodes < app->num_procs) {
362 orte_show_help("help-orte-rmaps-base.txt", "seq:not-enough-resources", true,
363 app->num_procs, num_nodes);
364 return ORTE_ERR_SILENT;
365 }
366
367 if (seq_list == &default_seq_list) {
368 sq = save;
369 } else {
370 sq = (seq_node_t*)opal_list_get_first(seq_list);
371 }
372 for (n=0; n < app->num_procs; n++) {
373
374
375
376
377 node = NULL;
378 for (j=0; j < orte_node_pool->size; j++) {
379 if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, j))) {
380 continue;
381 }
382 if (0 == strcmp(sq->hostname, node->name)) {
383 break;
384 }
385 }
386 if (NULL == node) {
387
388 orte_show_help("help-orte-rmaps-seq.txt",
389 "orte-rmaps-seq:resource-not-found",
390 true, sq->hostname);
391 rc = ORTE_ERR_SILENT;
392 goto error;
393 }
394
395 if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
396 OBJ_RETAIN(node);
397 opal_pointer_array_add(map->nodes, node);
398 jdata->map->num_nodes++;
399 ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
400 }
401 proc = orte_rmaps_base_setup_proc(jdata, node, i);
402 if ((node->slots < (int)node->num_procs) ||
403 (0 < node->slots_max && node->slots_max < (int)node->num_procs)) {
404 if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
405 orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
406 true, node->num_procs, app->app);
407 ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
408 rc = ORTE_ERR_SILENT;
409 goto error;
410 }
411
412
413
414 ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
415 ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_OVERSUBSCRIBED);
416
417 if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_SLOTS_GIVEN)) {
418
419
420
421 if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping))) {
422 orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
423 true, app->num_procs, app->app);
424 ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
425 return ORTE_ERR_SILENT;
426 } else if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
427
428 orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
429 true, app->num_procs, app->app);
430 ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
431 return ORTE_ERR_SILENT;
432 }
433 }
434 }
435
436 proc->name.vpid = vpid++;
437 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
438 "mca:rmaps:seq: assign proc %s to node %s for app %s",
439 ORTE_VPID_PRINT(proc->name.vpid), sq->hostname, app->app);
440
441
442 if (NULL != sq->cpuset) {
443 hwloc_cpuset_t bitmap;
444 char *cpu_bitmap;
445 if (NULL == node->topology || NULL == node->topology->topo) {
446
447
448
449 orte_show_help("help-orte-rmaps-base.txt", "rmaps:no-topology", true, node->name);
450 rc = ORTE_ERR_SILENT;
451 goto error;
452 }
453
454
455
456 if (opal_hwloc_use_hwthreads_as_cpus &&
457 OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
458 cpu_bitmap = strdup(sq->cpuset);
459 } else {
460
461 bitmap = hwloc_bitmap_alloc();
462
463 if (ORTE_SUCCESS != (rc = opal_hwloc_base_cpu_list_parse(sq->cpuset, node->topology->topo, rtype, bitmap))) {
464 ORTE_ERROR_LOG(rc);
465 hwloc_bitmap_free(bitmap);
466 goto error;
467 }
468
469
470
471
472
473 hwloc_bitmap_list_asprintf(&cpu_bitmap, bitmap);
474 hwloc_bitmap_free(bitmap);
475 }
476 orte_set_attribute(&proc->attributes, ORTE_PROC_CPU_BITMAP, ORTE_ATTR_GLOBAL, cpu_bitmap, OPAL_STRING);
477 opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
478 "mca:rmaps:seq: binding proc %s to cpuset %s bitmap %s",
479 ORTE_VPID_PRINT(proc->name.vpid), sq->cpuset, cpu_bitmap);
480
481 OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_CPUSET);
482
483 ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYUSER);
484 ORTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_MAPPING_GIVEN);
485
486 free(cpu_bitmap);
487 } else {
488 hwloc_obj_t locale;
489
490
491
492
493 if (NULL != node->topology && NULL != node->topology->topo) {
494 locale = hwloc_get_root_obj(node->topology->topo);
495 orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE,
496 ORTE_ATTR_LOCAL, locale, OPAL_PTR);
497 }
498 }
499
500
501 if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
502 ORTE_ERROR_LOG(rc);
503 goto error;
504 }
505
506 sq = (seq_node_t*)opal_list_get_next(&sq->super);
507 }
508
509
510 jdata->num_procs += app->num_procs;
511
512
513 if (seq_list != &default_seq_list) {
514 OPAL_LIST_DESTRUCT(seq_list);
515 } else {
516 save = sq;
517 }
518 }
519
520
521
522 orte_set_attribute(&jdata->attributes, ORTE_JOB_FULLY_DESCRIBED, ORTE_ATTR_GLOBAL, NULL, OPAL_BOOL);
523
524 return ORTE_SUCCESS;
525
526 error:
527 OPAL_LIST_DESTRUCT(&default_seq_list);
528 return rc;
529 }
530
531 static char *orte_getline(FILE *fp)
532 {
533 char *ret, *buff;
534 char input[1024];
535
536 ret = fgets(input, 1024, fp);
537 if (NULL != ret) {
538 input[strlen(input)-1] = '\0';
539 buff = strdup(input);
540 return buff;
541 }
542
543 return NULL;
544 }