orte/mca/schizo/ompi/schizo

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
define_cli
parse_cli
parse_env
setup_fork
setup_child
   1 /*
   2  * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
   3  *                         University Research and Technology
   4  *                         Corporation.  All rights reserved.
   5  * Copyright (c) 2004-2011 The University of Tennessee and The University
   6  *                         of Tennessee Research Foundation.  All rights
   7  *                         reserved.
   8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   9  *                         University of Stuttgart.  All rights reserved.
  10  * Copyright (c) 2004-2005 The Regents of the University of California.
  11  *                         All rights reserved.
  12  * Copyright (c) 2006-2017 Los Alamos National Security, LLC.
  13  *                         All rights reserved.
  14  * Copyright (c) 2009-2018 Cisco Systems, Inc.  All rights reserved
  15  * Copyright (c) 2011-2017 Oak Ridge National Labs.  All rights reserved.
  16  * Copyright (c) 2017      UT-Battelle, LLC. All rights reserved.
  17  * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  18  * Copyright (c) 2015-2019 Research Organization for Information Science
  19  *                         and Technology (RIST).  All rights reserved.
  20  * Copyright (c) 2018      IBM Corporation.  All rights reserved.
  21  * $COPYRIGHT$
  22  *
  23  * Additional copyrights may follow
  24  *
  25  * $HEADER$
  26  *
  27  */
  28 
  29 #include "orte_config.h"
  30 #include "orte/types.h"
  31 #include "opal/types.h"
  32 
  33 #ifdef HAVE_UNISTD_H
  34 #include <unistd.h>
  35 #endif
  36 #include <ctype.h>
  37 
  38 #include "opal/util/argv.h"
  39 #include "opal/util/opal_environ.h"
  40 #include "opal/util/os_dirpath.h"
  41 #include "opal/util/show_help.h"
  42 #include "opal/mca/shmem/base/base.h"
  43 
  44 #include "orte/mca/errmgr/errmgr.h"
  45 #include "orte/mca/ess/base/base.h"
  46 #include "orte/mca/rmaps/rmaps_types.h"
  47 #include "orte/orted/orted_submit.h"
  48 #include "orte/util/name_fns.h"
  49 #include "orte/util/session_dir.h"
  50 #include "orte/util/show_help.h"
  51 #include "orte/runtime/orte_globals.h"
  52 
  53 #include "orte/mca/schizo/base/base.h"
  54 
  55 static int define_cli(opal_cmd_line_t *cli);
  56 static int parse_cli(int argc, int start, char **argv);
  57 static int parse_env(char *path,
  58                      opal_cmd_line_t *cmd_line,
  59                      char **srcenv,
  60                      char ***dstenv);
  61 static int setup_fork(orte_job_t *jdata,
  62                       orte_app_context_t *context);
  63 static int setup_child(orte_job_t *jobdat,
  64                        orte_proc_t *child,
  65                        orte_app_context_t *app,
  66                        char ***env);
  67 
  68 orte_schizo_base_module_t orte_schizo_ompi_module = {
  69     .define_cli = define_cli,
  70     .parse_cli = parse_cli,
  71     .parse_env = parse_env,
  72     .setup_fork = setup_fork,
  73     .setup_child = setup_child
  74 };
  75 
  76 
  77 static opal_cmd_line_init_t cmd_line_init[] = {
  78     /* Various "obvious" options */
  79     { NULL, 'h', NULL, "help", 1,
  80       &orte_cmd_options.help, OPAL_CMD_LINE_TYPE_STRING,
  81       "This help message", OPAL_CMD_LINE_OTYPE_GENERAL },
  82     { NULL, 'V', NULL, "version", 0,
  83       &orte_cmd_options.version, OPAL_CMD_LINE_TYPE_BOOL,
  84       "Print version and exit", OPAL_CMD_LINE_OTYPE_GENERAL },
  85     { NULL, 'v', NULL, "verbose", 0,
  86       &orte_cmd_options.verbose, OPAL_CMD_LINE_TYPE_BOOL,
  87       "Be verbose", OPAL_CMD_LINE_OTYPE_GENERAL },
  88     { "orte_execute_quiet", 'q', NULL, "quiet", 0,
  89       NULL, OPAL_CMD_LINE_TYPE_BOOL,
  90       "Suppress helpful messages", OPAL_CMD_LINE_OTYPE_GENERAL },
  91     { NULL, '\0', "report-pid", "report-pid", 1,
  92       &orte_cmd_options.report_pid, OPAL_CMD_LINE_TYPE_STRING,
  93       "Printout pid on stdout [-], stderr [+], or a file [anything else]",
  94       OPAL_CMD_LINE_OTYPE_DEBUG },
  95     { NULL, '\0', "report-uri", "report-uri", 1,
  96       &orte_cmd_options.report_uri, OPAL_CMD_LINE_TYPE_STRING,
  97       "Printout URI on stdout [-], stderr [+], or a file [anything else]",
  98       OPAL_CMD_LINE_OTYPE_DEBUG },
  99 
 100     /* testing options */
 101     { NULL, '\0', "timeout", "timeout", 1,
 102       &orte_cmd_options.timeout, OPAL_CMD_LINE_TYPE_INT,
 103       "Timeout the job after the specified number of seconds",
 104       OPAL_CMD_LINE_OTYPE_DEBUG },
 105     { NULL, '\0', "report-state-on-timeout", "report-state-on-timeout", 0,
 106       &orte_cmd_options.report_state_on_timeout, OPAL_CMD_LINE_TYPE_BOOL,
 107       "Report all job and process states upon timeout",
 108       OPAL_CMD_LINE_OTYPE_DEBUG },
 109     { NULL, '\0', "get-stack-traces", "get-stack-traces", 0,
 110       &orte_cmd_options.get_stack_traces, OPAL_CMD_LINE_TYPE_BOOL,
 111       "Get stack traces of all application procs on timeout",
 112       OPAL_CMD_LINE_OTYPE_DEBUG },
 113 
 114 
 115     /* exit status reporting */
 116     { "orte_report_child_jobs_separately", '\0', "report-child-jobs-separately", "report-child-jobs-separately", 0,
 117       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 118       "Return the exit status of the primary job only", OPAL_CMD_LINE_OTYPE_OUTPUT },
 119 
 120     /* uri of the dvm, or at least where to get it */
 121     { NULL, '\0', "hnp", "hnp", 1,
 122       &orte_cmd_options.hnp, OPAL_CMD_LINE_TYPE_STRING,
 123       "Specify the URI of the HNP, or the name of the file (specified as file:filename) that contains that info",
 124       OPAL_CMD_LINE_OTYPE_DVM },
 125 
 126     /* select XML output */
 127     { "orte_xml_output", '\0', "xml", "xml", 0,
 128       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 129       "Provide all output in XML format", OPAL_CMD_LINE_OTYPE_OUTPUT },
 130     { "orte_xml_file", '\0', "xml-file", "xml-file", 1,
 131       NULL, OPAL_CMD_LINE_TYPE_STRING,
 132       "Provide all output in XML format to the specified file", OPAL_CMD_LINE_OTYPE_OUTPUT },
 133 
 134     /* tag output */
 135     { "orte_tag_output", '\0', "tag-output", "tag-output", 0,
 136       &orte_cmd_options.tag_output, OPAL_CMD_LINE_TYPE_BOOL,
 137       "Tag all output with [job,rank]", OPAL_CMD_LINE_OTYPE_OUTPUT },
 138     { "orte_timestamp_output", '\0', "timestamp-output", "timestamp-output", 0,
 139       &orte_cmd_options.timestamp_output, OPAL_CMD_LINE_TYPE_BOOL,
 140       "Timestamp all application process output", OPAL_CMD_LINE_OTYPE_OUTPUT },
 141     { "orte_output_filename", '\0', "output-filename", "output-filename", 1,
 142       &orte_cmd_options.output_filename, OPAL_CMD_LINE_TYPE_STRING,
 143       "Redirect output from application processes into filename/job/rank/std[out,err,diag]. A relative path value will be converted to an absolute path",
 144       OPAL_CMD_LINE_OTYPE_OUTPUT },
 145     { NULL, '\0', "merge-stderr-to-stdout", "merge-stderr-to-stdout", 0,
 146       &orte_cmd_options.merge, OPAL_CMD_LINE_TYPE_BOOL,
 147       "Merge stderr to stdout for each process", OPAL_CMD_LINE_OTYPE_OUTPUT },
 148     { "orte_xterm", '\0', "xterm", "xterm", 1,
 149       NULL, OPAL_CMD_LINE_TYPE_STRING,
 150       "Create a new xterm window and display output from the specified ranks there",
 151       OPAL_CMD_LINE_OTYPE_OUTPUT },
 152 
 153     /* select stdin option */
 154     { NULL, '\0', "stdin", "stdin", 1,
 155       &orte_cmd_options.stdin_target, OPAL_CMD_LINE_TYPE_STRING,
 156       "Specify procs to receive stdin [rank, all, none] (default: 0, indicating rank 0)",
 157       OPAL_CMD_LINE_OTYPE_INPUT },
 158 
 159     /* request that argv[0] be indexed */
 160     { NULL, '\0', "index-argv-by-rank", "index-argv-by-rank", 0,
 161       &orte_cmd_options.index_argv, OPAL_CMD_LINE_TYPE_BOOL,
 162       "Uniquely index argv[0] for each process using its rank",
 163       OPAL_CMD_LINE_OTYPE_INPUT },
 164 
 165     /* Specify the launch agent to be used */
 166     { "orte_launch_agent", '\0', "launch-agent", "launch-agent", 1,
 167       NULL, OPAL_CMD_LINE_TYPE_STRING,
 168       "Command used to start processes on remote nodes (default: orted)",
 169       OPAL_CMD_LINE_OTYPE_LAUNCH },
 170 
 171     /* Preload the binary on the remote machine */
 172     { NULL, 's', NULL, "preload-binary", 0,
 173       &orte_cmd_options.preload_binaries, OPAL_CMD_LINE_TYPE_BOOL,
 174       "Preload the binary on the remote machine before starting the remote process.",
 175       OPAL_CMD_LINE_OTYPE_LAUNCH },
 176 
 177     /* Preload files on the remote machine */
 178     { NULL, '\0', NULL, "preload-files", 1,
 179       &orte_cmd_options.preload_files, OPAL_CMD_LINE_TYPE_STRING,
 180       "Preload the comma separated list of files to the remote machines current working directory before starting the remote process.",
 181       OPAL_CMD_LINE_OTYPE_LAUNCH },
 182 
 183 #if OPAL_ENABLE_FT_CR == 1
 184     /* Tell SStore to preload a snapshot before launch */
 185     { NULL, '\0', NULL, "sstore-load", 1,
 186       &orte_cmd_options.sstore_load, OPAL_CMD_LINE_TYPE_STRING,
 187       "Internal Use Only! Tell SStore to preload a snapshot before launch." },
 188 #endif
 189 
 190     /* Use an appfile */
 191     { NULL, '\0', NULL, "app", 1,
 192       &orte_cmd_options.appfile, OPAL_CMD_LINE_TYPE_STRING,
 193       "Provide an appfile; ignore all other command line options",
 194       OPAL_CMD_LINE_OTYPE_LAUNCH },
 195 
 196     /* Number of processes; -c, -n, --n, -np, and --np are all
 197        synonyms */
 198     { NULL, 'c', "np", "np", 1,
 199       &orte_cmd_options.num_procs, OPAL_CMD_LINE_TYPE_INT,
 200       "Number of processes to run", OPAL_CMD_LINE_OTYPE_GENERAL },
 201     { NULL, '\0', "n", "n", 1,
 202       &orte_cmd_options.num_procs, OPAL_CMD_LINE_TYPE_INT,
 203       "Number of processes to run", OPAL_CMD_LINE_OTYPE_GENERAL },
 204 
 205     /* maximum size of VM - typically used to subdivide an allocation */
 206     { "orte_max_vm_size", '\0', "max-vm-size", "max-vm-size", 1,
 207       NULL, OPAL_CMD_LINE_TYPE_INT,
 208       "Number of processes to run", OPAL_CMD_LINE_OTYPE_DVM },
 209 
 210     /* Set a hostfile */
 211     { NULL, '\0', "hostfile", "hostfile", 1,
 212       NULL, OPAL_CMD_LINE_TYPE_STRING,
 213       "Provide a hostfile", OPAL_CMD_LINE_OTYPE_LAUNCH },
 214     { NULL, '\0', "machinefile", "machinefile", 1,
 215       NULL, OPAL_CMD_LINE_TYPE_STRING,
 216       "Provide a hostfile", OPAL_CMD_LINE_OTYPE_LAUNCH },
 217     { "orte_default_hostfile", '\0', "default-hostfile", "default-hostfile", 1,
 218         NULL, OPAL_CMD_LINE_TYPE_STRING,
 219       "Provide a default hostfile", OPAL_CMD_LINE_OTYPE_LAUNCH },
 220     { "opal_if_do_not_resolve", '\0', "do-not-resolve", "do-not-resolve", 0,
 221       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 222       "Do not attempt to resolve interfaces", OPAL_CMD_LINE_OTYPE_DEVEL },
 223 
 224     /* uri of PMIx publish/lookup server, or at least where to get it */
 225     { "pmix_server_uri", '\0', "ompi-server", "ompi-server", 1,
 226       NULL, OPAL_CMD_LINE_TYPE_STRING,
 227       "Specify the URI of the publish/lookup server, or the name of the file (specified as file:filename) that contains that info",
 228       OPAL_CMD_LINE_OTYPE_DVM },
 229 
 230     { "carto_file_path", '\0', "cf", "cartofile", 1,
 231       NULL, OPAL_CMD_LINE_TYPE_STRING,
 232       "Provide a cartography file", OPAL_CMD_LINE_OTYPE_MAPPING },
 233 
 234     { "orte_rankfile", '\0', "rf", "rankfile", 1,
 235       NULL, OPAL_CMD_LINE_TYPE_STRING,
 236       "Provide a rankfile file", OPAL_CMD_LINE_OTYPE_MAPPING },
 237 
 238     /* Export environment variables; potentially used multiple times,
 239        so it does not make sense to set into a variable */
 240     { NULL, 'x', NULL, NULL, 1,
 241       NULL, OPAL_CMD_LINE_TYPE_NULL,
 242       "Export an environment variable, optionally specifying a value (e.g., \"-x foo\" exports the environment variable foo and takes its value from the current environment; \"-x foo=bar\" exports the environment variable name foo and sets its value to \"bar\" in the started processes)", OPAL_CMD_LINE_OTYPE_LAUNCH },
 243 
 244       /* Mapping controls */
 245     { "rmaps_base_display_map", '\0', "display-map", "display-map", 0,
 246       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 247       "Display the process map just before launch", OPAL_CMD_LINE_OTYPE_DEBUG },
 248     { "rmaps_base_display_devel_map", '\0', "display-devel-map", "display-devel-map", 0,
 249        NULL, OPAL_CMD_LINE_TYPE_BOOL,
 250        "Display a detailed process map (mostly intended for developers) just before launch",
 251        OPAL_CMD_LINE_OTYPE_DEVEL },
 252     { "rmaps_base_display_topo_with_map", '\0', "display-topo", "display-topo", 0,
 253        NULL, OPAL_CMD_LINE_TYPE_BOOL,
 254        "Display the topology as part of the process map (mostly intended for developers) just before launch",
 255        OPAL_CMD_LINE_OTYPE_DEVEL },
 256     { "rmaps_base_display_diffable_map", '\0', "display-diffable-map", "display-diffable-map", 0,
 257        NULL, OPAL_CMD_LINE_TYPE_BOOL,
 258        "Display a diffable process map (mostly intended for developers) just before launch",
 259        OPAL_CMD_LINE_OTYPE_DEVEL },
 260     { NULL, 'H', "host", "host", 1,
 261       NULL, OPAL_CMD_LINE_TYPE_STRING,
 262       "List of hosts to invoke processes on",
 263       OPAL_CMD_LINE_OTYPE_MAPPING },
 264     { "rmaps_base_no_schedule_local", '\0', "nolocal", "nolocal", 0,
 265       &orte_cmd_options.nolocal, OPAL_CMD_LINE_TYPE_BOOL,
 266       "Do not run any MPI applications on the local node",
 267       OPAL_CMD_LINE_OTYPE_MAPPING },
 268     { "rmaps_base_no_oversubscribe", '\0', "nooversubscribe", "nooversubscribe", 0,
 269       &orte_cmd_options.no_oversubscribe, OPAL_CMD_LINE_TYPE_BOOL,
 270       "Nodes are not to be oversubscribed, even if the system supports such operation",
 271       OPAL_CMD_LINE_OTYPE_MAPPING },
 272     { "rmaps_base_oversubscribe", '\0', "oversubscribe", "oversubscribe", 0,
 273       &orte_cmd_options.oversubscribe, OPAL_CMD_LINE_TYPE_BOOL,
 274       "Nodes are allowed to be oversubscribed, even on a managed system, and overloading of processing elements",
 275       OPAL_CMD_LINE_OTYPE_MAPPING },
 276     { "rmaps_base_cpus_per_rank", '\0', "cpus-per-proc", "cpus-per-proc", 1,
 277       &orte_cmd_options.cpus_per_proc, OPAL_CMD_LINE_TYPE_INT,
 278       "Number of cpus to use for each process [default=1]",
 279       OPAL_CMD_LINE_OTYPE_MAPPING },
 280     { "rmaps_base_cpus_per_rank", '\0', "cpus-per-rank", "cpus-per-rank", 1,
 281       &orte_cmd_options.cpus_per_proc, OPAL_CMD_LINE_TYPE_INT,
 282       "Synonym for cpus-per-proc", OPAL_CMD_LINE_OTYPE_MAPPING },
 283 
 284     /* backward compatiblity */
 285     { "rmaps_base_bycore", '\0', "bycore", "bycore", 0,
 286       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 287       "Whether to map and rank processes round-robin by core",
 288       OPAL_CMD_LINE_OTYPE_COMPAT },
 289     { "rmaps_base_bynode", '\0', "bynode", "bynode", 0,
 290       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 291       "Whether to map and rank processes round-robin by node",
 292       OPAL_CMD_LINE_OTYPE_COMPAT },
 293     { "rmaps_base_byslot", '\0', "byslot", "byslot", 0,
 294       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 295       "Whether to map and rank processes round-robin by slot",
 296       OPAL_CMD_LINE_OTYPE_COMPAT },
 297 
 298     /* Nperxxx options that do not require topology and are always
 299      * available - included for backwards compatibility
 300      */
 301     { "rmaps_ppr_pernode", '\0', "pernode", "pernode", 0,
 302       &orte_cmd_options.pernode, OPAL_CMD_LINE_TYPE_BOOL,
 303       "Launch one process per available node",
 304       OPAL_CMD_LINE_OTYPE_COMPAT },
 305     { "rmaps_ppr_n_pernode", '\0', "npernode", "npernode", 1,
 306       &orte_cmd_options.npernode, OPAL_CMD_LINE_TYPE_INT,
 307       "Launch n processes per node on all allocated nodes",
 308       OPAL_CMD_LINE_OTYPE_COMPAT },
 309     { "rmaps_ppr_n_pernode", '\0', "N", NULL, 1,
 310       &orte_cmd_options.npernode, OPAL_CMD_LINE_TYPE_INT,
 311       "Launch n processes per node on all allocated nodes (synonym for 'map-by node')",
 312       OPAL_CMD_LINE_OTYPE_MAPPING },
 313 
 314     /* declare hardware threads as independent cpus */
 315     { "hwloc_base_use_hwthreads_as_cpus", '\0', "use-hwthread-cpus", "use-hwthread-cpus", 0,
 316       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 317       "Use hardware threads as independent cpus", OPAL_CMD_LINE_OTYPE_MAPPING },
 318 
 319     /* include npersocket for backwards compatibility */
 320     { "rmaps_ppr_n_persocket", '\0', "npersocket", "npersocket", 1,
 321       &orte_cmd_options.npersocket, OPAL_CMD_LINE_TYPE_INT,
 322       "Launch n processes per socket on all allocated nodes",
 323       OPAL_CMD_LINE_OTYPE_COMPAT },
 324 
 325     /* Mapping options */
 326     { "rmaps_base_mapping_policy", '\0', NULL, "map-by", 1,
 327       &orte_cmd_options.mapping_policy, OPAL_CMD_LINE_TYPE_STRING,
 328       "Mapping Policy [slot | hwthread | core | socket (default) | numa | board | node]",
 329       OPAL_CMD_LINE_OTYPE_MAPPING },
 330 
 331       /* Ranking options */
 332     { "rmaps_base_ranking_policy", '\0', NULL, "rank-by", 1,
 333       &orte_cmd_options.ranking_policy, OPAL_CMD_LINE_TYPE_STRING,
 334       "Ranking Policy [slot (default) | hwthread | core | socket | numa | board | node]",
 335       OPAL_CMD_LINE_OTYPE_RANKING },
 336 
 337       /* Binding options */
 338     { "hwloc_base_binding_policy", '\0', NULL, "bind-to", 1,
 339       &orte_cmd_options.binding_policy, OPAL_CMD_LINE_TYPE_STRING,
 340       "Policy for binding processes. Allowed values: none, hwthread, core, l1cache, l2cache, l3cache, socket, numa, board, cpu-list (\"none\" is the default when oversubscribed, \"core\" is the default when np<=2, and \"socket\" is the default when np>2). Allowed qualifiers: overload-allowed, if-supported, ordered", OPAL_CMD_LINE_OTYPE_BINDING },
 341 
 342     /* backward compatiblity */
 343     { "hwloc_base_bind_to_core", '\0', "bind-to-core", "bind-to-core", 0,
 344       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 345       "Bind processes to cores", OPAL_CMD_LINE_OTYPE_COMPAT },
 346     { "hwloc_base_bind_to_socket", '\0', "bind-to-socket", "bind-to-socket", 0,
 347       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 348       "Bind processes to sockets", OPAL_CMD_LINE_OTYPE_COMPAT },
 349 
 350     { "hwloc_base_report_bindings", '\0', "report-bindings", "report-bindings", 0,
 351       &orte_cmd_options.report_bindings, OPAL_CMD_LINE_TYPE_BOOL,
 352       "Whether to report process bindings to stderr",
 353       OPAL_CMD_LINE_OTYPE_BINDING },
 354 
 355     /* slot list option */
 356     { "hwloc_base_cpu_list", '\0', "cpu-list", "cpu-list", 1,
 357       &orte_cmd_options.cpu_list, OPAL_CMD_LINE_TYPE_STRING,
 358       "List of processor IDs to bind processes to [default=NULL]",
 359       OPAL_CMD_LINE_OTYPE_BINDING },
 360 
 361     /* generalized pattern mapping option */
 362     { "rmaps_ppr_pattern", '\0', NULL, "ppr", 1,
 363       NULL, OPAL_CMD_LINE_TYPE_STRING,
 364       "Comma-separated list of number of processes on a given resource type [default: none]",
 365       OPAL_CMD_LINE_OTYPE_MAPPING },
 366 
 367     /* Allocation options */
 368     { "orte_display_alloc", '\0', "display-allocation", "display-allocation", 0,
 369       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 370       "Display the allocation being used by this job", OPAL_CMD_LINE_OTYPE_DEBUG },
 371     { "orte_display_devel_alloc", '\0', "display-devel-allocation", "display-devel-allocation", 0,
 372       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 373       "Display a detailed list (mostly intended for developers) of the allocation being used by this job",
 374       OPAL_CMD_LINE_OTYPE_DEVEL },
 375     { "hwloc_base_cpu_set", '\0', "cpu-set", "cpu-set", 1,
 376       NULL, OPAL_CMD_LINE_TYPE_STRING,
 377       "Comma-separated list of ranges specifying logical cpus allocated to this job [default: none]",
 378       OPAL_CMD_LINE_OTYPE_DEBUG },
 379 
 380     /* mpiexec-like arguments */
 381     { NULL, '\0', "wdir", "wdir", 1,
 382       &orte_cmd_options.wdir, OPAL_CMD_LINE_TYPE_STRING,
 383       "Set the working directory of the started processes",
 384       OPAL_CMD_LINE_OTYPE_LAUNCH },
 385     { NULL, '\0', "wd", "wd", 1,
 386       &orte_cmd_options.wdir, OPAL_CMD_LINE_TYPE_STRING,
 387       "Synonym for --wdir", OPAL_CMD_LINE_OTYPE_LAUNCH },
 388     { NULL, '\0', "set-cwd-to-session-dir", "set-cwd-to-session-dir", 0,
 389       &orte_cmd_options.set_cwd_to_session_dir, OPAL_CMD_LINE_TYPE_BOOL,
 390       "Set the working directory of the started processes to their session directory",
 391       OPAL_CMD_LINE_OTYPE_LAUNCH },
 392     { NULL, '\0', "path", "path", 1,
 393       &orte_cmd_options.path, OPAL_CMD_LINE_TYPE_STRING,
 394       "PATH to be used to look for executables to start processes",
 395       OPAL_CMD_LINE_OTYPE_LAUNCH },
 396 
 397     /* User-level debugger arguments */
 398     { NULL, '\0', "tv", "tv", 0,
 399       &orte_cmd_options.debugger, OPAL_CMD_LINE_TYPE_BOOL,
 400       "Deprecated backwards compatibility flag; synonym for \"--debug\"",
 401       OPAL_CMD_LINE_OTYPE_DEBUG },
 402     { NULL, '\0', "debug", "debug", 0,
 403       &orte_cmd_options.debugger, OPAL_CMD_LINE_TYPE_BOOL,
 404       "Invoke the user-level debugger indicated by the orte_base_user_debugger MCA parameter",
 405       OPAL_CMD_LINE_OTYPE_DEBUG },
 406     { "orte_base_user_debugger", '\0', "debugger", "debugger", 1,
 407       NULL, OPAL_CMD_LINE_TYPE_STRING,
 408       "Sequence of debuggers to search for when \"--debug\" is used",
 409       OPAL_CMD_LINE_OTYPE_DEBUG },
 410     { "orte_output_debugger_proctable", '\0', "output-proctable", "output-proctable", 0,
 411       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 412       "Output the debugger proctable after launch",
 413       OPAL_CMD_LINE_OTYPE_DEBUG },
 414 
 415     /* OpenRTE arguments */
 416     { "orte_debug", 'd', "debug-devel", "debug-devel", 0,
 417       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 418       "Enable debugging of OpenRTE", OPAL_CMD_LINE_OTYPE_DEVEL },
 419 
 420     { "orte_debug_daemons", '\0', "debug-daemons", "debug-daemons", 0,
 421       NULL, OPAL_CMD_LINE_TYPE_INT,
 422       "Enable debugging of any OpenRTE daemons used by this application",
 423       OPAL_CMD_LINE_OTYPE_DEVEL },
 424 
 425     { "orte_debug_daemons_file", '\0', "debug-daemons-file", "debug-daemons-file", 0,
 426       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 427       "Enable debugging of any OpenRTE daemons used by this application, storing output in files",
 428       OPAL_CMD_LINE_OTYPE_DEVEL },
 429 
 430     { "orte_leave_session_attached", '\0', "leave-session-attached", "leave-session-attached", 0,
 431       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 432       "Enable debugging of OpenRTE", OPAL_CMD_LINE_OTYPE_DEBUG },
 433 
 434     { "orte_do_not_launch", '\0', "do-not-launch", "do-not-launch", 0,
 435       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 436       "Perform all necessary operations to prepare to launch the application, but do not actually launch it",
 437       OPAL_CMD_LINE_OTYPE_DEVEL },
 438 
 439     { NULL, '\0', NULL, "prefix", 1,
 440       NULL, OPAL_CMD_LINE_TYPE_STRING,
 441       "Prefix where Open MPI is installed on remote nodes",
 442       OPAL_CMD_LINE_OTYPE_LAUNCH },
 443     { NULL, '\0', NULL, "noprefix", 0,
 444       NULL, OPAL_CMD_LINE_TYPE_STRING,
 445       "Disable automatic --prefix behavior",
 446       OPAL_CMD_LINE_OTYPE_LAUNCH },
 447 
 448     { "orte_report_launch_progress", '\0', "show-progress", "show-progress", 0,
 449       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 450       "Output a brief periodic report on launch progress",
 451       OPAL_CMD_LINE_OTYPE_LAUNCH },
 452 
 453     { "orte_use_regexp", '\0', "use-regexp", "use-regexp", 0,
 454       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 455       "Use regular expressions for launch", OPAL_CMD_LINE_OTYPE_LAUNCH },
 456 
 457     { "orte_report_events", '\0', "report-events", "report-events", 1,
 458       NULL, OPAL_CMD_LINE_TYPE_STRING,
 459       "Report events to a tool listening at the specified URI", OPAL_CMD_LINE_OTYPE_DEBUG },
 460 
 461     { "orte_enable_recovery", '\0', "enable-recovery", "enable-recovery", 0,
 462       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 463       "Enable recovery from process failure [Default = disabled]",
 464       OPAL_CMD_LINE_OTYPE_UNSUPPORTED },
 465 
 466     { "orte_max_restarts", '\0', "max-restarts", "max-restarts", 1,
 467       NULL, OPAL_CMD_LINE_TYPE_INT,
 468       "Max number of times to restart a failed process",
 469       OPAL_CMD_LINE_OTYPE_UNSUPPORTED },
 470 
 471     { NULL, '\0', "continuous", "continuous", 0,
 472       &orte_cmd_options.continuous, OPAL_CMD_LINE_TYPE_BOOL,
 473       "Job is to run until explicitly terminated", OPAL_CMD_LINE_OTYPE_DEBUG },
 474 
 475 #if OPAL_ENABLE_CRDEBUG == 1
 476     { "opal_cr_enable_crdebug", '\0', "crdebug", "crdebug", 0,
 477       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 478       "Enable C/R Debugging" },
 479 #endif
 480 
 481     { NULL, '\0', "disable-recovery", "disable-recovery", 0,
 482       &orte_cmd_options.disable_recovery, OPAL_CMD_LINE_TYPE_BOOL,
 483       "Disable recovery (resets all recovery options to off)",
 484       OPAL_CMD_LINE_OTYPE_UNSUPPORTED },
 485 
 486     { "orte_no_vm", '\0', "novm", "novm", 0,
 487       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 488       "Execute without creating an allocation-spanning virtual machine (only start daemons on nodes hosting application procs)",
 489       OPAL_CMD_LINE_OTYPE_DVM },
 490 
 491     { NULL, '\0', "allow-run-as-root", "allow-run-as-root", 0,
 492       &orte_cmd_options.run_as_root, OPAL_CMD_LINE_TYPE_BOOL,
 493       "Allow execution as root (STRONGLY DISCOURAGED)",
 494       OPAL_CMD_LINE_OTYPE_LAUNCH },
 495 
 496     { NULL, '\0', "personality", "personality", 1,
 497       &orte_cmd_options.personality, OPAL_CMD_LINE_TYPE_STRING,
 498       "Comma-separated list of programming model, languages, and containers being used (default=\"ompi\")",
 499       OPAL_CMD_LINE_OTYPE_LAUNCH },
 500 
 501     { NULL, '\0', "dvm", "dvm", 0,
 502       &orte_cmd_options.create_dvm, OPAL_CMD_LINE_TYPE_BOOL,
 503       "Create a persistent distributed virtual machine (DVM)",
 504       OPAL_CMD_LINE_OTYPE_DVM },
 505 
 506     /* fwd mpirun port */
 507     { "orte_fwd_mpirun_port", '\0', "fwd-mpirun-port", "fwd-mpirun-port", 0,
 508       NULL, OPAL_CMD_LINE_TYPE_BOOL,
 509       "Forward mpirun port to compute node daemons so all will use it",
 510       OPAL_CMD_LINE_OTYPE_LAUNCH },
 511 
 512     /* End of list */
 513     { NULL, '\0', NULL, NULL, 0,
 514       NULL, OPAL_CMD_LINE_TYPE_NULL, NULL }
 515 };
 516 
 517 static int define_cli(opal_cmd_line_t *cli)
 518 {
 519     int i, rc;
 520     bool takeus = false;
 521 
 522     opal_output_verbose(1, orte_schizo_base_framework.framework_output,
 523                         "%s schizo:ompi: define_cli",
 524                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
 525 
 526     /* protect against bozo error */
 527     if (NULL == cli) {
 528         return ORTE_ERR_BAD_PARAM;
 529     }
 530 
 531     if (NULL != orte_schizo_base.personalities) {
 532         /* if we aren't included, then ignore us */
 533         for (i=0; NULL != orte_schizo_base.personalities[i]; i++) {
 534             if (0 == strcmp(orte_schizo_base.personalities[i], "ompi")) {
 535                 takeus = true;
 536                 break;
 537             }
 538         }
 539         if (!takeus) {
 540             return ORTE_ERR_TAKE_NEXT_OPTION;
 541         }
 542     }
 543 
 544     /*
 545      * Check if a HNP DVM URI is being passed via environment.
 546      * Note: Place before opal_cmd_line_parse() so that
 547      * if user passes both envvar & cmdln, the cmdln wins.
 548      */
 549     if (NULL != getenv("ORTE_HNP_DVM_URI")) {
 550         orte_cmd_options.hnp = strdup(getenv("ORTE_HNP_DVM_URI"));
 551     }
 552 
 553     /* just add ours to the end */
 554     rc = opal_cmd_line_add(cli, cmd_line_init);
 555     return rc;
 556 }
 557 
 558 static int parse_cli(int argc, int start, char **argv)
 559 {
 560     int i, j, k;
 561     bool ignore;
 562     char *no_dups[] = {
 563         "grpcomm",
 564         "odls",
 565         "rml",
 566         "routed",
 567         NULL
 568     };
 569     bool takeus = false;
 570 
 571     opal_output_verbose(1, orte_schizo_base_framework.framework_output,
 572                         "%s schizo:ompi: parse_cli",
 573                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
 574 
 575     /* if they gave us a list of personalities,
 576      * see if we are included */
 577     if (NULL != orte_schizo_base.personalities) {
 578         for (i=0; NULL != orte_schizo_base.personalities[i]; i++) {
 579             if (0 == strcmp(orte_schizo_base.personalities[i], "ompi")) {
 580                 takeus = true;
 581                 break;
 582             }
 583         }
 584         if (!takeus) {
 585             return ORTE_ERR_TAKE_NEXT_OPTION;
 586         }
 587     } else {
 588         /* attempt to auto-detect CLI options that
 589          * we recognize */
 590     }
 591 
 592     for (i = 0; i < (argc-start); ++i) {
 593         if (0 == strcmp("-mca",  argv[i]) ||
 594             0 == strcmp("--mca", argv[i]) ) {
 595             /* ignore this one */
 596             if (0 == strcmp(argv[i+1], "mca_base_env_list")) {
 597                 i += 2;
 598                 continue;
 599             }
 600             /* It would be nice to avoid increasing the length
 601              * of the orted cmd line by removing any non-ORTE
 602              * params. However, this raises a problem since
 603              * there could be OPAL directives that we really
 604              * -do- want the orted to see - it's only the OMPI
 605              * related directives we could ignore. This becomes
 606              * a very complicated procedure, however, since
 607              * the OMPI mca params are not cleanly separated - so
 608              * filtering them out is nearly impossible.
 609              *
 610              * see if this is already present so we at least can
 611              * avoid growing the cmd line with duplicates
 612              */
 613             ignore = false;
 614             if (NULL != orted_cmd_line) {
 615                 for (j=0; NULL != orted_cmd_line[j]; j++) {
 616                     if (0 == strcmp(argv[i+1], orted_cmd_line[j])) {
 617                         /* already here - if the value is the same,
 618                          * we can quitely ignore the fact that they
 619                          * provide it more than once. However, some
 620                          * frameworks are known to have problems if the
 621                          * value is different. We don't have a good way
 622                          * to know this, but we at least make a crude
 623                          * attempt here to protect ourselves.
 624                          */
 625                         if (0 == strcmp(argv[i+2], orted_cmd_line[j+1])) {
 626                             /* values are the same */
 627                             ignore = true;
 628                             break;
 629                         } else {
 630                             /* values are different - see if this is a problem */
 631                             for (k=0; NULL != no_dups[k]; k++) {
 632                                 if (0 == strcmp(no_dups[k], argv[i+1])) {
 633                                     /* print help message
 634                                      * and abort as we cannot know which one is correct
 635                                      */
 636                                     orte_show_help("help-orterun.txt", "orterun:conflicting-params",
 637                                                    true, orte_basename, argv[i+1],
 638                                                    argv[i+2], orted_cmd_line[j+1]);
 639                                     return ORTE_ERR_BAD_PARAM;
 640                                 }
 641                             }
 642                             /* this passed muster - just ignore it */
 643                             ignore = true;
 644                             break;
 645                         }
 646                     }
 647                 }
 648             }
 649             if (!ignore) {
 650                 opal_argv_append_nosize(&orted_cmd_line, argv[i]);
 651                 opal_argv_append_nosize(&orted_cmd_line, argv[i+1]);
 652                 opal_argv_append_nosize(&orted_cmd_line, argv[i+2]);
 653             }
 654             i += 2;
 655         }
 656     }
 657     return ORTE_SUCCESS;
 658 }
 659 
 660 static int parse_env(char *path,
 661                      opal_cmd_line_t *cmd_line,
 662                      char **srcenv,
 663                      char ***dstenv)
 664 {
 665     int i, j;
 666     char *param;
 667     char *value;
 668     char *env_set_flag;
 669     char **vars;
 670     bool takeus = false;
 671 
 672     opal_output_verbose(1, orte_schizo_base_framework.framework_output,
 673                         "%s schizo:ompi: parse_env",
 674                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
 675 
 676     if (NULL != orte_schizo_base.personalities) {
 677         /* see if we are included */
 678         for (i=0; NULL != orte_schizo_base.personalities[i]; i++) {
 679             if (0 == strcmp(orte_schizo_base.personalities[i], "ompi")) {
 680                 takeus = true;
 681                 break;
 682             }
 683         }
 684         if (!takeus) {
 685             return ORTE_ERR_TAKE_NEXT_OPTION;
 686         }
 687     }
 688 
 689     for (i = 0; NULL != srcenv[i]; ++i) {
 690         if (0 == strncmp("OMPI_", srcenv[i], 5) ||
 691             0 == strncmp("PMIX_", srcenv[i], 5)) {
 692             /* check for duplicate in app->env - this
 693              * would have been placed there by the
 694              * cmd line processor. By convention, we
 695              * always let the cmd line override the
 696              * environment
 697              */
 698             param = strdup(srcenv[i]);
 699             value = strchr(param, '=');
 700             *value = '\0';
 701             value++;
 702             opal_setenv(param, value, false, dstenv);
 703             free(param);
 704         }
 705     }
 706 
 707     /* set necessary env variables for external usage from tune conf file*/
 708     int set_from_file = 0;
 709     vars = NULL;
 710     if (OPAL_SUCCESS == mca_base_var_process_env_list_from_file(&vars) &&
 711             NULL != vars) {
 712         for (i=0; NULL != vars[i]; i++) {
 713             value = strchr(vars[i], '=');
 714             /* terminate the name of the param */
 715             *value = '\0';
 716             /* step over the equals */
 717             value++;
 718             /* overwrite any prior entry */
 719             opal_setenv(vars[i], value, true, dstenv);
 720             /* save it for any comm_spawn'd apps */
 721             opal_setenv(vars[i], value, true, &orte_forwarded_envars);
 722         }
 723         set_from_file = 1;
 724         opal_argv_free(vars);
 725     }
 726     /* Did the user request to export any environment variables on the cmd line? */
 727     env_set_flag = getenv("OMPI_MCA_mca_base_env_list");
 728     if (opal_cmd_line_is_taken(cmd_line, "x")) {
 729         if (NULL != env_set_flag) {
 730             orte_show_help("help-orterun.txt", "orterun:conflict-env-set", false);
 731             return ORTE_ERR_FATAL;
 732         }
 733         j = opal_cmd_line_get_ninsts(cmd_line, "x");
 734         for (i = 0; i < j; ++i) {
 735             param = opal_cmd_line_get_param(cmd_line, "x", i, 0);
 736 
 737             if (NULL != (value = strchr(param, '='))) {
 738                 /* terminate the name of the param */
 739                 *value = '\0';
 740                 /* step over the equals */
 741                 value++;
 742                 /* overwrite any prior entry */
 743                 opal_setenv(param, value, true, dstenv);
 744                 /* save it for any comm_spawn'd apps */
 745                 opal_setenv(param, value, true, &orte_forwarded_envars);
 746             } else {
 747                 value = getenv(param);
 748                 if (NULL != value) {
 749                     /* overwrite any prior entry */
 750                     opal_setenv(param, value, true, dstenv);
 751                     /* save it for any comm_spawn'd apps */
 752                     opal_setenv(param, value, true, &orte_forwarded_envars);
 753                 } else {
 754                     opal_output(0, "Warning: could not find environment variable \"%s\"\n", param);
 755                 }
 756             }
 757         }
 758     } else if (NULL != env_set_flag) {
 759         /* if mca_base_env_list was set, check if some of env vars were set via -x from a conf file.
 760          * If this is the case, error out.
 761          */
 762         if (!set_from_file) {
 763             /* set necessary env variables for external usage */
 764             vars = NULL;
 765             if (OPAL_SUCCESS == mca_base_var_process_env_list(env_set_flag, &vars) &&
 766                     NULL != vars) {
 767                 for (i=0; NULL != vars[i]; i++) {
 768                     value = strchr(vars[i], '=');
 769                     /* terminate the name of the param */
 770                     *value = '\0';
 771                     /* step over the equals */
 772                     value++;
 773                     /* overwrite any prior entry */
 774                     opal_setenv(vars[i], value, true, dstenv);
 775                     /* save it for any comm_spawn'd apps */
 776                     opal_setenv(vars[i], value, true, &orte_forwarded_envars);
 777                 }
 778                 opal_argv_free(vars);
 779             }
 780         } else {
 781             orte_show_help("help-orterun.txt", "orterun:conflict-env-set", false);
 782             return ORTE_ERR_FATAL;
 783         }
 784     }
 785 
 786     /* If the user specified --path, store it in the user's app
 787        environment via the OMPI_exec_path variable. */
 788     if (NULL != path) {
 789         opal_asprintf(&value, "OMPI_exec_path=%s", path);
 790         opal_argv_append_nosize(dstenv, value);
 791         /* save it for any comm_spawn'd apps */
 792         opal_argv_append_nosize(&orte_forwarded_envars, value);
 793         free(value);
 794     }
 795 
 796     return ORTE_SUCCESS;
 797 }
 798 
 799 static int setup_fork(orte_job_t *jdata,
 800                       orte_app_context_t *app)
 801 {
 802     int i;
 803     char *param, *p2, *saveptr;
 804     bool oversubscribed;
 805     orte_node_t *node;
 806     char **envcpy, **nps, **firstranks;
 807     char *npstring, *firstrankstring;
 808     char *num_app_ctx;
 809     bool takeus = false;
 810     bool exists;
 811     orte_app_context_t* tmp_app;
 812     orte_attribute_t *attr;
 813 
 814     opal_output_verbose(1, orte_schizo_base_framework.framework_output,
 815                         "%s schizo:ompi: setup_fork",
 816                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
 817 
 818     /* if no personality was specified, then nothing to do */
 819     if (NULL == jdata->personality) {
 820         return ORTE_ERR_TAKE_NEXT_OPTION;
 821     }
 822 
 823     if (NULL != orte_schizo_base.personalities) {
 824     /* see if we are included */
 825         for (i=0; NULL != jdata->personality[i]; i++) {
 826             if (0 == strcmp(jdata->personality[i], "ompi")) {
 827                 takeus = true;
 828                 break;
 829             }
 830         }
 831         if (!takeus) {
 832             return ORTE_ERR_TAKE_NEXT_OPTION;
 833         }
 834     }
 835 
 836     /* see if the mapper thinks we are oversubscribed */
 837     oversubscribed = false;
 838     if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid))) {
 839         ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
 840         return ORTE_ERR_NOT_FOUND;
 841     }
 842     if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_OVERSUBSCRIBED)) {
 843         oversubscribed = true;
 844     }
 845 
 846     /* setup base environment: copy the current environ and merge
 847        in the app context environ */
 848     if (NULL != app->env) {
 849         /* manually free original context->env to avoid a memory leak */
 850         char **tmp = app->env;
 851         envcpy = opal_environ_merge(orte_launch_environ, app->env);
 852         if (NULL != tmp) {
 853             opal_argv_free(tmp);
 854         }
 855     } else {
 856         envcpy = opal_argv_copy(orte_launch_environ);
 857     }
 858     app->env = envcpy;
 859 
 860     /* special case handling for --prefix: this is somewhat icky,
 861        but at least some users do this.  :-\ It is possible that
 862        when using --prefix, the user will also "-x PATH" and/or
 863        "-x LD_LIBRARY_PATH", which would therefore clobber the
 864        work that was done in the prior pls to ensure that we have
 865        the prefix at the beginning of the PATH and
 866        LD_LIBRARY_PATH.  So examine the context->env and see if we
 867        find PATH or LD_LIBRARY_PATH.  If found, that means the
 868        prior work was clobbered, and we need to re-prefix those
 869        variables. */
 870     param = NULL;
 871     orte_get_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, (void**)&param, OPAL_STRING);
 872     /* grab the parameter from the first app context because the current context does not have a prefix assigned */
 873     if (NULL == param) {
 874         tmp_app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, 0);
 875         assert (NULL != tmp_app);
 876         orte_get_attribute(&tmp_app->attributes, ORTE_APP_PREFIX_DIR, (void**)&param, OPAL_STRING);
 877     }
 878     for (i = 0; NULL != param && NULL != app->env && NULL != app->env[i]; ++i) {
 879         char *newenv;
 880 
 881         /* Reset PATH */
 882         if (0 == strncmp("PATH=", app->env[i], 5)) {
 883             opal_asprintf(&newenv, "%s/bin:%s", param, app->env[i] + 5);
 884             opal_setenv("PATH", newenv, true, &app->env);
 885             free(newenv);
 886         }
 887 
 888         /* Reset LD_LIBRARY_PATH */
 889         else if (0 == strncmp("LD_LIBRARY_PATH=", app->env[i], 16)) {
 890             opal_asprintf(&newenv, "%s/lib:%s", param, app->env[i] + 16);
 891             opal_setenv("LD_LIBRARY_PATH", newenv, true, &app->env);
 892             free(newenv);
 893         }
 894     }
 895     if (NULL != param) {
 896         free(param);
 897     }
 898 
 899     /* pass my contact info to the local proc so we can talk */
 900     opal_setenv("OMPI_MCA_orte_local_daemon_uri", orte_process_info.my_daemon_uri, true, &app->env);
 901 
 902     /* pass the hnp's contact info to the local proc in case it
 903      * needs it
 904      */
 905     if (NULL != orte_process_info.my_hnp_uri) {
 906         opal_setenv("OMPI_MCA_orte_hnp_uri", orte_process_info.my_hnp_uri, true, &app->env);
 907     }
 908 
 909     /* setup yield schedule - do not override any user-supplied directive! */
 910     if (oversubscribed) {
 911         opal_setenv("OMPI_MCA_mpi_oversubscribe", "1", true, &app->env);
 912     } else {
 913         opal_setenv("OMPI_MCA_mpi_oversubscribe", "0", true, &app->env);
 914     }
 915 
 916     /* set the app_context number into the environment */
 917     opal_asprintf(&param, "%ld", (long)app->idx);
 918     opal_setenv("OMPI_MCA_orte_app_num", param, true, &app->env);
 919     free(param);
 920 
 921     /* although the total_slots_alloc is the universe size, users
 922      * would appreciate being given a public environmental variable
 923      * that also represents this value - something MPI specific - so
 924      * do that here. Also required by the ompi_attributes code!
 925      *
 926      * AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
 927      * We know - just live with it
 928      */
 929     opal_asprintf(&param, "%ld", (long)jdata->total_slots_alloc);
 930     opal_setenv("OMPI_UNIVERSE_SIZE", param, true, &app->env);
 931     free(param);
 932 
 933     /* pass the number of nodes involved in this job */
 934     opal_asprintf(&param, "%ld", (long)(jdata->map->num_nodes));
 935     opal_setenv("OMPI_MCA_orte_num_nodes", param, true, &app->env);
 936     free(param);
 937 
 938     /* pass a param telling the child what type and model of cpu we are on,
 939      * if we know it. If hwloc has the value, use what it knows. Otherwise,
 940      * see if we were explicitly given it and use that value.
 941      */
 942     hwloc_obj_t obj;
 943     char *htmp;
 944     if (NULL != opal_hwloc_topology) {
 945         obj = hwloc_get_root_obj(opal_hwloc_topology);
 946         if (NULL != (htmp = (char*)hwloc_obj_get_info_by_name(obj, "CPUType")) ||
 947             NULL != (htmp = orte_local_cpu_type)) {
 948             opal_setenv("OMPI_MCA_orte_cpu_type", htmp, true, &app->env);
 949         }
 950         if (NULL != (htmp = (char*)hwloc_obj_get_info_by_name(obj, "CPUModel")) ||
 951             NULL != (htmp = orte_local_cpu_model)) {
 952             opal_setenv("OMPI_MCA_orte_cpu_model", htmp, true, &app->env);
 953         }
 954     } else {
 955         if (NULL != orte_local_cpu_type) {
 956             opal_setenv("OMPI_MCA_orte_cpu_type", orte_local_cpu_type, true, &app->env);
 957         }
 958         if (NULL != orte_local_cpu_model) {
 959             opal_setenv("OMPI_MCA_orte_cpu_model", orte_local_cpu_model, true, &app->env);
 960         }
 961     }
 962 
 963     /* get shmem's best component name so we can provide a hint to the shmem
 964      * framework. the idea here is to have someone figure out what component to
 965      * select (via the shmem framework) and then have the rest of the
 966      * components in shmem obey that decision. for more details take a look at
 967      * the shmem framework in opal.
 968      */
 969     if (NULL != (param = opal_shmem_base_best_runnable_component_name())) {
 970         opal_setenv("OMPI_MCA_shmem_RUNTIME_QUERY_hint", param, true, &app->env);
 971         free(param);
 972     }
 973 
 974     /* Set an info MCA param that tells the launched processes that
 975      * any binding policy was applied by us (e.g., so that
 976      * MPI_INIT doesn't try to bind itself)
 977      */
 978     if (OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(jdata->map->binding)) {
 979         opal_setenv("OMPI_MCA_orte_bound_at_launch", "1", true, &app->env);
 980     }
 981 
 982     /* tell the ESS to avoid the singleton component - but don't override
 983      * anything that may have been provided elsewhere
 984      */
 985     opal_setenv("OMPI_MCA_ess", "^singleton", false, &app->env);
 986 
 987     /* ensure that the spawned process ignores direct launch components,
 988      * but do not overrride anything we were given */
 989     opal_setenv("OMPI_MCA_pmix", "^s1,s2,cray", false, &app->env);
 990 
 991     /* since we want to pass the name as separate components, make sure
 992      * that the "name" environmental variable is cleared!
 993      */
 994     opal_unsetenv("OMPI_MCA_orte_ess_name", &app->env);
 995 
 996     opal_asprintf(&param, "%ld", (long)jdata->num_procs);
 997     opal_setenv("OMPI_MCA_orte_ess_num_procs", param, true, &app->env);
 998 
 999     /* although the num_procs is the comm_world size, users
1000      * would appreciate being given a public environmental variable
1001      * that also represents this value - something MPI specific - so
1002      * do that here.
1003      *
1004      * AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
1005      * We know - just live with it
1006      */
1007     opal_setenv("OMPI_COMM_WORLD_SIZE", param, true, &app->env);
1008     free(param);
1009 
1010     /* users would appreciate being given a public environmental variable
1011      * that also represents this value - something MPI specific - so
1012      * do that here.
1013      *
1014      * AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
1015      * We know - just live with it
1016      */
1017     opal_asprintf(&param, "%ld", (long)jdata->num_local_procs);
1018     opal_setenv("OMPI_COMM_WORLD_LOCAL_SIZE", param, true, &app->env);
1019     free(param);
1020 
1021     /* forcibly set the local tmpdir base and top session dir to match ours */
1022     opal_setenv("OMPI_MCA_orte_tmpdir_base", orte_process_info.tmpdir_base, true, &app->env);
1023     /* TODO: should we use PMIx key to pass this data? */
1024     opal_setenv("OMPI_MCA_orte_top_session_dir", orte_process_info.top_session_dir, true, &app->env);
1025     opal_setenv("OMPI_MCA_orte_jobfam_session_dir", orte_process_info.jobfam_session_dir, true, &app->env);
1026 
1027     /* MPI-3 requires we provide some further info to the procs,
1028      * so we pass them as envars to avoid introducing further
1029      * ORTE calls in the MPI layer
1030      */
1031     opal_asprintf(&num_app_ctx, "%lu", (unsigned long)jdata->num_apps);
1032 
1033     /* build some common envars we need to pass for MPI-3 compatibility */
1034     nps = NULL;
1035     firstranks = NULL;
1036     for (i=0; i < jdata->apps->size; i++) {
1037         if (NULL == (tmp_app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
1038             continue;
1039         }
1040         opal_argv_append_nosize(&nps, ORTE_VPID_PRINT(tmp_app->num_procs));
1041         opal_argv_append_nosize(&firstranks, ORTE_VPID_PRINT(tmp_app->first_rank));
1042     }
1043     npstring = opal_argv_join(nps, ' ');
1044     firstrankstring = opal_argv_join(firstranks, ' ');
1045     opal_argv_free(nps);
1046     opal_argv_free(firstranks);
1047 
1048     /* add the MPI-3 envars */
1049     opal_setenv("OMPI_NUM_APP_CTX", num_app_ctx, true, &app->env);
1050     opal_setenv("OMPI_FIRST_RANKS", firstrankstring, true, &app->env);
1051     opal_setenv("OMPI_APP_CTX_NUM_PROCS", npstring, true, &app->env);
1052     free(num_app_ctx);
1053     free(firstrankstring);
1054     free(npstring);
1055 
1056     /* now process any envar attributes - we begin with the job-level
1057      * ones as the app-specific ones can override them. We have to
1058      * process them in the order they were given to ensure we wind
1059      * up in the desired final state */
1060     OPAL_LIST_FOREACH(attr, &jdata->attributes, orte_attribute_t) {
1061         if (ORTE_JOB_SET_ENVAR == attr->key) {
1062             opal_setenv(attr->data.envar.envar, attr->data.envar.value, true, &app->env);
1063         } else if (ORTE_JOB_ADD_ENVAR == attr->key) {
1064             opal_setenv(attr->data.envar.envar, attr->data.envar.value, false, &app->env);
1065         } else if (ORTE_JOB_UNSET_ENVAR == attr->key) {
1066             opal_unsetenv(attr->data.string, &app->env);
1067         } else if (ORTE_JOB_PREPEND_ENVAR == attr->key) {
1068             /* see if the envar already exists */
1069             exists = false;
1070             for (i=0; NULL != app->env[i]; i++) {
1071                 saveptr = strchr(app->env[i], '=');   // cannot be NULL
1072                 *saveptr = '\0';
1073                 if (0 == strcmp(app->env[i], attr->data.envar.envar)) {
1074                     /* we have the var - prepend it */
1075                     param = saveptr;
1076                     ++param;  // move past where the '=' sign was
1077                     opal_asprintf(&p2, "%s%c%s", attr->data.envar.value,
1078                                    attr->data.envar.separator, param);
1079                     *saveptr = '=';  // restore the current envar setting
1080                     opal_setenv(attr->data.envar.envar, p2, true, &app->env);
1081                     free(p2);
1082                     exists = true;
1083                     break;
1084                 } else {
1085                     *saveptr = '=';  // restore the current envar setting
1086                 }
1087             }
1088             if (!exists) {
1089                 /* just insert it */
1090                 opal_setenv(attr->data.envar.envar, attr->data.envar.value, true, &app->env);
1091             }
1092         } else if (ORTE_JOB_APPEND_ENVAR == attr->key) {
1093             /* see if the envar already exists */
1094             exists = false;
1095             for (i=0; NULL != app->env[i]; i++) {
1096                 saveptr = strchr(app->env[i], '=');   // cannot be NULL
1097                 *saveptr = '\0';
1098                 if (0 == strcmp(app->env[i], attr->data.envar.envar)) {
1099                     /* we have the var - prepend it */
1100                     param = saveptr;
1101                     ++param;  // move past where the '=' sign was
1102                     opal_asprintf(&p2, "%s%c%s", param, attr->data.envar.separator,
1103                                    attr->data.envar.value);
1104                     *saveptr = '=';  // restore the current envar setting
1105                     opal_setenv(attr->data.envar.envar, p2, true, &app->env);
1106                     free(p2);
1107                     exists = true;
1108                     break;
1109                 } else {
1110                     *saveptr = '=';  // restore the current envar setting
1111                 }
1112             }
1113             if (!exists) {
1114                 /* just insert it */
1115                 opal_setenv(attr->data.envar.envar, attr->data.envar.value, true, &app->env);
1116             }
1117         }
1118     }
1119 
1120     /* now do the same thing for any app-level attributes */
1121     OPAL_LIST_FOREACH(attr, &app->attributes, orte_attribute_t) {
1122         if (ORTE_APP_SET_ENVAR == attr->key) {
1123             opal_setenv(attr->data.envar.envar, attr->data.envar.value, true, &app->env);
1124         } else if (ORTE_APP_ADD_ENVAR == attr->key) {
1125             opal_setenv(attr->data.envar.envar, attr->data.envar.value, false, &app->env);
1126         } else if (ORTE_APP_UNSET_ENVAR == attr->key) {
1127             opal_unsetenv(attr->data.string, &app->env);
1128         } else if (ORTE_APP_PREPEND_ENVAR == attr->key) {
1129             /* see if the envar already exists */
1130             exists = false;
1131             for (i=0; NULL != app->env[i]; i++) {
1132                 saveptr = strchr(app->env[i], '=');   // cannot be NULL
1133                 *saveptr = '\0';
1134                 if (0 == strcmp(app->env[i], attr->data.envar.envar)) {
1135                     /* we have the var - prepend it */
1136                     param = saveptr;
1137                     ++param;  // move past where the '=' sign was
1138                     opal_asprintf(&p2, "%s%c%s", attr->data.envar.value,
1139                                    attr->data.envar.separator, param);
1140                     *saveptr = '=';  // restore the current envar setting
1141                     opal_setenv(attr->data.envar.envar, p2, true, &app->env);
1142                     free(p2);
1143                     exists = true;
1144                     break;
1145                 } else {
1146                     *saveptr = '=';  // restore the current envar setting
1147                 }
1148             }
1149             if (!exists) {
1150                 /* just insert it */
1151                 opal_setenv(attr->data.envar.envar, attr->data.envar.value, true, &app->env);
1152             }
1153         } else if (ORTE_APP_APPEND_ENVAR == attr->key) {
1154             /* see if the envar already exists */
1155             exists = false;
1156             for (i=0; NULL != app->env[i]; i++) {
1157                 saveptr = strchr(app->env[i], '=');   // cannot be NULL
1158                 *saveptr = '\0';
1159                 if (0 == strcmp(app->env[i], attr->data.envar.envar)) {
1160                     /* we have the var - prepend it */
1161                     param = saveptr;
1162                     ++param;  // move past where the '=' sign was
1163                     opal_asprintf(&p2, "%s%c%s", param, attr->data.envar.separator,
1164                                    attr->data.envar.value);
1165                     *saveptr = '=';  // restore the current envar setting
1166                     opal_setenv(attr->data.envar.envar, p2, true, &app->env);
1167                     free(p2);
1168                     exists = true;
1169                     break;
1170                 } else {
1171                     *saveptr = '=';  // restore the current envar setting
1172                 }
1173             }
1174             if (!exists) {
1175                 /* just insert it */
1176                 opal_setenv(attr->data.envar.envar, attr->data.envar.value, true, &app->env);
1177             }
1178         }
1179     }
1180 
1181     return ORTE_SUCCESS;
1182 }
1183 
1184 
1185 static int setup_child(orte_job_t *jdata,
1186                        orte_proc_t *child,
1187                        orte_app_context_t *app,
1188                        char ***env)
1189 {
1190     char *param, *value;
1191     int rc, i;
1192     int32_t nrestarts=0, *nrptr;
1193     bool takeus = false;
1194 
1195     opal_output_verbose(1, orte_schizo_base_framework.framework_output,
1196                         "%s schizo:ompi: setup_child",
1197                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
1198 
1199     /* if no personality was specified, then nothing to do */
1200     if (NULL == jdata->personality) {
1201         return ORTE_ERR_TAKE_NEXT_OPTION;
1202     }
1203 
1204     if (NULL != orte_schizo_base.personalities) {
1205         /* see if we are included */
1206         for (i=0; NULL != jdata->personality[i]; i++) {
1207             if (0 == strcmp(jdata->personality[i], "ompi")) {
1208                 takeus = true;
1209                 break;
1210             }
1211         }
1212         if (!takeus) {
1213             return ORTE_ERR_TAKE_NEXT_OPTION;
1214         }
1215     }
1216 
1217     /* setup the jobid */
1218     if (ORTE_SUCCESS != (rc = orte_util_convert_jobid_to_string(&value, child->name.jobid))) {
1219         ORTE_ERROR_LOG(rc);
1220         return rc;
1221     }
1222     opal_setenv("OMPI_MCA_ess_base_jobid", value, true, env);
1223     free(value);
1224 
1225     /* setup the vpid */
1226     if (ORTE_SUCCESS != (rc = orte_util_convert_vpid_to_string(&value, child->name.vpid))) {
1227         ORTE_ERROR_LOG(rc);
1228         return rc;
1229     }
1230     opal_setenv("OMPI_MCA_ess_base_vpid", value, true, env);
1231 
1232     /* although the vpid IS the process' rank within the job, users
1233      * would appreciate being given a public environmental variable
1234      * that also represents this value - something MPI specific - so
1235      * do that here.
1236      *
1237      * AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
1238      * We know - just live with it
1239      */
1240     opal_setenv("OMPI_COMM_WORLD_RANK", value, true, env);
1241     free(value);  /* done with this now */
1242 
1243     /* users would appreciate being given a public environmental variable
1244      * that also represents the local rank value - something MPI specific - so
1245      * do that here.
1246      *
1247      * AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
1248      * We know - just live with it
1249      */
1250     if (ORTE_LOCAL_RANK_INVALID == child->local_rank) {
1251         ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
1252         rc = ORTE_ERR_VALUE_OUT_OF_BOUNDS;
1253         return rc;
1254     }
1255     opal_asprintf(&value, "%lu", (unsigned long) child->local_rank);
1256     opal_setenv("OMPI_COMM_WORLD_LOCAL_RANK", value, true, env);
1257     free(value);
1258 
1259     /* users would appreciate being given a public environmental variable
1260      * that also represents the node rank value - something MPI specific - so
1261      * do that here.
1262      *
1263      * AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
1264      * We know - just live with it
1265      */
1266     if (ORTE_NODE_RANK_INVALID == child->node_rank) {
1267         ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
1268         rc = ORTE_ERR_VALUE_OUT_OF_BOUNDS;
1269         return rc;
1270     }
1271     opal_asprintf(&value, "%lu", (unsigned long) child->node_rank);
1272     opal_setenv("OMPI_COMM_WORLD_NODE_RANK", value, true, env);
1273     /* set an mca param for it too */
1274     opal_setenv("OMPI_MCA_orte_ess_node_rank", value, true, env);
1275     free(value);
1276 
1277     /* provide the identifier for the PMIx connection - the
1278      * PMIx connection is made prior to setting the process
1279      * name itself. Although in most cases the ID and the
1280      * process name are the same, it isn't necessarily
1281      * required */
1282     orte_util_convert_process_name_to_string(&value, &child->name);
1283     opal_setenv("PMIX_ID", value, true, env);
1284     free(value);
1285 
1286     nrptr = &nrestarts;
1287     if (orte_get_attribute(&child->attributes, ORTE_PROC_NRESTARTS, (void**)&nrptr, OPAL_INT32)) {
1288         /* pass the number of restarts for this proc - will be zero for
1289          * an initial start, but procs would like to know if they are being
1290          * restarted so they can take appropriate action
1291          */
1292         opal_asprintf(&value, "%d", nrestarts);
1293         opal_setenv("OMPI_MCA_orte_num_restarts", value, true, env);
1294         free(value);
1295     }
1296 
1297     /* if the proc should not barrier in orte_init, tell it */
1298     if (orte_get_attribute(&child->attributes, ORTE_PROC_NOBARRIER, NULL, OPAL_BOOL)
1299         || 0 < nrestarts) {
1300         opal_setenv("OMPI_MCA_orte_do_not_barrier", "1", true, env);
1301     }
1302 
1303     /* if the proc isn't going to forward IO, then we need to flag that
1304      * it has "completed" iof termination as otherwise it will never fire
1305      */
1306     if (!ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_FORWARD_OUTPUT)) {
1307         ORTE_FLAG_SET(child, ORTE_PROC_FLAG_IOF_COMPLETE);
1308     }
1309 
1310     /* pass an envar so the proc can find any files it had prepositioned */
1311     param = orte_process_info.proc_session_dir;
1312     opal_setenv("OMPI_FILE_LOCATION", param, true, env);
1313 
1314     /* if the user wanted the cwd to be the proc's session dir, then
1315      * switch to that location now
1316      */
1317     if (orte_get_attribute(&app->attributes, ORTE_APP_SSNDIR_CWD, NULL, OPAL_BOOL)) {
1318         /* create the session dir - may not exist */
1319         if (OPAL_SUCCESS != (rc = opal_os_dirpath_create(param, S_IRWXU))) {
1320             ORTE_ERROR_LOG(rc);
1321             /* doesn't exist with correct permissions, and/or we can't
1322              * create it - either way, we are done
1323              */
1324             return rc;
1325         }
1326         /* change to it */
1327         if (0 != chdir(param)) {
1328             return ORTE_ERROR;
1329         }
1330         /* It seems that chdir doesn't
1331          * adjust the $PWD enviro variable when it changes the directory. This
1332          * can cause a user to get a different response when doing getcwd vs
1333          * looking at the enviro variable. To keep this consistent, we explicitly
1334          * ensure that the PWD enviro variable matches the CWD we moved to.
1335          *
1336          * NOTE: if a user's program does a chdir(), then $PWD will once
1337          * again not match getcwd! This is beyond our control - we are only
1338          * ensuring they start out matching.
1339          */
1340         opal_setenv("PWD", param, true, env);
1341         /* update the initial wdir value too */
1342         opal_setenv("OMPI_MCA_initial_wdir", param, true, env);
1343     } else if (NULL != app->cwd) {
1344         /* change to it */
1345         if (0 != chdir(app->cwd)) {
1346             return ORTE_ERROR;
1347         }
1348     }
1349     return ORTE_SUCCESS;
1350 }
/* [<][>][^][v][top][bottom][index][help] */
root/orte/mca/schizo/ompi/schizo_ompi.c

DEFINITIONS
