opal/mca/pmix/ext3x/ext3x

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
errreg_cbfunc
ext3x_client_init
dereg_cbfunc
ext3x_client_finalize
ext3x_tool_init
ext3x_tool_fini
ext3x_initialized
ext3x_abort
ext3x_store_local
ext3x_commit
opcbfunc
ext3x_fence
ext3x_fencenb
ext3x_put
ext3x_get
val_cbfunc
ext3x_getnb
ext3x_publish
ext3x_publishnb
ext3x_lookup
lk_cbfunc
ext3x_lookupnb
ext3x_unpublish
ext3x_unpublishnb
ext3x_spawn
spcbfunc
ext3x_spawnnb
ext3x_connect
ext3x_connectnb
ext3x_disconnect
ext3x_disconnectnb
ext3x_resolve_peers
ext3x_resolve_nodes
relcbfunc
infocbfunc
ext3x_allocate
ext3x_job_control
   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
   4  * Copyright (c) 2014-2017 Research Organization for Information Science
   5  *                         and Technology (RIST). All rights reserved.
   6  * Copyright (c) 2014-2017 Mellanox Technologies, Inc.
   7  *                         All rights reserved.
   8  * Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
   9  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  10  *                         reserved.
  11  * Copyright (c) 2018      The University of Tennessee and The University
  12  *                         of Tennessee Research Foundation.  All rights
  13  *                         reserved.
  14  * Copyright (c) 2019      IBM Corporation.  All rights reserved.
  15  * $COPYRIGHT$
  16  *
  17  * Additional copyrights may follow
  18  *
  19  * $HEADER$
  20  */
  21 
  22 #include "opal_config.h"
  23 #include "opal/constants.h"
  24 #include "opal/types.h"
  25 
  26 #ifdef HAVE_STRING_H
  27 #include <string.h>
  28 #endif
  29 #ifdef HAVE_UNISTD_H
  30 #include <unistd.h>
  31 #endif
  32 
  33 #include "opal/hash_string.h"
  34 #include "opal/threads/threads.h"
  35 #include "opal/util/argv.h"
  36 #include "opal/util/opal_environ.h"
  37 #include "opal/util/proc.h"
  38 #include "opal/util/show_help.h"
  39 #include "opal/util/string_copy.h"
  40 
  41 #include "opal/mca/pmix/base/base.h"
  42 #include "ext3x.h"
  43 #include "pmix.h"
  44 #include "pmix_tool.h"
  45 
  46 static char *dbgvalue=NULL;
  47 
  48 static void errreg_cbfunc (pmix_status_t status,
  49                            size_t errhandler_ref,
  50                            void *cbdata)
  51 {
  52     opal_ext3x_event_t *event = (opal_ext3x_event_t*)cbdata;
  53 
  54     OPAL_ACQUIRE_OBJECT(event);
  55 
  56     event->index = errhandler_ref;
  57     opal_output_verbose(5, opal_pmix_base_framework.framework_output,
  58                         "PMIX client errreg_cbfunc - error handler registered status=%d, reference=%lu",
  59                         status, (unsigned long)errhandler_ref);
  60     OPAL_POST_OBJECT(event);
  61     OPAL_PMIX_WAKEUP_THREAD(&event->lock);
  62 }
  63 
  64 int ext3x_client_init(opal_list_t *ilist)
  65 {
  66     opal_process_name_t pname;
  67     pmix_status_t rc;
  68     int dbg;
  69     opal_ext3x_jobid_trkr_t *job;
  70     opal_ext3x_event_t *event;
  71     pmix_info_t *pinfo;
  72     size_t ninfo, n;
  73     opal_value_t *ival;
  74 
  75     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
  76                         "PMIx_client init");
  77 
  78     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
  79 
  80     if (0 == opal_pmix_base.initialized) {
  81         if (0 < (dbg = opal_output_get_verbosity(opal_pmix_base_framework.framework_output))) {
  82             asprintf(&dbgvalue, "PMIX_DEBUG=%d", dbg);
  83             putenv(dbgvalue);
  84         }
  85         /* check the evars for a mismatch */
  86         if (OPAL_SUCCESS != (dbg = opal_pmix_ext3x_check_evars())) {
  87             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
  88             return dbg;
  89         }
  90     }
  91 
  92     /* convert the incoming list to info structs */
  93     if (NULL != ilist && 0 < (ninfo = opal_list_get_size(ilist))) {
  94         PMIX_INFO_CREATE(pinfo, ninfo);
  95         n=0;
  96         OPAL_LIST_FOREACH(ival, ilist, opal_value_t) {
  97             (void)opal_string_copy(pinfo[n].key, ival->key, PMIX_MAX_KEYLEN);
  98             ext3x_value_load(&pinfo[n].value, ival);
  99             ++n;
 100         }
 101     } else {
 102         pinfo = NULL;
 103         ninfo = 0;
 104     }
 105 
 106     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 107     rc = PMIx_Init(&mca_pmix_ext3x_component.myproc, pinfo, ninfo);
 108     if (NULL != pinfo) {
 109         PMIX_INFO_FREE(pinfo, ninfo);
 110     }
 111     if (PMIX_SUCCESS != rc) {
 112         dbg = ext3x_convert_rc(rc);
 113         OPAL_ERROR_LOG(dbg);
 114         return dbg;
 115     }
 116     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 117 
 118     ++opal_pmix_base.initialized;
 119     if (1 < opal_pmix_base.initialized) {
 120         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 121         return OPAL_SUCCESS;
 122     }
 123 
 124     /* store our jobid and rank */
 125     if (NULL != getenv(OPAL_MCA_PREFIX"orte_launch")) {
 126         /* if we were launched by the OMPI RTE, then
 127          * the jobid is in a special format - so get it */
 128         mca_pmix_ext3x_component.native_launch = true;
 129         opal_convert_string_to_jobid(&pname.jobid, mca_pmix_ext3x_component.myproc.nspace);
 130     } else {
 131         /* we were launched by someone else, so make the
 132          * jobid just be the hash of the nspace */
 133         OPAL_HASH_JOBID(mca_pmix_ext3x_component.myproc.nspace, pname.jobid);
 134     }
 135     /* insert this into our list of jobids - it will be the
 136      * first, and so we'll check it first */
 137     job = OBJ_NEW(opal_ext3x_jobid_trkr_t);
 138     (void)opal_string_copy(job->nspace, mca_pmix_ext3x_component.myproc.nspace, PMIX_MAX_NSLEN);
 139     job->jobid = pname.jobid;
 140     opal_list_append(&mca_pmix_ext3x_component.jobids, &job->super);
 141 
 142     pname.vpid = ext3x_convert_rank(mca_pmix_ext3x_component.myproc.rank);
 143     opal_proc_set_name(&pname);
 144 
 145     /* release the thread in case the event handler fires when
 146      * registered */
 147     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 148 
 149     /* register the default event handler */
 150     event = OBJ_NEW(opal_ext3x_event_t);
 151     opal_list_append(&mca_pmix_ext3x_component.events, &event->super);
 152     PMIX_INFO_CREATE(pinfo, 1);
 153     PMIX_INFO_LOAD(&pinfo[0], PMIX_EVENT_HDLR_NAME, "OPAL-PMIX-2X-DEFAULT", PMIX_STRING);
 154     PMIx_Register_event_handler(NULL, 0, NULL, 0, ext3x_event_hdlr, errreg_cbfunc, event);
 155     OPAL_PMIX_WAIT_THREAD(&event->lock);
 156     PMIX_INFO_FREE(pinfo, 1);
 157 
 158     return OPAL_SUCCESS;
 159 
 160 }
 161 
 162 static void dereg_cbfunc(pmix_status_t st, void *cbdata)
 163 {
 164     opal_ext3x_event_t *ev = (opal_ext3x_event_t*)cbdata;
 165     OPAL_PMIX_WAKEUP_THREAD(&ev->lock);
 166 }
 167 
 168 int ext3x_client_finalize(void)
 169 {
 170     pmix_status_t rc;
 171     opal_ext3x_event_t *event, *ev2;
 172     opal_list_t evlist;
 173     OBJ_CONSTRUCT(&evlist, opal_list_t);
 174 
 175     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 176                         "PMIx_client finalize");
 177 
 178     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 179     --opal_pmix_base.initialized;
 180 
 181     if (0 == opal_pmix_base.initialized) {
 182         /* deregister all event handlers */
 183         OPAL_LIST_FOREACH_SAFE(event, ev2, &mca_pmix_ext3x_component.events, opal_ext3x_event_t) {
 184             OPAL_PMIX_DESTRUCT_LOCK(&event->lock);
 185             OPAL_PMIX_CONSTRUCT_LOCK(&event->lock);
 186             PMIx_Deregister_event_handler(event->index, dereg_cbfunc, (void*)event);
 187             opal_list_remove_item(&mca_pmix_ext3x_component.events, &event->super);
 188             /* wait and release outside the loop to avoid double mutex
 189              * interlock */
 190             opal_list_append(&evlist, &event->super);
 191         }
 192     }
 193     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 194     OPAL_LIST_FOREACH_SAFE(event, ev2, &evlist, opal_ext3x_event_t) {
 195         OPAL_PMIX_WAIT_THREAD(&event->lock);
 196         opal_list_remove_item(&evlist, &event->super);
 197         OBJ_RELEASE(event);
 198     }
 199     OBJ_DESTRUCT(&evlist);
 200     rc = PMIx_Finalize(NULL, 0);
 201 
 202     return ext3x_convert_rc(rc);
 203 }
 204 
 205 int ext3x_tool_init(opal_list_t *info)
 206 {
 207     pmix_info_t *pinfo;
 208     size_t ninfo, n;
 209     opal_ext3x_jobid_trkr_t *job;
 210     opal_value_t *val;
 211     pmix_status_t rc;
 212     int ret;
 213     opal_process_name_t pname = {OPAL_JOBID_INVALID, OPAL_VPID_INVALID};
 214     opal_ext3x_event_t *event;
 215 
 216     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 217                         "PMIx_tool init");
 218 
 219     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 220 
 221     /* convert the incoming list to info structs */
 222     if (NULL != info && 0 < (ninfo = opal_list_get_size(info))) {
 223         PMIX_INFO_CREATE(pinfo, ninfo);
 224         n=0;
 225         OPAL_LIST_FOREACH(val, info, opal_value_t) {
 226             (void)opal_string_copy(pinfo[n].key, val->key, PMIX_MAX_KEYLEN);
 227             ext3x_value_load(&pinfo[n].value, val);
 228             ++n;
 229             /* check to see if our name is being given from above */
 230             if (0 == strcmp(val->key, OPAL_PMIX_TOOL_NSPACE)) {
 231                 opal_convert_string_to_jobid(&pname.jobid, val->data.string);
 232                 (void)opal_string_copy(mca_pmix_ext3x_component.myproc.nspace, val->data.string, PMIX_MAX_NSLEN);
 233             } else if (0 == strcmp(val->key, OPAL_PMIX_TOOL_RANK)) {
 234                 pname.vpid = val->data.name.vpid;
 235                 mca_pmix_ext3x_component.myproc.rank = pname.vpid;
 236             }
 237         }
 238     } else {
 239         pinfo = NULL;
 240         ninfo = 0;
 241     }
 242     /* we are going to get our name from the server, or we were given it by the tool,
 243      * so mark as native launch so we don't convert back/forth */
 244     mca_pmix_ext3x_component.native_launch = true;
 245 
 246     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 247     rc = PMIx_tool_init(&mca_pmix_ext3x_component.myproc, pinfo, ninfo);
 248     if (NULL != pinfo) {
 249         PMIX_INFO_FREE(pinfo, ninfo);
 250     }
 251     if (PMIX_SUCCESS != rc) {
 252         ret = ext3x_convert_rc(rc);
 253         OPAL_ERROR_LOG(ret);
 254         return ret;
 255     }
 256     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 257 
 258     ++opal_pmix_base.initialized;
 259     if (1 < opal_pmix_base.initialized) {
 260         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 261         return OPAL_SUCCESS;
 262     }
 263 
 264     /* store our jobid and rank */
 265     opal_convert_string_to_jobid(&pname.jobid, mca_pmix_ext3x_component.myproc.nspace);
 266     pname.vpid = ext3x_convert_rank(mca_pmix_ext3x_component.myproc.rank);
 267 
 268     /* insert this into our list of jobids - it will be the
 269      * first, and so we'll check it first */
 270     job = OBJ_NEW(opal_ext3x_jobid_trkr_t);
 271     (void)opal_string_copy(job->nspace, mca_pmix_ext3x_component.myproc.nspace, PMIX_MAX_NSLEN);
 272     job->jobid = pname.jobid;
 273     opal_list_append(&mca_pmix_ext3x_component.jobids, &job->super);
 274 
 275     opal_proc_set_name(&pname);
 276 
 277     /* release the thread in case the event handler fires when
 278      * registered */
 279     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 280 
 281     /* register the default event handler */
 282     event = OBJ_NEW(opal_ext3x_event_t);
 283     opal_list_append(&mca_pmix_ext3x_component.events, &event->super);
 284     PMIX_INFO_CREATE(pinfo, 1);
 285     PMIX_INFO_LOAD(&pinfo[0], PMIX_EVENT_HDLR_NAME, "OPAL-PMIX-2X-DEFAULT", PMIX_STRING);
 286     PMIx_Register_event_handler(NULL, 0, NULL, 0, ext3x_event_hdlr, errreg_cbfunc, event);
 287     OPAL_PMIX_WAIT_THREAD(&event->lock);
 288     PMIX_INFO_FREE(pinfo, 1);
 289 
 290     return OPAL_SUCCESS;
 291 }
 292 
 293 int ext3x_tool_fini(void)
 294 {
 295     pmix_status_t rc;
 296     opal_ext3x_event_t *event, *ev2;
 297 
 298     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 299                         "PMIx_tool finalize");
 300 
 301     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 302     --opal_pmix_base.initialized;
 303 
 304     if (0 == opal_pmix_base.initialized) {
 305         /* deregister all event handlers */
 306         OPAL_LIST_FOREACH_SAFE(event, ev2, &mca_pmix_ext3x_component.events, opal_ext3x_event_t) {
 307             OPAL_PMIX_DESTRUCT_LOCK(&event->lock);
 308             OPAL_PMIX_CONSTRUCT_LOCK(&event->lock);
 309             PMIx_Deregister_event_handler(event->index, dereg_cbfunc, (void*)event);
 310             OPAL_PMIX_WAIT_THREAD(&event->lock);
 311             opal_list_remove_item(&mca_pmix_ext3x_component.events, &event->super);
 312             OBJ_RELEASE(event);
 313         }
 314     }
 315     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 316     rc = PMIx_tool_finalize();
 317 
 318     return ext3x_convert_rc(rc);
 319 }
 320 
 321 
 322 int ext3x_initialized(void)
 323 {
 324     int init;
 325 
 326     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 327                         "PMIx_client initialized");
 328 
 329     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 330     init = opal_pmix_base.initialized;
 331     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 332 
 333     return init;
 334 }
 335 
 336 int ext3x_abort(int flag, const char *msg,
 337                  opal_list_t *procs)
 338 {
 339     pmix_status_t rc;
 340     pmix_proc_t *parray=NULL;
 341     size_t n, cnt=0;
 342     opal_namelist_t *ptr;
 343     char *nsptr;
 344 
 345     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 346                         "PMIx_client abort");
 347 
 348     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 349     if (0 >= opal_pmix_base.initialized) {
 350         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 351         return OPAL_ERR_NOT_INITIALIZED;
 352     }
 353     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 354 
 355     /* convert the list of procs to an array
 356      * of pmix_proc_t */
 357     if (NULL != procs && 0 < (cnt = opal_list_get_size(procs))) {
 358         PMIX_PROC_CREATE(parray, cnt);
 359         n=0;
 360         OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) {
 361             if (NULL == (nsptr = ext3x_convert_jobid(ptr->name.jobid))) {
 362                 PMIX_PROC_FREE(parray, cnt);
 363                 return OPAL_ERR_NOT_FOUND;
 364             }
 365             (void)opal_string_copy(parray[n].nspace, nsptr, PMIX_MAX_NSLEN);
 366             parray[n].rank = ext3x_convert_opalrank(ptr->name.vpid);
 367             ++n;
 368         }
 369     }
 370 
 371     /* call the library abort - this is a blocking call */
 372     rc = PMIx_Abort(flag, msg, parray, cnt);
 373 
 374     /* release the array */
 375     PMIX_PROC_FREE(parray, cnt);
 376 
 377     return ext3x_convert_rc(rc);
 378 }
 379 
 380 int ext3x_store_local(const opal_process_name_t *proc, opal_value_t *val)
 381 {
 382     pmix_value_t kv;
 383     pmix_status_t rc;
 384     pmix_proc_t p;
 385     char *nsptr;
 386     opal_ext3x_jobid_trkr_t *job;
 387 
 388     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 389 
 390     if (0 >= opal_pmix_base.initialized) {
 391         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 392         return OPAL_ERR_NOT_INITIALIZED;
 393     }
 394     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 395 
 396     if (NULL != proc) {
 397         if (NULL == (nsptr = ext3x_convert_jobid(proc->jobid))) {
 398             job = OBJ_NEW(opal_ext3x_jobid_trkr_t);
 399             (void)opal_snprintf_jobid(job->nspace, PMIX_MAX_NSLEN, proc->jobid);
 400             job->jobid = proc->jobid;
 401             OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 402             opal_list_append(&mca_pmix_ext3x_component.jobids, &job->super);
 403             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 404             nsptr = job->nspace;
 405         }
 406         (void)opal_string_copy(p.nspace, nsptr, PMIX_MAX_NSLEN);
 407         p.rank = ext3x_convert_opalrank(proc->vpid);
 408     } else {
 409         /* use our name */
 410         (void)opal_string_copy(p.nspace, mca_pmix_ext3x_component.myproc.nspace, PMIX_MAX_NSLEN);
 411         p.rank = ext3x_convert_opalrank(OPAL_PROC_MY_NAME.vpid);
 412     }
 413 
 414     PMIX_VALUE_CONSTRUCT(&kv);
 415     ext3x_value_load(&kv, val);
 416 
 417     /* call the library - this is a blocking call */
 418     rc = PMIx_Store_internal(&p, val->key, &kv);
 419     PMIX_VALUE_DESTRUCT(&kv);
 420 
 421     return ext3x_convert_rc(rc);
 422 }
 423 
 424 int ext3x_commit(void)
 425 {
 426     pmix_status_t rc;
 427 
 428     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 429     if (0 >= opal_pmix_base.initialized) {
 430         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 431         return OPAL_ERR_NOT_INITIALIZED;
 432     }
 433     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 434 
 435     rc = PMIx_Commit();
 436     return ext3x_convert_rc(rc);
 437 }
 438 
 439 static void opcbfunc(pmix_status_t status, void *cbdata)
 440 {
 441     ext3x_opcaddy_t *op = (ext3x_opcaddy_t*)cbdata;
 442 
 443     OPAL_ACQUIRE_OBJECT(op);
 444     if (NULL != op->opcbfunc) {
 445         op->opcbfunc(ext3x_convert_rc(status), op->cbdata);
 446     }
 447     OBJ_RELEASE(op);
 448 }
 449 
 450 int ext3x_fence(opal_list_t *procs, int collect_data)
 451 {
 452     pmix_status_t rc;
 453     opal_namelist_t *ptr;
 454     char *nsptr;
 455     size_t cnt = 0, n;
 456     pmix_proc_t *parray = NULL;
 457     pmix_info_t info, *iptr;
 458 
 459     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 460                         "PMIx_client fence");
 461 
 462     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 463     if (0 >= opal_pmix_base.initialized) {
 464         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 465         return OPAL_ERR_NOT_INITIALIZED;
 466     }
 467 
 468     /* convert the list of procs to an array
 469      * of pmix_proc_t */
 470     if (NULL != procs && 0 < (cnt = opal_list_get_size(procs))) {
 471         PMIX_PROC_CREATE(parray, cnt);
 472         n=0;
 473         OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) {
 474             if (NULL == (nsptr = ext3x_convert_jobid(ptr->name.jobid))) {
 475                 PMIX_PROC_FREE(parray, cnt);
 476                 OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 477                 return OPAL_ERR_NOT_FOUND;
 478             }
 479             (void)opal_string_copy(parray[n].nspace, nsptr, PMIX_MAX_NSLEN);
 480             parray[n].rank = ext3x_convert_opalrank(ptr->name.vpid);
 481             ++n;
 482         }
 483     }
 484     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 485 
 486     if (collect_data) {
 487         PMIX_INFO_CONSTRUCT(&info);
 488         (void)opal_string_copy(info.key, PMIX_COLLECT_DATA, PMIX_MAX_KEYLEN);
 489         info.value.type = PMIX_BOOL;
 490         info.value.data.flag = true;
 491         iptr = &info;
 492         n = 1;
 493     } else {
 494         iptr = NULL;
 495         n = 0;
 496     }
 497 
 498     rc = PMIx_Fence(parray, cnt, iptr, n);
 499     if (collect_data) {
 500         PMIX_INFO_DESTRUCT(&info);
 501     }
 502     if (NULL != parray) {
 503         PMIX_PROC_FREE(parray, cnt);
 504     }
 505 
 506     return ext3x_convert_rc(rc);
 507 }
 508 
 509 int ext3x_fencenb(opal_list_t *procs, int collect_data,
 510                    opal_pmix_op_cbfunc_t cbfunc, void *cbdata)
 511 {
 512     pmix_status_t rc;
 513     pmix_proc_t *parray=NULL;
 514     size_t n, cnt=0;
 515     opal_namelist_t *ptr;
 516     ext3x_opcaddy_t *op;
 517     char *nsptr;
 518 
 519     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 520                         "PMIx_client fencenb");
 521 
 522     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 523     if (0 >= opal_pmix_base.initialized) {
 524         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 525         return OPAL_ERR_NOT_INITIALIZED;
 526     }
 527 
 528     /* convert the list of procs to an array
 529      * of pmix_proc_t */
 530     if (NULL != procs && 0 < (cnt = opal_list_get_size(procs))) {
 531         PMIX_PROC_CREATE(parray, cnt);
 532         n=0;
 533         OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) {
 534             if (NULL == (nsptr = ext3x_convert_jobid(ptr->name.jobid))) {
 535                 PMIX_PROC_FREE(parray, cnt);
 536                 OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 537                 return OPAL_ERR_NOT_FOUND;
 538             }
 539             (void)opal_string_copy(parray[n].nspace, nsptr, PMIX_MAX_NSLEN);
 540             parray[n].rank = ext3x_convert_opalrank(ptr->name.vpid);
 541             ++n;
 542         }
 543     }
 544     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 545 
 546     /* create the caddy */
 547     op = OBJ_NEW(ext3x_opcaddy_t);
 548     op->opcbfunc = cbfunc;
 549     op->cbdata = cbdata;
 550     op->procs = parray;
 551     op->nprocs = cnt;
 552 
 553     if (collect_data) {
 554         op->ninfo = 1;
 555         PMIX_INFO_CREATE(op->info, op->ninfo);
 556         PMIX_INFO_LOAD(&op->info[0], PMIX_COLLECT_DATA, NULL, PMIX_BOOL);
 557     }
 558 
 559     /* call the library function */
 560     rc = PMIx_Fence_nb(op->procs, op->nprocs, op->info, op->ninfo, opcbfunc, op);
 561     return ext3x_convert_rc(rc);
 562 }
 563 
 564 int ext3x_put(opal_pmix_scope_t opal_scope,
 565                opal_value_t *val)
 566 {
 567     pmix_value_t kv;
 568     pmix_scope_t pmix_scope = ext3x_convert_opalscope(opal_scope);
 569     pmix_status_t rc;
 570 
 571     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 572                         "PMIx_client put");
 573 
 574     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 575     if (0 >= opal_pmix_base.initialized) {
 576         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 577         return OPAL_ERR_NOT_INITIALIZED;
 578     }
 579     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 580 
 581     PMIX_VALUE_CONSTRUCT(&kv);
 582     ext3x_value_load(&kv, val);
 583 
 584     rc = PMIx_Put(pmix_scope, val->key, &kv);
 585     PMIX_VALUE_DESTRUCT(&kv);
 586     return ext3x_convert_rc(rc);
 587 }
 588 
 589 int ext3x_get(const opal_process_name_t *proc, const char *key,
 590                opal_list_t *info, opal_value_t **val)
 591 {
 592     pmix_status_t rc;
 593     pmix_proc_t p;
 594     char *nsptr;
 595     pmix_info_t *pinfo = NULL;
 596     size_t sz = 0, n;
 597     opal_value_t *ival;
 598     pmix_value_t *pval = NULL;
 599     int ret;
 600 
 601     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 602                         "%s ext3x:client get on proc %s key %s",
 603                         OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
 604                         (NULL == proc) ? "NULL" : OPAL_NAME_PRINT(*proc), key);
 605 
 606     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 607     if (0 >= opal_pmix_base.initialized) {
 608         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 609         return OPAL_ERR_NOT_INITIALIZED;
 610     }
 611 
 612     if (NULL == proc) {
 613         /* if they are asking for our jobid, then return it */
 614         if (0 == strcmp(key, OPAL_PMIX_JOBID)) {
 615             (*val) = OBJ_NEW(opal_value_t);
 616             (*val)->key = strdup(key);
 617             (*val)->type = OPAL_UINT32;
 618             (*val)->data.uint32 = OPAL_PROC_MY_NAME.jobid;
 619             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 620             return OPAL_SUCCESS;
 621         }
 622         /* if they are asking for our rank, return it */
 623         if (0 == strcmp(key, OPAL_PMIX_RANK)) {
 624             (*val) = OBJ_NEW(opal_value_t);
 625             (*val)->key = strdup(key);
 626             (*val)->type = OPAL_INT;
 627             (*val)->data.integer = ext3x_convert_rank(mca_pmix_ext3x_component.myproc.rank);
 628             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 629             return OPAL_SUCCESS;
 630         }
 631     }
 632     *val = NULL;
 633 
 634     if (NULL == proc) {
 635         (void)opal_string_copy(p.nspace, mca_pmix_ext3x_component.myproc.nspace, PMIX_MAX_NSLEN);
 636         p.rank = ext3x_convert_rank(PMIX_RANK_WILDCARD);
 637     } else {
 638         if (NULL == (nsptr = ext3x_convert_jobid(proc->jobid))) {
 639             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 640             return OPAL_ERR_NOT_FOUND;
 641         }
 642         (void)opal_string_copy(p.nspace, nsptr, PMIX_MAX_NSLEN);
 643         p.rank = ext3x_convert_opalrank(proc->vpid);
 644     }
 645     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 646 
 647     if (NULL != info && 0 < (sz = opal_list_get_size(info))) {
 648         PMIX_INFO_CREATE(pinfo, sz);
 649         n=0;
 650         OPAL_LIST_FOREACH(ival, info, opal_value_t) {
 651             (void)opal_string_copy(pinfo[n].key, ival->key, PMIX_MAX_KEYLEN);
 652             ext3x_value_load(&pinfo[n].value, ival);
 653             ++n;
 654         }
 655     }
 656 
 657     rc = PMIx_Get(&p, key, pinfo, sz, &pval);
 658     if (PMIX_SUCCESS == rc) {
 659         ival = OBJ_NEW(opal_value_t);
 660         if (NULL != key) {
 661             ival->key = strdup(key);
 662         }
 663         if (OPAL_SUCCESS != (ret = ext3x_value_unload(ival, pval))) {
 664             rc = ext3x_convert_opalrc(ret);
 665         } else {
 666             *val = ival;
 667         }
 668         PMIX_VALUE_FREE(pval, 1);
 669     }
 670     PMIX_INFO_FREE(pinfo, sz);
 671 
 672     return ext3x_convert_rc(rc);
 673 }
 674 
 675 static void val_cbfunc(pmix_status_t status,
 676                        pmix_value_t *kv, void *cbdata)
 677 {
 678     ext3x_opcaddy_t *op = (ext3x_opcaddy_t*)cbdata;
 679     int rc;
 680     opal_value_t val, *v=NULL;
 681 
 682     OPAL_ACQUIRE_OBJECT(op);
 683     OBJ_CONSTRUCT(&val, opal_value_t);
 684     if (NULL != op->nspace) {
 685         val.key = strdup(op->nspace);
 686     }
 687     rc = ext3x_convert_opalrc(status);
 688     if (PMIX_SUCCESS == status && NULL != kv) {
 689         rc = ext3x_value_unload(&val, kv);
 690         v = &val;
 691     }
 692 
 693     if (NULL != op->valcbfunc) {
 694         op->valcbfunc(rc, v, op->cbdata);
 695     }
 696     OBJ_DESTRUCT(&val);
 697     OBJ_RELEASE(op);
 698 }
 699 
 700 int ext3x_getnb(const opal_process_name_t *proc, const char *key,
 701                  opal_list_t *info,
 702                  opal_pmix_value_cbfunc_t cbfunc, void *cbdata)
 703 {
 704     ext3x_opcaddy_t *op;
 705     opal_value_t *val;
 706     pmix_status_t rc;
 707     char *nsptr;
 708     size_t n;
 709 
 710     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 711                         "%s PMIx_client get_nb on proc %s key %s",
 712                         OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
 713                         (NULL == proc) ? "NULL" : OPAL_NAME_PRINT(*proc), key);
 714 
 715     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 716     if (0 >= opal_pmix_base.initialized) {
 717         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 718         return OPAL_ERR_NOT_INITIALIZED;
 719     }
 720 
 721     if (NULL == proc) {
 722         /* if they are asking for our jobid, then return it */
 723         if (0 == strcmp(key, OPAL_PMIX_JOBID)) {
 724             if (NULL != cbfunc) {
 725                 val = OBJ_NEW(opal_value_t);
 726                 val->key = strdup(key);
 727                 val->type = OPAL_UINT32;
 728                 val->data.uint32 = OPAL_PROC_MY_NAME.jobid;
 729                 cbfunc(OPAL_SUCCESS, val, cbdata);
 730             }
 731             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 732             return OPAL_SUCCESS;
 733         }
 734         /* if they are asking for our rank, return it */
 735         if (0 == strcmp(key, OPAL_PMIX_RANK)) {
 736             if (NULL != cbfunc) {
 737                 val = OBJ_NEW(opal_value_t);
 738                 val->key = strdup(key);
 739                 val->type = OPAL_INT;
 740                 val->data.integer = ext3x_convert_rank(mca_pmix_ext3x_component.myproc.rank);
 741                 cbfunc(OPAL_SUCCESS, val, cbdata);
 742             }
 743             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 744             return OPAL_SUCCESS;
 745         }
 746     }
 747 
 748     /* create the caddy */
 749     op = OBJ_NEW(ext3x_opcaddy_t);
 750     op->valcbfunc = cbfunc;
 751     op->cbdata = cbdata;
 752     if (NULL != key) {
 753         op->nspace = strdup(key);
 754     }
 755     if (NULL == proc) {
 756         (void)opal_string_copy(op->p.nspace, mca_pmix_ext3x_component.myproc.nspace, PMIX_MAX_NSLEN);
 757         op->p.rank = ext3x_convert_rank(PMIX_RANK_WILDCARD);
 758     } else {
 759         if (NULL == (nsptr = ext3x_convert_jobid(proc->jobid))) {
 760             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 761             return OPAL_ERR_NOT_FOUND;
 762         }
 763         (void)opal_string_copy(op->p.nspace, nsptr, PMIX_MAX_NSLEN);
 764         op->p.rank = ext3x_convert_opalrank(proc->vpid);
 765     }
 766     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 767 
 768     if (NULL != info && 0 < (op->sz = opal_list_get_size(info))) {
 769         PMIX_INFO_CREATE(op->info, op->sz);
 770         n=0;
 771         OPAL_LIST_FOREACH(val, info, opal_value_t) {
 772             (void)opal_string_copy(op->info[n].key, val->key, PMIX_MAX_KEYLEN);
 773             ext3x_value_load(&op->info[n].value, val);
 774             ++n;
 775         }
 776     }
 777 
 778     /* call the library function */
 779     rc = PMIx_Get_nb(&op->p, key, op->info, op->sz, val_cbfunc, op);
 780     if (PMIX_SUCCESS != rc) {
 781         OBJ_RELEASE(op);
 782     }
 783 
 784     return ext3x_convert_rc(rc);
 785 }
 786 
 787 int ext3x_publish(opal_list_t *info)
 788 {
 789     pmix_info_t *pinfo;
 790     pmix_status_t ret;
 791     opal_value_t *iptr;
 792     size_t sz, n;
 793 
 794     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 795                         "PMIx_client publish");
 796 
 797     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 798     if (0 >= opal_pmix_base.initialized) {
 799         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 800         return OPAL_ERR_NOT_INITIALIZED;
 801     }
 802     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 803 
 804     if (NULL == info) {
 805         return OPAL_ERR_BAD_PARAM;
 806     }
 807 
 808     sz = opal_list_get_size(info);
 809     if (0 < sz) {
 810         PMIX_INFO_CREATE(pinfo, sz);
 811         n=0;
 812         OPAL_LIST_FOREACH(iptr, info, opal_value_t) {
 813             (void)opal_string_copy(pinfo[n].key, iptr->key, PMIX_MAX_KEYLEN);
 814             ext3x_value_load(&pinfo[n].value, iptr);
 815             ++n;
 816         }
 817     } else {
 818         pinfo = NULL;
 819     }
 820 
 821     ret = PMIx_Publish(pinfo, sz);
 822     if (0 < sz) {
 823         PMIX_INFO_FREE(pinfo, sz);
 824     }
 825 
 826     return ext3x_convert_rc(ret);
 827 }
 828 
 829 int ext3x_publishnb(opal_list_t *info,
 830                      opal_pmix_op_cbfunc_t cbfunc, void *cbdata)
 831 {
 832     pmix_status_t ret;
 833     opal_value_t *iptr;
 834     size_t n;
 835     ext3x_opcaddy_t *op;
 836 
 837     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 838                         "PMIx_client publish_nb");
 839 
 840     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 841     if (0 >= opal_pmix_base.initialized) {
 842         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 843         return OPAL_ERR_NOT_INITIALIZED;
 844     }
 845     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 846 
 847     if (NULL == info) {
 848         return OPAL_ERR_BAD_PARAM;
 849     }
 850 
 851     /* create the caddy */
 852     op = OBJ_NEW(ext3x_opcaddy_t);
 853     op->opcbfunc = cbfunc;
 854     op->cbdata = cbdata;
 855 
 856     op->sz = opal_list_get_size(info);
 857     if (0 < op->sz) {
 858         PMIX_INFO_CREATE(op->info, op->sz);
 859         n=0;
 860         OPAL_LIST_FOREACH(iptr, info, opal_value_t) {
 861             (void)opal_string_copy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN);
 862             ext3x_value_load(&op->info[n].value, iptr);
 863             ++n;
 864         }
 865     }
 866 
 867     ret = PMIx_Publish_nb(op->info, op->sz, opcbfunc, op);
 868 
 869     return ext3x_convert_rc(ret);
 870 }
 871 
 872 int ext3x_lookup(opal_list_t *data, opal_list_t *info)
 873 {
 874     opal_pmix_pdata_t *d;
 875     pmix_pdata_t *pdata;
 876     pmix_info_t *pinfo = NULL;
 877     pmix_status_t rc;
 878     size_t cnt, n, sz = 0;
 879     opal_value_t *iptr;
 880     opal_ext3x_jobid_trkr_t *jptr, *job;
 881     int ret;
 882 
 883     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
 884                         "ext3x:client lookup");
 885 
 886     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 887     if (0 >= opal_pmix_base.initialized) {
 888         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 889         return OPAL_ERR_NOT_INITIALIZED;
 890     }
 891     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 892 
 893     if (NULL == data || 0 == (cnt = opal_list_get_size(data))) {
 894         return OPAL_ERR_BAD_PARAM;
 895     }
 896     PMIX_PDATA_CREATE(pdata, cnt);
 897     n = 0;
 898     OPAL_LIST_FOREACH(d, data, opal_pmix_pdata_t) {
 899         (void)opal_string_copy(pdata[n].key, d->value.key, PMIX_MAX_KEYLEN);
 900         ++n;
 901     }
 902 
 903     if (NULL != info && 0 < (sz = opal_list_get_size(info))) {
 904         PMIX_INFO_CREATE(pinfo, sz);
 905         n=0;
 906         OPAL_LIST_FOREACH(iptr, info, opal_value_t) {
 907             (void)opal_string_copy(pinfo[n].key, iptr->key, PMIX_MAX_KEYLEN);
 908             ext3x_value_load(&pinfo[n].value, iptr);
 909             ++n;
 910         }
 911     }
 912 
 913     rc = PMIx_Lookup(pdata, cnt, pinfo, sz);
 914     if (PMIX_SUCCESS == rc) {
 915         /* load the answers back into the list */
 916         n=0;
 917         OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 918         OPAL_LIST_FOREACH(d, data, opal_pmix_pdata_t) {
 919             if (mca_pmix_ext3x_component.native_launch) {
 920                 /* if we were launched by the OMPI RTE, then
 921                  * the jobid is in a special format - so get it */
 922                 opal_convert_string_to_jobid(&d->proc.jobid, pdata[n].proc.nspace);
 923             } else {
 924                 /* we were launched by someone else, so make the
 925                  * jobid just be the hash of the nspace */
 926                 OPAL_HASH_JOBID(pdata[n].proc.nspace, d->proc.jobid);
 927             }
 928             /* if we don't already have it, add this to our jobid tracker */
 929             job = NULL;
 930             OPAL_LIST_FOREACH(jptr, &mca_pmix_ext3x_component.jobids, opal_ext3x_jobid_trkr_t) {
 931                 if (jptr->jobid == d->proc.jobid) {
 932                     job = jptr;
 933                     break;
 934                 }
 935             }
 936             if (NULL == job) {
 937                 job = OBJ_NEW(opal_ext3x_jobid_trkr_t);
 938                 (void)opal_string_copy(job->nspace, pdata[n].proc.nspace, PMIX_MAX_NSLEN);
 939                 job->jobid = d->proc.jobid;
 940                 opal_list_append(&mca_pmix_ext3x_component.jobids, &job->super);
 941             }
 942             d->proc.vpid = ext3x_convert_rank(pdata[n].proc.rank);
 943             if (OPAL_SUCCESS != (ret = ext3x_value_unload(&d->value, &pdata[n].value))) {
 944                 OPAL_ERROR_LOG(ret);
 945             }
 946         }
 947         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
 948     }
 949     PMIX_PDATA_FREE(pdata, cnt);
 950     if (NULL != pinfo) {
 951         PMIX_INFO_FREE(pinfo, sz);
 952     }
 953     return ext3x_convert_rc(rc);
 954 }
 955 
 956 static void lk_cbfunc(pmix_status_t status,
 957                       pmix_pdata_t data[], size_t ndata,
 958                       void *cbdata)
 959 {
 960     ext3x_opcaddy_t *op = (ext3x_opcaddy_t*)cbdata;
 961     opal_pmix_pdata_t *d;
 962     opal_list_t results, *r = NULL;
 963     int rc;
 964     size_t n;
 965     opal_ext3x_jobid_trkr_t *job, *jptr;
 966 
 967     OPAL_ACQUIRE_OBJECT(op);
 968 
 969     if (NULL == op->lkcbfunc) {
 970         OBJ_RELEASE(op);
 971         return;
 972     }
 973 
 974     rc = ext3x_convert_rc(op->status);
 975     if (OPAL_SUCCESS == rc) {
 976         OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
 977         OBJ_CONSTRUCT(&results, opal_list_t);
 978         for (n=0; n < ndata; n++) {
 979             d = OBJ_NEW(opal_pmix_pdata_t);
 980             opal_list_append(&results, &d->super);
 981             if (mca_pmix_ext3x_component.native_launch) {
 982                 /* if we were launched by the OMPI RTE, then
 983                  * the jobid is in a special format - so get it */
 984                 opal_convert_string_to_jobid(&d->proc.jobid, data[n].proc.nspace);
 985             } else {
 986                 /* we were launched by someone else, so make the
 987                  * jobid just be the hash of the nspace */
 988                 OPAL_HASH_JOBID(data[n].proc.nspace, d->proc.jobid);
 989             }
 990             /* if we don't already have it, add this to our jobid tracker */
 991             job = NULL;
 992             OPAL_LIST_FOREACH(jptr, &mca_pmix_ext3x_component.jobids, opal_ext3x_jobid_trkr_t) {
 993                 if (jptr->jobid == d->proc.jobid) {
 994                     job = jptr;
 995                     break;
 996                 }
 997             }
 998             if (NULL == job) {
 999                 job = OBJ_NEW(opal_ext3x_jobid_trkr_t);
1000                 (void)opal_string_copy(job->nspace, data[n].proc.nspace, PMIX_MAX_NSLEN);
1001                 job->jobid = d->proc.jobid;
1002                 opal_list_append(&mca_pmix_ext3x_component.jobids, &job->super);
1003             }
1004             d->proc.vpid = ext3x_convert_rank(data[n].proc.rank);
1005             d->value.key = strdup(data[n].key);
1006             rc = ext3x_value_unload(&d->value, &data[n].value);
1007             if (OPAL_SUCCESS != rc) {
1008                 rc = OPAL_ERR_BAD_PARAM;
1009                 OPAL_ERROR_LOG(rc);
1010                 OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1011                 goto release;
1012             }
1013         }
1014         r = &results;
1015         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1016     }
1017 
1018   release:
1019     /* execute the callback */
1020     op->lkcbfunc(rc, r, op->cbdata);
1021 
1022     if (NULL != r) {
1023         OPAL_LIST_DESTRUCT(&results);
1024     }
1025     OBJ_RELEASE(op);
1026 }
1027 
1028 int ext3x_lookupnb(char **keys, opal_list_t *info,
1029                     opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata)
1030 {
1031     pmix_status_t ret;
1032     ext3x_opcaddy_t *op;
1033     opal_value_t *iptr;
1034     size_t n;
1035 
1036 
1037     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
1038                         "ext3x:client lookup_nb");
1039 
1040     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1041     if (0 >= opal_pmix_base.initialized) {
1042         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1043         return OPAL_ERR_NOT_INITIALIZED;
1044     }
1045     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1046 
1047     /* create the caddy */
1048     op = OBJ_NEW(ext3x_opcaddy_t);
1049     op->lkcbfunc = cbfunc;
1050     op->cbdata = cbdata;
1051 
1052     if (NULL != info && 0 < (op->sz = opal_list_get_size(info))) {
1053         PMIX_INFO_CREATE(op->info, op->sz);
1054         n=0;
1055         OPAL_LIST_FOREACH(iptr, info, opal_value_t) {
1056             (void)opal_string_copy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN);
1057             ext3x_value_load(&op->info[n].value, iptr);
1058             ++n;
1059         }
1060     }
1061     ret = PMIx_Lookup_nb(keys, op->info, op->sz, lk_cbfunc, op);
1062 
1063     return ext3x_convert_rc(ret);
1064 }
1065 
1066 int ext3x_unpublish(char **keys, opal_list_t *info)
1067 {
1068     pmix_status_t ret;
1069     size_t ninfo, n;
1070     pmix_info_t *pinfo;
1071     opal_value_t *iptr;
1072 
1073     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1074     if (0 >= opal_pmix_base.initialized) {
1075         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1076         return OPAL_ERR_NOT_INITIALIZED;
1077     }
1078     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1079 
1080     if (NULL != info && 0 < (ninfo = opal_list_get_size(info))) {
1081         PMIX_INFO_CREATE(pinfo, ninfo);
1082         n=0;
1083         OPAL_LIST_FOREACH(iptr, info, opal_value_t) {
1084             (void)opal_string_copy(pinfo[n].key, iptr->key, PMIX_MAX_KEYLEN);
1085             ext3x_value_load(&pinfo[n].value, iptr);
1086             ++n;
1087         }
1088     } else {
1089         pinfo = NULL;
1090         ninfo = 0;
1091     }
1092 
1093     ret = PMIx_Unpublish(keys, pinfo, ninfo);
1094     PMIX_INFO_FREE(pinfo, ninfo);
1095 
1096     return ext3x_convert_rc(ret);
1097 }
1098 
1099 int ext3x_unpublishnb(char **keys, opal_list_t *info,
1100                        opal_pmix_op_cbfunc_t cbfunc, void *cbdata)
1101 {
1102     pmix_status_t ret;
1103     ext3x_opcaddy_t *op;
1104     opal_value_t *iptr;
1105     size_t n;
1106 
1107     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1108     if (0 >= opal_pmix_base.initialized) {
1109         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1110         return OPAL_ERR_NOT_INITIALIZED;
1111     }
1112     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1113 
1114     /* create the caddy */
1115     op = OBJ_NEW(ext3x_opcaddy_t);
1116     op->opcbfunc = cbfunc;
1117     op->cbdata = cbdata;
1118 
1119     if (NULL != info && 0 < (op->sz = opal_list_get_size(info))) {
1120         PMIX_INFO_CREATE(op->info, op->sz);
1121         n=0;
1122         OPAL_LIST_FOREACH(iptr, info, opal_value_t) {
1123             (void)opal_string_copy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN);
1124             ext3x_value_load(&op->info[n].value, iptr);
1125             ++n;
1126         }
1127     }
1128 
1129     ret = PMIx_Unpublish_nb(keys, op->info, op->sz, opcbfunc, op);
1130 
1131     return ext3x_convert_rc(ret);
1132 }
1133 
1134 int ext3x_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid)
1135 {
1136     pmix_status_t rc;
1137     pmix_info_t *info = NULL;
1138     pmix_app_t *papps;
1139     size_t ninfo = 0, napps, n, m;
1140     opal_value_t *ival;
1141     opal_pmix_app_t *app;
1142     char nspace[PMIX_MAX_NSLEN+1];
1143     opal_ext3x_jobid_trkr_t *job;
1144 
1145     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1146     if (0 >= opal_pmix_base.initialized) {
1147         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1148         return OPAL_ERR_NOT_INITIALIZED;
1149     }
1150     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1151 
1152     *jobid = OPAL_JOBID_INVALID;
1153 
1154     if (NULL != job_info && 0 < (ninfo = opal_list_get_size(job_info))) {
1155         PMIX_INFO_CREATE(info, ninfo);
1156         n=0;
1157         OPAL_LIST_FOREACH(ival, job_info, opal_value_t) {
1158             (void)opal_string_copy(info[n].key, ival->key, PMIX_MAX_KEYLEN);
1159             ext3x_value_load(&info[n].value, ival);
1160             ++n;
1161         }
1162     }
1163 
1164     napps = opal_list_get_size(apps);
1165     PMIX_APP_CREATE(papps, napps);
1166     n=0;
1167     OPAL_LIST_FOREACH(app, apps, opal_pmix_app_t) {
1168         papps[n].cmd = strdup(app->cmd);
1169         if (NULL != app->argv) {
1170             papps[n].argv = opal_argv_copy(app->argv);
1171         }
1172         if (NULL != app->env) {
1173             papps[n].env = opal_argv_copy(app->env);
1174         }
1175         if (NULL != app->cwd) {
1176             papps[n].cwd = strdup(app->cwd);
1177         }
1178         papps[n].maxprocs = app->maxprocs;
1179         if (0 < (papps[n].ninfo = opal_list_get_size(&app->info))) {
1180             PMIX_INFO_CREATE(papps[n].info, papps[n].ninfo);
1181             m=0;
1182             OPAL_LIST_FOREACH(ival, &app->info, opal_value_t) {
1183                 (void)opal_string_copy(papps[n].info[m].key, ival->key, PMIX_MAX_KEYLEN);
1184                 ext3x_value_load(&papps[n].info[m].value, ival);
1185                 ++m;
1186             }
1187         }
1188         ++n;
1189     }
1190 
1191     rc = PMIx_Spawn(info, ninfo, papps, napps, nspace);
1192     if (PMIX_SUCCESS == rc) {
1193         OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1194         if (mca_pmix_ext3x_component.native_launch) {
1195             /* if we were launched by the OMPI RTE, then
1196              * the jobid is in a special format - so get it */
1197             opal_convert_string_to_jobid(jobid, nspace);
1198         } else {
1199             /* we were launched by someone else, so make the
1200              * jobid just be the hash of the nspace */
1201             OPAL_HASH_JOBID(nspace, *jobid);
1202         }
1203         /* add this to our jobid tracker */
1204         job = OBJ_NEW(opal_ext3x_jobid_trkr_t);
1205         (void)opal_string_copy(job->nspace, nspace, PMIX_MAX_NSLEN);
1206         job->jobid = *jobid;
1207         opal_list_append(&mca_pmix_ext3x_component.jobids, &job->super);
1208         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1209     }
1210     return rc;
1211 }
1212 
1213 static void spcbfunc(pmix_status_t status,
1214                      char *nspace, void *cbdata)
1215 {
1216     ext3x_opcaddy_t *op = (ext3x_opcaddy_t*)cbdata;
1217     opal_ext3x_jobid_trkr_t *job;
1218     opal_jobid_t jobid = OPAL_JOBID_INVALID;
1219     int rc;
1220 
1221     OPAL_ACQUIRE_OBJECT(op);
1222 
1223     rc = ext3x_convert_rc(status);
1224     if (PMIX_SUCCESS == status) {
1225         /* this is in the PMIx local thread - need to protect
1226          * the framework-level data */
1227         OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1228         if (mca_pmix_ext3x_component.native_launch) {
1229             /* if we were launched by the OMPI RTE, then
1230              * the jobid is in a special format - so get it */
1231             opal_convert_string_to_jobid(&jobid, nspace);
1232         } else {
1233             /* we were launched by someone else, so make the
1234              * jobid just be the hash of the nspace */
1235             OPAL_HASH_JOBID(nspace, jobid);
1236         }
1237         /* add this to our jobid tracker */
1238         job = OBJ_NEW(opal_ext3x_jobid_trkr_t);
1239         (void)opal_string_copy(job->nspace, nspace, PMIX_MAX_NSLEN);
1240         job->jobid = jobid;
1241         opal_list_append(&mca_pmix_ext3x_component.jobids, &job->super);
1242         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1243     }
1244 
1245     op->spcbfunc(rc, jobid, op->cbdata);
1246     OBJ_RELEASE(op);
1247 }
1248 
1249 int ext3x_spawnnb(opal_list_t *job_info, opal_list_t *apps,
1250                    opal_pmix_spawn_cbfunc_t cbfunc, void *cbdata)
1251 {
1252     pmix_status_t ret;
1253     ext3x_opcaddy_t *op;
1254     size_t n, m;
1255     opal_value_t *info;
1256     opal_pmix_app_t *app;
1257 
1258     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1259     if (0 >= opal_pmix_base.initialized) {
1260         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1261         return OPAL_ERR_NOT_INITIALIZED;
1262     }
1263     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1264 
1265     /* create the caddy */
1266     op = OBJ_NEW(ext3x_opcaddy_t);
1267     op->spcbfunc = cbfunc;
1268     op->cbdata = cbdata;
1269 
1270     if (NULL != job_info && 0 < (op->ninfo = opal_list_get_size(job_info))) {
1271         PMIX_INFO_CREATE(op->info, op->ninfo);
1272         n=0;
1273         OPAL_LIST_FOREACH(info, job_info, opal_value_t) {
1274             (void)opal_string_copy(op->info[n].key, info->key, PMIX_MAX_KEYLEN);
1275             ext3x_value_load(&op->info[n].value, info);
1276             ++n;
1277         }
1278     }
1279 
1280     op->sz = opal_list_get_size(apps);
1281     PMIX_APP_CREATE(op->apps, op->sz);
1282     n=0;
1283     OPAL_LIST_FOREACH(app, apps, opal_pmix_app_t) {
1284         op->apps[n].cmd = strdup(app->cmd);
1285         if (NULL != app->argv) {
1286             op->apps[n].argv = opal_argv_copy(app->argv);
1287         }
1288         if (NULL != app->env) {
1289             op->apps[n].env = opal_argv_copy(app->env);
1290         }
1291         op->apps[n].maxprocs = app->maxprocs;
1292         if (0 < (op->apps[n].ninfo = opal_list_get_size(&app->info))) {
1293             PMIX_INFO_CREATE(op->apps[n].info, op->apps[n].ninfo);
1294             m=0;
1295             OPAL_LIST_FOREACH(info, &app->info, opal_value_t) {
1296                 (void)opal_string_copy(op->apps[n].info[m].key, info->key, PMIX_MAX_KEYLEN);
1297                 ext3x_value_load(&op->apps[n].info[m].value, info);
1298                 ++m;
1299             }
1300         }
1301         ++n;
1302     }
1303 
1304     ret = PMIx_Spawn_nb(op->info, op->ninfo, op->apps, op->sz, spcbfunc, op);
1305 
1306     return ext3x_convert_rc(ret);
1307 }
1308 
1309 int ext3x_connect(opal_list_t *procs)
1310 {
1311     pmix_proc_t *p;
1312     size_t nprocs;
1313     opal_namelist_t *ptr;
1314     pmix_status_t ret;
1315     char *nsptr;
1316     size_t n;
1317 
1318     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
1319                         "ext3x:client connect");
1320 
1321     /* protect against bozo error */
1322     if (NULL == procs || 0 == (nprocs = opal_list_get_size(procs))) {
1323         return OPAL_ERR_BAD_PARAM;
1324     }
1325 
1326     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1327     if (0 >= opal_pmix_base.initialized) {
1328         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1329         return OPAL_ERR_NOT_INITIALIZED;
1330     }
1331 
1332     /* convert the list of procs to an array
1333      * of pmix_proc_t */
1334     PMIX_PROC_CREATE(p, nprocs);
1335     n=0;
1336     OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) {
1337         if (NULL == (nsptr = ext3x_convert_jobid(ptr->name.jobid))) {
1338             PMIX_PROC_FREE(p, nprocs);
1339             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1340             return OPAL_ERR_NOT_FOUND;
1341         }
1342         (void)opal_string_copy(p[n].nspace, nsptr, PMIX_MAX_NSLEN);
1343         p[n].rank = ext3x_convert_opalrank(ptr->name.vpid);
1344         ++n;
1345     }
1346     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1347 
1348     ret = PMIx_Connect(p, nprocs, NULL, 0);
1349     PMIX_PROC_FREE(p, nprocs);
1350 
1351     return ext3x_convert_rc(ret);
1352 }
1353 
1354 int ext3x_connectnb(opal_list_t *procs,
1355                      opal_pmix_op_cbfunc_t cbfunc,
1356                      void *cbdata)
1357 {
1358     ext3x_opcaddy_t *op;
1359     opal_namelist_t *ptr;
1360     pmix_status_t ret;
1361     char *nsptr;
1362     size_t n;
1363 
1364     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
1365                         "ext3x:client connect NB");
1366 
1367     /* protect against bozo error */
1368     if (NULL == procs || 0 == opal_list_get_size(procs)) {
1369         return OPAL_ERR_BAD_PARAM;
1370     }
1371 
1372     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1373     if (0 >= opal_pmix_base.initialized) {
1374         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1375         return OPAL_ERR_NOT_INITIALIZED;
1376     }
1377 
1378     /* create the caddy */
1379     op = OBJ_NEW(ext3x_opcaddy_t);
1380     op->opcbfunc = cbfunc;
1381     op->cbdata = cbdata;
1382     op->nprocs = opal_list_get_size(procs);
1383 
1384     /* convert the list of procs to an array
1385      * of pmix_proc_t */
1386     PMIX_PROC_CREATE(op->procs, op->nprocs);
1387     n=0;
1388     OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) {
1389         if (NULL == (nsptr = ext3x_convert_jobid(ptr->name.jobid))) {
1390             OBJ_RELEASE(op);
1391             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1392             return OPAL_ERR_NOT_FOUND;
1393         }
1394         (void)opal_string_copy(op->procs[n].nspace, nsptr, PMIX_MAX_NSLEN);
1395         op->procs[n].rank = ext3x_convert_opalrank(ptr->name.vpid);
1396         ++n;
1397     }
1398     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1399 
1400     ret = PMIx_Connect_nb(op->procs, op->nprocs, NULL, 0, opcbfunc, op);
1401     if (PMIX_SUCCESS != ret) {
1402         OBJ_RELEASE(op);
1403     }
1404     return ext3x_convert_rc(ret);
1405 }
1406 
1407 int ext3x_disconnect(opal_list_t *procs)
1408 {
1409     size_t nprocs, n;
1410     opal_namelist_t *ptr;
1411     pmix_status_t ret=PMIX_SUCCESS;
1412     pmix_proc_t *p;
1413     char *nsptr;
1414 
1415     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
1416                         "ext3x:client disconnect");
1417 
1418     /* protect against bozo error */
1419     if (NULL == procs || 0 == (nprocs = opal_list_get_size(procs))) {
1420         return OPAL_ERR_BAD_PARAM;
1421     }
1422 
1423     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1424     if (0 >= opal_pmix_base.initialized) {
1425         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1426         return OPAL_ERR_NOT_INITIALIZED;
1427     }
1428 
1429     /* convert the list of procs to an array
1430      * of pmix_proc_t */
1431     PMIX_PROC_CREATE(p, nprocs);
1432     n=0;
1433     OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) {
1434         if (NULL == (nsptr = ext3x_convert_jobid(ptr->name.jobid))) {
1435             PMIX_PROC_FREE(p, nprocs);
1436             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1437             return OPAL_ERR_NOT_FOUND;
1438         }
1439         (void)opal_string_copy(p[n].nspace, nsptr, PMIX_MAX_NSLEN);
1440         p[n].rank = ext3x_convert_opalrank(ptr->name.vpid);
1441         ++n;
1442     }
1443     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1444 
1445     ret = PMIx_Disconnect(p, nprocs, NULL, 0);
1446     PMIX_PROC_FREE(p, nprocs);
1447 
1448     return ext3x_convert_rc(ret);
1449 }
1450 
1451 int ext3x_disconnectnb(opal_list_t *procs,
1452                         opal_pmix_op_cbfunc_t cbfunc,
1453                         void *cbdata)
1454 {
1455     ext3x_opcaddy_t *op;
1456     opal_namelist_t *ptr;
1457     pmix_status_t ret;
1458     char *nsptr;
1459     size_t n;
1460 
1461     opal_output_verbose(1, opal_pmix_base_framework.framework_output,
1462                         "ext3x:client disconnect NB");
1463 
1464     /* protect against bozo error */
1465     if (NULL == procs || 0 == opal_list_get_size(procs)) {
1466         return OPAL_ERR_BAD_PARAM;
1467     }
1468 
1469     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1470     if (0 >= opal_pmix_base.initialized) {
1471         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1472         return OPAL_ERR_NOT_INITIALIZED;
1473     }
1474 
1475     /* create the caddy */
1476     op = OBJ_NEW(ext3x_opcaddy_t);
1477     op->opcbfunc = cbfunc;
1478     op->cbdata = cbdata;
1479     op->nprocs = opal_list_get_size(procs);
1480 
1481     /* convert the list of procs to an array
1482      * of pmix_proc_t */
1483     PMIX_PROC_CREATE(op->procs, op->nprocs);
1484     n=0;
1485     OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) {
1486         if (NULL == (nsptr = ext3x_convert_jobid(ptr->name.jobid))) {
1487             OBJ_RELEASE(op);
1488             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1489             return OPAL_ERR_NOT_FOUND;
1490         }
1491         (void)opal_string_copy(op->procs[n].nspace, nsptr, PMIX_MAX_NSLEN);
1492         op->procs[n].rank = ext3x_convert_opalrank(ptr->name.vpid);
1493         ++n;
1494     }
1495     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1496 
1497     ret = PMIx_Disconnect_nb(op->procs, op->nprocs, NULL, 0, opcbfunc, op);
1498     if (PMIX_SUCCESS != ret) {
1499         OBJ_RELEASE(op);
1500     }
1501     return ext3x_convert_rc(ret);
1502 }
1503 
1504 int ext3x_resolve_peers(const char *nodename,
1505                          opal_jobid_t jobid,
1506                          opal_list_t *procs)
1507 {
1508     pmix_status_t ret;
1509     char *nspace;
1510     pmix_proc_t *array=NULL;
1511     size_t nprocs, n;
1512     opal_namelist_t *nm;
1513     opal_ext3x_jobid_trkr_t *job;
1514 
1515     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1516     if (0 >= opal_pmix_base.initialized) {
1517         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1518         return OPAL_ERR_NOT_INITIALIZED;
1519     }
1520 
1521     if (OPAL_JOBID_WILDCARD != jobid) {
1522         if (NULL == (nspace = ext3x_convert_jobid(jobid))) {
1523             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1524             return OPAL_ERR_NOT_FOUND;
1525         }
1526     } else {
1527         nspace = NULL;
1528     }
1529     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1530 
1531     ret = PMIx_Resolve_peers(nodename, nspace, &array, &nprocs);
1532 
1533     if (NULL != array && 0 < nprocs) {
1534         OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1535         for (n=0; n < nprocs; n++) {
1536             nm = OBJ_NEW(opal_namelist_t);
1537             opal_list_append(procs, &nm->super);
1538             if (mca_pmix_ext3x_component.native_launch) {
1539                 /* if we were launched by the OMPI RTE, then
1540                  * the jobid is in a special format - so get it */
1541                 opal_convert_string_to_jobid(&nm->name.jobid, array[n].nspace);
1542             } else {
1543                 /* we were launched by someone else, so make the
1544                  * jobid just be the hash of the nspace */
1545                 OPAL_HASH_JOBID(array[n].nspace, nm->name.jobid);
1546             }
1547             /* if we don't already have it, add this to our jobid tracker */
1548             if (NULL == ext3x_convert_jobid(nm->name.jobid)) {
1549                 job = OBJ_NEW(opal_ext3x_jobid_trkr_t);
1550                 (void)opal_string_copy(job->nspace, array[n].nspace, PMIX_MAX_NSLEN);
1551                 job->jobid = nm->name.jobid;
1552                 opal_list_append(&mca_pmix_ext3x_component.jobids, &job->super);
1553             }
1554             nm->name.vpid = ext3x_convert_rank(array[n].rank);
1555         }
1556         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1557     }
1558     PMIX_PROC_FREE(array, nprocs);
1559     return ext3x_convert_rc(ret);
1560 }
1561 
1562 int ext3x_resolve_nodes(opal_jobid_t jobid, char **nodelist)
1563 {
1564     pmix_status_t ret;
1565     char *nsptr;
1566 
1567     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1568     if (0 >= opal_pmix_base.initialized) {
1569         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1570         return OPAL_ERR_NOT_INITIALIZED;
1571     }
1572 
1573     if (NULL == (nsptr = ext3x_convert_jobid(jobid))) {
1574         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1575         return OPAL_ERR_NOT_FOUND;
1576     }
1577     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1578 
1579     ret = PMIx_Resolve_nodes(nsptr, nodelist);
1580 
1581     return ext3x_convert_rc(ret);
1582 }
1583 
1584 static void relcbfunc(void *cbdata)
1585 {
1586     ext3x_opcaddy_t *op = (ext3x_opcaddy_t*)cbdata;
1587     OBJ_RELEASE(op);
1588 }
1589 
1590 static void infocbfunc(pmix_status_t status,
1591                        pmix_info_t *info, size_t ninfo,
1592                        void *cbdata,
1593                        pmix_release_cbfunc_t release_fn,
1594                        void *release_cbdata)
1595 {
1596     ext3x_opcaddy_t *op = (ext3x_opcaddy_t*)cbdata;
1597     int rc;
1598 
1599     if (NULL != release_fn) {
1600         release_fn(release_cbdata);
1601     }
1602     rc = ext3x_convert_rc(status);
1603     if (NULL != op->qcbfunc) {
1604         op->qcbfunc(rc, NULL, op->cbdata, relcbfunc, op);
1605     } else {
1606         OBJ_RELEASE(op);
1607     }
1608 }
1609 
1610 int ext3x_allocate(opal_pmix_alloc_directive_t directive,
1611                     opal_list_t *info,
1612                     opal_pmix_info_cbfunc_t cbfunc, void *cbdata)
1613 {
1614     return OPAL_ERR_NOT_SUPPORTED;
1615 }
1616 
1617 int ext3x_job_control(opal_list_t *targets,
1618                        opal_list_t *directives,
1619                        opal_pmix_info_cbfunc_t cbfunc, void *cbdata)
1620 {
1621     ext3x_opcaddy_t *op;
1622     size_t n;
1623     opal_namelist_t *ptr;
1624     opal_value_t *iptr;
1625     pmix_status_t rc;
1626     char *nsptr;
1627 
1628     OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
1629     if (0 >= opal_pmix_base.initialized) {
1630         OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1631         return OPAL_ERR_NOT_INITIALIZED;
1632     }
1633 
1634     /* create the caddy */
1635     op = OBJ_NEW(ext3x_opcaddy_t);
1636     op->qcbfunc = cbfunc;
1637     op->cbdata = cbdata;
1638     if (NULL != targets) {
1639         op->nprocs = opal_list_get_size(targets);
1640 
1641         /* convert the list of procs to an array
1642          * of pmix_proc_t */
1643         PMIX_PROC_CREATE(op->procs, op->nprocs);
1644         n=0;
1645         OPAL_LIST_FOREACH(ptr, targets, opal_namelist_t) {
1646             if (NULL == (nsptr = ext3x_convert_jobid(ptr->name.jobid))) {
1647                 OBJ_RELEASE(op);
1648                 OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1649                 return OPAL_ERR_NOT_FOUND;
1650             }
1651             (void)opal_string_copy(op->procs[n].nspace, nsptr, PMIX_MAX_NSLEN);
1652             op->procs[n].rank = ext3x_convert_opalrank(ptr->name.vpid);
1653             ++n;
1654         }
1655     }
1656     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
1657 
1658     if (NULL != directives && 0 < (op->ninfo = opal_list_get_size(directives))) {
1659         PMIX_INFO_CREATE(op->info, op->ninfo);
1660         n=0;
1661         OPAL_LIST_FOREACH(iptr, directives, opal_value_t) {
1662             (void)opal_string_copy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN);
1663             ext3x_value_load(&op->info[n].value, iptr);
1664             ++n;
1665         }
1666     }
1667 
1668     rc = PMIx_Job_control_nb(op->procs,op->nprocs, op->info, op->ninfo, infocbfunc, op);
1669     if (PMIX_SUCCESS != rc) {
1670         OBJ_RELEASE(op);
1671     }
1672     return ext3x_convert_rc(rc);
1673 }
/* [<][>][^][v][top][bottom][index][help] */
root/opal/mca/pmix/ext3x/ext3x_client.c

DEFINITIONS
