opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
atcon
atdes
rcon
rdes
opa_init
opa_finalize
transports_use_rand
transports_print
allocate
setup_local_network
setup_fork
child_finalized
local_app_finalized
deregister_nspace
collect_inventory
deliver_inventory
   1 /*
   2  * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
   3  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
   4  *
   5  * $COPYRIGHT$
   6  *
   7  * Additional copyrights may follow
   8  *
   9  * $HEADER$
  10  */
  11 
  12 #include <src/include/pmix_config.h>
  13 
  14 #include <string.h>
  15 #ifdef HAVE_UNISTD_H
  16 #include <unistd.h>
  17 #endif
  18 #ifdef HAVE_SYS_TYPES_H
  19 #include <sys/types.h>
  20 #endif
  21 #ifdef HAVE_SYS_STAT_H
  22 #include <sys/stat.h>
  23 #endif
  24 #ifdef HAVE_FCNTL_H
  25 #include <fcntl.h>
  26 #endif
  27 #include <time.h>
  28 
  29 #if PMIX_WANT_OPAMGT
  30 #include <opamgt/opamgt.h>
  31 #include <opamgt/opamgt_sa.h>
  32 #endif
  33 
  34 #include <pmix_common.h>
  35 
  36 #include "src/mca/base/pmix_mca_base_var.h"
  37 #include "src/class/pmix_list.h"
  38 #include "src/include/pmix_socket_errno.h"
  39 #include "src/include/pmix_globals.h"
  40 #include "src/class/pmix_list.h"
  41 #include "src/util/alfg.h"
  42 #include "src/util/argv.h"
  43 #include "src/util/error.h"
  44 #include "src/util/output.h"
  45 #include "src/util/pmix_environ.h"
  46 #include "src/mca/preg/preg.h"
  47 #include "src/hwloc/hwloc-internal.h"
  48 
  49 #include "src/mca/pnet/pnet.h"
  50 #include "src/mca/pnet/base/base.h"
  51 #include "pnet_opa.h"
  52 
  53 static pmix_status_t opa_init(void);
  54 static void opa_finalize(void);
  55 static pmix_status_t allocate(pmix_namespace_t *nptr,
  56                               pmix_info_t info[], size_t ninfo,
  57                               pmix_list_t *ilist);
  58 static pmix_status_t setup_local_network(pmix_namespace_t *nptr,
  59                                          pmix_info_t info[],
  60                                          size_t ninfo);
  61 static pmix_status_t setup_fork(pmix_namespace_t *nptr,
  62                                 const pmix_proc_t *proc,
  63                                 char ***env);
  64 static void child_finalized(pmix_proc_t *peer);
  65 static void local_app_finalized(pmix_namespace_t *nptr);
  66 static void deregister_nspace(pmix_namespace_t *nptr);
  67 static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs,
  68                                        pmix_inventory_cbfunc_t cbfunc, void *cbdata);
  69 static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo,
  70                                        pmix_info_t directives[], size_t ndirs,
  71                                        pmix_op_cbfunc_t cbfunc, void *cbdata);
  72 
  73 pmix_pnet_module_t pmix_opa_module = {
  74     .name = "opa",
  75     .init = opa_init,
  76     .finalize = opa_finalize,
  77     .allocate = allocate,
  78     .setup_local_network = setup_local_network,
  79     .setup_fork = setup_fork,
  80     .child_finalized = child_finalized,
  81     .local_app_finalized = local_app_finalized,
  82     .deregister_nspace = deregister_nspace,
  83     .collect_inventory = collect_inventory,
  84     .deliver_inventory = deliver_inventory
  85 };
  86 
  87 /* local object definitions */
  88 typedef struct {
  89     pmix_list_item_t super;
  90     char *name;
  91     char *value;
  92 } opa_attr_t;
  93 static void atcon(opa_attr_t *p)
  94 {
  95     p->name = NULL;
  96     p->value = NULL;
  97 }
  98 static void atdes(opa_attr_t *p)
  99 {
 100     if (NULL != p->name) {
 101         free(p->name);
 102     }
 103     if (NULL != p->value) {
 104         free(p->value);
 105     }
 106 }
 107 static PMIX_CLASS_INSTANCE(opa_attr_t,
 108                            pmix_list_item_t,
 109                            atcon, atdes);
 110 
 111 typedef struct {
 112     pmix_list_item_t super;
 113     char *device;
 114     pmix_list_t attributes;
 115 } opa_resource_t;
 116 static void rcon(opa_resource_t *p)
 117 {
 118     p->device = NULL;
 119     PMIX_CONSTRUCT(&p->attributes, pmix_list_t);
 120 }
 121 static void rdes(opa_resource_t *p)
 122 {
 123     if (NULL != p->device) {
 124         free(p->device);
 125     }
 126     PMIX_LIST_DESTRUCT(&p->attributes);
 127 }
 128 static PMIX_CLASS_INSTANCE(opa_resource_t,
 129                            pmix_list_item_t,
 130                            rcon, rdes);
 131 
 132 
 133 static pmix_status_t opa_init(void)
 134 {
 135     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 136                         "pnet: opa init");
 137     return PMIX_SUCCESS;
 138 }
 139 
 140 static void opa_finalize(void)
 141 {
 142     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 143                         "pnet: opa finalize");
 144 }
 145 
 146 /* some network transports require a little bit of information to
 147  * "pre-condition" them - i.e., to setup their individual transport
 148  * connections so they can generate their endpoint addresses. This
 149  * function provides a means for doing so. The resulting info is placed
 150  * into the app_context's env array so it will automatically be pushed
 151  * into the environment of every MPI process when launched.
 152  */
 153 
 154 static inline void transports_use_rand(uint64_t* unique_key) {
 155     pmix_rng_buff_t rng;
 156     pmix_srand(&rng,(unsigned int)time(NULL));
 157     unique_key[0] = pmix_rand(&rng);
 158     unique_key[1] = pmix_rand(&rng);
 159 }
 160 
 161 static char* transports_print(uint64_t *unique_key)
 162 {
 163     unsigned int *int_ptr;
 164     size_t i, j, string_key_len, written_len;
 165     char *string_key = NULL, *format = NULL;
 166 
 167     /* string is two 64 bit numbers printed in hex with a dash between
 168      * and zero padding.
 169      */
 170     string_key_len = (sizeof(uint64_t) * 2) * 2 + strlen("-") + 1;
 171     string_key = (char*) malloc(string_key_len);
 172     if (NULL == string_key) {
 173         return NULL;
 174     }
 175 
 176     string_key[0] = '\0';
 177     written_len = 0;
 178 
 179     /* get a format string based on the length of an unsigned int.  We
 180      * want to have zero padding for sizeof(unsigned int) * 2
 181      * characters -- when printing as a hex number, each byte is
 182      * represented by 2 hex characters.  Format will contain something
 183      * that looks like %08lx, where the number 8 might be a different
 184      * number if the system has a different sized long (8 would be for
 185      * sizeof(int) == 4)).
 186      */
 187     if (0 > asprintf(&format, "%%0%dx", (int)(sizeof(unsigned int)) * 2)) {
 188         return NULL;
 189     }
 190 
 191     /* print the first number */
 192     int_ptr = (unsigned int*) &unique_key[0];
 193     for (i = 0 ; i < sizeof(uint64_t) / sizeof(unsigned int) ; ++i) {
 194         if (0 == int_ptr[i]) {
 195             /* inject some energy */
 196             for (j=0; j < sizeof(unsigned int); j++) {
 197                 int_ptr[i] |= j << j;
 198             }
 199         }
 200         snprintf(string_key + written_len,
 201                  string_key_len - written_len,
 202                  format, int_ptr[i]);
 203         written_len = strlen(string_key);
 204     }
 205 
 206     /* print the middle dash */
 207     snprintf(string_key + written_len, string_key_len - written_len, "-");
 208     written_len = strlen(string_key);
 209 
 210     /* print the second number */
 211     int_ptr = (unsigned int*) &unique_key[1];
 212     for (i = 0 ; i < sizeof(uint64_t) / sizeof(unsigned int) ; ++i) {
 213         if (0 == int_ptr[i]) {
 214             /* inject some energy */
 215             for (j=0; j < sizeof(unsigned int); j++) {
 216                 int_ptr[i] |= j << j;
 217             }
 218         }
 219         snprintf(string_key + written_len,
 220                  string_key_len - written_len,
 221                  format, int_ptr[i]);
 222         written_len = strlen(string_key);
 223     }
 224     free(format);
 225 
 226     return string_key;
 227 }
 228 
 229 /* NOTE: if there is any binary data to be transferred, then
 230  * this function MUST pack it for transport as the host will
 231  * not know how to do so */
 232 static pmix_status_t allocate(pmix_namespace_t *nptr,
 233                               pmix_info_t info[], size_t ninfo,
 234                               pmix_list_t *ilist)
 235 {
 236     uint64_t unique_key[2];
 237     char *string_key, *cs_env;
 238     int fd_rand;
 239     size_t bytes_read, n, m, p;
 240     pmix_kval_t *kv;
 241     bool envars = false, seckeys = false, netalloc = false;
 242     pmix_status_t rc;
 243     pmix_proc_t pname;
 244     pmix_coord_t coord;
 245     pmix_buffer_t bucket;
 246     pmix_info_t *iptr;
 247     pmix_pnet_node_t *nd;
 248     pmix_pnet_local_procs_t *lp;
 249 
 250     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 251                         "pnet:opa:allocate for nspace %s", nptr->nspace);
 252 
 253     if (NULL == info) {
 254         return PMIX_ERR_TAKE_NEXT_OPTION;
 255     }
 256 
 257     for (n=0; n < ninfo; n++) {
 258         if (PMIX_CHECK_KEY(&info[n], PMIX_SETUP_APP_ENVARS)) {
 259             envars = PMIX_INFO_TRUE(&info[n]);
 260         } else if (PMIX_CHECK_KEY(&info[n], PMIX_SETUP_APP_ALL)) {
 261             envars = PMIX_INFO_TRUE(&info[n]);
 262             seckeys = PMIX_INFO_TRUE(&info[n]);
 263         } else if (PMIX_CHECK_KEY(&info[n], PMIX_SETUP_APP_NONENVARS)) {
 264             seckeys = PMIX_INFO_TRUE(&info[n]);
 265         } else if (PMIX_CHECK_KEY(&info[n], PMIX_ALLOC_NETWORK)) {
 266             iptr = (pmix_info_t*)info[n].value.data.darray->array;
 267             m = info[n].value.data.darray->size;
 268             for (p=0; p < m; p++) {
 269                 if (PMIX_CHECK_KEY(&iptr[p], PMIX_ALLOC_NETWORK_SEC_KEY)) {
 270                     seckeys = PMIX_INFO_TRUE(&iptr[p]);
 271                 } else if (PMIX_CHECK_KEY(&iptr[p], PMIX_ALLOC_NETWORK_ID)) {
 272                     /* need to track the request by this ID */
 273                 } else if (PMIX_CHECK_KEY(&iptr[p], PMIX_SETUP_APP_ENVARS)) {
 274                     envars = PMIX_INFO_TRUE(&iptr[p]);
 275                 } else if (PMIX_CHECK_KEY(&iptr[p], PMIX_SETUP_APP_ALL)) {
 276                     envars = PMIX_INFO_TRUE(&iptr[p]);
 277                     seckeys = PMIX_INFO_TRUE(&iptr[p]);
 278                 } else if (PMIX_CHECK_KEY(&iptr[p], PMIX_SETUP_APP_NONENVARS)) {
 279                     seckeys = PMIX_INFO_TRUE(&iptr[p]);
 280                 }
 281             }
 282             netalloc = true;
 283         }
 284     }
 285 
 286     if (seckeys) {
 287         pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 288                             "pnet: opa providing seckeys");
 289         /* put the number here - or else create an appropriate string. this just needs to
 290          * eventually be a string variable
 291          */
 292         if(-1 == (fd_rand = open("/dev/urandom", O_RDONLY))) {
 293             transports_use_rand(unique_key);
 294         } else {
 295             bytes_read = read(fd_rand, (char *) unique_key, 16);
 296             if(bytes_read != 16) {
 297                 transports_use_rand(unique_key);
 298             }
 299             close(fd_rand);
 300         }
 301 
 302         if (NULL == (string_key = transports_print(unique_key))) {
 303             PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE);
 304             return PMIX_ERR_OUT_OF_RESOURCE;
 305         }
 306 
 307         if (PMIX_SUCCESS != pmix_mca_base_var_env_name("opa_precondition_transports", &cs_env)) {
 308             PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE);
 309             free(string_key);
 310             return PMIX_ERR_OUT_OF_RESOURCE;
 311         }
 312 
 313         kv = PMIX_NEW(pmix_kval_t);
 314         if (NULL == kv) {
 315             free(string_key);
 316             free(cs_env);
 317             return PMIX_ERR_OUT_OF_RESOURCE;
 318         }
 319         kv->key = strdup(PMIX_SET_ENVAR);
 320         kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
 321         if (NULL == kv->value) {
 322             free(string_key);
 323             free(cs_env);
 324             PMIX_RELEASE(kv);
 325             return PMIX_ERR_OUT_OF_RESOURCE;
 326         }
 327         kv->value->type = PMIX_ENVAR;
 328         PMIX_ENVAR_LOAD(&kv->value->data.envar, cs_env, string_key, ':');
 329         pmix_list_append(ilist, &kv->super);
 330         free(cs_env);
 331         free(string_key);
 332     }
 333 
 334     if (envars) {
 335         pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 336                             "pnet: opa harvesting envars %s excluding %s",
 337                             (NULL == mca_pnet_opa_component.incparms) ? "NONE" : mca_pnet_opa_component.incparms,
 338                             (NULL == mca_pnet_opa_component.excparms) ? "NONE" : mca_pnet_opa_component.excparms);
 339         /* harvest envars to pass along */
 340         if (NULL != mca_pnet_opa_component.include) {
 341             rc = pmix_pnet_base_harvest_envars(mca_pnet_opa_component.include,
 342                                                mca_pnet_opa_component.exclude,
 343                                                ilist);
 344             if (PMIX_SUCCESS != rc) {
 345                 return rc;
 346             }
 347         }
 348     }
 349 
 350     if (netalloc) {
 351         /* assign a simulated coordinate to each process. For now, we
 352          * assume there is one device per node. Thus, the coordinate of
 353          * all procs on a node will be the network coord of the device
 354          * on that node. We'll assign device coordinates with a simple
 355          * round-robin algo */
 356         coord.y = 0;
 357         coord.z = 0;
 358         PMIX_LOAD_NSPACE(pname.nspace, nptr->nspace);
 359         PMIX_CONSTRUCT(&bucket, pmix_buffer_t);
 360         PMIX_LIST_FOREACH(nd, &pmix_pnet_globals.nodes, pmix_pnet_node_t) {
 361             /* find the job on this node */
 362             PMIX_LIST_FOREACH(lp, &nd->local_jobs, pmix_pnet_local_procs_t) {
 363                 if (0 == strcmp(nptr->nspace, lp->nspace)) {
 364                     /* assign the coord for each proc - in our case,
 365                      * we shall assign an x-coord based on local rank
 366                      * and the y-coord will represent the node */
 367                     for (n=0; n < lp->np; n++) {
 368                         coord.x = n;
 369                         pname.rank = lp->ranks[n];
 370                         /* pack this value */
 371                         PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &pname, 1, PMIX_PROC);
 372                         if (PMIX_SUCCESS != rc) {
 373                             PMIX_ERROR_LOG(rc);
 374                             PMIX_DESTRUCT(&bucket);
 375                             return rc;
 376                         }
 377                         PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &coord, 1, PMIX_COORD);
 378                         if (PMIX_SUCCESS != rc) {
 379                             PMIX_ERROR_LOG(rc);
 380                             PMIX_DESTRUCT(&bucket);
 381                             return rc;
 382                         }
 383                     }
 384                     break;
 385                 }
 386             }
 387             coord.y++;
 388         }
 389         /* pass that up */
 390         kv = PMIX_NEW(pmix_kval_t);
 391         if (NULL == kv) {
 392             PMIX_DESTRUCT(&bucket);
 393             return PMIX_ERR_OUT_OF_RESOURCE;
 394         }
 395         kv->key = strdup(PMIX_PNET_OPA_BLOB);
 396         kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
 397         if (NULL == kv->value) {
 398             PMIX_DESTRUCT(&bucket);
 399             PMIX_RELEASE(kv);
 400             return PMIX_ERR_OUT_OF_RESOURCE;
 401         }
 402         kv->value->type = PMIX_BYTE_OBJECT;
 403         /* unload the buffer into a byte object */
 404         PMIX_UNLOAD_BUFFER(&bucket, kv->value->data.bo.bytes, kv->value->data.bo.size);
 405         pmix_list_append(ilist, &kv->super);
 406     }
 407 
 408     /* we don't currently manage OPA resources */
 409     return PMIX_ERR_TAKE_NEXT_OPTION;
 410 }
 411 
 412 static pmix_status_t setup_local_network(pmix_namespace_t *nptr,
 413                                          pmix_info_t info[],
 414                                          size_t ninfo)
 415 {
 416     size_t n;
 417     pmix_kval_t *kv;
 418 
 419 
 420     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 421                         "pnet: opa setup_local_network for nspace %s", nptr->nspace);
 422 
 423     if (NULL != info) {
 424         for (n=0; n < ninfo; n++) {
 425             if (0 == strncmp(info[n].key, PMIX_PNET_OPA_BLOB, PMIX_MAX_KEYLEN)) {
 426                 /* the byte object contains a packed blob that needs to be
 427                  * cached until we determine we have local procs for this
 428                  * nspace */
 429                 kv = PMIX_NEW(pmix_kval_t);
 430                 if (NULL == kv) {
 431                     return PMIX_ERR_NOMEM;
 432                 }
 433                 kv->key = strdup(info[n].key);
 434                 kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
 435                 if (NULL == kv->value) {
 436                     PMIX_RELEASE(kv);
 437                     return PMIX_ERR_NOMEM;
 438                 }
 439                 pmix_value_xfer(kv->value, &info[n].value);
 440                 if (PMIX_ENVAR == kv->value->type) {
 441                     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 442                                         "pnet:opa:setup_local_network adding %s=%s to environment",
 443                                         kv->value->data.envar.envar, kv->value->data.envar.value);
 444                 } else {
 445                     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 446                                         "pnet:opa:setup_local_network loading blob");
 447                 }
 448                 pmix_list_append(&nptr->setup_data, &kv->super);
 449             }
 450         }
 451     }
 452 
 453     return PMIX_SUCCESS;
 454 }
 455 
 456 static pmix_status_t setup_fork(pmix_namespace_t *nptr,
 457                                 const pmix_proc_t *proc,
 458                                 char ***env)
 459 {
 460     pmix_kval_t *kv, *next;
 461     pmix_data_array_t dinfo;
 462     pmix_info_t info[2], stinfo;
 463     int cnt;
 464     pmix_status_t rc;
 465     pmix_buffer_t bkt;
 466     pmix_proc_t pname;
 467     pmix_coord_t coord;
 468 
 469     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 470                         "pnet: opa setup fork for nspace: %s", nptr->nspace);
 471 
 472     /* if there are any cached nspace prep blobs, execute them,
 473      * ensuring that we only do so once per nspace - note that
 474      * we don't expect to find any envars here, though we could
 475      * have included some if we needed to set them per-client */
 476     PMIX_LIST_FOREACH_SAFE(kv, next, &nptr->setup_data, pmix_kval_t) {
 477         if (0 == strcmp(kv->key, PMIX_PNET_OPA_BLOB)) {
 478             pmix_list_remove_item(&nptr->setup_data, &kv->super);
 479             /* setup to unpack the blob */
 480             PMIX_CONSTRUCT(&bkt,pmix_buffer_t);
 481             PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt,
 482                              kv->value->data.bo.bytes,
 483                              kv->value->data.bo.size);
 484             /* there will be an entry for each proc in the nspace */
 485             PMIX_INFO_CONSTRUCT(&stinfo);
 486             PMIX_LOAD_KEY(stinfo.key, PMIX_PROC_DATA);
 487             stinfo.value.type = PMIX_DATA_ARRAY;
 488             stinfo.value.data.darray = &dinfo;
 489             dinfo.type = PMIX_INFO;
 490             dinfo.size = 2;
 491             dinfo.array = info;
 492             /* unpack all the entries */
 493             cnt = 1;
 494             PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
 495                                &bkt, &pname, &cnt, PMIX_PROC);
 496             while (PMIX_SUCCESS == rc) {
 497                 /* unpack the coord of this proc */
 498                 cnt = 1;
 499                 PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
 500                                    &bkt, &coord, &cnt, PMIX_COORD);
 501                 if (PMIX_SUCCESS != rc) {
 502                     PMIX_ERROR_LOG(rc);
 503                     break;
 504                 }
 505                 /* cache the info on the job */
 506                 PMIX_INFO_LOAD(&info[0], PMIX_RANK, &pname.rank, PMIX_PROC_RANK);
 507                 PMIX_INFO_LOAD(&info[1], PMIX_NETWORK_COORDINATE, &coord, PMIX_COORD);
 508                 PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr,
 509                                         &stinfo, 1);
 510                 /* unpack next entry */
 511                 cnt = 1;
 512                 PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
 513                                    &bkt, &pname, &cnt, PMIX_PROC);
 514             }
 515             PMIX_RELEASE(kv);
 516             break;
 517         }
 518     }
 519     return PMIX_SUCCESS;
 520 }
 521 
 522 static void child_finalized(pmix_proc_t *peer)
 523 {
 524     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 525                         "pnet:opa child finalized");
 526 }
 527 
 528 static void local_app_finalized(pmix_namespace_t *nptr)
 529 {
 530     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 531                         "pnet:opa app finalized");
 532 
 533 }
 534 
 535 static void deregister_nspace(pmix_namespace_t *nptr)
 536 {
 537     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 538                         "pnet:opa deregister nspace");
 539 
 540 }
 541 
 542 static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs,
 543                                        pmix_inventory_cbfunc_t cbfunc, void *cbdata)
 544 {
 545     pmix_inventory_rollup_t *cd = (pmix_inventory_rollup_t*)cbdata;
 546 #if PMIX_HAVE_HWLOC
 547     hwloc_obj_t obj;
 548 #endif
 549     unsigned n;
 550     pmix_status_t rc;
 551     pmix_kval_t *kv;
 552     pmix_buffer_t bucket, pbkt;
 553     bool found = false;
 554     pmix_byte_object_t pbo;
 555     char nodename[PMIX_MAXHOSTNAMELEN], *foo;
 556 
 557     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 558                         "pnet:opa collect inventory");
 559 
 560     /* setup the bucket - we will pass the results as a blob */
 561     PMIX_CONSTRUCT(&bucket, pmix_buffer_t);
 562     /* pack our node name */
 563     gethostname(nodename, sizeof(nodename));
 564     foo = &nodename[0];
 565     PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &foo, 1, PMIX_STRING);
 566     if (PMIX_SUCCESS != rc) {
 567         PMIX_ERROR_LOG(rc);
 568         PMIX_DESTRUCT(&bucket);
 569         return rc;
 570     }
 571 
 572 #if PMIX_HAVE_HWLOC
 573     if (NULL == pmix_hwloc_topology) {
 574         goto query;
 575     }
 576 
 577     /* search the topology for OPA devices */
 578     obj = hwloc_get_next_osdev(pmix_hwloc_topology, NULL);
 579     while (NULL != obj) {
 580         if (obj->attr->osdev.type != HWLOC_OBJ_OSDEV_OPENFABRICS ||
 581             0 != strncmp(obj->name, "hfi", 3)) {
 582             obj = hwloc_get_next_osdev(pmix_hwloc_topology, obj);
 583             continue;
 584         }
 585         found = true;
 586         if (9 < pmix_output_get_verbosity(pmix_pnet_base_framework.framework_output)) {
 587             /* dump the discovered node resources */
 588             pmix_output(0, "OPA resource discovered on node: %s", nodename);
 589             pmix_output(0, "\tDevice name: %s", obj->name);
 590             for (n=0; n < obj->infos_count; n++) {
 591                 pmix_output(0, "\t\t%s: %s", obj->infos[n].name, obj->infos[n].value);
 592             }
 593         }
 594         /* pack the name of the device */
 595         PMIX_CONSTRUCT(&pbkt, pmix_buffer_t);
 596         PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->name, 1, PMIX_STRING);
 597         if (PMIX_SUCCESS != rc) {
 598             PMIX_ERROR_LOG(rc);
 599             PMIX_DESTRUCT(&pbkt);
 600             PMIX_DESTRUCT(&bucket);
 601             return rc;
 602         }
 603         /* pack the number of attributes */
 604         PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->infos_count, 1, PMIX_UINT);
 605         if (PMIX_SUCCESS != rc) {
 606             PMIX_ERROR_LOG(rc);
 607             PMIX_DESTRUCT(&pbkt);
 608             PMIX_DESTRUCT(&bucket);
 609             return rc;
 610         }
 611         /* pack each descriptive object */
 612         for (n=0; n < obj->infos_count; n++) {
 613             PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->infos[n].name, 1, PMIX_STRING);
 614             if (PMIX_SUCCESS != rc) {
 615                 PMIX_ERROR_LOG(rc);
 616                 PMIX_DESTRUCT(&pbkt);
 617                 PMIX_DESTRUCT(&bucket);
 618                 return rc;
 619             }
 620             PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->infos[n].value, 1, PMIX_STRING);
 621             if (PMIX_SUCCESS != rc) {
 622                 PMIX_ERROR_LOG(rc);
 623                 PMIX_DESTRUCT(&pbkt);
 624                 PMIX_DESTRUCT(&bucket);
 625                 return rc;
 626             }
 627         }
 628         /* extract the resulting blob - this is a device unit */
 629         PMIX_UNLOAD_BUFFER(&pbkt, pbo.bytes, pbo.size);
 630         /* now load that into the blob */
 631         PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &pbo, 1, PMIX_BYTE_OBJECT);
 632         if (PMIX_SUCCESS != rc) {
 633             PMIX_ERROR_LOG(rc);
 634             PMIX_BYTE_OBJECT_DESTRUCT(&pbo);
 635             PMIX_DESTRUCT(&bucket);
 636             return rc;
 637         }
 638         obj = hwloc_get_next_osdev(pmix_hwloc_topology, obj);
 639     }
 640 
 641   query:
 642 #if 0
 643 #if PMIX_WANT_OPAMGT
 644     if (PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) {
 645         /* collect the switch information from the FM */
 646         OMGT_STATUS_T status = OMGT_STATUS_SUCCESS;
 647         struct omgt_port * port = NULL;
 648         omgt_sa_selector_t selector;
 649 
 650         /* create a session */
 651         status = omgt_open_port_by_num(&port, 1 /* hfi */, 1 /* port */, NULL);
 652         if (OMGT_STATUS_SUCCESS != status) {
 653             pmix_output_verbose(1, pmix_pnet_base_framework.framework_output,
 654                                 "Unable to open port to FM");
 655             goto complete;
 656         }
 657         /* specify how and what we want to query by */
 658         selector.InputType = InputTypeLid;
 659         selector.InputValue.PortInfoRecord.Lid = 1;
 660 
 661     }
 662 #endif
 663 #endif
 664     /* if we found any devices, then return the blob */
 665     if (!found) {
 666         PMIX_DESTRUCT(&bucket);
 667         return PMIX_ERR_TAKE_NEXT_OPTION;
 668     }
 669 
 670     /* extract the resulting blob */
 671     PMIX_UNLOAD_BUFFER(&bucket, pbo.bytes, pbo.size);
 672     kv = PMIX_NEW(pmix_kval_t);
 673     kv->key = strdup(PMIX_OPA_INVENTORY_KEY);
 674     PMIX_VALUE_CREATE(kv->value, 1);
 675     pmix_value_load(kv->value, &pbo, PMIX_BYTE_OBJECT);
 676     PMIX_BYTE_OBJECT_DESTRUCT(&pbo);
 677     pmix_list_append(&cd->payload, &kv->super);
 678 
 679 #else  // have_hwloc
 680 #if 0
 681 #if PMIX_WANT_OPAMGT
 682     if (PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) {
 683         /* query the FM for the inventory */
 684     }
 685 
 686   complete:
 687     /* if we found any devices, then return the blob */
 688     if (!found) {
 689         PMIX_DESTRUCT(&bucket);
 690         return PMIX_ERR_TAKE_NEXT_OPTION;
 691     }
 692 
 693     /* extract the resulting blob */
 694     PMIX_UNLOAD_BUFFER(&bucket, pbo.bytes, pbo.size);
 695     kv = PMIX_NEW(pmix_kval_t);
 696     kv->key = strdup(PMIX_OPA_INVENTORY_KEY);
 697     PMIX_VALUE_CREATE(kv->value, 1);
 698     pmix_value_load(kv->value, &pbo, PMIX_BYTE_OBJECT);
 699     PMIX_BYTE_OBJECT_DESTRUCT(&pbo);
 700     pmix_list_append(&cd->payload, &kv->super);
 701 
 702 #endif
 703 #endif
 704     return PMIX_ERR_TAKE_NEXT_OPTION;
 705 #endif  // have_hwloc
 706 
 707     return PMIX_SUCCESS;
 708 }
 709 
 710 static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo,
 711                                        pmix_info_t directives[], size_t ndirs,
 712                                        pmix_op_cbfunc_t cbfunc, void *cbdata)
 713 {
 714     pmix_buffer_t bkt, pbkt;
 715     size_t n;
 716     int32_t cnt;
 717     unsigned m, nattrs;
 718     char *hostname;
 719     pmix_byte_object_t pbo;
 720     pmix_pnet_node_t *nd, *ndptr;
 721     pmix_pnet_resource_t *lt, *lst;
 722     opa_attr_t *attr;
 723     opa_resource_t *res;
 724     pmix_status_t rc;
 725 
 726     pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
 727                         "pnet:opa deliver inventory");
 728 
 729     for (n=0; n < ninfo; n++) {
 730         if (0 == strncmp(info[n].key, PMIX_OPA_INVENTORY_KEY, PMIX_MAX_KEYLEN)) {
 731             /* this is our inventory in the form of a blob */
 732             PMIX_CONSTRUCT(&bkt,pmix_buffer_t);
 733             PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt,
 734                              info[n].value.data.bo.bytes,
 735                              info[n].value.data.bo.size);
 736             /* first is the host this came from */
 737             cnt = 1;
 738             PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
 739                                &bkt, &hostname, &cnt, PMIX_STRING);
 740             if (PMIX_SUCCESS != rc) {
 741                 PMIX_ERROR_LOG(rc);
 742                 /* must _not_ destruct bkt as we don't
 743                  * own the bytes! */
 744                 return rc;
 745             }
 746             /* do we already have this node? */
 747             nd = NULL;
 748             PMIX_LIST_FOREACH(ndptr, &pmix_pnet_globals.nodes, pmix_pnet_node_t) {
 749                 if (0 == strcmp(hostname, ndptr->name)) {
 750                     nd = ndptr;
 751                     break;
 752                 }
 753             }
 754             if (NULL == nd) {
 755                 nd = PMIX_NEW(pmix_pnet_node_t);
 756                 nd->name = strdup(hostname);
 757                 pmix_list_append(&pmix_pnet_globals.nodes, &nd->super);
 758             }
 759             /* does this node already have an OPA entry? */
 760             lst = NULL;
 761             PMIX_LIST_FOREACH(lt, &nd->resources, pmix_pnet_resource_t) {
 762                 if (0 == strcmp(lt->name, "opa")) {
 763                     lst = lt;
 764                     break;
 765                 }
 766             }
 767             if (NULL == lst) {
 768                 lst = PMIX_NEW(pmix_pnet_resource_t);
 769                 lst->name = strdup("opa");
 770                 pmix_list_append(&nd->resources, &lst->super);
 771             }
 772             /* each device was packed as a "blob" */
 773             cnt = 1;
 774             PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
 775                                &bkt, &pbo, &cnt, PMIX_BYTE_OBJECT);
 776             while (PMIX_SUCCESS == rc) {
 777                 /* load the blob for unpacking */
 778                 PMIX_CONSTRUCT(&pbkt, pmix_buffer_t);
 779                 PMIX_LOAD_BUFFER(pmix_globals.mypeer, &pbkt,
 780                                  pbo.bytes, pbo.size);
 781 
 782                 res = PMIX_NEW(opa_resource_t);
 783                 /* starts with the name of the device */
 784                 cnt = 1;
 785                 PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
 786                                    &pbkt, &res->device, &cnt, PMIX_STRING);
 787                 if (PMIX_SUCCESS != rc) {
 788                     PMIX_ERROR_LOG(rc);
 789                     PMIX_DESTRUCT(&pbkt);
 790                     PMIX_RELEASE(res);
 791                     return rc;
 792                 }
 793                 /* next comes the numbers of attributes for that device */
 794                 cnt = 1;
 795                 PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
 796                                    &pbkt, &nattrs, &cnt, PMIX_UINT);
 797                 if (PMIX_SUCCESS != rc) {
 798                     PMIX_ERROR_LOG(rc);
 799                     PMIX_DESTRUCT(&pbkt);
 800                     PMIX_RELEASE(res);
 801                     return rc;
 802                 }
 803                 for (m=0; m < nattrs; m++) {
 804                     attr = PMIX_NEW(opa_attr_t);
 805                     /* unpack the name of the attribute */
 806                     cnt = 1;
 807                     PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
 808                                        &pbkt, &attr->name, &cnt, PMIX_STRING);
 809                     if (PMIX_SUCCESS != rc) {
 810                         PMIX_ERROR_LOG(rc);
 811                         PMIX_DESTRUCT(&pbkt);
 812                         PMIX_RELEASE(attr);
 813                         PMIX_RELEASE(res);
 814                         return rc;
 815                     }
 816                     /* unpack the attribute value */
 817                     cnt = 1;
 818                     PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
 819                                        &pbkt, &attr->value, &cnt, PMIX_STRING);
 820                     if (PMIX_SUCCESS != rc) {
 821                         PMIX_ERROR_LOG(rc);
 822                         PMIX_DESTRUCT(&pbkt);
 823                         PMIX_RELEASE(attr);
 824                         PMIX_RELEASE(res);
 825                         return rc;
 826                     }
 827                     pmix_list_append(&res->attributes, &attr->super);
 828                 }
 829                 pmix_list_append(&lst->resources, &res->super);
 830                 PMIX_DESTRUCT(&pbkt);
 831 
 832                 /* get the next device unit */
 833                 cnt = 1;
 834                 PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
 835                                    &bkt, &pbo, &cnt, PMIX_BYTE_OBJECT);
 836             }
 837             if (5 < pmix_output_get_verbosity(pmix_pnet_base_framework.framework_output)) {
 838                 /* dump the resulting node resources */
 839                 pmix_output(0, "OPA resources for node: %s", nd->name);
 840                 PMIX_LIST_FOREACH(lt, &nd->resources, pmix_pnet_resource_t) {
 841                     if (0 == strcmp(lt->name, "opa")) {
 842                         PMIX_LIST_FOREACH(res, &lt->resources, opa_resource_t) {
 843                             pmix_output(0, "\tDevice: %s", res->device);
 844                             PMIX_LIST_FOREACH(attr, &res->attributes, opa_attr_t) {
 845                                 pmix_output(0, "\t\t%s: %s", attr->name, attr->value);
 846                             }
 847                         }
 848                     }
 849                 }
 850             }
 851         }
 852     }
 853 
 854     return PMIX_SUCCESS;
 855 }
/* [<][>][^][v][top][bottom][index][help] */
root/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet_opa.c

DEFINITIONS
