root/opal/mca/pmix/pmix4x/pmix/src/include/pmix_globals.h

INCLUDED FROM

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2005 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
  14  * Copyright (c) 2019      Research Organization for Information Science
  15  *                         and Technology (RIST).  All rights reserved.
  16  * Copyright (c) 2019      Mellanox Technologies, Inc.
  17  *                         All rights reserved.
  18  * $COPYRIGHT$
  19  *
  20  * Additional copyrights may follow
  21  *
  22  * $HEADER$
  23  */
  24 
  25 #ifndef PMIX_GLOBALS_H
  26 #define PMIX_GLOBALS_H
  27 
  28 #include <src/include/pmix_config.h>
  29 
  30 #include <pmix_common.h>
  31 
  32 #include <src/include/types.h>
  33 
  34 #include <unistd.h>
  35 #ifdef HAVE_SYS_TYPES_H
  36 #include <sys/types.h>
  37 #endif
  38 #include PMIX_EVENT_HEADER
  39 
  40 #include <pmix.h>
  41 #include <pmix_common.h>
  42 
  43 #include "src/class/pmix_hash_table.h"
  44 #include "src/class/pmix_list.h"
  45 #include "src/class/pmix_hotel.h"
  46 #include "src/event/pmix_event.h"
  47 #include "src/threads/threads.h"
  48 
  49 #include "src/mca/bfrops/bfrops.h"
  50 #include "src/mca/gds/gds.h"
  51 #include "src/mca/psec/psec.h"
  52 #include "src/mca/ptl/ptl.h"
  53 
  54 BEGIN_C_DECLS
  55 
  56 /* some limits */
  57 #define PMIX_MAX_CRED_SIZE      131072              // set max at 128kbytes
  58 #define PMIX_MAX_ERR_CONSTANT   INT_MIN
  59 
  60 /* internal-only attributes */
  61 #define PMIX_BFROPS_MODULE                  "pmix.bfrops.mod"       // (char*) name of bfrops plugin in-use by a given nspace
  62 #define PMIX_PNET_SETUP_APP                 "pmix.pnet.setapp"      // (pmix_byte_object_t) blob containing info to be given to
  63                                                                     //      pnet framework on remote nodes
  64 
  65 #define PMIX_INFO_OP_COMPLETE    0x80000000
  66 #define PMIX_INFO_OP_COMPLETED(m)            \
  67     ((pmix_info_t*)(m))->flags |= PMIX_INFO_OP_COMPLETE
  68 #define PMIX_INFO_OP_IS_COMPLETE(m)          \
  69     ((m)->flags & PMIX_INFO_OP_COMPLETE)
  70 
  71 /* define an internal-only process name that has
  72  * a dynamically-sized nspace field to save memory */
  73 typedef struct {
  74     char *nspace;
  75     pmix_rank_t rank;
  76 } pmix_name_t;
  77 
  78 /* define an internal-only object for creating
  79  * lists of names */
  80 typedef struct {
  81     pmix_list_item_t super;
  82     pmix_name_t *pname;
  83 } pmix_namelist_t;
  84 PMIX_CLASS_DECLARATION(pmix_namelist_t);
  85 
  86 /* define a command type for communicating to the
  87  * pmix server */
  88 typedef uint8_t pmix_cmd_t;
  89 
  90 /* define some commands */
  91 #define PMIX_REQ_CMD                 0
  92 #define PMIX_ABORT_CMD               1
  93 #define PMIX_COMMIT_CMD              2
  94 #define PMIX_FENCENB_CMD             3
  95 #define PMIX_GETNB_CMD               4
  96 #define PMIX_FINALIZE_CMD            5
  97 #define PMIX_PUBLISHNB_CMD           6
  98 #define PMIX_LOOKUPNB_CMD            7
  99 #define PMIX_UNPUBLISHNB_CMD         8
 100 #define PMIX_SPAWNNB_CMD             9
 101 #define PMIX_CONNECTNB_CMD          10
 102 #define PMIX_DISCONNECTNB_CMD       11
 103 #define PMIX_NOTIFY_CMD             12
 104 #define PMIX_REGEVENTS_CMD          13
 105 #define PMIX_DEREGEVENTS_CMD        14
 106 #define PMIX_QUERY_CMD              15
 107 #define PMIX_LOG_CMD                16
 108 #define PMIX_ALLOC_CMD              17
 109 #define PMIX_JOB_CONTROL_CMD        18
 110 #define PMIX_MONITOR_CMD            19
 111 #define PMIX_GET_CREDENTIAL_CMD     20
 112 #define PMIX_VALIDATE_CRED_CMD      21
 113 #define PMIX_IOF_PULL_CMD           22
 114 #define PMIX_IOF_PUSH_CMD           23
 115 #define PMIX_GROUP_CONSTRUCT_CMD    24
 116 #define PMIX_GROUP_JOIN_CMD         25
 117 #define PMIX_GROUP_INVITE_CMD       26
 118 #define PMIX_GROUP_LEAVE_CMD        27
 119 #define PMIX_GROUP_DESTRUCT_CMD     28
 120 
 121 /* provide a "pretty-print" function for cmds */
 122 const char* pmix_command_string(pmix_cmd_t cmd);
 123 
 124 /* provide a hook to init tool data */
 125 PMIX_EXPORT extern pmix_status_t pmix_tool_init_info(void);
 126 
 127 /* define a set of flags to direct collection
 128  * of data during operations */
 129 typedef enum {
 130     PMIX_COLLECT_INVALID = -1,
 131     PMIX_COLLECT_NO,
 132     PMIX_COLLECT_YES,
 133     PMIX_COLLECT_MAX
 134 } pmix_collect_t;
 135 
 136 /****    PEER STRUCTURES    ****/
 137 
 138 /* clients can only talk to their server, and servers are
 139  * assumed to all have the same personality. Thus, each
 140  * process only needs to track a single set of personality
 141  * modules. All interactions between a client and its local
 142  * server, or between servers, are done thru these modules */
 143 typedef struct pmix_personality_t {
 144     pmix_bfrop_buffer_type_t type;
 145     pmix_bfrops_module_t *bfrops;
 146     pmix_psec_module_t *psec;
 147     pmix_ptl_module_t *ptl;
 148     pmix_gds_base_module_t *gds;
 149 } pmix_personality_t;
 150 
 151 /* define a set of structs for tracking post-termination cleanup */
 152 typedef struct pmix_epilog_t {
 153     uid_t uid;
 154     gid_t gid;
 155     pmix_list_t cleanup_dirs;
 156     pmix_list_t cleanup_files;
 157     pmix_list_t ignores;
 158 } pmix_epilog_t;
 159 
 160 typedef struct {
 161     pmix_list_item_t super;
 162     char *path;
 163 } pmix_cleanup_file_t;
 164 PMIX_CLASS_DECLARATION(pmix_cleanup_file_t);
 165 
 166 typedef struct {
 167     pmix_list_item_t super;
 168     char *path;
 169     bool recurse;
 170     bool leave_topdir;
 171 } pmix_cleanup_dir_t;
 172 PMIX_CLASS_DECLARATION(pmix_cleanup_dir_t);
 173 
 174 /* objects used by servers for tracking active nspaces */
 175 typedef struct {
 176     pmix_list_item_t super;
 177     char *nspace;
 178     pmix_rank_t nprocs;          // num procs in this nspace
 179     size_t nlocalprocs;
 180     bool all_registered;         // all local ranks have been defined
 181     bool version_stored;         // the version string used by this nspace has been stored
 182     pmix_buffer_t *jobbkt;       // packed version of jobinfo
 183     size_t ndelivered;           // count of #local clients that have received the jobinfo
 184     size_t nfinalized;           // count of #local clients that have finalized
 185     pmix_list_t ranks;           // list of pmix_rank_info_t for connection support of my clients
 186     /* all members of an nspace are required to have the
 187      * same personality, but it can differ between nspaces.
 188      * Since servers may support clients from multiple nspaces,
 189      * track their respective compatibility modules here */
 190     pmix_personality_t compat;
 191     pmix_epilog_t epilog;       // things to do upon termination of all local clients
 192                                 // from this nspace
 193     pmix_list_t setup_data;     // list of pmix_kval_t containing info structs having blobs
 194                                 // for setting up the local node for this nspace/application
 195 } pmix_namespace_t;
 196 PMIX_CLASS_DECLARATION(pmix_namespace_t);
 197 
 198 /* define a caddy for quickly creating a list of pmix_namespace_t
 199  * objects for local, dedicated purposes */
 200 typedef struct {
 201     pmix_list_item_t super;
 202     pmix_namespace_t *ns;
 203 } pmix_nspace_caddy_t;
 204 PMIX_CLASS_DECLARATION(pmix_nspace_caddy_t);
 205 
 206 typedef struct pmix_rank_info_t {
 207     pmix_list_item_t super;
 208     int peerid;                 // peer object index into the local clients array on the server
 209     pmix_name_t pname;
 210     uid_t uid;
 211     gid_t gid;
 212     bool modex_recvd;
 213     int proc_cnt;              // #clones of this rank we know about
 214     void *server_object;       // pointer to rank-specific object provided by server
 215 } pmix_rank_info_t;
 216 PMIX_CLASS_DECLARATION(pmix_rank_info_t);
 217 
 218 
 219 /* define a very simple caddy for dealing with pmix_info_t
 220  * objects when transferring portions of arrays */
 221 typedef struct {
 222     pmix_list_item_t super;
 223     pmix_info_t *info;
 224     size_t ninfo;
 225 } pmix_info_caddy_t;
 226 PMIX_CLASS_DECLARATION(pmix_info_caddy_t);
 227 
 228 typedef struct {
 229     pmix_list_item_t super;
 230     pmix_info_t info;
 231 } pmix_infolist_t;
 232 PMIX_CLASS_DECLARATION(pmix_infolist_t);
 233 
 234 /* object for tracking peers - each peer can have multiple
 235  * connections. This can occur if the initial app executes
 236  * a fork/exec, and the child initiates its own connection
 237  * back to the PMIx server. Thus, the trackers should be "indexed"
 238  * by the socket, not the process nspace/rank */
 239 typedef struct pmix_peer_t {
 240     pmix_object_t super;
 241     pmix_namespace_t *nptr;            // point to the nspace object for this process
 242     pmix_rank_info_t *info;
 243     pmix_proc_type_t proc_type;
 244     pmix_listener_protocol_t protocol;
 245     int proc_cnt;
 246     int index;                      // index into the local clients array on the server
 247     int sd;
 248     bool finalized;                 // peer has called finalize
 249     pmix_event_t send_event;        /**< registration with event thread for send events */
 250     bool send_ev_active;
 251     pmix_event_t recv_event;        /**< registration with event thread for recv events */
 252     bool recv_ev_active;
 253     pmix_list_t send_queue;         /**< list of messages to send */
 254     pmix_ptl_send_t *send_msg;      /**< current send in progress */
 255     pmix_ptl_recv_t *recv_msg;      /**< current recv in progress */
 256     int commit_cnt;
 257     pmix_epilog_t epilog;           /**< things to be performed upon
 258                                          termination of this peer */
 259 } pmix_peer_t;
 260 PMIX_CLASS_DECLARATION(pmix_peer_t);
 261 
 262 
 263 /* tracker for IOF requests */
 264 typedef struct {
 265     pmix_list_item_t super;
 266     pmix_peer_t *peer;
 267     pmix_name_t pname;
 268     pmix_iof_channel_t channels;
 269     pmix_iof_cbfunc_t cbfunc;
 270 } pmix_iof_req_t;
 271 PMIX_CLASS_DECLARATION(pmix_iof_req_t);
 272 
 273 
 274 /* caddy for query requests */
 275 typedef struct {
 276     pmix_object_t super;
 277     pmix_event_t ev;
 278     pmix_lock_t lock;
 279     pmix_status_t status;
 280     pmix_query_t *queries;
 281     size_t nqueries;
 282     pmix_proc_t *targets;
 283     size_t ntargets;
 284     pmix_info_t *info;
 285     size_t ninfo;
 286     pmix_list_t results;
 287     pmix_byte_object_t bo;
 288     pmix_info_cbfunc_t cbfunc;
 289     pmix_value_cbfunc_t valcbfunc;
 290     pmix_release_cbfunc_t relcbfunc;
 291     pmix_credential_cbfunc_t credcbfunc;
 292     pmix_validation_cbfunc_t validcbfunc;
 293     void *cbdata;
 294 } pmix_query_caddy_t;
 295 PMIX_CLASS_DECLARATION(pmix_query_caddy_t);
 296 
 297 /* define a tracker for collective operations
 298  * - instanced in pmix_server_ops.c */
 299 typedef struct {
 300     pmix_list_item_t super;
 301     pmix_event_t ev;
 302     bool event_active;
 303     bool host_called;               // tracker has been passed up to host
 304     bool local;                     // operation is strictly local
 305     char *id;                       // string identifier for the collective
 306     pmix_cmd_t type;
 307     pmix_proc_t pname;
 308     bool hybrid;                    // true if participating procs are from more than one nspace
 309     pmix_proc_t *pcs;               // copy of the original array of participants
 310     size_t   npcs;                  // number of procs in the array
 311     pmix_list_t nslist;             // unique nspace list of participants
 312     pmix_lock_t lock;               // flag for waiting for completion
 313     bool def_complete;              // all local procs have been registered and the trk definition is complete
 314     pmix_list_t local_cbs;          // list of pmix_server_caddy_t for sending result to the local participants
 315                                     //    Note: there may be multiple entries for a given proc if that proc
 316                                     //    has fork/exec'd clones that are also participating
 317     uint32_t nlocal;                // number of local participants
 318     uint32_t local_cnt;             // number of local participants who have contributed
 319     pmix_info_t *info;              // array of info structs
 320     size_t ninfo;                   // number of info structs in array
 321     pmix_collect_t collect_type;    // whether or not data is to be returned at completion
 322     pmix_modex_cbfunc_t modexcbfunc;
 323     pmix_op_cbfunc_t op_cbfunc;
 324     void *cbdata;
 325 } pmix_server_trkr_t;
 326 PMIX_CLASS_DECLARATION(pmix_server_trkr_t);
 327 
 328 /* define an object for moving a send
 329  * request into the server's event base and
 330  * dealing with some request timeouts
 331  * - instanced in pmix_server_ops.c */
 332 typedef struct {
 333     pmix_list_item_t super;
 334     pmix_event_t ev;
 335     bool event_active;
 336     pmix_server_trkr_t *trk;
 337     pmix_ptl_hdr_t hdr;
 338     pmix_peer_t *peer;
 339 } pmix_server_caddy_t;
 340 PMIX_CLASS_DECLARATION(pmix_server_caddy_t);
 341 
 342 /****    THREAD-RELATED    ****/
 343  /* define a caddy for thread-shifting operations */
 344  typedef struct {
 345     pmix_object_t super;
 346     pmix_event_t ev;
 347     pmix_lock_t lock;
 348     pmix_status_t status;
 349     pmix_status_t *codes;
 350     size_t ncodes;
 351     pmix_name_t pname;
 352     const char *data;
 353     size_t ndata;
 354     const char *key;
 355     pmix_info_t *info;
 356     size_t ninfo;
 357     pmix_info_t *directives;
 358     size_t ndirs;
 359     pmix_notification_fn_t evhdlr;
 360     pmix_iof_req_t *iofreq;
 361     pmix_kval_t *kv;
 362     pmix_value_t *vptr;
 363     pmix_server_caddy_t *cd;
 364     pmix_server_trkr_t *tracker;
 365     bool enviro;
 366     union {
 367        pmix_release_cbfunc_t relfn;
 368        pmix_hdlr_reg_cbfunc_t hdlrregcbfn;
 369        pmix_op_cbfunc_t opcbfn;
 370        pmix_modex_cbfunc_t modexcbfunc;
 371     } cbfunc;
 372     void *cbdata;
 373     size_t ref;
 374  } pmix_shift_caddy_t;
 375 PMIX_CLASS_DECLARATION(pmix_shift_caddy_t);
 376 
 377 /* struct for tracking ops */
 378 typedef struct {
 379     pmix_list_item_t super;
 380     pmix_event_t ev;
 381     pmix_lock_t lock;
 382     bool checked;
 383     int status;
 384     pmix_status_t pstatus;
 385     pmix_scope_t scope;
 386     pmix_buffer_t data;
 387     union {
 388         pmix_ptl_cbfunc_t ptlfn;
 389         pmix_op_cbfunc_t opfn;
 390         pmix_value_cbfunc_t valuefn;
 391         pmix_lookup_cbfunc_t lookupfn;
 392         pmix_spawn_cbfunc_t spawnfn;
 393         pmix_hdlr_reg_cbfunc_t hdlrregfn;
 394     } cbfunc;
 395     size_t errhandler_ref;
 396     void *cbdata;
 397     pmix_name_t pname;
 398     char *key;
 399     pmix_value_t *value;
 400     pmix_proc_t *proc;
 401     pmix_proc_t *procs;
 402     size_t nprocs;
 403     pmix_info_t *info;
 404     size_t ninfo;
 405     size_t nvals;
 406     pmix_list_t kvs;
 407     bool copy;
 408     bool timer_running;
 409 } pmix_cb_t;
 410 PMIX_CLASS_DECLARATION(pmix_cb_t);
 411 
 412 #define PMIX_THREADSHIFT(r, c)                              \
 413  do {                                                       \
 414     pmix_event_assign(&((r)->ev), pmix_globals.evbase,      \
 415                       -1, EV_WRITE, (c), (r));              \
 416     PMIX_POST_OBJECT((r));                                  \
 417     pmix_event_active(&((r)->ev), EV_WRITE, 1);             \
 418 } while (0)
 419 
 420 
 421 typedef struct {
 422     pmix_object_t super;
 423     pmix_event_t ev;
 424     pmix_lock_t lock;
 425     /* timestamp receipt of the notification so we
 426      * can evict the oldest one if we get overwhelmed */
 427     time_t ts;
 428     /* what room of the hotel they are in */
 429     int room;
 430     pmix_status_t status;
 431     pmix_proc_t source;
 432     pmix_data_range_t range;
 433     /* For notification, we use the targets field to track
 434      * any custom range of procs that are to receive the
 435      * event.
 436      */
 437     pmix_proc_t *targets;
 438     size_t ntargets;
 439     size_t nleft;   // number of targets left to be notified
 440     /* When generating a notification, the originator can
 441      * specify the range of procs affected by this event.
 442      * For example, when creating a JOB_TERMINATED event,
 443      * the RM can specify the nspace of the job that has
 444      * ended, thus allowing users to provide a different
 445      * callback object based on the nspace being monitored.
 446      * We use the "affected" field to track these values
 447      * when processing the event chain.
 448      */
 449     pmix_proc_t *affected;
 450     size_t naffected;
 451     /* track if the event generator stipulates that default
 452      * event handlers are/are not to be given the event */
 453     bool nondefault;
 454     /* carry along any other provided info so the individual
 455      * handlers can look at it */
 456     pmix_info_t *info;
 457     size_t ninfo;
 458     pmix_buffer_t *buf;
 459     pmix_op_cbfunc_t cbfunc;
 460     void *cbdata;
 461 } pmix_notify_caddy_t;
 462 PMIX_CLASS_DECLARATION(pmix_notify_caddy_t);
 463 
 464 
 465 /****    GLOBAL STORAGE    ****/
 466 /* define a global construct that includes values that must be shared
 467  * between various parts of the code library. The client, tool,
 468  * and server libraries must instance this structure */
 469 typedef struct {
 470     int init_cntr;                      // #times someone called Init - #times called Finalize
 471     pmix_proc_t myid;
 472     pmix_peer_t *mypeer;                // my own peer object
 473     uid_t uid;                          // my effective uid
 474     gid_t gid;                          // my effective gid
 475     char *hostname;                     // my hostname
 476     uint32_t nodeid;                    // my nodeid, if given
 477     int pindex;
 478     pmix_event_base_t *evbase;
 479     bool external_evbase;
 480     int debug_output;
 481     pmix_events_t events;               // my event handler registrations.
 482     bool connected;
 483     bool commits_pending;
 484     struct timeval event_window;
 485     pmix_list_t cached_events;          // events waiting in the window prior to processing
 486     pmix_list_t iof_requests;           // list of pmix_iof_req_t IOF requests
 487     int max_events;                     // size of the notifications hotel
 488     int event_eviction_time;            // max time to cache notifications
 489     pmix_hotel_t notifications;         // hotel of pending notifications
 490     /* processes also need a place where they can store
 491      * their own internal data - e.g., data provided by
 492      * the user via the store_internal interface, as well
 493      * as caching their own data obtained thru the "put"
 494      * interface so that other parts of the process can
 495      * look them up */
 496     pmix_gds_base_module_t *mygds;
 497     /* IOF controls */
 498     bool pushstdin;
 499     pmix_list_t stdin_targets;          // list of pmix_namelist_t
 500     bool tag_output;
 501     bool xml_output;
 502     bool timestamp_output;
 503     size_t output_limit;
 504 } pmix_globals_t;
 505 
 506 /* provide access to a function to cleanup epilogs */
 507 PMIX_EXPORT void pmix_execute_epilog(pmix_epilog_t *ep);
 508 
 509 PMIX_EXPORT extern pmix_globals_t pmix_globals;
 510 PMIX_EXPORT extern pmix_lock_t pmix_global_lock;
 511 
 512 END_C_DECLS
 513 
 514 #endif /* PMIX_GLOBALS_H */