1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2016-2019 Mellanox Technologies, Inc.
4 * All rights reserved.
5 * Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
6 * Copyright (c) 2018 IBM Corporation. All rights reserved.
7 * $COPYRIGHT$
8 *
9 * Additional copyrights may follow
10 *
11 * $HEADER$
12 */
13
14 #ifndef PMIX_GDS_H
15 #define PMIX_GDS_H
16
17 #include <src/include/pmix_config.h>
18
19
20 #include <pmix_common.h>
21 #include "src/mca/mca.h"
22 #include "src/mca/base/pmix_mca_base_var.h"
23 #include "src/mca/base/pmix_mca_base_framework.h"
24 #include "src/mca/bfrops/bfrops_types.h"
25
26
27 /* The client dictates the GDS module that will be used to interact
28 * with the server - this module is stored in pmix_globals.mypeer->compat.gds
29 * Because that is a long address to keep typing out, convenience macros
30 * are provided for when that module is to be used in an operation.
31 *
32 * However, an application can open any number of GDS modules for
33 * purposes other than exchanging info with the server. For example,
34 * an application may wish to utilize a DHT module for its own
35 * peer-to-peer data sharing. Thus, the public and private interfaces
36 * are deliberately designed to be generic. The macros should make
37 * things easier for the typical internal operations
38 *
39 * NOTE: ALTHOUGH SOME GDS COMPONENTS MAY UTILIZE THEIR OWN INTERNAL
40 * PROGRESS THREADS, THE GDS IS NOT GUARANTEED TO BE THREAD-SAFE.
41 * GDS FUNCTIONS SHOULD THEREFORE ALWAYS BE CALLED IN A THREAD-SAFE
42 * CONDITION - E.G., FROM WITHIN AN EVENT
43 */
44
45 BEGIN_C_DECLS
46 /* forward declaration */
47 struct pmix_peer_t;
48 struct pmix_namespace_t;
49
50 /* backdoor to base verbosity */
51 PMIX_EXPORT extern int pmix_gds_base_output;
52
53 /**
54 * Initialize the module. Returns an error if the module cannot
55 * run, success if it can.
56 */
57 typedef pmix_status_t (*pmix_gds_base_module_init_fn_t)(pmix_info_t info[], size_t ninfo);
58
59 /**
60 * Finalize the module. Tear down any allocated storage, disconnect
61 * from any system support.
62 */
63 typedef void (*pmix_gds_base_module_fini_fn_t)(void);
64
65 /**
66 * Assign a module per the requested directives. Modules should
67 * review the provided directives to determine if they can support
68 * the request. Modules are "scanned" in component priority order
69 * and given an opportunity to respond. If a module offers itself,
70 * it will provide a priority (which can be based on the directives
71 * and therefore different from the component priority). The highest
72 * returned priority received from a responder will be selected
73 * and a pointer to its module returned */
74 typedef pmix_status_t (*pmix_gds_base_assign_module_fn_t)(pmix_info_t *info,
75 size_t ninfo,
76 int *priority);
77
78 /* SERVER FN: assemble the keys buffer for server answer */
79 typedef pmix_status_t (*pmix_gds_base_module_assemb_kvs_req_fn_t)(const pmix_proc_t *proc,
80 pmix_list_t *kvs,
81 pmix_buffer_t *buf,
82 void *cbdata);
83
84 /* define a macro for server keys answer based on peer */
85 #define PMIX_GDS_ASSEMB_KVS_REQ(s, p, r, k, b, c) \
86 do { \
87 pmix_gds_base_module_t *_g = (p)->nptr->compat.gds; \
88 (s) = PMIX_SUCCESS; \
89 if (NULL != _g->assemb_kvs_req) { \
90 pmix_output_verbose(1, pmix_gds_base_output, \
91 "[%s:%d] GDS ASSEMBLE REQ WITH %s", \
92 __FILE__, __LINE__, _g->name); \
93 (s) = _g->assemb_kvs_req(r, k, b, (void*)c); \
94 } \
95 } while(0)
96
97
98 /* CLIENT FN: unpack buffer and key processing */
99 typedef pmix_status_t (*pmix_gds_base_module_accept_kvs_resp_fn_t)(pmix_buffer_t *buf);
100
101 /* define a macro for client key processing from a server response based on peer */
102 #define PMIX_GDS_ACCEPT_KVS_RESP(s, p, b) \
103 do { \
104 pmix_gds_base_module_t *_g = (p)->nptr->compat.gds; \
105 (s) = PMIX_SUCCESS; \
106 if (NULL != _g->accept_kvs_resp) { \
107 pmix_output_verbose(1, pmix_gds_base_output, \
108 "[%s:%d] GDS ACCEPT RESP WITH %s", \
109 __FILE__, __LINE__, _g->name); \
110 (s) = _g->accept_kvs_resp(b); \
111 } \
112 } while (0)
113
114
115 /* SERVER FN: cache job-level info in the server's GDS until client
116 * procs connect and we discover which GDS module to use for them.
117 * Note that this is essentially the same function as store_job_info,
118 * only we don't have packed data on the server side, and don't want
119 * to incur the overhead of packing it just to unpack it in the function.
120 */
121 typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_namespace_t *ns,
122 pmix_info_t info[], size_t ninfo);
123
124 /* define a convenience macro for caching job info */
125 #define PMIX_GDS_CACHE_JOB_INFO(s, p, n, i, ni) \
126 do { \
127 pmix_gds_base_module_t *_g = (p)->nptr->compat.gds; \
128 pmix_output_verbose(1, pmix_gds_base_output, \
129 "[%s:%d] GDS CACHE JOB INFO WITH %s", \
130 __FILE__, __LINE__, _g->name); \
131 (s) = _g->cache_job_info((struct pmix_namespace_t*)(n), (i), (ni)); \
132 } while(0)
133
134 /* register job-level info - this is provided as a special function
135 * to allow for optimization. Called solely by the server. We cannot
136 * prepare the job-level info provided at PMIx_Register_nspace, because
137 * we don't know the GDS component to use for that application until
138 * a local client contacts us. Thus, the module is required to process
139 * the job-level info cached in the pmix_namespace_t for this job and
140 * do whatever is necessary to support the client, packing any required
141 * return message into the provided buffer.
142 *
143 * This function will be called once for each local client of
144 * a given nspace. PMIx assumes that all peers of a given nspace
145 * will use the same GDS module. Thus, the module is free to perform
146 * any relevant optimizations (e.g., packing the data only once and
147 * then releasing the cached buffer once all local clients have
148 * been serviced, or storing it once in shared memory and simply
149 * returning the shared memory rendezvous information for subsequent
150 * calls).
151 *
152 * Info provided in the reply buffer will be given to the "store_job_info"
153 * API of the GDS module on the client. Since this should match the
154 * module used by the server, each module has full knowledge and control
155 * over what is in the reply buffer.
156 *
157 * The pmix_peer_t of the requesting client is provided here so that
158 * the module can access the job-level info cached on the corresponding
159 * pmix_namespace_t pointed to by the pmix_peer_t
160 */
161 typedef pmix_status_t (*pmix_gds_base_module_register_job_info_fn_t)(struct pmix_peer_t *pr,
162 pmix_buffer_t *reply);
163
164 /* define a convenience macro for registering job info for
165 * a given peer */
166 #define PMIX_GDS_REGISTER_JOB_INFO(s, p, b) \
167 do { \
168 pmix_gds_base_module_t *_g = (p)->nptr->compat.gds; \
169 pmix_output_verbose(1, pmix_gds_base_output, \
170 "[%s:%d] GDS REG JOB INFO WITH %s", \
171 __FILE__, __LINE__, _g->name); \
172 (s) = _g->register_job_info((struct pmix_peer_t*)(p), b); \
173 } while(0)
174
175
176 /* update job-level info - this is provided as a special function
177 * to allow for optimization. Called solely by the client. The buffer
178 * provided to this API is the same one given to the server by the
179 * corresponding "register_job_info" function
180 */
181 typedef pmix_status_t (*pmix_gds_base_module_store_job_info_fn_t)(const char *nspace,
182 pmix_buffer_t *buf);
183
184 /* define a convenience macro for storing job info based on peer */
185 #define PMIX_GDS_STORE_JOB_INFO(s, p, n, b) \
186 do { \
187 pmix_gds_base_module_t *_g = (p)->nptr->compat.gds; \
188 pmix_output_verbose(1, pmix_gds_base_output, \
189 "[%s:%d] GDS STORE JOB INFO WITH %s", \
190 __FILE__, __LINE__, _g->name); \
191 (s) = _g->store_job_info(n, b); \
192 } while(0)
193
194
195 /**
196 * store key/value pair - these will either be values committed by the peer
197 * and transmitted to the server, or values stored locally by the peer.
198 * The format of the data depends on the GDS module. Note that data stored
199 * with PMIX_INTERNAL scope should be stored solely within the process and
200 * is never shared.
201 *
202 * @param peer pointer to pmix_peer_t object of the peer that
203 * provided the data
204 *
205 * @param proc the proc that the data describes
206 *
207 * @param scope scope of the data
208 *
209 * @param kv key/value pair.
210 *
211 * @return PMIX_SUCCESS on success.
212 */
213 typedef pmix_status_t (*pmix_gds_base_module_store_fn_t)(const pmix_proc_t *proc,
214 pmix_scope_t scope,
215 pmix_kval_t *kv);
216
217 /* define a convenience macro for storing key-val pairs based on peer */
218 #define PMIX_GDS_STORE_KV(s, p, pc, sc, k) \
219 do { \
220 pmix_gds_base_module_t *_g = (p)->nptr->compat.gds; \
221 pmix_output_verbose(1, pmix_gds_base_output, \
222 "[%s:%d] GDS STORE KV WITH %s", \
223 __FILE__, __LINE__, _g->name); \
224 (s) = _g->store(pc, sc, k); \
225 } while(0)
226
227
228 /**
229 * unpack and store a data "blob" from a peer so that the individual
230 * elements can later be retrieved. This is an optimization path to
231 * avoid repeatedly storing pmix_kval_t's for multiple local procs
232 * from the same nspace.
233 *
234 * ranks - a list of pmix_rank_info_t for the local ranks from this
235 * nspace - this is to be used to filter the cbs list
236 *
237 * cbdata - pointer to modex callback data
238 *
239 * bo - pointer to the byte object containing the data
240 *
241 */
242 typedef pmix_status_t (*pmix_gds_base_module_store_modex_fn_t)(struct pmix_namespace_t *ns,
243 pmix_buffer_t *buff,
244 void *cbdata);
245
246 /**
247 * define a convenience macro for storing modex byte objects
248 *
249 * r - return status code
250 *
251 * n - pointer to the pmix_namespace_t this blob is to be stored for
252 *
253 * b - pointer to pmix_byte_object_t containing the data
254 *
255 * t - pointer to the modex server tracker
256 */
257 #define PMIX_GDS_STORE_MODEX(r, n, b, t) \
258 do { \
259 pmix_output_verbose(1, pmix_gds_base_output, \
260 "[%s:%d] GDS STORE MODEX WITH %s", \
261 __FILE__, __LINE__, (n)->compat.gds->name); \
262 (r) = (n)->compat.gds->store_modex((struct pmix_namespace_t*)n, b, t); \
263 } while (0)
264
265 /**
266 * fetch value corresponding to provided key from within the defined
267 * scope. A NULL key returns all values committed by the given peer
268 * for that scope.
269 *
270 * @param proc namespace and rank whose info is being requested
271 *
272 * @param key key.
273 *
274 * @param scope scope of the data to be considered
275 *
276 * @param copy true if the caller _requires_ a copy of the data. This
277 * is used when the requestor is off-node. If
278 * set to false, then the GDS component can provide
279 * either a copy of the data, or shmem contact info
280 * to the location of the data
281 *
282 * @param info array of pmix_info_t the caller provided as
283 * qualifiers to guide the request
284 *
285 * @param ninfo number of elements in the info array
286 *
287 * @param kvs pointer to a list that will be populated with the
288 * returned pmix_kval_t data
289 *
290 * @return PMIX_SUCCESS on success.
291 *
292 * Note: all available job-level data for a given nspace can be fetched
293 * by passing a proc with rank=PMIX_RANK_WILDCARD and a NULL key. Similarly,
294 * passing a NULL key for a non-wildcard rank will return all data "put"
295 * by that rank. Scope is ignored for job-level data requests.
296 *
297 * When a specific rank if provided with a NULL key, then data for only
298 * that rank is returned. If the scope is PMIX_LOCAL, then the returned
299 * data shall include only data that was specifically "put" to local scope,
300 * plus any data that was put to PMIX_GLOBAL scope. Similarly, a scope of
301 * PMIX_REMOTE will return data that was "put" to remote scope, plus
302 * any data that was put to PMIX_GLOBAL scope. A scope of PMIX_GLOBAL
303 * will return LOCAL, REMOTE, and GLOBAL data.
304 *
305 * Data stored with PMIX_INTERNAL scope can be retrieved with that scope.
306 */
307 typedef pmix_status_t (*pmix_gds_base_module_fetch_fn_t)(const pmix_proc_t *proc,
308 pmix_scope_t scope, bool copy,
309 const char *key,
310 pmix_info_t info[], size_t ninfo,
311 pmix_list_t *kvs);
312
313 /* define a convenience macro for fetch key-val pairs based on peer,
314 * passing a pmix_cb_t containing all the required info */
315 #define PMIX_GDS_FETCH_KV(s, p, c) \
316 do { \
317 pmix_gds_base_module_t *_g = (p)->nptr->compat.gds; \
318 pmix_output_verbose(1, pmix_gds_base_output, \
319 "[%s:%d] GDS FETCH KV WITH %s", \
320 __FILE__, __LINE__, _g->name); \
321 (s) = _g->fetch((c)->proc, (c)->scope, (c)->copy, \
322 (c)->key, (c)->info, (c)->ninfo, \
323 &(c)->kvs); \
324 } while(0)
325
326
327 /**
328 * Add any envars to a peer's environment that the module needs
329 * to communicate. The API stub will rotate across all active modules, giving
330 * each a chance to contribute
331 *
332 * @return PMIX_SUCCESS on success.
333 */
334 typedef pmix_status_t (*pmix_gds_base_module_setup_fork_fn_t)(const pmix_proc_t *proc,
335 char ***env);
336
337 /**
338 * Define a new nspace in the GDS
339 *
340 * @param nspace namespace string
341 *
342 * @return PMIX_SUCCESS on success.
343 */
344 typedef pmix_status_t (*pmix_gds_base_module_add_nspace_fn_t)(const char *nspace,
345 pmix_info_t info[],
346 size_t ninfo);
347
348 /* define a convenience macro for add_nspace based on peer */
349 #define PMIX_GDS_ADD_NSPACE(s, n, i, ni) \
350 do { \
351 pmix_gds_base_active_module_t *_g; \
352 pmix_status_t _s = PMIX_SUCCESS; \
353 (s) = PMIX_SUCCESS; \
354 pmix_output_verbose(1, pmix_gds_base_output, \
355 "[%s:%d] GDS ADD NSPACE %s", \
356 __FILE__, __LINE__, (n)); \
357 PMIX_LIST_FOREACH(_g, &pmix_gds_globals.actives, \
358 pmix_gds_base_active_module_t) { \
359 if (NULL != _g->module->add_nspace) { \
360 _s = _g->module->add_nspace(n, i, ni); \
361 } \
362 if (PMIX_SUCCESS != _s) { \
363 (s) = PMIX_ERROR; \
364 } \
365 } \
366 } while(0)
367
368
369 /**
370 * Delete nspace and its associated data
371 *
372 * @param nspace namespace string
373 *
374 * @return PMIX_SUCCESS on success.
375 */
376 typedef pmix_status_t (*pmix_gds_base_module_del_nspace_fn_t)(const char* nspace);
377
378 /* define a convenience macro for del_nspace based on peer */
379 #define PMIX_GDS_DEL_NSPACE(s, n) \
380 do { \
381 pmix_gds_base_active_module_t *_g; \
382 pmix_status_t _s = PMIX_SUCCESS; \
383 (s) = PMIX_SUCCESS; \
384 pmix_output_verbose(1, pmix_gds_base_output, \
385 "[%s:%d] GDS DEL NSPACE %s", \
386 __FILE__, __LINE__, (n)); \
387 PMIX_LIST_FOREACH(_g, &pmix_gds_globals.actives, \
388 pmix_gds_base_active_module_t) { \
389 if (NULL != _g->module->del_nspace) { \
390 _s = _g->module->del_nspace(n); \
391 } \
392 if (PMIX_SUCCESS != _s) { \
393 (s) = PMIX_ERROR; \
394 } \
395 } \
396 } while(0)
397
398 /* define a convenience macro for is_tsafe for fetch operation */
399 #define PMIX_GDS_FETCH_IS_TSAFE(s, p) \
400 do { \
401 pmix_gds_base_module_t *_g = (p)->nptr->compat.gds; \
402 pmix_output_verbose(1, pmix_gds_base_output, \
403 "[%s:%d] GDS FETCH IS THREAD SAFE WITH %s", \
404 __FILE__, __LINE__, _g->name); \
405 if (true == _g->is_tsafe) { \
406 (s) = PMIX_SUCCESS; \
407 } else { \
408 (s) = PMIX_ERR_NOT_SUPPORTED; \
409 } \
410 } while(0)
411
412 /**
413 * structure for gds modules
414 */
415 typedef struct {
416 const char *name;
417 const bool is_tsafe;
418 pmix_gds_base_module_init_fn_t init;
419 pmix_gds_base_module_fini_fn_t finalize;
420 pmix_gds_base_assign_module_fn_t assign_module;
421 pmix_gds_base_module_cache_job_info_fn_t cache_job_info;
422 pmix_gds_base_module_register_job_info_fn_t register_job_info;
423 pmix_gds_base_module_store_job_info_fn_t store_job_info;
424 pmix_gds_base_module_store_fn_t store;
425 pmix_gds_base_module_store_modex_fn_t store_modex;
426 pmix_gds_base_module_fetch_fn_t fetch;
427 pmix_gds_base_module_setup_fork_fn_t setup_fork;
428 pmix_gds_base_module_add_nspace_fn_t add_nspace;
429 pmix_gds_base_module_del_nspace_fn_t del_nspace;
430 pmix_gds_base_module_assemb_kvs_req_fn_t assemb_kvs_req;
431 pmix_gds_base_module_accept_kvs_resp_fn_t accept_kvs_resp;
432
433 } pmix_gds_base_module_t;
434
435 /* NOTE: there is no public GDS interface structure - all access is
436 * done directly to/from an assigned module */
437
438 /* define the component structure */
439 struct pmix_gds_base_component_t {
440 pmix_mca_base_component_t base;
441 pmix_mca_base_component_data_t data;
442 int priority;
443 };
444 typedef struct pmix_gds_base_component_t pmix_gds_base_component_t;
445
446
447 /*
448 * Macro for use in components that are of type gds
449 */
450 #define PMIX_GDS_BASE_VERSION_1_0_0 \
451 PMIX_MCA_BASE_VERSION_1_0_0("gds", 1, 0, 0)
452
453 END_C_DECLS
454
455 #endif