1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4 * University Research and Technology
5 * Corporation. All rights reserved.
6 * Copyright (c) 2004-2014 The University of Tennessee and The University
7 * of Tennessee Research Foundation. All rights
8 * reserved.
9 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10 * University of Stuttgart. All rights reserved.
11 * Copyright (c) 2004-2005 The Regents of the University of California.
12 * All rights reserved.
13 * Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
14 * Copyright (c) 2010-2018 Los Alamos National Security, LLC.
15 * All rights reserved.
16 * $COPYRIGHT$
17 *
18 * Additional copyrights may follow
19 *
20 * $HEADER$
21 */
22
23 #ifndef _COMMON_SM_H_
24 #define _COMMON_SM_H_
25
26 #include "opal_config.h"
27
28 #include "opal/mca/mca.h"
29 #include "opal/class/opal_object.h"
30 #include "opal/class/opal_list.h"
31 #include "opal/sys/atomic.h"
32 #include "opal/mca/shmem/shmem.h"
33 #include "opal/mca/btl/base/base.h"
34 #include "opal/util/proc.h"
35 #include "opal/mca/btl/base/btl_base_error.h"
36 #include "common_sm_mpool.h"
37
38 BEGIN_C_DECLS
39
40 struct mca_mpool_base_module_t;
41
42 typedef struct mca_common_sm_seg_header_t {
43 /* lock to control atomic access */
44 opal_atomic_lock_t seg_lock;
45 /* indicates whether or not the segment is ready for use */
46 opal_atomic_int32_t seg_inited;
47 /* number of local processes that are attached to the shared memory segment.
48 * this is primarily used as a way of determining whether or not it is safe
49 * to unlink the shared memory backing store. for example, once seg_att
50 * is equal to the number of local processes, then we can safely unlink.
51 */
52 opal_atomic_size_t seg_num_procs_inited;
53 /* offset to next available memory location available for allocation */
54 size_t seg_offset;
55 /* total size of the segment */
56 size_t seg_size;
57 } mca_common_sm_seg_header_t;
58
59 typedef struct mca_common_sm_module_t {
60 /* double link list element */
61 opal_list_item_t super;
62 /* pointer to header embedded in the shared memory segment */
63 mca_common_sm_seg_header_t *module_seg;
64 /* base address of the segment */
65 unsigned char *module_seg_addr;
66 /* base address of data segment */
67 unsigned char *module_data_addr;
68 /* shared memory backing facility object that encapsulates shmem info */
69 opal_shmem_ds_t shmem_ds;
70 /* memory pool interface to shared-memory region */
71 mca_mpool_base_module_t *mpool;
72 } mca_common_sm_module_t;
73
74 OBJ_CLASS_DECLARATION(mca_common_sm_module_t);
75
76 /**
77 * This routine reorders procs array to have all the local procs at the
78 * beginning and returns the number of local procs through out_num_local_procs.
79 * The proc with the lowest name is at the beginning of the reordered procs
80 * array.
81 *
82 * @returnvalue OPAL_SUCCESS on success, something else, otherwise.
83 */
84 OPAL_DECLSPEC extern int
85 mca_common_sm_local_proc_reorder(opal_proc_t **procs,
86 size_t num_procs,
87 size_t *out_num_local_procs);
88
89 /**
90 * This routine is used to create and attach to a shared memory segment
91 * (whether it's an mmaped file or a SYSV IPC segment). It is assumed that
92 * the shared memory segment does not exist before this call.
93 *
94 * @returnvalue pointer to control structure at head of shared memory segment.
95 * Returns NULL if an error occurred.
96 */
97 OPAL_DECLSPEC extern mca_common_sm_module_t *
98 mca_common_sm_module_create_and_attach(size_t size,
99 char *file_name,
100 size_t size_ctl_structure,
101 size_t data_seg_alignment);
102
103 /**
104 * This routine is used to attach to the shared memory segment associated with
105 * *seg_meta. It is assumed that the shared memory segment associated with
106 * *seg_meta was already created with mca_common_sm_module_create.
107 *
108 * @seg_meta points to shared memory information used for initialization and setup.
109 *
110 * @returnvalue pointer to control structure at head of shared memory segment.
111 * Returns NULL if an error occurred.
112 */
113 OPAL_DECLSPEC extern mca_common_sm_module_t *
114 mca_common_sm_module_attach(opal_shmem_ds_t *seg_meta,
115 size_t size_ctl_structure,
116 size_t data_seg_alignment);
117
118 /**
119 * A thin wrapper around opal_shmem_unlink.
120 *
121 * @ modp points to an initialized mca_common_sm_module_t.
122 *
123 * @returnvalue OPAL_SUCCESS if the operation completed successfully,
124 * OPAL_ERROR otherwise.
125 */
126 OPAL_DECLSPEC extern int
127 mca_common_sm_module_unlink(mca_common_sm_module_t *modp);
128
129 /**
130 * callback from the sm mpool
131 */
132 OPAL_DECLSPEC extern void *mca_common_sm_seg_alloc (void *ctx, size_t *size);
133
134 /**
135 * This function will release all local resources attached to the
136 * shared memory segment. We assume that the operating system will
137 * release the memory resources when the last process release it.
138 *
139 * @param mca_common_sm_module - instance that is shared between
140 * components that use shared memory.
141 *
142 * @return OPAL_SUCCESS if everything was okay, otherwise return OPAL_ERROR.
143 */
144
145 OPAL_DECLSPEC extern int
146 mca_common_sm_fini(mca_common_sm_module_t *mca_common_sm_module);
147
148 /**
149 * instance that is shared between components that use shared memory.
150 */
151 OPAL_DECLSPEC extern mca_common_sm_module_t *mca_common_sm_module;
152
153
154 END_C_DECLS
155
156 #endif /* _COMMON_SM_H_ */