This source file includes following definitions.
- oshmem_proc_init
- oshmem_proc_finalize
- oshmem_proc_group_init
- oshmem_proc_group_finalize_scoll
- oshmem_proc_group_finalize
- oshmem_proc_group_create
- oshmem_proc_group_destroy_internal
- oshmem_proc_group_destroy
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 #include "oshmem_config.h"
16 #include "oshmem/proc/proc.h"
17 #include "oshmem/constants.h"
18 #include "oshmem/runtime/runtime.h"
19 #include "oshmem/mca/scoll/base/base.h"
20 #include "oshmem/proc/proc_group_cache.h"
21
22 #ifdef HAVE_STRINGS_H
23 #include <strings.h>
24 #endif
25
26 #include "opal/datatype/opal_convertor.h"
27 #include "opal/threads/mutex.h"
28 #include "opal/dss/dss.h"
29 #include "opal/util/arch.h"
30 #include "opal/class/opal_list.h"
31
32
33 static opal_mutex_t oshmem_proc_lock;
34
35 int oshmem_proc_init(void)
36 {
37 OBJ_CONSTRUCT(&oshmem_proc_lock, opal_mutex_t);
38
39
40 assert(sizeof(oshmem_proc_data_t) <= OMPI_PROC_PADDING_SIZE);
41
42 assert(0 == (offsetof(ompi_proc_t, padding) & (sizeof(char *)-1)));
43
44 return OSHMEM_SUCCESS;
45 }
46
47 int oshmem_proc_finalize(void)
48 {
49 OBJ_DESTRUCT(&oshmem_proc_lock);
50
51 return OSHMEM_SUCCESS;
52 }
53
54 opal_pointer_array_t oshmem_group_array = {{0}};
55
56 oshmem_group_t* oshmem_group_all = NULL;
57 oshmem_group_t* oshmem_group_self = NULL;
58 oshmem_group_t* oshmem_group_null = NULL;
59
60 OBJ_CLASS_INSTANCE(oshmem_group_t, opal_object_t, NULL, NULL);
61
62 static void oshmem_proc_group_destroy_internal(oshmem_group_t* group,
63 int scoll_unselect);
64
65 int oshmem_proc_group_init(void)
66 {
67 int rc;
68
69 rc = oshmem_group_cache_init();
70 if (OSHMEM_SUCCESS != rc) {
71 return rc;
72 }
73
74
75 OBJ_CONSTRUCT(&oshmem_group_array, opal_pointer_array_t);
76
77 rc = opal_pointer_array_init(&oshmem_group_array, 0,
78 INT_MAX, 1);
79 if (OPAL_SUCCESS != rc) {
80 goto err1;
81 }
82
83
84 oshmem_group_all = oshmem_proc_group_create(0, 1, ompi_comm_size(oshmem_comm_world));
85 if (NULL == oshmem_group_all) {
86 goto err2;
87 }
88
89
90 oshmem_group_self = oshmem_proc_group_create(oshmem_proc_pe(oshmem_proc_local()), 0, 1);
91 if (NULL == oshmem_group_self) {
92 goto err3;
93 }
94
95
96 oshmem_group_null = NULL;
97
98 return OSHMEM_SUCCESS;
99
100 err3:
101 oshmem_proc_group_destroy_internal(oshmem_group_all, 1);
102 err2:
103 OBJ_DESTRUCT(&oshmem_group_array);
104 err1:
105 oshmem_group_cache_destroy();
106 return OSHMEM_ERROR;
107 }
108
109 void oshmem_proc_group_finalize_scoll(void)
110 {
111 int max, i;
112 oshmem_group_t *group;
113
114
115 max = opal_pointer_array_get_size(&oshmem_group_array);
116 for (i = 0; i < max; i++) {
117 group = (oshmem_group_t *) opal_pointer_array_get_item(&oshmem_group_array,
118 i);
119 if (NULL != group) {
120 mca_scoll_base_group_unselect(group);
121 }
122 }
123 }
124
125 int oshmem_proc_group_finalize(void)
126 {
127 int max, i;
128 oshmem_group_t *group;
129
130
131 max = opal_pointer_array_get_size(&oshmem_group_array);
132 for (i = 0; i < max; i++) {
133 group =
134 (oshmem_group_t *) opal_pointer_array_get_item(&oshmem_group_array,
135 i);
136 if (NULL != group) {
137
138 oshmem_proc_group_destroy_internal(group, 0);
139 }
140 }
141
142 OBJ_DESTRUCT(&oshmem_group_array);
143
144 oshmem_group_cache_destroy();
145 return OSHMEM_SUCCESS;
146 }
147
148 oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_size)
149 {
150 int cur_pe, count_pe;
151 int i;
152 oshmem_group_t* group = NULL;
153 ompi_proc_t** proc_array = NULL;
154 ompi_proc_t* proc = NULL;
155
156 assert(oshmem_proc_local());
157
158 group = oshmem_group_cache_find(pe_start, pe_stride, pe_size);
159 if (NULL != group) {
160 return group;
161 }
162
163 group = OBJ_NEW(oshmem_group_t);
164 if (NULL == group) {
165 return NULL;
166 }
167
168 cur_pe = 0;
169 count_pe = 0;
170
171 OPAL_THREAD_LOCK(&oshmem_proc_lock);
172
173
174 proc_array = (ompi_proc_t**) malloc(pe_size * sizeof(ompi_proc_t*));
175 if (NULL == proc_array) {
176 OBJ_RELEASE(group);
177 OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
178 return NULL ;
179 }
180
181 group->my_pe = oshmem_proc_pe(oshmem_proc_local());
182 group->is_member = 0;
183 for (i = 0 ; i < ompi_comm_size(oshmem_comm_world) ; i++) {
184 proc = oshmem_proc_find(i);
185 if (NULL == proc) {
186 opal_output(0,
187 "Error: Can not find proc object for pe = %d", i);
188 free(proc_array);
189 OBJ_RELEASE(group);
190 OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
191 return NULL;
192 }
193 if (count_pe >= (int) pe_size) {
194 break;
195 } else if ((cur_pe >= pe_start)
196 && ((pe_stride == 0)
197 || (((cur_pe - pe_start) % pe_stride) == 0))) {
198 proc_array[count_pe++] = proc;
199 if (oshmem_proc_pe(proc) == group->my_pe)
200 group->is_member = 1;
201 }
202 cur_pe++;
203 }
204 group->proc_array = proc_array;
205 group->proc_count = (int) count_pe;
206 group->ompi_comm = NULL;
207
208
209 OBJ_CONSTRUCT(&(group->peer_list), opal_list_t);
210 {
211 ompi_namelist_t *peer = NULL;
212
213 for (i = 0; i < group->proc_count; i++) {
214 peer = OBJ_NEW(ompi_namelist_t);
215 peer->name.jobid = OSHMEM_PROC_JOBID(group->proc_array[i]);
216 peer->name.vpid = OSHMEM_PROC_VPID(group->proc_array[i]);
217 opal_list_append(&(group->peer_list), &peer->super);
218 }
219 }
220 group->id = opal_pointer_array_add(&oshmem_group_array, group);
221
222 memset(&group->g_scoll, 0, sizeof(mca_scoll_base_group_scoll_t));
223
224 if (OSHMEM_SUCCESS != mca_scoll_base_select(group)) {
225 opal_output(0,
226 "Error: No collective modules are available: group is not created, returning NULL");
227 oshmem_proc_group_destroy_internal(group, 0);
228 OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
229 return NULL;
230 }
231
232 if (OSHMEM_SUCCESS != oshmem_group_cache_insert(group, pe_start,
233 pe_stride, pe_size)) {
234 oshmem_proc_group_destroy_internal(group, 1);
235 OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
236 return NULL;
237 }
238
239 OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
240 return group;
241 }
242
243 static void
244 oshmem_proc_group_destroy_internal(oshmem_group_t* group, int scoll_unselect)
245 {
246 if (NULL == group) {
247 return;
248 }
249
250 if (scoll_unselect) {
251 mca_scoll_base_group_unselect(group);
252 }
253
254
255 if (group->proc_array) {
256 free(group->proc_array);
257 }
258
259
260 {
261 opal_list_item_t *item;
262
263 while (NULL != (item = opal_list_remove_first(&(group->peer_list)))) {
264
265 OBJ_RELEASE(item);
266 }
267 OBJ_DESTRUCT(&(group->peer_list));
268 }
269
270
271
272 if (NULL
273 != opal_pointer_array_get_item(&oshmem_group_array,
274 group->id)) {
275 opal_pointer_array_set_item(&oshmem_group_array, group->id, NULL );
276 }
277
278 OBJ_RELEASE(group);
279 }
280
281 void oshmem_proc_group_destroy(oshmem_group_t* group)
282 {
283 if (oshmem_group_cache_enabled()) {
284 return;
285 }
286 oshmem_proc_group_destroy_internal(group, 1);
287 }