This source file includes following definitions.
- pmi_init
- pmi_get_local_ranks
- _get_maps_file
- is_dstor_region
- pmi_get_shmem_size
- _put_key
- pmi_put_key_loc
- pmi_put_key_rem
- pmi_put_double
- pmi_commit
- pmi_fence
- _get_key
- pmi_get_key_loc
- pmi_get_key_rem
- pmi_get_double
- pmi_fini
1
2
3
4
5
6
7
8
9
10
11
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <unistd.h>
15 #include <time.h>
16 #include <getopt.h>
17 #include <limits.h>
18 #include <string.h>
19 #include <pmix.h>
20
21 pmix_proc_t this_proc;
22 static int _int_size = 0;
23
24 void pmi_init(int *rank, int *size)
25 {
26 pmix_value_t value, *val = &value;
27 pmix_proc_t job_proc;
28 int rc;
29
30
31 #if (PMIX_VERSION_MAJOR == 1 )
32 if (PMIX_SUCCESS != (rc = PMIx_Init(&this_proc)))
33 #else
34 if (PMIX_SUCCESS != (rc = PMIx_Init(&this_proc, NULL, 0)))
35 #endif
36 {
37 fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d", this_proc.nspace, this_proc.rank, rc);
38 abort();
39 }
40
41 job_proc = this_proc;
42 #if (PMIX_VERSION_MAJOR > 1 )
43 job_proc.rank = PMIX_RANK_WILDCARD;
44 #endif
45
46 if (PMIX_SUCCESS != (rc = PMIx_Get(&job_proc, PMIX_JOB_SIZE, NULL, 0, &val))) {
47 fprintf(stderr, "Client ns %s rank %d: PMIx_Get job size failed: %d", this_proc.nspace, this_proc.rank, rc);
48 abort();
49 }
50 _int_size = *size = val->data.uint32;
51 *rank = this_proc.rank;
52 PMIX_VALUE_RELEASE(val);
53 }
54
55 void pmi_get_local_ranks(int **local_ranks, int *local_cnt)
56 {
57 pmix_value_t value, *val = &value;
58 char *ptr;
59 int i, rc;
60 pmix_proc_t job_proc = this_proc;
61 #if (PMIX_VERSION_MAJOR > 1 )
62 job_proc.rank = PMIX_RANK_WILDCARD;
63 #endif
64
65
66 if (PMIX_SUCCESS != (rc = PMIx_Get(&job_proc, PMIX_LOCAL_SIZE, NULL, 0, &val))) {
67 fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_SIZE failed: %d", this_proc.nspace, this_proc.rank, rc);
68 abort();
69 }
70 *local_cnt = val->data.uint32;
71 PMIX_VALUE_RELEASE(val);
72
73 *local_ranks = calloc(*local_cnt, sizeof(int));
74
75 if (PMIX_SUCCESS != (rc = PMIx_Get(&job_proc, PMIX_LOCAL_PEERS, NULL, 0, &val))) {
76 fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_PEERS failed: %d", this_proc.nspace, this_proc.rank, rc);
77 abort();
78 }
79 ptr = val->data.string;
80 for(i=0; NULL != ptr && i < *local_cnt; i++ ){
81 char *loc_rank = strsep(&ptr, ",");
82 (*local_ranks)[i] = atoi(loc_rank);
83 }
84 if( i != *local_cnt || NULL != ptr ){
85 fprintf(stderr, "Client ns %s rank %d: number of local peers doesn't match",
86 this_proc.nspace, this_proc.rank);
87 abort();
88 }
89 }
90
91
92
93
94 typedef struct {
95 size_t rank;
96 size_t offset;
97 size_t count;
98 } rank_meta_info;
99
100 #define DSTORE_INIT_SEG "initial-pmix_shared-segment"
101 #define DSTORE_META_SEG "smseg"
102 #define DSTORE_DATA_SEG "smdataseg"
103
104 typedef enum { other_seg, init_seg, meta_seg, data_seg } seg_type_t;
105
106 struct {
107 char *sign;
108 seg_type_t type;
109 } segments[] = {
110 {DSTORE_INIT_SEG, init_seg},
111 {DSTORE_META_SEG, meta_seg},
112 {DSTORE_DATA_SEG, data_seg},
113 };
114
115 static char *_get_maps_file(char *line)
116 {
117 char *token = NULL;
118 char *saveptr = NULL;
119 int i = 0;
120
121 token = strtok_r(line, "-\n", &saveptr);
122 if(NULL == token) {
123 return NULL;
124 }
125 return (char*)strtoul(token, &saveptr, 16);
126 }
127
128 static seg_type_t
129 is_dstor_region(char *line, char **base)
130 {
131 int i;
132 seg_type_t type = other_seg;
133 for(i=0; i<3; i++){
134 if( strstr(line, segments[i].sign) ){
135 if(!(*base = _get_maps_file(line)) ){
136 return other_seg;
137 }
138 return segments[i].type;
139 }
140 }
141 return other_seg;
142 }
143
144 void pmi_get_shmem_size(char *is_avail, size_t *cum_size)
145 {
146 char maps_path[PATH_MAX] = "/proc/self/maps";
147 FILE *maps_fp = NULL;
148 char *line = NULL;
149 size_t size = 0, meta_size = 0, data_size = 0;
150 char *base;
151
152 if (NULL == (maps_fp = fopen(maps_path, "r"))) {
153 abort();
154 }
155
156 while ((size = getline(&line, &size, maps_fp)) != -1) {
157 seg_type_t type = is_dstor_region(line, &base);
158 switch(type) {
159 case other_seg:
160 case init_seg:
161 case meta_seg:
162 break;
163 case data_seg:{
164 data_size += *((size_t*)base);
165 break;
166 }
167 }
168 }
169 free(line);
170 fclose(maps_fp);
171 *is_avail = 0;
172 *cum_size = data_size;
173 if( *cum_size > 0 ) {
174 *is_avail = 1;
175 *cum_size += sizeof(rank_meta_info) * _int_size;
176 }
177 }
178
179 static void _put_key(char *key, int *key_val, int key_size, pmix_scope_t scope)
180 {
181 pmix_value_t value;
182 int rc;
183
184 value.type = PMIX_BYTE_OBJECT;
185 value.data.bo.size = key_size * sizeof(int);
186 value.data.bo.bytes = (char*)key_val;
187 if (PMIX_SUCCESS != (rc = PMIx_Put(scope, key, &value))) {
188 fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d", this_proc.nspace, this_proc.rank, rc);
189 abort();
190 }
191 }
192
193 void pmi_put_key_loc(char *key, int *key_val, int key_size)
194 {
195 _put_key(key, key_val, key_size, PMIX_LOCAL);
196 }
197
198 void pmi_put_key_rem(char *key, int *key_val, int key_size)
199 {
200 _put_key(key, key_val, key_size, PMIX_REMOTE);
201 }
202
203 void pmi_put_double(char *key, double v)
204 {
205 pmix_value_t value;
206 int rc;
207
208 value.type = PMIX_DOUBLE;
209 value.data.dval = v;
210 if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_GLOBAL, key, &value))) {
211 fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d", this_proc.nspace, this_proc.rank, rc);
212 abort();
213 }
214 }
215
216 void pmi_commit()
217 {
218 int rc;
219
220 if (PMIX_SUCCESS != (rc = PMIx_Commit())) {
221 fprintf(stderr, "Client ns %s rank %d: PMIx_Commit failed: %d",
222 this_proc.nspace, this_proc.rank, rc);
223 abort();
224 }
225 }
226
227
228 void pmi_fence(int collect)
229 {
230 pmix_info_t *info = NULL;
231 pmix_proc_t proc;
232 bool value = 1;
233 int ninfo = 0;
234 int rc;
235
236 if( collect ){
237 PMIX_INFO_CREATE(info, 1);
238 (void)strncpy(info->key, PMIX_COLLECT_DATA, PMIX_MAX_KEYLEN);
239 pmix_value_load(&info->value, &value, PMIX_BOOL);
240 ninfo = 1;
241 }
242
243
244 PMIX_PROC_CONSTRUCT(&proc);
245 (void)strncpy(proc.nspace, this_proc.nspace, PMIX_MAX_NSLEN);
246 proc.rank = PMIX_RANK_WILDCARD;
247
248 if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, info, ninfo))) {
249 fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d",
250 this_proc.nspace, this_proc.rank, rc);
251 abort();
252 }
253
254 if( collect ){
255 PMIX_INFO_FREE(info, ninfo);
256 }
257 }
258
259 void _get_key(int rank, char *key_name, int **key_val, int *key_size)
260 {
261 pmix_proc_t proc;
262 pmix_value_t value, *val = &value;
263 int rc;
264
265 (void)strncpy(proc.nspace, this_proc.nspace, PMIX_MAX_NSLEN);
266 proc.rank = rank;
267 if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, key_name, NULL, 0, &val))) {
268 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d",
269 this_proc.nspace, this_proc.rank, key_name, rc);
270 abort();
271 }
272 if (PMIX_BYTE_OBJECT != val->type) {
273 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d",
274 this_proc.nspace, this_proc.rank, key_name, val->type);
275 PMIX_VALUE_RELEASE(val);
276 abort();
277 }
278 *key_val = (int*)val->data.bo.bytes;
279 *key_size = val->data.bo.size / sizeof(int);
280 val->data.bo.bytes = NULL;
281 PMIX_VALUE_RELEASE(val);
282 }
283
284 void pmi_get_key_loc(int rank, char *key_name, int **key_val, int *key_size)
285 {
286 _get_key(rank, key_name, key_val, key_size);
287 }
288
289 void pmi_get_key_rem(int rank, char *key_name, int **key_val, int *key_size)
290 {
291 _get_key(rank, key_name, key_val, key_size);
292 }
293
294 double pmi_get_double(int rank, char *key_name)
295 {
296 pmix_proc_t proc;
297 pmix_value_t value, *val = &value;
298 int rc;
299 double v;
300
301 (void)strncpy(proc.nspace, this_proc.nspace, PMIX_MAX_NSLEN);
302 proc.rank = rank;
303 if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, key_name, NULL, 0, &val))) {
304 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d",
305 this_proc.nspace, this_proc.rank, key_name, rc);
306 abort();
307 }
308 if (PMIX_DOUBLE != val->type) {
309 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d",
310 this_proc.nspace, this_proc.rank, key_name, val->type);
311 PMIX_VALUE_RELEASE(val);
312 abort();
313 }
314 v = val->data.dval;
315 PMIX_VALUE_RELEASE(val);
316 return v;
317 }
318
319 pmi_fini()
320 {
321 int rc;
322 #if (PMIX_VERSION_MAJOR == 1 )
323 if (PMIX_SUCCESS != (rc = PMIx_Finalize()))
324 #else
325 if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0)))
326 #endif
327 {
328 fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", this_proc.nspace, this_proc.rank, rc);
329 abort();
330 }
331 }