This source file includes following definitions.
- check_config_value_bool
- check_config_value_equal
- progress_callback
- component_open
- component_register
- component_init
- component_finalize
- component_query
- component_select
- ompi_osc_portals4_attach
- ompi_osc_portals4_detach
- ompi_osc_portals4_free
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 #include "ompi_config.h"
21
22 #include "opal/util/printf.h"
23
24 #include "ompi/mca/osc/osc.h"
25 #include "ompi/mca/osc/base/base.h"
26 #include "ompi/mca/osc/base/osc_base_obj_convert.h"
27 #include "ompi/request/request.h"
28
29 #include "osc_portals4.h"
30 #include "osc_portals4_request.h"
31
32 static int component_open(void);
33 static int component_register(void);
34 static int component_init(bool enable_progress_threads, bool enable_mpi_threads);
35 static int component_finalize(void);
36 static int component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
37 struct ompi_communicator_t *comm, struct opal_info_t *info,
38 int flavor);
39 static int component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
40 struct ompi_communicator_t *comm, struct opal_info_t *info,
41 int flavor, int *model);
42
43
44 ompi_osc_portals4_component_t mca_osc_portals4_component = {
45 {
46 .osc_version = {
47 OMPI_OSC_BASE_VERSION_3_0_0,
48 .mca_component_name = "portals4",
49 MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
50 OMPI_RELEASE_VERSION),
51 .mca_open_component = component_open,
52 .mca_register_component_params = component_register,
53 },
54 .osc_data = {
55
56 MCA_BASE_METADATA_PARAM_NONE
57 },
58 .osc_init = component_init,
59 .osc_query = component_query,
60 .osc_select = component_select,
61 .osc_finalize = component_finalize,
62 }
63 };
64
65
66 ompi_osc_portals4_module_t ompi_osc_portals4_module_template = {
67 {
68 .osc_win_attach = ompi_osc_portals4_attach,
69 .osc_win_detach = ompi_osc_portals4_detach,
70 .osc_free = ompi_osc_portals4_free,
71
72 .osc_put = ompi_osc_portals4_put,
73 .osc_get = ompi_osc_portals4_get,
74 .osc_accumulate = ompi_osc_portals4_accumulate,
75 .osc_compare_and_swap = ompi_osc_portals4_compare_and_swap,
76 .osc_fetch_and_op = ompi_osc_portals4_fetch_and_op,
77 .osc_get_accumulate = ompi_osc_portals4_get_accumulate,
78
79 .osc_rput = ompi_osc_portals4_rput,
80 .osc_rget = ompi_osc_portals4_rget,
81 .osc_raccumulate = ompi_osc_portals4_raccumulate,
82 .osc_rget_accumulate = ompi_osc_portals4_rget_accumulate,
83
84 .osc_fence = ompi_osc_portals4_fence,
85
86 .osc_start = ompi_osc_portals4_start,
87 .osc_complete = ompi_osc_portals4_complete,
88 .osc_post = ompi_osc_portals4_post,
89 .osc_wait = ompi_osc_portals4_wait,
90 .osc_test = ompi_osc_portals4_test,
91
92 .osc_lock = ompi_osc_portals4_lock,
93 .osc_unlock = ompi_osc_portals4_unlock,
94 .osc_lock_all = ompi_osc_portals4_lock_all,
95 .osc_unlock_all = ompi_osc_portals4_unlock_all,
96
97 .osc_sync = ompi_osc_portals4_sync,
98 .osc_flush = ompi_osc_portals4_flush,
99 .osc_flush_all = ompi_osc_portals4_flush_all,
100 .osc_flush_local = ompi_osc_portals4_flush_local,
101 .osc_flush_local_all = ompi_osc_portals4_flush_local_all,
102 }
103 };
104
105
106
107
108
109 static bool
110 check_config_value_bool(char *key, opal_info_t *info)
111 {
112 char *value_string;
113 int value_len, ret, flag, param;
114 const bool *flag_value;
115 bool result;
116
117 ret = opal_info_get_valuelen(info, key, &value_len, &flag);
118 if (OMPI_SUCCESS != ret) goto info_not_found;
119 if (flag == 0) goto info_not_found;
120 value_len++;
121
122 value_string = (char*)malloc(sizeof(char) * value_len + 1);
123 if (NULL == value_string) goto info_not_found;
124
125 ret = opal_info_get(info, key, value_len, value_string, &flag);
126 if (OMPI_SUCCESS != ret) {
127 free(value_string);
128 goto info_not_found;
129 }
130 assert(flag != 0);
131 ret = opal_info_value_to_bool(value_string, &result);
132 free(value_string);
133 if (OMPI_SUCCESS != ret) goto info_not_found;
134 return result;
135
136 info_not_found:
137 param = mca_base_var_find("ompi", "osc", "portals4", key);
138 if (0 > param) return false;
139
140 ret = mca_base_var_get_value(param, &flag_value, NULL, NULL);
141 if (OMPI_SUCCESS != ret) return false;
142
143 return flag_value[0];
144 }
145
146
147 static bool
148 check_config_value_equal(char *key, opal_info_t *info, char *value)
149 {
150 char *value_string;
151 int value_len, ret, flag, param;
152 const bool *flag_value;
153 bool result = false;
154
155 ret = opal_info_get_valuelen(info, key, &value_len, &flag);
156 if (OMPI_SUCCESS != ret) goto info_not_found;
157 if (flag == 0) goto info_not_found;
158 value_len++;
159
160 value_string = (char*)malloc(sizeof(char) * value_len + 1);
161 if (NULL == value_string) goto info_not_found;
162
163 ret = opal_info_get(info, key, value_len, value_string, &flag);
164 if (OMPI_SUCCESS != ret) {
165 free(value_string);
166 goto info_not_found;
167 }
168 assert(flag != 0);
169 if (0 == strcmp(value_string, value)) result = true;
170 free(value_string);
171 return result;
172
173 info_not_found:
174 param = mca_base_var_find("ompi", "osc", "portals4", key);
175 if (0 > param) return false;
176
177 ret = mca_base_var_get_value(param, &flag_value, NULL, NULL);
178 if (OMPI_SUCCESS != ret) return false;
179
180 if (0 == strcmp(value_string, value)) result = true;
181
182 return result;
183 }
184
185
186 static int
187 progress_callback(void)
188 {
189 int ret, count = 0;
190 ptl_event_t ev;
191 ompi_osc_portals4_request_t *req;
192 int32_t ops;
193
194 while (true) {
195 ret = PtlEQGet(mca_osc_portals4_component.matching_eq_h, &ev);
196 if (PTL_OK == ret) {
197 goto process;
198 } else if (PTL_EQ_DROPPED == ret) {
199 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
200 "%s:%d: PtlEQGet reported dropped event",
201 __FILE__, __LINE__);
202 goto process;
203 } else if (PTL_EQ_EMPTY == ret) {
204 return 0;
205 } else {
206 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
207 "%s:%d: PtlEQGet failed: %d\n",
208 __FILE__, __LINE__, ret);
209 return 0;
210 }
211
212 process:
213 if (ev.ni_fail_type != PTL_OK) {
214 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
215 "%s:%d: event failure: %d %d",
216 __FILE__, __LINE__, ev.type, ev.ni_fail_type);
217 return 0;
218 }
219
220 count++;
221
222 if (NULL != ev.user_ptr) {
223
224 if (ev.type == PTL_EVENT_LINK) {
225 *(int *)ev.user_ptr = *(int *)ev.user_ptr + 1;
226 opal_condition_broadcast(&mca_osc_portals4_component.cond);
227 continue;
228 }
229
230 req = (ompi_osc_portals4_request_t*) ev.user_ptr;
231 opal_atomic_add_fetch_size_t(&req->super.req_status._ucount, ev.mlength);
232 ops = opal_atomic_add_fetch_32(&req->ops_committed, 1);
233 if (ops == req->ops_expected) {
234 ompi_request_complete(&req->super, true);
235 }
236 }
237 }
238
239 return count;
240 }
241
242
243 static int
244 component_open(void)
245 {
246 return OMPI_SUCCESS;
247 }
248
249
250 static int
251 component_register(void)
252 {
253 mca_osc_portals4_component.no_locks = false;
254 (void) mca_base_component_var_register(&mca_osc_portals4_component.super.osc_version,
255 "no_locks",
256 "Enable optimizations available only if MPI_LOCK is "
257 "not used. "
258 "Info key of same name overrides this value.",
259 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
260 OPAL_INFO_LVL_9,
261 MCA_BASE_VAR_SCOPE_READONLY,
262 &mca_osc_portals4_component.no_locks);
263
264 mca_osc_portals4_component.ptl_max_msg_size = PTL_SIZE_MAX;
265 (void) mca_base_component_var_register(&mca_osc_portals4_component.super.osc_version,
266 "max_msg_size",
267 "Max size supported by portals4 (above that, a message is cut into messages less than that size)",
268 MCA_BASE_VAR_TYPE_UNSIGNED_LONG,
269 NULL,
270 0,
271 0,
272 OPAL_INFO_LVL_9,
273 MCA_BASE_VAR_SCOPE_READONLY,
274 &mca_osc_portals4_component.ptl_max_msg_size);
275
276 return OMPI_SUCCESS;
277 }
278
279
280 static int
281 component_init(bool enable_progress_threads, bool enable_mpi_threads)
282 {
283 int ret;
284 ptl_ni_limits_t actual;
285
286 ret = PtlInit();
287 if (PTL_OK != ret) {
288 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
289 "%s:%d: PtlInit failed: %d\n",
290 __FILE__, __LINE__, ret);
291 return OMPI_ERROR;
292 }
293
294 ret = PtlNIInit(PTL_IFACE_DEFAULT,
295 PTL_NI_PHYSICAL | PTL_NI_MATCHING,
296 PTL_PID_ANY,
297 NULL,
298 &actual,
299 &mca_osc_portals4_component.matching_ni_h);
300 if (PTL_OK != ret) {
301 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
302 "%s:%d: PtlNIInit failed: %d\n",
303 __FILE__, __LINE__, ret);
304 return ret;
305 }
306
307
308
309 if (mca_osc_portals4_component.ptl_max_msg_size > actual.max_msg_size)
310 mca_osc_portals4_component.ptl_max_msg_size = actual.max_msg_size;
311 OPAL_OUTPUT_VERBOSE((10, ompi_osc_base_framework.framework_output,
312 "max_size = %lu", mca_osc_portals4_component.ptl_max_msg_size));
313
314 mca_osc_portals4_component.matching_atomic_max = actual.max_atomic_size;
315 mca_osc_portals4_component.matching_fetch_atomic_max = actual.max_fetch_atomic_size;
316 mca_osc_portals4_component.matching_atomic_ordered_size =
317 MAX(actual.max_waw_ordered_size, actual.max_war_ordered_size);
318
319 ret = PtlEQAlloc(mca_osc_portals4_component.matching_ni_h,
320 4096,
321 &mca_osc_portals4_component.matching_eq_h);
322 if (PTL_OK != ret) {
323 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
324 "%s:%d: PtlEQAlloc failed: %d\n",
325 __FILE__, __LINE__, ret);
326 return ret;
327 }
328
329 ret = PtlPTAlloc(mca_osc_portals4_component.matching_ni_h,
330 0,
331 mca_osc_portals4_component.matching_eq_h,
332 REQ_OSC_TABLE_ID,
333 &mca_osc_portals4_component.matching_pt_idx);
334 if (PTL_OK != ret) {
335 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
336 "%s:%d: PtlPTAlloc failed: %d\n",
337 __FILE__, __LINE__, ret);
338 return ret;
339 }
340
341 if (mca_osc_portals4_component.matching_pt_idx != REQ_OSC_TABLE_ID) {
342 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
343 "%s:%d: PtlPTAlloc did not allocate the requested PT: %d\n",
344 __FILE__, __LINE__, mca_osc_portals4_component.matching_pt_idx);
345 return ret;
346 }
347
348 OBJ_CONSTRUCT(&mca_osc_portals4_component.requests, opal_free_list_t);
349 ret = opal_free_list_init (&mca_osc_portals4_component.requests,
350 sizeof(ompi_osc_portals4_request_t),
351 opal_cache_line_size,
352 OBJ_CLASS(ompi_osc_portals4_request_t),
353 0, 0, 8, 0, 8, NULL, 0, NULL, NULL, NULL);
354 if (OMPI_SUCCESS != ret) {
355 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
356 "%s:%d: opal_free_list_init failed: %d\n",
357 __FILE__, __LINE__, ret);
358 return ret;
359 }
360
361 ret = opal_progress_register(progress_callback);
362 if (OMPI_SUCCESS != ret) {
363 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
364 "%s:%d: opal_progress_register failed: %d\n",
365 __FILE__, __LINE__, ret);
366 return ret;
367 }
368
369 return OMPI_SUCCESS;
370 }
371
372
373 static int
374 component_finalize(void)
375 {
376 PtlNIFini(mca_osc_portals4_component.matching_ni_h);
377
378 return OMPI_SUCCESS;
379 }
380
381
382 static int
383 component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
384 struct ompi_communicator_t *comm, struct opal_info_t *info,
385 int flavor)
386 {
387 int ret;
388
389 if (MPI_WIN_FLAVOR_SHARED == flavor) return -1;
390
391 ret = PtlGetUid(mca_osc_portals4_component.matching_ni_h, &mca_osc_portals4_component.uid);
392 if (PTL_OK != ret) {
393 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
394 "%s:%d: PtlGetUid failed: %d\n",
395 __FILE__, __LINE__, ret);
396 return OMPI_ERROR;
397 }
398
399 return 20;
400 }
401
402
403 static int
404 component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
405 struct ompi_communicator_t *comm, struct opal_info_t *info,
406 int flavor, int *model)
407 {
408 ompi_osc_portals4_module_t *module = NULL;
409 int ret = OMPI_ERROR;
410 int tmp;
411 ptl_md_t md;
412 ptl_me_t me;
413 char *name;
414
415 if (MPI_WIN_FLAVOR_SHARED == flavor) return OMPI_ERR_NOT_SUPPORTED;
416
417
418 module = (ompi_osc_portals4_module_t*)
419 calloc(1, sizeof(ompi_osc_portals4_module_t));
420 if (NULL == module) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
421
422
423 memcpy(module, &ompi_osc_portals4_module_template,
424 sizeof(ompi_osc_base_module_t));
425
426
427 if (MPI_WIN_FLAVOR_ALLOCATE == flavor) {
428 module->free_after = *base = malloc(size);
429 if (NULL == *base) goto error;
430 } else {
431 module->free_after = NULL;
432 }
433
434 ret = ompi_comm_dup(comm, &module->comm);
435 if (OMPI_SUCCESS != ret) goto error;
436
437 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
438 "portals4 component creating window with id %d",
439 ompi_comm_get_cid(module->comm));
440
441 opal_asprintf(&name, "portals4 window %d", ompi_comm_get_cid(module->comm));
442 ompi_win_set_name(win, name);
443 free(name);
444
445
446
447 tmp = disp_unit;
448 ret = module->comm->c_coll->coll_bcast(&tmp, 1, MPI_INT, 0,
449 module->comm,
450 module->comm->c_coll->coll_bcast_module);
451 if (OMPI_SUCCESS != ret) {
452 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
453 "%s:%d: MPI_Bcast failed: %d\n",
454 __FILE__, __LINE__, ret);
455 goto error;
456 }
457 tmp = (tmp == disp_unit) ? 1 : 0;
458 ret = module->comm->c_coll->coll_allreduce(MPI_IN_PLACE, &tmp, 1, MPI_INT, MPI_LAND,
459 module->comm, module->comm->c_coll->coll_allreduce_module);
460 if (OMPI_SUCCESS != ret) goto error;
461 if (tmp == 1) {
462 module->disp_unit = disp_unit;
463 module->disp_units = NULL;
464 } else {
465 module->disp_unit = -1;
466 module->disp_units = malloc(sizeof(int) * ompi_comm_size(module->comm));
467 ret = module->comm->c_coll->coll_allgather(&disp_unit, 1, MPI_INT,
468 module->disp_units, 1, MPI_INT,
469 module->comm,
470 module->comm->c_coll->coll_allgather_module);
471 if (OMPI_SUCCESS != ret) goto error;
472 }
473
474 module->ni_h = mca_osc_portals4_component.matching_ni_h;
475 module->pt_idx = mca_osc_portals4_component.matching_pt_idx;
476
477 ret = PtlCTAlloc(module->ni_h, &(module->ct_h));
478 if (PTL_OK != ret) {
479 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
480 "%s:%d: PtlCTAlloc failed: %d\n",
481 __FILE__, __LINE__, ret);
482 goto error;
483 }
484
485 md.start = 0;
486 md.length = PTL_SIZE_MAX;
487 md.options = PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK;
488 md.eq_handle = mca_osc_portals4_component.matching_eq_h;
489 md.ct_handle = module->ct_h;
490 ret = PtlMDBind(module->ni_h, &md, &module->md_h);
491 if (PTL_OK != ret) {
492 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
493 "%s:%d: PtlMDBind failed: %d\n",
494 __FILE__, __LINE__, ret);
495 goto error;
496 }
497
498 md.start = 0;
499 md.length = PTL_SIZE_MAX;
500 md.options = PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK;
501 md.eq_handle = mca_osc_portals4_component.matching_eq_h;
502 md.ct_handle = module->ct_h;
503 ret = PtlMDBind(module->ni_h, &md, &module->req_md_h);
504 if (PTL_OK != ret) {
505 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
506 "%s:%d: PtlMDBind failed: %d\n",
507 __FILE__, __LINE__, ret);
508 goto error;
509 }
510
511 module->origin_iovec_list = NULL;
512 module->origin_iovec_md_h = PTL_INVALID_HANDLE;
513 module->result_iovec_list = NULL;
514 module->result_iovec_md_h = PTL_INVALID_HANDLE;
515
516 if (MPI_WIN_FLAVOR_DYNAMIC == flavor) {
517 me.start = 0;
518 me.length = PTL_SIZE_MAX;
519 } else {
520 me.start = *base;
521 me.length = size;
522 }
523 me.ct_handle = PTL_CT_NONE;
524 me.uid = mca_osc_portals4_component.uid;
525 me.options = PTL_ME_OP_PUT | PTL_ME_OP_GET | PTL_ME_NO_TRUNCATE | PTL_ME_EVENT_SUCCESS_DISABLE;
526 me.match_id.phys.nid = PTL_NID_ANY;
527 me.match_id.phys.pid = PTL_PID_ANY;
528 me.match_bits = module->comm->c_contextid;
529 me.ignore_bits = 0;
530
531 ret = PtlMEAppend(module->ni_h,
532 module->pt_idx,
533 &me,
534 PTL_PRIORITY_LIST,
535 &module->ct_link,
536 &module->data_me_h);
537 if (PTL_OK != ret) {
538 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
539 "%s:%d: PtlMEAppend failed: %d\n",
540 __FILE__, __LINE__, ret);
541 goto error;
542 }
543
544 me.start = &module->state;
545 me.length = sizeof(module->state);
546 me.ct_handle = PTL_CT_NONE;
547 me.uid = mca_osc_portals4_component.uid;
548 me.options = PTL_ME_OP_PUT | PTL_ME_OP_GET | PTL_ME_NO_TRUNCATE | PTL_ME_EVENT_SUCCESS_DISABLE;
549 me.match_id.phys.nid = PTL_NID_ANY;
550 me.match_id.phys.pid = PTL_PID_ANY;
551 me.match_bits = module->comm->c_contextid | OSC_PORTALS4_MB_CONTROL;
552 me.ignore_bits = 0;
553
554 ret = PtlMEAppend(module->ni_h,
555 module->pt_idx,
556 &me,
557 PTL_PRIORITY_LIST,
558 &module->ct_link,
559 &module->control_me_h);
560 if (PTL_OK != ret) {
561 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
562 "%s:%d: PtlMEAppend failed: %d\n",
563 __FILE__, __LINE__, ret);
564 goto error;
565 }
566
567 module->opcount = 0;
568 module->match_bits = module->comm->c_contextid;
569 module->atomic_max = (check_config_value_equal("accumulate_ordering", info, "none")) ?
570 mca_osc_portals4_component.matching_atomic_max :
571 MIN(mca_osc_portals4_component.matching_atomic_max,
572 mca_osc_portals4_component.matching_atomic_ordered_size);
573 module->fetch_atomic_max = (check_config_value_equal("accumulate_ordering", info, "none")) ?
574 mca_osc_portals4_component.matching_fetch_atomic_max :
575 MIN(mca_osc_portals4_component.matching_fetch_atomic_max,
576 mca_osc_portals4_component.matching_atomic_ordered_size);
577
578 module->zero = 0;
579 module->one = 1;
580 module->start_group = NULL;
581 module->post_group = NULL;
582
583 module->state.post_count = 0;
584 module->state.complete_count = 0;
585 if (check_config_value_bool("no_locks", info)) {
586 module->state.lock = LOCK_ILLEGAL;
587 } else {
588 module->state.lock = LOCK_UNLOCKED;
589 }
590
591 OBJ_CONSTRUCT(&module->outstanding_locks, opal_list_t);
592
593 module->passive_target_access_epoch = false;
594
595 #if OPAL_ASSEMBLY_ARCH == OPAL_X86_64 || OPAL_ASSEMBLY_ARCH == OPAL_IA32
596 *model = MPI_WIN_UNIFIED;
597 #else
598 *model = MPI_WIN_SEPARATE;
599 #endif
600
601 win->w_osc_module = &module->super;
602
603 PtlAtomicSync();
604
605
606 OPAL_THREAD_LOCK(&mca_osc_portals4_component.lock);
607 while (module->ct_link != 2) {
608 opal_condition_wait(&mca_osc_portals4_component.cond,
609 &mca_osc_portals4_component.lock);
610 }
611 OPAL_THREAD_UNLOCK(&mca_osc_portals4_component.lock);
612
613 module->comm->c_coll->coll_barrier(module->comm,
614 module->comm->c_coll->coll_barrier_module);
615
616 return OMPI_SUCCESS;
617
618 error:
619
620 if (0 != module->ct_h) PtlCTFree(module->ct_h);
621 if (0 != module->data_me_h) PtlMEUnlink(module->data_me_h);
622 if (0 != module->req_md_h) PtlMDRelease(module->req_md_h);
623 if (0 != module->md_h) PtlMDRelease(module->md_h);
624 if (NULL != module->comm) ompi_comm_free(&module->comm);
625 if (NULL != module) free(module);
626
627 return ret;
628 }
629
630
631 int
632 ompi_osc_portals4_attach(struct ompi_win_t *win, void *base, size_t len)
633 {
634 return OMPI_SUCCESS;
635 }
636
637
638 int
639 ompi_osc_portals4_detach(struct ompi_win_t *win, const void *base)
640 {
641 return OMPI_SUCCESS;
642 }
643
644
645 int
646 ompi_osc_portals4_free(struct ompi_win_t *win)
647 {
648 ompi_osc_portals4_module_t *module =
649 (ompi_osc_portals4_module_t*) win->w_osc_module;
650 int ret = OMPI_SUCCESS;
651
652
653 module->comm->c_coll->coll_barrier(module->comm,
654 module->comm->c_coll->coll_barrier_module);
655
656
657 PtlMEUnlink(module->control_me_h);
658 PtlMEUnlink(module->data_me_h);
659 PtlMDRelease(module->md_h);
660 if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) {
661 PtlMDRelease(module->origin_iovec_md_h);
662 free(module->origin_iovec_list);
663 }
664 if (module->result_iovec_md_h != PTL_INVALID_HANDLE) {
665 PtlMDRelease(module->result_iovec_md_h);
666 free(module->result_iovec_list);
667 }
668 PtlMDRelease(module->req_md_h);
669 PtlCTFree(module->ct_h);
670 if (NULL != module->disp_units) free(module->disp_units);
671 ompi_comm_free(&module->comm);
672 if (NULL != module->free_after) free(module->free_after);
673
674 if (!opal_list_is_empty(&module->outstanding_locks)) {
675 ret = OMPI_ERR_RMA_SYNC;
676 }
677 OBJ_DESTRUCT(&module->outstanding_locks);
678
679 free(module);
680
681 return ret;
682 }