This source file includes following definitions.
- mca_pml_ob1_param_register_int
- mca_pml_ob1_param_register_uint
- mca_pml_ob1_param_register_sizet
- mca_pml_ob1_comm_size_notify
- mca_pml_ob1_get_unex_msgq_size
- mca_pml_ob1_get_posted_recvq_size
- mca_pml_ob1_component_register
- mca_pml_ob1_component_open
- mca_pml_ob1_component_close
- mca_pml_ob1_component_init
- mca_pml_ob1_component_fini
- mca_pml_ob1_seg_alloc
- mca_pml_ob1_seg_free
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28 #include "ompi_config.h"
29 #include "opal/mca/event/event.h"
30 #include "mpi.h"
31 #include "ompi/runtime/params.h"
32 #include "ompi/mca/pml/pml.h"
33 #include "ompi/mca/pml/base/pml_base_bsend.h"
34 #include "pml_ob1.h"
35 #include "pml_ob1_hdr.h"
36 #include "pml_ob1_sendreq.h"
37 #include "pml_ob1_recvreq.h"
38 #include "pml_ob1_rdmafrag.h"
39 #include "pml_ob1_recvfrag.h"
40 #include "ompi/mca/bml/base/base.h"
41 #include "pml_ob1_component.h"
42 #include "opal/mca/allocator/base/base.h"
43 #include "opal/mca/base/mca_base_pvar.h"
44 #include "opal/runtime/opal_params.h"
45 #include "opal/mca/btl/base/base.h"
46
47 OBJ_CLASS_INSTANCE( mca_pml_ob1_pckt_pending_t,
48 opal_free_list_item_t,
49 NULL,
50 NULL );
51
52 static int mca_pml_ob1_component_register(void);
53 static int mca_pml_ob1_component_open(void);
54 static int mca_pml_ob1_component_close(void);
55 static mca_pml_base_module_t*
56 mca_pml_ob1_component_init( int* priority, bool enable_progress_threads,
57 bool enable_mpi_threads );
58 static int mca_pml_ob1_component_fini(void);
59 int mca_pml_ob1_output = 0;
60 static int mca_pml_ob1_verbose = 0;
61 bool mca_pml_ob1_matching_protection = false;
62
63 mca_pml_base_component_2_0_0_t mca_pml_ob1_component = {
64
65
66
67 .pmlm_version = {
68 MCA_PML_BASE_VERSION_2_0_0,
69
70 .mca_component_name = "ob1",
71 .mca_component_major_version = OMPI_MAJOR_VERSION,
72 .mca_component_minor_version = OMPI_MINOR_VERSION,
73 .mca_component_release_version = OMPI_RELEASE_VERSION,
74 .mca_open_component = mca_pml_ob1_component_open,
75 .mca_close_component = mca_pml_ob1_component_close,
76 .mca_register_component_params = mca_pml_ob1_component_register
77 },
78 .pmlm_data = {
79
80 MCA_BASE_METADATA_PARAM_CHECKPOINT
81 },
82
83 .pmlm_init = mca_pml_ob1_component_init,
84 .pmlm_finalize = mca_pml_ob1_component_fini,
85 };
86
87 void *mca_pml_ob1_seg_alloc (void *ctx, size_t* size);
88
89 void mca_pml_ob1_seg_free (void *ctx, void *segment);
90
91 static inline int mca_pml_ob1_param_register_int(
92 const char* param_name,
93 int default_value,
94 int *storage)
95 {
96 *storage = default_value;
97 (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, param_name,
98 NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
99 OPAL_INFO_LVL_9,
100 MCA_BASE_VAR_SCOPE_READONLY, storage);
101 return *storage;
102 }
103
104 static inline unsigned int mca_pml_ob1_param_register_uint(
105 const char* param_name,
106 unsigned int default_value,
107 unsigned int *storage)
108 {
109 *storage = default_value;
110 (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, param_name,
111 NULL, MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
112 OPAL_INFO_LVL_9,
113 MCA_BASE_VAR_SCOPE_READONLY, storage);
114 return *storage;
115 }
116
117 #if 0
118 static inline size_t mca_pml_ob1_param_register_sizet(
119 const char* param_name,
120 size_t default_value,
121 size_t *storage)
122 {
123 *storage = default_value;
124 (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, param_name,
125 NULL, MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
126 OPAL_INFO_LVL_9,
127 MCA_BASE_VAR_SCOPE_READONLY, storage);
128 return *storage;
129 }
130 #endif
131
132 static int mca_pml_ob1_comm_size_notify (mca_base_pvar_t *pvar, mca_base_pvar_event_t event, void *obj_handle, int *count)
133 {
134 if (MCA_BASE_PVAR_HANDLE_BIND == event) {
135
136 *count = ompi_comm_size ((ompi_communicator_t *) obj_handle);
137 }
138
139 return OMPI_SUCCESS;
140 }
141
142 static int mca_pml_ob1_get_unex_msgq_size (const struct mca_base_pvar_t *pvar, void *value, void *obj_handle)
143 {
144 ompi_communicator_t *comm = (ompi_communicator_t *) obj_handle;
145 mca_pml_ob1_comm_t *pml_comm = comm->c_pml_comm;
146 int comm_size = ompi_comm_size (comm);
147 unsigned *values = (unsigned *) value;
148 mca_pml_ob1_comm_proc_t *pml_proc;
149 int i;
150
151 for (i = 0 ; i < comm_size ; ++i) {
152 pml_proc = pml_comm->procs[i];
153 if (pml_proc) {
154 #if MCA_PML_OB1_CUSTOM_MATCH
155 values[i] = custom_match_umq_size(pml_comm->umq);
156
157 #else
158 values[i] = opal_list_get_size (&pml_proc->unexpected_frags);
159 #endif
160 } else {
161 values[i] = 0;
162 }
163 }
164
165 return OMPI_SUCCESS;
166 }
167
168 static int mca_pml_ob1_get_posted_recvq_size (const struct mca_base_pvar_t *pvar, void *value, void *obj_handle)
169 {
170 ompi_communicator_t *comm = (ompi_communicator_t *) obj_handle;
171 mca_pml_ob1_comm_t *pml_comm = comm->c_pml_comm;
172 int comm_size = ompi_comm_size (comm);
173 unsigned *values = (unsigned *) value;
174 mca_pml_ob1_comm_proc_t *pml_proc;
175 int i;
176
177 for (i = 0 ; i < comm_size ; ++i) {
178 pml_proc = pml_comm->procs[i];
179
180 if (pml_proc) {
181 #if MCA_PML_OB1_CUSTOM_MATCH
182 values[i] = custom_match_prq_size(pml_comm->prq);
183
184 #else
185 values[i] = opal_list_get_size (&pml_proc->specific_receives);
186 #endif
187 } else {
188 values[i] = 0;
189 }
190 }
191
192 return OMPI_SUCCESS;
193 }
194
195 static int mca_pml_ob1_component_register(void)
196 {
197 mca_pml_ob1_param_register_int("verbose", 0, &mca_pml_ob1_verbose);
198
199 mca_pml_ob1_param_register_int("free_list_num", 4, &mca_pml_ob1.free_list_num);
200 mca_pml_ob1_param_register_int("free_list_max", -1, &mca_pml_ob1.free_list_max);
201 mca_pml_ob1_param_register_int("free_list_inc", 64, &mca_pml_ob1.free_list_inc);
202 mca_pml_ob1_param_register_int("priority", 20, &mca_pml_ob1.priority);
203 mca_pml_ob1_param_register_int("send_pipeline_depth", 3, &mca_pml_ob1.send_pipeline_depth);
204 mca_pml_ob1_param_register_int("recv_pipeline_depth", 4, &mca_pml_ob1.recv_pipeline_depth);
205
206
207
208
209 mca_pml_ob1.rdma_retries_limit = 0;
210
211
212 mca_pml_ob1_param_register_int("max_rdma_per_request", 4, &mca_pml_ob1.max_rdma_per_request);
213 mca_pml_ob1_param_register_int("max_send_per_range", 4, &mca_pml_ob1.max_send_per_range);
214
215 mca_pml_ob1_param_register_uint("unexpected_limit", 128, &mca_pml_ob1.unexpected_limit);
216
217 mca_pml_ob1.use_all_rdma = false;
218 (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, "use_all_rdma",
219 "Use all available RDMA btls for the RDMA and RDMA pipeline protocols "
220 "(default: false)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
221 OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_GROUP, &mca_pml_ob1.use_all_rdma);
222
223 mca_pml_ob1.allocator_name = "bucket";
224 (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, "allocator",
225 "Name of allocator component for unexpected messages",
226 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9,
227 MCA_BASE_VAR_SCOPE_READONLY, &mca_pml_ob1.allocator_name);
228 (void)mca_base_component_pvar_register(&mca_pml_ob1_component.pmlm_version,
229 "unexpected_msgq_length", "Number of unexpected messages "
230 "received by each peer in a communicator", OPAL_INFO_LVL_4, MPI_T_PVAR_CLASS_SIZE,
231 MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, MPI_T_BIND_MPI_COMM,
232 MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS,
233 mca_pml_ob1_get_unex_msgq_size, NULL, mca_pml_ob1_comm_size_notify, NULL);
234
235 (void)mca_base_component_pvar_register(&mca_pml_ob1_component.pmlm_version,
236 "posted_recvq_length", "Number of unmatched receives "
237 "posted for each peer in a communicator", OPAL_INFO_LVL_4, MPI_T_PVAR_CLASS_SIZE,
238 MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, MPI_T_BIND_MPI_COMM,
239 MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS,
240 mca_pml_ob1_get_posted_recvq_size, NULL, mca_pml_ob1_comm_size_notify, NULL);
241
242 return OMPI_SUCCESS;
243 }
244
245 static int mca_pml_ob1_component_open(void)
246 {
247 mca_pml_ob1_output = opal_output_open(NULL);
248 opal_output_set_verbosity(mca_pml_ob1_output, mca_pml_ob1_verbose);
249
250 mca_pml_ob1.enabled = false;
251 return mca_base_framework_open(&ompi_bml_base_framework, 0);
252 }
253
254
255 static int mca_pml_ob1_component_close(void)
256 {
257 int rc;
258
259 if (OMPI_SUCCESS != (rc = mca_base_framework_close(&ompi_bml_base_framework))) {
260 return rc;
261 }
262 opal_output_close(mca_pml_ob1_output);
263
264 return OMPI_SUCCESS;
265 }
266
267
268 static mca_pml_base_module_t*
269 mca_pml_ob1_component_init( int* priority,
270 bool enable_progress_threads,
271 bool enable_mpi_threads )
272 {
273 mca_allocator_base_component_t* allocator_component;
274
275 opal_output_verbose( 10, mca_pml_ob1_output,
276 "in ob1, my priority is %d\n", mca_pml_ob1.priority);
277
278 *priority = mca_pml_ob1.priority;
279
280 allocator_component = mca_allocator_component_lookup( mca_pml_ob1.allocator_name );
281 if(NULL == allocator_component) {
282 opal_output(0, "mca_pml_ob1_component_init: can't find allocator: %s\n", mca_pml_ob1.allocator_name);
283 return NULL;
284 }
285
286 mca_pml_ob1.allocator = allocator_component->allocator_init(true,
287 mca_pml_ob1_seg_alloc,
288 mca_pml_ob1_seg_free, NULL);
289 if(NULL == mca_pml_ob1.allocator) {
290 opal_output(0, "mca_pml_ob1_component_init: unable to initialize allocator\n");
291 return NULL;
292 }
293
294 if(OMPI_SUCCESS != mca_bml_base_init( enable_progress_threads,
295 enable_mpi_threads)) {
296 return NULL;
297 }
298
299
300 mca_btl_base_selected_module_t* selected_btl;
301 OPAL_LIST_FOREACH(selected_btl, &mca_btl_base_modules_initialized, mca_btl_base_selected_module_t) {
302 mca_btl_base_module_t *btl = selected_btl->btl_module;
303
304 if (btl->btl_flags & MCA_BTL_FLAGS_BTL_PROGRESS_THREAD_ENABLED) {
305 mca_pml_ob1_matching_protection = true;
306 }
307
308 if (btl->btl_flags & MCA_BTL_FLAGS_SINGLE_ADD_PROCS) {
309 mca_pml_ob1.super.pml_flags |= MCA_PML_BASE_FLAG_REQUIRE_WORLD;
310 break;
311 }
312
313 }
314
315 return &mca_pml_ob1.super;
316 }
317
318 int mca_pml_ob1_component_fini(void)
319 {
320 int rc;
321
322
323 if(OMPI_SUCCESS != (rc = mca_bml.bml_finalize()))
324 return rc;
325
326 if(!mca_pml_ob1.enabled) {
327 if( NULL != mca_pml_ob1.allocator ) {
328 (void)mca_pml_ob1.allocator->alc_finalize(mca_pml_ob1.allocator);
329 mca_pml_ob1.allocator = NULL;
330 }
331
332 return OMPI_SUCCESS;
333 }
334 mca_pml_ob1.enabled = false;
335
336
337
338 if( NULL != mca_pml_ob1_recvreq ) {
339 opal_free_list_return (&mca_pml_base_recv_requests, (opal_free_list_item_t *) mca_pml_ob1_recvreq);
340 mca_pml_ob1_recvreq = NULL;
341 }
342
343 if( NULL != mca_pml_ob1_sendreq ) {
344 opal_free_list_return (&mca_pml_base_send_requests, (opal_free_list_item_t *) mca_pml_ob1_sendreq);
345 mca_pml_ob1_sendreq = NULL;
346 }
347
348 OBJ_DESTRUCT(&mca_pml_ob1.rdma_pending);
349 OBJ_DESTRUCT(&mca_pml_ob1.pckt_pending);
350 OBJ_DESTRUCT(&mca_pml_ob1.recv_pending);
351 OBJ_DESTRUCT(&mca_pml_ob1.send_pending);
352 OBJ_DESTRUCT(&mca_pml_ob1.non_existing_communicator_pending);
353 OBJ_DESTRUCT(&mca_pml_ob1.buffers);
354 OBJ_DESTRUCT(&mca_pml_ob1.pending_pckts);
355 OBJ_DESTRUCT(&mca_pml_ob1.recv_frags);
356 OBJ_DESTRUCT(&mca_pml_ob1.rdma_frags);
357 OBJ_DESTRUCT(&mca_pml_ob1.lock);
358 OBJ_DESTRUCT(&mca_pml_ob1.send_ranges);
359
360 if( NULL != mca_pml_ob1.allocator ) {
361 (void)mca_pml_ob1.allocator->alc_finalize(mca_pml_ob1.allocator);
362 mca_pml_ob1.allocator = NULL;
363 }
364
365 #if 0
366 if (mca_pml_base_send_requests.fl_num_allocated !=
367 mca_pml_base_send_requests.super.opal_list_length) {
368 opal_output(0, "ob1 send requests: %d allocated %d returned\n",
369 mca_pml_base_send_requests.fl_num_allocated,
370 mca_pml_base_send_requests.super.opal_list_length);
371 }
372 if (mca_pml_base_recv_requests.fl_num_allocated !=
373 mca_pml_base_recv_requests.super.opal_list_length) {
374 opal_output(0, "ob1 recv requests: %d allocated %d returned\n",
375 mca_pml_base_recv_requests.fl_num_allocated,
376 mca_pml_base_recv_requests.super.opal_list_length);
377 }
378 #endif
379
380 return OMPI_SUCCESS;
381 }
382
383 void *mca_pml_ob1_seg_alloc (void *ctx, size_t *size)
384 {
385 return malloc(*size);
386 }
387
388 void mca_pml_ob1_seg_free (void *ctx, void *segment)
389 {
390 free(segment);
391 }