This source file includes following definitions.
- mca_btl_ugni_add_procs
- mca_btl_ugni_del_procs
- mca_btl_ugni_get_ep
- ugni_reg_mem
- ugni_dereg_mem
- mca_btl_ugni_setup_mpools
- mca_btl_ugni_module_set_max_reg
- mca_btl_ugni_smsg_setup
1
2
3
4
5
6
7
8
9
10
11
12
13
14 #include "opal_config.h"
15
16 #include "btl_ugni.h"
17 #include "btl_ugni_frag.h"
18 #include "btl_ugni_smsg.h"
19
20 #include "opal/include/opal/align.h"
21 #include "opal/mca/pmix/pmix.h"
22
23 #define INITIAL_GNI_EPS 1024
24
25 static int
26 mca_btl_ugni_setup_mpools (mca_btl_ugni_module_t *ugni_module);
27 static void
28 mca_btl_ugni_module_set_max_reg (mca_btl_ugni_module_t *ugni_module, int nlocal_procs);
29 static int mca_btl_ugni_smsg_setup (int nprocs);
30
31 int mca_btl_ugni_add_procs (struct mca_btl_base_module_t* btl, size_t nprocs,
32 struct opal_proc_t **procs,
33 struct mca_btl_base_endpoint_t **peers,
34 opal_bitmap_t *reachable) {
35 mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
36 int rc;
37 void *mmap_start_addr;
38 struct timeval tv = {.tv_sec = 0, .tv_usec = MCA_BTL_UGNI_CONNECT_USEC};
39
40 if (false == ugni_module->initialized) {
41
42
43
44 rc = opal_pointer_array_init (&ugni_module->endpoints, INITIAL_GNI_EPS, 1 << 24, 512);
45 if (OPAL_SUCCESS != rc) {
46 BTL_ERROR(("error inializing the endpoint array. rc = %d", rc));
47 return rc;
48 }
49
50
51
52
53
54 rc = opal_hash_table_init (&ugni_module->id_to_endpoint, INITIAL_GNI_EPS);
55 if (OPAL_SUCCESS != rc) {
56 BTL_ERROR(("error initializing the endpoint hash. rc = %d", rc));
57 return rc;
58 }
59 }
60
61 for (size_t i = 0 ; i < nprocs ; ++i) {
62 peers[i] = mca_btl_ugni_get_ep (btl, procs[i]);
63 if (NULL == peers[i]) {
64 continue;
65 }
66
67 if (procs[i] == opal_proc_local_get ()) {
68 ugni_module->local_ep = peers[i];
69 }
70
71
72 if (reachable) {
73 (void) opal_bitmap_set_bit (reachable, i);
74 }
75 }
76
77 mca_btl_ugni_module_set_max_reg (ugni_module, ugni_module->nlocal_procs);
78
79 if (false == ugni_module->initialized) {
80 for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
81 mca_btl_ugni_device_t *device = ugni_module->devices + i;
82 rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_rdma_cq_size, 0,
83 GNI_CQ_NOBLOCK, NULL, NULL, &device->dev_rdma_local_cq.gni_handle);
84 if (GNI_RC_SUCCESS != rc) {
85 BTL_ERROR(("error creating local BTE/FMA CQ"));
86 return mca_btl_rc_ugni_to_opal (rc);
87 }
88
89 rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_cq_size,
90 0, GNI_CQ_NOBLOCK, NULL, NULL, &device->dev_smsg_local_cq.gni_handle);
91 if (GNI_RC_SUCCESS != rc) {
92 BTL_ERROR(("error creating local SMSG CQ"));
93 return mca_btl_rc_ugni_to_opal (rc);
94 }
95
96 if (mca_btl_ugni_component.progress_thread_enabled) {
97 rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_rdma_cq_size,
98 0, GNI_CQ_BLOCKING, NULL, NULL, &device->dev_rdma_local_irq_cq.gni_handle);
99 if (GNI_RC_SUCCESS != rc) {
100 BTL_ERROR(("error creating local BTE/FMA CQ"));
101 return mca_btl_rc_ugni_to_opal (rc);
102 }
103 }
104 }
105
106 rc = GNI_CqCreate (ugni_module->devices[0].dev_handle, mca_btl_ugni_component.remote_cq_size,
107 0, GNI_CQ_NOBLOCK, NULL, NULL, &ugni_module->smsg_remote_cq);
108 if (GNI_RC_SUCCESS != rc) {
109 BTL_ERROR(("error creating remote SMSG CQ"));
110 return mca_btl_rc_ugni_to_opal (rc);
111 }
112
113 if (mca_btl_ugni_component.progress_thread_enabled) {
114 rc = GNI_CqCreate (ugni_module->devices[0].dev_handle, mca_btl_ugni_component.remote_cq_size,
115 0, GNI_CQ_BLOCKING, NULL, NULL, &ugni_module->smsg_remote_irq_cq);
116 if (GNI_RC_SUCCESS != rc) {
117 BTL_ERROR(("error creating remote SMSG CQ"));
118 return mca_btl_rc_ugni_to_opal (rc);
119 }
120 }
121
122 rc = mca_btl_ugni_setup_mpools (ugni_module);
123 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
124 BTL_ERROR(("btl/ugni error setting up mpools/free lists"));
125 return rc;
126 }
127
128 rc = mca_btl_ugni_smsg_init (ugni_module);
129 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
130 BTL_ERROR(("btl/ugni error initializing SMSG"));
131 return rc;
132 }
133
134
135
136
137
138
139
140
141
142 if (mca_btl_ugni_component.progress_thread_enabled) {
143 mmap_start_addr = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
144 if (NULL == mmap_start_addr) {
145 BTL_ERROR(("btl/ugni mmap returned error"));
146 return OPAL_ERR_OUT_OF_RESOURCE;
147 }
148
149 rc = GNI_MemRegister(ugni_module->devices[0].dev_handle,
150 (unsigned long)mmap_start_addr,
151 4096,
152 ugni_module->smsg_remote_irq_cq,
153 GNI_MEM_READWRITE,
154 -1,
155 &ugni_module->devices[0].smsg_irq_mhndl);
156
157 mca_btl_ugni_spawn_progress_thread(btl);
158 }
159
160 opal_event_evtimer_add (&ugni_module->connection_event, &tv);
161
162 ugni_module->initialized = true;
163 }
164
165 return OPAL_SUCCESS;
166 }
167
168 int mca_btl_ugni_del_procs (struct mca_btl_base_module_t *btl,
169 size_t nprocs, struct opal_proc_t **procs,
170 struct mca_btl_base_endpoint_t **peers) {
171 mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
172
173 OPAL_THREAD_LOCK(&ugni_module->endpoint_lock);
174
175 for (size_t i = 0 ; i < nprocs ; ++i) {
176 struct opal_proc_t *opal_proc = procs[i];
177 uint64_t proc_id = mca_btl_ugni_proc_name_to_id(opal_proc->proc_name);
178 mca_btl_base_endpoint_t *ep = NULL;
179
180
181 (void) opal_hash_table_get_value_uint64 (&ugni_module->id_to_endpoint, proc_id, (void **) &ep);
182
183 BTL_VERBOSE(("deleting endpoint with proc id 0x%" PRIx64 ", ptr: %p", proc_id, (void *) ep));
184
185 if (NULL != ep) {
186 mca_btl_ugni_release_ep (ep);
187 --ugni_module->endpoint_count;
188 }
189
190 if (OPAL_PROC_ON_LOCAL_NODE(opal_proc->proc_flags)) {
191 --ugni_module->nlocal_procs;
192 }
193
194
195 opal_hash_table_set_value_uint64 (&ugni_module->id_to_endpoint, proc_id, NULL);
196 }
197
198 OPAL_THREAD_UNLOCK(&ugni_module->endpoint_lock);
199
200 mca_btl_ugni_module_set_max_reg (ugni_module, ugni_module->nlocal_procs);
201
202 return OPAL_SUCCESS;
203 }
204
205
206 struct mca_btl_base_endpoint_t *mca_btl_ugni_get_ep (struct mca_btl_base_module_t *module, opal_proc_t *proc)
207 {
208 mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) module;
209 uint64_t proc_id = mca_btl_ugni_proc_name_to_id(proc->proc_name);
210 mca_btl_base_endpoint_t *ep;
211 int rc;
212
213 OPAL_THREAD_LOCK(&ugni_module->endpoint_lock);
214
215 do {
216 rc = opal_hash_table_get_value_uint64 (&ugni_module->id_to_endpoint, proc_id, (void **) &ep);
217 if (OPAL_SUCCESS == rc) {
218 BTL_VERBOSE(("returning existing endpoint for proc %s", OPAL_NAME_PRINT(proc->proc_name)));
219 break;
220 }
221
222 BTL_VERBOSE(("initialized uGNI endpoint for proc id: 0x%" PRIx64 " ptr: %p", proc_id, (void *) proc));
223
224
225 rc = mca_btl_ugni_init_ep (ugni_module, &ep, ugni_module, proc);
226 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
227 BTL_ERROR(("btl/ugni error initializing endpoint"));
228 break;
229 }
230
231
232 if (OPAL_PROC_ON_LOCAL_NODE(proc->proc_flags)) {
233 ++ugni_module->nlocal_procs;
234 }
235 ++ugni_module->endpoint_count;
236
237
238 opal_hash_table_set_value_uint64 (&ugni_module->id_to_endpoint, proc_id, ep);
239 } while (0);
240
241 OPAL_THREAD_UNLOCK(&ugni_module->endpoint_lock);
242
243 return ep;
244 }
245
246
247 static int ugni_reg_mem (void *reg_data, void *base, size_t size,
248 mca_rcache_base_registration_t *reg)
249 {
250 mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) reg_data;
251 gni_cq_handle_t cq = 0;
252 int flags, rc;
253
254 if (ugni_module->reg_count >= ugni_module->reg_max) {
255 return OPAL_ERR_OUT_OF_RESOURCE;
256 }
257
258 if (reg->access_flags & (MCA_RCACHE_ACCESS_REMOTE_WRITE | MCA_RCACHE_ACCESS_LOCAL_WRITE |
259 MCA_RCACHE_ACCESS_REMOTE_ATOMIC)) {
260 flags = GNI_MEM_READWRITE;
261 } else {
262 flags = GNI_MEM_READ_ONLY;
263 }
264
265 if (!(reg->flags & MCA_RCACHE_FLAGS_SO_MEM)) {
266 flags |= GNI_MEM_RELAXED_PI_ORDERING;
267 }
268
269 if (reg->flags & MCA_RCACHE_FLAGS_RESV0) {
270 cq = ugni_module->smsg_remote_cq;
271 }
272
273 rc = mca_btl_ugni_reg_mem (ugni_module, base, size, (mca_btl_ugni_reg_t *) reg, cq, flags);
274 if (OPAL_LIKELY(OPAL_SUCCESS == rc)) {
275 opal_atomic_add_fetch_32(&ugni_module->reg_count,1);
276 }
277
278 return rc;
279 }
280
281 static int
282 ugni_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg)
283 {
284 mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) reg_data;
285 int rc;
286
287 rc = mca_btl_ugni_dereg_mem (ugni_module, (mca_btl_ugni_reg_t *) reg);
288 if (OPAL_LIKELY(OPAL_SUCCESS == rc)) {
289 opal_atomic_add_fetch_32(&ugni_module->reg_count,-1);
290 }
291
292 return rc;
293 }
294
295 static int
296 mca_btl_ugni_setup_mpools (mca_btl_ugni_module_t *ugni_module)
297 {
298 mca_rcache_udreg_resources_t rcache_resources;
299 unsigned int mbox_increment;
300 uint32_t nprocs, *u32;
301 char *rcache_name;
302 int rc;
303
304 rc = opal_pointer_array_init (&ugni_module->pending_smsg_frags_bb, 0,
305 1 << 30, 32768);
306 if (OPAL_SUCCESS != rc) {
307 return rc;
308 }
309
310
311 u32 = &nprocs;
312 OPAL_MODEX_RECV_VALUE(rc, OPAL_PMIX_UNIV_SIZE, &OPAL_PROC_MY_NAME,
313 &u32, OPAL_UINT32);
314 if (OPAL_SUCCESS != rc) {
315
316 nprocs = 512;
317 }
318
319 rc = mca_btl_ugni_smsg_setup (nprocs);
320 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
321 BTL_ERROR(("error setting up smsg"));
322 return rc;
323 }
324
325 rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_SMSG,
326 sizeof (mca_btl_ugni_smsg_frag_t),
327 opal_cache_line_size, OBJ_CLASS(mca_btl_ugni_smsg_frag_t),
328 mca_btl_ugni_component.ugni_smsg_limit,
329 opal_cache_line_size,
330 mca_btl_ugni_component.ugni_free_list_num,
331 mca_btl_ugni_component.ugni_free_list_max,
332 mca_btl_ugni_component.ugni_free_list_inc,
333 NULL, 0, NULL, (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
334 (void *) (intptr_t) MCA_BTL_UGNI_LIST_SMSG);
335 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
336 BTL_ERROR(("error creating smsg fragment free list"));
337 return rc;
338 }
339
340 rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_RDMA,
341 sizeof (mca_btl_ugni_rdma_frag_t), 64,
342 OBJ_CLASS(mca_btl_ugni_rdma_frag_t),
343 0, opal_cache_line_size,
344 mca_btl_ugni_component.ugni_free_list_num,
345 mca_btl_ugni_component.ugni_free_list_max,
346 mca_btl_ugni_component.ugni_free_list_inc,
347 NULL, 0, NULL, (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
348 (void *) (intptr_t) MCA_BTL_UGNI_LIST_RDMA);
349 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
350 return rc;
351 }
352
353 rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_RDMA_INT,
354 sizeof (mca_btl_ugni_rdma_frag_t), 8,
355 OBJ_CLASS(mca_btl_ugni_rdma_frag_t),
356 0, opal_cache_line_size, 0, -1, 64,
357 NULL, 0, NULL, (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
358 (void *) (intptr_t) MCA_BTL_UGNI_LIST_RDMA_INT);
359 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
360 return rc;
361 }
362
363 ugni_module->super.btl_mpool = mca_mpool_base_module_lookup (mca_btl_ugni_component.mpool_hints);
364 if (NULL == ugni_module->super.btl_mpool) {
365 BTL_ERROR(("could not find mpool matching hints %s", mca_btl_ugni_component.mpool_hints));
366 return OPAL_ERROR;
367 }
368
369 rcache_resources.base.cache_name = "ompi.ugni";
370 rcache_resources.base.reg_data = (void *) ugni_module;
371 rcache_resources.base.sizeof_reg = sizeof (mca_btl_ugni_reg_t);
372 rcache_resources.base.register_mem = ugni_reg_mem;
373 rcache_resources.base.deregister_mem = ugni_dereg_mem;
374
375 if (MCA_BTL_UGNI_RCACHE_UDREG == mca_btl_ugni_component.rcache_type) {
376
377
378 rcache_resources.max_entries = 4096;
379 rcache_resources.use_kernel_cache = true;
380
381 rcache_resources.use_evict_w_unreg = false;
382 rcache_name = "udreg";
383 } else {
384 rcache_name = "grdma";
385 }
386
387 ugni_module->rcache =
388 mca_rcache_base_module_create (rcache_name, ugni_module->devices, &rcache_resources.base);
389
390 if (NULL == ugni_module->rcache) {
391 BTL_ERROR(("error creating registration cache"));
392 return OPAL_ERROR;
393 }
394
395 rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_EAGER_SEND,
396 sizeof (mca_btl_ugni_eager_frag_t), 8,
397 OBJ_CLASS(mca_btl_ugni_eager_frag_t),
398 ugni_module->super.btl_eager_limit, 64,
399 mca_btl_ugni_component.ugni_eager_num,
400 mca_btl_ugni_component.ugni_eager_max,
401 mca_btl_ugni_component.ugni_eager_inc,
402 ugni_module->super.btl_mpool, 0, ugni_module->rcache,
403 (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
404 (void *) (intptr_t) MCA_BTL_UGNI_LIST_EAGER_SEND);
405 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
406 BTL_ERROR(("error creating eager send fragment free list"));
407 return rc;
408 }
409
410 rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_EAGER_RECV,
411 sizeof (mca_btl_ugni_eager_frag_t), 8,
412 OBJ_CLASS(mca_btl_ugni_eager_frag_t),
413 ugni_module->super.btl_eager_limit, 64,
414 mca_btl_ugni_component.ugni_eager_num,
415 mca_btl_ugni_component.ugni_eager_max,
416 mca_btl_ugni_component.ugni_eager_inc,
417 ugni_module->super.btl_mpool, 0, ugni_module->rcache,
418 (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
419 (void *) (intptr_t) MCA_BTL_UGNI_LIST_EAGER_RECV);
420 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
421 BTL_ERROR(("error creating eager receive fragment free list"));
422 return rc;
423 }
424
425 if (0 == mca_btl_ugni_component.mbox_increment) {
426
427
428 mbox_increment = (unsigned int) (2097152.0 / (float)mca_btl_ugni_component.smsg_mbox_size);
429
430
431 if (nprocs/mbox_increment > (unsigned int) ugni_module->reg_max / 8) {
432 mbox_increment = nprocs / (ugni_module->reg_max >> 3);
433 }
434 } else {
435 mbox_increment = mca_btl_ugni_component.mbox_increment;
436 }
437
438
439 rc = opal_free_list_init (&ugni_module->smsg_mboxes,
440 sizeof (mca_btl_ugni_smsg_mbox_t), 8,
441 OBJ_CLASS(mca_btl_ugni_smsg_mbox_t),
442 mca_btl_ugni_component.smsg_mbox_size, 128,
443 32, -1, mbox_increment, ugni_module->super.btl_mpool,
444 MCA_RCACHE_FLAGS_SO_MEM | MCA_RCACHE_FLAGS_RESV0,
445 ugni_module->rcache, NULL, NULL);
446 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
447 BTL_ERROR(("error creating smsg mailbox free list"));
448 return rc;
449 }
450
451 return OPAL_SUCCESS;
452 }
453
454 static void
455 mca_btl_ugni_module_set_max_reg (mca_btl_ugni_module_t *ugni_module, int nlocal_procs)
456 {
457 if (0 == mca_btl_ugni_component.max_mem_reg) {
458 #if defined(HAVE_GNI_GETJOBRESINFO)
459 gni_job_res_desc_t res_des;
460 gni_return_t grc;
461 int fuzz = 20;
462
463 grc = GNI_GetJobResInfo (0, mca_btl_ugni_component.ptag,
464 GNI_JOB_RES_MDD, &res_des);
465 if (GNI_RC_SUCCESS == grc) {
466 if (nlocal_procs) {
467 ugni_module->reg_max = (res_des.limit - fuzz) / nlocal_procs;
468 } else {
469 ugni_module->reg_max = 0;
470 }
471 }
472 #else
473
474 if (nlocal_procs) {
475 ugni_module->reg_max = 1200 / nlocal_procs;
476 } else {
477 ugni_module->reg_max = 0;
478 }
479 #endif
480 } else if (-1 == mca_btl_ugni_component.max_mem_reg) {
481 ugni_module->reg_max = INT_MAX;
482 } else {
483 ugni_module->reg_max = mca_btl_ugni_component.max_mem_reg;
484 }
485
486 ugni_module->reg_count = 0;
487 }
488
489 static int mca_btl_ugni_smsg_setup (int nprocs)
490 {
491 gni_smsg_attr_t tmp_smsg_attrib;
492 unsigned int mbox_size;
493 gni_return_t grc;
494
495 if (0 == mca_btl_ugni_component.ugni_smsg_limit) {
496
497 if (nprocs <= 512) {
498 mca_btl_ugni_component.ugni_smsg_limit = 8192;
499 } else if (nprocs <= 1024) {
500 mca_btl_ugni_component.ugni_smsg_limit = 2048;
501 } else if (nprocs <= 8192) {
502 mca_btl_ugni_component.ugni_smsg_limit = 1024;
503 } else if (nprocs <= 16384) {
504 mca_btl_ugni_component.ugni_smsg_limit = 512;
505 } else {
506 mca_btl_ugni_component.ugni_smsg_limit = 256;
507 }
508 }
509
510 mca_btl_ugni_component.smsg_max_data = mca_btl_ugni_component.ugni_smsg_limit -
511 sizeof (mca_btl_ugni_send_frag_hdr_t);
512
513 if (mca_btl_ugni_component.ugni_smsg_limit == mca_btl_ugni_module.super.btl_eager_limit) {
514 mca_btl_ugni_module.super.btl_eager_limit = mca_btl_ugni_component.smsg_max_data;
515 }
516
517
518 tmp_smsg_attrib.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
519 tmp_smsg_attrib.msg_maxsize = mca_btl_ugni_component.ugni_smsg_limit;
520 tmp_smsg_attrib.mbox_maxcredit = mca_btl_ugni_component.smsg_max_credits;
521
522 grc = GNI_SmsgBufferSizeNeeded (&tmp_smsg_attrib, &mbox_size);
523 if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc)) {
524 BTL_ERROR(("error in GNI_SmsgBufferSizeNeeded"));
525 return mca_btl_rc_ugni_to_opal (grc);
526 }
527
528 mca_btl_ugni_component.smsg_mbox_size = OPAL_ALIGN(mbox_size, 64, unsigned int);
529
530 return OPAL_SUCCESS;
531 }