wpool 30 ompi/mca/osc/ucx/osc_ucx.h opal_common_ucx_wpool_t *wpool; wpool 146 ompi/mca/osc/ucx/osc_ucx_active_target.c opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool); wpool 275 ompi/mca/osc/ucx/osc_ucx_active_target.c ucp_worker_progress(mca_osc_ucx_component.wpool->dflt_worker); wpool 302 ompi/mca/osc/ucx/osc_ucx_active_target.c opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool); wpool 256 ompi/mca/osc/ucx/osc_ucx_comm.c ucp_worker_progress(mca_osc_ucx_component.wpool->dflt_worker); wpool 57 ompi/mca/osc/ucx/osc_ucx_component.c .wpool = NULL, wpool 129 ompi/mca/osc/ucx/osc_ucx_component.c if (mca_osc_ucx_component.wpool != NULL) { wpool 130 ompi/mca/osc/ucx/osc_ucx_component.c opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool); wpool 137 ompi/mca/osc/ucx/osc_ucx_component.c mca_osc_ucx_component.wpool = opal_common_ucx_wpool_allocate(); wpool 144 ompi/mca/osc/ucx/osc_ucx_component.c opal_common_ucx_wpool_free(mca_osc_ucx_component.wpool); wpool 239 ompi/mca/osc/ucx/osc_ucx_component.c ret = opal_common_ucx_wpool_init(mca_osc_ucx_component.wpool, wpool 307 ompi/mca/osc/ucx/osc_ucx_component.c ret = opal_common_ucx_wpctx_create(mca_osc_ucx_component.wpool, comm_size, wpool 433 ompi/mca/osc/ucx/osc_ucx_component.c opal_common_ucx_wpool_finalize(mca_osc_ucx_component.wpool); wpool 579 ompi/mca/osc/ucx/osc_ucx_component.c opal_common_ucx_wpool_finalize(mca_osc_ucx_component.wpool); wpool 45 ompi/mca/osc/ucx/osc_ucx_passive_target.c ucp_worker_progress(mca_osc_ucx_component.wpool->dflt_worker); wpool 74 ompi/mca/osc/ucx/osc_ucx_passive_target.c ucp_worker_progress(mca_osc_ucx_component.wpool->dflt_worker); wpool 32 ompi/mca/osc/ucx/osc_ucx_request.h opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool); \ wpool 38 opal/mca/common/ucx/common_ucx_wpool.c _winfo_create(opal_common_ucx_wpool_t *wpool) wpool 48 opal/mca/common/ucx/common_ucx_wpool.c status = ucp_worker_create(wpool->ucp_ctx, &worker_params, &worker); wpool 126 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_wpool_free(opal_common_ucx_wpool_t *wpool) wpool 128 opal/mca/common/ucx/common_ucx_wpool.c assert(wpool->refcnt == 0); wpool 129 opal/mca/common/ucx/common_ucx_wpool.c free(wpool); wpool 133 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool, wpool 142 opal/mca/common/ucx/common_ucx_wpool.c wpool->refcnt++; wpool 144 opal/mca/common/ucx/common_ucx_wpool.c if (1 < wpool->refcnt) { wpool 148 opal/mca/common/ucx/common_ucx_wpool.c OBJ_CONSTRUCT(&wpool->mutex, opal_recursive_mutex_t); wpool 149 opal/mca/common/ucx/common_ucx_wpool.c OBJ_CONSTRUCT(&wpool->tls_list, opal_list_t); wpool 171 opal/mca/common/ucx/common_ucx_wpool.c status = ucp_init(&context_params, config, &wpool->ucp_ctx); wpool 180 opal/mca/common/ucx/common_ucx_wpool.c OBJ_CONSTRUCT(&wpool->idle_workers, opal_list_t); wpool 181 opal/mca/common/ucx/common_ucx_wpool.c OBJ_CONSTRUCT(&wpool->active_workers, opal_list_t); wpool 183 opal/mca/common/ucx/common_ucx_wpool.c winfo = _winfo_create(wpool); wpool 189 opal/mca/common/ucx/common_ucx_wpool.c wpool->dflt_worker = winfo->worker; wpool 191 opal/mca/common/ucx/common_ucx_wpool.c status = ucp_worker_get_address(wpool->dflt_worker, wpool 192 opal/mca/common/ucx/common_ucx_wpool.c &wpool->recv_waddr, &wpool->recv_waddr_len); wpool 199 opal/mca/common/ucx/common_ucx_wpool.c rc = _wpool_list_put(wpool, &wpool->idle_workers, winfo); wpool 204 opal/mca/common/ucx/common_ucx_wpool.c opal_tsd_key_create(&wpool->tls_key, _tlocal_cleanup); wpool 209 opal/mca/common/ucx/common_ucx_wpool.c free(wpool->recv_waddr); wpool 211 opal/mca/common/ucx/common_ucx_wpool.c if (NULL != wpool->dflt_worker) { wpool 212 opal/mca/common/ucx/common_ucx_wpool.c ucp_worker_destroy(wpool->dflt_worker); wpool 215 opal/mca/common/ucx/common_ucx_wpool.c ucp_cleanup(wpool->ucp_ctx); wpool 221 opal/mca/common/ucx/common_ucx_wpool.c void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool) wpool 225 opal/mca/common/ucx/common_ucx_wpool.c wpool->refcnt--; wpool 226 opal/mca/common/ucx/common_ucx_wpool.c if (wpool->refcnt > 0) { wpool 232 opal/mca/common/ucx/common_ucx_wpool.c opal_tsd_key_delete(wpool->tls_key); wpool 235 opal/mca/common/ucx/common_ucx_wpool.c OPAL_LIST_FOREACH_SAFE(tls_item, tls_next, &wpool->tls_list, wpool 237 opal/mca/common/ucx/common_ucx_wpool.c opal_list_remove_item(&wpool->tls_list, &tls_item->super); wpool 240 opal/mca/common/ucx/common_ucx_wpool.c OBJ_DESTRUCT(&wpool->tls_list); wpool 244 opal/mca/common/ucx/common_ucx_wpool.c ucp_worker_release_address(wpool->dflt_worker, wpool->recv_waddr); wpool 247 opal/mca/common/ucx/common_ucx_wpool.c if (!opal_list_is_empty(&wpool->idle_workers)) { wpool 249 opal/mca/common/ucx/common_ucx_wpool.c OPAL_LIST_FOREACH_SAFE(item, next, &wpool->idle_workers, wpool 251 opal/mca/common/ucx/common_ucx_wpool.c opal_list_remove_item(&wpool->idle_workers, &item->super); wpool 256 opal/mca/common/ucx/common_ucx_wpool.c OBJ_DESTRUCT(&wpool->idle_workers); wpool 260 opal/mca/common/ucx/common_ucx_wpool.c if (!opal_list_is_empty(&wpool->active_workers)) { wpool 262 opal/mca/common/ucx/common_ucx_wpool.c OPAL_LIST_FOREACH_SAFE(item, next, &wpool->active_workers, wpool 264 opal/mca/common/ucx/common_ucx_wpool.c opal_list_remove_item(&wpool->active_workers, &item->super); wpool 270 opal/mca/common/ucx/common_ucx_wpool.c OBJ_DESTRUCT(&wpool->active_workers); wpool 272 opal/mca/common/ucx/common_ucx_wpool.c OBJ_DESTRUCT(&wpool->mutex); wpool 273 opal/mca/common/ucx/common_ucx_wpool.c ucp_cleanup(wpool->ucp_ctx); wpool 278 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_wpool_progress(opal_common_ucx_wpool_t *wpool) wpool 285 opal/mca/common/ucx/common_ucx_wpool.c if (!opal_mutex_trylock (&wpool->mutex)) { wpool 286 opal/mca/common/ucx/common_ucx_wpool.c OPAL_LIST_FOREACH_SAFE(item, next, &wpool->active_workers, wpool 292 opal/mca/common/ucx/common_ucx_wpool.c opal_list_remove_item(&wpool->active_workers, &item->super); wpool 294 opal/mca/common/ucx/common_ucx_wpool.c opal_list_append(&wpool->idle_workers, &item->super); wpool 301 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_unlock(&wpool->mutex); wpool 306 opal/mca/common/ucx/common_ucx_wpool.c _wpool_list_put(opal_common_ucx_wpool_t *wpool, opal_list_t *list, wpool 318 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_lock(&wpool->mutex); wpool 320 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_unlock(&wpool->mutex); wpool 326 opal/mca/common/ucx/common_ucx_wpool.c _wpool_list_get(opal_common_ucx_wpool_t *wpool, opal_list_t *list) wpool 331 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_lock(&wpool->mutex); wpool 336 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_unlock(&wpool->mutex); wpool 346 opal/mca/common/ucx/common_ucx_wpool.c _wpool_get_idle(opal_common_ucx_wpool_t *wpool, size_t comm_size) wpool 349 opal/mca/common/ucx/common_ucx_wpool.c winfo = _wpool_list_get(wpool, &wpool->idle_workers); wpool 351 opal/mca/common/ucx/common_ucx_wpool.c winfo = _winfo_create(wpool); wpool 365 opal/mca/common/ucx/common_ucx_wpool.c _wpool_add_active(opal_common_ucx_wpool_t *wpool, opal_common_ucx_winfo_t *winfo) wpool 367 opal/mca/common/ucx/common_ucx_wpool.c return _wpool_list_put(wpool, &wpool->active_workers, winfo); wpool 375 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size, wpool 387 opal/mca/common/ucx/common_ucx_wpool.c ctx->wpool = wpool; wpool 392 opal/mca/common/ucx/common_ucx_wpool.c ret = exchange_func(wpool->recv_waddr, wpool->recv_waddr_len, wpool 533 opal/mca/common/ucx/common_ucx_wpool.c ret = _comm_ucx_wpmem_map(ctx->wpool, mem_base, mem_size, &mem->memh, wpool 540 opal/mca/common/ucx/common_ucx_wpool.c status = ucp_rkey_pack(ctx->wpool->ucp_ctx, mem->memh, wpool 565 opal/mca/common/ucx/common_ucx_wpool.c ucp_mem_unmap(ctx->wpool->ucp_ctx, mem->memh); wpool 593 opal/mca/common/ucx/common_ucx_wpool.c static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool, wpool 614 opal/mca/common/ucx/common_ucx_wpool.c status = ucp_mem_map(wpool->ucp_ctx, &mem_params, memh_ptr); wpool 638 opal/mca/common/ucx/common_ucx_wpool.c ucp_mem_unmap(wpool->ucp_ctx, (*memh_ptr)); wpool 647 opal/mca/common/ucx/common_ucx_wpool.c ucp_mem_unmap(mem->ctx->wpool->ucp_ctx, mem->memh); wpool 687 opal/mca/common/ucx/common_ucx_wpool.c static _tlocal_table_t* _common_ucx_tls_init(opal_common_ucx_wpool_t *wpool) wpool 703 opal/mca/common/ucx/common_ucx_wpool.c tls->wpool = wpool; wpool 704 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_lock(&wpool->mutex); wpool 705 opal/mca/common/ucx/common_ucx_wpool.c opal_list_append(&wpool->tls_list, &tls->super); wpool 706 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_unlock(&wpool->mutex); wpool 717 opal/mca/common/ucx/common_ucx_wpool.c opal_tsd_setspecific(wpool->tls_key, tls); wpool 723 opal/mca/common/ucx/common_ucx_wpool.c _tlocal_get_tls(opal_common_ucx_wpool_t *wpool){ wpool 725 opal/mca/common/ucx/common_ucx_wpool.c int rc = opal_tsd_getspecific(wpool->tls_key, (void**)&tls); wpool 732 opal/mca/common/ucx/common_ucx_wpool.c tls = _common_ucx_tls_init(wpool); wpool 741 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_wpool_t *wpool = NULL; wpool 746 opal/mca/common/ucx/common_ucx_wpool.c wpool = tls->wpool; wpool 749 opal/mca/common/ucx/common_ucx_wpool.c tls->wpool = wpool; wpool 750 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_lock(&wpool->mutex); wpool 751 opal/mca/common/ucx/common_ucx_wpool.c OPAL_LIST_FOREACH_SAFE(item, next, &wpool->tls_list, _tlocal_table_t) { wpool 753 opal/mca/common/ucx/common_ucx_wpool.c opal_list_remove_item(&wpool->tls_list, &item->super); wpool 757 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_unlock(&wpool->mutex); wpool 786 opal/mca/common/ucx/common_ucx_wpool.c opal_tsd_setspecific(tls->wpool->tls_key, NULL); wpool 895 opal/mca/common/ucx/common_ucx_wpool.c tls->ctx_tbl[free_idx]->winfo = _wpool_get_idle(tls->wpool, ctx->comm_size); wpool 909 opal/mca/common/ucx/common_ucx_wpool.c _wpool_add_active(tls->wpool, tls->ctx_tbl[free_idx]->winfo); wpool 1090 opal/mca/common/ucx/common_ucx_wpool.c tls = _tlocal_get_tls(mem->ctx->wpool); wpool 63 opal/mca/common/ucx/common_ucx_wpool.h opal_common_ucx_wpool_t *wpool; wpool 164 opal/mca/common/ucx/common_ucx_wpool.h OPAL_DECLSPEC void opal_common_ucx_wpool_free(opal_common_ucx_wpool_t *wpool); wpool 165 opal/mca/common/ucx/common_ucx_wpool.h OPAL_DECLSPEC int opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool, wpool 167 opal/mca/common/ucx/common_ucx_wpool.h OPAL_DECLSPEC void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool); wpool 168 opal/mca/common/ucx/common_ucx_wpool.h OPAL_DECLSPEC void opal_common_ucx_wpool_progress(opal_common_ucx_wpool_t *wpool); wpool 171 opal/mca/common/ucx/common_ucx_wpool.h OPAL_DECLSPEC int opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size, wpool 48 opal/mca/common/ucx/common_ucx_wpool_int.h opal_common_ucx_wpool_t *wpool; wpool 59 opal/mca/common/ucx/common_ucx_wpool_int.h static _tlocal_table_t* _common_ucx_tls_init(opal_common_ucx_wpool_t *wpool); wpool 80 opal/mca/common/ucx/common_ucx_wpool_int.h static opal_common_ucx_winfo_t *_winfo_create(opal_common_ucx_wpool_t *wpool); wpool 85 opal/mca/common/ucx/common_ucx_wpool_int.h static int _wpool_list_put(opal_common_ucx_wpool_t *wpool, opal_list_t *list, wpool 87 opal/mca/common/ucx/common_ucx_wpool_int.h static int _wpool_list_put(opal_common_ucx_wpool_t *wpool, opal_list_t *list, wpool 89 opal/mca/common/ucx/common_ucx_wpool_int.h static opal_common_ucx_winfo_t *_wpool_list_get(opal_common_ucx_wpool_t *wpool, wpool 91 opal/mca/common/ucx/common_ucx_wpool_int.h static opal_common_ucx_winfo_t *_wpool_get_idle(opal_common_ucx_wpool_t *wpool, wpool 93 opal/mca/common/ucx/common_ucx_wpool_int.h static int _wpool_add_active(opal_common_ucx_wpool_t *wpool, wpool 104 opal/mca/common/ucx/common_ucx_wpool_int.h static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool,