winfo 43 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_t *winfo = NULL; winfo 54 opal/mca/common/ucx/common_ucx_wpool.c winfo = calloc(1, sizeof(*winfo)); winfo 55 opal/mca/common/ucx/common_ucx_wpool.c if (NULL == winfo) { winfo 60 opal/mca/common/ucx/common_ucx_wpool.c OBJ_CONSTRUCT(&winfo->mutex, opal_recursive_mutex_t); winfo 61 opal/mca/common/ucx/common_ucx_wpool.c winfo->worker = worker; winfo 62 opal/mca/common/ucx/common_ucx_wpool.c winfo->endpoints = NULL; winfo 63 opal/mca/common/ucx/common_ucx_wpool.c winfo->comm_size = 0; winfo 64 opal/mca/common/ucx/common_ucx_wpool.c winfo->released = 0; winfo 65 opal/mca/common/ucx/common_ucx_wpool.c winfo->inflight_ops = NULL; winfo 66 opal/mca/common/ucx/common_ucx_wpool.c winfo->global_inflight_ops = 0; winfo 67 opal/mca/common/ucx/common_ucx_wpool.c winfo->inflight_req = UCS_OK; winfo 69 opal/mca/common/ucx/common_ucx_wpool.c return winfo; winfo 74 opal/mca/common/ucx/common_ucx_wpool.c return winfo; winfo 78 opal/mca/common/ucx/common_ucx_wpool.c _winfo_reset(opal_common_ucx_winfo_t *winfo) winfo 80 opal/mca/common/ucx/common_ucx_wpool.c if (winfo->inflight_req != UCS_OK) { winfo 81 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_wait_request_mt(winfo->inflight_req, winfo 83 opal/mca/common/ucx/common_ucx_wpool.c winfo->inflight_req = UCS_OK; winfo 86 opal/mca/common/ucx/common_ucx_wpool.c assert(winfo->global_inflight_ops == 0); winfo 88 opal/mca/common/ucx/common_ucx_wpool.c if(winfo->comm_size != 0) { winfo 90 opal/mca/common/ucx/common_ucx_wpool.c for (i = 0; i < winfo->comm_size; i++) { winfo 91 opal/mca/common/ucx/common_ucx_wpool.c if (NULL != winfo->endpoints[i]){ winfo 92 opal/mca/common/ucx/common_ucx_wpool.c ucp_ep_destroy(winfo->endpoints[i]); winfo 94 opal/mca/common/ucx/common_ucx_wpool.c assert(winfo->inflight_ops[i] == 0); winfo 96 opal/mca/common/ucx/common_ucx_wpool.c free(winfo->endpoints); winfo 97 opal/mca/common/ucx/common_ucx_wpool.c free(winfo->inflight_ops); winfo 99 opal/mca/common/ucx/common_ucx_wpool.c winfo->endpoints = NULL; winfo 100 opal/mca/common/ucx/common_ucx_wpool.c winfo->comm_size = 0; winfo 101 opal/mca/common/ucx/common_ucx_wpool.c winfo->released = 0; winfo 105 opal/mca/common/ucx/common_ucx_wpool.c _winfo_release(opal_common_ucx_winfo_t *winfo) winfo 107 opal/mca/common/ucx/common_ucx_wpool.c OBJ_DESTRUCT(&winfo->mutex); winfo 108 opal/mca/common/ucx/common_ucx_wpool.c ucp_worker_destroy(winfo->worker); winfo 109 opal/mca/common/ucx/common_ucx_wpool.c free(winfo); winfo 138 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_t *winfo; winfo 183 opal/mca/common/ucx/common_ucx_wpool.c winfo = _winfo_create(wpool); winfo 184 opal/mca/common/ucx/common_ucx_wpool.c if (NULL == winfo) { winfo 189 opal/mca/common/ucx/common_ucx_wpool.c wpool->dflt_worker = winfo->worker; winfo 199 opal/mca/common/ucx/common_ucx_wpool.c rc = _wpool_list_put(wpool, &wpool->idle_workers, winfo); winfo 288 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_t *winfo = item->ptr; winfo 289 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_lock(&winfo->mutex); winfo 290 opal/mca/common/ucx/common_ucx_wpool.c if( OPAL_UNLIKELY(winfo->released) ) { winfo 293 opal/mca/common/ucx/common_ucx_wpool.c _winfo_reset(winfo); winfo 297 opal/mca/common/ucx/common_ucx_wpool.c while(ucp_worker_progress(winfo->worker)); winfo 299 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_unlock(&winfo->mutex); winfo 307 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_t *winfo) winfo 316 opal/mca/common/ucx/common_ucx_wpool.c item->ptr = winfo; winfo 328 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_t *winfo = NULL; winfo 339 opal/mca/common/ucx/common_ucx_wpool.c winfo = item->ptr; winfo 342 opal/mca/common/ucx/common_ucx_wpool.c return winfo; winfo 348 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_t *winfo; winfo 349 opal/mca/common/ucx/common_ucx_wpool.c winfo = _wpool_list_get(wpool, &wpool->idle_workers); winfo 350 opal/mca/common/ucx/common_ucx_wpool.c if (!winfo) { winfo 351 opal/mca/common/ucx/common_ucx_wpool.c winfo = _winfo_create(wpool); winfo 352 opal/mca/common/ucx/common_ucx_wpool.c if (!winfo) { winfo 358 opal/mca/common/ucx/common_ucx_wpool.c winfo->endpoints = calloc(comm_size, sizeof(ucp_ep_h)); winfo 359 opal/mca/common/ucx/common_ucx_wpool.c winfo->inflight_ops = calloc(comm_size, sizeof(short)); winfo 360 opal/mca/common/ucx/common_ucx_wpool.c winfo->comm_size = comm_size; winfo 361 opal/mca/common/ucx/common_ucx_wpool.c return winfo; winfo 365 opal/mca/common/ucx/common_ucx_wpool.c _wpool_add_active(opal_common_ucx_wpool_t *wpool, opal_common_ucx_winfo_t *winfo) winfo 367 opal/mca/common/ucx/common_ucx_wpool.c return _wpool_list_put(wpool, &wpool->active_workers, winfo); winfo 449 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_t *winfo) winfo 459 opal/mca/common/ucx/common_ucx_wpool.c item->ptr = winfo; winfo 470 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_t *winfo) winfo 479 opal/mca/common/ucx/common_ucx_wpool.c if (winfo == item->ptr) { winfo 481 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_lock(&winfo->mutex); winfo 482 opal/mca/common/ucx/common_ucx_wpool.c winfo->released = 1; winfo 483 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_unlock(&winfo->mutex); winfo 853 opal/mca/common/ucx/common_ucx_wpool.c _common_ucx_wpctx_remove(ctx_rec->gctx, ctx_rec->winfo); winfo 895 opal/mca/common/ucx/common_ucx_wpool.c tls->ctx_tbl[free_idx]->winfo = _wpool_get_idle(tls->wpool, ctx->comm_size); winfo 896 opal/mca/common/ucx/common_ucx_wpool.c if (NULL == tls->ctx_tbl[free_idx]->winfo) { winfo 909 opal/mca/common/ucx/common_ucx_wpool.c _wpool_add_active(tls->wpool, tls->ctx_tbl[free_idx]->winfo); winfo 912 opal/mca/common/ucx/common_ucx_wpool.c rc = _common_ucx_wpctx_append(ctx, tls->ctx_tbl[free_idx]->winfo); winfo 925 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_t *winfo = ctx_rec->winfo; winfo 933 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_lock(&winfo->mutex); winfo 936 opal/mca/common/ucx/common_ucx_wpool.c status = ucp_ep_create(winfo->worker, &ep_params, &winfo->endpoints[target]); winfo 938 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_unlock(&winfo->mutex); winfo 942 opal/mca/common/ucx/common_ucx_wpool.c opal_mutex_unlock(&winfo->mutex); winfo 1034 opal/mca/common/ucx/common_ucx_wpool.c tls->mem_tbl[free_idx]->mem->worker = ctx_rec->winfo; winfo 1040 opal/mca/common/ucx/common_ucx_wpool.c tls->mem_tbl[free_idx]->mem_tls_ptr->winfo = ctx_rec->winfo; winfo 1084 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_t *winfo = NULL; winfo 1101 opal/mca/common/ucx/common_ucx_wpool.c winfo = ctx_rec->winfo; winfo 1104 opal/mca/common/ucx/common_ucx_wpool.c if (OPAL_UNLIKELY(NULL == winfo->endpoints[target])) { winfo 1110 opal/mca/common/ucx/common_ucx_wpool.c ep = winfo->endpoints[target]; winfo 1135 opal/mca/common/ucx/common_ucx_wpool.c opal_common_ucx_winfo_flush(opal_common_ucx_winfo_t *winfo, int target, winfo 1146 opal/mca/common/ucx/common_ucx_wpool.c req = ucp_ep_flush_nb(winfo->endpoints[target], 0, opal_common_ucx_empty_complete_cb); winfo 1148 opal/mca/common/ucx/common_ucx_wpool.c req = ucp_worker_flush_nb(winfo->worker, 0, opal_common_ucx_empty_complete_cb); winfo 1151 opal/mca/common/ucx/common_ucx_wpool.c ((opal_common_ucx_request_t *)req)->winfo = winfo; winfo 1165 opal/mca/common/ucx/common_ucx_wpool.c status = ucp_ep_flush(winfo->endpoints[target]); winfo 1167 opal/mca/common/ucx/common_ucx_wpool.c status = ucp_worker_flush(winfo->worker); winfo 1231 opal/mca/common/ucx/common_ucx_wpool.c req->winfo = NULL; winfo 120 opal/mca/common/ucx/common_ucx_wpool.h opal_common_ucx_winfo_t *winfo; winfo 133 opal/mca/common/ucx/common_ucx_wpool.h opal_common_ucx_winfo_t *winfo; winfo 197 opal/mca/common/ucx/common_ucx_wpool.h expr = fp && (NULL != fp->winfo) && (fp->winfo->endpoints[target]) && winfo 209 opal/mca/common/ucx/common_ucx_wpool.h MCA_COMMON_UCX_ASSERT(fp && (NULL != fp->winfo) && winfo 210 opal/mca/common/ucx/common_ucx_wpool.h (fp->winfo->endpoints[target]) winfo 214 opal/mca/common/ucx/common_ucx_wpool.h *_winfo = fp->winfo; winfo 215 opal/mca/common/ucx/common_ucx_wpool.h *_ep = fp->winfo->endpoints[target]; winfo 235 opal/mca/common/ucx/common_ucx_wpool.h OPAL_DECLSPEC int opal_common_ucx_winfo_flush(opal_common_ucx_winfo_t *winfo, int target, winfo 245 opal/mca/common/ucx/common_ucx_wpool.h opal_common_ucx_winfo_t *winfo; winfo 257 opal/mca/common/ucx/common_ucx_wpool.h winfo = ((opal_common_ucx_request_t *)request)->winfo; winfo 258 opal/mca/common/ucx/common_ucx_wpool.h assert(winfo != NULL); winfo 262 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_lock(&winfo->mutex); winfo 264 opal/mca/common/ucx/common_ucx_wpool.h ret = ucp_worker_progress(winfo->worker); winfo 273 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_unlock(&winfo->mutex); winfo 280 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_unlock(&winfo->mutex); winfo 288 opal/mca/common/ucx/common_ucx_wpool.h opal_common_ucx_winfo_t *winfo, winfo 292 opal/mca/common/ucx/common_ucx_wpool.h winfo->inflight_ops[target]++; winfo 293 opal/mca/common/ucx/common_ucx_wpool.h winfo->global_inflight_ops++; winfo 295 opal/mca/common/ucx/common_ucx_wpool.h if (OPAL_UNLIKELY(winfo->inflight_ops[target] >= MCA_COMMON_UCX_PER_TARGET_OPS_THRESHOLD) || winfo 296 opal/mca/common/ucx/common_ucx_wpool.h OPAL_UNLIKELY(winfo->global_inflight_ops >= MCA_COMMON_UCX_GLOBAL_OPS_THRESHOLD)) { winfo 299 opal/mca/common/ucx/common_ucx_wpool.h if (winfo->inflight_req != UCS_OK) { winfo 300 opal/mca/common/ucx/common_ucx_wpool.h rc = opal_common_ucx_wait_request_mt(winfo->inflight_req, winfo 306 opal/mca/common/ucx/common_ucx_wpool.h winfo->inflight_req = UCS_OK; winfo 309 opal/mca/common/ucx/common_ucx_wpool.h if (winfo->global_inflight_ops >= MCA_COMMON_UCX_GLOBAL_OPS_THRESHOLD) { winfo 311 opal/mca/common/ucx/common_ucx_wpool.h winfo->global_inflight_ops = 0; winfo 312 opal/mca/common/ucx/common_ucx_wpool.h memset(winfo->inflight_ops, 0, winfo->comm_size * sizeof(short)); winfo 315 opal/mca/common/ucx/common_ucx_wpool.h winfo->global_inflight_ops -= winfo->inflight_ops[target]; winfo 316 opal/mca/common/ucx/common_ucx_wpool.h winfo->inflight_ops[target] = 0; winfo 319 opal/mca/common/ucx/common_ucx_wpool.h rc = opal_common_ucx_winfo_flush(winfo, target, OPAL_COMMON_UCX_FLUSH_NB_PREFERRED, winfo 320 opal/mca/common/ucx/common_ucx_wpool.h scope, &winfo->inflight_req); winfo 325 opal/mca/common/ucx/common_ucx_wpool.h } else if (OPAL_UNLIKELY(winfo->inflight_req != UCS_OK)) { winfo 328 opal/mca/common/ucx/common_ucx_wpool.h ret = ucp_worker_progress(winfo->worker); winfo 342 opal/mca/common/ucx/common_ucx_wpool.h opal_common_ucx_winfo_t *winfo; winfo 346 opal/mca/common/ucx/common_ucx_wpool.h rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo); winfo 353 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_lock(&winfo->mutex); winfo 370 opal/mca/common/ucx/common_ucx_wpool.h rc = _periodical_flush_nb(mem, winfo, target); winfo 376 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_unlock(&winfo->mutex); winfo 389 opal/mca/common/ucx/common_ucx_wpool.h opal_common_ucx_winfo_t *winfo = NULL; winfo 393 opal/mca/common/ucx/common_ucx_wpool.h rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo); winfo 400 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_lock(&winfo->mutex); winfo 404 opal/mca/common/ucx/common_ucx_wpool.h winfo->worker); winfo 410 opal/mca/common/ucx/common_ucx_wpool.h rc = _periodical_flush_nb(mem, winfo, target); winfo 416 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_unlock(&winfo->mutex); winfo 427 opal/mca/common/ucx/common_ucx_wpool.h opal_common_ucx_winfo_t *winfo = NULL; winfo 432 opal/mca/common/ucx/common_ucx_wpool.h rc =opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo); winfo 439 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_lock(&winfo->mutex); winfo 447 opal/mca/common/ucx/common_ucx_wpool.h rc = _periodical_flush_nb(mem, winfo, target); winfo 453 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_unlock(&winfo->mutex); winfo 465 opal/mca/common/ucx/common_ucx_wpool.h opal_common_ucx_winfo_t *winfo = NULL; winfo 469 opal/mca/common/ucx/common_ucx_wpool.h rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo); winfo 476 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_lock(&winfo->mutex); winfo 480 opal/mca/common/ucx/common_ucx_wpool.h winfo->worker); winfo 486 opal/mca/common/ucx/common_ucx_wpool.h rc = _periodical_flush_nb(mem, winfo, target); winfo 492 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_unlock(&winfo->mutex); winfo 508 opal/mca/common/ucx/common_ucx_wpool.h opal_common_ucx_winfo_t *winfo = NULL; winfo 512 opal/mca/common/ucx/common_ucx_wpool.h rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo); winfo 518 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_lock(&winfo->mutex); winfo 521 opal/mca/common/ucx/common_ucx_wpool.h winfo->worker); winfo 525 opal/mca/common/ucx/common_ucx_wpool.h req->winfo = winfo; winfo 532 opal/mca/common/ucx/common_ucx_wpool.h rc = _periodical_flush_nb(mem, winfo, target); winfo 538 opal/mca/common/ucx/common_ucx_wpool.h opal_mutex_unlock(&winfo->mutex); winfo 10 opal/mca/common/ucx/common_ucx_wpool_int.h opal_common_ucx_winfo_t *winfo; winfo 81 opal/mca/common/ucx/common_ucx_wpool_int.h static void _winfo_release(opal_common_ucx_winfo_t *winfo); winfo 82 opal/mca/common/ucx/common_ucx_wpool_int.h static void _winfo_reset(opal_common_ucx_winfo_t *winfo); winfo 86 opal/mca/common/ucx/common_ucx_wpool_int.h opal_common_ucx_winfo_t *winfo); winfo 88 opal/mca/common/ucx/common_ucx_wpool_int.h opal_common_ucx_winfo_t *winfo); winfo 94 opal/mca/common/ucx/common_ucx_wpool_int.h opal_common_ucx_winfo_t *winfo); winfo 99 opal/mca/common/ucx/common_ucx_wpool_int.h opal_common_ucx_winfo_t *winfo); winfo 101 opal/mca/common/ucx/common_ucx_wpool_int.h opal_common_ucx_winfo_t *winfo);