This source file includes following definitions.
- mca_spml_ucx_aux_lock
- mca_spml_ucx_aux_unlock
- mca_spml_ucx_cache_mkey
- mca_spml_ucx_get_mkey
- ucx_status_to_oshmem
- ucx_status_to_oshmem_nb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 #ifndef MCA_SPML_UCX_H
18 #define MCA_SPML_UCX_H
19
20 #include "oshmem_config.h"
21 #include "oshmem/request/request.h"
22 #include "oshmem/mca/spml/base/base.h"
23 #include "oshmem/mca/spml/spml.h"
24 #include "oshmem/util/oshmem_util.h"
25 #include "oshmem/mca/spml/base/spml_base_putreq.h"
26 #include "oshmem/proc/proc.h"
27 #include "oshmem/mca/spml/base/spml_base_request.h"
28 #include "oshmem/mca/spml/base/spml_base_getreq.h"
29 #include "oshmem/runtime/runtime.h"
30
31 #include "oshmem/mca/memheap/memheap.h"
32 #include "oshmem/mca/memheap/base/base.h"
33
34 #include "opal/class/opal_free_list.h"
35 #include "opal/class/opal_list.h"
36
37 #include "opal/mca/common/ucx/common_ucx.h"
38
39 #include <ucp/api/ucp.h>
40
41 BEGIN_C_DECLS
42
43 #define SPML_UCX_ASSERT MCA_COMMON_UCX_ASSERT
44 #define SPML_UCX_ERROR MCA_COMMON_UCX_ERROR
45 #define SPML_UCX_VERBOSE MCA_COMMON_UCX_VERBOSE
46
47
48
49
50 struct spml_ucx_mkey {
51 ucp_rkey_h rkey;
52 ucp_mem_h mem_h;
53 };
54 typedef struct spml_ucx_mkey spml_ucx_mkey_t;
55
56 struct spml_ucx_cached_mkey {
57 mkey_segment_t super;
58 spml_ucx_mkey_t key;
59 };
60 typedef struct spml_ucx_cached_mkey spml_ucx_cached_mkey_t;
61
62 struct ucp_peer {
63 ucp_ep_h ucp_conn;
64 spml_ucx_cached_mkey_t mkeys[MCA_MEMHEAP_MAX_SEGMENTS];
65 };
66 typedef struct ucp_peer ucp_peer_t;
67
68 struct mca_spml_ucx_ctx {
69 ucp_worker_h ucp_worker;
70 ucp_peer_t *ucp_peers;
71 long options;
72 };
73 typedef struct mca_spml_ucx_ctx mca_spml_ucx_ctx_t;
74
75 extern mca_spml_ucx_ctx_t mca_spml_ucx_ctx_default;
76
77 typedef spml_ucx_mkey_t * (*mca_spml_ucx_get_mkey_slow_fn_t)(shmem_ctx_t ctx, int pe, void *va, void **rva);
78
79 typedef struct mca_spml_ucx_ctx_array {
80 int ctxs_count;
81 int ctxs_num;
82 mca_spml_ucx_ctx_t **ctxs;
83 } mca_spml_ucx_ctx_array_t;
84
85 struct mca_spml_ucx {
86 mca_spml_base_module_t super;
87 ucp_context_h ucp_context;
88 int num_disconnect;
89 int heap_reg_nb;
90 bool enabled;
91 mca_spml_ucx_get_mkey_slow_fn_t get_mkey_slow;
92 char **remote_addrs_tbl;
93 mca_spml_ucx_ctx_array_t active_array;
94 mca_spml_ucx_ctx_array_t idle_array;
95 int priority;
96 shmem_internal_mutex_t internal_mutex;
97 pthread_mutex_t ctx_create_mutex;
98
99 bool async_progress;
100 int async_tick;
101 opal_event_base_t *async_event_base;
102 opal_event_t *tick_event;
103 mca_spml_ucx_ctx_t *aux_ctx;
104 pthread_spinlock_t async_lock;
105 int aux_refcnt;
106
107 };
108 typedef struct mca_spml_ucx mca_spml_ucx_t;
109
110 extern mca_spml_ucx_t mca_spml_ucx;
111
112 extern int mca_spml_ucx_enable(bool enable);
113 extern int mca_spml_ucx_ctx_create(long options,
114 shmem_ctx_t *ctx);
115 extern void mca_spml_ucx_ctx_destroy(shmem_ctx_t ctx);
116 extern int mca_spml_ucx_get(shmem_ctx_t ctx,
117 void* dst_addr,
118 size_t size,
119 void* src_addr,
120 int src);
121 extern int mca_spml_ucx_get_nb(shmem_ctx_t ctx,
122 void* dst_addr,
123 size_t size,
124 void* src_addr,
125 int src,
126 void **handle);
127
128 extern int mca_spml_ucx_put(shmem_ctx_t ctx,
129 void* dst_addr,
130 size_t size,
131 void* src_addr,
132 int dst);
133
134 extern int mca_spml_ucx_put_nb(shmem_ctx_t ctx,
135 void* dst_addr,
136 size_t size,
137 void* src_addr,
138 int dst,
139 void **handle);
140
141 extern int mca_spml_ucx_recv(void* buf, size_t size, int src);
142 extern int mca_spml_ucx_send(void* buf,
143 size_t size,
144 int dst,
145 mca_spml_base_put_mode_t mode);
146
147 extern int mca_spml_ucx_put_all_nb(void *target,
148 const void *source,
149 size_t size,
150 long *counter);
151
152 extern sshmem_mkey_t *mca_spml_ucx_register(void* addr,
153 size_t size,
154 uint64_t shmid,
155 int *count);
156 extern int mca_spml_ucx_deregister(sshmem_mkey_t *mkeys);
157
158 extern void mca_spml_ucx_memuse_hook(void *addr, size_t length);
159
160 extern void mca_spml_ucx_rmkey_unpack(shmem_ctx_t ctx, sshmem_mkey_t *mkey, uint32_t segno, int pe, int tr_id);
161 extern void mca_spml_ucx_rmkey_free(sshmem_mkey_t *mkey);
162 extern void *mca_spml_ucx_rmkey_ptr(const void *dst_addr, sshmem_mkey_t *, int pe);
163
164 extern int mca_spml_ucx_add_procs(ompi_proc_t** procs, size_t nprocs);
165 extern int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs);
166 extern int mca_spml_ucx_fence(shmem_ctx_t ctx);
167 extern int mca_spml_ucx_quiet(shmem_ctx_t ctx);
168 extern int spml_ucx_default_progress(void);
169 extern int spml_ucx_ctx_progress(void);
170 extern int spml_ucx_progress_aux_ctx(void);
171 void mca_spml_ucx_async_cb(int fd, short event, void *cbdata);
172
173 static inline void mca_spml_ucx_aux_lock(void)
174 {
175 if (mca_spml_ucx.async_progress) {
176 pthread_spin_lock(&mca_spml_ucx.async_lock);
177 }
178 }
179
180 static inline void mca_spml_ucx_aux_unlock(void)
181 {
182 if (mca_spml_ucx.async_progress) {
183 pthread_spin_unlock(&mca_spml_ucx.async_lock);
184 }
185 }
186
187 static inline void mca_spml_ucx_cache_mkey(mca_spml_ucx_ctx_t *ucx_ctx,
188 sshmem_mkey_t *mkey, uint32_t segno, int dst_pe)
189 {
190 ucp_peer_t *peer;
191
192 peer = &(ucx_ctx->ucp_peers[dst_pe]);
193 mkey_segment_init(&peer->mkeys[segno].super, mkey, segno);
194 }
195
196 static inline spml_ucx_mkey_t *
197 mca_spml_ucx_get_mkey(shmem_ctx_t ctx, int pe, void *va, void **rva, mca_spml_ucx_t* module)
198 {
199 spml_ucx_cached_mkey_t *mkey;
200 mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
201
202 mkey = ucx_ctx->ucp_peers[pe].mkeys;
203 mkey = (spml_ucx_cached_mkey_t *)map_segment_find_va(&mkey->super.super, sizeof(*mkey), va);
204 assert(mkey != NULL);
205 *rva = map_segment_va2rva(&mkey->super, va);
206 return &mkey->key;
207 }
208
209 static inline int ucx_status_to_oshmem(ucs_status_t status)
210 {
211 #if OSHMEM_PARAM_CHECK == 1
212 return OPAL_LIKELY(UCS_OK == status) ? OSHMEM_SUCCESS : OSHMEM_ERROR;
213 #else
214 return OSHMEM_SUCCESS;
215 #endif
216 }
217
218 static inline int ucx_status_to_oshmem_nb(ucs_status_t status)
219 {
220 #if OSHMEM_PARAM_CHECK == 1
221 return OPAL_LIKELY(status >= 0) ? OSHMEM_SUCCESS : OSHMEM_ERROR;
222 #else
223 return OSHMEM_SUCCESS;
224 #endif
225 }
226
227 #define MCA_SPML_UCX_CTXS_ARRAY_SIZE 64
228 #define MCA_SPML_UCX_CTXS_ARRAY_INC 64
229
230 END_C_DECLS
231
232 #endif
233