This source file includes following definitions.
- mca_btl_vader_knem_reg
- mca_btl_vader_knem_dereg
- mca_btl_vader_register_mem_knem
- mca_btl_vader_deregister_mem_knem
- mca_btl_vader_knem_init
- mca_btl_vader_knem_fini
- mca_btl_vader_knem_progress
1
2
3
4
5
6
7
8
9
10
11
12 #include "btl_vader.h"
13
14 #if OPAL_BTL_VADER_HAVE_KNEM
15
16 #include <stdio.h>
17 #include <fcntl.h>
18 #include <sys/stat.h>
19 #include <unistd.h>
20
21 #include "opal/util/show_help.h"
22
23 OBJ_CLASS_INSTANCE(mca_btl_vader_registration_handle_t, mca_rcache_base_registration_t, NULL, NULL);
24
25 static int mca_btl_vader_knem_reg (void *reg_data, void *base, size_t size,
26 mca_rcache_base_registration_t *reg)
27 {
28 mca_btl_vader_registration_handle_t *knem_reg = (mca_btl_vader_registration_handle_t *) reg;
29 struct knem_cmd_create_region knem_cr;
30 struct knem_cmd_param_iovec knem_iov;
31
32 knem_iov.base = (uintptr_t) base;
33 knem_iov.len = size;
34
35 knem_cr.iovec_array = (uintptr_t) &knem_iov;
36 knem_cr.iovec_nr = 1;
37 knem_cr.protection = 0;
38
39 if (reg->access_flags & (MCA_RCACHE_ACCESS_LOCAL_WRITE | MCA_RCACHE_ACCESS_REMOTE_WRITE)) {
40 knem_cr.protection |= PROT_WRITE;
41 }
42
43 if (reg->access_flags & MCA_RCACHE_ACCESS_REMOTE_READ) {
44 knem_cr.protection |= PROT_READ;
45 }
46
47
48 knem_cr.flags = 0;
49 if (OPAL_UNLIKELY(ioctl(mca_btl_vader.knem_fd, KNEM_CMD_CREATE_REGION, &knem_cr) < 0)) {
50 return OPAL_ERROR;
51 }
52
53 knem_reg->btl_handle.cookie = knem_cr.cookie;
54 knem_reg->btl_handle.base_addr = (intptr_t) base;
55
56 return OPAL_SUCCESS;
57 }
58
59 static int mca_btl_vader_knem_dereg (void *reg_data, mca_rcache_base_registration_t *reg)
60 {
61 mca_btl_vader_registration_handle_t *knem_reg = (mca_btl_vader_registration_handle_t *) reg;
62
63
64 (void) ioctl(mca_btl_vader.knem_fd, KNEM_CMD_DESTROY_REGION, &knem_reg->btl_handle.cookie);
65
66 return OPAL_SUCCESS;
67 }
68
69 static mca_btl_base_registration_handle_t *
70 mca_btl_vader_register_mem_knem (struct mca_btl_base_module_t* btl,
71 struct mca_btl_base_endpoint_t *endpoint,
72 void *base, size_t size, uint32_t flags)
73 {
74 mca_btl_vader_t *vader_module = (mca_btl_vader_t *) btl;
75 mca_btl_vader_registration_handle_t *reg = NULL;
76 int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
77 int rc;
78
79 rc = vader_module->knem_rcache->rcache_register (vader_module->knem_rcache, base, size, 0,
80 access_flags,
81 (mca_rcache_base_registration_t **) ®);
82 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
83 return NULL;
84 }
85
86 return ®->btl_handle;
87 }
88
89 static int
90 mca_btl_vader_deregister_mem_knem (struct mca_btl_base_module_t *btl, struct mca_btl_base_registration_handle_t *handle)
91 {
92 mca_btl_vader_t *vader_module = (mca_btl_vader_t *) btl;
93 mca_btl_vader_registration_handle_t *reg =
94 (mca_btl_vader_registration_handle_t *)((intptr_t) handle - offsetof (mca_btl_vader_registration_handle_t, btl_handle));
95
96 vader_module->knem_rcache->rcache_deregister (vader_module->knem_rcache, ®->base);
97
98 return OPAL_SUCCESS;
99 }
100
101 int mca_btl_vader_knem_init (void)
102 {
103 mca_rcache_base_resources_t rcache_resources = {
104 .cache_name = "vader", .reg_data = NULL,
105 .sizeof_reg = sizeof (mca_btl_vader_registration_handle_t),
106 .register_mem = mca_btl_vader_knem_reg,
107 .deregister_mem = mca_btl_vader_knem_dereg
108 };
109 struct knem_cmd_info knem_info;
110 int rc;
111
112
113
114 mca_btl_vader.knem_fd = open("/dev/knem", O_RDWR);
115 if (mca_btl_vader.knem_fd < 0) {
116 if (EACCES == errno) {
117 struct stat sbuf;
118 if (0 != stat("/dev/knem", &sbuf)) {
119 sbuf.st_mode = 0;
120 }
121 opal_show_help("help-btl-vader.txt", "knem permission denied",
122 true, opal_process_info.nodename, sbuf.st_mode);
123 } else {
124 opal_show_help("help-btl-vader.txt", "knem fail open",
125 true, opal_process_info.nodename, errno,
126 strerror(errno));
127 }
128
129 return OPAL_ERR_NOT_AVAILABLE;
130 }
131
132 do {
133
134
135 memset (&knem_info, 0, sizeof (knem_info));
136 rc = ioctl(mca_btl_vader.knem_fd, KNEM_CMD_GET_INFO, &knem_info);
137 if (rc < 0) {
138 opal_show_help("help-btl-vader.txt", "knem get ABI fail",
139 true, opal_process_info.nodename, errno,
140 strerror(errno));
141 break;
142 }
143
144 if (KNEM_ABI_VERSION != knem_info.abi) {
145 opal_show_help("help-btl-vader.txt", "knem ABI mismatch",
146 true, opal_process_info.nodename, KNEM_ABI_VERSION,
147 knem_info.abi);
148 break;
149 }
150
151 if (!(mca_btl_vader_component.knem_dma_min && (knem_info.features & KNEM_FEATURE_DMA))) {
152
153 mca_btl_vader_component.knem_dma_min = UINT_MAX;
154 }
155
156
157
158
159 mca_btl_vader.super.btl_get = mca_btl_vader_get_knem;
160 mca_btl_vader.super.btl_put = mca_btl_vader_put_knem;
161
162
163 mca_btl_vader.super.btl_register_mem = mca_btl_vader_register_mem_knem;
164 mca_btl_vader.super.btl_deregister_mem = mca_btl_vader_deregister_mem_knem;
165 mca_btl_vader.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t);
166
167 mca_btl_vader.knem_rcache = mca_rcache_base_module_create ("grdma", NULL,
168 &rcache_resources);
169 if (NULL == mca_btl_vader.knem_rcache) {
170 return OPAL_ERR_OUT_OF_RESOURCE;
171 }
172
173 return OPAL_SUCCESS;
174 } while (0);
175
176 mca_btl_vader_knem_fini ();
177
178 return OPAL_ERR_NOT_AVAILABLE;;
179 }
180
181 int mca_btl_vader_knem_fini (void)
182 {
183 if (-1 != mca_btl_vader.knem_fd) {
184 close (mca_btl_vader.knem_fd);
185 mca_btl_vader.knem_fd = -1;
186 }
187
188 if (mca_btl_vader.knem_rcache) {
189 (void) mca_rcache_base_module_destroy (mca_btl_vader.knem_rcache);
190 mca_btl_vader.knem_rcache = NULL;
191 }
192
193 return OPAL_SUCCESS;
194 }
195
196 int mca_btl_vader_knem_progress (void)
197 {
198
199 return OPAL_SUCCESS;
200 }
201
202 #endif