This source file includes following definitions.
- module_init
- module_finalize
- segment_create
- segment_attach
- segment_detach
- segment_unlink
- sshmem_sysv_gethugepagesize
1
2
3
4
5
6
7
8
9
10
11
12
13
14 #include "oshmem_config.h"
15
16 #include <errno.h>
17 #ifdef HAVE_FCNTL_H
18 #include <fcntl.h>
19 #endif
20 #ifdef HAVE_SYS_MMAN_H
21 #include <sys/mman.h>
22 #endif
23 #ifdef HAVE_UNISTD_H
24 #include <unistd.h>
25 #endif
26 #ifdef HAVE_SYS_TYPES_H
27 #include <sys/types.h>
28 #endif
29 #ifdef HAVE_SYS_IPC_H
30 #include <sys/ipc.h>
31 #endif
32 #if HAVE_SYS_SHM_H
33 #include <sys/shm.h>
34 #endif
35 #if HAVE_SYS_STAT_H
36 #include <sys/stat.h>
37 #endif
38 #include <string.h>
39 #ifdef HAVE_NETDB_H
40 #include <netdb.h>
41 #endif
42
43 #include "opal/constants.h"
44 #include "opal_stdint.h"
45 #include "opal/util/output.h"
46 #include "opal/util/path.h"
47 #include "opal/util/show_help.h"
48
49 #include "oshmem/proc/proc.h"
50 #include "oshmem/mca/sshmem/sshmem.h"
51 #include "oshmem/mca/sshmem/base/base.h"
52
53 #include "sshmem_sysv.h"
54
55
56
57
58 static int
59 module_init(void);
60
61 static int
62 segment_create(map_segment_t *ds_buf,
63 const char *file_name,
64 size_t size, long hint);
65
66 static void *
67 segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
68
69 static int
70 segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
71
72 static int
73 segment_unlink(map_segment_t *ds_buf);
74
75 static int
76 module_finalize(void);
77
78
79 mca_sshmem_sysv_module_t mca_sshmem_sysv_module = {
80
81 {
82 module_init,
83 segment_create,
84 segment_attach,
85 segment_detach,
86 segment_unlink,
87 module_finalize
88 }
89 };
90
91
92
93 static int
94 module_init(void)
95 {
96
97 return OSHMEM_SUCCESS;
98 }
99
100
101 static int
102 module_finalize(void)
103 {
104
105 return OSHMEM_SUCCESS;
106 }
107
108
109
110 static int
111 segment_create(map_segment_t *ds_buf,
112 const char *file_name,
113 size_t size, long hint)
114 {
115 int rc = OSHMEM_SUCCESS;
116 void *addr = NULL;
117 int shmid = MAP_SEGMENT_SHM_INVALID;
118 int flags;
119 int try_hp;
120
121 assert(ds_buf);
122
123 if (hint) {
124 return OSHMEM_ERR_NOT_IMPLEMENTED;
125 }
126
127
128 shmem_ds_reset(ds_buf);
129
130
131
132
133
134
135
136
137 flags = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR;
138 try_hp = mca_sshmem_sysv_component.use_hp;
139 #if defined (SHM_HUGETLB)
140 flags |= ((0 != try_hp) ? SHM_HUGETLB : 0);
141 size = ((size + sshmem_sysv_gethugepagesize() - 1) / sshmem_sysv_gethugepagesize()) * sshmem_sysv_gethugepagesize();
142 #endif
143
144
145 retry_alloc:
146 shmid = shmget(IPC_PRIVATE, size, flags);
147 if (shmid == MAP_SEGMENT_SHM_INVALID) {
148
149
150
151 if (-1 == try_hp) {
152 OPAL_OUTPUT_VERBOSE(
153 (10, oshmem_sshmem_base_framework.framework_output,
154 "failed to allocate %llu bytes with huge pages. "
155 "Using regular pages", (unsigned long long)size));
156 flags = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR;
157 try_hp = 0;
158 goto retry_alloc;
159 }
160 opal_show_help("help-oshmem-sshmem.txt",
161 "create segment failure",
162 true,
163 "sysv",
164 ompi_process_info.nodename, (unsigned long long) size,
165 strerror(errno), errno);
166 opal_show_help("help-oshmem-sshmem-sysv.txt",
167 "sysv:create segment failure",
168 true);
169 return OSHMEM_ERROR;
170 }
171
172
173 addr = shmat(shmid, (void *) mca_sshmem_base_start_address, 0);
174 if (addr == (void *) -1L) {
175 opal_show_help("help-oshmem-sshmem.txt",
176 "create segment failure",
177 true,
178 "sysv",
179 ompi_process_info.nodename, (unsigned long long) size,
180 strerror(errno), errno);
181 opal_show_help("help-oshmem-sshmem-sysv.txt",
182 "sysv:create segment failure",
183 true);
184 shmctl(shmid, IPC_RMID, NULL);
185 return OSHMEM_ERR_OUT_OF_RESOURCE;
186 }
187
188 shmctl(shmid, IPC_RMID, NULL );
189
190 ds_buf->type = MAP_SEGMENT_ALLOC_SHM;
191 ds_buf->seg_id = shmid;
192 ds_buf->super.va_base = addr;
193 ds_buf->seg_size = size;
194 ds_buf->super.va_end = (void*)((uintptr_t)ds_buf->super.va_base + ds_buf->seg_size);
195
196 OPAL_OUTPUT_VERBOSE(
197 (70, oshmem_sshmem_base_framework.framework_output,
198 "%s: %s: create %s "
199 "(id: %d, addr: %p size: %lu)\n",
200 mca_sshmem_sysv_component.super.base_version.mca_type_name,
201 mca_sshmem_sysv_component.super.base_version.mca_component_name,
202 (rc ? "failure" : "successful"),
203 ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
204 );
205
206 return rc;
207 }
208
209
210
211
212
213 static void *
214 segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
215 {
216 assert(ds_buf);
217 assert(mkey->va_base == 0);
218
219 if (MAP_SEGMENT_SHM_INVALID == (int)(mkey->u.key)) {
220 return (mkey->va_base);
221 }
222
223 mkey->va_base = shmat((int)(mkey->u.key), 0, 0);
224
225 OPAL_OUTPUT_VERBOSE(
226 (70, oshmem_sshmem_base_framework.framework_output,
227 "%s: %s: attach successful "
228 "(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n",
229 mca_sshmem_sysv_component.super.base_version.mca_type_name,
230 mca_sshmem_sysv_component.super.base_version.mca_component_name,
231 ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size,
232 mkey->va_base, mkey->len, (unsigned long long)mkey->u.key)
233 );
234
235
236 return (mkey->va_base);
237 }
238
239
240 static int
241 segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
242 {
243 int rc = OSHMEM_SUCCESS;
244
245 assert(ds_buf);
246
247 OPAL_OUTPUT_VERBOSE(
248 (70, oshmem_sshmem_base_framework.framework_output,
249 "%s: %s: detaching "
250 "(id: %d, addr: %p size: %lu)\n",
251 mca_sshmem_sysv_component.super.base_version.mca_type_name,
252 mca_sshmem_sysv_component.super.base_version.mca_component_name,
253 ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
254 );
255
256 if (ds_buf->seg_id != MAP_SEGMENT_SHM_INVALID) {
257 shmctl(ds_buf->seg_id, IPC_RMID, NULL );
258 }
259
260 if (mca_sshmem_sysv_component.use_hp != 0) {
261
262
263
264
265
266 }
267
268
269
270
271 shmem_ds_reset(ds_buf);
272
273 return rc;
274 }
275
276
277 static int
278 segment_unlink(map_segment_t *ds_buf)
279 {
280
281
282 OPAL_OUTPUT_VERBOSE(
283 (70, oshmem_sshmem_base_framework.framework_output,
284 "%s: %s: unlinking "
285 "(id: %d, size: %lu)\n",
286 mca_sshmem_sysv_component.super.base_version.mca_type_name,
287 mca_sshmem_sysv_component.super.base_version.mca_component_name,
288 ds_buf->seg_id, (unsigned long)ds_buf->seg_size)
289 );
290
291
292
293
294
295 ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
296
297 MAP_SEGMENT_INVALIDATE(ds_buf);
298
299 return OSHMEM_SUCCESS;
300 }
301
302
303
304
305
306 size_t sshmem_sysv_gethugepagesize(void)
307 {
308 static size_t huge_page_size = 0;
309 char buf[256];
310 int size_kb;
311 FILE *f;
312
313
314 if (huge_page_size == 0) {
315 f = fopen("/proc/meminfo", "r");
316 if (f != NULL) {
317 while (fgets(buf, sizeof(buf), f)) {
318 if (sscanf(buf, "Hugepagesize: %d kB", &size_kb) == 1) {
319 huge_page_size = size_kb * 1024L;
320 break;
321 }
322 }
323 fclose(f);
324 }
325
326 if (huge_page_size == 0) {
327 huge_page_size = 2 * 1024L *1024L;
328 }
329 }
330
331 return huge_page_size;
332 }
333
334