oshmem/mca/sshmem/sysv/sshmem_sysv

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
module_init
module_finalize
segment_create
segment_attach
segment_detach
segment_unlink
sshmem_sysv_gethugepagesize
   1 /*
   2  * Copyright (c) 2014      Mellanox Technologies, Inc.
   3  *                         All rights reserved.
   4  * Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
   5  * Copyright (c) 2019      Research Organization for Information Science
   6  *                         and Technology (RIST).  All rights reserved.
   7  * $COPYRIGHT$
   8  *
   9  * Additional copyrights may follow
  10  *
  11  * $HEADER$
  12  */
  13 
  14 #include "oshmem_config.h"
  15 
  16 #include <errno.h>
  17 #ifdef HAVE_FCNTL_H
  18 #include <fcntl.h>
  19 #endif  /* HAVE_FCNTL_H */
  20 #ifdef HAVE_SYS_MMAN_H
  21 #include <sys/mman.h>
  22 #endif /* HAVE_SYS_MMAN_H */
  23 #ifdef HAVE_UNISTD_H
  24 #include <unistd.h>
  25 #endif /* HAVE_UNISTD_H */
  26 #ifdef HAVE_SYS_TYPES_H
  27 #include <sys/types.h>
  28 #endif /* HAVE_SYS_TYPES_H */
  29 #ifdef HAVE_SYS_IPC_H
  30 #include <sys/ipc.h>
  31 #endif /* HAVE_SYS_IPC_H */
  32 #if HAVE_SYS_SHM_H
  33 #include <sys/shm.h>
  34 #endif /* HAVE_SYS_SHM_H */
  35 #if HAVE_SYS_STAT_H
  36 #include <sys/stat.h>
  37 #endif /* HAVE_SYS_STAT_H */
  38 #include <string.h>
  39 #ifdef HAVE_NETDB_H
  40 #include <netdb.h>
  41 #endif /* HAVE_NETDB_H */
  42 
  43 #include "opal/constants.h"
  44 #include "opal_stdint.h"
  45 #include "opal/util/output.h"
  46 #include "opal/util/path.h"
  47 #include "opal/util/show_help.h"
  48 
  49 #include "oshmem/proc/proc.h"
  50 #include "oshmem/mca/sshmem/sshmem.h"
  51 #include "oshmem/mca/sshmem/base/base.h"
  52 
  53 #include "sshmem_sysv.h"
  54 
  55 
  56 /* ////////////////////////////////////////////////////////////////////////// */
  57 /* local functions */
  58 static int
  59 module_init(void);
  60 
  61 static int
  62 segment_create(map_segment_t *ds_buf,
  63                const char *file_name,
  64                size_t size, long hint);
  65 
  66 static void *
  67 segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
  68 
  69 static int
  70 segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
  71 
  72 static int
  73 segment_unlink(map_segment_t *ds_buf);
  74 
  75 static int
  76 module_finalize(void);
  77 
  78 /* sysv shmem module */
  79 mca_sshmem_sysv_module_t mca_sshmem_sysv_module = {
  80     /* super */
  81     {
  82         module_init,
  83         segment_create,
  84         segment_attach,
  85         segment_detach,
  86         segment_unlink,
  87         module_finalize
  88     }
  89 };
  90 
  91 
  92 /* ////////////////////////////////////////////////////////////////////////// */
  93 static int
  94 module_init(void)
  95 {
  96     /* nothing to do */
  97     return OSHMEM_SUCCESS;
  98 }
  99 
 100 /* ////////////////////////////////////////////////////////////////////////// */
 101 static int
 102 module_finalize(void)
 103 {
 104     /* nothing to do */
 105     return OSHMEM_SUCCESS;
 106 }
 107 
 108 
 109 /* ////////////////////////////////////////////////////////////////////////// */
 110 static int
 111 segment_create(map_segment_t *ds_buf,
 112                const char *file_name,
 113                size_t size, long hint)
 114 {
 115     int rc = OSHMEM_SUCCESS;
 116     void *addr = NULL;
 117     int shmid = MAP_SEGMENT_SHM_INVALID;
 118     int flags;
 119     int try_hp;
 120 
 121     assert(ds_buf);
 122 
 123     if (hint) {
 124         return OSHMEM_ERR_NOT_IMPLEMENTED;
 125     }
 126 
 127     /* init the contents of map_segment_t */
 128     shmem_ds_reset(ds_buf);
 129 
 130     /* for sysv shared memory we don't have to worry about the backing store
 131      * being located on a network file system... so no check is needed here.
 132      */
 133 
 134     /* create a new shared memory segment and save the shmid. note the use of
 135      * real_size here
 136      */
 137     flags = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR;
 138     try_hp = mca_sshmem_sysv_component.use_hp;
 139 #if defined (SHM_HUGETLB)
 140     flags |= ((0 != try_hp) ? SHM_HUGETLB : 0);
 141     size = ((size + sshmem_sysv_gethugepagesize() - 1) / sshmem_sysv_gethugepagesize()) * sshmem_sysv_gethugepagesize();
 142 #endif
 143 
 144     /* Create a new shared memory segment and save the shmid. */
 145 retry_alloc:
 146     shmid = shmget(IPC_PRIVATE, size, flags);
 147     if (shmid == MAP_SEGMENT_SHM_INVALID) {
 148         /* hugepage alloc was set to auto. Hopefully it failed because there are no
 149          * enough hugepages on the system. Turn it off and retry.
 150          */
 151         if (-1 == try_hp) {
 152             OPAL_OUTPUT_VERBOSE(
 153                     (10, oshmem_sshmem_base_framework.framework_output,
 154                      "failed to allocate %llu bytes with huge pages. "
 155                      "Using regular pages", (unsigned long long)size));
 156             flags = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR;
 157             try_hp = 0;
 158             goto retry_alloc;
 159         }
 160         opal_show_help("help-oshmem-sshmem.txt",
 161                        "create segment failure",
 162                        true,
 163                        "sysv",
 164                        ompi_process_info.nodename, (unsigned long long) size,
 165                        strerror(errno), errno);
 166         opal_show_help("help-oshmem-sshmem-sysv.txt",
 167                        "sysv:create segment failure",
 168                        true);
 169         return OSHMEM_ERROR;
 170     }
 171 
 172     /* Attach to the segment */
 173     addr = shmat(shmid, (void *) mca_sshmem_base_start_address, 0);
 174     if (addr == (void *) -1L) {
 175         opal_show_help("help-oshmem-sshmem.txt",
 176                        "create segment failure",
 177                        true,
 178                        "sysv",
 179                        ompi_process_info.nodename, (unsigned long long) size,
 180                        strerror(errno), errno);
 181         opal_show_help("help-oshmem-sshmem-sysv.txt",
 182                        "sysv:create segment failure",
 183                        true);
 184         shmctl(shmid, IPC_RMID, NULL);
 185         return OSHMEM_ERR_OUT_OF_RESOURCE;
 186     }
 187 
 188     shmctl(shmid, IPC_RMID, NULL );
 189 
 190     ds_buf->type = MAP_SEGMENT_ALLOC_SHM;
 191     ds_buf->seg_id = shmid;
 192     ds_buf->super.va_base = addr;
 193     ds_buf->seg_size = size;
 194     ds_buf->super.va_end = (void*)((uintptr_t)ds_buf->super.va_base + ds_buf->seg_size);
 195 
 196     OPAL_OUTPUT_VERBOSE(
 197           (70, oshmem_sshmem_base_framework.framework_output,
 198            "%s: %s: create %s "
 199            "(id: %d, addr: %p size: %lu)\n",
 200            mca_sshmem_sysv_component.super.base_version.mca_type_name,
 201            mca_sshmem_sysv_component.super.base_version.mca_component_name,
 202            (rc ? "failure" : "successful"),
 203            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
 204       );
 205 
 206     return rc;
 207 }
 208 
 209 /* ////////////////////////////////////////////////////////////////////////// */
 210 /**
 211  * segment_attach can only be called after a successful call to segment_create
 212  */
 213 static void *
 214 segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
 215 {
 216     assert(ds_buf);
 217     assert(mkey->va_base == 0);
 218 
 219     if (MAP_SEGMENT_SHM_INVALID == (int)(mkey->u.key)) {
 220         return (mkey->va_base);
 221     }
 222 
 223     mkey->va_base = shmat((int)(mkey->u.key), 0, 0);
 224 
 225     OPAL_OUTPUT_VERBOSE(
 226         (70, oshmem_sshmem_base_framework.framework_output,
 227          "%s: %s: attach successful "
 228             "(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n",
 229             mca_sshmem_sysv_component.super.base_version.mca_type_name,
 230             mca_sshmem_sysv_component.super.base_version.mca_component_name,
 231             ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size,
 232             mkey->va_base, mkey->len, (unsigned long long)mkey->u.key)
 233     );
 234 
 235     /* update returned base pointer with an offset that hides our stuff */
 236     return (mkey->va_base);
 237 }
 238 
 239 /* ////////////////////////////////////////////////////////////////////////// */
 240 static int
 241 segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
 242 {
 243     int rc = OSHMEM_SUCCESS;
 244 
 245     assert(ds_buf);
 246 
 247     OPAL_OUTPUT_VERBOSE(
 248         (70, oshmem_sshmem_base_framework.framework_output,
 249          "%s: %s: detaching "
 250             "(id: %d, addr: %p size: %lu)\n",
 251             mca_sshmem_sysv_component.super.base_version.mca_type_name,
 252             mca_sshmem_sysv_component.super.base_version.mca_component_name,
 253             ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
 254     );
 255 
 256     if (ds_buf->seg_id != MAP_SEGMENT_SHM_INVALID) {
 257         shmctl(ds_buf->seg_id, IPC_RMID, NULL );
 258     }
 259 
 260     if (mca_sshmem_sysv_component.use_hp != 0) {
 261         /**
 262          *  Workaround kernel panic when detaching huge pages from user space simultanously from several processes
 263          *  dont detach here instead let kernel do it during process cleanup
 264          */
 265         /* shmdt((void *)ds_buf->seg_base_addr); */
 266     }
 267 
 268     /* reset the contents of the map_segment_t associated with this
 269      * shared memory segment.
 270      */
 271     shmem_ds_reset(ds_buf);
 272 
 273     return rc;
 274 }
 275 
 276 /* ////////////////////////////////////////////////////////////////////////// */
 277 static int
 278 segment_unlink(map_segment_t *ds_buf)
 279 {
 280     /* not much unlink work needed for sysv */
 281 
 282     OPAL_OUTPUT_VERBOSE(
 283         (70, oshmem_sshmem_base_framework.framework_output,
 284          "%s: %s: unlinking "
 285          "(id: %d, size: %lu)\n",
 286          mca_sshmem_sysv_component.super.base_version.mca_type_name,
 287          mca_sshmem_sysv_component.super.base_version.mca_component_name,
 288          ds_buf->seg_id, (unsigned long)ds_buf->seg_size)
 289     );
 290 
 291     /* don't completely reset.  in particular, only reset
 292      * the id and flip the invalid bit.  size and name values will remain valid
 293      * across unlinks. other information stored in flags will remain untouched.
 294      */
 295     ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
 296     /* note: this is only changing the valid bit to 0. */
 297     MAP_SEGMENT_INVALIDATE(ds_buf);
 298 
 299     return OSHMEM_SUCCESS;
 300 }
 301 
 302 /*
 303  * Get current huge page size
 304  *
 305  */
 306 size_t sshmem_sysv_gethugepagesize(void)
 307 {
 308     static size_t huge_page_size = 0;
 309     char buf[256];
 310     int size_kb;
 311     FILE *f;
 312 
 313     /* Cache the huge page size value */
 314     if (huge_page_size == 0) {
 315         f = fopen("/proc/meminfo", "r");
 316         if (f != NULL) {
 317             while (fgets(buf, sizeof(buf), f)) {
 318                 if (sscanf(buf, "Hugepagesize: %d kB", &size_kb) == 1) {
 319                     huge_page_size = size_kb * 1024L;
 320                     break;
 321                 }
 322             }
 323             fclose(f);
 324         }
 325 
 326         if (huge_page_size == 0) {
 327             huge_page_size = 2 * 1024L *1024L;
 328         }
 329     }
 330 
 331     return huge_page_size;
 332 }
 333 
 334
/* [<][>][^][v][top][bottom][index][help] */
root/oshmem/mca/sshmem/sysv/sshmem_sysv_module.c

DEFINITIONS
