root/opal/mca/shmem/posix/shmem_posix_module.c

DEFINITIONS

1. shmem_ds_reset
2. module_init
3. module_finalize
4. ds_copy
5. segment_create
6. segment_attach
7. segment_detach
8. segment_unlink

   1 /*
   2  * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
   3  *                         University Research and Technology
   4  *                         Corporation.  All rights reserved.
   5  * Copyright (c) 2004-2005 The University of Tennessee and The University
   6  *                         of Tennessee Research Foundation.  All rights
   7  *                         reserved.
   8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   9  *                         University of Stuttgart.  All rights reserved.
  10  * Copyright (c) 2004-2005 The Regents of the University of California.
  11  *                         All rights reserved.
  12  * Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
  13  * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
  14  * Copyright (c) 2010-2012 Los Alamos National Security, LLC.
  15  *                         All rights reserved.
  16  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  17  * $COPYRIGHT$
  18  *
  19  * Additional copyrights may follow
  20  *
  21  * $HEADER$
  22  */
  23 
  24 #include "opal_config.h"
  25 
  26 #include <errno.h>
  27 #ifdef HAVE_FCNTL_H
  28 #include <fcntl.h>
  29 #endif  /* HAVE_FCNTL_H */
  30 #if OPAL_HAVE_SOLARIS && !defined(_POSIX_C_SOURCE)
  31   #define _POSIX_C_SOURCE 200112L /* Required for shm_{open,unlink} decls */
  32   #include <sys/mman.h>
  33   #undef _POSIX_C_SOURCE
  34 #else
  35 #ifdef HAVE_SYS_MMAN_H
  36 #include <sys/mman.h>
  37 #endif /* HAVE_SYS_MMAN_H */
  38 #endif
  39 #ifdef HAVE_UNISTD_H
  40 #include <unistd.h>
  41 #endif /* HAVE_UNISTD_H */
  42 #ifdef HAVE_SYS_TYPES_H
  43 #include <sys/types.h>
  44 #endif /* HAVE_SYS_TYPES_H */
  45 #include <string.h>
  46 #ifdef HAVE_NETDB_H
  47 #include <netdb.h>
  48 #endif /* HAVE_NETDB_H */
  49 
  50 #include "opal/constants.h"
  51 #include "opal_stdint.h"
  52 #include "opal/util/output.h"
  53 #include "opal/util/path.h"
  54 #include "opal/util/show_help.h"
  55 #include "opal/mca/shmem/shmem.h"
  56 #include "opal/mca/shmem/base/base.h"
  57 
  58 #include "shmem_posix.h"
  59 #include "shmem_posix_common_utils.h"
  60 
  61 /* for tons of debug output: -mca shmem_base_verbose 70 */
  62 
  63 /* ////////////////////////////////////////////////////////////////////////// */
  64 /* local functions */
  65 static int
  66 module_init(void);
  67 
  68 static int
  69 segment_create(opal_shmem_ds_t *ds_buf,
  70                const char *file_name,
  71                size_t size);
  72 
  73 static int
  74 ds_copy(const opal_shmem_ds_t *from,
  75         opal_shmem_ds_t *to);
  76 
  77 static void *
  78 segment_attach(opal_shmem_ds_t *ds_buf);
  79 
  80 static int
  81 segment_detach(opal_shmem_ds_t *ds_buf);
  82 
  83 static int
  84 segment_unlink(opal_shmem_ds_t *ds_buf);
  85 
  86 static int
  87 module_finalize(void);
  88 
  89 /* posix shmem module */
  90 opal_shmem_posix_module_t opal_shmem_posix_module = {
  91     .super = {
  92         .module_init = module_init,
  93         .segment_create = segment_create,
  94         .ds_copy = ds_copy,
  95         .segment_attach = segment_attach,
  96         .segment_detach = segment_detach,
  97         .unlink = segment_unlink,
  98         .module_finalize = module_finalize
  99     }
 100 };
 101 
 102 /* ////////////////////////////////////////////////////////////////////////// */
 103 /* private utility functions */
 104 /* ////////////////////////////////////////////////////////////////////////// */
 105 
 106 /* ////////////////////////////////////////////////////////////////////////// */
 107 /**
 108  * completely resets the contents of *ds_buf
 109  */
 110 static inline void
 111 shmem_ds_reset(opal_shmem_ds_t *ds_buf)
 112 {
 113     /* don't print ds_buf info here, as we may be printing garbage. */
 114     OPAL_OUTPUT_VERBOSE(
 115         (70, opal_shmem_base_framework.framework_output,
 116          "%s: %s: shmem_ds_resetting\n",
 117          mca_shmem_posix_component.super.base_version.mca_type_name,
 118          mca_shmem_posix_component.super.base_version.mca_component_name)
 119     );
 120 
 121     ds_buf->seg_cpid = 0;
 122     OPAL_SHMEM_DS_RESET_FLAGS(ds_buf);
 123     ds_buf->seg_id = OPAL_SHMEM_DS_ID_INVALID;
 124     ds_buf->seg_size = 0;
 125     memset(ds_buf->seg_name, '\0', OPAL_PATH_MAX);
 126     ds_buf->seg_base_addr = (unsigned char *)MAP_FAILED;
 127 }
 128 
 129 /* ////////////////////////////////////////////////////////////////////////// */
 130 static int
 131 module_init(void)
 132 {
 133     /* nothing to do */
 134     return OPAL_SUCCESS;
 135 }
 136 
 137 /* ////////////////////////////////////////////////////////////////////////// */
 138 static int
 139 module_finalize(void)
 140 {
 141     /* nothing to do */
 142     return OPAL_SUCCESS;
 143 }
 144 
 145 /* ////////////////////////////////////////////////////////////////////////// */
 146 static int
 147 ds_copy(const opal_shmem_ds_t *from,
 148         opal_shmem_ds_t *to)
 149 {
 150     memcpy(to, from, sizeof(opal_shmem_ds_t));
 151 
 152     OPAL_OUTPUT_VERBOSE(
 153         (70, opal_shmem_base_framework.framework_output,
 154          "%s: %s: ds_copy complete "
 155          "from: (id: %d, size: %lu, "
 156          "name: %s flags: 0x%02x) "
 157          "to: (id: %d, size: %lu, "
 158          "name: %s flags: 0x%02x)\n",
 159          mca_shmem_posix_component.super.base_version.mca_type_name,
 160          mca_shmem_posix_component.super.base_version.mca_component_name,
 161          from->seg_id, (unsigned long)from->seg_size, from->seg_name,
 162          from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
 163          to->flags)
 164     );
 165 
 166     return OPAL_SUCCESS;
 167 }
 168 
 169 /* ////////////////////////////////////////////////////////////////////////// */
 170 static int
 171 segment_create(opal_shmem_ds_t *ds_buf,
 172                const char *file_name,
 173                size_t size)
 174 {
 175     int rc = OPAL_SUCCESS;
 176     pid_t my_pid = getpid();
 177     /* the real size of the shared memory segment.  this includes enough space
 178      * to store our segment header.
 179      */
 180     size_t real_size = size + sizeof(opal_shmem_seg_hdr_t);
 181     opal_shmem_seg_hdr_t *seg_hdrp = MAP_FAILED;
 182 
 183     /* init the contents of opal_shmem_ds_t */
 184     shmem_ds_reset(ds_buf);
 185 
 186     /* for posix shared memory we don't have to worry about the backing store
 187      * being located on a network file system... so no check is needed here.
 188      */
 189 
 190     /* calling shmem_posix_shm_open searches for an available posix shared
 191      * memory object name and upon successful completion populates the name
 192      * buffer
 193      */
 194     if (-1 == (ds_buf->seg_id = shmem_posix_shm_open(
 195                                     ds_buf->seg_name,
 196                                     OPAL_SHMEM_POSIX_FILE_LEN_MAX - 1))) {
 197         /* snaps!  something happened in posix_shm_open.  don't report anything
 198          * here because posix_shm_open will display all the necessary info.
 199          */
 200         rc = OPAL_ERROR;
 201         goto out;
 202     }
 203     /* size backing file - note the use of real_size here */
 204     else if (0 != ftruncate(ds_buf->seg_id, real_size)) {
 205         int err = errno;
 206         char hn[OPAL_MAXHOSTNAMELEN];
 207         gethostname(hn, sizeof(hn));
 208         opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
 209                        "ftruncate(2)", "", strerror(err), err);
 210         rc = OPAL_ERROR;
 211         goto out;
 212     }
 213     else if (MAP_FAILED == (seg_hdrp = (opal_shmem_seg_hdr_t*)mmap(NULL, real_size,
 214                                                                    PROT_READ | PROT_WRITE, MAP_SHARED,
 215                                                                    ds_buf->seg_id, 0))) {
 216         int err = errno;
 217         char hn[OPAL_MAXHOSTNAMELEN];
 218         gethostname(hn, sizeof(hn));
 219         opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
 220                        "mmap(2)", "", strerror(err), err);
 221         rc = OPAL_ERROR;
 222         goto out;
 223     }
 224     /* all is well */
 225     else {
 226         /* -- initialize the shared memory segment -- */
 227         opal_atomic_rmb();
 228 
 229         /* init segment lock */
 230         opal_atomic_lock_init(&seg_hdrp->lock, OPAL_ATOMIC_LOCK_UNLOCKED);
 231         /* i was the creator of this segment, so note that fact */
 232         seg_hdrp->cpid = my_pid;
 233 
 234         opal_atomic_wmb();
 235 
 236         /* -- initialize the contents of opal_shmem_ds_t -- */
 237         ds_buf->seg_cpid = my_pid;
 238         ds_buf->seg_size = real_size;
 239         ds_buf->seg_base_addr = (unsigned char *)seg_hdrp;
 240 
 241         /* notice that we are not setting ds_buf->name here.  at this point,
 242          * posix_shm_open was successful, so the contents of ds_buf->name are
 243          * already set for us :-)
 244          */
 245 
 246         /* set "valid" bit because setment creation was successful */
 247         OPAL_SHMEM_DS_SET_VALID(ds_buf);
 248 
 249         OPAL_OUTPUT_VERBOSE(
 250             (70, opal_shmem_base_framework.framework_output,
 251              "%s: %s: create successful "
 252              "(id: %d, size: %lu, name: %s)\n",
 253              mca_shmem_posix_component.super.base_version.mca_type_name,
 254              mca_shmem_posix_component.super.base_version.mca_component_name,
 255              ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
 256         );
 257     }
 258 
 259 out:
 260     /* in this component, the id is the file descriptor returned by open.  this
 261      * check is here to see if it is safe to call close on the file descriptor.
 262      * that is, we are making sure that our call to open was successful and
 263      * we are not not in an error path.
 264      */
 265     if (-1 != ds_buf->seg_id) {
 266         if (0 != close(ds_buf->seg_id)) {
 267             int err = errno;
 268             char hn[OPAL_MAXHOSTNAMELEN];
 269             gethostname(hn, sizeof(hn));
 270             opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
 271                            "close(2)", "", strerror(err), err);
 272             rc = OPAL_ERROR;
 273          }
 274      }
 275     /* an error occured, so invalidate the shmem object and release any
 276      * allocated resources.
 277      */
 278     if (OPAL_SUCCESS != rc) {
 279         /* posix_shm_open was successful, but something else wasn't.
 280          * note: if the id is not equal to -1 and we are here, name will be
 281          * valid.  that is, we can safely call shm_unlink with ds_buf->name.
 282          */
 283         if (-1 != ds_buf->seg_id) {
 284             shm_unlink(ds_buf->seg_name);
 285         }
 286         if (MAP_FAILED != seg_hdrp) {
 287             munmap((void*)seg_hdrp, real_size);
 288         }
 289         /* always invalidate in this error path */
 290         shmem_ds_reset(ds_buf);
 291     }
 292     return rc;
 293 }
 294 
 295 /* ////////////////////////////////////////////////////////////////////////// */
 296 /**
 297  * segment_attach can only be called after a successful call to segment_create
 298  */
 299 static void *
 300 segment_attach(opal_shmem_ds_t *ds_buf)
 301 {
 302     pid_t my_pid = getpid();
 303 
 304     if (my_pid != ds_buf->seg_cpid) {
 305         if (-1 == (ds_buf->seg_id = shm_open(ds_buf->seg_name, O_RDWR, 0600))) {
 306             int err = errno;
 307             char hn[OPAL_MAXHOSTNAMELEN];
 308             gethostname(hn, sizeof(hn));
 309             opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
 310                            "open(2)", "", strerror(err), err);
 311             return NULL;
 312         }
 313         else if (MAP_FAILED == (ds_buf->seg_base_addr =
 314                                 (unsigned char*)mmap(NULL, ds_buf->seg_size,
 315                                                      PROT_READ | PROT_WRITE, MAP_SHARED,
 316                                                      ds_buf->seg_id, 0))) {
 317             int err = errno;
 318             char hn[OPAL_MAXHOSTNAMELEN];
 319             gethostname(hn, sizeof(hn));
 320             opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
 321                            "mmap(2)", "", strerror(err), err);
 322             /* mmap failed, so shm_unlink and return NULL - no error check here
 323              * because we are already in an error path...
 324              */
 325             shm_unlink(ds_buf->seg_name);
 326             return NULL;
 327         }
 328         /* all is well */
 329         else {
 330             /* if close fails here, that's okay.  just let the user know and
 331              * continue.  if we got this far, open and mmap were successful...
 332              */
 333             if (0 != close(ds_buf->seg_id)) {
 334                 int err = errno;
 335                 char hn[OPAL_MAXHOSTNAMELEN];
 336                 gethostname(hn, sizeof(hn));
 337                 opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1,
 338                                hn, "close(2)", "", strerror(err), err);
 339              }
 340         }
 341     }
 342     /* else i was the segment creator.  nothing to do here because all the hard
 343      * work was done in segment_create :-).
 344      */
 345 
 346     OPAL_OUTPUT_VERBOSE(
 347         (70, opal_shmem_base_framework.framework_output,
 348          "%s: %s: attach successful "
 349          "(id: %d, size: %lu, name: %s)\n",
 350          mca_shmem_posix_component.super.base_version.mca_type_name,
 351          mca_shmem_posix_component.super.base_version.mca_component_name,
 352          ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
 353     );
 354 
 355     /* update returned base pointer with an offset that hides our stuff */
 356     return (ds_buf->seg_base_addr + sizeof(opal_shmem_seg_hdr_t));
 357 }
 358 
 359 /* ////////////////////////////////////////////////////////////////////////// */
 360 static int
 361 segment_detach(opal_shmem_ds_t *ds_buf)
 362 {
 363     int rc = OPAL_SUCCESS;
 364 
 365     OPAL_OUTPUT_VERBOSE(
 366         (70, opal_shmem_base_framework.framework_output,
 367          "%s: %s: detaching "
 368          "(id: %d, size: %lu, name: %s)\n",
 369          mca_shmem_posix_component.super.base_version.mca_type_name,
 370          mca_shmem_posix_component.super.base_version.mca_component_name,
 371          ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
 372     );
 373 
 374     if (0 != munmap((void*)ds_buf->seg_base_addr, ds_buf->seg_size)) {
 375         int err = errno;
 376         char hn[OPAL_MAXHOSTNAMELEN];
 377         gethostname(hn, sizeof(hn));
 378         opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
 379                        "munmap(2)", "", strerror(err), err);
 380         rc = OPAL_ERROR;
 381     }
 382     /* reset the contents of the opal_shmem_ds_t associated with this
 383      * shared memory segment.
 384      */
 385     shmem_ds_reset(ds_buf);
 386     return rc;
 387 }
 388 
 389 /* ////////////////////////////////////////////////////////////////////////// */
 390 static int
 391 segment_unlink(opal_shmem_ds_t *ds_buf)
 392 {
 393     OPAL_OUTPUT_VERBOSE(
 394         (70, opal_shmem_base_framework.framework_output,
 395          "%s: %s: unlinking "
 396          "(id: %d, size: %lu, name: %s)\n",
 397          mca_shmem_posix_component.super.base_version.mca_type_name,
 398          mca_shmem_posix_component.super.base_version.mca_component_name,
 399          ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
 400     );
 401 
 402     if (-1 == shm_unlink(ds_buf->seg_name)) {
 403         int err = errno;
 404         char hn[OPAL_MAXHOSTNAMELEN];
 405         gethostname(hn, sizeof(hn));
 406         opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
 407                        "shm_unlink(2)", ds_buf->seg_name, strerror(err), err);
 408         return OPAL_ERROR;
 409     }
 410 
 411     /* don't completely reset the opal_shmem_ds_t.  in particular, only reset
 412      * the id and flip the invalid bit.  size and name values will remain valid
 413      * across unlinks. other information stored in flags will remain untouched.
 414      */
 415     ds_buf->seg_id = OPAL_SHMEM_DS_ID_INVALID;
 416     /* note: this is only chaning the valid bit to 0. */
 417     OPAL_SHMEM_DS_INVALIDATE(ds_buf);
 418     return OPAL_SUCCESS;
 419 }
 420