This source file includes following definitions.
- shmem_ds_reset
- enough_space
- module_init
- module_finalize
- ds_copy
- sdbm_hash
- path_usable
- get_uniq_file_name
- segment_create
- segment_attach
- segment_detach
- segment_unlink
   1 
   2 
   3 
   4 
   5 
   6 
   7 
   8 
   9 
  10 
  11 
  12 
  13 
  14 
  15 
  16 
  17 
  18 
  19 
  20 
  21 
  22 
  23 
  24 
  25 
  26 #include "opal_config.h"
  27 
  28 #include <errno.h>
  29 #ifdef HAVE_FCNTL_H
  30 #include <fcntl.h>
  31 #endif  
  32 #ifdef HAVE_SYS_MMAN_H
  33 #include <sys/mman.h>
  34 #endif 
  35 #ifdef HAVE_UNISTD_H
  36 #include <unistd.h>
  37 #endif 
  38 #ifdef HAVE_SYS_TYPES_H
  39 #include <sys/types.h>
  40 #endif 
  41 #include <string.h>
  42 #ifdef HAVE_NETDB_H
  43 #include <netdb.h>
  44 #endif 
  45 #include <time.h>
  46 #ifdef HAVE_SYS_STAT_H
  47 #include <sys/stat.h>
  48 #endif 
  49 
  50 #include "opal_stdint.h"
  51 #include "opal/constants.h"
  52 #include "opal/util/alfg.h"
  53 #include "opal/util/output.h"
  54 #include "opal/util/path.h"
  55 #include "opal/util/show_help.h"
  56 #include "opal/util/string_copy.h"
  57 #include "opal/mca/shmem/shmem.h"
  58 #include "opal/mca/shmem/base/base.h"
  59 
  60 #include "shmem_mmap.h"
  61 
  62 
  63 
  64 
  65 
  66 
  67 static int
  68 module_init(void);
  69 
  70 static int
  71 segment_create(opal_shmem_ds_t *ds_buf,
  72                const char *file_name,
  73                size_t size);
  74 
  75 static int
  76 ds_copy(const opal_shmem_ds_t *from,
  77         opal_shmem_ds_t *to);
  78 
  79 static void *
  80 segment_attach(opal_shmem_ds_t *ds_buf);
  81 
  82 static int
  83 segment_detach(opal_shmem_ds_t *ds_buf);
  84 
  85 static int
  86 segment_unlink(opal_shmem_ds_t *ds_buf);
  87 
  88 static int
  89 module_finalize(void);
  90 
  91 static int
  92 enough_space(const char *filename,
  93              size_t space_req,
  94              uint64_t *space_avail,
  95              bool *result);
  96 
  97 
  98 
  99 
 100 opal_shmem_mmap_module_t opal_shmem_mmap_module = {
 101     .super = {
 102         .module_init = module_init,
 103         .segment_create = segment_create,
 104         .ds_copy = ds_copy,
 105         .segment_attach = segment_attach,
 106         .segment_detach = segment_detach,
 107         .unlink = segment_unlink,
 108         .module_finalize = module_finalize
 109     }
 110 };
 111 
 112 
 113 
 114 
 115 
 116 
 117 
 118 
 119 
 120 static inline void
 121 shmem_ds_reset(opal_shmem_ds_t *ds_buf)
 122 {
 123     
 124     OPAL_OUTPUT_VERBOSE(
 125         (70, opal_shmem_base_framework.framework_output,
 126          "%s: %s: shmem_ds_resetting\n",
 127          mca_shmem_mmap_component.super.base_version.mca_type_name,
 128          mca_shmem_mmap_component.super.base_version.mca_component_name)
 129     );
 130 
 131     ds_buf->seg_cpid = 0;
 132     OPAL_SHMEM_DS_RESET_FLAGS(ds_buf);
 133     ds_buf->seg_id = OPAL_SHMEM_DS_ID_INVALID;
 134     ds_buf->seg_size = 0;
 135     memset(ds_buf->seg_name, '\0', OPAL_PATH_MAX);
 136     ds_buf->seg_base_addr = (unsigned char *)MAP_FAILED;
 137 }
 138 
 139 
 140 static int
 141 enough_space(const char *filename,
 142              size_t space_req,
 143              uint64_t *space_avail,
 144              bool *result)
 145 {
 146     uint64_t avail = 0;
 147     size_t fluff = (size_t)(.05 * space_req);
 148     bool enough = false;
 149     char *last_sep = NULL;
 150     
 151 
 152     char *target_dir = strdup(filename);
 153     int rc;
 154 
 155     if (NULL == target_dir) {
 156         rc = OPAL_ERR_OUT_OF_RESOURCE;
 157         goto out;
 158     }
 159     
 160     last_sep = strrchr(target_dir, OPAL_PATH_SEP[0]);
 161     *last_sep = '\0';
 162     
 163     if (OPAL_SUCCESS != (rc = opal_path_df(target_dir, &avail))) {
 164         OPAL_OUTPUT_VERBOSE(
 165             (70, opal_shmem_base_framework.framework_output,
 166              "WARNING: opal_path_df failure!")
 167         );
 168         goto out;
 169     }
 170     
 171     if (avail >= space_req + fluff) {
 172         enough = true;
 173     }
 174     else {
 175         OPAL_OUTPUT_VERBOSE(
 176             (70, opal_shmem_base_framework.framework_output,
 177              "WARNING: not enough space on %s to meet request!"
 178              "available: %"PRIu64 "requested: %lu", target_dir,
 179              avail, (unsigned long)space_req + fluff)
 180         );
 181     }
 182 
 183 out:
 184     if (NULL != target_dir) {
 185         free(target_dir);
 186     }
 187     *result = enough;
 188     *space_avail = avail;
 189     return rc;
 190 }
 191 
 192 
 193 static int
 194 module_init(void)
 195 {
 196     
 197     return OPAL_SUCCESS;
 198 }
 199 
 200 
 201 static int
 202 module_finalize(void)
 203 {
 204     
 205     return OPAL_SUCCESS;
 206 }
 207 
 208 
 209 static int
 210 ds_copy(const opal_shmem_ds_t *from,
 211         opal_shmem_ds_t *to)
 212 {
 213     memcpy(to, from, sizeof(opal_shmem_ds_t));
 214 
 215     OPAL_OUTPUT_VERBOSE(
 216         (70, opal_shmem_base_framework.framework_output,
 217          "%s: %s: ds_copy complete "
 218          "from: (id: %d, size: %lu, "
 219          "name: %s flags: 0x%02x) "
 220          "to: (id: %d, size: %lu, "
 221          "name: %s flags: 0x%02x)\n",
 222          mca_shmem_mmap_component.super.base_version.mca_type_name,
 223          mca_shmem_mmap_component.super.base_version.mca_component_name,
 224          from->seg_id, (unsigned long)from->seg_size, from->seg_name,
 225          from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
 226          to->flags)
 227     );
 228 
 229     return OPAL_SUCCESS;
 230 }
 231 
 232 
 233 static unsigned long
 234 sdbm_hash(const unsigned char *hash_key)
 235 {
 236     unsigned long str_hash = 0;
 237     int c;
 238 
 239     
 240     while ((c = *hash_key++)) {
 241         str_hash = c + (str_hash << 6) + (str_hash << 16) - str_hash;
 242     }
 243     return str_hash;
 244 }
 245 
 246 
 247 static bool
 248 path_usable(const char *path, int *stat_errno)
 249 {
 250     struct stat buf;
 251     int rc;
 252 
 253     rc = stat(path, &buf);
 254     *stat_errno = errno;
 255     return (0 == rc);
 256 }
 257 
 258 
 259 
 260 
 261 
 262 
 263 
 264 
 265 
 266 static char *
 267 get_uniq_file_name(const char *base_path, const char *hash_key)
 268 {
 269     char *uniq_name_buf = NULL;
 270     unsigned long str_hash = 0;
 271     pid_t my_pid;
 272     opal_rng_buff_t rand_buff;
 273     uint32_t rand_num;
 274 
 275     
 276     if (NULL == hash_key) {
 277         return NULL;
 278     }
 279     if (NULL == (uniq_name_buf = calloc(OPAL_PATH_MAX, sizeof(char)))) {
 280         
 281         return NULL;
 282     }
 283 
 284     my_pid = getpid();
 285     opal_srand(&rand_buff,((uint32_t)(time(NULL) + my_pid)));
 286     rand_num = opal_rand(&rand_buff) % 1024;
 287     str_hash = sdbm_hash((unsigned char *)hash_key);
 288     
 289     snprintf(uniq_name_buf, OPAL_PATH_MAX, "%s/open_mpi_shmem_mmap.%d_%lu_%d",
 290              base_path, (int)my_pid, str_hash, rand_num);
 291 
 292     return uniq_name_buf;
 293 }
 294 
 295 
 296 static int
 297 segment_create(opal_shmem_ds_t *ds_buf,
 298                const char *file_name,
 299                size_t size)
 300 {
 301     int rc = OPAL_SUCCESS;
 302     char *real_file_name = NULL;
 303     pid_t my_pid = getpid();
 304     bool space_available = false;
 305     uint64_t amount_space_avail = 0;
 306 
 307     
 308 
 309 
 310     size_t real_size = size + sizeof(opal_shmem_seg_hdr_t);
 311     opal_shmem_seg_hdr_t *seg_hdrp = MAP_FAILED;
 312 
 313     
 314     shmem_ds_reset(ds_buf);
 315 
 316     
 317     if (0 != opal_shmem_mmap_relocate_backing_file) {
 318         int err;
 319         if (path_usable(opal_shmem_mmap_backing_file_base_dir, &err)) {
 320             if (NULL ==
 321                 (real_file_name =
 322                      get_uniq_file_name(opal_shmem_mmap_backing_file_base_dir,
 323                                         file_name))) {
 324                 
 325                 return OPAL_ERROR;
 326             }
 327         }
 328         
 329 
 330 
 331 
 332         else if (opal_shmem_mmap_relocate_backing_file < 0) {
 333             opal_output(0, "shmem: mmap: WARNING: could not relocate "
 334                         "backing store to \"%s\" (%s).  Continuing with "
 335                         "default path.\n",
 336                         opal_shmem_mmap_backing_file_base_dir, strerror(err));
 337         }
 338         
 339         else {
 340             opal_output(0, "shmem: mmap: WARNING: could not relocate "
 341                         "backing store to \"%s\" (%s).  Cannot continue with "
 342                         "shmem mmap.\n", opal_shmem_mmap_backing_file_base_dir,
 343                         strerror(err));
 344             return OPAL_ERROR;
 345         }
 346     }
 347     
 348     if (NULL == real_file_name) {
 349         
 350         if (NULL == (real_file_name = strdup(file_name))) {
 351             
 352             return OPAL_ERROR;
 353         }
 354     }
 355 
 356     OPAL_OUTPUT_VERBOSE(
 357         (70, opal_shmem_base_framework.framework_output,
 358          "%s: %s: backing store base directory: %s\n",
 359          mca_shmem_mmap_component.super.base_version.mca_type_name,
 360          mca_shmem_mmap_component.super.base_version.mca_component_name,
 361          real_file_name)
 362     );
 363 
 364     
 365 
 366 
 367 
 368     if (opal_shmem_mmap_nfs_warning && opal_path_nfs(real_file_name, NULL)) {
 369         char hn[OPAL_MAXHOSTNAMELEN];
 370         gethostname(hn, sizeof(hn));
 371         opal_show_help("help-opal-shmem-mmap.txt", "mmap on nfs", 1, hn,
 372                        real_file_name);
 373     }
 374     
 375     if (OPAL_SUCCESS != (rc = enough_space(real_file_name,
 376                                            real_size,
 377                                            &amount_space_avail,
 378                                            &space_available))) {
 379         opal_output(0, "shmem: mmap: an error occurred while determining "
 380                     "whether or not %s could be created.", real_file_name);
 381         
 382         goto out;
 383     }
 384     if (!space_available) {
 385         char hn[OPAL_MAXHOSTNAMELEN];
 386         gethostname(hn, sizeof(hn));
 387         rc = OPAL_ERR_OUT_OF_RESOURCE;
 388         opal_show_help("help-opal-shmem-mmap.txt", "target full", 1,
 389                        real_file_name, hn, (unsigned long)real_size,
 390                        (unsigned long long)amount_space_avail);
 391         goto out;
 392     }
 393     
 394     if (-1 == (ds_buf->seg_id = open(real_file_name, O_CREAT | O_RDWR, 0600))) {
 395         int err = errno;
 396         char hn[OPAL_MAXHOSTNAMELEN];
 397         gethostname(hn, sizeof(hn));
 398         opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
 399                        "open(2)", "", strerror(err), err);
 400         rc = OPAL_ERROR;
 401         goto out;
 402     }
 403     
 404     if (0 != ftruncate(ds_buf->seg_id, real_size)) {
 405         int err = errno;
 406         char hn[OPAL_MAXHOSTNAMELEN];
 407         gethostname(hn, sizeof(hn));
 408         opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
 409                        "ftruncate(2)", "", strerror(err), err);
 410         rc = OPAL_ERROR;
 411         goto out;
 412     }
 413     if (MAP_FAILED == (seg_hdrp = (opal_shmem_seg_hdr_t *)
 414                                   mmap(NULL, real_size,
 415                                        PROT_READ | PROT_WRITE, MAP_SHARED,
 416                                        ds_buf->seg_id, 0))) {
 417         int err = errno;
 418         char hn[OPAL_MAXHOSTNAMELEN];
 419         gethostname(hn, sizeof(hn));
 420         opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
 421                        "mmap(2)", "", strerror(err), err);
 422         rc = OPAL_ERROR;
 423         goto out;
 424     }
 425     
 426     else {
 427         
 428         opal_atomic_rmb();
 429 
 430         
 431         opal_atomic_lock_init(&seg_hdrp->lock, OPAL_ATOMIC_LOCK_UNLOCKED);
 432         
 433         seg_hdrp->cpid = my_pid;
 434 
 435         opal_atomic_wmb();
 436 
 437         
 438         ds_buf->seg_cpid = my_pid;
 439         ds_buf->seg_size = real_size;
 440         ds_buf->seg_base_addr = (unsigned char *)seg_hdrp;
 441         (void)opal_string_copy(ds_buf->seg_name, real_file_name, OPAL_PATH_MAX);
 442 
 443         
 444         OPAL_SHMEM_DS_SET_VALID(ds_buf);
 445 
 446         OPAL_OUTPUT_VERBOSE(
 447             (70, opal_shmem_base_framework.framework_output,
 448              "%s: %s: create successful "
 449              "(id: %d, size: %lu, name: %s)\n",
 450              mca_shmem_mmap_component.super.base_version.mca_type_name,
 451              mca_shmem_mmap_component.super.base_version.mca_component_name,
 452              ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
 453         );
 454     }
 455 
 456 out:
 457     
 458 
 459 
 460 
 461 
 462     if (-1 != ds_buf->seg_id) {
 463         if (0 != close(ds_buf->seg_id)) {
 464             int err = errno;
 465             char hn[OPAL_MAXHOSTNAMELEN];
 466             gethostname(hn, sizeof(hn));
 467             opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
 468                            "close(2)", "", strerror(err), err);
 469             rc = OPAL_ERROR;
 470          }
 471      }
 472     
 473     if (OPAL_SUCCESS != rc) {
 474         if (MAP_FAILED != seg_hdrp) {
 475             munmap((void *)seg_hdrp, real_size);
 476         }
 477         shmem_ds_reset(ds_buf);
 478     }
 479     
 480     if (NULL != real_file_name) {
 481         free(real_file_name);
 482     }
 483     return rc;
 484 }
 485 
 486 
 487 
 488 
 489 
 490 static void *
 491 segment_attach(opal_shmem_ds_t *ds_buf)
 492 {
 493     pid_t my_pid = getpid();
 494 
 495     if (my_pid != ds_buf->seg_cpid) {
 496         if (-1 == (ds_buf->seg_id = open(ds_buf->seg_name, O_RDWR))) {
 497             int err = errno;
 498             char hn[OPAL_MAXHOSTNAMELEN];
 499             gethostname(hn, sizeof(hn));
 500             opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
 501                            "open(2)", "", strerror(err), err);
 502             return NULL;
 503         }
 504         if (MAP_FAILED == (ds_buf->seg_base_addr = (unsigned char *)
 505                               mmap(NULL, ds_buf->seg_size,
 506                                    PROT_READ | PROT_WRITE, MAP_SHARED,
 507                                    ds_buf->seg_id, 0))) {
 508             int err = errno;
 509             char hn[OPAL_MAXHOSTNAMELEN];
 510             gethostname(hn, sizeof(hn));
 511             opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
 512                            "mmap(2)", "", strerror(err), err);
 513             
 514 
 515 
 516             close(ds_buf->seg_id);
 517             return NULL;
 518         }
 519         
 520         
 521 
 522 
 523         if (0 != close(ds_buf->seg_id)) {
 524             int err = errno;
 525             char hn[OPAL_MAXHOSTNAMELEN];
 526             gethostname(hn, sizeof(hn));
 527             opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1,
 528                            hn, "close(2)", "", strerror(err), err);
 529         }
 530     }
 531     
 532 
 533 
 534 
 535     OPAL_OUTPUT_VERBOSE(
 536         (70, opal_shmem_base_framework.framework_output,
 537          "%s: %s: attach successful "
 538          "(id: %d, size: %lu, name: %s)\n",
 539          mca_shmem_mmap_component.super.base_version.mca_type_name,
 540          mca_shmem_mmap_component.super.base_version.mca_component_name,
 541          ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
 542     );
 543 
 544     
 545     return (ds_buf->seg_base_addr + sizeof(opal_shmem_seg_hdr_t));
 546 }
 547 
 548 
 549 static int
 550 segment_detach(opal_shmem_ds_t *ds_buf)
 551 {
 552     int rc = OPAL_SUCCESS;
 553 
 554     OPAL_OUTPUT_VERBOSE(
 555         (70, opal_shmem_base_framework.framework_output,
 556          "%s: %s: detaching "
 557          "(id: %d, size: %lu, name: %s)\n",
 558          mca_shmem_mmap_component.super.base_version.mca_type_name,
 559          mca_shmem_mmap_component.super.base_version.mca_component_name,
 560          ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
 561     );
 562 
 563     if (0 != munmap((void *)ds_buf->seg_base_addr, ds_buf->seg_size)) {
 564         int err = errno;
 565         char hn[OPAL_MAXHOSTNAMELEN];
 566         gethostname(hn, sizeof(hn));
 567         opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
 568                        "munmap(2)", "", strerror(err), err);
 569         rc = OPAL_ERROR;
 570     }
 571     
 572 
 573 
 574     shmem_ds_reset(ds_buf);
 575     return rc;
 576 }
 577 
 578 
 579 static int
 580 segment_unlink(opal_shmem_ds_t *ds_buf)
 581 {
 582     OPAL_OUTPUT_VERBOSE(
 583         (70, opal_shmem_base_framework.framework_output,
 584          "%s: %s: unlinking"
 585          "(id: %d, size: %lu, name: %s)\n",
 586          mca_shmem_mmap_component.super.base_version.mca_type_name,
 587          mca_shmem_mmap_component.super.base_version.mca_component_name,
 588          ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
 589     );
 590 
 591     if (-1 == unlink(ds_buf->seg_name)) {
 592         int err = errno;
 593         char hn[OPAL_MAXHOSTNAMELEN];
 594         gethostname(hn, sizeof(hn));
 595         opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
 596                        "unlink(2)", ds_buf->seg_name, strerror(err), err);
 597         return OPAL_ERROR;
 598     }
 599 
 600     
 601 
 602 
 603 
 604     ds_buf->seg_id = OPAL_SHMEM_DS_ID_INVALID;
 605     
 606     OPAL_SHMEM_DS_INVALIDATE(ds_buf);
 607     return OPAL_SUCCESS;
 608 }
 609