ompi/mca/sharedfp/sm/sharedfp_sm_file

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
mca_sharedfp_sm_file_open
mca_sharedfp_sm_file_close
   1 /*
   2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   3  *                         University Research and Technology
   4  *                         Corporation.  All rights reserved.
   5  * Copyright (c) 2004-2017 The University of Tennessee and The University
   6  *                         of Tennessee Research Foundation.  All rights
   7  *                         reserved.
   8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   9  *                         University of Stuttgart.  All rights reserved.
  10  * Copyright (c) 2004-2005 The Regents of the University of California.
  11  *                         All rights reserved.
  12  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  13  * Copyright (c) 2013      Intel, Inc. All rights reserved.
  14  * Copyright (c) 2015-2018 Research Organization for Information Science
  15  *                         and Technology (RIST). All rights reserved.
  16  * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
  17  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
  18  * $COPYRIGHT$
  19  *
  20  * Additional copyrights may follow
  21  *
  22  * $HEADER$
  23  */
  24 
  25 
  26 #include "ompi_config.h"
  27 
  28 #if HAVE_LIBGEN_H
  29 #include <libgen.h>
  30 #endif
  31 #if HAVE_SYS_STAT_H
  32 #include <sys/stat.h>
  33 #endif
  34 
  35 #include "sharedfp_sm.h"
  36 
  37 #include "mpi.h"
  38 #include "ompi/constants.h"
  39 #include "ompi/group/group.h"
  40 #include "ompi/proc/proc.h"
  41 #include "ompi/mca/sharedfp/sharedfp.h"
  42 #include "ompi/mca/sharedfp/base/base.h"
  43 
  44 #include <semaphore.h>
  45 #include <sys/mman.h>
  46 #include <libgen.h>
  47 #include <unistd.h>
  48 
  49 int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
  50                                const char* filename,
  51                                int amode,
  52                                struct opal_info_t *info,
  53                                ompio_file_t *fh)
  54 {
  55     int err = OMPI_SUCCESS;
  56     struct mca_sharedfp_base_data_t* sh;
  57     struct mca_sharedfp_sm_data * sm_data = NULL;
  58     char * filename_basename;
  59     char * sm_filename;
  60     int sm_filename_length;
  61     struct mca_sharedfp_sm_offset * sm_offset_ptr;
  62     struct mca_sharedfp_sm_offset sm_offset;
  63     int sm_fd;
  64     uint32_t comm_cid;
  65     int int_pid;
  66     pid_t my_pid;
  67 
  68     /*Memory is allocated here for the sh structure*/
  69     if ( mca_sharedfp_sm_verbose ) {
  70         opal_output(ompi_sharedfp_base_framework.framework_output,
  71                     "mca_sharedfp_sm_file_open: malloc f_sharedfp_ptr struct\n");
  72     }
  73 
  74     sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t));
  75     if ( NULL == sh ) {
  76         opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc f_sharedfp  struct\n");
  77         return OMPI_ERR_OUT_OF_RESOURCE;
  78     }
  79 
  80     /*Populate the sh file structure based on the implementation*/
  81     sh->global_offset = 0;                        /* Global Offset*/
  82     sh->selected_module_data = NULL;
  83 
  84     /*Open a shared memory segment which will hold the shared file pointer*/
  85     if ( mca_sharedfp_sm_verbose ) {
  86         opal_output(ompi_sharedfp_base_framework.framework_output,
  87                     "mca_sharedfp_sm_file_open: allocatge shared memory segment.\n");
  88     }
  89 
  90 
  91     sm_data = (struct mca_sharedfp_sm_data*) malloc ( sizeof(struct mca_sharedfp_sm_data));
  92     if ( NULL == sm_data ){
  93         opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc sm_data struct\n");
  94         free(sh);
  95         return OMPI_ERR_OUT_OF_RESOURCE;
  96     }
  97     sm_data->sm_filename=NULL;
  98 
  99 
 100     /* the shared memory segment is identified opening a file
 101     ** and then mapping it to memory
 102     ** For sharedfp we also want to put the file backed shared memory into the tmp directory
 103     */
 104     filename_basename = basename((char*)filename);
 105     /* format is "%s/%s_cid-%d-%d.sm", see below */
 106     sm_filename_length = strlen(ompi_process_info.job_session_dir) + 1 + strlen(filename_basename) + 5 + (3*sizeof(uint32_t)+1) + 4;
 107     sm_filename = (char*) malloc( sizeof(char) * sm_filename_length);
 108     if (NULL == sm_filename) {
 109         opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc sm_filename\n");
 110         free(sm_data);
 111         free(sh);
 112         return OMPI_ERR_OUT_OF_RESOURCE;
 113     }
 114 
 115     comm_cid = ompi_comm_get_cid(comm);
 116     if ( 0 == fh->f_rank ) {
 117         my_pid = getpid();
 118         int_pid = (int) my_pid;
 119     }
 120     err = comm->c_coll->coll_bcast (&int_pid, 1, MPI_INT, 0, comm, comm->c_coll->coll_bcast_module );
 121     if ( OMPI_SUCCESS != err ) {
 122         opal_output(0,"mca_sharedfp_sm_file_open: Error in bcast operation \n");
 123         free(sm_filename);
 124         free(sm_data);
 125         free(sh);
 126         return err;
 127     }
 128 
 129     snprintf(sm_filename, sm_filename_length, "%s/%s_cid-%d-%d.sm", ompi_process_info.job_session_dir,
 130              filename_basename, comm_cid, int_pid);
 131     /* open shared memory file, initialize to 0, map into memory */
 132     sm_fd = open(sm_filename, O_RDWR | O_CREAT,
 133                  S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
 134     if ( sm_fd == -1){
 135         /*error opening file*/
 136         opal_output(0,"mca_sharedfp_sm_file_open: Error, unable to open file for mmap: %s\n",sm_filename);
 137         free(sm_filename);
 138         free(sm_data);
 139         free(sh);
 140         return OMPI_ERROR;
 141     }
 142 
 143     sm_data->sm_filename = sm_filename;
 144 
 145     /* TODO: is it necessary to write to the file first? */
 146     if( 0 == fh->f_rank ){
 147         memset ( &sm_offset, 0, sizeof (struct mca_sharedfp_sm_offset ));
 148         write ( sm_fd, &sm_offset, sizeof(struct mca_sharedfp_sm_offset));
 149     }
 150     err = comm->c_coll->coll_barrier (comm, comm->c_coll->coll_barrier_module );
 151     if ( OMPI_SUCCESS != err ) {
 152         opal_output(0,"mca_sharedfp_sm_file_open: Error in barrier operation \n");
 153         free(sm_filename);
 154         free(sm_data);
 155         free(sh);
 156         close (sm_fd);
 157         return err;
 158     }
 159 
 160     /*the file has been written to, now we can map*/
 161     sm_offset_ptr = mmap(NULL, sizeof(struct mca_sharedfp_sm_offset), PROT_READ | PROT_WRITE,
 162                          MAP_SHARED, sm_fd, 0);
 163 
 164     close(sm_fd);
 165 
 166     if ( sm_offset_ptr==MAP_FAILED){
 167         err = OMPI_ERROR;
 168         opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to mmap file: %s\n",sm_filename);
 169         opal_output(0, "%s\n", strerror(errno));
 170         free(sm_filename);
 171         free(sm_data);
 172         free(sh);
 173         return OMPI_ERROR;
 174     }
 175 
 176     /* Initialize semaphore so that is shared between processes.           */
 177     /* the semaphore is shared by keeping it in the shared memory segment  */
 178 
 179 #if defined(HAVE_SEM_OPEN)
 180 
 181 #if defined (__APPLE__)
 182     sm_data->sem_name = (char*) malloc( sizeof(char) * 32);
 183     snprintf(sm_data->sem_name,31,"OMPIO_%s",filename_basename);
 184 #else
 185     sm_data->sem_name = (char*) malloc( sizeof(char) * 253);
 186     snprintf(sm_data->sem_name,252,"OMPIO_%s",filename_basename);
 187 #endif
 188 
 189     if( (sm_data->mutex = sem_open(sm_data->sem_name, O_CREAT, 0644, 1)) != SEM_FAILED ) {
 190 #elif defined(HAVE_SEM_INIT)
 191     sm_data->mutex = &sm_offset_ptr->mutex;
 192     if(sem_init(&sm_offset_ptr->mutex, 1, 1) != -1){
 193 #endif
 194         /*If opening was successful*/
 195         /*Store the new file handle*/
 196         sm_data->sm_offset_ptr = sm_offset_ptr;
 197         /* Assign the sm_data to sh->selected_module_data*/
 198         sh->selected_module_data   = sm_data;
 199         /*remember the shared file handle*/
 200         fh->f_sharedfp_data = sh;
 201 
 202         /*write initial zero*/
 203         if(fh->f_rank==0){
 204             MPI_Offset position=0;
 205 
 206             sem_wait(sm_data->mutex);
 207             sm_offset_ptr->offset=position;
 208             sem_post(sm_data->mutex);
 209         }
 210     }else{
 211         free(sm_filename);
 212         free(sm_data);
 213         free(sh);
 214         munmap(sm_offset_ptr, sizeof(struct mca_sharedfp_sm_offset));
 215         return OMPI_ERROR;
 216     }
 217 
 218     err = comm->c_coll->coll_barrier (comm, comm->c_coll->coll_barrier_module );
 219     if ( OMPI_SUCCESS != err ) {
 220         opal_output(0,"mca_sharedfp_sm_file_open: Error in barrier operation \n");
 221         free(sm_filename);
 222         free(sm_data);
 223         free(sh);
 224         munmap(sm_offset_ptr, sizeof(struct mca_sharedfp_sm_offset));
 225         return err;
 226     }
 227 
 228 #if defined(HAVE_SEM_OPEN)
 229     if ( 0 == fh->f_rank ) {
 230         sem_unlink ( sm_data->sem_name);
 231     }
 232 #endif
 233 
 234     return OMPI_SUCCESS;
 235 }
 236 
 237 int mca_sharedfp_sm_file_close (ompio_file_t *fh)
 238 {
 239     int err = OMPI_SUCCESS;
 240     /*sharedfp data structure*/
 241     struct mca_sharedfp_base_data_t *sh=NULL;
 242     /*sharedfp sm module data structure*/
 243     struct mca_sharedfp_sm_data * file_data=NULL;
 244 
 245     if( NULL == fh->f_sharedfp_data ){
 246         return OMPI_SUCCESS;
 247     }
 248     sh = fh->f_sharedfp_data;
 249 
 250     /* Use an MPI Barrier in order to make sure that
 251      * all processes are ready to release the
 252      * shared file pointer resources
 253      */
 254     fh->f_comm->c_coll->coll_barrier (fh->f_comm, fh->f_comm->c_coll->coll_barrier_module );
 255 
 256     file_data = (sm_data*)(sh->selected_module_data);
 257     if (file_data)  {
 258         /*Close sm handle*/
 259         if (file_data->sm_offset_ptr) {
 260             /* destroy semaphore */
 261 #if defined(HAVE_SEM_OPEN)
 262              sem_close ( file_data->mutex);
 263              free (file_data->sem_name);
 264 #elif defined(HAVE_SEM_INIT)
 265             sem_destroy(&file_data->sm_offset_ptr->mutex);
 266 #endif
 267             /*Release the shared memory segment.*/
 268             munmap(file_data->sm_offset_ptr,sizeof(struct mca_sharedfp_sm_offset));
 269             /*Q: Do we need to delete the file? */
 270             remove(file_data->sm_filename);
 271         }
 272         /*free our sm data structure*/
 273         if(file_data->sm_filename){
 274             free(file_data->sm_filename);
 275         }
 276         free(file_data);
 277     }
 278 
 279     /*free shared file pointer data struct*/
 280     free(sh);
 281 
 282     return err;
 283 }
/* [<][>][^][v][top][bottom][index][help] */
root/ompi/mca/sharedfp/sm/sharedfp_sm_file_open.c

DEFINITIONS
