root/ompi/mca/io/romio321/romio/adio/ad_lustre/ad_lustre_open.c

DEFINITIONS

1. ADIOI_LUSTRE_Open

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
   2 /*
   3  *   Copyright (C) 1997 University of Chicago.
   4  *   See COPYRIGHT notice in top-level directory.
   5  *
   6  *   Copyright (C) 2007 Oak Ridge National Laboratory
   7  *
   8  *   Copyright (C) 2008 Sun Microsystems, Lustre group
   9  */
  10 
  11 #include "ad_lustre.h"
  12 
  13 /* what is the basis for this define?
  14  * what happens if there are more than 1k UUIDs? */
  15 
  16 #define MAX_LOV_UUID_COUNT      1000
  17 
  18 void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code)
  19 {
  20     int perm, old_mask, amode, amode_direct;
  21     int lumlen, myrank, flag, set_layout=0, err;
  22     struct lov_user_md *lum = NULL;
  23     char *value;
  24     ADIO_Offset str_factor = -1, str_unit=0, start_iodev=-1;
  25 
  26 #if defined(MPICH) || !defined(PRINT_ERR_MSG)
  27     static char myname[] = "ADIOI_LUSTRE_OPEN";
  28 #endif
  29 
  30     MPI_Comm_rank(fd->comm, &myrank);
  31 
  32     if (fd->perm == ADIO_PERM_NULL) {
  33         old_mask = umask(022);
  34         umask(old_mask);
  35         perm = old_mask ^ 0666;
  36     }
  37     else perm = fd->perm;
  38 
  39     amode = 0;
  40     if (fd->access_mode & ADIO_CREATE)
  41         amode = amode | O_CREAT;
  42     if (fd->access_mode & ADIO_RDONLY)
  43         amode = amode | O_RDONLY;
  44     if (fd->access_mode & ADIO_WRONLY)
  45         amode = amode | O_WRONLY;
  46     if (fd->access_mode & ADIO_RDWR)
  47         amode = amode | O_RDWR;
  48     if (fd->access_mode & ADIO_EXCL)
  49         amode = amode | O_EXCL;
  50 
  51     amode_direct = amode | O_DIRECT;
  52 
  53     /* odd length here because lov_user_md contains some fixed data and
  54      * then a list of 'lmm_objects' representing stripe */
  55     lumlen = sizeof(struct lov_user_md) +
  56             MAX_LOV_UUID_COUNT * sizeof(struct lov_user_ost_data);
  57     lum = (struct lov_user_md *)ADIOI_Calloc(1,lumlen);
  58 
  59      value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
  60     /* we already validated in LUSTRE_SetInfo that these are going to be the same */
  61     if (fd->info != MPI_INFO_NULL) {
  62         /* striping information */
  63         ADIOI_Info_get(fd->info, "striping_unit", MPI_MAX_INFO_VAL,
  64                 value, &flag);
  65         if (flag)
  66             str_unit=atoll(value);
  67 
  68         ADIOI_Info_get(fd->info, "striping_factor", MPI_MAX_INFO_VAL,
  69                 value, &flag);
  70         if (flag)
  71             str_factor=atoll(value);
  72 
  73         ADIOI_Info_get(fd->info, "romio_lustre_start_iodevice",
  74                 MPI_MAX_INFO_VAL, value, &flag);
  75         if (flag)
  76             start_iodev=atoll(value);
  77     }
  78     if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0))
  79         set_layout = 1;
  80 
  81     /* if hints were set, we need to delay creation of any lustre objects.
  82      * However, if we open the file with O_LOV_DELAY_CREATE and don't call the
  83      * follow-up ioctl, subsequent writes will fail */
  84     if (myrank == 0 && set_layout)
  85         amode = amode | O_LOV_DELAY_CREATE;
  86 
  87     fd->fd_sys = open(fd->filename, amode, perm);
  88     if (fd->fd_sys == -1) goto fn_exit;
  89 
  90     /* we can only set these hints on new files */
  91     /* It was strange and buggy to open the file in the hint path.  Instead,
  92      * we'll apply the file tunings at open time */
  93     if ((amode & O_CREAT) && set_layout ) {
  94         /* if user has specified striping info, process 0 tries to set it */
  95         if (!myrank) {
  96             lum->lmm_magic = LOV_USER_MAGIC;
  97             lum->lmm_pattern = 0;
  98             /* crude check for overflow of lustre internal datatypes.
  99                  * Silently cap to large value if user provides a value
 100                  * larger than lustre supports */
 101             if (str_unit > UINT_MAX)
 102                     lum->lmm_stripe_size = UINT_MAX;
 103             else
 104                     lum->lmm_stripe_size = str_unit;
 105 
 106             if (str_factor > USHRT_MAX)
 107                     lum->lmm_stripe_count = USHRT_MAX;
 108             else
 109                     lum->lmm_stripe_count = str_factor;
 110 
 111             if (start_iodev > USHRT_MAX)
 112                      lum->lmm_stripe_offset = USHRT_MAX;
 113             else
 114                     lum->lmm_stripe_offset = start_iodev;
 115             err = ioctl(fd->fd_sys, LL_IOC_LOV_SETSTRIPE, lum);
 116             if (err == -1 && errno != EEXIST) {
 117                 fprintf(stderr, "Failure to set stripe info %s \n", strerror(errno));
 118                 /* not a fatal error, but user might care to know */
 119             }
 120         } /* End of striping parameters validation */
 121     }
 122 
 123     /* Pascal Deveze reports that, even though we pass a
 124      * "GETSTRIPE" (read) flag to the ioctl, if some of the values of this
 125      * struct are uninitialzed, the call can give an error.  zero it out in case
 126      * there are other members that must be initialized and in case
 127      * lov_user_md struct changes in future */
 128     memset(lum, 0, lumlen);
 129     lum->lmm_magic = LOV_USER_MAGIC;
 130     err = ioctl(fd->fd_sys, LL_IOC_LOV_GETSTRIPE, (void *)lum);
 131     if (!err) {
 132 
 133         fd->hints->striping_unit = lum->lmm_stripe_size;
 134         sprintf(value, "%d", lum->lmm_stripe_size);
 135         ADIOI_Info_set(fd->info, "striping_unit", value);
 136 
 137         fd->hints->striping_factor = lum->lmm_stripe_count;
 138         sprintf(value, "%d", lum->lmm_stripe_count);
 139         ADIOI_Info_set(fd->info, "striping_factor", value);
 140 
 141         fd->hints->fs_hints.lustre.start_iodevice = lum->lmm_stripe_offset;
 142         sprintf(value, "%d", lum->lmm_stripe_offset);
 143         ADIOI_Info_set(fd->info, "romio_lustre_start_iodevice", value);
 144 
 145     }
 146 
 147     if (fd->access_mode & ADIO_APPEND)
 148         fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
 149 
 150     fd->fd_direct = -1;
 151     if (fd->direct_write || fd->direct_read) {
 152         fd->fd_direct = open(fd->filename, amode_direct, perm);
 153         if (fd->fd_direct != -1) {
 154             fd->d_mem = fd->d_miniosz = (1<<12);
 155         } else {
 156             perror("cannot open file with O_Direct");
 157             fd->direct_write = fd->direct_read = 0;
 158         }
 159     }
 160 
 161 fn_exit:
 162     ADIOI_Free(lum);
 163     ADIOI_Free(value);
 164     /* --BEGIN ERROR HANDLING-- */
 165     if (fd->fd_sys == -1 || ((fd->fd_direct == -1) && 
 166                 (fd->direct_write || fd->direct_read))) {
 167         *error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
 168     }
 169     /* --END ERROR HANDLING-- */
 170     else *error_code = MPI_SUCCESS;
 171 
 172 }