root/ompi/mca/io/romio321/romio/test/noncontig_coll2.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. handle_error
  2. cb_gather_name_array
  3. default_str
  4. reverse_str
  5. reverse_alternating_str
  6. simple_shuffle_str
  7. main
  8. test_file
   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
   2 /*  
   3  *  (C) 2001 by Argonne National Laboratory.
   4  *      See COPYRIGHT in top-level directory.
   5  */
   6 #include "mpi.h"
   7 #include <stdio.h>
   8 #include <string.h>
   9 #include <stdlib.h>
  10 
  11 /* tests noncontiguous reads/writes using collective I/O */
  12 
  13 /* this test is almost exactly like noncontig_coll.c with the following changes:
  14  *
  15  * . generalized file writing/reading to handle arbitrary number of processors
  16  * . provides the "cb_config_list" hint with several permutations of the
  17  *   avaliable processors.  
  18  *   [ makes use of code copied from ROMIO's ADIO code to collect the names of
  19  *   the processors ]
  20  */
  21 
  22 /* we are going to muck with this later to make it evenly divisible by however many compute nodes we have */
  23 #define STARTING_SIZE 5000
  24 
  25 int test_file(char *filename, int mynod, int nprocs, char * cb_hosts, 
  26                 const char *msg, int verbose); 
  27 
  28 #define ADIOI_Free free
  29 #define ADIOI_Malloc malloc
  30 #define FPRINTF fprintf
  31 /* I have no idea what the "D" stands for; it's how things are done in adio.h 
  32  */ 
  33 struct ADIO_cb_name_arrayD {   
  34        int refct;              
  35        int namect;
  36        char **names;
  37 };  
  38 typedef struct ADIO_cb_name_arrayD *ADIO_cb_name_array;
  39 
  40 void handle_error(int errcode, const char *str);
  41 int cb_gather_name_array(MPI_Comm comm, ADIO_cb_name_array *arrayp);
  42 void default_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
  43 void reverse_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
  44 void reverse_alternating_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
  45 void simple_shuffle_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
  46 
  47 
  48 void handle_error(int errcode, const char *str) 
  49 {
  50         char msg[MPI_MAX_ERROR_STRING];
  51         int resultlen;
  52         MPI_Error_string(errcode, msg, &resultlen);
  53         fprintf(stderr, "%s: %s\n", str, msg);
  54         MPI_Abort(MPI_COMM_WORLD, 1);
  55 }
  56    
  57 
  58 /* cb_gather_name_array() - gather a list of processor names from all processes
  59  *                          in a communicator and store them on rank 0.
  60  *                   
  61  * This is a collective call on the communicator(s) passed in.
  62  *
  63  * Obtains a rank-ordered list of processor names from the processes in
  64  * "dupcomm".
  65  *
  66  * Returns 0 on success, -1 on failure.
  67  * 
  68  * NOTE: Needs some work to cleanly handle out of memory cases!  
  69  */
  70 int cb_gather_name_array(MPI_Comm comm, ADIO_cb_name_array *arrayp)
  71 {
  72         /* this is copied from ROMIO, but since this test is for correctness,
  73          * not performance, note that we have removed the parts where ROMIO
  74          * uses a keyval to cache the name array.  We'll just rebuild it if we
  75          * need to */
  76 
  77     char my_procname[MPI_MAX_PROCESSOR_NAME], **procname = 0;
  78     int *procname_len = NULL, my_procname_len, *disp = NULL, i;
  79     int commsize, commrank;
  80     ADIO_cb_name_array array = NULL;
  81 
  82     MPI_Comm_size(comm, &commsize);
  83     MPI_Comm_rank(comm, &commrank);
  84 
  85     MPI_Get_processor_name(my_procname, &my_procname_len);
  86 
  87     /* allocate space for everything */
  88     array = (ADIO_cb_name_array) malloc(sizeof(*array));
  89     if (array == NULL) {
  90         return -1;
  91     }
  92     array->refct = 1; 
  93 
  94     if (commrank == 0) {
  95         /* process 0 keeps the real list */
  96         array->namect = commsize;
  97 
  98         array->names = (char **) ADIOI_Malloc(sizeof(char *) * commsize);
  99         if (array->names == NULL) {
 100             return -1;
 101         }
 102         procname = array->names; /* simpler to read */
 103 
 104         procname_len = (int *) ADIOI_Malloc(commsize * sizeof(int));
 105         if (procname_len == NULL) { 
 106             return -1;
 107         }
 108     }
 109     else {
 110         /* everyone else just keeps an empty list as a placeholder */
 111         array->namect = 0;
 112         array->names = NULL;
 113     }
 114     /* gather lengths first */
 115     MPI_Gather(&my_procname_len, 1, MPI_INT, 
 116                procname_len, 1, MPI_INT, 0, comm);
 117 
 118     if (commrank == 0) {
 119 #ifdef CB_CONFIG_LIST_DEBUG
 120         for (i=0; i < commsize; i++) {
 121             FPRINTF(stderr, "len[%d] = %d\n", i, procname_len[i]);
 122         }
 123 #endif
 124 
 125         for (i=0; i < commsize; i++) {
 126             /* add one to the lengths because we need to count the
 127              * terminator, and we are going to use this list of lengths
 128              * again in the gatherv.  
 129              */
 130             procname_len[i]++;
 131             procname[i] = malloc(procname_len[i]);
 132             if (procname[i] == NULL) {
 133                 return -1;
 134             }
 135         }
 136         
 137         /* create our list of displacements for the gatherv.  we're going
 138          * to do everything relative to the start of the region allocated
 139          * for procname[0]
 140          *
 141          * I suppose it is theoretically possible that the distance between 
 142          * malloc'd regions could be more than will fit in an int.  We don't
 143          * cover that case.
 144          */
 145         disp = malloc(commsize * sizeof(int));
 146         disp[0] = 0;
 147         for (i=1; i < commsize; i++) {
 148             disp[i] = (int) (procname[i] - procname[0]);
 149         }
 150 
 151     }
 152 
 153     /* now gather strings */
 154     if (commrank == 0) {
 155         MPI_Gatherv(my_procname, my_procname_len + 1, MPI_CHAR, 
 156                     procname[0], procname_len, disp, MPI_CHAR,
 157                     0, comm);
 158     }
 159     else {
 160         /* if we didn't do this, we would need to allocate procname[]
 161          * on all processes...which seems a little silly.
 162          */
 163         MPI_Gatherv(my_procname, my_procname_len + 1, MPI_CHAR, 
 164                     NULL, NULL, NULL, MPI_CHAR, 0, comm);
 165     }
 166 
 167     if (commrank == 0) {
 168         /* no longer need the displacements or lengths */
 169         free(disp);
 170         free(procname_len);
 171 
 172 #ifdef CB_CONFIG_LIST_DEBUG
 173         for (i=0; i < commsize; i++) {
 174             fprintf(stderr, "name[%d] = %s\n", i, procname[i]);
 175         }
 176 #endif
 177     }
 178 
 179     *arrayp = array;
 180     return 0;
 181 }
 182 
 183 void default_str(int mynod, int len, ADIO_cb_name_array array, char *dest)
 184 {
 185         char *ptr;
 186         int i, p;
 187         if (!mynod) {
 188             ptr = dest;
 189             for (i=0; i<array->namect; i++ ) {
 190                     p = snprintf(ptr, len, "%s,", array->names[i]);
 191                     ptr += p;
 192             }
 193             /* chop off that last comma */
 194             dest[strlen(dest) - 1] = '\0';
 195         }
 196         MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
 197 }
 198 void reverse_str(int mynod, int len, ADIO_cb_name_array array, char *dest) 
 199 {
 200         char *ptr;
 201         int i, p;
 202         if (!mynod) {
 203             ptr = dest;
 204             for (i=(array->namect - 1); i >= 0; i-- ) {
 205                     p = snprintf(ptr, len, "%s,", array->names[i]);
 206                     ptr += p;
 207             }
 208             dest[strlen(dest) - 1] = '\0';
 209         }
 210         MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
 211 }
 212 
 213 void reverse_alternating_str(int mynod, int len, ADIO_cb_name_array array, char *dest)
 214 {
 215         char *ptr;
 216         int i, p;
 217         if (!mynod) {
 218             ptr = dest;
 219             /* evens */
 220             for (i=(array->namect - 1); i>= 0; i-=2 ) {
 221                     p = snprintf(ptr, len, "%s,", array->names[i]);
 222                     ptr += p;
 223             }
 224             /* odds */
 225             for (i=(array->namect - 2); i > 0; i-=2 ) {
 226                     p = snprintf(ptr, len, "%s,", array->names[i]);
 227                     ptr += p;
 228             }
 229             dest[strlen(dest) - 1] = '\0';
 230         }
 231         MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
 232 }
 233 
 234 void simple_shuffle_str(int mynod, int len, ADIO_cb_name_array array, char *dest)
 235 {
 236         char *ptr;
 237         int i, p;
 238         if (!mynod) {
 239             ptr = dest;
 240             for (i=(array->namect / 2 ); i < array->namect; i++) {
 241                     p = snprintf(ptr, len, "%s,", array->names[i]);
 242                     ptr += p;
 243             }
 244             for (i=0; i < (array->namect / 2); i++ ) {
 245                     p = snprintf(ptr, len, "%s,", array->names[i]);
 246                     ptr += p;
 247             }
 248             dest[strlen(dest) - 1] = '\0';
 249         }
 250         MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
 251 }
 252 
 253 int main(int argc, char **argv)
 254 {
 255     int i, mynod, nprocs, len, errs=0, sum_errs=0, verbose=0;
 256     char *filename;
 257     char * cb_config_string;
 258     int cb_config_len;
 259     ADIO_cb_name_array array;
 260 
 261 
 262     MPI_Init(&argc,&argv);
 263     MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
 264     MPI_Comm_rank(MPI_COMM_WORLD, &mynod); 
 265 
 266     
 267     /* process 0 takes the file name as a command-line argument and 
 268    broadcasts it to other processes */
 269     if (!mynod) {
 270         i = 1;
 271         /* TODO: at some point, accept -v for verbose */
 272         while ((i < argc) && strcmp("-fname", *argv)) {
 273             i++;
 274             argv++;
 275         }
 276         if (i >= argc) {
 277             fprintf(stderr, "\n*#  Usage: noncontig_coll -fname filename\n\n");
 278             MPI_Abort(MPI_COMM_WORLD, 1);
 279         }
 280         argv++;
 281         len = strlen(*argv);
 282         filename = (char *) malloc(len+1);
 283         strcpy(filename, *argv);
 284         MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
 285         MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
 286     }
 287     else {
 288         MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
 289         filename = (char *) malloc(len+1);
 290         MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
 291     }
 292 
 293     /* want to hint the cb_config_list, but do so in a non-sequential way */
 294     cb_gather_name_array(MPI_COMM_WORLD,  &array);
 295 
 296     /* sanity check */
 297     if (!mynod) {
 298             if (array->namect < 2 ) {
 299                     fprintf(stderr, "Run this test on two or more hosts\n");
 300                     MPI_Abort(MPI_COMM_WORLD, 1);
 301             }
 302     }
 303     /* get space for the permuted cb_config_string */
 304     if (!mynod) {
 305             cb_config_len = 0;
 306             for (i=0; i < array->namect; i++) {
 307                     /* +1: space for either a , or \0 if last */
 308                     cb_config_len += strlen(array->names[i]) + 1;
 309             }
 310             ++cb_config_len;
 311     }
 312     MPI_Bcast(&cb_config_len, 1, MPI_INT, 0, MPI_COMM_WORLD);
 313     if ( (cb_config_string = malloc(cb_config_len)) == NULL ) {
 314             perror("malloc");
 315             MPI_Abort(MPI_COMM_WORLD, 1);
 316     }
 317 
 318     /* first, no hinting */
 319     errs += test_file(filename, mynod, nprocs, NULL, "collective w/o hinting", verbose);
 320 
 321     /* hint, but no change in order */
 322     default_str(mynod, cb_config_len, array, cb_config_string);
 323     errs += test_file(filename, mynod, nprocs, cb_config_string, "collective w/ hinting: default order", verbose);
 324 
 325     /*  reverse order */
 326     reverse_str(mynod, cb_config_len, array, cb_config_string); 
 327     errs += test_file(filename, mynod, nprocs, cb_config_string, "collective w/ hinting: reverse order", verbose);
 328 
 329     /* reverse, every other */
 330     reverse_alternating_str(mynod, cb_config_len, array, cb_config_string);
 331     errs += test_file(filename, mynod, nprocs, cb_config_string,"collective w/ hinting: permutation1", verbose);
 332 
 333     /* second half, first half */
 334     simple_shuffle_str(mynod, cb_config_len, array, cb_config_string);
 335     errs += test_file(filename, mynod, nprocs, cb_config_string, "collective w/ hinting: permutation2", verbose);
 336 
 337     MPI_Allreduce(&errs, &sum_errs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
 338          
 339     if (!mynod) {
 340             if (sum_errs) fprintf(stderr, "Found %d error cases\n", sum_errs);
 341             else printf(" No Errors\n");
 342     }
 343     free(filename);
 344     free(cb_config_string);
 345     MPI_Finalize();
 346     return 0;
 347 }
 348 
 349 #define SEEDER(x,y,z) ((x)*1000000 + (y) + (x)*(z))
 350 
 351 int test_file(char *filename, int mynod, int nprocs, char * cb_hosts, const char *msg, int verbose) 
 352 {
 353     MPI_Datatype typevec, newtype, t[3];
 354     int *buf, i, b[3], errcode, errors=0;
 355     MPI_File fh;
 356     MPI_Aint d[3];
 357     MPI_Status status;
 358     int SIZE = (STARTING_SIZE/nprocs)*nprocs;
 359     MPI_Info info;
 360 
 361     if (mynod==0 && verbose) fprintf(stderr, "%s\n", msg);
 362 
 363     buf = (int *) malloc(SIZE*sizeof(int));
 364     if (buf == NULL) {
 365             perror("test_file");
 366             MPI_Abort(MPI_COMM_WORLD, -1);
 367     }
 368 
 369 
 370     if (cb_hosts != NULL ) {
 371             MPI_Info_create(&info);
 372             MPI_Info_set(info, "cb_config_list", cb_hosts);
 373     } else {
 374             info = MPI_INFO_NULL;
 375     }
 376 
 377     MPI_Type_vector(SIZE/nprocs, 1, nprocs, MPI_INT, &typevec);
 378 
 379     b[0] = b[1] = b[2] = 1;
 380     d[0] = 0;
 381     d[1] = mynod*sizeof(int);
 382     d[2] = SIZE*sizeof(int);
 383     t[0] = MPI_LB;
 384     t[1] = typevec;
 385     t[2] = MPI_UB;
 386 
 387     MPI_Type_struct(3, b, d, t, &newtype);
 388     MPI_Type_commit(&newtype);
 389     MPI_Type_free(&typevec);
 390 
 391     if (!mynod) {
 392         if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using collective I/O\n");
 393         MPI_File_delete(filename, info);
 394     }
 395     MPI_Barrier(MPI_COMM_WORLD);
 396 
 397     errcode = MPI_File_open(MPI_COMM_WORLD, filename, 
 398                     MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
 399     if (errcode != MPI_SUCCESS) {
 400             handle_error(errcode, "MPI_File_open");
 401     }
 402 
 403     MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
 404 
 405     for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE);
 406     errcode = MPI_File_write_all(fh, buf, 1, newtype, &status);
 407     if (errcode != MPI_SUCCESS) {
 408             handle_error(errcode, "nc mem - nc file: MPI_File_write_all");
 409     }
 410 
 411     MPI_Barrier(MPI_COMM_WORLD);
 412 
 413     for (i=0; i<SIZE; i++) buf[i] = -1;
 414 
 415     errcode = MPI_File_read_at_all(fh, 0, buf, 1, newtype, &status);
 416     if (errcode != MPI_SUCCESS) {
 417             handle_error(errcode, "nc mem - nc file: MPI_File_read_at_all");
 418     }
 419 
 420     /* the verification for N compute nodes is tricky. Say we have 3
 421      * processors.  
 422      * process 0 sees: 0 -1 -1 3 -1 -1 ...
 423      * process 1 sees: -1 34 -1 -1 37 -1 ...
 424      * process 2 sees: -1 -1 68 -1 -1 71 ... */
 425 
 426     /* verify those leading -1s exist if they should */
 427     for (i=0; i<mynod; i++ ) {
 428             if ( buf[i] != -1 ) {
 429                     if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]);
 430                     errors++;
 431             }
 432     }
 433     /* now the modulo games are hairy.  processor 0 sees real data in the 0th,
 434      * 3rd, 6th... elements of the buffer (assuming nprocs==3 ).  proc 1 sees
 435      * the data in 1st, 4th, 7th..., and proc 2 sees it in 2nd, 5th, 8th */
 436 
 437     for(/* 'i' set in above loop */; i<SIZE; i++) {
 438             if ( ((i-mynod)%nprocs) && buf[i] != -1)  {
 439                     if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
 440                                     mynod, i, buf[i]);
 441                     errors++;
 442             }
 443             if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE) ) {
 444                     if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
 445                                     mynod, i, buf[i], SEEDER(mynod,i,SIZE));
 446                     errors++;
 447             }
 448     }
 449     MPI_File_close(&fh);
 450 
 451     MPI_Barrier(MPI_COMM_WORLD);
 452 
 453     if (!mynod) {
 454         if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using collective I/O\n");
 455         MPI_File_delete(filename, info);
 456     }
 457     MPI_Barrier(MPI_COMM_WORLD);
 458 
 459     MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
 460                   info, &fh);
 461 
 462     for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE);
 463     errcode = MPI_File_write_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int), 
 464                     buf, 1, newtype, &status);
 465     if (errcode != MPI_SUCCESS)
 466             handle_error(errcode, "nc mem - c file: MPI_File_write_at_all");
 467 
 468     MPI_Barrier(MPI_COMM_WORLD);
 469 
 470     for (i=0; i<SIZE; i++) buf[i] = -1;
 471 
 472     errcode = MPI_File_read_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int), 
 473                     buf, 1, newtype, &status);
 474     if (errcode != MPI_SUCCESS)
 475             handle_error(errcode, "nc mem - c file: MPI_File_read_at_all");
 476 
 477     /* just like as above */
 478     for (i=0; i<mynod; i++ ) {
 479             if ( buf[i] != -1 ) {
 480                     if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]);
 481                     errors++;
 482             }
 483     }
 484     for(/* i set in above loop */; i<SIZE; i++) {
 485             if ( ((i-mynod)%nprocs) && buf[i] != -1)  {
 486                     if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
 487                                     mynod, i, buf[i]);
 488                     errors++;
 489             }
 490             if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE)) {
 491                     if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
 492                                     mynod, i, buf[i], SEEDER(mynod,i,SIZE) );
 493                     errors++;
 494             }
 495     }
 496 
 497     MPI_File_close(&fh);
 498 
 499     MPI_Barrier(MPI_COMM_WORLD);
 500 
 501     if (!mynod) {
 502         if(verbose) fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using collective I/O\n");
 503         MPI_File_delete(filename, info);
 504     }
 505     MPI_Barrier(MPI_COMM_WORLD);
 506 
 507     MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
 508                   info, &fh);
 509 
 510     MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
 511 
 512     for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod, i, SIZE);
 513     errcode = MPI_File_write_all(fh, buf, SIZE, MPI_INT, &status);
 514     if (errcode != MPI_SUCCESS)
 515             handle_error(errcode, "c mem - nc file: MPI_File_write_all");
 516 
 517     MPI_Barrier(MPI_COMM_WORLD);
 518 
 519     for (i=0; i<SIZE; i++) buf[i] = -1;
 520 
 521     errcode = MPI_File_read_at_all(fh, 0, buf, SIZE, MPI_INT, &status);
 522     if (errcode != MPI_SUCCESS)
 523             handle_error(errcode, "c mem - nc file: MPI_File_read_at_all");
 524 
 525     /* same crazy checking */
 526     for (i=0; i<SIZE; i++) {
 527             if (buf[i] != SEEDER(mynod, i, SIZE)) {
 528                 if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], SEEDER(mynod, i, SIZE));
 529                 errors++;
 530             }
 531     }
 532 
 533     MPI_File_close(&fh);
 534 
 535     MPI_Type_free(&newtype);
 536     free(buf);
 537     if (info != MPI_INFO_NULL) MPI_Info_free(&info);
 538     return errors;
 539 }
/* [<][>][^][v][top][bottom][index][help] */