1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 /*
3 *
4 * (C) 2008 by Argonne National Laboratory.
5 * See COPYRIGHT in top-level directory.
6 */
7 /* Wei-keng Liao (wkliao@ece.northwestern.edu) September 8, 2008 */
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <mpi.h>
12
13 #define YLEN 5
14 #define XLEN 10
15 #define SUB_XLEN 3
16
17 /* rjl: I was just too lazy to compute this at run-time */
18 char compare_buf[XLEN*4][YLEN*4] = {
19 {'0','1','2',0,0,'3','4','5',0,0,'D','E','F',0,0,'G','H','I'},
20 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
21 {'6','7','8',0,0,'9',':',';',0,0,'J','K','L',0,0,'M','N','O'},
22 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
23 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
24 {'X','Y','Z',0,0,'[','\\',']',0,0,'l','m','n',0,0,'o','p','q'},
25 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
26 {'^','_','`',0,0,'a','b','c',0,0,'r','s','t',0,0,'u','v','w'},
27 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
28 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
29 {'0','1','2',0,0,'3','4','5',0,0,'D','E','F',0,0,'G','H','I'},
30 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
31 {'6','7','8',0,0,'9',':',';',0,0,'J','K','L',0,0,'M','N','O'},
32 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
33 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
34 {'X','Y','Z',0,0,'[','\\',']',0,0,'l','m','n',0,0,'o','p','q'},
35 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
36 {'^','_','`',0,0,'a','b','c',0,0,'r','s','t',0,0,'u','v','w'},
37 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
38 {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}
39 };
40
41
42 /* set this if you want a dump of the global array
43 #define VERBOSE 1
44 */
45
46 /*----< main() >------------------------------------------------------------*/
47 int main(int argc, char **argv) {
48 int i, j, err, rank, np, num_io;
49 char *buf, *filename;
50 int rank_dim[2], array_of_sizes[2];
51 int array_of_subsizes[2];
52 int count, *blocklengths, global_array_size;
53 MPI_Count ftype_size;
54 MPI_Aint *displacements;
55 MPI_File fh;
56 MPI_Datatype ftype;
57 MPI_Status status;
58 MPI_Offset offset=0;
59 int nr_errors=0;
60 #ifdef VERBOSE
61 int k;
62 #endif
63
64 MPI_Init(&argc, &argv);
65 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
66 MPI_Comm_size(MPI_COMM_WORLD, &np);
67
68 if (np != 4) {
69 if (!rank) printf("Please run with 4 processes. Exiting ...\n\n");
70 MPI_Finalize();
71 return 1;
72 }
73
74 filename = argv[1];
75
76 num_io = 2;
77
78 /*-----------------------------------------------------------------------*/
79 /* process rank in each dimension */
80 rank_dim[0] = rank / 2;
81 rank_dim[1] = rank % 2;
82
83 /* global 2D array size */
84 array_of_sizes[0] = YLEN * 2;
85 array_of_sizes[1] = XLEN * 2;
86
87 global_array_size = array_of_sizes[0] * array_of_sizes[1];
88
89 array_of_subsizes[0] = YLEN / 2;
90 array_of_subsizes[1] = XLEN * SUB_XLEN / 5;
91
92 offset = rank_dim[0] * YLEN * array_of_sizes[1] +
93 rank_dim[1] * XLEN;
94
95 /* define data type for file view */
96 count = array_of_subsizes[0] * 2; /* 2 is the no. blocks along X */
97 blocklengths = (int*) malloc(count*sizeof(int));
98 displacements = (MPI_Aint*)malloc(count*sizeof(MPI_Aint));
99 for (i=0; i<count; i++)
100 blocklengths[i] = array_of_subsizes[1] / 2;
101 for (i=0; i<array_of_subsizes[0]; i++)
102 for (j=0; j<2; j++)
103 displacements[i*2+j] = offset + i*2*array_of_sizes[1] + j * XLEN/2;
104 MPI_Type_create_hindexed(count, blocklengths, displacements, MPI_CHAR, &ftype);
105 MPI_Type_commit(&ftype);
106 MPI_Type_size_x(ftype, &ftype_size);
107
108 /* subarray's layout in the global array
109
110 P0's 's layout P1's layout
111 [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9] | [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9]
112 [ 0] 0 1 2 3 4 5 | D E F G H I
113 [ 1] |
114 [ 2] 6 7 8 9 : ; | J K L M N O
115 [ 3] |
116 [ 4] |
117 [ 5] |
118 [ 6] |
119 [ 7] |
120 [ 8] |
121 [ 9] |
122
123 P2's 's layout P3's layout
124 [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9] | [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9]
125 [ 0] |
126 [ 1] |
127 [ 2] |
128 [ 3] |
129 [ 4] |
130 [ 5] X Y Z [ \ ] | l m n o p q
131 [ 6] |
132 [ 7] ^ _ ` a b c | r s t u v w
133 [ 8] |
134 [ 9] |
135 */
136
137 /* initialize the write buffer */
138 buf = (char*) malloc(array_of_subsizes[0]*array_of_subsizes[1]);
139 for (i=0; i<array_of_subsizes[0]*array_of_subsizes[1]; i++)
140 buf[i] = '0' + rank*20 + i%79;
141
142 /* zero file contents ---------------------------------------------------*/
143 if (rank == 0) {
144 char *wr_buf = (char*) calloc(num_io*global_array_size,1);
145 MPI_File_open(MPI_COMM_SELF, filename,
146 MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
147 MPI_File_write(fh, wr_buf, num_io*global_array_size, MPI_CHAR, &status);
148 MPI_File_close(&fh);
149 free(wr_buf);
150 }
151 /* open the file --------------------------------------------------------*/
152 err = MPI_File_open(MPI_COMM_WORLD, filename,
153 MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
154 if (err != MPI_SUCCESS) {
155 printf("Error: MPI_File_open() filename %s\n",filename);
156 MPI_Abort(MPI_COMM_WORLD, -1);
157 exit(1);
158 }
159
160 /* MPI collective write */
161 for (i=0; i<num_io; i++) {
162 offset = i * global_array_size;
163 /* set the file view */
164 MPI_File_set_view(fh, offset, MPI_BYTE, ftype, "native", MPI_INFO_NULL);
165 MPI_File_write_all(fh, buf, ftype_size, MPI_CHAR, &status);
166 }
167 MPI_File_close(&fh);
168
169 /* read and print file contents -----------------------------------------*/
170 if (rank == 0) {
171 char *ptr;
172 char *rd_buf = (char*) calloc(num_io*global_array_size,1);
173 MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
174 MPI_File_read(fh, rd_buf, num_io*global_array_size, MPI_CHAR, &status);
175 MPI_File_close(&fh);
176
177 #ifdef VERBOSE
178 printf("-------------------------------------------------------\n");
179 printf(" [");
180 for (i=0; i<2; i++) {
181 for (j=0; j<XLEN; j++) printf(" %d",j);
182 printf(" ");
183 }
184 printf("]\n\n");
185
186
187 ptr = rd_buf;
188 for (k=0; k<num_io; k++) {
189 for (i=0; i<2*YLEN; i++) {
190 printf("[%2d]",k*2*YLEN+i);
191 for (j=0; j<2*XLEN; j++) {
192 if (j>0 && j%XLEN==0) printf(" ");
193 if (*ptr != 0)
194 printf(" %c",*ptr);
195 else
196 printf(" ");
197 ptr++;
198 }
199 printf("\n");
200 }
201 printf("\n");
202 }
203 #endif
204 ptr = rd_buf;
205 for(i=0; i<2*YLEN*num_io; i++) {
206 for(j=0; j<2*XLEN; j++) {
207 if( *ptr != compare_buf[i][j]) {
208 fprintf(stderr, "expected %d got %d at [%d][%d]\n",
209 *ptr, compare_buf[i][j], i, j);
210 nr_errors++;
211 }
212 ptr++;
213 }
214 }
215 free(rd_buf);
216
217 if (nr_errors == 0)
218 fprintf(stdout, " No Errors\n");
219 else
220 fprintf(stderr, "Found %d errors\n", nr_errors);
221 }
222
223 free(blocklengths);
224 free(displacements);
225 free(buf);
226 MPI_Type_free(&ftype);
227 MPI_Finalize();
228 return 0;
229 }
230
231 /* command-line outputs are: (the global array is written twice)
232
233 % mpiexec -n 4 wkl_subarray
234 -------------------------------------------------------
235 [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 ]
236
237 [ 0] 0 1 2 3 4 5 D E F G H I
238 [ 1]
239 [ 2] 6 7 8 9 : ; J K L M N O
240 [ 3]
241 [ 4]
242 [ 5] X Y Z [ \ ] l m n o p q
243 [ 6]
244 [ 7] ^ _ ` a b c r s t u v w
245 [ 8]
246 [ 9]
247
248 [10] 0 1 2 3 4 5 D E F G H I
249 [11]
250 [12] 6 7 8 9 : ; J K L M N O
251 [13]
252 [14]
253 [15] X Y Z [ \ ] l m n o p q
254 [16]
255 [17] ^ _ ` a b c r s t u v w
256 [18]
257 [19]
258
259 */
260