This source file includes following definitions.
- MPI_Init
- MPI_Finalize
- init_monitoring_result
- start_monitoring_result
- stop_monitoring_result
- get_monitoring_result
- destroy_monitoring_result
- write_mat
- monitoring_prof_mpi_init_f2c
- monitoring_prof_mpi_finalize_f2c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45 #include "ompi_config.h"
46
47 #include <stdio.h>
48 #include <stdlib.h>
49 #include <string.h>
50 #include <stdbool.h>
51
52 #if OMPI_BUILD_FORTRAN_BINDINGS
53
54
55
56
57 #define OMPI_BUILD_MPI_PROFILING 0
58 #define OMPI_COMPILING_FORTRAN_WRAPPERS 1
59 #endif
60
61 #include "opal/threads/thread_usage.h"
62
63 #include "ompi/include/mpi.h"
64 #include "ompi/mpi/fortran/base/constants.h"
65 #include "ompi/mpi/fortran/base/fint_2_int.h"
66 #if OMPI_BUILD_FORTRAN_BINDINGS
67 #include "ompi/mpi/fortran/mpif-h/bindings.h"
68 #endif
69
70 static MPI_T_pvar_session session;
71 static int comm_world_size;
72 static int comm_world_rank;
73
74 struct monitoring_result
75 {
76 char * pvar_name;
77 int pvar_idx;
78 MPI_T_pvar_handle pvar_handle;
79 size_t * vector;
80 };
81 typedef struct monitoring_result monitoring_result;
82
83
84 static monitoring_result pml_counts;
85 static monitoring_result pml_sizes;
86
87 static monitoring_result osc_scounts;
88 static monitoring_result osc_ssizes;
89
90 static monitoring_result osc_rcounts;
91 static monitoring_result osc_rsizes;
92
93 static monitoring_result coll_counts;
94 static monitoring_result coll_sizes;
95
96 static int write_mat(char *, size_t *, unsigned int);
97 static void init_monitoring_result(const char *, monitoring_result *);
98 static void start_monitoring_result(monitoring_result *);
99 static void stop_monitoring_result(monitoring_result *);
100 static void get_monitoring_result(monitoring_result *);
101 static void destroy_monitoring_result(monitoring_result *);
102
103 int MPI_Init(int* argc, char*** argv)
104 {
105 int result, MPIT_result;
106 int provided;
107
108 result = PMPI_Init(argc, argv);
109
110 PMPI_Comm_size(MPI_COMM_WORLD, &comm_world_size);
111 PMPI_Comm_rank(MPI_COMM_WORLD, &comm_world_rank);
112
113 MPIT_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
114 if (MPIT_result != MPI_SUCCESS) {
115 fprintf(stderr, "ERROR : failed to intialize MPI_T interface, preventing to get monitoring results: check your OpenMPI installation\n");
116 PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
117 }
118
119 MPIT_result = MPI_T_pvar_session_create(&session);
120 if (MPIT_result != MPI_SUCCESS) {
121 fprintf(stderr, "ERROR : failed to create MPI_T session, preventing to get monitoring results: check your OpenMPI installation\n");
122 PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
123 }
124
125 init_monitoring_result("pml_monitoring_messages_count", &pml_counts);
126 init_monitoring_result("pml_monitoring_messages_size", &pml_sizes);
127 init_monitoring_result("osc_monitoring_messages_sent_count", &osc_scounts);
128 init_monitoring_result("osc_monitoring_messages_sent_size", &osc_ssizes);
129 init_monitoring_result("osc_monitoring_messages_recv_count", &osc_rcounts);
130 init_monitoring_result("osc_monitoring_messages_recv_size", &osc_rsizes);
131 init_monitoring_result("coll_monitoring_messages_count", &coll_counts);
132 init_monitoring_result("coll_monitoring_messages_size", &coll_sizes);
133
134 start_monitoring_result(&pml_counts);
135 start_monitoring_result(&pml_sizes);
136 start_monitoring_result(&osc_scounts);
137 start_monitoring_result(&osc_ssizes);
138 start_monitoring_result(&osc_rcounts);
139 start_monitoring_result(&osc_rsizes);
140 start_monitoring_result(&coll_counts);
141 start_monitoring_result(&coll_sizes);
142
143 return result;
144 }
145
146 int MPI_Finalize(void)
147 {
148 int result, MPIT_result;
149 size_t * exchange_count_matrix_1 = NULL;
150 size_t * exchange_size_matrix_1 = NULL;
151 size_t * exchange_count_matrix_2 = NULL;
152 size_t * exchange_size_matrix_2 = NULL;
153 size_t * exchange_all_size_matrix = NULL;
154 size_t * exchange_all_count_matrix = NULL;
155 size_t * exchange_all_avg_matrix = NULL;
156
157 stop_monitoring_result(&pml_counts);
158 stop_monitoring_result(&pml_sizes);
159 stop_monitoring_result(&osc_scounts);
160 stop_monitoring_result(&osc_ssizes);
161 stop_monitoring_result(&osc_rcounts);
162 stop_monitoring_result(&osc_rsizes);
163 stop_monitoring_result(&coll_counts);
164 stop_monitoring_result(&coll_sizes);
165
166 get_monitoring_result(&pml_counts);
167 get_monitoring_result(&pml_sizes);
168 get_monitoring_result(&osc_scounts);
169 get_monitoring_result(&osc_ssizes);
170 get_monitoring_result(&osc_rcounts);
171 get_monitoring_result(&osc_rsizes);
172 get_monitoring_result(&coll_counts);
173 get_monitoring_result(&coll_sizes);
174
175 if (0 == comm_world_rank) {
176 exchange_count_matrix_1 = (size_t *) calloc(comm_world_size * comm_world_size, sizeof(size_t));
177 exchange_size_matrix_1 = (size_t *) calloc(comm_world_size * comm_world_size, sizeof(size_t));
178 exchange_count_matrix_2 = (size_t *) calloc(comm_world_size * comm_world_size, sizeof(size_t));
179 exchange_size_matrix_2 = (size_t *) calloc(comm_world_size * comm_world_size, sizeof(size_t));
180 exchange_all_size_matrix = (size_t *) calloc(comm_world_size * comm_world_size, sizeof(size_t));
181 exchange_all_count_matrix = (size_t *) calloc(comm_world_size * comm_world_size, sizeof(size_t));
182 exchange_all_avg_matrix = (size_t *) calloc(comm_world_size * comm_world_size, sizeof(size_t));
183 }
184
185
186 PMPI_Gather(pml_counts.vector, comm_world_size, MPI_UNSIGNED_LONG, exchange_count_matrix_1, comm_world_size, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
187 PMPI_Gather(pml_sizes.vector, comm_world_size, MPI_UNSIGNED_LONG, exchange_size_matrix_1, comm_world_size, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
188 PMPI_Gather(coll_counts.vector, comm_world_size, MPI_UNSIGNED_LONG, exchange_count_matrix_2, comm_world_size, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
189 PMPI_Gather(coll_sizes.vector, comm_world_size, MPI_UNSIGNED_LONG, exchange_size_matrix_2, comm_world_size, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
190
191 if (0 == comm_world_rank) {
192 int i, j;
193
194 for (i = 0; i < comm_world_size; ++i) {
195 for (j = i + 1; j < comm_world_size; ++j) {
196
197 exchange_count_matrix_1[i * comm_world_size + j] = exchange_count_matrix_1[j * comm_world_size + i] = (exchange_count_matrix_1[i * comm_world_size + j] + exchange_count_matrix_1[j * comm_world_size + i]) / 2;
198 exchange_size_matrix_1[i * comm_world_size + j] = exchange_size_matrix_1[j * comm_world_size + i] = (exchange_size_matrix_1[i * comm_world_size + j] + exchange_size_matrix_1[j * comm_world_size + i]) / 2;
199 if (exchange_count_matrix_1[i * comm_world_size + j] != 0)
200 exchange_all_size_matrix[i * comm_world_size + j] = exchange_all_size_matrix[j * comm_world_size + i] = exchange_size_matrix_1[i * comm_world_size + j] / exchange_count_matrix_1[i * comm_world_size + j];
201
202
203 exchange_count_matrix_2[i * comm_world_size + j] = exchange_count_matrix_2[j * comm_world_size + i] = (exchange_count_matrix_2[i * comm_world_size + j] + exchange_count_matrix_2[j * comm_world_size + i]) / 2;
204 exchange_size_matrix_2[i * comm_world_size + j] = exchange_size_matrix_2[j * comm_world_size + i] = (exchange_size_matrix_2[i * comm_world_size + j] + exchange_size_matrix_2[j * comm_world_size + i]) / 2;
205 if (exchange_count_matrix_2[i * comm_world_size + j] != 0)
206 exchange_all_count_matrix[i * comm_world_size + j] = exchange_all_count_matrix[j * comm_world_size + i] = exchange_size_matrix_2[i * comm_world_size + j] / exchange_count_matrix_2[i * comm_world_size + j];
207 }
208 }
209
210
211 write_mat("monitoring_pml_msg.mat", exchange_count_matrix_1, comm_world_size);
212 write_mat("monitoring_pml_size.mat", exchange_size_matrix_1, comm_world_size);
213 write_mat("monitoring_pml_avg.mat", exchange_all_size_matrix, comm_world_size);
214
215
216 write_mat("monitoring_coll_msg.mat", exchange_count_matrix_2, comm_world_size);
217 write_mat("monitoring_coll_size.mat", exchange_size_matrix_2, comm_world_size);
218 write_mat("monitoring_coll_avg.mat", exchange_all_count_matrix, comm_world_size);
219
220
221 for (i = 0; i < comm_world_size; ++i) {
222 for (j = i + 1; j < comm_world_size; ++j) {
223 exchange_all_size_matrix[i * comm_world_size + j] = exchange_all_size_matrix[j * comm_world_size + i] = exchange_size_matrix_1[i * comm_world_size + j] + exchange_size_matrix_2[i * comm_world_size + j];
224 exchange_all_count_matrix[i * comm_world_size + j] = exchange_all_count_matrix[j * comm_world_size + i] = exchange_count_matrix_1[i * comm_world_size + j] + exchange_count_matrix_2[i * comm_world_size + j];
225 }
226 }
227 }
228
229
230 PMPI_Gather(osc_scounts.vector, comm_world_size, MPI_UNSIGNED_LONG, exchange_count_matrix_1, comm_world_size, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
231 PMPI_Gather(osc_ssizes.vector, comm_world_size, MPI_UNSIGNED_LONG, exchange_size_matrix_1, comm_world_size, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
232 PMPI_Gather(osc_rcounts.vector, comm_world_size, MPI_UNSIGNED_LONG, exchange_count_matrix_2, comm_world_size, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
233 PMPI_Gather(osc_rsizes.vector, comm_world_size, MPI_UNSIGNED_LONG, exchange_size_matrix_2, comm_world_size, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
234
235 if (0 == comm_world_rank) {
236 int i, j;
237
238 for (i = 0; i < comm_world_size; ++i) {
239 for (j = i + 1; j < comm_world_size; ++j) {
240
241 exchange_count_matrix_1[i * comm_world_size + j] = exchange_count_matrix_1[j * comm_world_size + i] = (exchange_count_matrix_1[i * comm_world_size + j] + exchange_count_matrix_1[j * comm_world_size + i] + exchange_count_matrix_2[i * comm_world_size + j] + exchange_count_matrix_2[j * comm_world_size + i]) / 2;
242 exchange_size_matrix_1[i * comm_world_size + j] = exchange_size_matrix_1[j * comm_world_size + i] = (exchange_size_matrix_1[i * comm_world_size + j] + exchange_size_matrix_1[j * comm_world_size + i] + exchange_size_matrix_2[i * comm_world_size + j] + exchange_size_matrix_2[j * comm_world_size + i]) / 2;
243 if (exchange_count_matrix_1[i * comm_world_size + j] != 0)
244 exchange_all_avg_matrix[i * comm_world_size + j] = exchange_all_avg_matrix[j * comm_world_size + i] = exchange_size_matrix_1[i * comm_world_size + j] / exchange_count_matrix_1[i * comm_world_size + j];
245 }
246 }
247
248
249 write_mat("monitoring_osc_msg.mat", exchange_count_matrix_1, comm_world_size);
250 write_mat("monitoring_osc_size.mat", exchange_size_matrix_1, comm_world_size);
251 write_mat("monitoring_osc_avg.mat", exchange_all_avg_matrix, comm_world_size);
252
253
254 for (i = 0; i < comm_world_size; ++i) {
255 for (j = i + 1; j < comm_world_size; ++j) {
256 exchange_all_size_matrix[i * comm_world_size + j] = exchange_all_size_matrix[j * comm_world_size + i] += exchange_size_matrix_1[i * comm_world_size + j];
257 exchange_all_count_matrix[i * comm_world_size + j] = exchange_all_count_matrix[j * comm_world_size + i] += exchange_count_matrix_1[i * comm_world_size + j];
258 if (exchange_all_count_matrix[i * comm_world_size + j] != 0)
259 exchange_all_avg_matrix[i * comm_world_size + j] = exchange_all_avg_matrix[j * comm_world_size + i] = exchange_all_size_matrix[i * comm_world_size + j] / exchange_all_count_matrix[i * comm_world_size + j];
260 }
261 }
262
263
264 write_mat("monitoring_all_msg.mat", exchange_all_count_matrix, comm_world_size);
265 write_mat("monitoring_all_size.mat", exchange_all_size_matrix, comm_world_size);
266 write_mat("monitoring_all_avg.mat", exchange_all_avg_matrix, comm_world_size);
267
268
269 free(exchange_count_matrix_1);
270 free(exchange_size_matrix_1);
271 free(exchange_count_matrix_2);
272 free(exchange_size_matrix_2);
273 free(exchange_all_count_matrix);
274 free(exchange_all_size_matrix);
275 free(exchange_all_avg_matrix);
276 }
277
278 destroy_monitoring_result(&pml_counts);
279 destroy_monitoring_result(&pml_sizes);
280 destroy_monitoring_result(&osc_scounts);
281 destroy_monitoring_result(&osc_ssizes);
282 destroy_monitoring_result(&osc_rcounts);
283 destroy_monitoring_result(&osc_rsizes);
284 destroy_monitoring_result(&coll_counts);
285 destroy_monitoring_result(&coll_sizes);
286
287 MPIT_result = MPI_T_pvar_session_free(&session);
288 if (MPIT_result != MPI_SUCCESS) {
289 fprintf(stderr, "WARNING : failed to free MPI_T session, monitoring results may be impacted : check your OpenMPI installation\n");
290 }
291
292 MPIT_result = MPI_T_finalize();
293 if (MPIT_result != MPI_SUCCESS) {
294 fprintf(stderr, "WARNING : failed to finalize MPI_T interface, monitoring results may be impacted : check your OpenMPI installation\n");
295 }
296
297 result = PMPI_Finalize();
298
299 return result;
300 }
301
302 void init_monitoring_result(const char * pvar_name, monitoring_result * res)
303 {
304 int count;
305 int MPIT_result;
306 MPI_Comm comm_world = MPI_COMM_WORLD;
307
308 res->pvar_name = strdup(pvar_name);
309
310 MPIT_result = MPI_T_pvar_get_index(res->pvar_name, MPI_T_PVAR_CLASS_SIZE, &(res->pvar_idx));
311 if (MPIT_result != MPI_SUCCESS) {
312 fprintf(stderr, "ERROR : cannot find monitoring MPI_T \"%s\" pvar, check that you have monitoring pml\n", pvar_name);
313 PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
314 }
315
316 MPIT_result = MPI_T_pvar_handle_alloc(session, res->pvar_idx, comm_world, &(res->pvar_handle), &count);
317 if (MPIT_result != MPI_SUCCESS) {
318 fprintf(stderr, "ERROR : failed to allocate handle on \"%s\" pvar, check that you have monitoring pml\n", pvar_name);
319 PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
320 }
321
322 if (count != comm_world_size) {
323 fprintf(stderr, "ERROR : COMM_WORLD has %d ranks \"%s\" pvar contains %d values, check that you have monitoring pml\n", comm_world_size, pvar_name, count);
324 PMPI_Abort(MPI_COMM_WORLD, count);
325 }
326
327 res->vector = (size_t *) malloc(comm_world_size * sizeof(size_t));
328 }
329
330 void start_monitoring_result(monitoring_result * res)
331 {
332 int MPIT_result;
333
334 MPIT_result = MPI_T_pvar_start(session, res->pvar_handle);
335 if (MPIT_result != MPI_SUCCESS) {
336 fprintf(stderr, "ERROR : failed to start handle on \"%s\" pvar, check that you have enabled the monitoring pml\n", res->pvar_name);
337 PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
338 }
339 }
340
341 void stop_monitoring_result(monitoring_result * res)
342 {
343 int MPIT_result;
344
345 MPIT_result = MPI_T_pvar_stop(session, res->pvar_handle);
346 if (MPIT_result != MPI_SUCCESS) {
347 fprintf(stderr, "ERROR : failed to stop handle on \"%s\" pvar, check that you have enabled the monitoring pml\n", res->pvar_name);
348 MPI_Abort(MPI_COMM_WORLD, MPIT_result);
349 }
350 }
351
352 void get_monitoring_result(monitoring_result * res)
353 {
354 int MPIT_result;
355
356 MPIT_result = MPI_T_pvar_read(session, res->pvar_handle, res->vector);
357 if (MPIT_result != MPI_SUCCESS) {
358 fprintf(stderr, "ERROR : failed to read \"%s\" pvar, check that you have enabled the monitoring pml\n", res->pvar_name);
359 PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
360 }
361 }
362
363 void destroy_monitoring_result(monitoring_result * res)
364 {
365 int MPIT_result;
366
367 MPIT_result = MPI_T_pvar_handle_free(session, &(res->pvar_handle));
368 if (MPIT_result != MPI_SUCCESS) {
369 printf("ERROR : failed to free handle on \"%s\" pvar, check that you have enabled the monitoring pml\n", res->pvar_name);
370 MPI_Abort(MPI_COMM_WORLD, MPIT_result);
371 }
372
373 free(res->pvar_name);
374 free(res->vector);
375 }
376
377 int write_mat(char * filename, size_t * mat, unsigned int dim)
378 {
379 FILE *matrix_file;
380 int i, j;
381
382 matrix_file = fopen(filename, "w");
383 if (!matrix_file) {
384 fprintf(stderr, "ERROR : failed to open \"%s\" file in write mode, check your permissions\n", filename);
385 return -1;
386 }
387
388 printf("writing %ux%u matrix to %s\n", dim, dim, filename);
389
390 for (i = 0; i < comm_world_size; ++i) {
391 for (j = 0; j < comm_world_size; ++j) {
392 fprintf(matrix_file, "%zu ", mat[i * comm_world_size + j]);
393 }
394 fprintf(matrix_file, "\n");
395 }
396 fflush(matrix_file);
397 fclose(matrix_file);
398
399 return 0;
400 }
401
402
403
404
405
406 void monitoring_prof_mpi_init_f2c( MPI_Fint * );
407 void monitoring_prof_mpi_finalize_f2c( MPI_Fint * );
408
409 void monitoring_prof_mpi_init_f2c( MPI_Fint *ierr ) {
410 int c_ierr;
411 int argc = 0;
412 char ** argv = NULL;
413
414 c_ierr = MPI_Init(&argc, &argv);
415 if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
416 }
417
418 void monitoring_prof_mpi_finalize_f2c( MPI_Fint *ierr ) {
419 int c_ierr;
420
421 c_ierr = MPI_Finalize();
422 if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
423 }
424
425 #if OPAL_HAVE_WEAK_SYMBOLS
426 #pragma weak MPI_INIT = monitoring_prof_mpi_init_f2c
427 #pragma weak mpi_init = monitoring_prof_mpi_init_f2c
428 #pragma weak mpi_init_ = monitoring_prof_mpi_init_f2c
429 #pragma weak mpi_init__ = monitoring_prof_mpi_init_f2c
430 #pragma weak MPI_Init_f = monitoring_prof_mpi_init_f2c
431 #pragma weak MPI_Init_f08 = monitoring_prof_mpi_init_f2c
432
433 #pragma weak MPI_FINALIZE = monitoring_prof_mpi_finalize_f2c
434 #pragma weak mpi_finalize = monitoring_prof_mpi_finalize_f2c
435 #pragma weak mpi_finalize_ = monitoring_prof_mpi_finalize_f2c
436 #pragma weak mpi_finalize__ = monitoring_prof_mpi_finalize_f2c
437 #pragma weak MPI_Finalize_f = monitoring_prof_mpi_finalize_f2c
438 #pragma weak MPI_Finalize_f08 = monitoring_prof_mpi_finalize_f2c
439 #elif OMPI_BUILD_FORTRAN_BINDINGS
440
441 OMPI_GENERATE_F77_BINDINGS (MPI_INIT,
442 mpi_init,
443 mpi_init_,
444 mpi_init__,
445 monitoring_prof_mpi_init_f2c,
446 (MPI_Fint *ierr),
447 (ierr) )
448
449 OMPI_GENERATE_F77_BINDINGS (MPI_FINALIZE,
450 mpi_finalize,
451 mpi_finalize_,
452 mpi_finalize__,
453 monitoring_prof_mpi_finalize_f2c,
454 (MPI_Fint *ierr),
455 (ierr) )
456 #endif