1 /*
2 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2017 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2006-2007 University of Houston. All rights reserved.
13 * Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
14 * Copyright (c) 2015-2017 Research Organization for Information Science
15 * and Technology (RIST). All rights reserved.
16 * $COPYRIGHT$
17 *
18 * Additional copyrights may follow
19 *
20 * $HEADER$
21 */
22
23 #include "ompi_config.h"
24 #include "coll_inter.h"
25
26 #include "mpi.h"
27 #include "ompi/constants.h"
28 #include "ompi/datatype/ompi_datatype.h"
29 #include "ompi/communicator/communicator.h"
30 #include "ompi/op/op.h"
31 #include "ompi/mca/coll/coll.h"
32 #include "ompi/mca/coll/base/coll_tags.h"
33 #include "ompi/mca/coll/base/coll_base_util.h"
34 #include "ompi/mca/pml/pml.h"
35
36 /*
37 * allreduce_inter
38 *
39 * Function: - allreduce using other MPI collectives
40 * Accepts: - same as MPI_Allreduce()
41 * Returns: - MPI_SUCCESS or error code
42 */
43 int
44 mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, int count,
45 struct ompi_datatype_t *dtype,
46 struct ompi_op_t *op,
47 struct ompi_communicator_t *comm,
48 mca_coll_base_module_t *module)
49 {
50 int err, rank, root = 0;
51 char *tmpbuf = NULL, *pml_buffer = NULL;
52 ptrdiff_t gap, span;
53
54 rank = ompi_comm_rank(comm);
55
56 /* Perform the reduction locally */
57 span = opal_datatype_span(&dtype->super, count, &gap);
58
59 tmpbuf = (char *) malloc(span);
60 if (NULL == tmpbuf) {
61 return OMPI_ERR_OUT_OF_RESOURCE;
62 }
63 pml_buffer = tmpbuf - gap;
64
65 err = comm->c_local_comm->c_coll->coll_reduce(sbuf, pml_buffer, count,
66 dtype, op, root,
67 comm->c_local_comm,
68 comm->c_local_comm->c_coll->coll_reduce_module);
69 if (OMPI_SUCCESS != err) {
70 goto exit;
71 }
72
73 if (rank == root) {
74 /* Do a send-recv between the two root procs. to avoid deadlock */
75 err = ompi_coll_base_sendrecv_actual(pml_buffer, count, dtype, 0,
76 MCA_COLL_BASE_TAG_ALLREDUCE,
77 rbuf, count, dtype, 0,
78 MCA_COLL_BASE_TAG_ALLREDUCE,
79 comm, MPI_STATUS_IGNORE);
80 if (OMPI_SUCCESS != err) {
81 goto exit;
82 }
83 }
84
85 /* bcast the message to all the local processes */
86 err = comm->c_local_comm->c_coll->coll_bcast(rbuf, count, dtype,
87 root, comm->c_local_comm,
88 comm->c_local_comm->c_coll->coll_bcast_module);
89 if (OMPI_SUCCESS != err) {
90 goto exit;
91 }
92
93 exit:
94 if (NULL != tmpbuf) {
95 free(tmpbuf);
96 }
97
98 return err;
99 }