1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ 2 /* 3 * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana 4 * University Research and Technology 5 * Corporation. All rights reserved. 6 * Copyright (c) 2004-2005 The University of Tennessee and The University 7 * of Tennessee Research Foundation. All rights 8 * reserved. 9 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 10 * University of Stuttgart. All rights reserved. 11 * Copyright (c) 2004-2005 The Regents of the University of California. 12 * All rights reserved. 13 * Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights 14 * reserved. 15 * Copyright (c) 2016 Research Organization for Information Science 16 * and Technology (RIST). All rights reserved. 17 * Copyright (c) 2017-2018 Intel, Inc. All rights reserved. 18 * $COPYRIGHT$ 19 * 20 * Additional copyrights may follow 21 * 22 * $HEADER$ 23 */ 24 /** @file: 25 * 26 * The OpenRTE Group Communications 27 * 28 * The OpenRTE Group Comm framework provides communication services that 29 * span entire jobs or collections of processes. It is not intended to be 30 * used for point-to-point communications (the RML does that), nor should 31 * it be viewed as a high-performance communication channel for large-scale 32 * data transfers. 33 */ 34 35 #ifndef MCA_GRPCOMM_H 36 #define MCA_GRPCOMM_H 37 38 /* 39 * includes 40 */ 41 42 #include "orte_config.h" 43 #include "orte/constants.h" 44 #include "orte/types.h" 45 46 #include "orte/mca/mca.h" 47 #include "opal/class/opal_list.h" 48 #include "opal/class/opal_bitmap.h" 49 #include "opal/dss/dss_types.h" 50 51 #include "orte/mca/rml/rml_types.h" 52 53 BEGIN_C_DECLS 54 55 /* define a callback function to be invoked upon 56 * collective completion */ 57 typedef void (*orte_grpcomm_cbfunc_t)(int status, opal_buffer_t *buf, void *cbdata); 58 59 /* Define a collective signature so we don't need to 60 * track global collective id's */ 61 typedef struct { 62 opal_object_t super; 63 orte_process_name_t *signature; 64 size_t sz; 65 } orte_grpcomm_signature_t; 66 OBJ_CLASS_DECLARATION(orte_grpcomm_signature_t); 67 68 /* Internal component object for tracking ongoing 69 * allgather operations */ 70 typedef struct { 71 opal_list_item_t super; 72 /* collective's signature */ 73 orte_grpcomm_signature_t *sig; 74 /* collection bucket */ 75 opal_buffer_t bucket; 76 /* participating daemons */ 77 orte_vpid_t *dmns; 78 /** number of participating daemons */ 79 size_t ndmns; 80 /** my index in the dmns array */ 81 unsigned long my_rank; 82 /* number of buckets expected */ 83 size_t nexpected; 84 /* number reported in */ 85 size_t nreported; 86 /* distance masks for receive */ 87 opal_bitmap_t distance_mask_recv; 88 /* received buckets */ 89 opal_buffer_t ** buffers; 90 /* callback function */ 91 orte_grpcomm_cbfunc_t cbfunc; 92 /* user-provided callback data */ 93 void *cbdata; 94 } orte_grpcomm_coll_t; 95 OBJ_CLASS_DECLARATION(orte_grpcomm_coll_t); 96 97 /* 98 * Component functions - all MUST be provided! 99 */ 100 101 102 /* initialize the selected module */ 103 typedef int (*orte_grpcomm_base_module_init_fn_t)(void); 104 105 /* finalize the selected module */ 106 typedef void (*orte_grpcomm_base_module_finalize_fn_t)(void); 107 108 /* Scalably send a message. Caller will provide an array 109 * of daemon vpids that are to receive the message. A NULL 110 * pointer indicates that all daemons are participating. */ 111 typedef int (*orte_grpcomm_base_module_xcast_fn_t)(orte_vpid_t *vpids, 112 size_t nprocs, 113 opal_buffer_t *msg); 114 115 /* allgather - gather data from all specified daemons. Barrier operations 116 * will provide a zero-byte buffer. Caller will provide an array 117 * of daemon vpids that are participating in the allgather via the 118 * orte_grpcomm_coll_t object. A NULL pointer indicates that all daemons 119 * are participating. 120 * 121 * NOTE: this is a non-blocking call. The callback function cached in 122 * the orte_grpcomm_coll_t will be invoked upon completion. */ 123 typedef int (*orte_grpcomm_base_module_allgather_fn_t)(orte_grpcomm_coll_t *coll, 124 opal_buffer_t *buf); 125 126 /* 127 * Ver 3.0 - internal modules 128 */ 129 typedef struct { 130 orte_grpcomm_base_module_init_fn_t init; 131 orte_grpcomm_base_module_finalize_fn_t finalize; 132 /* collective operations */ 133 orte_grpcomm_base_module_xcast_fn_t xcast; 134 orte_grpcomm_base_module_allgather_fn_t allgather; 135 } orte_grpcomm_base_module_t; 136 137 /* the Public APIs */ 138 /* Scalably send a message. Caller will provide an array 139 * of process names that are to receive the message. A NULL 140 * pointer indicates that all known procs are to receive 141 * the message. A pointer to a name that includes ORTE_VPID_WILDCARD 142 * will send the message to all procs in the specified jobid. 143 * The message will be sent to the daemons hosting the specified 144 * procs for processing and relay. */ 145 typedef int (*orte_grpcomm_base_API_xcast_fn_t)(orte_grpcomm_signature_t *sig, 146 orte_rml_tag_t tag, 147 opal_buffer_t *msg); 148 149 /* allgather - gather data from all specified procs. Barrier operations 150 * will provide a zero-byte buffer. Caller will provide an array 151 * of application proc vpids that are participating in the allgather. A NULL 152 * pointer indicates that all known procs are participating. A pointer 153 * to a name that includes ORTE_VPID_WILDCARD indicates that all procs 154 * in the specified jobid are contributing. 155 * 156 * NOTE: this is a non-blocking call. The provided callback function 157 * will be invoked upon completion. */ 158 typedef int (*orte_grpcomm_base_API_allgather_fn_t)(orte_grpcomm_signature_t *sig, 159 opal_buffer_t *buf, 160 orte_grpcomm_cbfunc_t cbfunc, 161 void *cbdata); 162 typedef struct { 163 /* collective operations */ 164 orte_grpcomm_base_API_xcast_fn_t xcast; 165 orte_grpcomm_base_API_allgather_fn_t allgather; 166 } orte_grpcomm_API_module_t; 167 168 169 /* 170 * the standard component data structure 171 */ 172 struct orte_grpcomm_base_component_3_0_0_t { 173 mca_base_component_t base_version; 174 mca_base_component_data_t base_data; 175 }; 176 typedef struct orte_grpcomm_base_component_3_0_0_t orte_grpcomm_base_component_3_0_0_t; 177 typedef orte_grpcomm_base_component_3_0_0_t orte_grpcomm_base_component_t; 178 179 180 181 /* 182 * Macro for use in components that are of type grpcomm v3.0.0 183 */ 184 #define ORTE_GRPCOMM_BASE_VERSION_3_0_0 \ 185 /* grpcomm v3.0 is chained to MCA v2.0 */ \ 186 ORTE_MCA_BASE_VERSION_2_1_0("grpcomm", 3, 0, 0) 187 188 /* Global structure for accessing grpcomm functions */ 189 ORTE_DECLSPEC extern orte_grpcomm_API_module_t orte_grpcomm; 190 191 END_C_DECLS 192 193 #endif