root/ompi/group/group.h

INCLUDED FROM

DEFINITIONS

1. ompi_group_size
2. ompi_group_rank
3. ompi_group_dense_lookup
4. ompi_group_get_proc_ptr
5. ompi_group_get_proc_name
6. ompi_group_peer_lookup
7. ompi_group_peer_lookup_existing

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2007 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2006-2007 University of Houston. All rights reserved.
  14  * Copyright (c) 2007-2017 Cisco Systems, Inc.  All rights reserved
  15  * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
  16  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
  17  * Copyright (c) 2013-2017 Los Alamos National Security, LLC.  All rights
  18  *                         reserved.
  19  * Copyright (c) 2016      Research Organization for Information Science
  20  *                         and Technology (RIST). All rights reserved.
  21  * $COPYRIGHT$
  22  *
  23  * Additional copyrights may follow
  24  *
  25  * $HEADER$
  26  */
  27 
  28 /**
  29  * @file:
  30  *
  31  * Infrastructure for MPI group support.
  32  */
  33 #ifndef OMPI_GROUP_H
  34 #define OMPI_GROUP_H
  35 
  36 #include "ompi_config.h"
  37 #include "ompi/proc/proc.h"
  38 #include "mpi.h"
  39 #include "opal/class/opal_pointer_array.h"
  40 #include "opal/util/output.h"
  41 
  42 BEGIN_C_DECLS
  43 
  44 #define BSIZE ((int)sizeof(unsigned char)*8)
  45 
  46 struct ompi_group_sporadic_list_t
  47 {
  48   int rank_first;
  49   int length;
  50 };
  51 
  52 struct ompi_group_sporadic_data_t
  53 {
  54     struct ompi_group_sporadic_list_t  *grp_sporadic_list;
  55                                             /** list to hold the sporadic struct */
  56     int                        grp_sporadic_list_len;/** length of the structure*/
  57 };
  58 struct ompi_group_strided_data_t
  59 {
  60     int grp_strided_offset;         /** offset to start from when including or excluding */
  61     int grp_strided_stride;         /** stride for including or excluding */
  62     int grp_strided_last_element;       /** the last element to be included for */
  63 };
  64 struct ompi_group_bitmap_data_t
  65 {
  66     unsigned char *grp_bitmap_array;     /* the bit map array for sparse groups of type BMAP */
  67     int            grp_bitmap_array_len; /* length of the bit array */
  68 };
  69 
  70 /**
  71  * Group structure
  72  * Currently we have four formats for storing the process pointers that are members
  73  * of the group.
  74  * PList: a dense format that stores all the process pointers of the group.
  75  * Sporadic: a sparse format that stores the ranges of the ranks from the parent group,
  76  *           that are included in the current group.
  77  * Strided: a sparse format that stores three integers that describe a red-black pattern
  78  *          that the current group is formed from its parent group.
  79  * Bitmap: a sparse format that maintains a bitmap of the included processes from the
  80  *         parent group. For each process that is included from the parent group
  81  *         its corresponding rank is set in the bitmap array.
  82  */
  83 struct ompi_group_t {
  84     opal_object_t super;    /**< base class */
  85     int grp_proc_count;     /**< number of processes in group */
  86     int grp_my_rank;        /**< rank in group */
  87     int grp_f_to_c_index;   /**< index in Fortran <-> C translation array */
  88     struct ompi_proc_t **grp_proc_pointers;
  89                             /**< list of pointers to ompi_proc_t structures
  90                                  for each process in the group */
  91     uint32_t grp_flags;     /**< flags, e.g. freed, cannot be freed etc.*/
  92     /** pointer to the original group when using sparse storage */
  93     struct ompi_group_t *grp_parent_group_ptr;
  94     union
  95     {
  96         struct ompi_group_sporadic_data_t grp_sporadic;
  97         struct ompi_group_strided_data_t  grp_strided;
  98         struct ompi_group_bitmap_data_t   grp_bitmap;
  99     } sparse_data;
 100 };
 101 
 102 typedef struct ompi_group_t ompi_group_t;
 103 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_group_t);
 104 
 105 /**
 106  * Padded struct to maintain back compatibiltiy.
 107  * See ompi/communicator/communicator.h comments with struct ompi_communicator_t
 108  * for full explanation why we chose the following padding construct for predefines.
 109  */
 110 #define PREDEFINED_GROUP_PAD 256
 111 
 112 struct ompi_predefined_group_t {
 113     struct ompi_group_t group;
 114     char padding[PREDEFINED_GROUP_PAD - sizeof(ompi_group_t)];
 115 };
 116 
 117 typedef struct ompi_predefined_group_t ompi_predefined_group_t;
 118 
 119 /*
 120  * The following include pulls in shared typedefs with debugger plugins.
 121  * For more information on why we do this see the Notice to developers
 122  * comment at the top of the ompi_msgq_dll.c file.
 123  */
 124 #include "group_dbg.h"
 125 
 126 #define OMPI_GROUP_IS_INTRINSIC(_group) ((_group)->grp_flags&OMPI_GROUP_INTRINSIC)
 127 #define OMPI_GROUP_IS_DENSE(_group) ((_group)->grp_flags & OMPI_GROUP_DENSE)
 128 #define OMPI_GROUP_IS_SPORADIC(_group) ((_group)->grp_flags & OMPI_GROUP_SPORADIC)
 129 #define OMPI_GROUP_IS_STRIDED(_group) ((_group)->grp_flags & OMPI_GROUP_STRIDED)
 130 #define OMPI_GROUP_IS_BITMAP(_group) ((_group)->grp_flags & OMPI_GROUP_BITMAP)
 131 
 132 #define OMPI_GROUP_SET_INTRINSIC(_group) ( (_group)->grp_flags |= OMPI_GROUP_INTRINSIC)
 133 #define OMPI_GROUP_SET_DENSE(_group) ( (_group)->grp_flags |= OMPI_GROUP_DENSE)
 134 #define OMPI_GROUP_SET_SPORADIC(_group) ( (_group)->grp_flags |= OMPI_GROUP_SPORADIC)
 135 #define OMPI_GROUP_SET_STRIDED(_group) ( (_group)->grp_flags |= OMPI_GROUP_STRIDED)
 136 #define OMPI_GROUP_SET_BITMAP(_group) ( (_group)->grp_flags |= OMPI_GROUP_BITMAP)
 137 
 138 /**
 139  * Table for Fortran <-> C group handle conversion
 140  */
 141 OMPI_DECLSPEC extern struct opal_pointer_array_t ompi_group_f_to_c_table;
 142 OMPI_DECLSPEC extern struct ompi_predefined_group_t ompi_mpi_group_null;
 143 OMPI_DECLSPEC extern struct ompi_predefined_group_t *ompi_mpi_group_null_addr;
 144 
 145 
 146 /*
 147  * function prototypes
 148  */
 149 
 150 /**
 151  * Allocate a new group structure.
 152  *
 153  * @param group_size Number of MPI processes in the group
 154  *
 155  * @return Pointer to new group structure
 156  */
 157 OMPI_DECLSPEC ompi_group_t *ompi_group_allocate(int group_size);
 158 ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_size);
 159 ompi_group_t *ompi_group_allocate_sporadic(int group_size);
 160 ompi_group_t *ompi_group_allocate_strided(void);
 161 ompi_group_t *ompi_group_allocate_bmap(int orig_group_size, int group_size);
 162 
 163 /**
 164  * Increment the reference count of the proc structures.
 165  *
 166  * @param group Pointer to ompi_group_t structute (IN)
 167  *
 168  */
 169 OMPI_DECLSPEC void ompi_group_increment_proc_count(ompi_group_t *group);
 170 
 171 /**
 172  * Decrement the reference count of the proc structures.
 173  *
 174  * @param group Pointer to ompi_group_t structute (IN)
 175  *
 176  */
 177 OMPI_DECLSPEC void ompi_group_decrement_proc_count(ompi_group_t *group);
 178 
 179 
 180 /**
 181  * Initialize OMPI group infrastructure.
 182  *
 183  * @return Error code
 184  */
 185 int ompi_group_init(void);
 186 
 187 
 188 /**
 189  * Clean up OMPI group infrastructure.
 190  *
 191  * @return Error code
 192  */
 193 int ompi_group_finalize(void);
 194 
 195 
 196 /**
 197  * Get group size.
 198  *
 199  * @param group Pointer to ompi_group_t structute (IN)
 200  *
 201  * @return Group size
 202  */
 203 static inline int ompi_group_size(ompi_group_t *group)
 204 {
 205     return group->grp_proc_count;
 206 }
 207 
 208 
 209 /**
 210  * Get group rank
 211  *
 212  * @param group Pointer to ompi_group_t structure (IN)
 213  *
 214  * @return Group rank
 215  */
 216 static inline int ompi_group_rank(ompi_group_t *group)
 217 {
 218     return group->grp_my_rank;
 219 }
 220 
 221 
 222 
 223 /**
 224  * Set group rank in the input group structure
 225  *
 226  * @param group Group Pointer to ompi_group_t structure (IN)
 227  * @param proc_pointer Pointer to ompi_proc_t structure for process.
 228  *                     MPI_PROC_NULL may be used to indicate proc not
 229  *                     in group
 230  *
 231  * @return Error code
 232  */
 233 void ompi_set_group_rank(ompi_group_t *group, struct ompi_proc_t *proc_pointer);
 234 
 235 /**
 236  * Abstracting MPI_Group_translate_ranks to an ompi function for internal use
 237  */
 238 OMPI_DECLSPEC int ompi_group_translate_ranks ( ompi_group_t *group1,
 239                                                int n_ranks, const int *ranks1,
 240                                                ompi_group_t *group2,
 241                                                int *ranks2);
 242 
 243 /**
 244  * Abstracting MPI_Group_compare to an ompi function for internal use
 245  */
 246 OMPI_DECLSPEC int ompi_group_compare(ompi_group_t *group1,
 247                                      ompi_group_t *group2,
 248                                      int *result);
 249 
 250 /**
 251  * Abstracting MPI_Group_free, since it is required by some internal functions...
 252  */
 253 int ompi_group_free (ompi_group_t **group);
 254 
 255 /**
 256  * Functions to handle process pointers for sparse group formats
 257  */
 258 int ompi_group_translate_ranks_sporadic ( ompi_group_t *group1,
 259                                  int n_ranks, const int *ranks1,
 260                                  ompi_group_t *group2,
 261                                  int *ranks2);
 262 int ompi_group_translate_ranks_sporadic_reverse ( ompi_group_t *group1,
 263                                  int n_ranks, const int *ranks1,
 264                                  ompi_group_t *group2,
 265                                  int *ranks2);
 266 int ompi_group_translate_ranks_strided ( ompi_group_t *group1,
 267                                  int n_ranks, const int *ranks1,
 268                                  ompi_group_t *group2,
 269                                  int *ranks2);
 270 int ompi_group_translate_ranks_strided_reverse ( ompi_group_t *group1,
 271                                  int n_ranks, const int *ranks1,
 272                                  ompi_group_t *group2,
 273                                  int *ranks2);
 274 int ompi_group_translate_ranks_bmap ( ompi_group_t *group1,
 275                                  int n_ranks, const int *ranks1,
 276                                  ompi_group_t *group2,
 277                                  int *ranks2);
 278 int ompi_group_translate_ranks_bmap_reverse ( ompi_group_t *group1,
 279                                  int n_ranks, const int *ranks1,
 280                                  ompi_group_t *group2,
 281                                  int *ranks2);
 282 
 283 /**
 284  *  Prototypes for the group back-end functions. Argument lists
 285  are similar to the according  C MPI functions.
 286  */
 287 int ompi_group_incl(ompi_group_t* group, int n, const int *ranks,
 288                     ompi_group_t **new_group);
 289 int ompi_group_excl(ompi_group_t* group, int n, const int *ranks,
 290                     ompi_group_t **new_group);
 291 int ompi_group_range_incl(ompi_group_t* group, int n_triplets,
 292                           int ranges[][3],ompi_group_t **new_group);
 293 int ompi_group_range_excl(ompi_group_t* group, int n_triplets,
 294                           int ranges[][3],ompi_group_t **new_group);
 295 int ompi_group_union (ompi_group_t* group1, ompi_group_t* group2,
 296                       ompi_group_t **new_group);
 297 int ompi_group_intersection(ompi_group_t* group1,ompi_group_t* group2,
 298                             ompi_group_t **new_group);
 299 int ompi_group_difference(ompi_group_t* group1, ompi_group_t* group2,
 300                           ompi_group_t **new_group);
 301 
 302 
 303 /**
 304  *  Include Functions to handle Sparse storage formats
 305  */
 306 int ompi_group_incl_plist(ompi_group_t* group, int n, const int *ranks,
 307                           ompi_group_t **new_group);
 308 int ompi_group_incl_spor(ompi_group_t* group, int n, const int *ranks,
 309                          ompi_group_t **new_group);
 310 int ompi_group_incl_strided(ompi_group_t* group, int n, const int *ranks,
 311                             ompi_group_t **new_group);
 312 int ompi_group_incl_bmap(ompi_group_t* group, int n, const int *ranks,
 313                          ompi_group_t **new_group);
 314 
 315 /**
 316  *  Functions to calculate storage spaces
 317  */
 318 int ompi_group_calc_plist ( int n, const int *ranks );
 319 int ompi_group_calc_strided ( int n, const int *ranks );
 320 int ompi_group_calc_sporadic ( int n, const int *ranks );
 321 int ompi_group_calc_bmap ( int n, int orig_size , const int *ranks );
 322 
 323 /**
 324  * Function to return the minimum value in an array
 325  */
 326 int ompi_group_minloc (int list[], int length);
 327 
 328 /**
 329  * @brief Helper function for retreiving the proc of a group member in a dense group
 330  *
 331  * This function exists to handle the translation of sentinel group members to real
 332  * ompi_proc_t's. If a sentinel value is found and allocate is true then this function
 333  * looks for an existing ompi_proc_t using ompi_proc_for_name which will allocate a
 334  * ompi_proc_t if one does not exist. If allocate is false then sentinel values translate
 335  * to NULL.
 336  */
 337 static inline struct ompi_proc_t *ompi_group_dense_lookup (ompi_group_t *group, const int peer_id, const bool allocate)
 338 {
 339     ompi_proc_t *proc;
 340 
 341 #if OPAL_ENABLE_DEBUG
 342     if (peer_id >= group->grp_proc_count) {
 343         opal_output(0, "ompi_group_dense_lookup: invalid peer index (%d)", peer_id);
 344         return (struct ompi_proc_t *) NULL;
 345     }
 346 #endif
 347 
 348     proc = group->grp_proc_pointers[peer_id];
 349 
 350     if (OPAL_UNLIKELY(ompi_proc_is_sentinel (proc))) {
 351         if (!allocate) {
 352             return NULL;
 353         }
 354 
 355         /* replace sentinel value with an actual ompi_proc_t */
 356         ompi_proc_t *real_proc =
 357             (ompi_proc_t *) ompi_proc_for_name (ompi_proc_sentinel_to_name ((uintptr_t) proc));
 358 
 359         if (opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *)(group->grp_proc_pointers + peer_id),
 360                                                      (intptr_t *) &proc, (intptr_t) real_proc)) {
 361             OBJ_RETAIN(real_proc);
 362         }
 363 
 364         proc = real_proc;
 365     }
 366 
 367     return proc;
 368 }
 369 
 370 /*
 371  * This is the function that iterates through the sparse groups to the dense group
 372  * to reach the process pointer
 373  */
 374 static inline ompi_proc_t *ompi_group_get_proc_ptr (ompi_group_t *group, int rank, const bool allocate)
 375 {
 376 #if OMPI_GROUP_SPARSE
 377     do {
 378         if (OMPI_GROUP_IS_DENSE(group)) {
 379             return ompi_group_dense_lookup (group, rank, allocate);
 380         }
 381         int ranks1 = rank;
 382         ompi_group_translate_ranks (group, 1, &ranks1, group->grp_parent_group_ptr, &rank);
 383         group = group->grp_parent_group_ptr;
 384     } while (1);
 385 #else
 386     return ompi_group_dense_lookup (group, rank, allocate);
 387 #endif
 388 }
 389 
 390 /**
 391  * @brief Get the raw proc pointer from the group
 392  *
 393  * This function will either return a ompi_proc_t if one exists (either stored in the group
 394  * or cached in the proc hash table) or a sentinel value representing the proc. This
 395  * differs from ompi_group_get_proc_ptr() which returns the ompi_proc_t or NULL.
 396  */
 397 ompi_proc_t *ompi_group_get_proc_ptr_raw (ompi_group_t *group, int rank);
 398 
 399 static inline opal_process_name_t ompi_group_get_proc_name (ompi_group_t *group, int rank)
 400 {
 401     ompi_proc_t *proc = ompi_group_get_proc_ptr_raw (group, rank);
 402     if (ompi_proc_is_sentinel (proc)) {
 403         return ompi_proc_sentinel_to_name ((intptr_t) proc);
 404     }
 405 
 406     return proc->super.proc_name;
 407 }
 408 
 409 /**
 410  * Inline function to check if sparse groups are enabled and return the direct access
 411  * to the proc pointer, otherwise the lookup function
 412  */
 413 static inline struct ompi_proc_t* ompi_group_peer_lookup(ompi_group_t *group, int peer_id)
 414 {
 415     return ompi_group_get_proc_ptr (group, peer_id, true);
 416 }
 417 
 418 static inline struct ompi_proc_t *ompi_group_peer_lookup_existing (ompi_group_t *group, int peer_id)
 419 {
 420     return ompi_group_get_proc_ptr (group, peer_id, false);
 421 }
 422 
 423 bool ompi_group_have_remote_peers (ompi_group_t *group);
 424 
 425 /**
 426  *  Function to print the group info
 427  */
 428 int ompi_group_dump (ompi_group_t* group);
 429 
 430 /**
 431  * Ceil Function so not to include the math.h lib
 432  */
 433 int ompi_group_div_ceil (int num, int den);
 434 
 435 END_C_DECLS
 436 #endif /* OMPI_GROUP_H */