root/ompi/mca/coll/tuned/coll_tuned_dynamic_rules.c

DEFINITIONS

1. ompi_coll_tuned_mk_alg_rules
2. ompi_coll_tuned_mk_com_rules
3. ompi_coll_tuned_mk_msg_rules
4. ompi_coll_tuned_dump_msg_rule
5. ompi_coll_tuned_dump_com_rule
6. ompi_coll_tuned_dump_alg_rule
7. ompi_coll_tuned_dump_all_rules
8. ompi_coll_tuned_free_msg_rules_in_com_rule
9. ompi_coll_tuned_free_coms_in_alg_rule
10. ompi_coll_tuned_free_all_rules
11. ompi_coll_tuned_get_com_rule_ptr
12. ompi_coll_tuned_get_target_method_params

   1 /*
   2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   3  *                         University Research and Technology
   4  *                         Corporation.  All rights reserved.
   5  * Copyright (c) 2004-2015 The University of Tennessee and The University
   6  *                         of Tennessee Research Foundation.  All rights
   7  *                         reserved.
   8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   9  *                         University of Stuttgart.  All rights reserved.
  10  * Copyright (c) 2004-2005 The Regents of the University of California.
  11  *                         All rights reserved.
  12  * Copyright (c) 2011-2012 FUJITSU LIMITED.  All rights reserved.
  13  * Copyright (c) 2017      Research Organization for Information Science
  14  *                         and Technology (RIST). All rights reserved.
  15  * $COPYRIGHT$
  16  *
  17  * Additional copyrights may follow
  18  *
  19  * $HEADER$
  20  */
  21 
  22 #include "ompi_config.h"
  23 
  24 #include "mpi.h"
  25 #include "ompi/mca/mca.h"
  26 #include "ompi/constants.h"
  27 #include "coll_tuned.h"
  28 
  29 /* need to include our own topo prototypes so we can malloc data on the comm correctly */
  30 #include "ompi/mca/coll/base/coll_base_topo.h"
  31 
  32 /* also need the dynamic rule structures */
  33 #include "coll_tuned_dynamic_rules.h"
  34 
  35 #include <stdlib.h>
  36 #include <stdio.h>
  37 
  38 #include "ompi/mca/coll/base/coll_base_util.h"
  39 
  40 
  41 ompi_coll_alg_rule_t* ompi_coll_tuned_mk_alg_rules (int n_alg)
  42 {
  43     int i;
  44     ompi_coll_alg_rule_t* alg_rules;
  45 
  46     alg_rules = (ompi_coll_alg_rule_t *) calloc (n_alg, sizeof (ompi_coll_alg_rule_t));
  47     if (!alg_rules) return (alg_rules);
  48 
  49     /* set all we can at this point */
  50     for (i=0;i<n_alg;i++) {
  51         alg_rules[i].alg_rule_id = i;
  52     }
  53     return (alg_rules);
  54 }
  55 
  56 
  57 ompi_coll_com_rule_t* ompi_coll_tuned_mk_com_rules (int n_com_rules, int alg_rule_id)
  58 {
  59     int i;
  60     ompi_coll_com_rule_t * com_rules;
  61 
  62     com_rules = (ompi_coll_com_rule_t *) calloc (n_com_rules, sizeof (ompi_coll_com_rule_t));
  63     if (!com_rules) return (com_rules);
  64 
  65     for (i=0;i<n_com_rules;i++) {
  66         com_rules[i].mpi_comsize = 0;   /* unknown */
  67         com_rules[i].alg_rule_id = alg_rule_id;
  68         com_rules[i].com_rule_id = i;
  69         com_rules[i].n_msg_sizes = 0;   /* unknown */
  70         com_rules[i].msg_rules = (ompi_coll_msg_rule_t *) NULL;
  71     }
  72     return (com_rules);
  73 }
  74 
  75 
  76 ompi_coll_msg_rule_t* ompi_coll_tuned_mk_msg_rules (int n_msg_rules, int alg_rule_id, int com_rule_id, int mpi_comsize)
  77 {
  78     int i;
  79     ompi_coll_msg_rule_t *msg_rules;
  80 
  81     msg_rules = (ompi_coll_msg_rule_t *) calloc (n_msg_rules, sizeof (ompi_coll_msg_rule_t));
  82     if (!msg_rules) return (msg_rules);
  83 
  84     for( i = 0; i < n_msg_rules; i++ ) {
  85         msg_rules[i].mpi_comsize = mpi_comsize;
  86         msg_rules[i].alg_rule_id = alg_rule_id;
  87         msg_rules[i].com_rule_id = com_rule_id;
  88         msg_rules[i].msg_rule_id = i;
  89         msg_rules[i].msg_size = 0;               /* unknown */
  90         msg_rules[i].result_alg = 0;             /* unknown */
  91         msg_rules[i].result_topo_faninout = 0;   /* unknown */
  92         msg_rules[i].result_segsize = 0;         /* unknown */
  93         msg_rules[i].result_max_requests = 0;    /* unknown & default */
  94     }
  95     return (msg_rules);
  96 }
  97 
  98 
  99 /*
 100  * Debug / IO routines
 101  *
 102  */
 103 int ompi_coll_tuned_dump_msg_rule (ompi_coll_msg_rule_t* msg_p)
 104 {
 105     if (!msg_p) {
 106         OPAL_OUTPUT((ompi_coll_tuned_stream,"Message rule was a NULL ptr?!\n"));
 107         return (-1);
 108     }
 109 
 110     OPAL_OUTPUT((ompi_coll_tuned_stream,"alg_id %3d\tcom_id %3d\tcom_size %3d\tmsg_id %3d\t", msg_p->alg_rule_id,
 111                  msg_p->com_rule_id, msg_p->mpi_comsize, msg_p->msg_rule_id));
 112 
 113     OPAL_OUTPUT((ompi_coll_tuned_stream,"msg_size %10lu -> algorithm %2d\ttopo in/out %2d\tsegsize %5ld\tmax_requests %4d\n",
 114                  msg_p->msg_size, msg_p->result_alg, msg_p->result_topo_faninout, msg_p->result_segsize,
 115                  msg_p->result_max_requests));
 116 
 117     return (0);
 118 }
 119 
 120 
 121 int ompi_coll_tuned_dump_com_rule (ompi_coll_com_rule_t* com_p)
 122 {
 123     int i;
 124 
 125     if (!com_p) {
 126         OPAL_OUTPUT((ompi_coll_tuned_stream,"Com rule was a NULL ptr?!\n"));
 127         return (-1);
 128     }
 129 
 130     OPAL_OUTPUT((ompi_coll_tuned_stream, "alg_id %3d\tcom_id %3d\tcom_size %3d\t", com_p->alg_rule_id, com_p->com_rule_id, com_p->mpi_comsize));
 131 
 132     if (!com_p->n_msg_sizes) {
 133         OPAL_OUTPUT((ompi_coll_tuned_stream,"no msgsizes defined\n"));
 134         return (0);
 135     }
 136 
 137     OPAL_OUTPUT((ompi_coll_tuned_stream,"number of message sizes %3d\n", com_p->n_msg_sizes));
 138 
 139     for (i=0;i<com_p->n_msg_sizes;i++) {
 140         ompi_coll_tuned_dump_msg_rule (&(com_p->msg_rules[i]));
 141     }
 142 
 143     return (0);
 144 }
 145 
 146 
 147 int ompi_coll_tuned_dump_alg_rule (ompi_coll_alg_rule_t* alg_p)
 148 {
 149     int i;
 150 
 151     if (!alg_p) {
 152         OPAL_OUTPUT((ompi_coll_tuned_stream,"Algorithm rule was a NULL ptr?!\n"));
 153         return (-1);
 154     }
 155 
 156     OPAL_OUTPUT((ompi_coll_tuned_stream,"alg_id %3d\t", alg_p->alg_rule_id));
 157 
 158     if (!alg_p->n_com_sizes) {
 159         OPAL_OUTPUT((ompi_coll_tuned_stream,"no coms defined\n"));
 160         return (0);
 161     }
 162 
 163     OPAL_OUTPUT((ompi_coll_tuned_stream,"number of com sizes %3d\n", alg_p->n_com_sizes));
 164 
 165     for (i=0;i<alg_p->n_com_sizes;i++) {
 166         ompi_coll_tuned_dump_com_rule (&(alg_p->com_rules[i]));
 167     }
 168 
 169     return (0);
 170 }
 171 
 172 
 173 int ompi_coll_tuned_dump_all_rules (ompi_coll_alg_rule_t* alg_p, int n_rules)
 174 {
 175     int i;
 176 
 177     if (!alg_p) {
 178         OPAL_OUTPUT((ompi_coll_tuned_stream,"Algorithm rule was a NULL ptr?!\n"));
 179         return (-1);
 180     }
 181 
 182     OPAL_OUTPUT((ompi_coll_tuned_stream,"Number of algorithm rules %3d\n", n_rules));
 183 
 184     for (i=0;i<n_rules;i++) {
 185         ompi_coll_tuned_dump_alg_rule (&(alg_p[i]));
 186     }
 187 
 188     return (0);
 189 }
 190 
 191 
 192 /*
 193  * Memory free routines
 194  *
 195  */
 196 int ompi_coll_tuned_free_msg_rules_in_com_rule (ompi_coll_com_rule_t* com_p)
 197 {
 198     int rc=0;
 199     ompi_coll_msg_rule_t* msg_p;
 200 
 201     if (!com_p) {
 202         OPAL_OUTPUT((ompi_coll_tuned_stream,"attempt to free NULL com_rule ptr\n"));
 203         return (-1);
 204     }
 205 
 206     if (com_p->n_msg_sizes) {
 207         msg_p = com_p->msg_rules;
 208 
 209         if (!msg_p) {
 210             OPAL_OUTPUT((ompi_coll_tuned_stream,"attempt to free NULL msg_rules when msg count was %d\n", com_p->n_msg_sizes));
 211             rc = -1; /* some error */
 212         }
 213         else {
 214             /* ok, memory exists for the msg rules so free that first */
 215             free (com_p->msg_rules);
 216             com_p->msg_rules = (ompi_coll_msg_rule_t*) NULL;
 217         }
 218 
 219     } /* if we have msg rules to free as well */
 220 
 221     return (rc);
 222 }
 223 
 224 
 225 int ompi_coll_tuned_free_coms_in_alg_rule (ompi_coll_alg_rule_t* alg_p)
 226 {
 227     int rc=0;
 228     int i;
 229 
 230     ompi_coll_com_rule_t* com_p;
 231 
 232     if (!alg_p) {
 233         OPAL_OUTPUT((ompi_coll_tuned_stream,"attempt to free NULL alg_rule ptr\n"));
 234         return (-1);
 235     }
 236 
 237     if (alg_p->n_com_sizes) {
 238         com_p = alg_p->com_rules;
 239 
 240         if (!com_p) {
 241             OPAL_OUTPUT((ompi_coll_tuned_stream,"attempt to free NULL com_rules when com count was %d\n", alg_p->n_com_sizes));
 242         } else {
 243             /* ok, memory exists for the com rules so free their message rules first */
 244             for( i = 0; i < alg_p->n_com_sizes; i++ ) {
 245                 com_p = &(alg_p->com_rules[i]);
 246                 ompi_coll_tuned_free_msg_rules_in_com_rule (com_p);
 247             }
 248             /* we are now free to free the com rules themselives */
 249             free (alg_p->com_rules);
 250             alg_p->com_rules = (ompi_coll_com_rule_t*) NULL;
 251         }
 252 
 253     } /* if we have msg rules to free as well */
 254 
 255     return (rc);
 256 }
 257 
 258 
 259 int ompi_coll_tuned_free_all_rules (ompi_coll_alg_rule_t* alg_p, int n_algs)
 260 {
 261     int i;
 262     int rc = 0;
 263 
 264     for( i = 0; i < n_algs; i++ ) {
 265         rc += ompi_coll_tuned_free_coms_in_alg_rule (&(alg_p[i]));
 266     }
 267 
 268     free (alg_p);
 269 
 270     return (rc);
 271 }
 272 
 273 /*
 274  * query functions
 275  * i.e. the functions that get me the algorithm, topo fanin/out and segment size fast
 276  * and also get the rules that are needed by each communicator as needed
 277  *
 278  */
 279 
 280 /*
 281  * This function is used to get the pointer to the nearest (less than or equal)
 282  * com rule for this MPI collective (alg_id) for a given
 283  * MPI communicator size. The complete rule base must be presented.
 284  *
 285  * If no rule exits returns NULL, else the com rule ptr
 286  * (which can be used in the coll_tuned_get_target_method_params() call)
 287  *
 288  */
 289 ompi_coll_com_rule_t* ompi_coll_tuned_get_com_rule_ptr (ompi_coll_alg_rule_t* rules, int alg_id, int mpi_comsize)
 290 {
 291     ompi_coll_alg_rule_t*  alg_p = (ompi_coll_alg_rule_t*) NULL;
 292     ompi_coll_com_rule_t*  com_p = (ompi_coll_com_rule_t*) NULL;
 293     ompi_coll_com_rule_t*  best_com_p = (ompi_coll_com_rule_t*) NULL;
 294     int i;
 295 
 296     if (!rules) {                    /* no rule base no resulting com rule */
 297         return ((ompi_coll_com_rule_t*)NULL);
 298     }
 299 
 300     alg_p = &(rules[alg_id]); /* get the algorithm rule pointer */
 301 
 302     if (!alg_p->n_com_sizes) {   /* check for count of communicator sizes */
 303         return ((ompi_coll_com_rule_t*)NULL);    /* no com sizes so no rule */
 304     }
 305 
 306     /* ok have some com sizes, now to find the one closest to my mpi_comsize */
 307 
 308     /* make a copy of the first com rule */
 309     best_com_p = com_p = alg_p->com_rules;
 310     i = 0;
 311 
 312     while( i < alg_p->n_com_sizes ) {
 313         if (com_p->mpi_comsize > mpi_comsize) {
 314             break;
 315         }
 316         best_com_p = com_p;
 317         /* go to the next entry */
 318         com_p++;
 319         i++;
 320     }
 321 
 322     OPAL_OUTPUT((ompi_coll_tuned_stream,"Selected the following com rule id %d\n", best_com_p->com_rule_id));
 323     ompi_coll_tuned_dump_com_rule (best_com_p);
 324 
 325     return (best_com_p);
 326 }
 327 
 328 /*
 329  * This function takes a com_rule ptr (from the communicators coll tuned data structure)
 330  * (Which is chosen for a particular MPI collective)
 331  * and a (total_)msg_size and it returns (0) and a algorithm to use and a recommended topo faninout and segment size
 332  * all based on the user supplied rules
 333  *
 334  * Just like the above functions it uses a less than or equal msg size
 335  * (hense config file must have a default defined for '0' if we reach this point)
 336  * else if no rules match we return '0' + '0,0' or used fixed decision table with no topo chand and no segmentation
 337  * of users data.. shame.
 338  *
 339  * On error return 0 so we default to fixed rules anyway :)
 340  *
 341  */
 342 
 343 int ompi_coll_tuned_get_target_method_params (ompi_coll_com_rule_t* base_com_rule, size_t mpi_msgsize, int *result_topo_faninout,
 344                                               int* result_segsize, int* max_requests)
 345 {
 346     ompi_coll_msg_rule_t*  msg_p = (ompi_coll_msg_rule_t*) NULL;
 347     ompi_coll_msg_rule_t*  best_msg_p = (ompi_coll_msg_rule_t*) NULL;
 348     int i;
 349 
 350     /* No rule or zero rules */
 351     if( (NULL == base_com_rule) || (0 == base_com_rule->n_msg_sizes)) {
 352         return (0);
 353     }
 354 
 355     /* ok have some msg sizes, now to find the one closest to my mpi_msgsize */
 356 
 357     /* make a copy of the first msg rule */
 358     best_msg_p = msg_p = base_com_rule->msg_rules;
 359     i = 0;
 360 
 361     while (i<base_com_rule->n_msg_sizes) {
 362         /*       OPAL_OUTPUT((ompi_coll_tuned_stream,"checking mpi_msgsize %d against com_id %d msg_id %d index %d msg_size %d",  */
 363         /*             mpi_msgsize, msg_p->com_rule_id, msg_p->msg_rule_id, i, msg_p->msg_size)); */
 364         if (msg_p->msg_size <= mpi_msgsize) {
 365             best_msg_p = msg_p;
 366             /*          OPAL_OUTPUT((ompi_coll_tuned_stream(":ok\n")); */
 367         }
 368         else {
 369             /*          OPAL_OUTPUT((ompi_coll_tuned_stream(":nop\n")); */
 370             break;
 371         }
 372         /* go to the next entry */
 373         msg_p++;
 374         i++;
 375     }
 376 
 377     OPAL_OUTPUT((ompi_coll_tuned_stream,"Selected the following msg rule id %d\n", best_msg_p->msg_rule_id));
 378     ompi_coll_tuned_dump_msg_rule (best_msg_p);
 379 
 380     /* return the segment size */
 381     *result_topo_faninout = best_msg_p->result_topo_faninout;
 382 
 383     /* return the segment size */
 384     *result_segsize = best_msg_p->result_segsize;
 385 
 386     /* return the maximum requests */
 387     *max_requests = best_msg_p->result_max_requests;
 388 
 389     /* return the algorithm/method to use */
 390     return (best_msg_p->result_alg);
 391 }