root/orte/test/system/threads.c

DEFINITIONS

1. get_ts_gettimeofday
2. main
3. value_xfer_thread
4. bind_me_to

   1 /*
   2  * Test program for memory consistency in a thread shifting design
   3  *
   4  *
   5  * Run:
   6  *  ./threads ITERATIONS [MODE]
   7  *  ./threads 9000000 3
   8  *
   9  * Example:
  10  *  ./threads 9000000 0 --> Will fail, no memory barriers
  11  *  ./threads 9000000 1 --> Will fail, no WMB
  12  *  ./threads 9000000 2 --> Will fail, no RMB
  13  *  ./threads 9000000 3 --> Success
  14  *  ./threads 9000000 4 --> Success
  15  *  ./threads 9000000 5 --> N/A
  16  */
  17 #include <stdio.h>
  18 #include <stdlib.h>
  19 #include <stdbool.h>
  20 #include <string.h>
  21 #include <pthread.h>
  22 #include <stdint.h>
  23 #include <hwloc.h>
  24 #include <sys/time.h>
  25 
  26 #include "opal/sys/atomic.h"
  27 
  28 
  29 // Max value for an int16_t
  30 #define MAX_VAL 32767
  31 
  32 typedef struct {
  33     int type;
  34     union {
  35         bool flag;
  36         int integer;
  37         int8_t int8;
  38         int16_t int16;
  39         int32_t int32;
  40         int64_t int64;
  41         //char padding[1];
  42     } data;
  43 } my_value_t;
  44 
  45 // Structure to handoff work to the peer thread
  46 typedef struct {
  47     volatile bool working;
  48     void *ptr; // Note that adding a volatile here has no effect
  49 } thread_handoff_t;
  50 
  51 // Shared object to handoff work
  52 thread_handoff_t handoff;
  53 
  54 // Indicates if the test has finished
  55 bool time_to_stop = false;
  56 
  57 // Progress reporting
  58 #define PERC_INC 10.0
  59 double perc_report_after = PERC_INC;
  60 double perc_current = 0.0;
  61 
  62 // Memory barrier modes
  63 #define MB_MODE_NONE 0x0
  64 #define MB_MODE_RMB  0x1
  65 #define MB_MODE_WMB  0x2
  66 #define MB_MODE_MB   0x4
  67 #define MB_MODE_XMB  0x8
  68 #define MB_MODE_ALL (MB_MODE_RMB | MB_MODE_WMB)
  69 int mb_mode = MB_MODE_ALL;
  70 
  71 
  72 // Shared hwloc topology (so we only have to read it once)
  73 static hwloc_topology_t topo;
  74 // Which object we are binding to
  75 //  4 - sockets with 5 cores each
  76 // 20 - cores with 8 PUs each
  77 //#define OBJ_TYPE HWLOC_OBJ_SOCKET
  78 #define OBJ_TYPE HWLOC_OBJ_CORE
  79 
  80 /*
  81  * Some basic timing support
  82  */
  83 double acc_time, start_time, stop_time, delta;
  84 static double get_ts_gettimeofday(void) {
  85     double ret;
  86     struct timeval tv;
  87     gettimeofday(&tv, NULL);
  88     ret = tv.tv_sec;
  89     ret += (double)tv.tv_usec / 1000000.0;
  90     return ret;
  91 }
  92 
  93 /*
  94  * Bind either the main or support thread far away from each other
  95  */
  96 void bind_me_to(bool main_thread);
  97 
  98 /*
  99  * Support thread to do the memory allocation and xfer
 100  */
 101 void *value_xfer_thread(void *arg);
 102 
 103 /*
 104  * Main thread
 105  */
 106 int main(int argc, char **argv) {
 107     pthread_t support_thread;
 108     int rc, i, max_iters = 10, cur_iter;
 109     my_value_t *val = NULL;
 110     int mode;
 111 
 112     /*
 113      * Parse command line arguments
 114      */
 115     if( argc > 1 ) {
 116         max_iters = atoi(argv[1]);
 117     }
 118     if( argc > 2 ) {
 119         mode = atoi(argv[2]);
 120         if( 0 > mode || mode > 5 ) {
 121             printf("Error: Invalid mode %d\n"
 122                    "\tNone = 0\n"
 123                    "\tRMB = 1\n"
 124                    "\tWMB = 2\n"
 125                    "\tBoth = 3\n"
 126                    "\tMB Only = 4\n",
 127                    "\tXMB Only = 5\n",
 128                    mode);
 129             exit(-1);
 130         }
 131     }
 132     else {
 133         mode = 3;
 134     }
 135     switch(mode) {
 136     case 0:
 137         mb_mode = MB_MODE_NONE;
 138         break;
 139     case 1:
 140         mb_mode = MB_MODE_RMB;
 141         break;
 142     case 2:
 143         mb_mode = MB_MODE_WMB;
 144         break;
 145     case 3:
 146         mb_mode = MB_MODE_ALL;
 147         break;
 148     case 4:
 149         mb_mode = MB_MODE_MB;
 150         break;
 151     case 5:
 152         mb_mode = MB_MODE_XMB;
 153         break;
 154     }
 155 
 156     // Load hwloc topology
 157     hwloc_topology_init(&topo);
 158     hwloc_topology_load(topo);
 159 
 160     // Display banner
 161     printf("---------------------------\n");
 162     printf("Iterations: %10d\n", max_iters);
 163     printf("Mode R MB : %10s\n", (mb_mode & MB_MODE_RMB ? "Enabled" : "Disabled") );
 164     printf("Mode W MB : %10s\n", (mb_mode & MB_MODE_WMB ? "Enabled" : "Disabled") );
 165     printf("Mode - MB : %10s\n", (mb_mode & MB_MODE_MB  ? "Enabled" : "Disabled") );
 166     printf("Mode X MB : %10s\n", (mb_mode & MB_MODE_XMB ? "Enabled" : "Disabled") );
 167     printf("---------------------------\n");
 168 
 169     bind_me_to(true);
 170     handoff.working = false;
 171 
 172     /*
 173      * Launch supporting thread
 174      */
 175     rc = pthread_create(&support_thread, NULL, value_xfer_thread, NULL);
 176     if( 0 != rc ) {
 177         printf("Error: Failed to create a thread! %d\n", rc);
 178         exit(-1);
 179     }
 180 
 181     /*
 182      * Main work loop
 183      */
 184     acc_time = 0.0;
 185     for(cur_iter = 0; cur_iter < max_iters; ++cur_iter) {
 186         perc_current = (cur_iter / ((double)max_iters)) * 100.0;
 187         if( perc_current > perc_report_after ) {
 188             delta = (acc_time / cur_iter) * 1000000;
 189             printf("%6.1f %% complete : Iteration %10d / %10d : %6.1f usec / iter\n",
 190                    perc_current, cur_iter+1, max_iters, delta);
 191             perc_report_after += PERC_INC;
 192         }
 193 
 194         start_time = get_ts_gettimeofday();
 195         // Initialize values
 196         val = NULL;
 197         handoff.ptr = &val;
 198         if( mb_mode & MB_MODE_RMB ) {
 199             opal_atomic_rmb();
 200         }
 201         if( mb_mode & MB_MODE_MB ) {
 202             opal_atomic_mb();
 203         }
 204         handoff.working = true;
 205 
 206         // Wait for work to finish
 207         while( handoff.working ) {
 208             usleep(1);
 209         }
 210         if( mb_mode & MB_MODE_WMB ) {
 211             opal_atomic_wmb();
 212         }
 213         if( mb_mode & MB_MODE_MB ) {
 214             opal_atomic_mb();
 215         }
 216 
 217         // Inspect values for correctness
 218         if( NULL == val ) {
 219             printf("[%10d / %10d] Error: val = %s\n", cur_iter+1, max_iters,
 220                    (NULL == val ? "NULL" : "Valid") );
 221             exit(-1);
 222         }
 223         else if( 999 != val->type ) {
 224             printf("[%10d / %10d] Error: val->type = %d\n", cur_iter+1, max_iters, val->type);
 225             exit(-1);
 226         }
 227         else if( (cur_iter+1)%MAX_VAL != val->data.int16 ) {
 228             printf("[%10d / %10d] Error: val->data.int16 = %d\n", cur_iter+1, max_iters, val->data.int16);
 229             exit(-1);
 230         }
 231 
 232         stop_time = get_ts_gettimeofday();
 233         acc_time += (stop_time - start_time);
 234 
 235         // Yes, this is a memory leak!
 236         // I need to make sure that the supporting thread is not reusing a
 237         // previous storage location when it calls malloc. This is to emulate
 238         // a program that calls malloc after the value was acquired, possibly
 239         // reusing this memory location.
 240         //free(val);
 241         val = NULL;
 242     }
 243     delta = (acc_time / max_iters) * 1000000;
 244 
 245     /*
 246      * All done - Cleanup
 247      */
 248     time_to_stop = true;
 249 
 250     rc = pthread_join(support_thread, NULL);
 251     if( 0 != rc ) {
 252         printf("Error: Failed to join a thread! %d\n", rc);
 253         exit(-1);
 254     }
 255 
 256     hwloc_topology_destroy(topo);
 257 
 258     printf("Success - %6.1f usec / iter\n", delta);
 259 
 260     return 0;
 261 }
 262 
 263 void *value_xfer_thread(void *arg) {
 264     my_value_t **val = NULL;
 265     static int var = 0;
 266 
 267     // Bind this thread away from the main thread
 268     bind_me_to(false);
 269 
 270     while( !time_to_stop ) {
 271         if( handoff.working ) {
 272             // Make sure I have the right pointer
 273             if( mb_mode & MB_MODE_WMB ) {
 274                 opal_atomic_wmb();
 275             }
 276             if( mb_mode & MB_MODE_MB ) {
 277                 opal_atomic_mb();
 278             }
 279 
 280             // Allocate and set the value
 281             val = (my_value_t**)handoff.ptr;
 282             (*val) = malloc(sizeof(my_value_t));
 283             (*val)->type = 999;
 284             (*val)->data.int16 = (++var)%MAX_VAL;
 285 
 286             // Make sure main thread can see the value
 287             // See 'Examples' -> 'Global thread flag' discussion here:
 288             // https://www.ibm.com/developerworks/systems/articles/powerpc.html
 289             if( mb_mode & MB_MODE_RMB ) {
 290                 opal_atomic_rmb();
 291             }
 292             if( mb_mode & MB_MODE_MB ) {
 293                 opal_atomic_mb();
 294             }
 295             // Release main thread
 296             handoff.working = false;
 297         }
 298         else {
 299             // wait for work
 300             usleep(1);
 301         }
 302     }
 303     pthread_exit(NULL);
 304 }
 305 
 306 void bind_me_to(bool main_thread) {
 307     int num_objs;
 308     hwloc_cpuset_t set;
 309     char *buffer = NULL;
 310     hwloc_obj_t obj;
 311 
 312     num_objs = hwloc_get_nbobjs_by_type(topo, OBJ_TYPE);
 313 
 314     if( main_thread ) {
 315         obj = hwloc_get_obj_by_type(topo, OBJ_TYPE, 0);
 316     }
 317     else {
 318         obj = hwloc_get_obj_by_type(topo, OBJ_TYPE, num_objs-1);
 319     }
 320 
 321     if( obj->type == OBJ_TYPE ) {
 322         hwloc_set_cpubind(topo, obj->cpuset, HWLOC_CPUBIND_THREAD);
 323     }
 324     else {
 325         printf("Error: Invalid object\n");
 326         exit(-1);
 327     }
 328 
 329     set = hwloc_bitmap_alloc();
 330     hwloc_get_cpubind(topo, set, HWLOC_CPUBIND_THREAD);
 331     hwloc_bitmap_opal_asprintf(&buffer, set);
 332     printf("%s : [objs = %d] : cpuset is %s\n", (main_thread ? "Main" : "Peer"), num_objs, buffer);
 333     free(buffer);
 334     hwloc_bitmap_free(set);
 335 }