root/opal/mca/pmix/pmix4x/pmix/src/util/timings.c

DEFINITIONS

1. pmix_init_id
2. get_ts_gettimeofday
3. _init_timestamping
4. pmix_timing_event_alloc
5. pmix_timing_init
6. pmix_timing_prep_ev
7. pmix_timing_prep_ev_end
8. pmix_timing_add_step
9. pmix_timing_descr
10. pmix_timing_start_id
11. pmix_timing_end
12. pmix_timing_end_prep
13. _prepare_descriptions
14. pmix_timing_report
15. pmix_timing_deltas
16. pmix_timing_release

   1 /*
   2  * Copyright (C) 2014      Artem Polyakov <artpol84@gmail.com>
   3  * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
   4  * $COPYRIGHT$
   5  *
   6  * Additional copyrights may follow
   7  *
   8  * $HEADER$
   9  */
  10 
  11 #include <src/include/pmix_config.h>
  12 
  13 #include <pmix_common.h>
  14 
  15 #include <stdlib.h>
  16 #include <stdarg.h>
  17 #include <stdio.h>
  18 #include <unistd.h>
  19 
  20 #ifdef HAVE_STRING_H
  21 #include <string.h>
  22 #endif
  23 
  24 #include <errno.h>
  25 #ifdef HAVE_SYS_TYPES_H
  26 #include <sys/types.h>
  27 #endif
  28 #ifdef HAVE_SYS_TIME_H
  29 #include <sys/time.h>
  30 #endif
  31 #ifdef HAVE_SYS_RESOURCE_H
  32 #include <sys/resource.h>
  33 #endif
  34 
  35 
  36 #if PMIX_ENABLE_TIMING
  37 
  38 #include "src/class/pmix_pointer_array.h"
  39 #include "src/class/pmix_list.h"
  40 #include "src/util/output.h"
  41 #include "src/util/basename.h"
  42 
  43 #include "src/util/timings.h"
  44 
  45 #define DELTAS_SANE_LIMIT (10*1024*1024)
  46 
  47 struct interval_descr{
  48     pmix_timing_event_t *descr_ev, *begin_ev;
  49     double interval, overhead;
  50 };
  51 
  52 pmix_timing_event_t *pmix_timing_event_alloc(pmix_timing_t *t);
  53 void pmix_timing_init(pmix_timing_t *t);
  54 pmix_timing_prep_t pmix_timing_prep_ev(pmix_timing_t *t, const char *fmt, ...);
  55 
  56 static PMIX_CLASS_INSTANCE(pmix_timing_event_t, pmix_list_item_t, NULL, NULL);
  57 
  58 
  59 static char *nodename = NULL;
  60 static char *jobid = "";
  61 static double hnp_offs = 0;
  62 static bool pmix_timing_overhead = false;
  63 
  64 void pmix_init_id(char* nspace, int rank)
  65 {
  66     asprintf(&jobid, "%s:%d", nspace, rank);
  67 }
  68 
  69 /* Get current timestamp. Derived from MPI_Wtime */
  70 
  71 static double get_ts_gettimeofday(void)
  72 {
  73     double ret;
  74     /* Use gettimeofday() if we pmix wasn't initialized */
  75     struct timeval tv;
  76     gettimeofday(&tv, NULL);
  77     ret = tv.tv_sec;
  78     ret += (double)tv.tv_usec / 1000000.0;
  79     return ret;
  80 }
  81 
  82 static get_ts_t _init_timestamping(void)
  83 {
  84     return get_ts_gettimeofday;
  85 }
  86 
  87 
  88 pmix_timing_event_t *pmix_timing_event_alloc(pmix_timing_t *t)
  89 {
  90     if( t->buffer_offset >= t->buffer_size ){
  91         // notch timings overhead
  92         double alloc_begin = t->get_ts();
  93 
  94         t->buffer = malloc(sizeof(pmix_timing_event_t)*t->buffer_size);
  95         if( t->buffer == NULL ){
  96             return NULL;
  97         }
  98         memset(t->buffer, 0, sizeof(pmix_timing_event_t)*t->buffer_size);
  99 
 100         double alloc_end = t->get_ts();
 101 
 102         t->buffer_offset = 0;
 103         t->buffer[0].fib = 1;
 104         t->buffer[0].ts_ovh = alloc_end - alloc_begin;
 105     }
 106     int tmp = t->buffer_offset;
 107     (t->buffer_offset)++;
 108     return t->buffer + tmp;
 109 }
 110 
 111 void pmix_timing_init(pmix_timing_t *t)
 112 {
 113     memset(t,0,sizeof(*t));
 114 
 115     t->next_id_cntr = 0;
 116     t->current_id = -1;
 117     /* initialize events list */
 118     t->events = PMIX_NEW(pmix_list_t);
 119     /* Set buffer size */
 120     t->buffer_size = PMIX_TIMING_BUFSIZE;
 121     /* Set buffer_offset = buffer_size so new buffer
 122      * will be allocated at first event report */
 123     t->buffer_offset = t->buffer_size;
 124     /* initialize gettime function */
 125     t->get_ts = _init_timestamping();
 126 
 127 }
 128 
 129 pmix_timing_prep_t pmix_timing_prep_ev(pmix_timing_t *t, const char *fmt, ...)
 130 {
 131     pmix_timing_event_t *ev = pmix_timing_event_alloc(t);
 132     if( ev == NULL ){
 133         pmix_timing_prep_t p = { t, NULL, PMIX_ERR_OUT_OF_RESOURCE };
 134         return p;
 135     }
 136     PMIX_CONSTRUCT(ev, pmix_timing_event_t);
 137     ev->ts = t->get_ts();
 138     va_list args;
 139     va_start( args, fmt );
 140     vsnprintf(ev->descr, PMIX_TIMING_DESCR_MAX - 1, fmt, args);
 141     ev->descr[PMIX_TIMING_DESCR_MAX-1] = '\0';
 142     va_end( args );
 143     pmix_timing_prep_t p = { t, ev, 0 };
 144     return p;
 145 }
 146 
 147 pmix_timing_prep_t pmix_timing_prep_ev_end(pmix_timing_t *t, const char *fmt, ...)
 148 {
 149     pmix_timing_prep_t p = { t, NULL, 0 };
 150 
 151     if( t->current_id >= 0 ){
 152         pmix_timing_event_t *ev = pmix_timing_event_alloc(t);
 153         if( ev == NULL ){
 154             pmix_timing_prep_t p = { t, NULL, PMIX_ERR_OUT_OF_RESOURCE };
 155             return p;
 156         }
 157         PMIX_CONSTRUCT(ev, pmix_timing_event_t);
 158         ev->ts = t->get_ts();
 159         p.ev = ev;
 160     }
 161     return p;
 162 }
 163 
 164 void pmix_timing_add_step(pmix_timing_prep_t p,
 165                           const char *func, const char *file, int line)
 166 {
 167     if( !p.errcode ) {
 168         p.ev->func = func;
 169         p.ev->file = file;
 170         p.ev->line = line;
 171         p.ev->type = PMIX_TIMING_TRACE;
 172         pmix_list_append(p.t->events, (pmix_list_item_t*)p.ev);
 173     }
 174 }
 175 
 176 /* Add description of the interval */
 177 int pmix_timing_descr(pmix_timing_prep_t p,
 178                            const char *func, const char *file, int line)
 179 {
 180     if( !p.errcode ){
 181         p.ev->func = func;
 182         p.ev->file = file;
 183         p.ev->line = line;
 184         p.ev->type = PMIX_TIMING_INTDESCR;
 185         p.ev->id = p.t->next_id_cntr;
 186         (p.t->next_id_cntr)++;
 187         pmix_list_append(p.t->events, (pmix_list_item_t*)p.ev);
 188         return p.ev->id;
 189     }
 190     return -1;
 191 }
 192 
 193 void pmix_timing_start_id(pmix_timing_t *t, int id, const char *func, const char *file, int line)
 194 {
 195     /* No description is needed. If everything is OK
 196      * it'll be included in pmix_timing_start_init */
 197     pmix_timing_event_t *ev = pmix_timing_event_alloc(t);
 198     if( ev == NULL ){
 199         return;
 200     }
 201     PMIX_CONSTRUCT(ev, pmix_timing_event_t);
 202 
 203     t->current_id = id;
 204     ev->ts = t->get_ts();
 205     ev->func = func;
 206     ev->file = file;
 207     ev->line = line;
 208     ev->type = PMIX_TIMING_INTBEGIN;
 209     ev->id = id;
 210     pmix_list_append(t->events, (pmix_list_item_t*)ev);
 211 }
 212 
 213 void pmix_timing_end(pmix_timing_t *t, int id, const char *func, const char *file, int line )
 214 {
 215     /* No description is needed. If everything is OK
 216      * it'll be included in pmix_timing_start_init */
 217     pmix_timing_event_t *ev = pmix_timing_event_alloc(t);
 218     if( ev == NULL ){
 219         return;
 220     }
 221     PMIX_CONSTRUCT(ev, pmix_timing_event_t);
 222 
 223     if( id < 0 ){
 224         ev->id = t->current_id;
 225         t->current_id = -1;
 226     } else {
 227         if( t->current_id == id ){
 228             t->current_id = -1;
 229         }
 230         ev->id = id;
 231     }
 232     ev->ts = t->get_ts();
 233     ev->func = func;
 234     ev->file = file;
 235     ev->line = line;
 236     ev->type = PMIX_TIMING_INTEND;
 237     pmix_list_append(t->events, (pmix_list_item_t*)ev);
 238 }
 239 
 240 void pmix_timing_end_prep(pmix_timing_prep_t p,
 241                                         const char *func, const char *file, int line)
 242 {
 243     pmix_timing_event_t *ev = p.ev;
 244 
 245     if( !p.errcode && ( NULL != ev ) ){
 246         assert(  p.t->current_id >=0 );
 247         ev->id = p.t->current_id;
 248         p.t->current_id = -1;
 249         ev->func = func;
 250         ev->file = file;
 251         ev->line = line;
 252         ev->type = PMIX_TIMING_INTEND;
 253         pmix_list_append(p.t->events, (pmix_list_item_t*)ev);
 254     }
 255 }
 256 
 257 static int _prepare_descriptions(pmix_timing_t *t, struct interval_descr **__descr)
 258 {
 259     struct interval_descr *descr;
 260     pmix_timing_event_t *ev, *next;
 261 
 262     if( t->next_id_cntr == 0 ){
 263         return 0;
 264     }
 265 
 266     *__descr = malloc(sizeof(struct interval_descr) * t->next_id_cntr);
 267     descr = *__descr;
 268     memset(descr, 0, sizeof(struct interval_descr) * t->next_id_cntr);
 269 
 270     PMIX_LIST_FOREACH_SAFE(ev, next, t->events, pmix_timing_event_t){
 271 
 272         /* pmix_output(0,"EVENT: type = %d, id=%d, ts = %.12le, ovh = %.12le %s",
 273                     ev->type, ev->id, ev->ts, ev->ts_ovh,
 274                     ev->descr );
 275         */
 276         switch(ev->type){
 277         case PMIX_TIMING_INTDESCR:{
 278             if( ev->id >= t->next_id_cntr){
 279                 char *file = pmix_basename(ev->file);
 280                 pmix_output(0,"pmix_timing: bad event id at %s:%d:%s, ignore and remove",
 281                             file, ev->line, ev->func);
 282                 free(file);
 283                 pmix_list_remove_item(t->events, (pmix_list_item_t *)ev);
 284                 continue;
 285             }
 286             if( NULL != descr[ev->id].descr_ev ){
 287                 pmix_timing_event_t *prev = descr[ev->id].descr_ev;
 288                 char *file = pmix_basename(ev->file);
 289                 char *file_prev = pmix_basename(prev->file);
 290                 pmix_output(0,"pmix_timing: duplicated description at %s:%d:%s, "
 291                             "previous: %s:%d:%s, ignore and remove", file, ev->line, ev->func,
 292                             file_prev, prev->line, prev->func);
 293                 free(file);
 294                 free(file_prev);
 295                 pmix_list_remove_item(t->events, (pmix_list_item_t *)ev);
 296                 continue;
 297             }
 298 
 299             descr[ev->id].descr_ev = ev;
 300             descr[ev->id].begin_ev = NULL;
 301             descr[ev->id].interval = 0;
 302             descr[ev->id].overhead = 0;
 303             break;
 304         }
 305         case PMIX_TIMING_INTBEGIN:
 306         case PMIX_TIMING_INTEND:{
 307             if( ev->id >= t->next_id_cntr || (NULL == descr[ev->id].descr_ev ) ){
 308                 char *file = pmix_basename(ev->file);
 309                 pmix_output(0,"pmix_timing: bad event id at %s:%d:%s, ignore and remove",
 310                             file, ev->line, ev->func);
 311                 free(file);
 312                 pmix_list_remove_item(t->events, (pmix_list_item_t *)ev);
 313                 continue;
 314             }
 315             break;
 316         }
 317         case PMIX_TIMING_TRACE:
 318             break;
 319         }
 320     }
 321     return t->next_id_cntr;
 322 }
 323 
 324 /* Output lines in portions that doesn't
 325  * exceed PMIX_TIMING_OUTBUF_SIZE for later automatic processing */
 326 int pmix_timing_report(pmix_timing_t *t, char *fname)
 327 {
 328     pmix_timing_event_t *ev;
 329     FILE *fp = NULL;
 330     char *buf = NULL;
 331     int buf_size = 0;
 332     struct interval_descr *descr = NULL;
 333     int rc = PMIX_SUCCESS;
 334 
 335     if( fname != NULL ){
 336         fp = fopen(fname,"a");
 337         if( fp == NULL ){
 338             pmix_output(0, "pmix_timing_report: Cannot open %s file"
 339                         " for writing timing information!",fname);
 340             rc = PMIX_ERROR;
 341             goto err_exit;
 342         }
 343     }
 344 
 345     _prepare_descriptions(t, &descr);
 346 
 347     buf = malloc(PMIX_TIMING_OUTBUF_SIZE+1);
 348     if( buf == NULL ){
 349         rc = PMIX_ERR_OUT_OF_RESOURCE;
 350         goto err_exit;
 351     }
 352     buf[0] = '\0';
 353 
 354     double overhead = 0;
 355     PMIX_LIST_FOREACH(ev, t->events, pmix_timing_event_t){
 356         char *line, *file;
 357         if( ev->fib && pmix_timing_overhead ){
 358             overhead += ev->ts_ovh;
 359         }
 360         file = pmix_basename(ev->file);
 361         switch( ev->type ){
 362         case PMIX_TIMING_INTDESCR:
 363             // Service event, skip it.
 364             continue;
 365         case PMIX_TIMING_TRACE:
 366             rc = asprintf(&line,"[%s:%d] %s \"%s\" [PMIX_TRACE] %s:%d %.10lf\n",
 367                           nodename, getpid(), jobid, ev->descr, file, ev->line,
 368                           ev->ts + hnp_offs + overhead);
 369             break;
 370         case PMIX_TIMING_INTBEGIN:
 371             rc = asprintf(&line,"[%s:%d] %s \"%s [start]\" [PMIX_TRACE] %s:%d %.10lf\n",
 372                           nodename, getpid(), jobid, descr[ev->id].descr_ev->descr,
 373                           file, ev->line, ev->ts + hnp_offs + overhead);
 374             break;
 375         case PMIX_TIMING_INTEND:
 376             rc = asprintf(&line,"[%s:%d] %s \"%s [stop]\" [PMIX_TRACE] %s:%d %.10lf\n",
 377                           nodename, getpid(), jobid, descr[ev->id].descr_ev->descr,
 378                           file, ev->line, ev->ts + hnp_offs + overhead);
 379             break;
 380         }
 381         free(file);
 382 
 383         if( rc < 0 ){
 384             rc = PMIX_ERR_OUT_OF_RESOURCE;
 385             goto err_exit;
 386         }
 387         rc = 0;
 388 
 389         /* Sanity check: this shouldn't happen since description
 390              * is event only 1KB long and other fields should never
 391              * exceed 9KB */
 392         assert( strlen(line) <= PMIX_TIMING_OUTBUF_SIZE );
 393 
 394 
 395         if( buf_size + strlen(line) > PMIX_TIMING_OUTBUF_SIZE ){
 396             // flush buffer to the file
 397             if( fp != NULL ){
 398                 fprintf(fp,"%s", buf);
 399                 fprintf(fp,"\n");
 400             } else {
 401                 pmix_output(0,"\n%s", buf);
 402             }
 403             buf[0] = '\0';
 404             buf_size = 0;
 405         }
 406         sprintf(buf,"%s%s", buf, line);
 407         buf_size += strlen(line);
 408         free(line);
 409     }
 410 
 411     if( buf_size > 0 ){
 412         // flush buffer to the file
 413         if( fp != NULL ){
 414             fprintf(fp,"%s", buf);
 415             fprintf(fp,"\n");
 416         } else {
 417             pmix_output(0,"\n%s", buf);
 418         }
 419         buf[0] = '\0';
 420         buf_size = 0;
 421     }
 422 
 423 err_exit:
 424     if( NULL != descr ){
 425         free(descr);
 426     }
 427     if( buf != NULL ){
 428         free(buf);
 429     }
 430     if( fp != NULL ){
 431         fflush(fp);
 432         fclose(fp);
 433     }
 434     return rc;
 435 }
 436 
 437 /* Output events as one buffer so the data won't be mixed
 438  * with other output. This function is supposed to be human readable.
 439  * The output goes only to stdout. */
 440 int pmix_timing_deltas(pmix_timing_t *t, char *fname)
 441 {
 442     pmix_timing_event_t *ev;
 443     FILE *fp = NULL;
 444     char *buf = NULL;
 445     struct interval_descr *descr = NULL;
 446     int i, rc = PMIX_SUCCESS;
 447     size_t buf_size = 0, buf_used = 0;
 448 
 449     if( fname != NULL ){
 450         fp = fopen(fname,"a");
 451         if( fp == NULL ){
 452             pmix_output(0, "pmix_timing_report: Cannot open %s file"
 453                         " for writing timing information!",fname);
 454             rc = PMIX_ERROR;
 455             goto err_exit;
 456         }
 457     }
 458 
 459     _prepare_descriptions(t, &descr);
 460 
 461     PMIX_LIST_FOREACH(ev, t->events, pmix_timing_event_t){
 462         int id;
 463         if( ev->fib ){
 464             /* this event caused buffered memory allocation
 465              * for events. Account the overhead for all active
 466              * intervals. */
 467             int i;
 468             for( i = 0; i < t->next_id_cntr; i++){
 469                 if( (NULL != descr[i].descr_ev) && (NULL != descr[i].begin_ev) ){
 470                     if( pmix_timing_overhead ){
 471                         descr[i].overhead += ev->ts_ovh;
 472                     }
 473                 }
 474             }
 475         }
 476 
 477         /* we already process all PMIX_TIMING_DESCR events
 478          * and we ignore PMIX_TIMING_EVENT */
 479         if( ev->type == PMIX_TIMING_INTDESCR ||
 480                 ev->type == PMIX_TIMING_TRACE){
 481             /* skip */
 482             continue;
 483         }
 484 
 485         id = ev->id;
 486         if( id < 0 || id >= t->next_id_cntr ){
 487             char *file = pmix_basename(ev->file);
 488             pmix_output(0,"pmix_timing_deltas: bad interval event id: %d at %s:%d:%s (maxid=%d)",
 489                         id, file, ev->line, ev->func, t->next_id_cntr - 1 );
 490             free(file);
 491             /* skip */
 492             continue;
 493         }
 494 
 495         /* id's assigned auomatically. Ther shouldn't be any gaps in descr[] */
 496         assert( NULL != descr[id].descr_ev);
 497 
 498         if( ev->type == PMIX_TIMING_INTBEGIN ){
 499             if( NULL != descr[id].begin_ev ){
 500                 /* the measurement on this interval was already
 501                  * started! */
 502                 pmix_timing_event_t *prev = descr[ev->id].begin_ev;
 503                 char *file = pmix_basename(ev->file);
 504                 char *file_prev = pmix_basename(prev->file);
 505                 pmix_output(0,"pmix_timing_deltas: duplicated start statement at %s:%d:%s, "
 506                             "previous: %s:%d:%s", file, ev->line, ev->func,
 507                             file_prev, prev->line, prev->func);
 508                 free(file);
 509                 free(file_prev);
 510             } else {
 511                 /* save pointer to the start of measurement event */
 512                 descr[id].begin_ev = ev;
 513             }
 514             /* done, go to the next event */
 515             continue;
 516         }
 517 
 518         if( ev->type == PMIX_TIMING_INTEND ){
 519             if( NULL == descr[id].begin_ev ){
 520                 /* the measurement on this interval wasn't started! */
 521                 char *file = pmix_basename(ev->file);
 522                 pmix_output(0,"pmix_timing_deltas: inteval end without start at %s:%d:%s",
 523                             file, ev->line, ev->func );
 524                 free(file);
 525             } else {
 526                 descr[id].interval += ev->ts - descr[id].begin_ev->ts;
 527                 descr[id].begin_ev = NULL;
 528                 if( ev->fib ){
 529                     descr[id].overhead += ev->ts_ovh;
 530                 }
 531             }
 532             continue;
 533         }
 534 
 535         /* shouldn't ever get here: bad ev->type */
 536         pmix_output(0, "pmix_timing_deltas: bad event type %d", ev->type);
 537         assert(0);
 538     }
 539 
 540     buf = malloc(PMIX_TIMING_OUTBUF_SIZE + 1);
 541     if( buf == NULL ){
 542         rc = PMIX_ERR_OUT_OF_RESOURCE;
 543         goto err_exit;
 544     }
 545     buf[0] = '\0';
 546     buf_size = PMIX_TIMING_OUTBUF_SIZE + 1;
 547     buf_used = 0;
 548     for(i = 0; i < t->next_id_cntr; i++){
 549         char *line = NULL;
 550         size_t line_size;
 551         rc = asprintf(&line,"[%s:%d] %s \"%s\" [PMIX_OVHD] %le\n",
 552                       nodename, getpid(), jobid, descr[i].descr_ev->descr,
 553                       descr[i].interval - descr[i].overhead);
 554         if( rc < 0 ){
 555             rc = PMIX_ERR_OUT_OF_RESOURCE;
 556             goto err_exit;
 557         }
 558         rc = 0;
 559         line_size = strlen(line);
 560 
 561         /* Sanity check: this shouldn't happen since description
 562          * is event only 1KB long and other fields should never
 563          * exceed 9KB */
 564         assert( line_size <= PMIX_TIMING_OUTBUF_SIZE );
 565 
 566         if( buf_used + strlen(line) > buf_size ){
 567             // Increase output buffer
 568             while( buf_used + line_size > buf_size && buf_size < DELTAS_SANE_LIMIT){
 569                 buf_size += PMIX_TIMING_OUTBUF_SIZE + 1;
 570             }
 571             if( buf_size > DELTAS_SANE_LIMIT ){
 572                 pmix_output(0, "pmix_timing_report: delta sane limit overflow (%u > %u)!\n",
 573                             (unsigned int)buf_size, DELTAS_SANE_LIMIT);
 574                 free(line);
 575                 rc = PMIX_ERR_OUT_OF_RESOURCE;
 576                 goto err_exit;
 577             }
 578             buf = realloc(buf, buf_size);
 579             if( buf == NULL ){
 580                 pmix_output(0, "pmix_timing_deltas: Out of memory!\n");
 581                 rc = PMIX_ERR_OUT_OF_RESOURCE;
 582                 goto err_exit;
 583             }
 584         }
 585         sprintf(buf,"%s%s", buf, line);
 586         buf_used += line_size;
 587         free(line);
 588     }
 589 
 590 
 591     if( buf_used > 0 ){
 592         // flush buffer to the file
 593         if( fp != NULL ){
 594             fprintf(fp,"%s", buf);
 595             fprintf(fp,"\n");
 596         } else {
 597             pmix_output(0,"\n%s", buf);
 598         }
 599         buf[0] = '\0';
 600         buf_size = 0;
 601     }
 602 
 603 err_exit:
 604     if( NULL != descr ){
 605         free(descr);
 606     }
 607     if( NULL != buf ){
 608         free(buf);
 609     }
 610     if( fp != NULL ){
 611         fflush(fp);
 612         fclose(fp);
 613     }
 614     return rc;
 615 }
 616 
 617 void pmix_timing_release(pmix_timing_t *t)
 618 {
 619     int cnt = pmix_list_get_size(t->events);
 620 
 621     if( cnt > 0 ){
 622         pmix_list_t *tmp = PMIX_NEW(pmix_list_t);
 623         int i;
 624         for(i=0; i<cnt; i++){
 625             pmix_timing_event_t *ev = (pmix_timing_event_t *)pmix_list_remove_first(t->events);
 626             if( ev->fib ){
 627                 pmix_list_append(tmp,(pmix_list_item_t*)ev);
 628             }
 629         }
 630 
 631         cnt = pmix_list_get_size(tmp);
 632         for(i=0; i<cnt; i++){
 633             pmix_timing_event_t *ev = (pmix_timing_event_t *)pmix_list_remove_first(tmp);
 634             free(ev);
 635         }
 636         PMIX_RELEASE(tmp);
 637     } else {
 638         // Error case. At list one event was inserted at initialization.
 639 
 640     }
 641 
 642     PMIX_RELEASE(t->events);
 643     t->events = NULL;
 644 }
 645 #endif