root/oshmem/shmem/c/shmem_lock.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. shmem_lock_init
  2. shmem_lock_finalize
  3. shmem_lock_get_server
  4. get_lock_value
  5. set_lock_value
  6. extract_2_words
  7. pack_2_words
  8. extract_first_word
  9. extract_second_word
  10. shmem_lock_cswap
  11. shmem_lock_cswap_poll
  12. shmem_lock_fadd
  13. pack_first_word
  14. pack_second_word
  15. lock_extract_pe_next_counter
  16. shmem_lock_extract_pe_next
  17. lock_extract_pe_last
  18. lock_extract_counter
  19. shmem_lock_pack_pe_next_pe_last
  20. lock_pack_pe_next_counter
  21. shmem_lock_pack_pe_next
  22. lock_pack_counter
  23. lock_pack_pe_last
  24. lock_find_counter
  25. shmem_lock_insert_counter
  26. shmem_lock_remove_counter
  27. shmem_lock_increment_counter
  28. shmem_lock_decrement_counter
  29. lock_get_count
  30. shmem_lock_is_mine
  31. shmem_lock_get_ticket
  32. shmem_get_wrapper
  33. shmem_lock_wait_for_ticket
  34. shmem_lock_subscribe_for_informing
  35. shmem_wait_wrapper
  36. shmem_lock_wait_for_informing
  37. shmem_lock_inform_next
  38. lock_save_prev_pe
  39. lock_restore_prev_pe
  40. shmem_lock_try_inform_server
  41. _shmem_set_lock
  42. _shmem_test_lock
  43. _shmem_clear_lock

   1 /*
   2  * Copyright (c) 2013      Mellanox Technologies, Inc.
   3  *                         All rights reserved.
   4  * Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
   5  * Copyright (c) 2015      Research Organization for Information Science
   6  *                         and Technology (RIST). All rights reserved.
   7  * $COPYRIGHT$
   8  *
   9  * Additional copyrights may follow
  10  *
  11  * $HEADER$
  12  */
  13 /**
  14  * @file
  15  */
  16 #include "oshmem_config.h"
  17 
  18 #include "oshmem/constants.h"
  19 #include "oshmem/include/shmem.h"
  20 #include "oshmem/runtime/params.h"
  21 #include "oshmem/runtime/runtime.h"
  22 #include <stdlib.h>
  23 #include <memory.h>
  24 
  25 #include "oshmem/shmem/shmem_api_logger.h"
  26 #include "oshmem/shmem/shmem_lock.h"
  27 #include "oshmem/mca/memheap/memheap.h"
  28 #include "oshmem/mca/memheap/base/base.h"
  29 #include "oshmem/mca/atomic/atomic.h"
  30 
  31 #define OPAL_BITWISE_SIZEOF_LONG (SIZEOF_LONG * 8)
  32 
  33 struct oshmem_lock_counter {
  34     void *lock;
  35     int counter;
  36     struct oshmem_lock_counter *next;
  37     struct oshmem_lock_counter *prev;
  38 };
  39 typedef struct oshmem_lock_counter oshmem_lock_counter_t;
  40 
  41 struct oshmem_lock_prev_pe_container {
  42     void *lock;
  43     int prev_pe;
  44     struct oshmem_lock_prev_pe_container *next;
  45     struct oshmem_lock_prev_pe_container *prev;
  46 };
  47 typedef struct oshmem_lock_prev_pe_container oshmem_lock_prev_pe_container_t;
  48 
  49 oshmem_lock_counter_t *lock_counter_head = NULL;
  50 oshmem_lock_prev_pe_container_t *lock_prev_pe_container_head = NULL;
  51 static int *lock_turn;
  52 static int *lock_inform;
  53 static int *lock_last_ticket;
  54 
  55 static int lock_save_prev_pe(void *lock, int prev_pe);
  56 static int lock_restore_prev_pe(void *lock, int *prev_pe);
  57 
  58 static int shmem_lock_try_inform_server(void *lock, int lock_size);
  59 static void shmem_get_wrapper(uint64_t *target,
  60                               const void *source,
  61                               int source_size,
  62                               size_t nelems,
  63                               int pe);
  64 static void shmem_wait_wrapper(void *target, int target_size, uint64_t value);
  65 
  66 static int shmem_lock_extract_pe_next(void *lock, int lock_size, int *pe_next);
  67 static int shmem_lock_pack_pe_next_pe_last(void *lock,
  68                                            int lock_size,
  69                                            const int *pe_next,
  70                                            const int *pe_last);
  71 static int shmem_lock_pack_pe_next(void *lock,
  72                                    int lock_size,
  73                                    const int *pe_next);
  74 
  75 static void shmem_lock_increment_counter(void *lock, int lock_size);
  76 static int shmem_lock_decrement_counter(void *lock, int lock_size);
  77 
  78 static int shmem_lock_get_server(void *lock);
  79 static int shmem_lock_is_mine(void *lock, int lock_size);
  80 
  81 static int shmem_lock_get_ticket(void *lock);
  82 static int shmem_lock_wait_for_ticket(void *lock,
  83                                       int lock_size,
  84                                       int ticket,
  85                                       int *pe_last);
  86 static int shmem_lock_subscribe_for_informing(void *lock,
  87                                               int lock_size,
  88                                               int pe_last);
  89 static int shmem_lock_wait_for_informing(void *lock, int lock_size);
  90 static int shmem_lock_inform_next(void *lock, int lock_size, int pe_next);
  91 
  92 /***************************************************************************/
  93 /**************************Init/Finalize************************************/
  94 /***************************************************************************/
  95 
  96 int shmem_lock_init()
  97 {
  98     void* ptr = 0;
  99 
 100 #if (OPAL_BITWISE_SIZEOF_LONG == 32)
 101     int number_of_pes = shmem_n_pes();
 102     if (number_of_pes >= 65534)
 103     {
 104         SHMEM_API_ERROR("SHMEM distributed locking implementation does not support total number of PEs greater than 65534 if sizeof(long) = 4");
 105         return OSHMEM_ERROR;
 106     }
 107 #endif
 108 
 109 #if ((OPAL_BITWISE_SIZEOF_LONG != 64) && (OPAL_BITWISE_SIZEOF_LONG != 32))
 110     SHMEM_API_ERROR("SHMEM distributed locking implementation does not support sizeof(long) = %i",
 111                     sizeof(long));
 112     return OSHMEM_ERROR;
 113 #endif
 114 
 115     ptr = (void *) lock_turn;
 116     MCA_MEMHEAP_CALL(private_alloc(sizeof(int), &ptr));
 117     lock_turn = (int *) ptr;
 118     *lock_turn = 0;
 119     ptr = (void *) lock_last_ticket;
 120     MCA_MEMHEAP_CALL(private_alloc(sizeof(int), &ptr));
 121     lock_last_ticket = (int *) ptr;
 122     *lock_last_ticket = 0;
 123     ptr = (void *) lock_inform;
 124     MCA_MEMHEAP_CALL(private_alloc(sizeof(int), &ptr));
 125     lock_inform = (int *) ptr;
 126     *lock_inform = 0;
 127 
 128     lock_counter_head = 0;
 129     lock_prev_pe_container_head = 0;
 130 
 131     return OSHMEM_SUCCESS;
 132 }
 133 
 134 int shmem_lock_finalize()
 135 {
 136     oshmem_lock_counter_t *current_counter = lock_counter_head;
 137     oshmem_lock_prev_pe_container_t *current_pe_container =
 138             lock_prev_pe_container_head;
 139 
 140     if (0 != lock_turn) {
 141         MCA_MEMHEAP_CALL(private_free(lock_turn));
 142     }
 143     if (0 != lock_last_ticket) {
 144         MCA_MEMHEAP_CALL(private_free(lock_last_ticket));
 145     }
 146     if (0 != lock_inform) {
 147         MCA_MEMHEAP_CALL(private_free(lock_inform));
 148     }
 149 
 150     lock_turn = 0;
 151     lock_last_ticket = 0;
 152     lock_inform = 0;
 153 
 154     while (0 != current_counter) {
 155         oshmem_lock_counter_t *counter_to_free = current_counter;
 156         current_counter = current_counter->next;
 157         free(counter_to_free);
 158     }
 159     lock_counter_head = 0;
 160 
 161     while (0 != current_pe_container) {
 162         oshmem_lock_prev_pe_container_t *container_to_free =
 163                 current_pe_container;
 164         current_pe_container = current_pe_container->next;
 165         free(container_to_free);
 166     }
 167     lock_prev_pe_container_head = 0;
 168 
 169     return OSHMEM_SUCCESS;
 170 }
 171 
 172 static int shmem_lock_get_server(void *lock)
 173 {
 174     map_segment_t *s;
 175 
 176     s = memheap_find_va(lock);
 177     if (NULL == s) {
 178         SHMEM_API_ERROR("PE#%i lock %p is not a shared variable", shmem_my_pe(), lock);
 179         oshmem_shmem_abort(-1);
 180         return 0;
 181     }
 182 
 183     return ((int)((uintptr_t)lock - (uintptr_t)s->super.va_base)/8) % shmem_n_pes();
 184 }
 185 
 186 static uint64_t get_lock_value(const void *lock, int lock_size)
 187 {
 188     uint64_t lock_value = 0;
 189 
 190     if (lock_size == 4) {
 191         lock_value = *(uint32_t *) lock;
 192     } else if (lock_size == 8) {
 193         lock_value = *(uint64_t *) lock;
 194     }
 195 
 196     return lock_value;
 197 }
 198 
 199 static void set_lock_value(void *lock, int lock_size, uint64_t lock_value)
 200 {
 201     if (lock_size == 8) {
 202         memcpy(lock, &lock_value, 8);
 203     } else if (lock_size == 4) {
 204         uint32_t lock_value_32 = (uint32_t) lock_value;
 205         memcpy(lock, &lock_value_32, 4);
 206     }
 207 }
 208 
 209 /***************************************************************************/
 210 /**************************Pack/Extract*************************************/
 211 /***************************************************************************/
 212 
 213 static int extract_2_words(const void *lock, int lock_size, int *one, int *two)
 214 {
 215     int lock_bitwise_size = lock_size * 8;
 216     uint64_t lock_value = get_lock_value(lock, lock_size);
 217 
 218     if (lock == 0 || one == 0 || two == 0) {
 219         return OSHMEM_ERROR;
 220     }
 221 
 222     *one = (int) (lock_value >> (lock_bitwise_size / 2));
 223     if (lock_size == 8) {
 224         *two = (int) ((lock_value << 32) >> 32);
 225     } else if (lock_size == 4) {
 226         *two = (int) ((lock_value << 48) >> 48);
 227     }
 228 
 229     return OSHMEM_SUCCESS;
 230 }
 231 
 232 static int pack_2_words(void *lock,
 233                         int lock_size,
 234                         const int *one,
 235                         const int *two)
 236 {
 237     uint64_t lock_value = 0;
 238     int lock_bitwise_size = lock_size * 8;
 239 
 240     if (lock == 0 || one == 0 || two == 0) {
 241         return OSHMEM_ERROR;
 242     }
 243 
 244     lock_value = (uint64_t) *two
 245             | (((uint64_t) *one) << (lock_bitwise_size / 2));
 246     set_lock_value(lock, lock_size, lock_value);
 247 
 248     return OSHMEM_SUCCESS;
 249 }
 250 
 251 static int extract_first_word(void *lock, int lock_size, int *one)
 252 {
 253     int two = 0;
 254     return extract_2_words(lock, lock_size, one, &two);
 255 }
 256 
 257 static int extract_second_word(void *lock, int lock_size, int *two)
 258 {
 259     int one = 0;
 260     return extract_2_words(lock, lock_size, &one, two);
 261 }
 262 
 263 static uint64_t shmem_lock_cswap(void *target,
 264                                  int target_size,
 265                                  uint64_t cond,
 266                                  uint64_t value,
 267                                  int pe)
 268 {
 269     uint64_t prev_value = 0;
 270 
 271     if (target_size == 8) {
 272         MCA_ATOMIC_CALL(cswap(oshmem_ctx_default, target, (void*)&prev_value, cond, value, target_size, pe));
 273     } else if (target_size == 4) {
 274         uint32_t prev_value_32 = 0;
 275         uint32_t cond32 = (uint32_t) cond;
 276         uint32_t value32 = (uint32_t) value;
 277 
 278         MCA_ATOMIC_CALL(cswap(oshmem_ctx_default, target, (void*)&prev_value_32, cond32, value32, target_size, pe));
 279 
 280         prev_value = prev_value_32;
 281     }
 282     return prev_value;
 283 }
 284 
 285 /* function is used to busy wait for the value. 
 286  * Call opal_progress() so that ompi will no deadlock
 287  * (for example may need to respond to rkey requests)
 288  */
 289 static uint64_t shmem_lock_cswap_poll(void *target,
 290                                       int target_size,
 291                                       uint64_t cond,
 292                                       uint64_t value,
 293                                       int pe)
 294 {
 295     uint64_t prev_value;
 296 
 297     prev_value = shmem_lock_cswap(target, target_size, cond, value, pe);
 298     opal_progress();
 299     return prev_value;
 300 }
 301 
 302 static uint64_t shmem_lock_fadd(void *target,
 303                                 int target_size,
 304                                 uint64_t value,
 305                                 int pe)
 306 {
 307     uint64_t prev_value = 0;
 308 
 309     if (target_size == sizeof(int)) {
 310         prev_value = (uint64_t) shmem_int_fadd((int *) target, (int) value, pe);
 311     } else if (target_size == sizeof(long)) {
 312         prev_value = (uint64_t) shmem_long_fadd((long *) target,
 313                                                 (long) value,
 314                                                 pe);
 315     } else if (target_size == sizeof(long long)) {
 316         prev_value = (uint64_t) shmem_longlong_fadd((long long *) target,
 317                                                     (long long) value,
 318                                                     pe);
 319     }
 320 
 321     return prev_value;
 322 }
 323 
 324 static int pack_first_word(void *lock,
 325                            int lock_size,
 326                            const int *one,
 327                            int use_atomic)
 328 {
 329     int my_pe = shmem_my_pe();
 330     uint64_t lock_value = 0;
 331     uint64_t new_long_value = 0;
 332     uint64_t temp = 0;
 333     int two = 0;
 334 
 335     if (lock == 0 || one == 0) {
 336         return OSHMEM_ERROR;
 337     }
 338 
 339     if (use_atomic) {
 340         lock_value = get_lock_value(lock, lock_size);
 341         extract_second_word(&lock_value, lock_size, &two);
 342         pack_2_words(&new_long_value, lock_size, one, &two);
 343         while (lock_value
 344                 != (temp = shmem_lock_cswap_poll(lock,
 345                                                  lock_size,
 346                                                  lock_value,
 347                                                  new_long_value,
 348                                                  my_pe))) {
 349             lock_value = temp;
 350             extract_second_word(&lock_value, lock_size, &two);
 351             pack_2_words(&new_long_value, lock_size, one, &two);
 352         }
 353     } else {
 354         uint64_t zero_mask = 0xFFFFFFFF;
 355         if (lock_size == 4) {
 356             zero_mask = 0xFFFF;
 357         }
 358 
 359         int zero = 0;
 360         int written_one = 0;
 361 
 362         pack_2_words(&new_long_value, lock_size, one, &zero);
 363         do {
 364             lock_value = get_lock_value(lock, lock_size);
 365             lock_value &= zero_mask;
 366             lock_value |= new_long_value;
 367             set_lock_value(lock, lock_size, lock_value);
 368             extract_first_word(lock, lock_size, &written_one);
 369         } while (written_one != *one);
 370     }
 371 
 372     return OSHMEM_SUCCESS;
 373 }
 374 
 375 static int pack_second_word(void *lock,
 376                             int lock_size,
 377                             const int *two,
 378                             int use_atomic)
 379 {
 380     int my_pe = shmem_my_pe();
 381     uint64_t lock_value = 0;
 382     uint64_t new_long_value = 0;
 383     uint64_t temp = 0;
 384     int one = 0;
 385 
 386     if (lock == 0 || two == 0) {
 387         return OSHMEM_ERROR;
 388     }
 389 
 390     if (use_atomic) {
 391         lock_value = get_lock_value(lock, lock_size);
 392         extract_first_word(&lock_value, lock_size, &one);
 393         pack_2_words(&new_long_value, lock_size, &one, two);
 394         while (lock_value
 395                 != (temp = shmem_lock_cswap_poll(lock,
 396                                                  lock_size,
 397                                                  lock_value,
 398                                                  new_long_value,
 399                                                  my_pe))) {
 400             lock_value = temp;
 401             extract_first_word(&lock_value, lock_size, &one);
 402             pack_2_words(&new_long_value, lock_size, &one, two);
 403         }
 404     } else {
 405         uint64_t zero_mask = 0xFFFFFFFF00000000;
 406         if (lock_size == 4) {
 407             zero_mask = 0xFFFF0000;
 408         }
 409 
 410         int zero = 0;
 411         int written_two = 0;
 412 
 413         pack_2_words(&new_long_value, lock_size, &zero, two);
 414         do {
 415             lock_value = get_lock_value(lock, lock_size);
 416             lock_value &= zero_mask;
 417             lock_value |= new_long_value;
 418             set_lock_value(lock, lock_size, lock_value);
 419             extract_second_word(lock, lock_size, &written_two);
 420         } while (written_two != *two);
 421     }
 422 
 423     return OSHMEM_SUCCESS;
 424 }
 425 
 426 static int lock_extract_pe_next_counter(void *lock,
 427                                         int lock_size,
 428                                         int *pe_next,
 429                                         int *counter)
 430 {
 431     int status = extract_2_words(lock, lock_size, counter, pe_next);
 432 
 433     /* make sure counter does not have the last bit - infoming bit */
 434     *counter &= ~(((unsigned int) 1) << (SIZEOF_INT * 8 - 1));
 435     if (*pe_next >= 0) {
 436         *pe_next -= 1;
 437     }
 438 
 439     return status;
 440 }
 441 
 442 static int shmem_lock_extract_pe_next(void *lock, int lock_size, int *pe_next)
 443 {
 444     int status = extract_second_word(lock, lock_size, pe_next);
 445     if (*pe_next >= 0) {
 446         *pe_next -= 1;
 447     }
 448 
 449     return status;
 450 }
 451 
 452 static int lock_extract_pe_last(void *lock, int lock_size, int *pe_last)
 453 {
 454     int status = extract_first_word(lock, lock_size, pe_last);
 455     if (*pe_last >= 0) {
 456         *pe_last -= 1;
 457     }
 458 
 459     return status;
 460 }
 461 
 462 static int lock_extract_counter(void *lock, int lock_size, int *count)
 463 {
 464     int status = extract_first_word(lock, lock_size, count);
 465 
 466     /* make sure counter does not have the last bit - infoming bit */
 467     *count &= ~(((unsigned int) 1) << (SIZEOF_INT * 8 - 1));
 468     return status;
 469 }
 470 
 471 static int shmem_lock_pack_pe_next_pe_last(void *lock,
 472                                            int lock_size,
 473                                            const int *pe_next,
 474                                            const int *pe_last)
 475 {
 476     int pe_next_plus_one = *pe_next + 1;
 477     int pe_last_plus_one = *pe_last + 1;
 478 
 479     return pack_2_words(lock, lock_size, &pe_last_plus_one, &pe_next_plus_one);
 480 }
 481 
 482 static int lock_pack_pe_next_counter(void *lock,
 483                                      int lock_size,
 484                                      const int *pe_next,
 485                                      const int *counter)
 486 {
 487     int pe_next_plus_one = *pe_next + 1;
 488 
 489     return pack_2_words(lock, lock_size, counter, &pe_next_plus_one);
 490 }
 491 
 492 static int shmem_lock_pack_pe_next(void *lock,
 493                                    int lock_size,
 494                                    const int *pe_next)
 495 {
 496     int pe_next_plus_one = *pe_next + 1;
 497 
 498     return pack_second_word(lock, lock_size, &pe_next_plus_one, 1);
 499 }
 500 
 501 static int lock_pack_counter(void *lock,
 502                              int lock_size,
 503                              const int *counter,
 504                              int use_atomic)
 505 {
 506     return pack_first_word(lock, lock_size, counter, use_atomic);
 507 }
 508 
 509 static int lock_pack_pe_last(void *lock,
 510                              int lock_size,
 511                              const int *pe_last,
 512                              int use_atomic)
 513 {
 514     int pe_last_plus_one = *pe_last + 1;
 515 
 516     return pack_first_word(lock, lock_size, &pe_last_plus_one, use_atomic);
 517 }
 518 
 519 /***************************************************************************/
 520 /**************************Lock counters************************************/
 521 /***************************************************************************/
 522 
 523 static oshmem_lock_counter_t *lock_find_counter(void *lock)
 524 {
 525     oshmem_lock_counter_t *current_counter = lock_counter_head;
 526 
 527     if (0 == lock_counter_head) {
 528         return 0;
 529     }
 530 
 531     while (0 != current_counter) {
 532         if (current_counter->lock == lock) {
 533             return current_counter;
 534         }
 535 
 536         current_counter = current_counter->next;
 537     }
 538 
 539     return 0;
 540 }
 541 
 542 static int shmem_lock_insert_counter(void *lock)
 543 {
 544     oshmem_lock_counter_t *counter = lock_find_counter(lock);
 545 
 546     if (counter) {
 547         counter->counter += 1;
 548     } else if (lock_counter_head) {
 549         counter = malloc(sizeof(oshmem_lock_counter_t));
 550         counter->lock = lock;
 551         counter->counter = 1;
 552         counter->next = lock_counter_head;
 553         counter->prev = lock_counter_head->prev;
 554         lock_counter_head->prev = counter;
 555         lock_counter_head = counter;
 556     } else {
 557         lock_counter_head = malloc(sizeof(oshmem_lock_counter_t));
 558         lock_counter_head->lock = lock;
 559         lock_counter_head->counter = 1;
 560         lock_counter_head->next = 0;
 561         lock_counter_head->prev = 0;
 562     }
 563 
 564     return OSHMEM_SUCCESS;
 565 }
 566 
 567 static int shmem_lock_remove_counter(void *lock)
 568 {
 569     oshmem_lock_counter_t *counter = lock_find_counter(lock);
 570 
 571     if (counter) {
 572         oshmem_lock_counter_t *prev = counter->prev;
 573         oshmem_lock_counter_t *next = counter->next;
 574 
 575         if (next) {
 576             next->prev = prev;
 577         }
 578 
 579         if (prev) {
 580             prev->next = next;
 581         }
 582 
 583         if (lock_counter_head == counter) {
 584             lock_counter_head = next;
 585         }
 586 
 587         free(counter);
 588     }
 589 
 590     return OSHMEM_SUCCESS;
 591 }
 592 
 593 static void shmem_lock_increment_counter(void *lock, int lock_size)
 594 {
 595     int my_pe = shmem_my_pe();
 596     int server_pe = shmem_lock_get_server(lock);
 597 
 598     if (my_pe == server_pe) {
 599         shmem_lock_insert_counter(lock);
 600     } else {
 601         int counter = 0;
 602         lock_extract_counter(lock, lock_size, &counter);
 603         counter++;
 604         lock_pack_counter(lock, lock_size, &counter, 1);
 605     }
 606 }
 607 
 608 static int shmem_lock_decrement_counter(void *lock, int lock_size)
 609 {
 610     int my_pe = shmem_my_pe();
 611     int server_pe = shmem_lock_get_server(lock);
 612     int current_lock_counter = -1;
 613 
 614     if (my_pe == server_pe) {
 615         oshmem_lock_counter_t *counter = lock_find_counter(lock);
 616 
 617         if (counter) {
 618             if (oshmem_shmem_lock_recursive) {
 619                 counter->counter -= 1;
 620             } else {
 621                 counter->counter = 0;
 622             }
 623 
 624             if ((current_lock_counter = counter->counter) <= 0) {
 625                 shmem_lock_remove_counter(lock);
 626                 current_lock_counter = 0;
 627             }
 628         }
 629     } else {
 630         int pe_next = 0, counter = 0;
 631         lock_extract_pe_next_counter(lock, lock_size, &pe_next, &counter);
 632         if (counter > 0) {
 633             if (oshmem_shmem_lock_recursive) {
 634                 current_lock_counter = counter - 1;
 635             } else {
 636                 current_lock_counter = 0;
 637             }
 638 
 639             lock_pack_counter(lock, lock_size, &current_lock_counter, 1);
 640         }
 641     }
 642 
 643     return current_lock_counter;
 644 }
 645 
 646 static int lock_get_count(void *lock, int lock_size)
 647 {
 648     int my_pe = shmem_my_pe();
 649     int server_pe = shmem_lock_get_server(lock);
 650     int count = 0;
 651 
 652     if (my_pe == server_pe) {
 653         oshmem_lock_counter_t *counter = lock_find_counter(lock);
 654 
 655         if (counter) {
 656             count = counter->counter;
 657         }
 658     } else {
 659         lock_extract_counter(lock, lock_size, &count);
 660     }
 661 
 662     return count;
 663 }
 664 
 665 static int shmem_lock_is_mine(void *lock, int lock_size)
 666 {
 667     return (lock_get_count(lock, lock_size) > 0);
 668 }
 669 
 670 /***************************************************************************/
 671 /**************************Ticket utilities*********************************/
 672 /***************************************************************************/
 673 static int shmem_lock_get_ticket(void *lock)
 674 {
 675     int server_pe = shmem_lock_get_server(lock);
 676     int my_ticket = shmem_int_finc(lock_last_ticket, server_pe);
 677 
 678     return my_ticket;
 679 }
 680 
 681 static void shmem_get_wrapper(uint64_t *target,
 682                               const void *source,
 683                               int source_size,
 684                               size_t nelems,
 685                               int pe)
 686 {
 687     if (source_size == 8) {
 688         shmem_get64(target, source, nelems, pe);
 689     } else if (source_size == 4) {
 690         uint32_t temp32 = 0;
 691         shmem_get32(&temp32, source, nelems, pe);
 692         *target = temp32;
 693     }
 694 }
 695 
 696 static int shmem_lock_wait_for_ticket(void *lock,
 697                                       int lock_size,
 698                                       int ticket,
 699                                       int *pe_last)
 700 {
 701     int my_pe = shmem_my_pe();
 702     int server_pe = shmem_lock_get_server(lock);
 703     int remote_new_pe_last = 0;
 704     int remote_turn = 0;
 705     uint64_t server_lock = 0;
 706     uint64_t new_server_lock = 0;
 707     uint64_t temp = 0;
 708 
 709     do {
 710         shmem_int_get(&remote_turn, lock_turn, 1, server_pe);
 711         opal_progress();
 712     } while (remote_turn != ticket);
 713 
 714     shmem_get_wrapper(&temp, lock, lock_size, 1, server_pe);
 715     do {
 716         /* Another process has ignored the queue, possibly due to shmem_test_lock */
 717         new_server_lock = server_lock = temp;
 718         lock_pack_pe_last(&new_server_lock, lock_size, &my_pe, 0);
 719     } while (server_lock
 720             != (temp = shmem_lock_cswap_poll(lock,
 721                                              lock_size,
 722                                              server_lock,
 723                                              new_server_lock,
 724                                              server_pe)));
 725     lock_extract_pe_last(&server_lock, lock_size, pe_last);
 726     if (*pe_last == -1) {
 727         /* we are first in queue for the lock */
 728         *pe_last = my_pe;
 729     }
 730 
 731     /* Since quiet is too slow in ikrit then check directly
 732      * shmem_quiet();
 733      */
 734     do {
 735         shmem_get_wrapper(&new_server_lock, lock, lock_size, 1, server_pe);
 736         lock_extract_pe_last(&new_server_lock, lock_size, &remote_new_pe_last);
 737     } while (remote_new_pe_last != my_pe);
 738 
 739     shmem_int_finc(lock_turn, server_pe);
 740 
 741     return OSHMEM_SUCCESS;
 742 }
 743 
 744 static int shmem_lock_subscribe_for_informing(void *lock,
 745                                               int lock_size,
 746                                               int pe_last)
 747 {
 748     int my_pe = shmem_my_pe();
 749     int server_pe = shmem_lock_get_server(lock);
 750     int remote_prev_pe_next = 0;
 751     uint64_t prev_remote_value = 1;
 752 
 753     prev_remote_value = shmem_lock_fadd(lock, lock_size, my_pe + 1, pe_last);
 754     if (my_pe == server_pe) {
 755         lock_save_prev_pe(lock, pe_last);
 756     }
 757 
 758     /* Check the previous value of pe_next is -1
 759      * if not 0 report a bug in distributed locking implementation
 760      */
 761     shmem_lock_extract_pe_next(&prev_remote_value,
 762                                lock_size,
 763                                &remote_prev_pe_next);
 764     if (remote_prev_pe_next != -1) {
 765         int remote_counter = 0;
 766         uint64_t new_remote_value = 0;
 767         uint64_t temp_value = 0;
 768         SHMEM_API_ERROR("PE #%i noticed incorrect pe_next value=%i on PE#%i",
 769                         my_pe, remote_prev_pe_next, pe_last);
 770 
 771         /* Trying to restore */
 772         lock_extract_counter(&prev_remote_value, lock_size, &remote_counter);
 773         lock_pack_pe_next_counter(&new_remote_value,
 774                                   lock_size,
 775                                   &my_pe,
 776                                   &remote_counter);
 777         prev_remote_value += my_pe + 1;
 778 
 779         while (prev_remote_value
 780                 != (temp_value = shmem_lock_cswap_poll(lock,
 781                                                        lock_size,
 782                                                        prev_remote_value,
 783                                                        new_remote_value,
 784                                                        pe_last))) {
 785             prev_remote_value = temp_value;
 786             lock_extract_counter(&prev_remote_value,
 787                                  lock_size,
 788                                  &remote_counter);
 789             lock_pack_pe_next_counter(&new_remote_value,
 790                                       lock_size,
 791                                       &my_pe,
 792                                       &remote_counter);
 793         }
 794     }
 795 
 796     return OSHMEM_SUCCESS;
 797 }
 798 
 799 static void shmem_wait_wrapper(void *target, int target_size, uint64_t value)
 800 {
 801     if (target_size == sizeof(int)) {
 802         shmem_int_wait((int *) target, (int) value);
 803     } else if (target_size == sizeof(long)) {
 804         shmem_long_wait((long *) target, (long) value);
 805     } else if (target_size == sizeof(long long)) {
 806         shmem_longlong_wait((long long *) target, (long long) value);
 807     }
 808 }
 809 
 810 /***************************************************************************/
 811 /********************Release lock informing functions***********************/
 812 /***************************************************************************/
 813 
 814 static int shmem_lock_wait_for_informing(void *lock, int lock_size)
 815 {
 816     int lock_bitwise_size = lock_size * 8;
 817     int my_pe = shmem_my_pe();
 818     int server_pe = shmem_lock_get_server(lock);
 819 
 820     if (my_pe != server_pe) {
 821         int original_counter = 1;
 822         uint64_t prev_value = get_lock_value(lock, lock_size);
 823         int informed = (prev_value >> (lock_bitwise_size - 1));
 824 
 825         lock_extract_counter(&prev_value, lock_size, &original_counter);
 826         while (!informed) {
 827             shmem_wait_wrapper(lock, lock_size, prev_value);
 828             prev_value = get_lock_value(lock, lock_size);
 829             informed = (prev_value >> (lock_bitwise_size - 1));
 830         }
 831 
 832         lock_pack_counter(lock, lock_size, &original_counter, 1);
 833     } else {
 834         int prev_value = *lock_inform;
 835         int prev_pe = -1;
 836         int remote_pe_next = 0;
 837         int remote_counter = 1;
 838 
 839         if (OSHMEM_SUCCESS != lock_restore_prev_pe(lock, &prev_pe)) {
 840             SHMEM_API_ERROR("Unable to restore prev_pe on server PE#%i", my_pe);
 841             oshmem_shmem_abort(-1);
 842 
 843         }
 844 
 845         if (prev_pe == server_pe) {
 846             SHMEM_API_ERROR("prev_pe (%i) is me", prev_pe);
 847         }
 848 
 849         do {
 850             uint64_t remote_lock = 0;
 851             shmem_get_wrapper(&remote_lock, lock, lock_size, 1, prev_pe);
 852             lock_extract_pe_next_counter(&remote_lock,
 853                                          lock_size,
 854                                          &remote_pe_next,
 855                                          &remote_counter);
 856             if ((remote_counter > 0) && (remote_pe_next == my_pe)) {
 857                 shmem_int_wait(lock_inform, prev_value);
 858                 prev_value = *lock_inform;
 859             }
 860         } while ((remote_counter > 0) && (remote_pe_next == my_pe));
 861     }
 862 
 863     return OSHMEM_SUCCESS;
 864 }
 865 
 866 static int shmem_lock_inform_next(void *lock, int lock_size, int pe_next)
 867 {
 868     int lock_bitwise_size = lock_size * 8;
 869     int server_pe = shmem_lock_get_server(lock);
 870 
 871     if (server_pe != pe_next) {
 872         uint64_t temp_value = 0, remote_value = 0;
 873         shmem_get_wrapper(&remote_value, lock, lock_size, 1, pe_next);
 874         uint64_t new_remote_value = remote_value
 875                 | (((uint64_t) 1) << (lock_bitwise_size - 1));
 876 
 877         while (remote_value
 878                 != (temp_value = shmem_lock_cswap_poll(lock,
 879                                                        lock_size,
 880                                                        remote_value,
 881                                                        new_remote_value,
 882                                                        pe_next))) {
 883             remote_value = temp_value;
 884             new_remote_value = remote_value
 885                     | (((uint64_t) 1) << (lock_bitwise_size - 1));
 886         }
 887     } else {
 888         shmem_int_inc(lock_inform, pe_next);
 889     }
 890 
 891     return OSHMEM_SUCCESS;
 892 }
 893 
 894 static int lock_save_prev_pe(void *lock, int prev_pe)
 895 {
 896     oshmem_lock_prev_pe_container_t *container = lock_prev_pe_container_head;
 897 
 898     while (container != 0) {
 899         if (container->lock == lock) {
 900             break;
 901         }
 902         container = container->next;
 903     }
 904 
 905     if (container) {
 906         container->prev_pe = prev_pe;
 907     } else {
 908         container = malloc(sizeof(oshmem_lock_prev_pe_container_t));
 909         container->lock = lock;
 910         container->prev_pe = prev_pe;
 911         container->next = lock_prev_pe_container_head;
 912         container->prev = 0;
 913         if (lock_prev_pe_container_head) {
 914             lock_prev_pe_container_head->prev = container;
 915         }
 916         lock_prev_pe_container_head = container;
 917     }
 918 
 919     return OSHMEM_SUCCESS;
 920 }
 921 
 922 static int lock_restore_prev_pe(void *lock, int *prev_pe)
 923 {
 924     oshmem_lock_prev_pe_container_t *container = lock_prev_pe_container_head;
 925     while (container != 0) {
 926         if (container->lock == lock) {
 927             break;
 928         }
 929         container = container->next;
 930     }
 931 
 932     if (container) {
 933         oshmem_lock_prev_pe_container_t *next = container->next;
 934         oshmem_lock_prev_pe_container_t *prev = container->prev;
 935         *prev_pe = container->prev_pe;
 936 
 937         if (prev) {
 938             prev->next = next;
 939         }
 940         if (next) {
 941             next->prev = prev;
 942         }
 943         if (lock_prev_pe_container_head == container) {
 944             lock_prev_pe_container_head = next;
 945         }
 946         free(container);
 947         return OSHMEM_SUCCESS;
 948     } else {
 949         *prev_pe = -1;
 950         return OSHMEM_ERROR;
 951     }
 952 }
 953 
 954 static int shmem_lock_try_inform_server(void *lock, int lock_size)
 955 {
 956     int my_pe = shmem_my_pe();
 957     int server_pe = shmem_lock_get_server(lock);
 958     int zero = 0;
 959     int incorrect_pe = -1;
 960     uint64_t remote_value = 0;
 961 
 962     shmem_lock_pack_pe_next_pe_last(&remote_value,
 963                                     lock_size,
 964                                     &incorrect_pe,
 965                                     &my_pe);
 966     return !(remote_value
 967             == shmem_lock_cswap_poll(lock, lock_size, remote_value, zero, server_pe));
 968 }
 969 
 970 /***************************************************************************/
 971 /**************************API wrappers*************************************/
 972 /***************************************************************************/
 973 
 974 void _shmem_set_lock(void *lock, int lock_size)
 975 {
 976     int my_pe = shmem_my_pe();
 977 
 978     if (!shmem_lock_is_mine(lock, lock_size)) {
 979         int has_lock = !_shmem_test_lock(lock, lock_size);
 980 
 981         if (!has_lock) {
 982             int pe_last = -1;
 983             int ticket = shmem_lock_get_ticket(lock);
 984 
 985             shmem_lock_increment_counter(lock, lock_size);
 986             shmem_lock_wait_for_ticket(lock, lock_size, ticket, &pe_last);
 987             if (pe_last != my_pe) {
 988                 shmem_lock_subscribe_for_informing(lock, lock_size, pe_last);
 989                 shmem_lock_wait_for_informing(lock, lock_size);
 990             }
 991         }
 992     } else {
 993         shmem_lock_increment_counter(lock, lock_size);
 994     }
 995 }
 996 
 997 int _shmem_test_lock(void *lock, int lock_size)
 998 {
 999     int status = 1;
1000     int server_pe = shmem_lock_get_server(lock);
1001     int incorrect_value = -1;
1002     int my_pe = shmem_my_pe();
1003     uint64_t new_lock_value = 0;
1004     uint64_t prev_lock_value = 1;
1005     int my_lock = shmem_lock_is_mine(lock, lock_size);
1006 
1007     shmem_lock_increment_counter(lock, lock_size);
1008     if (!my_lock) {
1009         if (shmem_lock_pack_pe_next_pe_last(&new_lock_value,
1010                                             lock_size,
1011                                             &incorrect_value,
1012                                             &my_pe)) {
1013             goto FreeMemory;
1014         }
1015 
1016         prev_lock_value = shmem_lock_cswap(lock,
1017                                            lock_size,
1018                                            0,
1019                                            new_lock_value,
1020                                            server_pe);
1021     }
1022 
1023     if (0 == prev_lock_value || my_lock) {
1024         status = 0;
1025     } else {
1026         shmem_lock_decrement_counter(lock, lock_size);
1027     }
1028 
1029     FreeMemory: return status;
1030 }
1031 
1032 void _shmem_clear_lock(void *lock, int lock_size)
1033 {
1034     int current_lock_counter = shmem_lock_decrement_counter(lock, lock_size);
1035 
1036     if (0 == current_lock_counter) {
1037         int next_informed = 0;
1038         int pe_next = 0;
1039 
1040         while (!next_informed) {
1041             shmem_lock_extract_pe_next(lock, lock_size, &pe_next);
1042             if (pe_next >= 0) {
1043                 shmem_lock_inform_next(lock, lock_size, pe_next);
1044                 next_informed = 1;
1045             } else {
1046                 /* It seems I'm the last in queue */
1047                 if (!shmem_lock_try_inform_server(lock, lock_size)) {
1048                     next_informed = 1;
1049                 }
1050             }
1051         }
1052 
1053         pe_next = -1;
1054         shmem_lock_pack_pe_next(lock, lock_size, &pe_next);
1055     }
1056 }

/* [<][>][^][v][top][bottom][index][help] */