This source file includes following definitions.
- shmem_lock_init
- shmem_lock_finalize
- shmem_lock_get_server
- get_lock_value
- set_lock_value
- extract_2_words
- pack_2_words
- extract_first_word
- extract_second_word
- shmem_lock_cswap
- shmem_lock_cswap_poll
- shmem_lock_fadd
- pack_first_word
- pack_second_word
- lock_extract_pe_next_counter
- shmem_lock_extract_pe_next
- lock_extract_pe_last
- lock_extract_counter
- shmem_lock_pack_pe_next_pe_last
- lock_pack_pe_next_counter
- shmem_lock_pack_pe_next
- lock_pack_counter
- lock_pack_pe_last
- lock_find_counter
- shmem_lock_insert_counter
- shmem_lock_remove_counter
- shmem_lock_increment_counter
- shmem_lock_decrement_counter
- lock_get_count
- shmem_lock_is_mine
- shmem_lock_get_ticket
- shmem_get_wrapper
- shmem_lock_wait_for_ticket
- shmem_lock_subscribe_for_informing
- shmem_wait_wrapper
- shmem_lock_wait_for_informing
- shmem_lock_inform_next
- lock_save_prev_pe
- lock_restore_prev_pe
- shmem_lock_try_inform_server
- _shmem_set_lock
- _shmem_test_lock
- _shmem_clear_lock
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 #include "oshmem_config.h"
17
18 #include "oshmem/constants.h"
19 #include "oshmem/include/shmem.h"
20 #include "oshmem/runtime/params.h"
21 #include "oshmem/runtime/runtime.h"
22 #include <stdlib.h>
23 #include <memory.h>
24
25 #include "oshmem/shmem/shmem_api_logger.h"
26 #include "oshmem/shmem/shmem_lock.h"
27 #include "oshmem/mca/memheap/memheap.h"
28 #include "oshmem/mca/memheap/base/base.h"
29 #include "oshmem/mca/atomic/atomic.h"
30
31 #define OPAL_BITWISE_SIZEOF_LONG (SIZEOF_LONG * 8)
32
33 struct oshmem_lock_counter {
34 void *lock;
35 int counter;
36 struct oshmem_lock_counter *next;
37 struct oshmem_lock_counter *prev;
38 };
39 typedef struct oshmem_lock_counter oshmem_lock_counter_t;
40
41 struct oshmem_lock_prev_pe_container {
42 void *lock;
43 int prev_pe;
44 struct oshmem_lock_prev_pe_container *next;
45 struct oshmem_lock_prev_pe_container *prev;
46 };
47 typedef struct oshmem_lock_prev_pe_container oshmem_lock_prev_pe_container_t;
48
49 oshmem_lock_counter_t *lock_counter_head = NULL;
50 oshmem_lock_prev_pe_container_t *lock_prev_pe_container_head = NULL;
51 static int *lock_turn;
52 static int *lock_inform;
53 static int *lock_last_ticket;
54
55 static int lock_save_prev_pe(void *lock, int prev_pe);
56 static int lock_restore_prev_pe(void *lock, int *prev_pe);
57
58 static int shmem_lock_try_inform_server(void *lock, int lock_size);
59 static void shmem_get_wrapper(uint64_t *target,
60 const void *source,
61 int source_size,
62 size_t nelems,
63 int pe);
64 static void shmem_wait_wrapper(void *target, int target_size, uint64_t value);
65
66 static int shmem_lock_extract_pe_next(void *lock, int lock_size, int *pe_next);
67 static int shmem_lock_pack_pe_next_pe_last(void *lock,
68 int lock_size,
69 const int *pe_next,
70 const int *pe_last);
71 static int shmem_lock_pack_pe_next(void *lock,
72 int lock_size,
73 const int *pe_next);
74
75 static void shmem_lock_increment_counter(void *lock, int lock_size);
76 static int shmem_lock_decrement_counter(void *lock, int lock_size);
77
78 static int shmem_lock_get_server(void *lock);
79 static int shmem_lock_is_mine(void *lock, int lock_size);
80
81 static int shmem_lock_get_ticket(void *lock);
82 static int shmem_lock_wait_for_ticket(void *lock,
83 int lock_size,
84 int ticket,
85 int *pe_last);
86 static int shmem_lock_subscribe_for_informing(void *lock,
87 int lock_size,
88 int pe_last);
89 static int shmem_lock_wait_for_informing(void *lock, int lock_size);
90 static int shmem_lock_inform_next(void *lock, int lock_size, int pe_next);
91
92
93
94
95
96 int shmem_lock_init()
97 {
98 void* ptr = 0;
99
100 #if (OPAL_BITWISE_SIZEOF_LONG == 32)
101 int number_of_pes = shmem_n_pes();
102 if (number_of_pes >= 65534)
103 {
104 SHMEM_API_ERROR("SHMEM distributed locking implementation does not support total number of PEs greater than 65534 if sizeof(long) = 4");
105 return OSHMEM_ERROR;
106 }
107 #endif
108
109 #if ((OPAL_BITWISE_SIZEOF_LONG != 64) && (OPAL_BITWISE_SIZEOF_LONG != 32))
110 SHMEM_API_ERROR("SHMEM distributed locking implementation does not support sizeof(long) = %i",
111 sizeof(long));
112 return OSHMEM_ERROR;
113 #endif
114
115 ptr = (void *) lock_turn;
116 MCA_MEMHEAP_CALL(private_alloc(sizeof(int), &ptr));
117 lock_turn = (int *) ptr;
118 *lock_turn = 0;
119 ptr = (void *) lock_last_ticket;
120 MCA_MEMHEAP_CALL(private_alloc(sizeof(int), &ptr));
121 lock_last_ticket = (int *) ptr;
122 *lock_last_ticket = 0;
123 ptr = (void *) lock_inform;
124 MCA_MEMHEAP_CALL(private_alloc(sizeof(int), &ptr));
125 lock_inform = (int *) ptr;
126 *lock_inform = 0;
127
128 lock_counter_head = 0;
129 lock_prev_pe_container_head = 0;
130
131 return OSHMEM_SUCCESS;
132 }
133
134 int shmem_lock_finalize()
135 {
136 oshmem_lock_counter_t *current_counter = lock_counter_head;
137 oshmem_lock_prev_pe_container_t *current_pe_container =
138 lock_prev_pe_container_head;
139
140 if (0 != lock_turn) {
141 MCA_MEMHEAP_CALL(private_free(lock_turn));
142 }
143 if (0 != lock_last_ticket) {
144 MCA_MEMHEAP_CALL(private_free(lock_last_ticket));
145 }
146 if (0 != lock_inform) {
147 MCA_MEMHEAP_CALL(private_free(lock_inform));
148 }
149
150 lock_turn = 0;
151 lock_last_ticket = 0;
152 lock_inform = 0;
153
154 while (0 != current_counter) {
155 oshmem_lock_counter_t *counter_to_free = current_counter;
156 current_counter = current_counter->next;
157 free(counter_to_free);
158 }
159 lock_counter_head = 0;
160
161 while (0 != current_pe_container) {
162 oshmem_lock_prev_pe_container_t *container_to_free =
163 current_pe_container;
164 current_pe_container = current_pe_container->next;
165 free(container_to_free);
166 }
167 lock_prev_pe_container_head = 0;
168
169 return OSHMEM_SUCCESS;
170 }
171
172 static int shmem_lock_get_server(void *lock)
173 {
174 map_segment_t *s;
175
176 s = memheap_find_va(lock);
177 if (NULL == s) {
178 SHMEM_API_ERROR("PE#%i lock %p is not a shared variable", shmem_my_pe(), lock);
179 oshmem_shmem_abort(-1);
180 return 0;
181 }
182
183 return ((int)((uintptr_t)lock - (uintptr_t)s->super.va_base)/8) % shmem_n_pes();
184 }
185
186 static uint64_t get_lock_value(const void *lock, int lock_size)
187 {
188 uint64_t lock_value = 0;
189
190 if (lock_size == 4) {
191 lock_value = *(uint32_t *) lock;
192 } else if (lock_size == 8) {
193 lock_value = *(uint64_t *) lock;
194 }
195
196 return lock_value;
197 }
198
199 static void set_lock_value(void *lock, int lock_size, uint64_t lock_value)
200 {
201 if (lock_size == 8) {
202 memcpy(lock, &lock_value, 8);
203 } else if (lock_size == 4) {
204 uint32_t lock_value_32 = (uint32_t) lock_value;
205 memcpy(lock, &lock_value_32, 4);
206 }
207 }
208
209
210
211
212
213 static int extract_2_words(const void *lock, int lock_size, int *one, int *two)
214 {
215 int lock_bitwise_size = lock_size * 8;
216 uint64_t lock_value = get_lock_value(lock, lock_size);
217
218 if (lock == 0 || one == 0 || two == 0) {
219 return OSHMEM_ERROR;
220 }
221
222 *one = (int) (lock_value >> (lock_bitwise_size / 2));
223 if (lock_size == 8) {
224 *two = (int) ((lock_value << 32) >> 32);
225 } else if (lock_size == 4) {
226 *two = (int) ((lock_value << 48) >> 48);
227 }
228
229 return OSHMEM_SUCCESS;
230 }
231
232 static int pack_2_words(void *lock,
233 int lock_size,
234 const int *one,
235 const int *two)
236 {
237 uint64_t lock_value = 0;
238 int lock_bitwise_size = lock_size * 8;
239
240 if (lock == 0 || one == 0 || two == 0) {
241 return OSHMEM_ERROR;
242 }
243
244 lock_value = (uint64_t) *two
245 | (((uint64_t) *one) << (lock_bitwise_size / 2));
246 set_lock_value(lock, lock_size, lock_value);
247
248 return OSHMEM_SUCCESS;
249 }
250
251 static int extract_first_word(void *lock, int lock_size, int *one)
252 {
253 int two = 0;
254 return extract_2_words(lock, lock_size, one, &two);
255 }
256
257 static int extract_second_word(void *lock, int lock_size, int *two)
258 {
259 int one = 0;
260 return extract_2_words(lock, lock_size, &one, two);
261 }
262
263 static uint64_t shmem_lock_cswap(void *target,
264 int target_size,
265 uint64_t cond,
266 uint64_t value,
267 int pe)
268 {
269 uint64_t prev_value = 0;
270
271 if (target_size == 8) {
272 MCA_ATOMIC_CALL(cswap(oshmem_ctx_default, target, (void*)&prev_value, cond, value, target_size, pe));
273 } else if (target_size == 4) {
274 uint32_t prev_value_32 = 0;
275 uint32_t cond32 = (uint32_t) cond;
276 uint32_t value32 = (uint32_t) value;
277
278 MCA_ATOMIC_CALL(cswap(oshmem_ctx_default, target, (void*)&prev_value_32, cond32, value32, target_size, pe));
279
280 prev_value = prev_value_32;
281 }
282 return prev_value;
283 }
284
285
286
287
288
289 static uint64_t shmem_lock_cswap_poll(void *target,
290 int target_size,
291 uint64_t cond,
292 uint64_t value,
293 int pe)
294 {
295 uint64_t prev_value;
296
297 prev_value = shmem_lock_cswap(target, target_size, cond, value, pe);
298 opal_progress();
299 return prev_value;
300 }
301
302 static uint64_t shmem_lock_fadd(void *target,
303 int target_size,
304 uint64_t value,
305 int pe)
306 {
307 uint64_t prev_value = 0;
308
309 if (target_size == sizeof(int)) {
310 prev_value = (uint64_t) shmem_int_fadd((int *) target, (int) value, pe);
311 } else if (target_size == sizeof(long)) {
312 prev_value = (uint64_t) shmem_long_fadd((long *) target,
313 (long) value,
314 pe);
315 } else if (target_size == sizeof(long long)) {
316 prev_value = (uint64_t) shmem_longlong_fadd((long long *) target,
317 (long long) value,
318 pe);
319 }
320
321 return prev_value;
322 }
323
324 static int pack_first_word(void *lock,
325 int lock_size,
326 const int *one,
327 int use_atomic)
328 {
329 int my_pe = shmem_my_pe();
330 uint64_t lock_value = 0;
331 uint64_t new_long_value = 0;
332 uint64_t temp = 0;
333 int two = 0;
334
335 if (lock == 0 || one == 0) {
336 return OSHMEM_ERROR;
337 }
338
339 if (use_atomic) {
340 lock_value = get_lock_value(lock, lock_size);
341 extract_second_word(&lock_value, lock_size, &two);
342 pack_2_words(&new_long_value, lock_size, one, &two);
343 while (lock_value
344 != (temp = shmem_lock_cswap_poll(lock,
345 lock_size,
346 lock_value,
347 new_long_value,
348 my_pe))) {
349 lock_value = temp;
350 extract_second_word(&lock_value, lock_size, &two);
351 pack_2_words(&new_long_value, lock_size, one, &two);
352 }
353 } else {
354 uint64_t zero_mask = 0xFFFFFFFF;
355 if (lock_size == 4) {
356 zero_mask = 0xFFFF;
357 }
358
359 int zero = 0;
360 int written_one = 0;
361
362 pack_2_words(&new_long_value, lock_size, one, &zero);
363 do {
364 lock_value = get_lock_value(lock, lock_size);
365 lock_value &= zero_mask;
366 lock_value |= new_long_value;
367 set_lock_value(lock, lock_size, lock_value);
368 extract_first_word(lock, lock_size, &written_one);
369 } while (written_one != *one);
370 }
371
372 return OSHMEM_SUCCESS;
373 }
374
375 static int pack_second_word(void *lock,
376 int lock_size,
377 const int *two,
378 int use_atomic)
379 {
380 int my_pe = shmem_my_pe();
381 uint64_t lock_value = 0;
382 uint64_t new_long_value = 0;
383 uint64_t temp = 0;
384 int one = 0;
385
386 if (lock == 0 || two == 0) {
387 return OSHMEM_ERROR;
388 }
389
390 if (use_atomic) {
391 lock_value = get_lock_value(lock, lock_size);
392 extract_first_word(&lock_value, lock_size, &one);
393 pack_2_words(&new_long_value, lock_size, &one, two);
394 while (lock_value
395 != (temp = shmem_lock_cswap_poll(lock,
396 lock_size,
397 lock_value,
398 new_long_value,
399 my_pe))) {
400 lock_value = temp;
401 extract_first_word(&lock_value, lock_size, &one);
402 pack_2_words(&new_long_value, lock_size, &one, two);
403 }
404 } else {
405 uint64_t zero_mask = 0xFFFFFFFF00000000;
406 if (lock_size == 4) {
407 zero_mask = 0xFFFF0000;
408 }
409
410 int zero = 0;
411 int written_two = 0;
412
413 pack_2_words(&new_long_value, lock_size, &zero, two);
414 do {
415 lock_value = get_lock_value(lock, lock_size);
416 lock_value &= zero_mask;
417 lock_value |= new_long_value;
418 set_lock_value(lock, lock_size, lock_value);
419 extract_second_word(lock, lock_size, &written_two);
420 } while (written_two != *two);
421 }
422
423 return OSHMEM_SUCCESS;
424 }
425
426 static int lock_extract_pe_next_counter(void *lock,
427 int lock_size,
428 int *pe_next,
429 int *counter)
430 {
431 int status = extract_2_words(lock, lock_size, counter, pe_next);
432
433
434 *counter &= ~(((unsigned int) 1) << (SIZEOF_INT * 8 - 1));
435 if (*pe_next >= 0) {
436 *pe_next -= 1;
437 }
438
439 return status;
440 }
441
442 static int shmem_lock_extract_pe_next(void *lock, int lock_size, int *pe_next)
443 {
444 int status = extract_second_word(lock, lock_size, pe_next);
445 if (*pe_next >= 0) {
446 *pe_next -= 1;
447 }
448
449 return status;
450 }
451
452 static int lock_extract_pe_last(void *lock, int lock_size, int *pe_last)
453 {
454 int status = extract_first_word(lock, lock_size, pe_last);
455 if (*pe_last >= 0) {
456 *pe_last -= 1;
457 }
458
459 return status;
460 }
461
462 static int lock_extract_counter(void *lock, int lock_size, int *count)
463 {
464 int status = extract_first_word(lock, lock_size, count);
465
466
467 *count &= ~(((unsigned int) 1) << (SIZEOF_INT * 8 - 1));
468 return status;
469 }
470
471 static int shmem_lock_pack_pe_next_pe_last(void *lock,
472 int lock_size,
473 const int *pe_next,
474 const int *pe_last)
475 {
476 int pe_next_plus_one = *pe_next + 1;
477 int pe_last_plus_one = *pe_last + 1;
478
479 return pack_2_words(lock, lock_size, &pe_last_plus_one, &pe_next_plus_one);
480 }
481
482 static int lock_pack_pe_next_counter(void *lock,
483 int lock_size,
484 const int *pe_next,
485 const int *counter)
486 {
487 int pe_next_plus_one = *pe_next + 1;
488
489 return pack_2_words(lock, lock_size, counter, &pe_next_plus_one);
490 }
491
492 static int shmem_lock_pack_pe_next(void *lock,
493 int lock_size,
494 const int *pe_next)
495 {
496 int pe_next_plus_one = *pe_next + 1;
497
498 return pack_second_word(lock, lock_size, &pe_next_plus_one, 1);
499 }
500
501 static int lock_pack_counter(void *lock,
502 int lock_size,
503 const int *counter,
504 int use_atomic)
505 {
506 return pack_first_word(lock, lock_size, counter, use_atomic);
507 }
508
509 static int lock_pack_pe_last(void *lock,
510 int lock_size,
511 const int *pe_last,
512 int use_atomic)
513 {
514 int pe_last_plus_one = *pe_last + 1;
515
516 return pack_first_word(lock, lock_size, &pe_last_plus_one, use_atomic);
517 }
518
519
520
521
522
523 static oshmem_lock_counter_t *lock_find_counter(void *lock)
524 {
525 oshmem_lock_counter_t *current_counter = lock_counter_head;
526
527 if (0 == lock_counter_head) {
528 return 0;
529 }
530
531 while (0 != current_counter) {
532 if (current_counter->lock == lock) {
533 return current_counter;
534 }
535
536 current_counter = current_counter->next;
537 }
538
539 return 0;
540 }
541
542 static int shmem_lock_insert_counter(void *lock)
543 {
544 oshmem_lock_counter_t *counter = lock_find_counter(lock);
545
546 if (counter) {
547 counter->counter += 1;
548 } else if (lock_counter_head) {
549 counter = malloc(sizeof(oshmem_lock_counter_t));
550 counter->lock = lock;
551 counter->counter = 1;
552 counter->next = lock_counter_head;
553 counter->prev = lock_counter_head->prev;
554 lock_counter_head->prev = counter;
555 lock_counter_head = counter;
556 } else {
557 lock_counter_head = malloc(sizeof(oshmem_lock_counter_t));
558 lock_counter_head->lock = lock;
559 lock_counter_head->counter = 1;
560 lock_counter_head->next = 0;
561 lock_counter_head->prev = 0;
562 }
563
564 return OSHMEM_SUCCESS;
565 }
566
567 static int shmem_lock_remove_counter(void *lock)
568 {
569 oshmem_lock_counter_t *counter = lock_find_counter(lock);
570
571 if (counter) {
572 oshmem_lock_counter_t *prev = counter->prev;
573 oshmem_lock_counter_t *next = counter->next;
574
575 if (next) {
576 next->prev = prev;
577 }
578
579 if (prev) {
580 prev->next = next;
581 }
582
583 if (lock_counter_head == counter) {
584 lock_counter_head = next;
585 }
586
587 free(counter);
588 }
589
590 return OSHMEM_SUCCESS;
591 }
592
593 static void shmem_lock_increment_counter(void *lock, int lock_size)
594 {
595 int my_pe = shmem_my_pe();
596 int server_pe = shmem_lock_get_server(lock);
597
598 if (my_pe == server_pe) {
599 shmem_lock_insert_counter(lock);
600 } else {
601 int counter = 0;
602 lock_extract_counter(lock, lock_size, &counter);
603 counter++;
604 lock_pack_counter(lock, lock_size, &counter, 1);
605 }
606 }
607
608 static int shmem_lock_decrement_counter(void *lock, int lock_size)
609 {
610 int my_pe = shmem_my_pe();
611 int server_pe = shmem_lock_get_server(lock);
612 int current_lock_counter = -1;
613
614 if (my_pe == server_pe) {
615 oshmem_lock_counter_t *counter = lock_find_counter(lock);
616
617 if (counter) {
618 if (oshmem_shmem_lock_recursive) {
619 counter->counter -= 1;
620 } else {
621 counter->counter = 0;
622 }
623
624 if ((current_lock_counter = counter->counter) <= 0) {
625 shmem_lock_remove_counter(lock);
626 current_lock_counter = 0;
627 }
628 }
629 } else {
630 int pe_next = 0, counter = 0;
631 lock_extract_pe_next_counter(lock, lock_size, &pe_next, &counter);
632 if (counter > 0) {
633 if (oshmem_shmem_lock_recursive) {
634 current_lock_counter = counter - 1;
635 } else {
636 current_lock_counter = 0;
637 }
638
639 lock_pack_counter(lock, lock_size, ¤t_lock_counter, 1);
640 }
641 }
642
643 return current_lock_counter;
644 }
645
646 static int lock_get_count(void *lock, int lock_size)
647 {
648 int my_pe = shmem_my_pe();
649 int server_pe = shmem_lock_get_server(lock);
650 int count = 0;
651
652 if (my_pe == server_pe) {
653 oshmem_lock_counter_t *counter = lock_find_counter(lock);
654
655 if (counter) {
656 count = counter->counter;
657 }
658 } else {
659 lock_extract_counter(lock, lock_size, &count);
660 }
661
662 return count;
663 }
664
665 static int shmem_lock_is_mine(void *lock, int lock_size)
666 {
667 return (lock_get_count(lock, lock_size) > 0);
668 }
669
670
671
672
673 static int shmem_lock_get_ticket(void *lock)
674 {
675 int server_pe = shmem_lock_get_server(lock);
676 int my_ticket = shmem_int_finc(lock_last_ticket, server_pe);
677
678 return my_ticket;
679 }
680
681 static void shmem_get_wrapper(uint64_t *target,
682 const void *source,
683 int source_size,
684 size_t nelems,
685 int pe)
686 {
687 if (source_size == 8) {
688 shmem_get64(target, source, nelems, pe);
689 } else if (source_size == 4) {
690 uint32_t temp32 = 0;
691 shmem_get32(&temp32, source, nelems, pe);
692 *target = temp32;
693 }
694 }
695
696 static int shmem_lock_wait_for_ticket(void *lock,
697 int lock_size,
698 int ticket,
699 int *pe_last)
700 {
701 int my_pe = shmem_my_pe();
702 int server_pe = shmem_lock_get_server(lock);
703 int remote_new_pe_last = 0;
704 int remote_turn = 0;
705 uint64_t server_lock = 0;
706 uint64_t new_server_lock = 0;
707 uint64_t temp = 0;
708
709 do {
710 shmem_int_get(&remote_turn, lock_turn, 1, server_pe);
711 opal_progress();
712 } while (remote_turn != ticket);
713
714 shmem_get_wrapper(&temp, lock, lock_size, 1, server_pe);
715 do {
716
717 new_server_lock = server_lock = temp;
718 lock_pack_pe_last(&new_server_lock, lock_size, &my_pe, 0);
719 } while (server_lock
720 != (temp = shmem_lock_cswap_poll(lock,
721 lock_size,
722 server_lock,
723 new_server_lock,
724 server_pe)));
725 lock_extract_pe_last(&server_lock, lock_size, pe_last);
726 if (*pe_last == -1) {
727
728 *pe_last = my_pe;
729 }
730
731
732
733
734 do {
735 shmem_get_wrapper(&new_server_lock, lock, lock_size, 1, server_pe);
736 lock_extract_pe_last(&new_server_lock, lock_size, &remote_new_pe_last);
737 } while (remote_new_pe_last != my_pe);
738
739 shmem_int_finc(lock_turn, server_pe);
740
741 return OSHMEM_SUCCESS;
742 }
743
744 static int shmem_lock_subscribe_for_informing(void *lock,
745 int lock_size,
746 int pe_last)
747 {
748 int my_pe = shmem_my_pe();
749 int server_pe = shmem_lock_get_server(lock);
750 int remote_prev_pe_next = 0;
751 uint64_t prev_remote_value = 1;
752
753 prev_remote_value = shmem_lock_fadd(lock, lock_size, my_pe + 1, pe_last);
754 if (my_pe == server_pe) {
755 lock_save_prev_pe(lock, pe_last);
756 }
757
758
759
760
761 shmem_lock_extract_pe_next(&prev_remote_value,
762 lock_size,
763 &remote_prev_pe_next);
764 if (remote_prev_pe_next != -1) {
765 int remote_counter = 0;
766 uint64_t new_remote_value = 0;
767 uint64_t temp_value = 0;
768 SHMEM_API_ERROR("PE #%i noticed incorrect pe_next value=%i on PE#%i",
769 my_pe, remote_prev_pe_next, pe_last);
770
771
772 lock_extract_counter(&prev_remote_value, lock_size, &remote_counter);
773 lock_pack_pe_next_counter(&new_remote_value,
774 lock_size,
775 &my_pe,
776 &remote_counter);
777 prev_remote_value += my_pe + 1;
778
779 while (prev_remote_value
780 != (temp_value = shmem_lock_cswap_poll(lock,
781 lock_size,
782 prev_remote_value,
783 new_remote_value,
784 pe_last))) {
785 prev_remote_value = temp_value;
786 lock_extract_counter(&prev_remote_value,
787 lock_size,
788 &remote_counter);
789 lock_pack_pe_next_counter(&new_remote_value,
790 lock_size,
791 &my_pe,
792 &remote_counter);
793 }
794 }
795
796 return OSHMEM_SUCCESS;
797 }
798
799 static void shmem_wait_wrapper(void *target, int target_size, uint64_t value)
800 {
801 if (target_size == sizeof(int)) {
802 shmem_int_wait((int *) target, (int) value);
803 } else if (target_size == sizeof(long)) {
804 shmem_long_wait((long *) target, (long) value);
805 } else if (target_size == sizeof(long long)) {
806 shmem_longlong_wait((long long *) target, (long long) value);
807 }
808 }
809
810
811
812
813
814 static int shmem_lock_wait_for_informing(void *lock, int lock_size)
815 {
816 int lock_bitwise_size = lock_size * 8;
817 int my_pe = shmem_my_pe();
818 int server_pe = shmem_lock_get_server(lock);
819
820 if (my_pe != server_pe) {
821 int original_counter = 1;
822 uint64_t prev_value = get_lock_value(lock, lock_size);
823 int informed = (prev_value >> (lock_bitwise_size - 1));
824
825 lock_extract_counter(&prev_value, lock_size, &original_counter);
826 while (!informed) {
827 shmem_wait_wrapper(lock, lock_size, prev_value);
828 prev_value = get_lock_value(lock, lock_size);
829 informed = (prev_value >> (lock_bitwise_size - 1));
830 }
831
832 lock_pack_counter(lock, lock_size, &original_counter, 1);
833 } else {
834 int prev_value = *lock_inform;
835 int prev_pe = -1;
836 int remote_pe_next = 0;
837 int remote_counter = 1;
838
839 if (OSHMEM_SUCCESS != lock_restore_prev_pe(lock, &prev_pe)) {
840 SHMEM_API_ERROR("Unable to restore prev_pe on server PE#%i", my_pe);
841 oshmem_shmem_abort(-1);
842
843 }
844
845 if (prev_pe == server_pe) {
846 SHMEM_API_ERROR("prev_pe (%i) is me", prev_pe);
847 }
848
849 do {
850 uint64_t remote_lock = 0;
851 shmem_get_wrapper(&remote_lock, lock, lock_size, 1, prev_pe);
852 lock_extract_pe_next_counter(&remote_lock,
853 lock_size,
854 &remote_pe_next,
855 &remote_counter);
856 if ((remote_counter > 0) && (remote_pe_next == my_pe)) {
857 shmem_int_wait(lock_inform, prev_value);
858 prev_value = *lock_inform;
859 }
860 } while ((remote_counter > 0) && (remote_pe_next == my_pe));
861 }
862
863 return OSHMEM_SUCCESS;
864 }
865
866 static int shmem_lock_inform_next(void *lock, int lock_size, int pe_next)
867 {
868 int lock_bitwise_size = lock_size * 8;
869 int server_pe = shmem_lock_get_server(lock);
870
871 if (server_pe != pe_next) {
872 uint64_t temp_value = 0, remote_value = 0;
873 shmem_get_wrapper(&remote_value, lock, lock_size, 1, pe_next);
874 uint64_t new_remote_value = remote_value
875 | (((uint64_t) 1) << (lock_bitwise_size - 1));
876
877 while (remote_value
878 != (temp_value = shmem_lock_cswap_poll(lock,
879 lock_size,
880 remote_value,
881 new_remote_value,
882 pe_next))) {
883 remote_value = temp_value;
884 new_remote_value = remote_value
885 | (((uint64_t) 1) << (lock_bitwise_size - 1));
886 }
887 } else {
888 shmem_int_inc(lock_inform, pe_next);
889 }
890
891 return OSHMEM_SUCCESS;
892 }
893
894 static int lock_save_prev_pe(void *lock, int prev_pe)
895 {
896 oshmem_lock_prev_pe_container_t *container = lock_prev_pe_container_head;
897
898 while (container != 0) {
899 if (container->lock == lock) {
900 break;
901 }
902 container = container->next;
903 }
904
905 if (container) {
906 container->prev_pe = prev_pe;
907 } else {
908 container = malloc(sizeof(oshmem_lock_prev_pe_container_t));
909 container->lock = lock;
910 container->prev_pe = prev_pe;
911 container->next = lock_prev_pe_container_head;
912 container->prev = 0;
913 if (lock_prev_pe_container_head) {
914 lock_prev_pe_container_head->prev = container;
915 }
916 lock_prev_pe_container_head = container;
917 }
918
919 return OSHMEM_SUCCESS;
920 }
921
922 static int lock_restore_prev_pe(void *lock, int *prev_pe)
923 {
924 oshmem_lock_prev_pe_container_t *container = lock_prev_pe_container_head;
925 while (container != 0) {
926 if (container->lock == lock) {
927 break;
928 }
929 container = container->next;
930 }
931
932 if (container) {
933 oshmem_lock_prev_pe_container_t *next = container->next;
934 oshmem_lock_prev_pe_container_t *prev = container->prev;
935 *prev_pe = container->prev_pe;
936
937 if (prev) {
938 prev->next = next;
939 }
940 if (next) {
941 next->prev = prev;
942 }
943 if (lock_prev_pe_container_head == container) {
944 lock_prev_pe_container_head = next;
945 }
946 free(container);
947 return OSHMEM_SUCCESS;
948 } else {
949 *prev_pe = -1;
950 return OSHMEM_ERROR;
951 }
952 }
953
954 static int shmem_lock_try_inform_server(void *lock, int lock_size)
955 {
956 int my_pe = shmem_my_pe();
957 int server_pe = shmem_lock_get_server(lock);
958 int zero = 0;
959 int incorrect_pe = -1;
960 uint64_t remote_value = 0;
961
962 shmem_lock_pack_pe_next_pe_last(&remote_value,
963 lock_size,
964 &incorrect_pe,
965 &my_pe);
966 return !(remote_value
967 == shmem_lock_cswap_poll(lock, lock_size, remote_value, zero, server_pe));
968 }
969
970
971
972
973
974 void _shmem_set_lock(void *lock, int lock_size)
975 {
976 int my_pe = shmem_my_pe();
977
978 if (!shmem_lock_is_mine(lock, lock_size)) {
979 int has_lock = !_shmem_test_lock(lock, lock_size);
980
981 if (!has_lock) {
982 int pe_last = -1;
983 int ticket = shmem_lock_get_ticket(lock);
984
985 shmem_lock_increment_counter(lock, lock_size);
986 shmem_lock_wait_for_ticket(lock, lock_size, ticket, &pe_last);
987 if (pe_last != my_pe) {
988 shmem_lock_subscribe_for_informing(lock, lock_size, pe_last);
989 shmem_lock_wait_for_informing(lock, lock_size);
990 }
991 }
992 } else {
993 shmem_lock_increment_counter(lock, lock_size);
994 }
995 }
996
997 int _shmem_test_lock(void *lock, int lock_size)
998 {
999 int status = 1;
1000 int server_pe = shmem_lock_get_server(lock);
1001 int incorrect_value = -1;
1002 int my_pe = shmem_my_pe();
1003 uint64_t new_lock_value = 0;
1004 uint64_t prev_lock_value = 1;
1005 int my_lock = shmem_lock_is_mine(lock, lock_size);
1006
1007 shmem_lock_increment_counter(lock, lock_size);
1008 if (!my_lock) {
1009 if (shmem_lock_pack_pe_next_pe_last(&new_lock_value,
1010 lock_size,
1011 &incorrect_value,
1012 &my_pe)) {
1013 goto FreeMemory;
1014 }
1015
1016 prev_lock_value = shmem_lock_cswap(lock,
1017 lock_size,
1018 0,
1019 new_lock_value,
1020 server_pe);
1021 }
1022
1023 if (0 == prev_lock_value || my_lock) {
1024 status = 0;
1025 } else {
1026 shmem_lock_decrement_counter(lock, lock_size);
1027 }
1028
1029 FreeMemory: return status;
1030 }
1031
1032 void _shmem_clear_lock(void *lock, int lock_size)
1033 {
1034 int current_lock_counter = shmem_lock_decrement_counter(lock, lock_size);
1035
1036 if (0 == current_lock_counter) {
1037 int next_informed = 0;
1038 int pe_next = 0;
1039
1040 while (!next_informed) {
1041 shmem_lock_extract_pe_next(lock, lock_size, &pe_next);
1042 if (pe_next >= 0) {
1043 shmem_lock_inform_next(lock, lock_size, pe_next);
1044 next_informed = 1;
1045 } else {
1046
1047 if (!shmem_lock_try_inform_server(lock, lock_size)) {
1048 next_informed = 1;
1049 }
1050 }
1051 }
1052
1053 pe_next = -1;
1054 shmem_lock_pack_pe_next(lock, lock_size, &pe_next);
1055 }
1056 }