root/opal/mca/btl/vader/btl_vader_fifo.h

INCLUDED FROM

DEFINITIONS

1. virtual2relative
2. virtual2relativepeer
3. relative2virtual
4. vader_fifo_read
5. vader_fifo_init
6. vader_fifo_write
7. vader_fifo_write_ep
8. vader_fifo_write_back

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2009 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
  14  * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
  15  * Copyright (c) 2010-2018 Los Alamos National Security, LLC.
  16  *                         All rights reserved.
  17  * $COPYRIGHT$
  18  *
  19  * Additional copyrights may follow
  20  *
  21  * $HEADER$
  22  */
  23 /**
  24  * @file
  25  */
  26 #ifndef MCA_BTL_VADER_FIFO_H
  27 #define MCA_BTL_VADER_FIFO_H
  28 
  29 #include "btl_vader.h"
  30 #include "btl_vader_endpoint.h"
  31 #include "btl_vader_frag.h"
  32 
  33 #define vader_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) (x), (intptr_t *) (y), (intptr_t) (z))
  34 
  35 #if SIZEOF_VOID_P == 8
  36   #define vader_item_swap(x, y)      opal_atomic_swap_64((opal_atomic_int64_t *)(x), (int64_t)(y))
  37 
  38   #define MCA_BTL_VADER_OFFSET_MASK 0xffffffffll
  39   #define MCA_BTL_VADER_OFFSET_BITS 32
  40   #define MCA_BTL_VADER_BITNESS     64
  41 #else
  42   #define vader_item_swap(x, y)      opal_atomic_swap_32((opal_atomic_int32_t *)(x), (int32_t)(y))
  43 
  44   #define MCA_BTL_VADER_OFFSET_MASK 0x00ffffffl
  45   #define MCA_BTL_VADER_OFFSET_BITS 24
  46   #define MCA_BTL_VADER_BITNESS     32
  47 #endif
  48 
  49 typedef opal_atomic_intptr_t atomic_fifo_value_t;
  50 typedef intptr_t fifo_value_t;
  51 
  52 #define VADER_FIFO_FREE  ((fifo_value_t)-2)
  53 
  54 /*
  55  * Shared Memory FIFOs
  56  *
  57  * The FIFO is implemented as a linked list of frag headers. The fifo has multiple
  58  * producers and a single consumer (in the single thread case) so the tail needs
  59  * to be modified by an atomic or protected by a atomic lock.
  60  *
  61  * Since the frags live in shared memory that is mapped differently into
  62  * each address space, the head and tail pointers are relative (each process must
  63  * add its own offset).
  64  *
  65  * We introduce some padding at the end of the structure but it is probably unnecessary.
  66  */
  67 
  68 /* lock free fifo */
  69 typedef struct vader_fifo_t {
  70     atomic_fifo_value_t fifo_head;
  71     atomic_fifo_value_t fifo_tail;
  72     opal_atomic_int32_t fbox_available;
  73 } vader_fifo_t;
  74 
  75 /* large enough to ensure the fifo is on its own cache line */
  76 #define MCA_BTL_VADER_FIFO_SIZE 128
  77 
  78 /***
  79  * One or more FIFO components may be a pointer that must be
  80  * accessed by multiple processes.  Since the shared region may
  81  * be mmapped differently into each process's address space,
  82  * these pointers will be relative to some base address.  Here,
  83  * we define inline functions to translate between relative
  84  * addresses and virtual addresses.
  85  */
  86 
  87 /* This only works for finding the relative address for a pointer within my_segment */
  88 static inline fifo_value_t virtual2relative (char *addr)
  89 {
  90     return (fifo_value_t) ((intptr_t) (addr - mca_btl_vader_component.my_segment)) | ((fifo_value_t)MCA_BTL_VADER_LOCAL_RANK << MCA_BTL_VADER_OFFSET_BITS);
  91 }
  92 
  93 static inline fifo_value_t virtual2relativepeer (struct mca_btl_base_endpoint_t *endpoint, char *addr)
  94 {
  95     return (fifo_value_t) ((intptr_t) (addr - endpoint->segment_base)) | ((fifo_value_t)endpoint->peer_smp_rank << MCA_BTL_VADER_OFFSET_BITS);
  96 }
  97 
  98 static inline void *relative2virtual (fifo_value_t offset)
  99 {
 100     return (void *)(intptr_t)((offset & MCA_BTL_VADER_OFFSET_MASK) + mca_btl_vader_component.endpoints[offset >> MCA_BTL_VADER_OFFSET_BITS].segment_base);
 101 }
 102 
 103 #include "btl_vader_fbox.h"
 104 
 105 /**
 106  * vader_fifo_read:
 107  *
 108  * @brief reads a single fragment from a local fifo
 109  *
 110  * @param[inout]   fifo - FIFO to read from
 111  * @param[out]     ep   - returns the endpoint the fifo element was read from
 112  *
 113  * @returns a fragment header or NULL
 114  *
 115  * This function does not currently support multiple readers.
 116  */
 117 static inline mca_btl_vader_hdr_t *vader_fifo_read (vader_fifo_t *fifo, struct mca_btl_base_endpoint_t **ep)
 118 {
 119     mca_btl_vader_hdr_t *hdr;
 120     fifo_value_t value;
 121 
 122     if (VADER_FIFO_FREE == fifo->fifo_head) {
 123         return NULL;
 124     }
 125 
 126     opal_atomic_rmb ();
 127 
 128     value = fifo->fifo_head;
 129 
 130     *ep = &mca_btl_vader_component.endpoints[value >> MCA_BTL_VADER_OFFSET_BITS];
 131     hdr = (mca_btl_vader_hdr_t *) relative2virtual (value);
 132 
 133     fifo->fifo_head = VADER_FIFO_FREE;
 134 
 135     assert (hdr->next != value);
 136 
 137     if (OPAL_UNLIKELY(VADER_FIFO_FREE == hdr->next)) {
 138         opal_atomic_rmb();
 139 
 140         if (!vader_item_compare_exchange (&fifo->fifo_tail, &value, VADER_FIFO_FREE)) {
 141             while (VADER_FIFO_FREE == hdr->next) {
 142                 opal_atomic_rmb ();
 143             }
 144 
 145             fifo->fifo_head = hdr->next;
 146         }
 147     } else {
 148         fifo->fifo_head = hdr->next;
 149     }
 150 
 151     opal_atomic_wmb ();
 152     return hdr;
 153 }
 154 
 155 static inline void vader_fifo_init (vader_fifo_t *fifo)
 156 {
 157     /* due to a compiler bug in Oracle C 5.15 the following line was broken into two. Not
 158      * ideal but oh well. See #5814 */
 159     /* fifo->fifo_head = fifo->fifo_tail = VADER_FIFO_FREE; */
 160     fifo->fifo_head = VADER_FIFO_FREE;
 161     fifo->fifo_tail = VADER_FIFO_FREE;
 162     fifo->fbox_available = mca_btl_vader_component.fbox_max;
 163     mca_btl_vader_component.my_fifo = fifo;
 164 }
 165 
 166 static inline void vader_fifo_write (vader_fifo_t *fifo, fifo_value_t value)
 167 {
 168     fifo_value_t prev;
 169 
 170     opal_atomic_wmb ();
 171     prev = vader_item_swap (&fifo->fifo_tail, value);
 172     opal_atomic_rmb ();
 173 
 174     assert (prev != value);
 175 
 176     if (OPAL_LIKELY(VADER_FIFO_FREE != prev)) {
 177         mca_btl_vader_hdr_t *hdr = (mca_btl_vader_hdr_t *) relative2virtual (prev);
 178         hdr->next = value;
 179     } else {
 180         fifo->fifo_head = value;
 181     }
 182 
 183     opal_atomic_wmb ();
 184 }
 185 
 186 /**
 187  * vader_fifo_write_ep:
 188  *
 189  * @brief write a frag (relative to this process' base) to another rank's fifo
 190  *
 191  * @param[in]  hdr - fragment header to write
 192  * @param[in]  ep  - endpoint to write the fragment to
 193  *
 194  * This function is used to send a fragment to a remote peer. {hdr} must belong
 195  * to the current process.
 196  */
 197 static inline bool vader_fifo_write_ep (mca_btl_vader_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep)
 198 {
 199     fifo_value_t rhdr = virtual2relative ((char *) hdr);
 200     if (ep->fbox_out.buffer) {
 201         /* if there is a fast box for this peer then use the fast box to send the fragment header.
 202          * this is done to ensure fragment ordering */
 203         opal_atomic_wmb ();
 204         return mca_btl_vader_fbox_sendi (ep, 0xfe, &rhdr, sizeof (rhdr), NULL, 0);
 205     }
 206     mca_btl_vader_try_fbox_setup (ep, hdr);
 207     hdr->next = VADER_FIFO_FREE;
 208     vader_fifo_write (ep->fifo, rhdr);
 209 
 210     return true;
 211 }
 212 
 213 /**
 214  * vader_fifo_write_back:
 215  *
 216  * @brief write a frag (relative to the remote process' base) to the remote fifo
 217  *
 218  * @param[in]  hdr - fragment header to write
 219  * @param[in]  ep  - endpoint the fragment belongs to
 220  *
 221  * This function is used to return a fragment to the sending process. It differs from vader_fifo_write_ep
 222  * in that it uses the {ep} to produce the relative address.
 223  */
 224 static inline void vader_fifo_write_back (mca_btl_vader_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep)
 225 {
 226     hdr->next = VADER_FIFO_FREE;
 227     vader_fifo_write(ep->fifo, virtual2relativepeer (ep, (char *) hdr));
 228 }
 229 
 230 #endif /* MCA_BTL_VADER_FIFO_H */