oshmem/mca/memheap/base/base.h

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
memheap_log2
memheap_down_align_addr
memheap_up_align_addr
memheap_align
map_segment_is_va_in
memheap_find_seg
memheap_is_va_in_segment
memheap_find_segnum
memheap_va2rva
map_segment_va2rva
map_segment_find_va
memheap_find_va
mca_memheap_base_get_cached_mkey
mca_memheap_base_num_transports
mca_memheap_seg2base_va
   1 /*
   2  * Copyright (c) 2013      Mellanox Technologies, Inc.
   3  *                         All rights reserved.
   4  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
   5  * $COPYRIGHT$
   6  *
   7  * Additional copyrights may follow
   8  *
   9  * $HEADER$
  10  */
  11 /**
  12  * @file
  13  */
  14 #ifndef MCA_MEMHEAP_BASE_H
  15 #define MCA_MEMHEAP_BASE_H
  16 
  17 #include "oshmem_config.h"
  18 #include "opal/class/opal_list.h"
  19 #include "opal/class/opal_value_array.h"
  20 #include "oshmem/mca/mca.h"
  21 
  22 #include "oshmem/mca/memheap/memheap.h"
  23 
  24 BEGIN_C_DECLS
  25 
  26 /*
  27  * Global functions for MCA: overall MEMHEAP open and close
  28  */
  29 OSHMEM_DECLSPEC int mca_memheap_base_select(void);
  30 
  31 /*
  32  * Globals
  33  */
  34 
  35 /* only used within base -- no need to DECLSPEC */
  36 #define MEMHEAP_BASE_MIN_ORDER         3                                /* forces 64 bit alignment */
  37 #define MEMHEAP_BASE_PAGE_ORDER        21
  38 #define MEMHEAP_BASE_PRIVATE_SIZE      (1ULL << MEMHEAP_BASE_PAGE_ORDER) /* should be at least the same as a huge page size */
  39 #define MEMHEAP_BASE_MIN_SIZE          (1ULL << MEMHEAP_BASE_PAGE_ORDER)    /* must fit into at least one huge page */
  40 
  41 extern int mca_memheap_base_already_opened;
  42 extern int mca_memheap_base_key_exchange;
  43 
  44 #define MCA_MEMHEAP_MAX_SEGMENTS    8
  45 #define HEAP_SEG_INDEX              0
  46 
  47 #define MEMHEAP_SEG_INVALID  0xFFFF
  48 
  49 
  50 typedef struct mca_memheap_base_config {
  51     long            device_nic_mem_seg_size; /* Used for SHMEM_HINT_DEVICE_NIC_MEM */
  52 } mca_memheap_base_config_t;
  53 
  54 
  55 typedef struct mca_memheap_map {
  56     map_segment_t   mem_segs[MCA_MEMHEAP_MAX_SEGMENTS]; /* TODO: change into pointer array */
  57     int             n_segments;
  58     int             num_transports;
  59 } mca_memheap_map_t;
  60 
  61 extern mca_memheap_map_t mca_memheap_base_map;
  62 extern mca_memheap_base_config_t mca_memheap_base_config;
  63 
  64 int mca_memheap_base_alloc_init(mca_memheap_map_t *, size_t, long);
  65 void mca_memheap_base_alloc_exit(mca_memheap_map_t *);
  66 int mca_memheap_base_static_init(mca_memheap_map_t *);
  67 void mca_memheap_base_static_exit(mca_memheap_map_t *);
  68 int mca_memheap_base_reg(mca_memheap_map_t *);
  69 int mca_memheap_base_dereg(mca_memheap_map_t *);
  70 int memheap_oob_init(mca_memheap_map_t *);
  71 void memheap_oob_destruct(void);
  72 
  73 OSHMEM_DECLSPEC int mca_memheap_base_is_symmetric_addr(const void* va);
  74 OSHMEM_DECLSPEC sshmem_mkey_t *mca_memheap_base_get_mkey(void* va,
  75                                                            int tr_id);
  76 OSHMEM_DECLSPEC sshmem_mkey_t * mca_memheap_base_get_cached_mkey_slow(shmem_ctx_t ctx,
  77                                                                       map_segment_t *s,
  78                                                                       int pe,
  79                                                                       void* va,
  80                                                                       int btl_id,
  81                                                                       void** rva);
  82 OSHMEM_DECLSPEC void mca_memheap_modex_recv_all(void);
  83 
  84 /* This function is for internal usage only
  85  * return value:
  86  * 0 - addr is not symmetric address
  87  * 1 - addr is part of user memheap
  88  * 2 - addr is part of private memheap
  89  * 3 - addr is static variable
  90  */
  91 typedef enum {
  92     ADDR_INVALID = 0, ADDR_USER, ADDR_PRIVATE, ADDR_STATIC,
  93 } addr_type_t;
  94 
  95 OSHMEM_DECLSPEC int mca_memheap_base_detect_addr_type(void* va);
  96 
  97 static inline unsigned memheap_log2(unsigned long long val)
  98 {
  99     /* add 1 if val is NOT a power of 2 (to do the ceil) */
 100     unsigned int count = (val & (val - 1) ? 1 : 0);
 101 
 102     while (val > 0) {
 103         val = val >> 1;
 104         count++;
 105     }
 106 
 107     return count > 0 ? count - 1 : 0;
 108 }
 109 
 110 static inline void *memheap_down_align_addr(void* addr, unsigned int shift)
 111 {
 112     return (void*) (((intptr_t) addr) & (~(intptr_t) 0) << shift);
 113 }
 114 
 115 static inline void *memheap_up_align_addr(void*addr, unsigned int shift)
 116 {
 117     return (void*) ((((intptr_t) addr) | ~((~(intptr_t) 0) << shift)));
 118 }
 119 
 120 static inline unsigned long long memheap_align(unsigned long top)
 121 {
 122     return ((top + MEMHEAP_BASE_MIN_SIZE - 1) & ~(MEMHEAP_BASE_MIN_SIZE - 1));
 123 }
 124 
 125 /*
 126  * MCA framework
 127  */
 128 OSHMEM_DECLSPEC extern mca_base_framework_t oshmem_memheap_base_framework;
 129 
 130 /* ******************************************************************** */
 131 #ifdef __BASE_FILE__
 132 #define __SPML_FILE__ __BASE_FILE__
 133 #else
 134 #define __SPML_FILE__ __FILE__
 135 #endif
 136 
 137 #ifdef OPAL_ENABLE_DEBUG
 138 #define MEMHEAP_VERBOSE(level, ...) \
 139     oshmem_output_verbose(level, oshmem_memheap_base_framework.framework_output, \
 140         "%s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
 141 #else
 142 #define MEMHEAP_VERBOSE(level, ...)
 143 #endif
 144 
 145 #define MEMHEAP_ERROR(...) \
 146     oshmem_output(oshmem_memheap_base_framework.framework_output, \
 147         "Error %s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
 148 
 149 #define MEMHEAP_WARN(...) \
 150     oshmem_output_verbose(0, oshmem_memheap_base_framework.framework_output, \
 151         "Warning %s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
 152 
 153 extern int mca_memheap_seg_cmp(const void *k, const void *v);
 154 
 155 /* Turn ON/OFF debug output from build (default 0) */
 156 #ifndef MEMHEAP_BASE_DEBUG
 157 #define MEMHEAP_BASE_DEBUG    0
 158 #endif
 159 #define MEMHEAP_VERBOSE_FASTPATH(...)
 160 
 161 extern mca_memheap_map_t* memheap_map;
 162 
 163 static inline int map_segment_is_va_in(map_base_segment_t *s, void *va)
 164 {
 165     return (va >= s->va_base && va < s->va_end);
 166 }
 167 
 168 static inline map_segment_t *memheap_find_seg(int segno)
 169 {
 170     return &mca_memheap_base_map.mem_segs[segno];
 171 }
 172 
 173 static inline int memheap_is_va_in_segment(void *va, int segno) 
 174 {
 175     return map_segment_is_va_in(&memheap_find_seg(segno)->super, va);
 176 }
 177 
 178 static inline int memheap_find_segnum(void *va)
 179 {
 180     int i;
 181 
 182     for (i = 0; i < mca_memheap_base_map.n_segments; i++) {
 183         if (memheap_is_va_in_segment(va, i)) {
 184             return i;
 185         }
 186     }
 187     return MEMHEAP_SEG_INVALID;
 188 }
 189 
 190 static inline void* memheap_va2rva(void* va, void* local_base, void* remote_base)
 191 {
 192     return (void*) (remote_base > local_base ?
 193             (uintptr_t)va + ((uintptr_t)remote_base - (uintptr_t)local_base) :
 194             (uintptr_t)va - ((uintptr_t)local_base - (uintptr_t)remote_base));
 195 }
 196 
 197 static inline void *map_segment_va2rva(mkey_segment_t *seg, void *va)
 198 {
 199     return memheap_va2rva(va, seg->super.va_base, seg->rva_base);
 200 }
 201 
 202 static inline map_base_segment_t *map_segment_find_va(map_base_segment_t *segs,
 203                                                       size_t elem_size, void *va)
 204 {
 205     map_base_segment_t *rseg;
 206     int i;
 207 
 208     for (i = 0; i < MCA_MEMHEAP_MAX_SEGMENTS; i++) {
 209         rseg = (map_base_segment_t *)((char *)segs + elem_size * i);
 210         if (OPAL_LIKELY(map_segment_is_va_in(rseg, va))) {
 211             return rseg;
 212         }
 213     }
 214 
 215     return NULL;
 216 }
 217 
 218 void mkey_segment_init(mkey_segment_t *seg, sshmem_mkey_t *mkey, uint32_t segno);
 219 
 220 static inline map_segment_t *memheap_find_va(void* va)
 221 {
 222     map_segment_t *s = NULL;
 223     int i;
 224 
 225     for (i = 0; i < memheap_map->n_segments; i++) {
 226         if (memheap_is_va_in_segment(va, i)) {
 227             s = &memheap_map->mem_segs[i];
 228             break;
 229         }
 230     }
 231 
 232 #if MEMHEAP_BASE_DEBUG == 1
 233     if (s) {
 234         MEMHEAP_VERBOSE(5, "match seg#%02ld: 0x%llX - 0x%llX %llu bytes va=%p",
 235                 s - memheap_map->mem_segs,
 236                 (long long)s->super.va_base,
 237                 (long long)s->super.va_end,
 238                 (long long)(s->super.va_end - s->super.va_base),
 239                 (void *)va);
 240     }
 241 #endif
 242     return s;
 243 }
 244 
 245 static inline  sshmem_mkey_t *mca_memheap_base_get_cached_mkey(shmem_ctx_t ctx,
 246                                                                int pe,
 247                                                                 void* va,
 248                                                                 int btl_id,
 249                                                                 void** rva)
 250 {
 251     map_segment_t *s;
 252     sshmem_mkey_t *mkey;
 253 
 254     MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p", pe, va);
 255     s = memheap_find_va(va);
 256     if (OPAL_UNLIKELY(NULL == s))
 257         return NULL ;
 258 
 259     if (OPAL_UNLIKELY(!MAP_SEGMENT_IS_VALID(s)))
 260         return NULL ;
 261 
 262     if (OPAL_UNLIKELY(pe == oshmem_my_proc_id())) {
 263         *rva = va;
 264         MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p -> (local) %lx %p", pe, va, 
 265                 s->mkeys[btl_id].u.key, *rva);
 266         return &s->mkeys[btl_id];
 267     }
 268 
 269     if (OPAL_LIKELY(s->mkeys_cache[pe])) {
 270         mkey = &s->mkeys_cache[pe][btl_id];
 271         *rva = memheap_va2rva(va, s->super.va_base, mkey->va_base);
 272         MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p -> (cached) %lx %p", pe, (void *)va, mkey->u.key, (void *)*rva);
 273         return mkey;
 274     }
 275 
 276     return mca_memheap_base_get_cached_mkey_slow(ctx, s, pe, va, btl_id, rva);
 277 }
 278 
 279 static inline int mca_memheap_base_num_transports(void) 
 280 {
 281     return memheap_map->num_transports;
 282 }
 283 
 284 static inline void* mca_memheap_seg2base_va(int seg)
 285 {
 286     return memheap_map->mem_segs[seg].super.va_base;
 287 }
 288 
 289 END_C_DECLS
 290 
 291 #endif /* MCA_MEMHEAP_BASE_H */
/* [<][>][^][v][top][bottom][index][help] */
root/oshmem/mca/memheap/base/base.h

INCLUDED FROM

DEFINITIONS
