1 /*
2 * Copyright (c) 2013 Mellanox Technologies, Inc.
3 * All rights reserved.
4 * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
5 * $COPYRIGHT$
6 *
7 * Additional copyrights may follow
8 *
9 * $HEADER$
10 */
11 /**
12 * @file
13 */
14 #ifndef MCA_MEMHEAP_BASE_H
15 #define MCA_MEMHEAP_BASE_H
16
17 #include "oshmem_config.h"
18 #include "opal/class/opal_list.h"
19 #include "opal/class/opal_value_array.h"
20 #include "oshmem/mca/mca.h"
21
22 #include "oshmem/mca/memheap/memheap.h"
23
24 BEGIN_C_DECLS
25
26 /*
27 * Global functions for MCA: overall MEMHEAP open and close
28 */
29 OSHMEM_DECLSPEC int mca_memheap_base_select(void);
30
31 /*
32 * Globals
33 */
34
35 /* only used within base -- no need to DECLSPEC */
36 #define MEMHEAP_BASE_MIN_ORDER 3 /* forces 64 bit alignment */
37 #define MEMHEAP_BASE_PAGE_ORDER 21
38 #define MEMHEAP_BASE_PRIVATE_SIZE (1ULL << MEMHEAP_BASE_PAGE_ORDER) /* should be at least the same as a huge page size */
39 #define MEMHEAP_BASE_MIN_SIZE (1ULL << MEMHEAP_BASE_PAGE_ORDER) /* must fit into at least one huge page */
40
41 extern int mca_memheap_base_already_opened;
42 extern int mca_memheap_base_key_exchange;
43
44 #define MCA_MEMHEAP_MAX_SEGMENTS 4
45 #define HEAP_SEG_INDEX 0
46 #define SYMB_SEG_INDEX 1
47 #define MCA_MEMHEAP_SEG_COUNT (SYMB_SEG_INDEX+1)
48
49 #define MEMHEAP_SEG_INVALID 0xFFFF
50
51
52 typedef struct mca_memheap_map {
53 map_segment_t mem_segs[MCA_MEMHEAP_MAX_SEGMENTS]; /* TODO: change into pointer array */
54 int n_segments;
55 int num_transports;
56 } mca_memheap_map_t;
57
58 extern mca_memheap_map_t mca_memheap_base_map;
59
60 int mca_memheap_base_alloc_init(mca_memheap_map_t *, size_t);
61 void mca_memheap_base_alloc_exit(mca_memheap_map_t *);
62 int mca_memheap_base_static_init(mca_memheap_map_t *);
63 void mca_memheap_base_static_exit(mca_memheap_map_t *);
64 int mca_memheap_base_reg(mca_memheap_map_t *);
65 int mca_memheap_base_dereg(mca_memheap_map_t *);
66 int memheap_oob_init(mca_memheap_map_t *);
67 void memheap_oob_destruct(void);
68
69 OSHMEM_DECLSPEC int mca_memheap_base_is_symmetric_addr(const void* va);
70 OSHMEM_DECLSPEC sshmem_mkey_t *mca_memheap_base_get_mkey(void* va,
71 int tr_id);
72 OSHMEM_DECLSPEC sshmem_mkey_t * mca_memheap_base_get_cached_mkey_slow(map_segment_t *s,
73 int pe,
74 void* va,
75 int btl_id,
76 void** rva);
77 OSHMEM_DECLSPEC void mca_memheap_modex_recv_all(void);
78
79 /* This function is for internal usage only
80 * return value:
81 * 0 - addr is not symmetric address
82 * 1 - addr is part of user memheap
83 * 2 - addr is part of private memheap
84 * 3 - addr is static variable
85 */
86 typedef enum {
87 ADDR_INVALID = 0, ADDR_USER, ADDR_PRIVATE, ADDR_STATIC,
88 } addr_type_t;
89
90 OSHMEM_DECLSPEC int mca_memheap_base_detect_addr_type(void* va);
91
memheap_log2(unsigned long long val)92 static inline unsigned memheap_log2(unsigned long long val)
93 {
94 /* add 1 if val is NOT a power of 2 (to do the ceil) */
95 unsigned int count = (val & (val - 1) ? 1 : 0);
96
97 while (val > 0) {
98 val = val >> 1;
99 count++;
100 }
101
102 return count > 0 ? count - 1 : 0;
103 }
104
memheap_down_align_addr(void * addr,unsigned int shift)105 static inline void *memheap_down_align_addr(void* addr, unsigned int shift)
106 {
107 return (void*) (((intptr_t) addr) & (~(intptr_t) 0) << shift);
108 }
109
memheap_up_align_addr(void * addr,unsigned int shift)110 static inline void *memheap_up_align_addr(void*addr, unsigned int shift)
111 {
112 return (void*) ((((intptr_t) addr) | ~((~(intptr_t) 0) << shift)));
113 }
114
memheap_align(unsigned long top)115 static inline unsigned long long memheap_align(unsigned long top)
116 {
117 return ((top + MEMHEAP_BASE_MIN_SIZE - 1) & ~(MEMHEAP_BASE_MIN_SIZE - 1));
118 }
119
120 /*
121 * MCA framework
122 */
123 OSHMEM_DECLSPEC extern mca_base_framework_t oshmem_memheap_base_framework;
124
125 /* ******************************************************************** */
126 #ifdef __BASE_FILE__
127 #define __SPML_FILE__ __BASE_FILE__
128 #else
129 #define __SPML_FILE__ __FILE__
130 #endif
131
132 #ifdef OPAL_ENABLE_DEBUG
133 #define MEMHEAP_VERBOSE(level, ...) \
134 oshmem_output_verbose(level, oshmem_memheap_base_framework.framework_output, \
135 "%s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
136 #else
137 #define MEMHEAP_VERBOSE(level, ...)
138 #endif
139
140 #define MEMHEAP_ERROR(...) \
141 oshmem_output(oshmem_memheap_base_framework.framework_output, \
142 "Error %s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
143
144 #define MEMHEAP_WARN(...) \
145 oshmem_output_verbose(0, oshmem_memheap_base_framework.framework_output, \
146 "Warning %s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
147
148 extern int mca_memheap_seg_cmp(const void *k, const void *v);
149
150 /* Turn ON/OFF debug output from build (default 0) */
151 #ifndef MEMHEAP_BASE_DEBUG
152 #define MEMHEAP_BASE_DEBUG 0
153 #endif
154 #define MEMHEAP_VERBOSE_FASTPATH(...)
155
156 extern mca_memheap_map_t* memheap_map;
157
map_segment_is_va_in(map_base_segment_t * s,void * va)158 static inline int map_segment_is_va_in(map_base_segment_t *s, void *va)
159 {
160 return (va >= s->va_base && va < s->va_end);
161 }
162
memheap_find_seg(int segno)163 static inline map_segment_t *memheap_find_seg(int segno)
164 {
165 return &mca_memheap_base_map.mem_segs[segno];
166 }
167
memheap_is_va_in_segment(void * va,int segno)168 static inline int memheap_is_va_in_segment(void *va, int segno)
169 {
170 return map_segment_is_va_in(&memheap_find_seg(segno)->super, va);
171 }
172
memheap_find_segnum(void * va)173 static inline int memheap_find_segnum(void *va)
174 {
175 if (OPAL_LIKELY(memheap_is_va_in_segment(va, SYMB_SEG_INDEX))) {
176 return SYMB_SEG_INDEX;
177 } else if (memheap_is_va_in_segment(va, HEAP_SEG_INDEX)) {
178 return HEAP_SEG_INDEX;
179 }
180 return MEMHEAP_SEG_INVALID;
181 }
182
memheap_va2rva(void * va,void * local_base,void * remote_base)183 static inline void* memheap_va2rva(void* va, void* local_base, void* remote_base)
184 {
185 return (void*) (remote_base > local_base ?
186 (uintptr_t)va + ((uintptr_t)remote_base - (uintptr_t)local_base) :
187 (uintptr_t)va - ((uintptr_t)local_base - (uintptr_t)remote_base));
188 }
189
map_segment_va2rva(mkey_segment_t * seg,void * va)190 static inline void *map_segment_va2rva(mkey_segment_t *seg, void *va)
191 {
192 return memheap_va2rva(va, seg->super.va_base, seg->rva_base);
193 }
194
map_segment_find_va(map_base_segment_t * segs,size_t elem_size,void * va)195 static inline map_base_segment_t *map_segment_find_va(map_base_segment_t *segs, size_t elem_size, void *va)
196 {
197 map_base_segment_t *rseg;
198
199 rseg = (map_base_segment_t *)((char *)segs + elem_size * HEAP_SEG_INDEX);
200 if (OPAL_LIKELY(map_segment_is_va_in(rseg, va))) {
201 return rseg;
202 }
203
204 rseg = (map_base_segment_t *)((char *)segs + elem_size * SYMB_SEG_INDEX);
205 if (OPAL_LIKELY(map_segment_is_va_in(rseg, va))) {
206 return rseg;
207 }
208
209 return NULL;
210 }
211
212 void mkey_segment_init(mkey_segment_t *seg, sshmem_mkey_t *mkey, uint32_t segno);
213
memheap_find_va(void * va)214 static inline map_segment_t *memheap_find_va(void* va)
215 {
216 map_segment_t *s;
217
218 /* most probably there will be only two segments: heap and global data */
219 if (OPAL_LIKELY(memheap_is_va_in_segment(va, SYMB_SEG_INDEX))) {
220 s = &memheap_map->mem_segs[SYMB_SEG_INDEX];
221 } else if (memheap_is_va_in_segment(va, HEAP_SEG_INDEX)) {
222 s = &memheap_map->mem_segs[HEAP_SEG_INDEX];
223 } else if (memheap_map->n_segments - 2 > 0) {
224 s = bsearch(va,
225 &memheap_map->mem_segs[SYMB_SEG_INDEX+1],
226 memheap_map->n_segments - 2,
227 sizeof(*s),
228 mca_memheap_seg_cmp);
229 } else {
230 s = NULL;
231 }
232
233 #if MEMHEAP_BASE_DEBUG == 1
234 if (s) {
235 MEMHEAP_VERBOSE(5, "match seg#%02ld: 0x%llX - 0x%llX %llu bytes va=%p",
236 s - memheap_map->mem_segs,
237 (long long)s->super.va_base,
238 (long long)s->super.va_end,
239 (long long)(s->super.va_end - s->super.va_base),
240 (void *)va);
241 }
242 #endif
243 return s;
244 }
245
mca_memheap_base_get_cached_mkey(int pe,void * va,int btl_id,void ** rva)246 static inline sshmem_mkey_t *mca_memheap_base_get_cached_mkey(int pe,
247 void* va,
248 int btl_id,
249 void** rva)
250 {
251 map_segment_t *s;
252 sshmem_mkey_t *mkey;
253
254 MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p", pe, va);
255 s = memheap_find_va(va);
256 if (OPAL_UNLIKELY(NULL == s))
257 return NULL ;
258
259 if (OPAL_UNLIKELY(!MAP_SEGMENT_IS_VALID(s)))
260 return NULL ;
261
262 if (OPAL_UNLIKELY(pe == oshmem_my_proc_id())) {
263 *rva = va;
264 MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p -> (local) %lx %p", pe, va,
265 s->mkeys[btl_id].u.key, *rva);
266 return &s->mkeys[btl_id];
267 }
268
269 if (OPAL_LIKELY(s->mkeys_cache[pe])) {
270 mkey = &s->mkeys_cache[pe][btl_id];
271 *rva = memheap_va2rva(va, s->super.va_base, mkey->va_base);
272 MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p -> (cached) %lx %p", pe, (void *)va, mkey->u.key, (void *)*rva);
273 return mkey;
274 }
275
276 return mca_memheap_base_get_cached_mkey_slow(s, pe, va, btl_id, rva);
277 }
278
mca_memheap_base_num_transports(void)279 static inline int mca_memheap_base_num_transports(void)
280 {
281 return memheap_map->num_transports;
282 }
283
mca_memheap_seg2base_va(int seg)284 static inline void* mca_memheap_seg2base_va(int seg)
285 {
286 return memheap_map->mem_segs[seg].super.va_base;
287 }
288
289 END_C_DECLS
290
291 #endif /* MCA_MEMHEAP_BASE_H */
292