1 #ifndef JEMALLOC_INTERNAL_EXTENT_INLINES_H 2 #define JEMALLOC_INTERNAL_EXTENT_INLINES_H 3 4 #include "jemalloc/internal/mutex.h" 5 #include "jemalloc/internal/mutex_pool.h" 6 #include "jemalloc/internal/pages.h" 7 #include "jemalloc/internal/prng.h" 8 #include "jemalloc/internal/ql.h" 9 #include "jemalloc/internal/sz.h" 10 11 static inline void 12 extent_lock(tsdn_t *tsdn, extent_t *extent) { 13 assert(extent != NULL); 14 mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)extent); 15 } 16 17 static inline void 18 extent_unlock(tsdn_t *tsdn, extent_t *extent) { 19 assert(extent != NULL); 20 mutex_pool_unlock(tsdn, &extent_mutex_pool, (uintptr_t)extent); 21 } 22 23 static inline void 24 extent_lock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { 25 assert(extent1 != NULL && extent2 != NULL); 26 mutex_pool_lock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1, 27 (uintptr_t)extent2); 28 } 29 30 static inline void 31 extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { 32 assert(extent1 != NULL && extent2 != NULL); 33 mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1, 34 (uintptr_t)extent2); 35 } 36 37 static inline arena_t * 38 extent_arena_get(const extent_t *extent) { 39 unsigned arena_ind = (unsigned)((extent->e_bits & 40 EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); 41 /* 42 * The following check is omitted because we should never actually read 43 * a NULL arena pointer. 44 */ 45 if (false && arena_ind >= MALLOCX_ARENA_LIMIT) { 46 return NULL; 47 } 48 assert(arena_ind < MALLOCX_ARENA_LIMIT); 49 return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE); 50 } 51 52 static inline szind_t 53 extent_szind_get_maybe_invalid(const extent_t *extent) { 54 szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >> 55 EXTENT_BITS_SZIND_SHIFT); 56 assert(szind <= NSIZES); 57 return szind; 58 } 59 60 static inline szind_t 61 extent_szind_get(const extent_t *extent) { 62 szind_t szind = extent_szind_get_maybe_invalid(extent); 63 assert(szind < NSIZES); /* Never call when "invalid". */ 64 return szind; 65 } 66 67 static inline size_t 68 extent_usize_get(const extent_t *extent) { 69 return sz_index2size(extent_szind_get(extent)); 70 } 71 72 static inline size_t 73 extent_sn_get(const extent_t *extent) { 74 return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >> 75 EXTENT_BITS_SN_SHIFT); 76 } 77 78 static inline extent_state_t 79 extent_state_get(const extent_t *extent) { 80 return (extent_state_t)((extent->e_bits & EXTENT_BITS_STATE_MASK) >> 81 EXTENT_BITS_STATE_SHIFT); 82 } 83 84 static inline bool 85 extent_zeroed_get(const extent_t *extent) { 86 return (bool)((extent->e_bits & EXTENT_BITS_ZEROED_MASK) >> 87 EXTENT_BITS_ZEROED_SHIFT); 88 } 89 90 static inline bool 91 extent_committed_get(const extent_t *extent) { 92 return (bool)((extent->e_bits & EXTENT_BITS_COMMITTED_MASK) >> 93 EXTENT_BITS_COMMITTED_SHIFT); 94 } 95 96 static inline bool 97 extent_slab_get(const extent_t *extent) { 98 return (bool)((extent->e_bits & EXTENT_BITS_SLAB_MASK) >> 99 EXTENT_BITS_SLAB_SHIFT); 100 } 101 102 static inline unsigned 103 extent_nfree_get(const extent_t *extent) { 104 assert(extent_slab_get(extent)); 105 return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK) >> 106 EXTENT_BITS_NFREE_SHIFT); 107 } 108 109 static inline void * 110 extent_base_get(const extent_t *extent) { 111 assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || 112 !extent_slab_get(extent)); 113 return PAGE_ADDR2BASE(extent->e_addr); 114 } 115 116 static inline void * 117 extent_addr_get(const extent_t *extent) { 118 assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || 119 !extent_slab_get(extent)); 120 return extent->e_addr; 121 } 122 123 static inline size_t 124 extent_size_get(const extent_t *extent) { 125 return (extent->e_size_esn & EXTENT_SIZE_MASK); 126 } 127 128 static inline size_t 129 extent_esn_get(const extent_t *extent) { 130 return (extent->e_size_esn & EXTENT_ESN_MASK); 131 } 132 133 static inline size_t 134 extent_bsize_get(const extent_t *extent) { 135 return extent->e_bsize; 136 } 137 138 static inline void * 139 extent_before_get(const extent_t *extent) { 140 return (void *)((uintptr_t)extent_base_get(extent) - PAGE); 141 } 142 143 static inline void * 144 extent_last_get(const extent_t *extent) { 145 return (void *)((uintptr_t)extent_base_get(extent) + 146 extent_size_get(extent) - PAGE); 147 } 148 149 static inline void * 150 extent_past_get(const extent_t *extent) { 151 return (void *)((uintptr_t)extent_base_get(extent) + 152 extent_size_get(extent)); 153 } 154 155 static inline arena_slab_data_t * 156 extent_slab_data_get(extent_t *extent) { 157 assert(extent_slab_get(extent)); 158 return &extent->e_slab_data; 159 } 160 161 static inline const arena_slab_data_t * 162 extent_slab_data_get_const(const extent_t *extent) { 163 assert(extent_slab_get(extent)); 164 return &extent->e_slab_data; 165 } 166 167 static inline prof_tctx_t * 168 extent_prof_tctx_get(const extent_t *extent) { 169 return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx, 170 ATOMIC_ACQUIRE); 171 } 172 173 static inline void 174 extent_arena_set(extent_t *extent, arena_t *arena) { 175 unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U << 176 MALLOCX_ARENA_BITS) - 1); 177 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ARENA_MASK) | 178 ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT); 179 } 180 181 static inline void 182 extent_addr_set(extent_t *extent, void *addr) { 183 extent->e_addr = addr; 184 } 185 186 static inline void 187 extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) { 188 assert(extent_base_get(extent) == extent_addr_get(extent)); 189 190 if (alignment < PAGE) { 191 unsigned lg_range = LG_PAGE - 192 lg_floor(CACHELINE_CEILING(alignment)); 193 size_t r = 194 prng_lg_range_zu(&extent_arena_get(extent)->offset_state, 195 lg_range, true); 196 uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE - 197 lg_range); 198 extent->e_addr = (void *)((uintptr_t)extent->e_addr + 199 random_offset); 200 assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) == 201 extent->e_addr); 202 } 203 } 204 205 static inline void 206 extent_size_set(extent_t *extent, size_t size) { 207 assert((size & ~EXTENT_SIZE_MASK) == 0); 208 extent->e_size_esn = size | (extent->e_size_esn & ~EXTENT_SIZE_MASK); 209 } 210 211 static inline void 212 extent_esn_set(extent_t *extent, size_t esn) { 213 extent->e_size_esn = (extent->e_size_esn & ~EXTENT_ESN_MASK) | (esn & 214 EXTENT_ESN_MASK); 215 } 216 217 static inline void 218 extent_bsize_set(extent_t *extent, size_t bsize) { 219 extent->e_bsize = bsize; 220 } 221 222 static inline void 223 extent_szind_set(extent_t *extent, szind_t szind) { 224 assert(szind <= NSIZES); /* NSIZES means "invalid". */ 225 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) | 226 ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT); 227 } 228 229 static inline void 230 extent_nfree_set(extent_t *extent, unsigned nfree) { 231 assert(extent_slab_get(extent)); 232 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) | 233 ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); 234 } 235 236 static inline void 237 extent_nfree_inc(extent_t *extent) { 238 assert(extent_slab_get(extent)); 239 extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); 240 } 241 242 static inline void 243 extent_nfree_dec(extent_t *extent) { 244 assert(extent_slab_get(extent)); 245 extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); 246 } 247 248 static inline void 249 extent_sn_set(extent_t *extent, size_t sn) { 250 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | 251 ((uint64_t)sn << EXTENT_BITS_SN_SHIFT); 252 } 253 254 static inline void 255 extent_state_set(extent_t *extent, extent_state_t state) { 256 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_STATE_MASK) | 257 ((uint64_t)state << EXTENT_BITS_STATE_SHIFT); 258 } 259 260 static inline void 261 extent_zeroed_set(extent_t *extent, bool zeroed) { 262 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ZEROED_MASK) | 263 ((uint64_t)zeroed << EXTENT_BITS_ZEROED_SHIFT); 264 } 265 266 static inline void 267 extent_committed_set(extent_t *extent, bool committed) { 268 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_COMMITTED_MASK) | 269 ((uint64_t)committed << EXTENT_BITS_COMMITTED_SHIFT); 270 } 271 272 static inline void 273 extent_slab_set(extent_t *extent, bool slab) { 274 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SLAB_MASK) | 275 ((uint64_t)slab << EXTENT_BITS_SLAB_SHIFT); 276 } 277 278 static inline void 279 extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { 280 atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE); 281 } 282 283 static inline void 284 extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, 285 bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, 286 bool committed) { 287 assert(addr == PAGE_ADDR2BASE(addr) || !slab); 288 289 extent_arena_set(extent, arena); 290 extent_addr_set(extent, addr); 291 extent_size_set(extent, size); 292 extent_slab_set(extent, slab); 293 extent_szind_set(extent, szind); 294 extent_sn_set(extent, sn); 295 extent_state_set(extent, state); 296 extent_zeroed_set(extent, zeroed); 297 extent_committed_set(extent, committed); 298 ql_elm_new(extent, ql_link); 299 if (config_prof) { 300 extent_prof_tctx_set(extent, NULL); 301 } 302 } 303 304 static inline void 305 extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { 306 extent_arena_set(extent, NULL); 307 extent_addr_set(extent, addr); 308 extent_bsize_set(extent, bsize); 309 extent_slab_set(extent, false); 310 extent_szind_set(extent, NSIZES); 311 extent_sn_set(extent, sn); 312 extent_state_set(extent, extent_state_active); 313 extent_zeroed_set(extent, true); 314 extent_committed_set(extent, true); 315 } 316 317 static inline void 318 extent_list_init(extent_list_t *list) { 319 ql_new(list); 320 } 321 322 static inline extent_t * 323 extent_list_first(const extent_list_t *list) { 324 return ql_first(list); 325 } 326 327 static inline extent_t * 328 extent_list_last(const extent_list_t *list) { 329 return ql_last(list, ql_link); 330 } 331 332 static inline void 333 extent_list_append(extent_list_t *list, extent_t *extent) { 334 ql_tail_insert(list, extent, ql_link); 335 } 336 337 static inline void 338 extent_list_replace(extent_list_t *list, extent_t *to_remove, 339 extent_t *to_insert) { 340 ql_after_insert(to_remove, to_insert, ql_link); 341 ql_remove(list, to_remove, ql_link); 342 } 343 344 static inline void 345 extent_list_remove(extent_list_t *list, extent_t *extent) { 346 ql_remove(list, extent, ql_link); 347 } 348 349 static inline int 350 extent_sn_comp(const extent_t *a, const extent_t *b) { 351 size_t a_sn = extent_sn_get(a); 352 size_t b_sn = extent_sn_get(b); 353 354 return (a_sn > b_sn) - (a_sn < b_sn); 355 } 356 357 static inline int 358 extent_esn_comp(const extent_t *a, const extent_t *b) { 359 size_t a_esn = extent_esn_get(a); 360 size_t b_esn = extent_esn_get(b); 361 362 return (a_esn > b_esn) - (a_esn < b_esn); 363 } 364 365 static inline int 366 extent_ad_comp(const extent_t *a, const extent_t *b) { 367 uintptr_t a_addr = (uintptr_t)extent_addr_get(a); 368 uintptr_t b_addr = (uintptr_t)extent_addr_get(b); 369 370 return (a_addr > b_addr) - (a_addr < b_addr); 371 } 372 373 static inline int 374 extent_ead_comp(const extent_t *a, const extent_t *b) { 375 uintptr_t a_eaddr = (uintptr_t)a; 376 uintptr_t b_eaddr = (uintptr_t)b; 377 378 return (a_eaddr > b_eaddr) - (a_eaddr < b_eaddr); 379 } 380 381 static inline int 382 extent_snad_comp(const extent_t *a, const extent_t *b) { 383 int ret; 384 385 ret = extent_sn_comp(a, b); 386 if (ret != 0) { 387 return ret; 388 } 389 390 ret = extent_ad_comp(a, b); 391 return ret; 392 } 393 394 static inline int 395 extent_esnead_comp(const extent_t *a, const extent_t *b) { 396 int ret; 397 398 ret = extent_esn_comp(a, b); 399 if (ret != 0) { 400 return ret; 401 } 402 403 ret = extent_ead_comp(a, b); 404 return ret; 405 } 406 407 #endif /* JEMALLOC_INTERNAL_EXTENT_INLINES_H */ 408