1 #ifndef JEMALLOC_INTERNAL_EXTENT_INLINES_H 2 #define JEMALLOC_INTERNAL_EXTENT_INLINES_H 3 4 #include "jemalloc/internal/mutex.h" 5 #include "jemalloc/internal/mutex_pool.h" 6 #include "jemalloc/internal/pages.h" 7 #include "jemalloc/internal/prng.h" 8 #include "jemalloc/internal/ql.h" 9 #include "jemalloc/internal/sc.h" 10 #include "jemalloc/internal/sz.h" 11 12 static inline void 13 extent_lock(tsdn_t *tsdn, extent_t *extent) { 14 assert(extent != NULL); 15 mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)extent); 16 } 17 18 static inline void 19 extent_unlock(tsdn_t *tsdn, extent_t *extent) { 20 assert(extent != NULL); 21 mutex_pool_unlock(tsdn, &extent_mutex_pool, (uintptr_t)extent); 22 } 23 24 static inline void 25 extent_lock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { 26 assert(extent1 != NULL && extent2 != NULL); 27 mutex_pool_lock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1, 28 (uintptr_t)extent2); 29 } 30 31 static inline void 32 extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { 33 assert(extent1 != NULL && extent2 != NULL); 34 mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1, 35 (uintptr_t)extent2); 36 } 37 38 static inline unsigned 39 extent_arena_ind_get(const extent_t *extent) { 40 unsigned arena_ind = (unsigned)((extent->e_bits & 41 EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); 42 assert(arena_ind < MALLOCX_ARENA_LIMIT); 43 44 return arena_ind; 45 } 46 47 static inline arena_t * 48 extent_arena_get(const extent_t *extent) { 49 unsigned arena_ind = extent_arena_ind_get(extent); 50 51 return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE); 52 } 53 54 static inline szind_t 55 extent_szind_get_maybe_invalid(const extent_t *extent) { 56 szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >> 57 EXTENT_BITS_SZIND_SHIFT); 58 assert(szind <= SC_NSIZES); 59 return szind; 60 } 61 62 static inline szind_t 63 extent_szind_get(const extent_t *extent) { 64 szind_t szind = extent_szind_get_maybe_invalid(extent); 65 assert(szind < SC_NSIZES); /* Never call when "invalid". */ 66 return szind; 67 } 68 69 static inline size_t 70 extent_usize_get(const extent_t *extent) { 71 return sz_index2size(extent_szind_get(extent)); 72 } 73 74 static inline unsigned 75 extent_binshard_get(const extent_t *extent) { 76 unsigned binshard = (unsigned)((extent->e_bits & 77 EXTENT_BITS_BINSHARD_MASK) >> EXTENT_BITS_BINSHARD_SHIFT); 78 assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); 79 return binshard; 80 } 81 82 static inline size_t 83 extent_sn_get(const extent_t *extent) { 84 return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >> 85 EXTENT_BITS_SN_SHIFT); 86 } 87 88 static inline extent_state_t 89 extent_state_get(const extent_t *extent) { 90 return (extent_state_t)((extent->e_bits & EXTENT_BITS_STATE_MASK) >> 91 EXTENT_BITS_STATE_SHIFT); 92 } 93 94 static inline bool 95 extent_zeroed_get(const extent_t *extent) { 96 return (bool)((extent->e_bits & EXTENT_BITS_ZEROED_MASK) >> 97 EXTENT_BITS_ZEROED_SHIFT); 98 } 99 100 static inline bool 101 extent_committed_get(const extent_t *extent) { 102 return (bool)((extent->e_bits & EXTENT_BITS_COMMITTED_MASK) >> 103 EXTENT_BITS_COMMITTED_SHIFT); 104 } 105 106 static inline bool 107 extent_dumpable_get(const extent_t *extent) { 108 return (bool)((extent->e_bits & EXTENT_BITS_DUMPABLE_MASK) >> 109 EXTENT_BITS_DUMPABLE_SHIFT); 110 } 111 112 static inline bool 113 extent_slab_get(const extent_t *extent) { 114 return (bool)((extent->e_bits & EXTENT_BITS_SLAB_MASK) >> 115 EXTENT_BITS_SLAB_SHIFT); 116 } 117 118 static inline unsigned 119 extent_nfree_get(const extent_t *extent) { 120 assert(extent_slab_get(extent)); 121 return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK) >> 122 EXTENT_BITS_NFREE_SHIFT); 123 } 124 125 static inline void * 126 extent_base_get(const extent_t *extent) { 127 assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || 128 !extent_slab_get(extent)); 129 return PAGE_ADDR2BASE(extent->e_addr); 130 } 131 132 static inline void * 133 extent_addr_get(const extent_t *extent) { 134 assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || 135 !extent_slab_get(extent)); 136 return extent->e_addr; 137 } 138 139 static inline size_t 140 extent_size_get(const extent_t *extent) { 141 return (extent->e_size_esn & EXTENT_SIZE_MASK); 142 } 143 144 static inline size_t 145 extent_esn_get(const extent_t *extent) { 146 return (extent->e_size_esn & EXTENT_ESN_MASK); 147 } 148 149 static inline size_t 150 extent_bsize_get(const extent_t *extent) { 151 return extent->e_bsize; 152 } 153 154 static inline void * 155 extent_before_get(const extent_t *extent) { 156 return (void *)((uintptr_t)extent_base_get(extent) - PAGE); 157 } 158 159 static inline void * 160 extent_last_get(const extent_t *extent) { 161 return (void *)((uintptr_t)extent_base_get(extent) + 162 extent_size_get(extent) - PAGE); 163 } 164 165 static inline void * 166 extent_past_get(const extent_t *extent) { 167 return (void *)((uintptr_t)extent_base_get(extent) + 168 extent_size_get(extent)); 169 } 170 171 static inline arena_slab_data_t * 172 extent_slab_data_get(extent_t *extent) { 173 assert(extent_slab_get(extent)); 174 return &extent->e_slab_data; 175 } 176 177 static inline const arena_slab_data_t * 178 extent_slab_data_get_const(const extent_t *extent) { 179 assert(extent_slab_get(extent)); 180 return &extent->e_slab_data; 181 } 182 183 static inline prof_tctx_t * 184 extent_prof_tctx_get(const extent_t *extent) { 185 return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx, 186 ATOMIC_ACQUIRE); 187 } 188 189 static inline nstime_t 190 extent_prof_alloc_time_get(const extent_t *extent) { 191 return extent->e_alloc_time; 192 } 193 194 static inline void 195 extent_arena_set(extent_t *extent, arena_t *arena) { 196 unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U << 197 MALLOCX_ARENA_BITS) - 1); 198 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ARENA_MASK) | 199 ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT); 200 } 201 202 static inline void 203 extent_binshard_set(extent_t *extent, unsigned binshard) { 204 /* The assertion assumes szind is set already. */ 205 assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); 206 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_BINSHARD_MASK) | 207 ((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT); 208 } 209 210 static inline void 211 extent_addr_set(extent_t *extent, void *addr) { 212 extent->e_addr = addr; 213 } 214 215 static inline void 216 extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) { 217 assert(extent_base_get(extent) == extent_addr_get(extent)); 218 219 if (alignment < PAGE) { 220 unsigned lg_range = LG_PAGE - 221 lg_floor(CACHELINE_CEILING(alignment)); 222 size_t r; 223 if (!tsdn_null(tsdn)) { 224 tsd_t *tsd = tsdn_tsd(tsdn); 225 r = (size_t)prng_lg_range_u64( 226 tsd_offset_statep_get(tsd), lg_range); 227 } else { 228 r = prng_lg_range_zu( 229 &extent_arena_get(extent)->offset_state, 230 lg_range, true); 231 } 232 uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE - 233 lg_range); 234 extent->e_addr = (void *)((uintptr_t)extent->e_addr + 235 random_offset); 236 assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) == 237 extent->e_addr); 238 } 239 } 240 241 static inline void 242 extent_size_set(extent_t *extent, size_t size) { 243 assert((size & ~EXTENT_SIZE_MASK) == 0); 244 extent->e_size_esn = size | (extent->e_size_esn & ~EXTENT_SIZE_MASK); 245 } 246 247 static inline void 248 extent_esn_set(extent_t *extent, size_t esn) { 249 extent->e_size_esn = (extent->e_size_esn & ~EXTENT_ESN_MASK) | (esn & 250 EXTENT_ESN_MASK); 251 } 252 253 static inline void 254 extent_bsize_set(extent_t *extent, size_t bsize) { 255 extent->e_bsize = bsize; 256 } 257 258 static inline void 259 extent_szind_set(extent_t *extent, szind_t szind) { 260 assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */ 261 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) | 262 ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT); 263 } 264 265 static inline void 266 extent_nfree_set(extent_t *extent, unsigned nfree) { 267 assert(extent_slab_get(extent)); 268 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) | 269 ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); 270 } 271 272 static inline void 273 extent_nfree_binshard_set(extent_t *extent, unsigned nfree, unsigned binshard) { 274 /* The assertion assumes szind is set already. */ 275 assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); 276 extent->e_bits = (extent->e_bits & 277 (~EXTENT_BITS_NFREE_MASK & ~EXTENT_BITS_BINSHARD_MASK)) | 278 ((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT) | 279 ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); 280 } 281 282 static inline void 283 extent_nfree_inc(extent_t *extent) { 284 assert(extent_slab_get(extent)); 285 extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); 286 } 287 288 static inline void 289 extent_nfree_dec(extent_t *extent) { 290 assert(extent_slab_get(extent)); 291 extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); 292 } 293 294 static inline void 295 extent_nfree_sub(extent_t *extent, uint64_t n) { 296 assert(extent_slab_get(extent)); 297 extent->e_bits -= (n << EXTENT_BITS_NFREE_SHIFT); 298 } 299 300 static inline void 301 extent_sn_set(extent_t *extent, size_t sn) { 302 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | 303 ((uint64_t)sn << EXTENT_BITS_SN_SHIFT); 304 } 305 306 static inline void 307 extent_state_set(extent_t *extent, extent_state_t state) { 308 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_STATE_MASK) | 309 ((uint64_t)state << EXTENT_BITS_STATE_SHIFT); 310 } 311 312 static inline void 313 extent_zeroed_set(extent_t *extent, bool zeroed) { 314 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ZEROED_MASK) | 315 ((uint64_t)zeroed << EXTENT_BITS_ZEROED_SHIFT); 316 } 317 318 static inline void 319 extent_committed_set(extent_t *extent, bool committed) { 320 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_COMMITTED_MASK) | 321 ((uint64_t)committed << EXTENT_BITS_COMMITTED_SHIFT); 322 } 323 324 static inline void 325 extent_dumpable_set(extent_t *extent, bool dumpable) { 326 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_DUMPABLE_MASK) | 327 ((uint64_t)dumpable << EXTENT_BITS_DUMPABLE_SHIFT); 328 } 329 330 static inline void 331 extent_slab_set(extent_t *extent, bool slab) { 332 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SLAB_MASK) | 333 ((uint64_t)slab << EXTENT_BITS_SLAB_SHIFT); 334 } 335 336 static inline void 337 extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { 338 atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE); 339 } 340 341 static inline void 342 extent_prof_alloc_time_set(extent_t *extent, nstime_t t) { 343 nstime_copy(&extent->e_alloc_time, &t); 344 } 345 346 static inline bool 347 extent_is_head_get(extent_t *extent) { 348 if (maps_coalesce) { 349 not_reached(); 350 } 351 352 return (bool)((extent->e_bits & EXTENT_BITS_IS_HEAD_MASK) >> 353 EXTENT_BITS_IS_HEAD_SHIFT); 354 } 355 356 static inline void 357 extent_is_head_set(extent_t *extent, bool is_head) { 358 if (maps_coalesce) { 359 not_reached(); 360 } 361 362 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_IS_HEAD_MASK) | 363 ((uint64_t)is_head << EXTENT_BITS_IS_HEAD_SHIFT); 364 } 365 366 static inline void 367 extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, 368 bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, 369 bool committed, bool dumpable, extent_head_state_t is_head) { 370 assert(addr == PAGE_ADDR2BASE(addr) || !slab); 371 372 extent_arena_set(extent, arena); 373 extent_addr_set(extent, addr); 374 extent_size_set(extent, size); 375 extent_slab_set(extent, slab); 376 extent_szind_set(extent, szind); 377 extent_sn_set(extent, sn); 378 extent_state_set(extent, state); 379 extent_zeroed_set(extent, zeroed); 380 extent_committed_set(extent, committed); 381 extent_dumpable_set(extent, dumpable); 382 ql_elm_new(extent, ql_link); 383 if (!maps_coalesce) { 384 extent_is_head_set(extent, (is_head == EXTENT_IS_HEAD) ? true : 385 false); 386 } 387 if (config_prof) { 388 extent_prof_tctx_set(extent, NULL); 389 } 390 } 391 392 static inline void 393 extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { 394 extent_arena_set(extent, NULL); 395 extent_addr_set(extent, addr); 396 extent_bsize_set(extent, bsize); 397 extent_slab_set(extent, false); 398 extent_szind_set(extent, SC_NSIZES); 399 extent_sn_set(extent, sn); 400 extent_state_set(extent, extent_state_active); 401 extent_zeroed_set(extent, true); 402 extent_committed_set(extent, true); 403 extent_dumpable_set(extent, true); 404 } 405 406 static inline void 407 extent_list_init(extent_list_t *list) { 408 ql_new(list); 409 } 410 411 static inline extent_t * 412 extent_list_first(const extent_list_t *list) { 413 return ql_first(list); 414 } 415 416 static inline extent_t * 417 extent_list_last(const extent_list_t *list) { 418 return ql_last(list, ql_link); 419 } 420 421 static inline void 422 extent_list_append(extent_list_t *list, extent_t *extent) { 423 ql_tail_insert(list, extent, ql_link); 424 } 425 426 static inline void 427 extent_list_prepend(extent_list_t *list, extent_t *extent) { 428 ql_head_insert(list, extent, ql_link); 429 } 430 431 static inline void 432 extent_list_replace(extent_list_t *list, extent_t *to_remove, 433 extent_t *to_insert) { 434 ql_after_insert(to_remove, to_insert, ql_link); 435 ql_remove(list, to_remove, ql_link); 436 } 437 438 static inline void 439 extent_list_remove(extent_list_t *list, extent_t *extent) { 440 ql_remove(list, extent, ql_link); 441 } 442 443 static inline int 444 extent_sn_comp(const extent_t *a, const extent_t *b) { 445 size_t a_sn = extent_sn_get(a); 446 size_t b_sn = extent_sn_get(b); 447 448 return (a_sn > b_sn) - (a_sn < b_sn); 449 } 450 451 static inline int 452 extent_esn_comp(const extent_t *a, const extent_t *b) { 453 size_t a_esn = extent_esn_get(a); 454 size_t b_esn = extent_esn_get(b); 455 456 return (a_esn > b_esn) - (a_esn < b_esn); 457 } 458 459 static inline int 460 extent_ad_comp(const extent_t *a, const extent_t *b) { 461 uintptr_t a_addr = (uintptr_t)extent_addr_get(a); 462 uintptr_t b_addr = (uintptr_t)extent_addr_get(b); 463 464 return (a_addr > b_addr) - (a_addr < b_addr); 465 } 466 467 static inline int 468 extent_ead_comp(const extent_t *a, const extent_t *b) { 469 uintptr_t a_eaddr = (uintptr_t)a; 470 uintptr_t b_eaddr = (uintptr_t)b; 471 472 return (a_eaddr > b_eaddr) - (a_eaddr < b_eaddr); 473 } 474 475 static inline int 476 extent_snad_comp(const extent_t *a, const extent_t *b) { 477 int ret; 478 479 ret = extent_sn_comp(a, b); 480 if (ret != 0) { 481 return ret; 482 } 483 484 ret = extent_ad_comp(a, b); 485 return ret; 486 } 487 488 static inline int 489 extent_esnead_comp(const extent_t *a, const extent_t *b) { 490 int ret; 491 492 ret = extent_esn_comp(a, b); 493 if (ret != 0) { 494 return ret; 495 } 496 497 ret = extent_ead_comp(a, b); 498 return ret; 499 } 500 501 #endif /* JEMALLOC_INTERNAL_EXTENT_INLINES_H */ 502