1 /* 2 * %CopyrightBegin% 3 * 4 * Copyright Ericsson AB 2002-2020. All Rights Reserved. 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * %CopyrightEnd% 19 */ 20 21 #ifndef ERL_ALLOC_UTIL__ 22 #define ERL_ALLOC_UTIL__ 23 24 #define ERTS_ALCU_VSN_STR "3.0" 25 26 #include "erl_alloc_types.h" 27 #include "erl_alloc.h" 28 #define ERL_THREADS_EMU_INTERNAL__ 29 #include "erl_threads.h" 30 31 #include "erl_mseg.h" 32 #include "lttng-wrapper.h" 33 34 #define ERTS_AU_PREF_ALLOC_BITS 11 35 #define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS) 36 37 typedef struct Allctr_t_ Allctr_t; 38 39 typedef struct { 40 UWord ycs; 41 UWord mmc; 42 int sac; 43 } AlcUInit_t; 44 45 typedef struct { 46 char *name_prefix; 47 ErtsAlcType_t alloc_no; 48 ErtsAlcStrat_t alloc_strat; 49 int force; 50 int ix; 51 int ts; 52 int tspec; 53 int tpref; 54 int ramv; 55 int atags; 56 int cp; 57 UWord sbct; 58 UWord asbcst; 59 UWord rsbcst; 60 UWord rsbcmt; 61 UWord rmbcmt; 62 UWord mmbcs; 63 UWord mmsbc; 64 UWord mmmbc; 65 UWord lmbcs; 66 UWord smbcs; 67 UWord mbcgs; 68 UWord acul; 69 UWord acnl; 70 UWord acfml; 71 72 void *fix; 73 size_t *fix_type_size; 74 75 #if HAVE_ERTS_MSEG 76 void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags); 77 void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); 78 void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); 79 ErtsMemMapper *mseg_mmapper; 80 #endif 81 void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign); 82 void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); 83 void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); 84 } AllctrInit_t; 85 86 typedef struct { 87 UWord blocks; 88 UWord carriers; 89 } AllctrSize_t; 90 91 typedef struct { 92 UWord allocated; 93 UWord used; 94 } ErtsAlcUFixInfo_t; 95 96 #ifndef SMALL_MEMORY 97 98 #define ERTS_DEFAULT_ALCU_INIT { \ 99 1024*1024, /* (bytes) ycs: sys_alloc carrier size */\ 100 ~((UWord) 0), /* (amount) mmc: max mseg carriers */\ 101 1 /* (bool) sac: sys_alloc carriers */\ 102 } 103 104 #define ERTS_DEFAULT_ALLCTR_INIT { \ 105 NULL, \ 106 ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ 107 ERTS_ALC_S_INVALID, /* (number) alloc_strat: allocator strategy */\ 108 0, /* (bool) force: force enabled */\ 109 0, /* (number) ix: instance index */\ 110 1, /* (bool) ts: thread safe */\ 111 0, /* (bool) tspec: thread specific */\ 112 0, /* (bool) tpref: thread preferred */\ 113 0, /* (bool) ramv: realloc always moves */\ 114 0, /* (bool) atags: tagged allocations */\ 115 -1, /* (ix) cp: carrier pool */\ 116 512*1024, /* (bytes) sbct: sbc threshold */\ 117 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 118 20, /* (%) rsbcst: rel sbc shrink threshold */\ 119 80, /* (%) rsbcmt: rel sbc move threshold */\ 120 50, /* (%) rmbcmt: rel mbc move threshold */\ 121 1024*1024, /* (bytes) mmbcs: main multiblock carrier size */\ 122 256, /* (amount) mmsbc: max mseg sbcs */\ 123 ~((UWord) 0), /* (amount) mmmbc: max mseg mbcs */ \ 124 10*1024*1024, /* (bytes) lmbcs: largest mbc size */\ 125 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 126 10, /* (amount) mbcgs: mbc growth stages */\ 127 0, /* (%) acul: abandon carrier utilization limit */\ 128 1000, /* (amount) acnl: abandoned carriers number limit */\ 129 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ 130 /* --- Data not options -------------------------------------------- */\ 131 NULL, /* (ptr) fix */\ 132 NULL /* (ptr) fix_type_size */\ 133 } 134 135 #else /* if SMALL_MEMORY */ 136 137 #define ERTS_DEFAULT_ALCU_INIT { \ 138 128*1024, /* (bytes) ycs: sys_alloc carrier size */\ 139 1024, /* (amount) mmc: max mseg carriers */\ 140 1 /* (bool) sac: sys_alloc carriers */\ 141 } 142 143 #define ERTS_DEFAULT_ALLCTR_INIT { \ 144 NULL, \ 145 ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ 146 ERTS_ALC_S_INVALID, /* (number) alloc_strat: allocator strategy */\ 147 0, /* (bool) force: force enabled */\ 148 0, /* (number) ix: instance index */\ 149 1, /* (bool) ts: thread safe */\ 150 0, /* (bool) tspec: thread specific */\ 151 0, /* (bool) tpref: thread preferred */\ 152 0, /* (bool) ramv: realloc always moves */\ 153 0, /* (bool) atags: tagged allocations */\ 154 -1, /* (ix) cp: carrier pool */\ 155 64*1024, /* (bytes) sbct: sbc threshold */\ 156 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 157 20, /* (%) rsbcst: rel sbc shrink threshold */\ 158 80, /* (%) rsbcmt: rel sbc move threshold */\ 159 50, /* (%) rmbcmt: rel mbc move threshold */\ 160 128*1024, /* (bytes) mmbcs: main multiblock carrier size */\ 161 256, /* (amount) mmsbc: max mseg sbcs */\ 162 ~((UWord) 0), /* (amount) mmmbc: max mseg mbcs */ \ 163 1024*1024, /* (bytes) lmbcs: largest mbc size */\ 164 128*1024, /* (bytes) smbcs: smallest mbc size */\ 165 10, /* (amount) mbcgs: mbc growth stages */\ 166 0, /* (%) acul: abandon carrier utilization limit */\ 167 1000, /* (amount) acnl: abandoned carriers number limit */\ 168 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ 169 /* --- Data not options -------------------------------------------- */\ 170 NULL, /* (ptr) fix */\ 171 NULL /* (ptr) fix_type_size */\ 172 } 173 174 #endif 175 176 void * erts_alcu_alloc(ErtsAlcType_t, void *, Uint); 177 void * erts_alcu_realloc(ErtsAlcType_t, void *, void *, Uint); 178 void * erts_alcu_realloc_mv(ErtsAlcType_t, void *, void *, Uint); 179 void erts_alcu_free(ErtsAlcType_t, void *, void *); 180 void * erts_alcu_alloc_ts(ErtsAlcType_t, void *, Uint); 181 void * erts_alcu_realloc_ts(ErtsAlcType_t, void *, void *, Uint); 182 void * erts_alcu_realloc_mv_ts(ErtsAlcType_t, void *, void *, Uint); 183 void erts_alcu_free_ts(ErtsAlcType_t, void *, void *); 184 void * erts_alcu_alloc_thr_spec(ErtsAlcType_t, void *, Uint); 185 void * erts_alcu_realloc_thr_spec(ErtsAlcType_t, void *, void *, Uint); 186 void * erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t, void *, void *, Uint); 187 void erts_alcu_free_thr_spec(ErtsAlcType_t, void *, void *); 188 void * erts_alcu_alloc_thr_pref(ErtsAlcType_t, void *, Uint); 189 void * erts_alcu_realloc_thr_pref(ErtsAlcType_t, void *, void *, Uint); 190 void * erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t, void *, void *, Uint); 191 void erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *); 192 Eterm erts_alcu_au_info_options(fmtfn_t *, void *, Uint **, Uint *); 193 Eterm erts_alcu_info_options(Allctr_t *, fmtfn_t *, void *, Uint **, Uint *); 194 Eterm erts_alcu_sz_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *); 195 Eterm erts_alcu_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *); 196 void erts_alcu_init(AlcUInit_t *); 197 void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, 198 ErtsAlcUFixInfo_t *, int); 199 void erts_alcu_foreign_size(Allctr_t *, ErtsAlcType_t, AllctrSize_t *); 200 void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *); 201 erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); 202 203 #ifdef ARCH_32 204 extern UWord erts_literal_vspace_map[]; 205 # define ERTS_VSPACE_WORD_BITS (sizeof(UWord)*8) 206 #endif 207 208 #if HAVE_ERTS_MSEG 209 # if defined(ARCH_32) 210 void* erts_alcu_literal_32_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); 211 void* erts_alcu_literal_32_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); 212 void erts_alcu_literal_32_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); 213 214 # elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) 215 void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); 216 void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); 217 void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); 218 # endif 219 220 # if defined(ERTS_ALC_A_EXEC) 221 void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); 222 void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); 223 void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); 224 # endif 225 #endif /* HAVE_ERTS_MSEG */ 226 227 #ifdef ARCH_32 228 void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint *size_p, int superalign); 229 void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign); 230 void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); 231 #endif 232 233 #ifdef ERTS_ENABLE_LOCK_COUNT 234 void erts_lcnt_update_allocator_locks(int enable); 235 #endif 236 237 int erts_alcu_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); 238 239 /* Gathers per-tag allocation histograms from the given allocator number 240 * (ERTS_ALC_A_*) and scheduler id. An id of 0 means the global instance will 241 * be used. 242 * 243 * The results are sent to `p`, and it returns the number of messages to wait 244 * for. */ 245 int erts_alcu_gather_alloc_histograms(struct process *p, int allocator_num, 246 int sched_id, int hist_width, 247 UWord hist_start, Eterm ref); 248 249 /* Gathers per-carrier info from the given allocator number (ERTS_ALC_A_*) and 250 * scheduler id. An id of 0 means the global instance will be used. 251 * 252 * The results are sent to `p`, and it returns the number of messages to wait 253 * for. */ 254 int erts_alcu_gather_carrier_info(struct process *p, int allocator_num, 255 int sched_id, int hist_width, 256 UWord hist_start, Eterm ref); 257 258 struct alcu_blockscan; 259 260 typedef struct { 261 struct alcu_blockscan *current; 262 struct alcu_blockscan *last; 263 } ErtsAlcuBlockscanYieldData; 264 265 int erts_handle_yielded_alcu_blockscan(struct ErtsSchedulerData_ *esdp, 266 ErtsAlcuBlockscanYieldData *yield); 267 void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp); 268 269 #endif /* !ERL_ALLOC_UTIL__ */ 270 271 #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) 272 #define ERL_ALLOC_UTIL_IMPL__ 273 274 #define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE (((Uint32) 1) << 0) 275 276 #undef ERTS_ALLOC_UTIL_HARD_DEBUG 277 #ifdef DEBUG 278 # if 0 279 # define ERTS_ALLOC_UTIL_HARD_DEBUG 280 # endif 281 #endif 282 283 #define FLOOR(X, I) (((X)/(I))*(I)) 284 #define CEILING(X, I) ((((X) - 1)/(I) + 1)*(I)) 285 286 #undef WORD_MASK 287 #define INV_WORD_MASK ((UWord) (sizeof(UWord) - 1)) 288 #define WORD_MASK (~INV_WORD_MASK) 289 #define WORD_FLOOR(X) ((X) & WORD_MASK) 290 #define WORD_CEILING(X) WORD_FLOOR((X) + INV_WORD_MASK) 291 292 #undef UNIT_MASK 293 #define INV_UNIT_MASK ((UWord) (sizeof(Unit_t) - 1)) 294 #define UNIT_MASK (~INV_UNIT_MASK) 295 #define UNIT_FLOOR(X) ((X) & UNIT_MASK) 296 #define UNIT_CEILING(X) UNIT_FLOOR((X) + INV_UNIT_MASK) 297 298 /* We store flags in the bits that no one will ever use. Generally these are 299 * the bits below the alignment size, but for blocks we also steal the highest 300 * bit since the header's a size and no one can expect to be able to allocate 301 * objects that large. */ 302 #define HIGHEST_WORD_BIT (((UWord) 1) << (sizeof(UWord) * CHAR_BIT - 1)) 303 304 #define BLK_FLG_MASK (INV_UNIT_MASK | HIGHEST_WORD_BIT) 305 #define SBC_BLK_SZ_MASK (~BLK_FLG_MASK) 306 #define MBC_FBLK_SZ_MASK (~BLK_FLG_MASK) 307 308 #define CRR_FLG_MASK INV_UNIT_MASK 309 #define CRR_SZ_MASK UNIT_MASK 310 311 #if ERTS_HAVE_MSEG_SUPER_ALIGNED \ 312 || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC) 313 # ifdef MSEG_ALIGN_BITS 314 # define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS 315 # else 316 # define ERTS_SUPER_ALIGN_BITS 18 317 # endif 318 # ifdef ARCH_64 319 # define MBC_ABLK_OFFSET_BITS 23 320 # else 321 # define MBC_ABLK_OFFSET_BITS 8 322 /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ 323 # endif 324 # define ERTS_SACRR_UNIT_SHIFT ERTS_SUPER_ALIGN_BITS 325 # define ERTS_SACRR_UNIT_SZ (1 << ERTS_SACRR_UNIT_SHIFT) 326 # define ERTS_SACRR_UNIT_MASK ((~(UWord)0) << ERTS_SACRR_UNIT_SHIFT) 327 # define ERTS_SACRR_UNIT_FLOOR(X) ((X) & ERTS_SACRR_UNIT_MASK) 328 # define ERTS_SACRR_UNIT_CEILING(X) ERTS_SACRR_UNIT_FLOOR((X) + ~ERTS_SACRR_UNIT_MASK) 329 # define ERTS_SA_MB_CARRIERS 1 330 #else 331 # define ERTS_SA_MB_CARRIERS 0 332 # define MBC_ABLK_OFFSET_BITS 0 /* no carrier offset in block header */ 333 #endif 334 #if ERTS_HAVE_MSEG_SUPER_ALIGNED && !ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 335 # define ERTS_SUPER_ALIGNED_MSEG_ONLY 1 336 #else 337 # define ERTS_SUPER_ALIGNED_MSEG_ONLY 0 338 #endif 339 340 #if MBC_ABLK_OFFSET_BITS 341 /* The shift is reduced by 1 since the highest bit is used for a flag. */ 342 # define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - 1 - MBC_ABLK_OFFSET_BITS) 343 # define MBC_ABLK_OFFSET_MASK \ 344 (((UWORD_CONSTANT(1) << MBC_ABLK_OFFSET_BITS) - UWORD_CONSTANT(1)) \ 345 << MBC_ABLK_OFFSET_SHIFT) 346 # define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~BLK_FLG_MASK) 347 #else 348 # define MBC_ABLK_SZ_MASK (~BLK_FLG_MASK) 349 #endif 350 351 #define MBC_ABLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) 352 #define MBC_FBLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK) 353 #define SBC_BLK_SZ(B) (ASSERT(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) 354 355 #define CARRIER_SZ(C) ((C)->chdr & CRR_SZ_MASK) 356 357 typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; 358 359 typedef struct Carrier_t_ Carrier_t; 360 361 typedef struct { 362 UWord bhdr; 363 #if !MBC_ABLK_OFFSET_BITS 364 Carrier_t *carrier; 365 #else 366 union { 367 Carrier_t *carrier; /* if free */ 368 char udata__[1]; /* if allocated */ 369 }u; 370 #endif 371 } Block_t; 372 373 typedef struct ErtsAllctrDDBlock__ { 374 union { 375 struct ErtsAllctrDDBlock__ *ptr_next; 376 erts_atomic_t atmc_next; 377 } u; 378 ErtsAlcType_t type; 379 Uint32 flags; 380 } ErtsAllctrDDBlock_t; 381 382 /* Deallocation was caused by shrinking a fix-list, so usage statistics has 383 * already been updated. */ 384 #define DEALLOC_FLG_FIX_SHRINK (1 << 0) 385 /* Deallocation was redirected to another instance. */ 386 #define DEALLOC_FLG_REDIRECTED (1 << 1) 387 388 typedef struct { 389 Block_t blk; 390 #if !MBC_ABLK_OFFSET_BITS 391 ErtsAllctrDDBlock_t umem_; 392 #endif 393 } ErtsFakeDDBlock_t; 394 395 #define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) 396 #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) 397 #define LAST_BLK_HDR_FLG (((UWord) 1) << 2) 398 #define ATAG_BLK_HDR_FLG HIGHEST_WORD_BIT 399 400 #define SBC_BLK_HDR_FLG /* Special flag combo for (allocated) SBC blocks */\ 401 (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) 402 403 /* 404 * HOMECOMING_MBC_BLK_HDR is a special block header combo used for 405 * distinguishing MBC's from allocated blocks in handle_delayed_dealloc(). 406 */ 407 #define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) 408 409 #define IS_FREE_LAST_MBC_BLK(B) \ 410 (((B)->bhdr & BLK_FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)) 411 412 #define IS_SBC_BLK(B) (((B)->bhdr & SBC_BLK_HDR_FLG) == SBC_BLK_HDR_FLG) 413 #define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) 414 #define IS_FREE_BLK(B) (ASSERT(IS_MBC_BLK(B)), \ 415 (B)->bhdr & THIS_FREE_BLK_HDR_FLG) 416 417 #if MBC_ABLK_OFFSET_BITS 418 # define FBLK_TO_MBC(B) (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \ 419 (B)->u.carrier) 420 # define ABLK_TO_MBC(B) \ 421 (ASSERT(IS_MBC_BLK(B) && !IS_FREE_BLK(B)), \ 422 (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \ 423 ((((B)->bhdr & ~BLK_FLG_MASK) >> MBC_ABLK_OFFSET_SHIFT) \ 424 << ERTS_SACRR_UNIT_SHIFT)))) 425 # define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) 426 #else 427 # define FBLK_TO_MBC(B) ((B)->carrier) 428 # define ABLK_TO_MBC(B) ((B)->carrier) 429 # define BLK_TO_MBC(B) ((B)->carrier) 430 #endif 431 #define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B)) 432 433 typedef UWord FreeBlkFtr_t; /* Footer of a free block */ 434 435 /* This AOFF stuff really belong in erl_ao_firstfit_alloc.h */ 436 typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; 437 struct AOFF_RBTree_t_ { 438 Block_t hdr; 439 AOFF_RBTree_t *parent; 440 AOFF_RBTree_t *left; 441 AOFF_RBTree_t *right; 442 Uint32 flags; 443 Uint32 max_sz; /* of all blocks in this sub-tree */ 444 union { 445 AOFF_RBTree_t* next; /* for best fit */ 446 Sint64 birth_time; /* for age first fit */ 447 } u; 448 }; 449 450 #if ERTS_ALC_A_INVALID != 0 451 # error "Carrier pool implementation assumes ERTS_ALC_A_INVALID == 0" 452 #endif 453 #if ERTS_ALC_A_MIN <= ERTS_ALC_A_INVALID 454 # error "Carrier pool implementation assumes ERTS_ALC_A_MIN > ERTS_ALC_A_INVALID" 455 #endif 456 457 /* The pools are only allowed to be manipulated by managed threads except in 458 * the alloc_SUITE:cpool test, where only ERTS_ALC_TEST_CPOOL_IX pool is used. */ 459 460 #define ERTS_ALC_TEST_CPOOL_IX ERTS_ALC_A_INVALID 461 /* 462 * System is not an alloc_util allocator, so we use its slot for 463 * the common cpool... 464 */ 465 #define ERTS_ALC_COMMON_CPOOL_IX ERTS_ALC_A_SYSTEM 466 #define ERTS_ALC_NO_CPOOLS (ERTS_ALC_A_MAX+1) 467 468 void aoff_add_pooled_mbc(Allctr_t*, Carrier_t*); 469 void aoff_remove_pooled_mbc(Allctr_t*, Carrier_t*); 470 Carrier_t* aoff_lookup_pooled_mbc(Allctr_t*, Uint size); 471 void erts_aoff_larger_max_size(AOFF_RBTree_t *node); 472 473 typedef struct { 474 ErtsFakeDDBlock_t homecoming_dd; 475 erts_atomic_t next; 476 erts_atomic_t prev; 477 Allctr_t *orig_allctr; /* read-only while carrier is alive */ 478 ErtsThrPrgrVal thr_prgr; 479 erts_atomic_t max_size; 480 UWord abandon_limit; 481 UWord blocks[ERTS_ALC_A_COUNT]; 482 UWord blocks_size[ERTS_ALC_A_COUNT]; 483 UWord total_blocks_size; 484 enum { 485 ERTS_MBC_IS_HOME, 486 ERTS_MBC_WAS_POOLED, 487 ERTS_MBC_WAS_TRAITOR 488 } state; 489 AOFF_RBTree_t pooled; /* node in pooled_tree */ 490 } ErtsAlcCPoolData_t; 491 492 struct Carrier_t_ { 493 UWord chdr; 494 Carrier_t *next; 495 Carrier_t *prev; 496 erts_atomic_t allctr; 497 ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ 498 }; 499 500 #define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ 501 ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~CRR_FLG_MASK)) 502 503 typedef struct { 504 Carrier_t *first; 505 Carrier_t *last; 506 } CarrierList_t; 507 508 509 typedef Uint64 CallCounter_t; 510 511 typedef struct { 512 UWord no; 513 UWord size; 514 } StatValues_t; 515 516 typedef struct { 517 StatValues_t curr; 518 StatValues_t max; 519 StatValues_t max_ever; 520 } BlockStats_t; 521 522 enum { 523 ERTS_CRR_ALLOC_MIN = 0, 524 525 ERTS_CRR_ALLOC_MSEG = ERTS_CRR_ALLOC_MIN, 526 ERTS_CRR_ALLOC_SYS = 1, 527 528 ERTS_CRR_ALLOC_MAX, 529 ERTS_CRR_ALLOC_COUNT = ERTS_CRR_ALLOC_MAX + 1 530 }; 531 532 typedef struct { 533 StatValues_t carriers[ERTS_CRR_ALLOC_COUNT]; 534 535 StatValues_t max; 536 StatValues_t max_ever; 537 538 BlockStats_t blocks[ERTS_ALC_A_COUNT]; 539 } CarriersStats_t; 540 541 #ifdef USE_LTTNG_VM_TRACEPOINTS 542 #define LTTNG_CARRIER_STATS_TO_LTTNG_STATS(CSP, LSP) \ 543 do { \ 544 UWord no_sum__, size_sum__; \ 545 int alloc_no__, i__; \ 546 /* Carrier counters */ \ 547 no_sum__ = size_sum__ = 0; \ 548 for (i__ = ERTS_CRR_ALLOC_MIN; i__ <= ERTS_CRR_ALLOC_MAX; i__++) { \ 549 StatValues_t *curr__ = &((CSP)->carriers[i__]); \ 550 no_sum__ += curr__->no; \ 551 size_sum__ += curr__->size; \ 552 } \ 553 (LSP)->carriers.size = size_sum__; \ 554 (LSP)->carriers.no = no_sum__; \ 555 /* Block counters */ \ 556 no_sum__ = size_sum__ = 0; \ 557 for (alloc_no__ = ERTS_ALC_A_MIN; \ 558 alloc_no__ <= ERTS_ALC_A_MAX; \ 559 alloc_no__++) { \ 560 StatValues_t *curr__; \ 561 i__ = alloc_no__ - ERTS_ALC_A_MIN; \ 562 curr__ = &((CSP)->blocks[i__].curr); \ 563 no_sum__ += curr__->no; \ 564 size_sum__ += curr__->size; \ 565 } \ 566 (LSP)->blocks.size = size_sum__; \ 567 (LSP)->blocks.no = no_sum__; \ 568 } while (0) 569 #endif 570 571 572 typedef struct { 573 ErtsAllctrDDBlock_t marker; 574 erts_atomic_t last; 575 erts_atomic_t um_refc[2]; 576 erts_atomic32_t um_refc_ix; 577 } ErtsDDTail_t; 578 579 typedef struct { 580 /* 581 * This structure needs to be cache line aligned for best 582 * performance. 583 */ 584 union { 585 /* Modified by threads returning memory to this allocator */ 586 ErtsDDTail_t data; 587 char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsDDTail_t))]; 588 } tail; 589 /* 590 * Everything below this point is *only* accessed by the 591 * thread owning the allocator. 592 */ 593 struct { 594 ErtsAllctrDDBlock_t *first; 595 ErtsAllctrDDBlock_t *unref_end; 596 struct { 597 ErtsThrPrgrVal thr_progress; 598 int thr_progress_reached; 599 int um_refc_ix; 600 ErtsAllctrDDBlock_t *unref_end; 601 } next; 602 int used_marker; 603 } head; 604 } ErtsAllctrDDQueue_t; 605 606 typedef struct { 607 size_t type_size; 608 SWord list_size; 609 void *list; 610 union { 611 struct { 612 SWord max_used; 613 SWord limit; 614 SWord allocated; 615 SWord used; 616 } nocpool; 617 struct { 618 int min_list_size; 619 int shrink_list; 620 UWord allocated; 621 UWord used; 622 } cpool; 623 } u; 624 ErtsAlcType_t type; 625 } ErtsAlcFixList_t; 626 627 struct Allctr_t_ { 628 struct { 629 /* 630 * We want the queue at the beginning of 631 * the Allctr_t struct, due to cache line 632 * alignment reasons. 633 */ 634 ErtsAllctrDDQueue_t q; 635 int use; 636 int ix; 637 } dd; 638 639 /* Allocator name prefix */ 640 char * name_prefix; 641 642 /* Allocator number */ 643 ErtsAlcType_t alloc_no; 644 645 /* Allocator strategy */ 646 ErtsAlcStrat_t alloc_strat; 647 648 /* Instance index */ 649 int ix; 650 651 /* Alloc, realloc and free names as atoms */ 652 struct { 653 Eterm alloc; 654 Eterm realloc; 655 Eterm free; 656 } name; 657 658 /* Version string */ 659 char * vsn_str; 660 661 /* Options */ 662 int t; 663 int ramv; 664 int atags; 665 Uint sbc_threshold; 666 Uint sbc_move_threshold; 667 Uint mbc_move_threshold; 668 Uint main_carrier_size; 669 Uint max_mseg_sbcs; 670 Uint max_mseg_mbcs; 671 Uint largest_mbc_size; 672 Uint smallest_mbc_size; 673 Uint mbc_growth_stages; 674 675 #if HAVE_ERTS_MSEG 676 ErtsMsegOpt_t mseg_opt; 677 #endif 678 679 /* */ 680 Uint mbc_header_size; 681 Uint min_mbc_size; 682 Uint min_mbc_first_free_size; 683 Uint min_block_size; 684 UWord crr_set_flgs; 685 UWord crr_clr_flgs; 686 687 /* Carriers *employed* by this allocator */ 688 CarrierList_t mbc_list; 689 CarrierList_t sbc_list; 690 struct { 691 /* pooled_tree and dc_list contain only 692 carriers *created* by this allocator */ 693 AOFF_RBTree_t* pooled_tree; 694 CarrierList_t dc_list; 695 696 /* the sentinel of the cpool we're attached to */ 697 ErtsAlcCPoolData_t *sentinel; 698 699 UWord abandon_limit; 700 int disable_abandon; 701 int check_limit_count; 702 UWord util_limit; /* acul */ 703 UWord in_pool_limit; /* acnl */ 704 UWord fblk_min_limit; /* acmfl */ 705 int carrier_pool; /* cp */ 706 struct { 707 erts_atomic_t blocks_size[ERTS_ALC_A_COUNT]; 708 erts_atomic_t no_blocks[ERTS_ALC_A_COUNT]; 709 erts_atomic_t carriers_size; 710 erts_atomic_t no_carriers; 711 CallCounter_t fail_pooled; 712 CallCounter_t fail_shared; 713 CallCounter_t fail_pend_dealloc; 714 CallCounter_t fail; 715 CallCounter_t fetch; 716 CallCounter_t skip_size; 717 CallCounter_t skip_busy; 718 CallCounter_t skip_not_pooled; 719 CallCounter_t skip_homecoming; 720 CallCounter_t skip_race; 721 CallCounter_t entrance_removed; 722 } stat; 723 } cpool; 724 725 /* Main carrier (if there is one) */ 726 Carrier_t * main_carrier; 727 728 /* Callback functions (first 4 are mandatory) */ 729 Block_t * (*get_free_block) (Allctr_t *, Uint, 730 Block_t *, Uint); 731 void (*link_free_block) (Allctr_t *, Block_t *); 732 void (*unlink_free_block) (Allctr_t *, Block_t *); 733 Eterm (*info_options) (Allctr_t *, char *, fmtfn_t *, 734 void *, Uint **, Uint *); 735 736 Uint (*get_next_mbc_size) (Allctr_t *); 737 void (*creating_mbc) (Allctr_t *, Carrier_t *); 738 void (*destroying_mbc) (Allctr_t *, Carrier_t *); 739 740 /* The five callbacks below are needed to support carrier migration. */ 741 void (*add_mbc) (Allctr_t *, Carrier_t *); 742 void (*remove_mbc) (Allctr_t *, Carrier_t *); 743 UWord (*largest_fblk_in_mbc) (Allctr_t *, Carrier_t *); 744 Block_t * (*first_fblk_in_mbc) (Allctr_t *, Carrier_t *); 745 Block_t * (*next_fblk_in_mbc) (Allctr_t *, Carrier_t *, Block_t *); 746 747 #if HAVE_ERTS_MSEG 748 void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags); 749 void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); 750 void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); 751 ErtsMemMapper *mseg_mmapper; 752 #endif 753 void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign); 754 void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); 755 void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); 756 757 int (*try_set_dyn_param)(Allctr_t*, Eterm param, Uint value); 758 759 void (*init_atoms) (void); 760 761 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG 762 void (*check_block) (Allctr_t *, Block_t *, int); 763 void (*check_mbc) (Allctr_t *, Carrier_t *); 764 #endif 765 766 int fix_n_base; 767 int fix_shrink_scheduled; 768 ErtsAlcFixList_t *fix; 769 770 /* Mutex for this allocator */ 771 erts_mtx_t mutex; 772 int thread_safe; 773 struct { 774 Allctr_t *prev; 775 Allctr_t *next; 776 } ts_list; 777 778 779 int atoms_initialized; 780 781 int stopped; 782 783 /* Some statistics ... */ 784 struct { 785 CallCounter_t this_alloc; 786 CallCounter_t this_free; 787 CallCounter_t this_realloc; 788 CallCounter_t mseg_alloc; 789 CallCounter_t mseg_dealloc; 790 CallCounter_t mseg_realloc; 791 CallCounter_t sys_alloc; 792 CallCounter_t sys_free; 793 CallCounter_t sys_realloc; 794 } calls; 795 796 CarriersStats_t sbcs; 797 CarriersStats_t mbcs; 798 799 #ifdef DEBUG 800 struct { 801 int saved_tid; 802 erts_tid_t tid; 803 } debug; 804 #endif 805 }; 806 807 808 int erts_alcu_start(Allctr_t *, AllctrInit_t *); 809 void erts_alcu_stop(Allctr_t *); 810 811 void erts_alcu_verify_unused(Allctr_t *); 812 void erts_alcu_verify_unused_ts(Allctr_t *allctr); 813 814 UWord erts_alcu_test(UWord, UWord, UWord); 815 816 void erts_alcu_assert_failed(char* expr, char* file, int line, char *func); 817 818 #ifdef DEBUG 819 int is_sbc_blk(Block_t*); 820 #endif 821 822 #endif /* #if defined(GET_ERL_ALLOC_UTIL_IMPL) 823 && !defined(ERL_ALLOC_UTIL_IMPL__) */ 824 825