1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2002-2018. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 #ifndef ERL_ALLOC_H__
22 #define ERL_ALLOC_H__
23
24 #include "erl_alloc_types.h"
25 #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
26 #define ERL_THR_PROGRESS_TSD_TYPE_ONLY
27 #include "erl_thr_progress.h"
28 #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
29 #include "erl_threads.h"
30 #include "erl_mmap.h"
31
32 typedef enum {
33 ERTS_ALC_S_INVALID = 0,
34
35 ERTS_ALC_S_GOODFIT,
36 ERTS_ALC_S_BESTFIT,
37 ERTS_ALC_S_AFIT,
38 ERTS_ALC_S_FIRSTFIT,
39
40 ERTS_ALC_S_MIN = ERTS_ALC_S_GOODFIT,
41 ERTS_ALC_S_MAX = ERTS_ALC_S_FIRSTFIT
42 } ErtsAlcStrat_t;
43
44 #include "erl_alloc_util.h"
45
46 #ifdef DEBUG
47 # undef ERTS_ALC_WANT_INLINE
48 # define ERTS_ALC_WANT_INLINE 0
49 #endif
50
51 #ifndef ERTS_ALC_WANT_INLINE
52 # define ERTS_ALC_WANT_INLINE 1
53 #endif
54
55 #if ERTS_CAN_INLINE && ERTS_ALC_WANT_INLINE
56 # define ERTS_ALC_DO_INLINE 1
57 # define ERTS_ALC_INLINE static ERTS_INLINE
58 # define ERTS_ALC_FORCE_INLINE static ERTS_FORCE_INLINE
59 #else
60 # define ERTS_ALC_DO_INLINE 0
61 # define ERTS_ALC_INLINE
62 # define ERTS_ALC_FORCE_INLINE
63 #endif
64
65 #define ERTS_ALC_NO_FIXED_SIZES \
66 (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1)
67
68 #define ERTS_ALC_IS_FIX_TYPE(T) \
69 (ERTS_ALC_T2N(T) >= ERTS_ALC_N_MIN_A_FIXED_SIZE && \
70 ERTS_ALC_T2N(T) <= ERTS_ALC_N_MAX_A_FIXED_SIZE)
71
72 #define ERTS_ALC_FIX_TYPE_IX(T) \
73 (ASSERT(ERTS_ALC_IS_FIX_TYPE(T)), \
74 ERTS_ALC_T2N((T)) - ERTS_ALC_N_MIN_A_FIXED_SIZE)
75
76 void erts_sys_alloc_init(void);
77 void *erts_sys_alloc(ErtsAlcType_t, void *, Uint);
78 void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint);
79 void erts_sys_free(ErtsAlcType_t, void *, void *);
80 #if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
81 /*
82 * Note 'alignment' must remain the same in calls to
83 * 'erts_sys_aligned_realloc()' and 'erts_sys_aligned_free()'
84 * as in the initial call to 'erts_sys_aligned_alloc()'.
85 */
86 void *erts_sys_aligned_alloc(UWord alignment, UWord size);
87 void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size);
88 void erts_sys_aligned_free(UWord alignment, void *ptr);
89 #endif
90
91 Eterm erts_memory(fmtfn_t *, void *, void *, Eterm);
92 Eterm erts_allocated_areas(fmtfn_t *, void *, void *);
93
94 Eterm erts_alloc_util_allocators(void *proc);
95 void erts_allocator_info(fmtfn_t, void *);
96 Eterm erts_allocator_options(void *proc);
97
98 struct process;
99
100 int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs,
101 int only_sz, int internal);
102
103 #define ERTS_ALLOC_INIT_DEF_OPTS_INITER {0}
104 typedef struct {
105 int ncpu;
106 } ErtsAllocInitOpts;
107
108 typedef struct {
109 Allctr_t *deallctr[ERTS_ALC_A_MAX+1];
110 int pref_ix[ERTS_ALC_A_MAX+1];
111 int flist_ix[ERTS_ALC_A_MAX+1];
112 int pre_alc_ix;
113 } ErtsSchedAllocData;
114
115 void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop);
116 void erts_alloc_late_init(void);
117
118 #if defined(GET_ERTS_ALC_TEST) || defined(ERTS_ALC_INTERNAL__)
119 /* Only for testing */
120 UWord erts_alc_test(UWord,
121 UWord,
122 UWord,
123 UWord);
124 #endif
125
126 #define ERTS_ALC_O_ALLOC 0
127 #define ERTS_ALC_O_REALLOC 1
128 #define ERTS_ALC_O_FREE 2
129
130 #define ERTS_ALC_E_NOTSUP 0
131 #define ERTS_ALC_E_NOMEM 1
132 #define ERTS_ALC_E_NOALLCTR 2
133
134 #define ERTS_ALC_MIN_LONG_LIVED_TIME (10*60*1000)
135
136 typedef struct {
137 int alloc_util;
138 int enabled;
139 int thr_spec;
140 void *extra;
141 } ErtsAllocatorInfo_t;
142
143 typedef struct {
144 void * (*alloc) (ErtsAlcType_t, void *, Uint);
145 void * (*realloc) (ErtsAlcType_t, void *, void *, Uint);
146 void (*free) (ErtsAlcType_t, void *, void *);
147 void *extra;
148 } ErtsAllocatorFunctions_t;
149
150 extern ErtsAllocatorFunctions_t
151 ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]);
152 extern ErtsAllocatorInfo_t
153 ERTS_WRITE_UNLIKELY(erts_allctrs_info[ERTS_ALC_A_MAX+1]);
154
155 typedef struct {
156 int enabled;
157 int dd;
158 int aix;
159 int size;
160 Allctr_t **allctr;
161 } ErtsAllocatorThrSpec_t;
162
163 extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1];
164
165 typedef struct ErtsAllocatorWrapper_t_ {
166 void (*lock)(void);
167 void (*unlock)(void);
168 struct ErtsAllocatorWrapper_t_* next;
169 }ErtsAllocatorWrapper_t;
170 extern ErtsAllocatorWrapper_t *erts_allctr_wrappers;
171 extern int erts_allctr_wrapper_prelocked;
172 extern erts_tsd_key_t erts_allctr_prelock_tsd_key;
173 void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper);
174 void erts_allctr_wrapper_pre_lock(void);
175 void erts_allctr_wrapper_pre_unlock(void);
176
177 void erts_alloc_register_scheduler(void *vesdp);
178 void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
179 int *need_thr_progress,
180 ErtsThrPrgrVal *thr_prgr_p,
181 int *more_work);
182 erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs);
183
184 __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint)
185 __noreturn;
186 __decl_noreturn void erts_alloc_n_enomem(ErtsAlcType_t,Uint)
187 __noreturn;
188 __decl_noreturn void erts_realloc_enomem(ErtsAlcType_t,void*,Uint)
189 __noreturn;
190 __decl_noreturn void erts_realloc_n_enomem(ErtsAlcType_t,void*,Uint)
191 __noreturn;
192 __decl_noreturn void erts_alc_fatal_error(int,int,ErtsAlcType_t,...)
193 __noreturn;
194
195 Eterm erts_alloc_set_dyn_param(struct process*, Eterm);
196
197 #undef ERTS_HAVE_IS_IN_LITERAL_RANGE
198 #if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
199 # define ERTS_HAVE_IS_IN_LITERAL_RANGE
200 #endif
201
202
203 /*
204 * erts_alloc[_fnf](), erts_realloc[_fnf](), erts_free() works as
205 * malloc(), realloc(), and free() with the following exceptions:
206 *
207 * * They take an extra type argument as first argument which is
208 * the memory type to operate on. Memory types are generated
209 * (as ERTS_ALC_T_[SOMETHING] defines) from the erl_alloc.types
210 * configuration file.
211 * * The erts_alloc() and erts_realloc() functions terminate the
212 * emulator if memory cannot be obtained. The _fnf (Failure Not
213 * Fatal) suffixed versions return NULL if memory cannot be
214 * obtained.
215 * * They may be static functions so function pointers to "the same"
216 * function may differ.
217 *
218 * IMPORTANT: Memory allocated or reallocated as type X, can only
219 * be reallocated or deallocated as type X.
220 */
221
222 #if !ERTS_ALC_DO_INLINE
223
224 void *erts_alloc(ErtsAlcType_t type, Uint size);
225 void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size);
226 void erts_free(ErtsAlcType_t type, void *ptr);
227 void *erts_alloc_fnf(ErtsAlcType_t type, Uint size);
228 void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size);
229 int erts_is_allctr_wrapper_prelocked(void);
230 #ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
231 int erts_is_in_literal_range(void* ptr);
232 #endif
233
234 #endif /* #if !ERTS_ALC_DO_INLINE */
235
236 void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size);
237
238 #ifndef ERTS_CACHE_LINE_SIZE
239 /* Assumed cache line size */
240 # define ERTS_CACHE_LINE_SIZE ((UWord) ASSUMED_CACHE_LINE_SIZE)
241 # define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1)
242 #endif
243
244 #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__)
245
246 ERTS_ALC_INLINE
erts_alloc(ErtsAlcType_t type,Uint size)247 void *erts_alloc(ErtsAlcType_t type, Uint size)
248 {
249 void *res;
250 ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
251 res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)(
252 type,
253 erts_allctrs[ERTS_ALC_T2A(type)].extra,
254 size);
255 if (!res)
256 erts_alloc_n_enomem(ERTS_ALC_T2N(type), size);
257 ERTS_MSACC_POP_STATE_X();
258 return res;
259 }
260
261 ERTS_ALC_INLINE
erts_realloc(ErtsAlcType_t type,void * ptr,Uint size)262 void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size)
263 {
264 void *res;
265 ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
266 res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)(
267 type,
268 erts_allctrs[ERTS_ALC_T2A(type)].extra,
269 ptr,
270 size);
271 if (!res)
272 erts_realloc_n_enomem(ERTS_ALC_T2N(type), ptr, size);
273 ERTS_MSACC_POP_STATE_X();
274 return res;
275 }
276
277 ERTS_ALC_INLINE
erts_free(ErtsAlcType_t type,void * ptr)278 void erts_free(ErtsAlcType_t type, void *ptr)
279 {
280 ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
281 (*erts_allctrs[ERTS_ALC_T2A(type)].free)(
282 type,
283 erts_allctrs[ERTS_ALC_T2A(type)].extra,
284 ptr);
285 ERTS_MSACC_POP_STATE_X();
286 }
287
288
289 ERTS_ALC_INLINE
erts_alloc_fnf(ErtsAlcType_t type,Uint size)290 void *erts_alloc_fnf(ErtsAlcType_t type, Uint size)
291 {
292 void *res;
293 ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
294 res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)(
295 type,
296 erts_allctrs[ERTS_ALC_T2A(type)].extra,
297 size);
298 ERTS_MSACC_POP_STATE_X();
299 return res;
300 }
301
302
303 ERTS_ALC_INLINE
erts_realloc_fnf(ErtsAlcType_t type,void * ptr,Uint size)304 void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size)
305 {
306 void *res;
307 ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC);
308 res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)(
309 type,
310 erts_allctrs[ERTS_ALC_T2A(type)].extra,
311 ptr,
312 size);
313 ERTS_MSACC_POP_STATE_X();
314 return res;
315 }
316
317 ERTS_ALC_INLINE
erts_is_allctr_wrapper_prelocked(void)318 int erts_is_allctr_wrapper_prelocked(void)
319 {
320 return erts_allctr_wrapper_prelocked /* locked */
321 && !!erts_tsd_get(erts_allctr_prelock_tsd_key); /* by me */
322 }
323
324 #ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
325
326 ERTS_ALC_FORCE_INLINE
erts_is_in_literal_range(void * ptr)327 int erts_is_in_literal_range(void* ptr)
328 {
329 #if defined(ARCH_32)
330 Uint ix = (UWord)ptr >> ERTS_MMAP_SUPERALIGNED_BITS;
331
332 return erts_literal_vspace_map[ix / ERTS_VSPACE_WORD_BITS]
333 & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
334
335 #elif defined(ARCH_64)
336 extern char* erts_literals_start;
337 extern UWord erts_literals_size;
338 return ErtsInArea(ptr, erts_literals_start, erts_literals_size);
339 #else
340 # error No ARCH_xx
341 #endif
342 }
343
344 #endif /* ERTS_HAVE_IS_IN_LITERAL_RANGE */
345
346 #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */
347
348 #define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id())
349
350 typedef void (*erts_alloc_verify_func_t)(Allctr_t *);
351
352 erts_alloc_verify_func_t
353 erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr);
354
355 #define ERTS_ALC_DATA_ALIGN_SIZE(SZ) \
356 (((((SZ) - 1) / 8) + 1) * 8)
357
358 #define ERTS_ALC_CACHE_LINE_ALIGN_SIZE(SZ) \
359 (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE)
360
361 #if !defined(VALGRIND) && !defined(ADDRESS_SANITIZER)
362
363 #define ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \
364 ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, (void) 0, (void) 0, (void) 0)
365
366
367 #define ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, ILCK, LCK, ULCK) \
368 ERTS_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ, ILCK, LCK, ULCK) \
369 static void \
370 init_##NAME##_alloc(void) \
371 { \
372 init_##NAME##_pre_alloc(); \
373 } \
374 static ERTS_INLINE TYPE * \
375 NAME##_alloc(void) \
376 { \
377 TYPE *res = NAME##_pre_alloc(); \
378 if (!res) \
379 res = erts_alloc(ALCT, sizeof(TYPE)); \
380 return res; \
381 } \
382 static ERTS_INLINE void \
383 NAME##_free(TYPE *p) \
384 { \
385 if (!NAME##_pre_free(p)) \
386 erts_free(ALCT, (void *) p); \
387 }
388
389 #define ERTS_SCHED_PREF_PALLOC_IMPL(NAME, TYPE, PASZ) \
390 ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ)
391
392 #define ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ) \
393 ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ)
394
395 #define ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \
396 ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ) \
397 static void \
398 init_##NAME##_alloc(void) \
399 { \
400 init_##NAME##_pre_alloc(); \
401 } \
402 static ERTS_INLINE TYPE * \
403 NAME##_alloc(void) \
404 { \
405 TYPE *res = NAME##_pre_alloc(); \
406 if (!res) \
407 res = erts_alloc(ALCT, sizeof(TYPE)); \
408 return res; \
409 } \
410 static ERTS_INLINE void \
411 NAME##_free(TYPE *p) \
412 { \
413 if (!NAME##_pre_free(p)) \
414 erts_free(ALCT, (void *) p); \
415 }
416
417 #define ERTS_THR_PREF_AUX(NAME, TYPE, PASZ) \
418 ERTS_THR_PREF_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ)
419
420 #define ERTS_THR_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \
421 ERTS_THR_PREF_AUX(NAME, TYPE, PASZ) \
422 static void \
423 init_##NAME##_alloc(int nthreads) \
424 { \
425 init_##NAME##_pre_alloc(nthreads); \
426 } \
427 static ERTS_INLINE TYPE * \
428 NAME##_alloc(void) \
429 { \
430 TYPE *res = NAME##_pre_alloc(); \
431 if (!res) \
432 res = erts_alloc(ALCT, sizeof(TYPE)); \
433 return res; \
434 } \
435 static ERTS_INLINE void \
436 NAME##_free(TYPE *p) \
437 { \
438 if (!NAME##_pre_free(p)) \
439 erts_free(ALCT, (void *) p); \
440 }
441
442
443 #ifdef DEBUG
444 #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) < 1000 ? (SZ)/10 + 10 : 100)
445 #define ERTS_PRE_ALLOC_CLOBBER(P, T) sys_memset((void *) (P), 0xfd, sizeof(T))
446 #else
447 #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) > 1 ? (SZ) : 1)
448 #define ERTS_PRE_ALLOC_CLOBBER(P, T)
449 #endif
450
451 #define ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, ILCK, LCK, ULCK) \
452 union erts_qa_##NAME##__ { \
453 TYPE type; \
454 union erts_qa_##NAME##__ *next; \
455 }; \
456 static union erts_qa_##NAME##__ \
457 qa_prealcd_##NAME[ERTS_PRE_ALLOC_SIZE((PASZ))]; \
458 static union erts_qa_##NAME##__ *qa_freelist_##NAME; \
459 static void \
460 init_##NAME##_alloc(void) \
461 { \
462 int i; \
463 qa_freelist_##NAME = &qa_prealcd_##NAME[0]; \
464 for (i = 1; i < ERTS_PRE_ALLOC_SIZE((PASZ)); i++) { \
465 ERTS_PRE_ALLOC_CLOBBER(&qa_prealcd_##NAME[i-1], \
466 union erts_qa_##NAME##__); \
467 qa_prealcd_##NAME[i-1].next = &qa_prealcd_##NAME[i]; \
468 } \
469 ERTS_PRE_ALLOC_CLOBBER(&qa_prealcd_##NAME[ERTS_PRE_ALLOC_SIZE((PASZ))-1],\
470 union erts_qa_##NAME##__); \
471 qa_prealcd_##NAME[ERTS_PRE_ALLOC_SIZE((PASZ))-1].next = NULL; \
472 ILCK; \
473 } \
474 static ERTS_INLINE TYPE * \
475 NAME##_alloc(void) \
476 { \
477 TYPE *res; \
478 LCK; \
479 if (!qa_freelist_##NAME) \
480 res = NULL; \
481 else { \
482 res = &qa_freelist_##NAME->type; \
483 qa_freelist_##NAME = qa_freelist_##NAME->next; \
484 } \
485 ULCK; \
486 return res; \
487 } \
488 static ERTS_INLINE int \
489 NAME##_free(TYPE *p) \
490 { \
491 union erts_qa_##NAME##__ * up; \
492 up = ((union erts_qa_##NAME##__ *) \
493 (((char *) p) \
494 - ((char *) &((union erts_qa_##NAME##__ *) 0)->type))); \
495 if (up > &qa_prealcd_##NAME[ERTS_PRE_ALLOC_SIZE((PASZ))-1] \
496 || up < &qa_prealcd_##NAME[0]) \
497 return 0; \
498 else { \
499 LCK; \
500 ERTS_PRE_ALLOC_CLOBBER(up, union erts_qa_##NAME##__); \
501 up->next = qa_freelist_##NAME; \
502 qa_freelist_##NAME = up; \
503 ULCK; \
504 return 1; \
505 } \
506 }
507
508 #include "erl_sched_spec_pre_alloc.h"
509
510 #define ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ) \
511 union erts_sspa_##NAME##__ { \
512 erts_sspa_blk_t next; \
513 TYPE type; \
514 }; \
515 \
516 static erts_sspa_data_t *sspa_data_##NAME##__; \
517 \
518 static void \
519 init_##NAME##_alloc(void) \
520 { \
521 sspa_data_##NAME##__ = \
522 erts_sspa_create(sizeof(union erts_sspa_##NAME##__), \
523 ERTS_PRE_ALLOC_SIZE((PASZ)), \
524 0, NULL); \
525 } \
526 \
527 static TYPE * \
528 NAME##_alloc(void) \
529 { \
530 ErtsSchedulerData *esdp = erts_get_scheduler_data(); \
531 if (!esdp || ERTS_SCHEDULER_IS_DIRTY(esdp)) \
532 return NULL; \
533 return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__, \
534 (int) esdp->no - 1); \
535 } \
536 \
537 static int \
538 NAME##_free(TYPE *p) \
539 { \
540 ErtsSchedulerData *esdp = erts_get_scheduler_data(); \
541 return erts_sspa_free(sspa_data_##NAME##__, \
542 esdp ? (int) esdp->no - 1 : -1, \
543 (char *) p); \
544 }
545
546
547 #define ERTS_THR_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ) \
548 union erts_sspa_##NAME##__ { \
549 erts_sspa_blk_t next; \
550 TYPE type; \
551 }; \
552 \
553 static erts_sspa_data_t *sspa_data_##NAME##__; \
554 \
555 static void \
556 init_##NAME##_alloc(int nthreads) \
557 { \
558 sspa_data_##NAME##__ = \
559 erts_sspa_create(sizeof(union erts_sspa_##NAME##__), \
560 ERTS_PRE_ALLOC_SIZE((PASZ)), \
561 nthreads, \
562 #NAME); \
563 } \
564 \
565 void \
566 erts_##NAME##_alloc_init_thread(void) \
567 { \
568 int id = erts_atomic_inc_read_nob(&sspa_data_##NAME##__->id_generator);\
569 if (id > sspa_data_##NAME##__->nthreads) { \
570 erts_exit(ERTS_ABORT_EXIT, \
571 "%s:%d:%s(): Too many threads for '" #NAME "'\n", \
572 __FILE__, __LINE__, __func__); \
573 } \
574 erts_tsd_set(sspa_data_##NAME##__->tsd_key, (void*)(SWord)id); \
575 } \
576 \
577 static TYPE * \
578 NAME##_alloc(void) \
579 { \
580 int id = (int)(SWord)erts_tsd_get(sspa_data_##NAME##__->tsd_key); \
581 if (id == 0) \
582 return NULL; \
583 return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__, \
584 id-1); \
585 } \
586 \
587 static int \
588 NAME##_free(TYPE *p) \
589 { \
590 int id = (int)(SWord)erts_tsd_get(sspa_data_##NAME##__->tsd_key); \
591 return erts_sspa_free(sspa_data_##NAME##__, \
592 id - 1, \
593 (char *) p); \
594 }
595
596 #else /* !defined(VALGRIND) && !defined(ADDRESS_SANITIZER) */
597
598 /*
599 * For VALGRIND and ADDRESS_SANITIZER we short circuit all preallocation
600 * with dummy wrappers around malloc and free.
601 */
602
603 #define ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \
604 ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, (void) 0, (void) 0, (void) 0)
605
606 #define ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, ILCK, LCK, ULCK) \
607 static void init_##NAME##_alloc(void) \
608 { \
609 } \
610 static ERTS_INLINE TYPE* NAME##_alloc(void) \
611 { \
612 return malloc(sizeof(TYPE)); \
613 } \
614 static ERTS_INLINE void NAME##_free(TYPE *p) \
615 { \
616 free((void *) p); \
617 }
618
619 #define ERTS_SCHED_PREF_PALLOC_IMPL(NAME, TYPE, PASZ) \
620 ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ)
621
622 #define ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ) \
623 ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ)
624
625 #define ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \
626 ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)
627
628 #define ERTS_THR_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \
629 void erts_##NAME##_pre_alloc_init_thread(void) \
630 { \
631 } \
632 static void init_##NAME##_alloc(int nthreads) \
633 { \
634 } \
635 static ERTS_INLINE TYPE* NAME##_alloc(void) \
636 { \
637 return malloc(sizeof(TYPE)); \
638 } \
639 static ERTS_INLINE void NAME##_free(TYPE *p) \
640 { \
641 free(p); \
642 }
643
644 #define ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ) \
645 static void init_##NAME##_alloc(void) \
646 { \
647 } \
648 static TYPE* NAME##_alloc(void) \
649 { \
650 return (TYPE *) malloc(sizeof(TYPE)); \
651 } \
652 static int NAME##_free(TYPE *p) \
653 { \
654 free(p); \
655 return 1; \
656 }
657
658 #endif /* VALGRIND || ADDRESS_SANITIZER */
659
660 #ifdef DEBUG
661 #define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((UWord *) (PTR)) - 2))
662 #endif /* #ifdef DEBUG */
663
664 #undef ERTS_ALC_INLINE
665 #undef ERTS_ALC_ATTRIBUTES
666
667 #endif /* #ifndef ERL_ALLOC_H__ */
668