1 /*
2  * %CopyrightBegin%
3  *
4  * Copyright Ericsson AB 2002-2020. All Rights Reserved.
5  *
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at
9  *
10  *     http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *
18  * %CopyrightEnd%
19  */
20 
21 
22 /*
23  * Description:	Management of memory allocators.
24  *
25  * Author: 	Rickard Green
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #  include "config.h"
30 #endif
31 #define ERTS_ALLOC_C__
32 #define ERTS_ALC_INTERNAL__
33 #define ERTS_WANT_MEM_MAPPERS
34 #include "sys.h"
35 #define ERL_THREADS_EMU_INTERNAL__
36 #include "erl_threads.h"
37 #include "global.h"
38 #include "erl_db.h"
39 #include "erl_binary.h"
40 #include "erl_bits.h"
41 #include "erl_mtrace.h"
42 #include "erl_mseg.h"
43 #include "erl_monitor_link.h"
44 #include "erl_hl_timer.h"
45 #include "erl_cpu_topology.h"
46 #include "erl_thr_queue.h"
47 #include "erl_nfunc_sched.h"
48 #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
49 #include "erl_check_io.h"
50 #endif
51 #include "erl_bif_unique.h"
52 
53 #define GET_ERL_GF_ALLOC_IMPL
54 #include "erl_goodfit_alloc.h"
55 #define GET_ERL_BF_ALLOC_IMPL
56 #include "erl_bestfit_alloc.h"
57 #define GET_ERL_AF_ALLOC_IMPL
58 #include "erl_afit_alloc.h"
59 #define GET_ERL_AOFF_ALLOC_IMPL
60 #include "erl_ao_firstfit_alloc.h"
61 
62 
63 #if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_AU_MAX_PREF_ALLOC_INSTANCES
64 #  error "Too many schedulers; cannot create that many pref alloc instances"
65 #endif
66 
67 #define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS
68 
69 #if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND) || defined(ADDRESS_SANITIZER)
70 #define AU_ALLOC_DEFAULT_ENABLE(X)	0
71 #else
72 #define AU_ALLOC_DEFAULT_ENABLE(X)	(X)
73 #endif
74 
75 #define ERTS_ALC_DEFAULT_ENABLED_ACUL 60
76 #define ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC 45
77 #define ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC 85
78 
79 #define ERTS_ALC_DEFAULT_ACUL ERTS_ALC_DEFAULT_ENABLED_ACUL
80 #define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC
81 #define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC
82 
83 
84 #ifdef DEBUG
85 static Uint install_debug_functions(void);
86 #if 0
87 #define HARD_DEBUG
88 #ifdef __GNUC__
89 #warning "* * * * * * * * * * * * * *"
90 #warning "* HARD DEBUG IS ENABLED!  *"
91 #warning "* * * * * * * * * * * * * *"
92 #endif
93 #endif
94 #endif
95 
96 static int lock_all_physical_memory = 0;
97 
98 ErtsAllocatorFunctions_t ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]);
99 ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
100 ErtsAllocatorThrSpec_t ERTS_WRITE_UNLIKELY(erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]);
101 
102 #define ERTS_MIN(A, B) ((A) < (B) ? (A) : (B))
103 #define ERTS_MAX(A, B) ((A) > (B) ? (A) : (B))
104 
105 typedef union {
106     GFAllctr_t gfa;
107     char align_gfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(GFAllctr_t))];
108     BFAllctr_t bfa;
109     char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))];
110     AFAllctr_t afa;
111     char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))];
112     AOFFAllctr_t aoffa;
113     char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))];
114 } ErtsAllocatorState_t erts_align_attribute(ERTS_CACHE_LINE_SIZE);
115 
116 static ErtsAllocatorState_t std_alloc_state;
117 static ErtsAllocatorState_t ll_alloc_state;
118 static ErtsAllocatorState_t sl_alloc_state;
119 static ErtsAllocatorState_t temp_alloc_state;
120 static ErtsAllocatorState_t eheap_alloc_state;
121 static ErtsAllocatorState_t binary_alloc_state;
122 static ErtsAllocatorState_t ets_alloc_state;
123 static ErtsAllocatorState_t driver_alloc_state;
124 static ErtsAllocatorState_t fix_alloc_state;
125 static ErtsAllocatorState_t literal_alloc_state;
126 #ifdef ERTS_ALC_A_EXEC
127 static ErtsAllocatorState_t exec_alloc_state;
128 #endif
129 static ErtsAllocatorState_t test_alloc_state;
130 
131 enum {
132     ERTS_ALC_INFO_A_ALLOC_UTIL = ERTS_ALC_A_MAX + 1,
133     ERTS_ALC_INFO_A_MSEG_ALLOC,
134     ERTS_ALC_INFO_A_ERTS_MMAP,
135     ERTS_ALC_INFO_A_DISABLED_EXEC,  /* fake a disabled "exec_alloc" */
136     ERTS_ALC_INFO_A_END
137 };
138 
139 typedef struct {
140     erts_atomic32_t refc;
141     int only_sz;
142     int internal;
143     Uint req_sched;
144     Process *proc;
145     ErtsIRefStorage iref;
146     int allocs[ERTS_ALC_INFO_A_END - ERTS_ALC_A_MIN + 1];
147 } ErtsAllocInfoReq;
148 
149 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq,
150                                  ErtsAllocInfoReq,
151                                  5,
152                                  ERTS_ALC_T_AINFO_REQ)
153 
154 ErtsAlcType_t erts_fix_core_allocator_ix;
155 
156 struct au_init {
157     int enable;
158     int thr_spec;
159     int disable_allowed;
160     int thr_spec_allowed;
161     int carrier_migration_allowed;
162     ErtsAlcStrat_t	astrat;
163     struct {
164 	AllctrInit_t	util;
165 	GFAllctrInit_t	gf;
166 	BFAllctrInit_t	bf;
167 	AFAllctrInit_t	af;
168 	AOFFAllctrInit_t aoff;
169     } init;
170     struct {
171 	int mmbcs;
172 	int lmbcs;
173 	int smbcs;
174 	int mmmbc;
175     } default_;
176 };
177 
178 #define DEFAULT_ALLCTR_INIT {		\
179     ERTS_DEFAULT_ALLCTR_INIT,		\
180     ERTS_DEFAULT_GF_ALLCTR_INIT,	\
181     ERTS_DEFAULT_BF_ALLCTR_INIT,	\
182     ERTS_DEFAULT_AF_ALLCTR_INIT,	\
183     ERTS_DEFAULT_AOFF_ALLCTR_INIT       \
184 }
185 
186 typedef struct {
187     int erts_alloc_config;
188 #if HAVE_ERTS_MSEG
189     ErtsMsegInit_t mseg;
190 #endif
191     int trim_threshold;
192     int top_pad;
193     AlcUInit_t alloc_util;
194     struct {
195 	char *mtrace;
196 	char *nodename;
197     } instr;
198     struct au_init sl_alloc;
199     struct au_init std_alloc;
200     struct au_init ll_alloc;
201     struct au_init temp_alloc;
202     struct au_init eheap_alloc;
203     struct au_init binary_alloc;
204     struct au_init ets_alloc;
205     struct au_init driver_alloc;
206     struct au_init fix_alloc;
207     struct au_init literal_alloc;
208     struct au_init exec_alloc;
209     struct au_init test_alloc;
210 } erts_alc_hndl_args_init_t;
211 
212 #define ERTS_AU_INIT__ {0, 0, 1, 1, 1, \
213                         ERTS_ALC_S_GOODFIT, DEFAULT_ALLCTR_INIT, \
214                         {1,1,1,1}}
215 
216 #define SET_DEFAULT_ALLOC_OPTS(IP)					\
217 do {									\
218     struct au_init aui__ = ERTS_AU_INIT__;				\
219     sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init));	\
220 } while (0)
221 
222 static void
set_default_sl_alloc_opts(struct au_init * ip)223 set_default_sl_alloc_opts(struct au_init *ip)
224 {
225     SET_DEFAULT_ALLOC_OPTS(ip);
226     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
227     ip->thr_spec		= 1;
228     ip->astrat			= ERTS_ALC_S_GOODFIT;
229     ip->init.util.name_prefix	= "sl_";
230     ip->init.util.alloc_no	= ERTS_ALC_A_SHORT_LIVED;
231     ip->init.util.cp            = ERTS_ALC_COMMON_CPOOL_IX;
232 #ifndef SMALL_MEMORY
233     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
234 #else
235     ip->init.util.mmbcs 	= 32*1024; /* Main carrier size */
236 #endif
237     ip->init.util.ts 		= ERTS_ALC_MTA_SHORT_LIVED;
238     ip->init.util.rsbcst	= 80;
239     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
240 }
241 
242 static void
set_default_std_alloc_opts(struct au_init * ip)243 set_default_std_alloc_opts(struct au_init *ip)
244 {
245     SET_DEFAULT_ALLOC_OPTS(ip);
246     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
247     ip->thr_spec		= 1;
248     ip->astrat			= ERTS_ALC_S_BESTFIT;
249     ip->init.util.name_prefix	= "std_";
250     ip->init.util.alloc_no	= ERTS_ALC_A_STANDARD;
251     ip->init.util.cp            = ERTS_ALC_COMMON_CPOOL_IX;
252 #ifndef SMALL_MEMORY
253     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
254 #else
255     ip->init.util.mmbcs 	= 32*1024; /* Main carrier size */
256 #endif
257     ip->init.util.ts 		= ERTS_ALC_MTA_STANDARD;
258     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
259 }
260 
261 static void
set_default_ll_alloc_opts(struct au_init * ip)262 set_default_ll_alloc_opts(struct au_init *ip)
263 {
264     SET_DEFAULT_ALLOC_OPTS(ip);
265     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
266     ip->thr_spec		= 0;
267     ip->astrat			= ERTS_ALC_S_BESTFIT;
268     ip->init.bf.ao		= 1;
269     ip->init.util.ramv		= 0;
270     ip->init.util.mmsbc		= 0;
271     ip->init.util.sbct		= ~((UWord) 0);
272     ip->init.util.name_prefix	= "ll_";
273     ip->init.util.alloc_no	= ERTS_ALC_A_LONG_LIVED;
274     ip->init.util.cp            = ERTS_ALC_COMMON_CPOOL_IX;
275 #ifndef SMALL_MEMORY
276     ip->init.util.mmbcs 	= 2*1024*1024; /* Main carrier size */
277 #else
278     ip->init.util.mmbcs 	= 1*1024*1024; /* Main carrier size */
279 #endif
280     ip->init.util.ts 		= ERTS_ALC_MTA_LONG_LIVED;
281     ip->init.util.asbcst	= 0;
282     ip->init.util.rsbcst	= 0;
283     ip->init.util.rsbcmt	= 0;
284     ip->init.util.rmbcmt	= 0;
285     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL_LL_ALLOC;
286 }
287 
288 static void
set_default_literal_alloc_opts(struct au_init * ip)289 set_default_literal_alloc_opts(struct au_init *ip)
290 {
291     SET_DEFAULT_ALLOC_OPTS(ip);
292 #ifdef ADDRESS_SANITIZER
293     ip->enable			= 0;
294 #else
295     ip->enable			= 1;
296 #endif
297     ip->thr_spec		= 0;
298     ip->disable_allowed         = 0;
299     ip->thr_spec_allowed        = 0;
300     ip->carrier_migration_allowed = 0;
301     ip->astrat			= ERTS_ALC_S_BESTFIT;
302     ip->init.bf.ao		= 1;
303     ip->init.util.ramv		= 0;
304     ip->init.util.mmsbc		= 0;
305     ip->init.util.sbct		= ~((UWord) 0);
306     ip->init.util.name_prefix	= "literal_";
307     ip->init.util.alloc_no	= ERTS_ALC_A_LITERAL;
308 #ifndef SMALL_MEMORY
309     ip->init.util.mmbcs 	= 1024*1024; /* Main carrier size */
310 #else
311     ip->init.util.mmbcs 	= 256*1024; /* Main carrier size */
312 #endif
313     ip->init.util.ts 		= ERTS_ALC_MTA_LITERAL;
314     ip->init.util.asbcst	= 0;
315     ip->init.util.rsbcst	= 0;
316     ip->init.util.rsbcmt	= 0;
317     ip->init.util.rmbcmt	= 0;
318     ip->init.util.acul		= 0;
319 
320 #if defined(ARCH_32)
321 # if HAVE_ERTS_MSEG
322     ip->init.util.mseg_alloc   = &erts_alcu_literal_32_mseg_alloc;
323     ip->init.util.mseg_realloc = &erts_alcu_literal_32_mseg_realloc;
324     ip->init.util.mseg_dealloc = &erts_alcu_literal_32_mseg_dealloc;
325 # endif
326     ip->init.util.sys_alloc    = &erts_alcu_literal_32_sys_alloc;
327     ip->init.util.sys_realloc  = &erts_alcu_literal_32_sys_realloc;
328     ip->init.util.sys_dealloc  = &erts_alcu_literal_32_sys_dealloc;
329 #elif defined(ARCH_64)
330 # ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
331     ip->init.util.mseg_alloc    = &erts_alcu_mmapper_mseg_alloc;
332     ip->init.util.mseg_realloc  = &erts_alcu_mmapper_mseg_realloc;
333     ip->init.util.mseg_dealloc  = &erts_alcu_mmapper_mseg_dealloc;
334     ip->init.util.mseg_mmapper  = &erts_literal_mmapper;
335 # endif
336 #else
337 # error Unknown architecture
338 #endif
339 }
340 
341 #ifdef ERTS_ALC_A_EXEC
342 static void
set_default_exec_alloc_opts(struct au_init * ip)343 set_default_exec_alloc_opts(struct au_init *ip)
344 {
345     SET_DEFAULT_ALLOC_OPTS(ip);
346     ip->enable			= 1;
347     ip->thr_spec		= 0;
348     ip->disable_allowed         = 0;
349     ip->thr_spec_allowed        = 0;
350     ip->carrier_migration_allowed = 0;
351     ip->astrat			= ERTS_ALC_S_BESTFIT;
352     ip->init.bf.ao		= 1;
353     ip->init.util.ramv		= 0;
354     ip->init.util.mmsbc		= 0;
355     ip->init.util.sbct		= ~((UWord) 0);
356     ip->init.util.name_prefix	= "exec_";
357     ip->init.util.alloc_no	= ERTS_ALC_A_EXEC;
358     ip->init.util.mmbcs 	= 0; /* No main carrier */
359     ip->init.util.ts 		= ERTS_ALC_MTA_EXEC;
360     ip->init.util.asbcst	= 0;
361     ip->init.util.rsbcst	= 0;
362     ip->init.util.rsbcmt	= 0;
363     ip->init.util.rmbcmt	= 0;
364     ip->init.util.acul		= 0;
365 
366     ip->init.util.mseg_alloc    = &erts_alcu_exec_mseg_alloc;
367     ip->init.util.mseg_realloc  = &erts_alcu_exec_mseg_realloc;
368     ip->init.util.mseg_dealloc  = &erts_alcu_exec_mseg_dealloc;
369 }
370 #endif /* ERTS_ALC_A_EXEC */
371 
372 static void
set_default_temp_alloc_opts(struct au_init * ip)373 set_default_temp_alloc_opts(struct au_init *ip)
374 {
375     SET_DEFAULT_ALLOC_OPTS(ip);
376     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
377     ip->thr_spec		= 1;
378     ip->disable_allowed         = 0;
379     ip->carrier_migration_allowed = 0;
380     ip->astrat			= ERTS_ALC_S_AFIT;
381     ip->init.util.name_prefix	= "temp_";
382     ip->init.util.alloc_no	= ERTS_ALC_A_TEMPORARY;
383 #ifndef SMALL_MEMORY
384     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
385 #else
386     ip->init.util.mmbcs 	= 32*1024; /* Main carrier size */
387 #endif
388     ip->init.util.ts 		= ERTS_ALC_MTA_TEMPORARY;
389     ip->init.util.rsbcst	= 90;
390     ip->init.util.rmbcmt	= 100;
391 }
392 
393 static void
set_default_eheap_alloc_opts(struct au_init * ip)394 set_default_eheap_alloc_opts(struct au_init *ip)
395 {
396     SET_DEFAULT_ALLOC_OPTS(ip);
397     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
398     ip->thr_spec		= 1;
399     ip->astrat			= ERTS_ALC_S_GOODFIT;
400     ip->init.util.name_prefix	= "eheap_";
401     ip->init.util.alloc_no	= ERTS_ALC_A_EHEAP;
402     ip->init.util.cp            = ERTS_ALC_COMMON_CPOOL_IX;
403 #ifndef SMALL_MEMORY
404     ip->init.util.mmbcs 	= 512*1024; /* Main carrier size */
405 #else
406     ip->init.util.mmbcs 	= 256*1024; /* Main carrier size */
407 #endif
408     ip->init.util.ts 		= ERTS_ALC_MTA_EHEAP;
409     ip->init.util.rsbcst	= 50;
410     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC;
411 }
412 
413 static void
set_default_binary_alloc_opts(struct au_init * ip)414 set_default_binary_alloc_opts(struct au_init *ip)
415 {
416     SET_DEFAULT_ALLOC_OPTS(ip);
417     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
418     ip->thr_spec		= 1;
419     ip->astrat			= ERTS_ALC_S_BESTFIT;
420     ip->init.util.name_prefix	= "binary_";
421     ip->init.util.alloc_no	= ERTS_ALC_A_BINARY;
422     ip->init.util.cp            = ERTS_ALC_COMMON_CPOOL_IX;
423 #ifndef SMALL_MEMORY
424     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
425 #else
426     ip->init.util.mmbcs 	= 32*1024; /* Main carrier size */
427 #endif
428     ip->init.util.ts 		= ERTS_ALC_MTA_BINARY;
429     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
430     ip->init.util.atags		= 1;
431 }
432 
433 static void
set_default_ets_alloc_opts(struct au_init * ip)434 set_default_ets_alloc_opts(struct au_init *ip)
435 {
436     SET_DEFAULT_ALLOC_OPTS(ip);
437     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
438     ip->thr_spec		= 1;
439     ip->astrat			= ERTS_ALC_S_BESTFIT;
440     ip->init.util.name_prefix	= "ets_";
441     ip->init.util.alloc_no	= ERTS_ALC_A_ETS;
442     ip->init.util.cp            = ERTS_ALC_COMMON_CPOOL_IX;
443 #ifndef SMALL_MEMORY
444     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
445 #else
446     ip->init.util.mmbcs 	= 32*1024; /* Main carrier size */
447 #endif
448     ip->init.util.ts 		= ERTS_ALC_MTA_ETS;
449     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
450 }
451 
452 static void
set_default_driver_alloc_opts(struct au_init * ip)453 set_default_driver_alloc_opts(struct au_init *ip)
454 {
455     SET_DEFAULT_ALLOC_OPTS(ip);
456     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
457     ip->thr_spec		= 1;
458     ip->astrat			= ERTS_ALC_S_BESTFIT;
459     ip->init.util.name_prefix	= "driver_";
460     ip->init.util.alloc_no	= ERTS_ALC_A_DRIVER;
461     ip->init.util.cp            = ERTS_ALC_COMMON_CPOOL_IX;
462 #ifndef SMALL_MEMORY
463     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
464 #else
465     ip->init.util.mmbcs 	= 32*1024; /* Main carrier size */
466 #endif
467     ip->init.util.ts 		= ERTS_ALC_MTA_DRIVER;
468     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
469     ip->init.util.atags		= 1;
470 }
471 
472 static void
set_default_fix_alloc_opts(struct au_init * ip,size_t * fix_type_sizes)473 set_default_fix_alloc_opts(struct au_init *ip,
474 			   size_t *fix_type_sizes)
475 {
476     SET_DEFAULT_ALLOC_OPTS(ip);
477     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
478     ip->thr_spec		= 1;
479     ip->astrat			= ERTS_ALC_S_BESTFIT;
480     ip->init.bf.ao = 1;
481     ip->init.util.name_prefix	= "fix_";
482     ip->init.util.fix_type_size	= fix_type_sizes;
483     ip->init.util.alloc_no	= ERTS_ALC_A_FIXED_SIZE;
484     ip->init.util.cp            = ERTS_ALC_COMMON_CPOOL_IX;
485 #ifndef SMALL_MEMORY
486     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
487 #else
488     ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
489 #endif
490     ip->init.util.ts 		= ERTS_ALC_MTA_FIXED_SIZE;
491     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
492 }
493 
494 static void
set_default_test_alloc_opts(struct au_init * ip)495 set_default_test_alloc_opts(struct au_init *ip)
496 {
497     SET_DEFAULT_ALLOC_OPTS(ip);
498     ip->enable			= 0; /* Disabled by default */
499     ip->thr_spec		= -1 * erts_no_schedulers;
500     ip->astrat			= ERTS_ALC_S_FIRSTFIT;
501     ip->init.aoff.crr_order     = FF_AOFF;
502     ip->init.aoff.blk_order     = FF_BF;
503     ip->init.util.name_prefix	= "test_";
504     ip->init.util.alloc_no	= ERTS_ALC_A_TEST;
505     ip->init.util.cp            = ERTS_ALC_A_TEST;
506     ip->init.util.mmbcs 	= 0; /* Main carrier size */
507     ip->init.util.ts 		= ERTS_ALC_MTA_TEST;
508     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
509     ip->init.util.atags		= 1;
510 
511     /* Use a constant minimal MBC size */
512 #if ERTS_SA_MB_CARRIERS
513     ip->init.util.smbcs = ERTS_SACRR_UNIT_SZ;
514     ip->init.util.lmbcs = ERTS_SACRR_UNIT_SZ;
515     ip->init.util.sbct  = ERTS_SACRR_UNIT_SZ;
516 #else
517     ip->init.util.smbcs = 1 << 12;
518     ip->init.util.lmbcs = 1 << 12;
519     ip->init.util.sbct  = 1 << 12;
520 #endif
521 }
522 
523 
524 
525 static void
adjust_tpref(struct au_init * ip,int no_sched)526 adjust_tpref(struct au_init *ip, int no_sched)
527 {
528     if (ip->thr_spec) {
529 	ip->thr_spec = no_sched;
530 	ip->thr_spec *= -1; /* thread preferred */
531 
532 	/* If default ... */
533 
534 	/* ... shrink main multi-block carrier size */
535 	if (ip->default_.mmbcs)
536 	    ip->init.util.mmbcs /= ERTS_MIN(4, no_sched);
537 	/* ... shrink largest multi-block carrier size */
538 	if (ip->default_.lmbcs)
539 	    ip->init.util.lmbcs /= ERTS_MIN(2, no_sched);
540 	/* ... shrink smallest multi-block carrier size */
541 	if (ip->default_.smbcs)
542 	    ip->init.util.smbcs /= ERTS_MIN(4, no_sched);
543     }
544 }
545 
546 
547 static void handle_args(int *, char **, erts_alc_hndl_args_init_t *);
548 
549 static void
550 set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu);
551 
552 static void
553 start_au_allocator(ErtsAlcType_t alctr_n,
554 		   struct au_init *init,
555 		   ErtsAllocatorState_t *state);
556 
557 static void
refuse_af_strategy(struct au_init * init)558 refuse_af_strategy(struct au_init *init)
559 {
560     if (init->astrat == ERTS_ALC_S_AFIT)
561 	init->astrat = ERTS_ALC_S_GOODFIT;
562 }
563 
564 #ifdef HARD_DEBUG
565 static void hdbg_init(void);
566 #endif
567 
adjust_fix_alloc_sizes(UWord extra_block_size)568 static void adjust_fix_alloc_sizes(UWord extra_block_size)
569 {
570 
571     if (extra_block_size && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled) {
572 	int j;
573 
574 	if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec) {
575 	    int i;
576 	    ErtsAllocatorThrSpec_t* tspec;
577 
578 	    tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
579 	    ASSERT(tspec->enabled);
580 
581 	    for (i=0; i < tspec->size; i++) {
582 		Allctr_t* allctr = tspec->allctr[i];
583 		for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) {
584                     size_t size = allctr->fix[j].type_size;
585                     size = MAX(size + extra_block_size,
586                                sizeof(ErtsAllctrDDBlock_t));
587 		    allctr->fix[j].type_size = size;
588 		}
589 	    }
590 	}
591 	else
592 	{
593 	    Allctr_t* allctr = erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra;
594 	    for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) {
595                 size_t size = allctr->fix[j].type_size;
596                 size = MAX(size + extra_block_size,
597                            sizeof(ErtsAllctrDDBlock_t));
598                 allctr->fix[j].type_size = size;
599 	    }
600 	}
601     }
602 }
603 
604 static ERTS_INLINE int
strategy_support_carrier_migration(struct au_init * auip)605 strategy_support_carrier_migration(struct au_init *auip)
606 {
607     /*
608      * Currently only aoff* and ageff* support carrier
609      * migration, i.e, type AOFIRSTFIT.
610      */
611     return auip->astrat == ERTS_ALC_S_FIRSTFIT;
612 }
613 
614 static ERTS_INLINE void
adjust_carrier_migration_support(struct au_init * auip)615 adjust_carrier_migration_support(struct au_init *auip)
616 {
617     if (auip->init.util.acul) {
618 	auip->thr_spec = -1; /* Need thread preferred */
619 
620 	/*
621 	 * If strategy cannot handle carrier migration,
622 	 * default to a strategy that can...
623 	 */
624 	if (!strategy_support_carrier_migration(auip)) {
625 	    /* Default to aoffcbf */
626 	    auip->astrat = ERTS_ALC_S_FIRSTFIT;
627 	    auip->init.aoff.crr_order = FF_AOFF;
628 	    auip->init.aoff.blk_order = FF_BF;
629 	}
630     }
631 }
632 
633 void
erts_alloc_init(int * argc,char ** argv,ErtsAllocInitOpts * eaiop)634 erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
635 {
636     UWord extra_block_size = 0;
637     int i, ncpu;
638     erts_alc_hndl_args_init_t init = {
639 	0,
640 #if HAVE_ERTS_MSEG
641 	ERTS_MSEG_INIT_DEFAULT_INITIALIZER,
642 #endif
643 	ERTS_DEFAULT_TRIM_THRESHOLD,
644 	ERTS_DEFAULT_TOP_PAD,
645 	ERTS_DEFAULT_ALCU_INIT,
646     };
647     size_t fix_type_sizes[ERTS_ALC_NO_FIXED_SIZES] = {0};
648 
649     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)]
650 	= sizeof(Process);
651     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR)]
652 	= sizeof(ErtsMonitorDataHeap);
653     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LINK)]
654 	= sizeof(ErtsILink);
655     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
656 	= sizeof(ErtsDrvSelectDataState);
657     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_SEL_D_STATE)]
658         = sizeof(ErtsNifSelectDataState);
659     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)]
660 	= sizeof(ErtsMessageRef);
661     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)]
662 	= sizeof(ErtsThrQElement_t);
663     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LL_PTIMER)]
664 	= erts_timer_type_size(ERTS_ALC_T_LL_PTIMER);
665     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_HL_PTIMER)]
666 	= erts_timer_type_size(ERTS_ALC_T_HL_PTIMER);
667     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)]
668 	= erts_timer_type_size(ERTS_ALC_T_BIF_TIMER);
669     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)]
670 	= sizeof(ErtsNSchedMagicRefTableEntry);
671     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MINDIRECTION)]
672 	= ERTS_MAGIC_BIN_UNALIGNED_SIZE(sizeof(ErtsMagicIndirectionWord));
673 
674 #ifdef HARD_DEBUG
675     hdbg_init();
676 #endif
677 
678     lock_all_physical_memory = 0;
679 
680     ncpu = eaiop->ncpu;
681     if (ncpu < 1)
682 	ncpu = 1;
683 
684     erts_tsd_key_create(&erts_allctr_prelock_tsd_key,
685 			"erts_allctr_prelock_tsd_key");
686 
687     erts_sys_alloc_init();
688     erts_init_utils_mem();
689 
690     set_default_sl_alloc_opts(&init.sl_alloc);
691     set_default_std_alloc_opts(&init.std_alloc);
692     set_default_ll_alloc_opts(&init.ll_alloc);
693     set_default_temp_alloc_opts(&init.temp_alloc);
694     set_default_eheap_alloc_opts(&init.eheap_alloc);
695     set_default_binary_alloc_opts(&init.binary_alloc);
696     set_default_ets_alloc_opts(&init.ets_alloc);
697     set_default_driver_alloc_opts(&init.driver_alloc);
698     set_default_fix_alloc_opts(&init.fix_alloc,
699 			       fix_type_sizes);
700     set_default_literal_alloc_opts(&init.literal_alloc);
701 #ifdef ERTS_ALC_A_EXEC
702     set_default_exec_alloc_opts(&init.exec_alloc);
703 #endif
704     set_default_test_alloc_opts(&init.test_alloc);
705 
706     if (argc && argv)
707 	handle_args(argc, argv, &init);
708 
709     if (lock_all_physical_memory) {
710 #ifdef HAVE_MLOCKALL
711 	errno = 0;
712 	if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) {
713 	    int err = errno;
714 	    char *errstr = err ? strerror(err) : "unknown";
715 	    erts_exit(1, "Failed to lock physical memory: %s (%d)\n",
716 		     errstr, err);
717 	}
718 #else
719 	erts_exit(1, "Failed to lock physical memory: Not supported\n");
720 #endif
721     }
722 
723 
724     /* Make adjustments for carrier migration support */
725     init.temp_alloc.init.util.acul = 0;
726     adjust_carrier_migration_support(&init.sl_alloc);
727     adjust_carrier_migration_support(&init.std_alloc);
728     adjust_carrier_migration_support(&init.ll_alloc);
729     adjust_carrier_migration_support(&init.eheap_alloc);
730     adjust_carrier_migration_support(&init.binary_alloc);
731     adjust_carrier_migration_support(&init.ets_alloc);
732     adjust_carrier_migration_support(&init.driver_alloc);
733     adjust_carrier_migration_support(&init.fix_alloc);
734     adjust_carrier_migration_support(&init.literal_alloc);
735 #ifdef ERTS_ALC_A_EXEC
736     adjust_carrier_migration_support(&init.exec_alloc);
737 #endif
738 
739     if (init.erts_alloc_config) {
740 	/* Adjust flags that erts_alloc_config won't like */
741 
742 	/* No thread specific instances */
743 	init.temp_alloc.thr_spec = 0;
744 	init.sl_alloc.thr_spec = 0;
745 	init.std_alloc.thr_spec = 0;
746 	init.ll_alloc.thr_spec = 0;
747 	init.eheap_alloc.thr_spec = 0;
748 	init.binary_alloc.thr_spec = 0;
749 	init.ets_alloc.thr_spec = 0;
750 	init.driver_alloc.thr_spec = 0;
751 	init.fix_alloc.thr_spec = 0;
752         init.literal_alloc.thr_spec = 0;
753 #ifdef ERTS_ALC_A_EXEC
754         init.exec_alloc.thr_spec = 0;
755 #endif
756 
757 	/* No carrier migration */
758 	init.temp_alloc.init.util.acul = 0;
759 	init.sl_alloc.init.util.acul = 0;
760 	init.std_alloc.init.util.acul = 0;
761 	init.ll_alloc.init.util.acul = 0;
762 	init.eheap_alloc.init.util.acul = 0;
763 	init.binary_alloc.init.util.acul = 0;
764 	init.ets_alloc.init.util.acul = 0;
765 	init.driver_alloc.init.util.acul = 0;
766 	init.fix_alloc.init.util.acul = 0;
767         init.literal_alloc.init.util.acul = 0;
768 #ifdef ERTS_ALC_A_EXEC
769         init.exec_alloc.init.util.acul = 0;
770 #endif
771     }
772 
773     /* Only temp_alloc can use thread specific interface */
774     if (init.temp_alloc.thr_spec)
775 	init.temp_alloc.thr_spec = erts_no_schedulers;
776 
777     /* Others must use thread preferred interface */
778     adjust_tpref(&init.sl_alloc, erts_no_schedulers);
779     adjust_tpref(&init.std_alloc, erts_no_schedulers);
780     adjust_tpref(&init.ll_alloc, erts_no_schedulers);
781     adjust_tpref(&init.eheap_alloc, erts_no_schedulers);
782     adjust_tpref(&init.binary_alloc, erts_no_schedulers);
783     adjust_tpref(&init.ets_alloc, erts_no_schedulers);
784     adjust_tpref(&init.driver_alloc, erts_no_schedulers);
785     adjust_tpref(&init.fix_alloc, erts_no_schedulers);
786     adjust_tpref(&init.literal_alloc, erts_no_schedulers);
787 #ifdef ERTS_ALC_A_EXEC
788     adjust_tpref(&init.exec_alloc, erts_no_schedulers);
789 #endif
790 
791 
792     /*
793      * The following allocators cannot be run with afit strategy.
794      * Make sure they don't...
795      */
796     refuse_af_strategy(&init.sl_alloc);
797     refuse_af_strategy(&init.std_alloc);
798     refuse_af_strategy(&init.ll_alloc);
799     refuse_af_strategy(&init.eheap_alloc);
800     refuse_af_strategy(&init.binary_alloc);
801     refuse_af_strategy(&init.ets_alloc);
802     refuse_af_strategy(&init.driver_alloc);
803     refuse_af_strategy(&init.fix_alloc);
804     refuse_af_strategy(&init.literal_alloc);
805 #ifdef ERTS_ALC_A_EXEC
806     refuse_af_strategy(&init.exec_alloc);
807 #endif
808 
809     if (!init.temp_alloc.thr_spec)
810 	refuse_af_strategy(&init.temp_alloc);
811 
812     erts_mtrace_pre_init();
813 #if HAVE_ERTS_MSEG
814     init.mseg.nos = erts_no_schedulers;
815     erts_mseg_init(&init.mseg);
816 #endif
817 
818     erts_alcu_init(&init.alloc_util);
819     erts_afalc_init();
820     erts_bfalc_init();
821     erts_gfalc_init();
822     erts_aoffalc_init();
823 
824     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
825 	erts_allctrs[i].alloc		= NULL;
826 	erts_allctrs[i].realloc		= NULL;
827 	erts_allctrs[i].free		= NULL;
828 	erts_allctrs[i].extra		= NULL;
829 	erts_allctrs_info[i].alloc_util	= 0;
830 	erts_allctrs_info[i].enabled	= 0;
831 	erts_allctrs_info[i].thr_spec	= 0;
832 	erts_allctrs_info[i].extra	= NULL;
833     }
834 
835     erts_allctrs[ERTS_ALC_A_SYSTEM].alloc		= erts_sys_alloc;
836     erts_allctrs[ERTS_ALC_A_SYSTEM].realloc		= erts_sys_realloc;
837     erts_allctrs[ERTS_ALC_A_SYSTEM].free		= erts_sys_free;
838     erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled	= 1;
839 
840     set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu);
841     set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu);
842     set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu);
843     set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu);
844     set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, ncpu);
845     set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc, ncpu);
846     set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu);
847     set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
848     set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
849     set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu);
850 #ifdef ERTS_ALC_A_EXEC
851     set_au_allocator(ERTS_ALC_A_EXEC, &init.exec_alloc, ncpu);
852 #endif
853     set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu);
854 
855     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
856 	if (!erts_allctrs[i].alloc)
857 	    erts_exit(ERTS_ABORT_EXIT,
858 		     "Missing alloc function for %s\n", ERTS_ALC_A2AD(i));
859 	if (!erts_allctrs[i].realloc)
860 	    erts_exit(ERTS_ABORT_EXIT,
861 		     "Missing realloc function for %s\n", ERTS_ALC_A2AD(i));
862 	if (!erts_allctrs[i].free)
863 	    erts_exit(ERTS_ABORT_EXIT,
864 		     "Missing free function for %s\n", ERTS_ALC_A2AD(i));
865     }
866 
867     sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold);
868     sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad);
869 
870     erts_mtrace_init(init.instr.mtrace, init.instr.nodename);
871 
872     start_au_allocator(ERTS_ALC_A_TEMPORARY,
873 		       &init.temp_alloc,
874 		       &temp_alloc_state);
875 
876     start_au_allocator(ERTS_ALC_A_SHORT_LIVED,
877 		       &init.sl_alloc,
878 		       &sl_alloc_state);
879 
880     start_au_allocator(ERTS_ALC_A_STANDARD,
881 		       &init.std_alloc,
882 		       &std_alloc_state);
883 
884     start_au_allocator(ERTS_ALC_A_LONG_LIVED,
885 		       &init.ll_alloc,
886 		       &ll_alloc_state);
887     start_au_allocator(ERTS_ALC_A_EHEAP,
888 		       &init.eheap_alloc,
889 		       &eheap_alloc_state);
890 
891     start_au_allocator(ERTS_ALC_A_BINARY,
892 		       &init.binary_alloc,
893 		       &binary_alloc_state);
894 
895     start_au_allocator(ERTS_ALC_A_ETS,
896 		       &init.ets_alloc,
897 		       &ets_alloc_state);
898 
899     start_au_allocator(ERTS_ALC_A_DRIVER,
900 		       &init.driver_alloc,
901 		       &driver_alloc_state);
902 
903     start_au_allocator(ERTS_ALC_A_FIXED_SIZE,
904 		       &init.fix_alloc,
905 		       &fix_alloc_state);
906     start_au_allocator(ERTS_ALC_A_LITERAL,
907                        &init.literal_alloc,
908                        &literal_alloc_state);
909 #ifdef ERTS_ALC_A_EXEC
910     start_au_allocator(ERTS_ALC_A_EXEC,
911                        &init.exec_alloc,
912                        &exec_alloc_state);
913 #endif
914     start_au_allocator(ERTS_ALC_A_TEST,
915 		       &init.test_alloc,
916 		       &test_alloc_state);
917 
918     erts_mtrace_install_wrapper_functions();
919 
920     init_aireq_alloc();
921 
922 #ifdef DEBUG
923     extra_block_size += install_debug_functions();
924 #endif
925     adjust_fix_alloc_sizes(extra_block_size);
926 }
927 
928 void
erts_alloc_late_init(void)929 erts_alloc_late_init(void)
930 {
931 
932 }
933 
934 static void *
erts_realloc_fixed_size(ErtsAlcType_t type,void * extra,void * p,Uint size)935 erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size)
936 {
937     erts_exit(ERTS_ABORT_EXIT,
938 	     "Attempt to reallocate a block of the fixed size type %s\n",
939 	     ERTS_ALC_T2TD(type));
940 }
941 
942 
943 static void
set_au_allocator(ErtsAlcType_t alctr_n,struct au_init * init,int ncpu)944 set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
945 {
946     ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
947     ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
948     ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
949 
950     /*
951      * Some allocators are forced on if halfword heap is used.
952      */
953     if (init->init.util.force)
954 	init->enable = 1;
955 
956     tspec->enabled = 0;
957     tspec->dd = 0;
958     tspec->aix = alctr_n;
959     tspec->size	= 0;
960     ai->thr_spec = 0;
961 
962     if (!init->enable) {
963 	af->alloc = erts_sys_alloc;
964 	af->realloc = erts_sys_realloc;
965 	af->free = erts_sys_free;
966 	af->extra = NULL;
967 	ai->alloc_util = 0;
968 	ai->enabled = 0;
969 	ai->extra = NULL;
970 	return;
971     }
972 
973     if (init->thr_spec) {
974 	if (init->thr_spec > 0) {
975 	    af->alloc = erts_alcu_alloc_thr_spec;
976 	    if (init->init.util.fix_type_size)
977 		af->realloc = erts_realloc_fixed_size;
978 	    else if (init->init.util.ramv)
979 		af->realloc = erts_alcu_realloc_mv_thr_spec;
980 	    else
981 		af->realloc = erts_alcu_realloc_thr_spec;
982 	    af->free = erts_alcu_free_thr_spec;
983 	}
984 	else {
985 	    af->alloc = erts_alcu_alloc_thr_pref;
986 	    if (init->init.util.fix_type_size)
987 		af->realloc = erts_realloc_fixed_size;
988 	    else if (init->init.util.ramv)
989 		af->realloc = erts_alcu_realloc_mv_thr_pref;
990 	    else
991 		af->realloc = erts_alcu_realloc_thr_pref;
992 	    af->free = erts_alcu_free_thr_pref;
993 	    tspec->dd = 1;
994 	}
995 
996 	tspec->enabled	= 1;
997 	tspec->size	= abs(init->thr_spec) + 1;
998 
999 	ai->thr_spec	= tspec->size;
1000     }
1001     else
1002 	if (init->init.util.ts) {
1003 	af->alloc = erts_alcu_alloc_ts;
1004 	if (init->init.util.fix_type_size)
1005 	    af->realloc = erts_realloc_fixed_size;
1006 	else if (init->init.util.ramv)
1007 	    af->realloc = erts_alcu_realloc_mv_ts;
1008 	else
1009 	    af->realloc = erts_alcu_realloc_ts;
1010 	af->free = erts_alcu_free_ts;
1011     }
1012     else
1013     {
1014         erts_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n",
1015                  init->init.util.name_prefix);
1016     }
1017     af->extra	= NULL;
1018     ai->alloc_util	= 1;
1019     ai->enabled		= 1;
1020 }
1021 
1022 static void
start_au_allocator(ErtsAlcType_t alctr_n,struct au_init * init,ErtsAllocatorState_t * state)1023 start_au_allocator(ErtsAlcType_t alctr_n,
1024 		   struct au_init *init,
1025 		   ErtsAllocatorState_t *state)
1026 {
1027     int i;
1028     int size = 1;
1029     void *as0;
1030     ErtsAlcStrat_t astrat;
1031     ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
1032     ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
1033     ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
1034     ErtsAlcFixList_t *fix_lists = NULL;
1035     size_t fix_list_size = 0;
1036 
1037     if (!init->enable)
1038 	return;
1039 
1040     if (init->thr_spec) {
1041 	char *states = erts_sys_alloc(0,
1042 				      NULL,
1043 				      ((sizeof(Allctr_t *)
1044 					* (tspec->size + 1))
1045 				       + (sizeof(ErtsAllocatorState_t)
1046 					  * tspec->size)
1047 				       + ERTS_CACHE_LINE_SIZE - 1));
1048 	if (!states)
1049 	    erts_exit(ERTS_ABORT_EXIT,
1050 		     "Failed to allocate allocator states for %salloc\n",
1051 		     init->init.util.name_prefix);
1052 	tspec->allctr = (Allctr_t **) states;
1053 	states += sizeof(Allctr_t *) * (tspec->size + 1);
1054 	states = ((((UWord) states) & ERTS_CACHE_LINE_MASK)
1055 		  ? (char *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK)
1056 			      + ERTS_CACHE_LINE_SIZE)
1057 		  : (char *) states);
1058 	tspec->allctr[0] = (Allctr_t *) state;
1059 	size = tspec->size;
1060 	for (i = 1; i < size; i++)
1061 	    tspec->allctr[i] = (Allctr_t *)
1062 		&((ErtsAllocatorState_t *) states)[i-1];
1063     }
1064 
1065     if (init->init.util.fix_type_size) {
1066 	size_t tot_fix_list_size;
1067 	fix_list_size = sizeof(ErtsAlcFixList_t)*ERTS_ALC_NO_FIXED_SIZES;
1068 	fix_list_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(fix_list_size);
1069 	tot_fix_list_size = fix_list_size;
1070 	if (init->thr_spec)
1071 	    tot_fix_list_size *= tspec->size;
1072 	fix_lists = erts_sys_alloc(0,
1073 				   NULL,
1074 				   (tot_fix_list_size
1075 				    + ERTS_CACHE_LINE_SIZE - 1));
1076 	if (!fix_lists)
1077 	    erts_exit(ERTS_ABORT_EXIT,
1078 		     "Failed to allocate fix lists for %salloc\n",
1079 		     init->init.util.name_prefix);
1080 
1081 	if (((UWord) fix_lists) & ERTS_CACHE_LINE_MASK)
1082 		fix_lists = ((ErtsAlcFixList_t *)
1083 		       ((((UWord) fix_lists) & ~ERTS_CACHE_LINE_MASK)
1084 			+ ERTS_CACHE_LINE_SIZE));
1085     }
1086 
1087     for (i = 0; i < size; i++) {
1088 	Allctr_t *as;
1089 	astrat = init->astrat;
1090 
1091 	if (!init->thr_spec)
1092 	    as0 = state;
1093 	else {
1094 	    as0 = (void *) tspec->allctr[i];
1095 	    if (!as0)
1096 		continue;
1097 	    if (init->thr_spec < 0) {
1098 		init->init.util.ts = i == 0;
1099 		init->init.util.tspec = 0;
1100 		init->init.util.tpref = -1*init->thr_spec + 1;
1101 	    }
1102 	    else {
1103 		if (i != 0)
1104 		    init->init.util.ts = 0;
1105 		else {
1106 		    if (astrat == ERTS_ALC_S_AFIT)
1107 			astrat = ERTS_ALC_S_GOODFIT;
1108 		    init->init.util.ts = 1;
1109 		}
1110 		init->init.util.tspec = init->thr_spec + 1;
1111 		init->init.util.tpref = 0;
1112 	    }
1113 	}
1114 
1115 	if (fix_lists) {
1116 	    init->init.util.fix = fix_lists;
1117 	    fix_lists = ((ErtsAlcFixList_t *)
1118 			 (((char *) fix_lists) + fix_list_size));
1119 	}
1120 
1121         init->init.util.alloc_strat = astrat;
1122 	init->init.util.ix = i;
1123 
1124 	switch (astrat) {
1125 	case ERTS_ALC_S_GOODFIT:
1126 	    as = erts_gfalc_start((GFAllctr_t *) as0,
1127 					   &init->init.gf,
1128 					   &init->init.util);
1129 	    break;
1130 	case ERTS_ALC_S_BESTFIT:
1131 	    as = erts_bfalc_start((BFAllctr_t *) as0,
1132 					   &init->init.bf,
1133 					   &init->init.util);
1134 	    break;
1135 	case ERTS_ALC_S_AFIT:
1136 	    as = erts_afalc_start((AFAllctr_t *) as0,
1137 					   &init->init.af,
1138 					   &init->init.util);
1139 	    break;
1140 	case ERTS_ALC_S_FIRSTFIT:
1141 	    as = erts_aoffalc_start((AOFFAllctr_t *) as0,
1142 					     &init->init.aoff,
1143 					     &init->init.util);
1144 	    break;
1145 
1146 	default:
1147 	    as = NULL;
1148 	    ASSERT(0);
1149 	}
1150 
1151 	if (!as)
1152 	    erts_exit(ERTS_ABORT_EXIT,
1153 		     "Failed to start %salloc\n", init->init.util.name_prefix);
1154 
1155 	ASSERT(as == (void *) as0);
1156 	af->extra = as;
1157     }
1158 
1159     if (init->thr_spec)
1160 	af->extra = tspec;
1161     ai->extra = af->extra;
1162 }
1163 
1164 
bad_param(char * param_start,char * param_end)1165 static void bad_param(char *param_start, char *param_end)
1166 {
1167     size_t len = param_end - param_start;
1168     char param[100];
1169     if (len > 99)
1170 	len = 99;
1171     sys_memcpy((void *) param, (void *) param_start, len);
1172     param[len] = '\0';
1173     erts_fprintf(stderr, "bad \"%s\" parameter\n", param);
1174     erts_usage();
1175 }
1176 
bad_value(char * param_start,char * param_end,char * value)1177 static void bad_value(char *param_start, char *param_end, char *value)
1178 {
1179     size_t len = param_end - param_start;
1180     char param[100];
1181     if (len > 99)
1182 	len = 99;
1183     sys_memcpy((void *) param, (void *) param_start, len);
1184     param[len] = '\0';
1185     erts_fprintf(stderr, "bad \"%s\" value: %s\n", param, value);
1186     erts_usage();
1187 }
1188 
1189 /* Get arg marks argument as handled by
1190    putting NULL in argv */
1191 static char *
get_value(char * rest,char ** argv,int * ip)1192 get_value(char* rest, char** argv, int* ip)
1193 {
1194     char *param = argv[*ip]+1;
1195     argv[*ip] = NULL;
1196     if (*rest == '\0') {
1197 	char *next = argv[*ip + 1];
1198 	if (next[0] == '-'
1199 	    && next[1] == '-'
1200 	    &&  next[2] == '\0') {
1201 	    bad_value(param, rest, "");
1202 	}
1203 	(*ip)++;
1204 	argv[*ip] = NULL;
1205 	return next;
1206     }
1207     return rest;
1208 }
1209 
1210 static ERTS_INLINE int
has_prefix(const char * prefix,const char * string)1211 has_prefix(const char *prefix, const char *string)
1212 {
1213     int i;
1214     for (i = 0; prefix[i]; i++)
1215 	if (prefix[i] != string[i])
1216 	    return 0;
1217     return 1;
1218 }
1219 
1220 static int
get_bool_value(char * param_end,char ** argv,int * ip)1221 get_bool_value(char *param_end, char** argv, int* ip)
1222 {
1223     char *param = argv[*ip]+1;
1224     char *value = get_value(param_end, argv, ip);
1225     if (sys_strcmp(value, "true") == 0)
1226 	return 1;
1227     else if (sys_strcmp(value, "false") == 0)
1228 	return 0;
1229     else
1230 	bad_value(param, param_end, value);
1231     return -1;
1232 }
1233 
kb_to_bytes(Sint kb,Uint * bytes)1234 static Uint kb_to_bytes(Sint kb, Uint *bytes)
1235 {
1236     const Uint max = ((~((Uint) 0))/1024) + 1;
1237 
1238     if (kb < 0 || (Uint)kb > max)
1239         return 0;
1240     if ((Uint)kb == max)
1241         *bytes = ~((Uint) 0);
1242     else
1243         *bytes = ((Uint) kb)*1024;
1244     return 1;
1245 }
1246 
1247 static Uint
get_kb_value(char * param_end,char ** argv,int * ip)1248 get_kb_value(char *param_end, char** argv, int* ip)
1249 {
1250     Sint tmp;
1251     Uint bytes = 0;
1252     char *rest;
1253     char *param = argv[*ip]+1;
1254     char *value = get_value(param_end, argv, ip);
1255     errno = 0;
1256     tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1257     if (errno != 0 || rest == value || !kb_to_bytes(tmp, &bytes))
1258 	bad_value(param, param_end, value);
1259     return bytes;
1260 }
1261 
1262 static UWord
get_mb_value(char * param_end,char ** argv,int * ip)1263 get_mb_value(char *param_end, char** argv, int* ip)
1264 {
1265     SWord tmp;
1266     UWord max = ((~((UWord) 0))/(1024*1024)) + 1;
1267     char *rest;
1268     char *param = argv[*ip]+1;
1269     char *value = get_value(param_end, argv, ip);
1270     errno = 0;
1271     tmp = (SWord) ErtsStrToSint(value, &rest, 10);
1272     if (errno != 0 || rest == value || tmp < 0 || max < ((UWord) tmp))
1273 	bad_value(param, param_end, value);
1274     if (max == (UWord) tmp)
1275 	return ~((UWord) 0);
1276     else
1277 	return ((UWord) tmp)*1024*1024;
1278 }
1279 
1280 
1281 #if 0
1282 static Uint
1283 get_byte_value(char *param_end, char** argv, int* ip)
1284 {
1285     Sint tmp;
1286     char *rest;
1287     char *param = argv[*ip]+1;
1288     char *value = get_value(param_end, argv, ip);
1289     errno = 0;
1290     tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1291     if (errno != 0 || rest == value || tmp < 0)
1292 	bad_value(param, param_end, value);
1293     return (Uint) tmp;
1294 }
1295 #endif
1296 
1297 static Uint
get_amount_value(char * param_end,char ** argv,int * ip)1298 get_amount_value(char *param_end, char** argv, int* ip)
1299 {
1300     Sint tmp;
1301     char *rest;
1302     char *param = argv[*ip]+1;
1303     char *value = get_value(param_end, argv, ip);
1304     errno = 0;
1305     tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1306     if (errno != 0 || rest == value || tmp < 0)
1307 	bad_value(param, param_end, value);
1308     return (Uint) tmp;
1309 }
1310 
1311 static Uint
get_acul_value(struct au_init * auip,char * param_end,char ** argv,int * ip)1312 get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip)
1313 {
1314     Sint tmp;
1315     char *rest;
1316     char *param = argv[*ip]+1;
1317     char *value = get_value(param_end, argv, ip);
1318     if (sys_strcmp(value, "de") == 0) {
1319 	switch (auip->init.util.alloc_no) {
1320 	case ERTS_ALC_A_LONG_LIVED:
1321 	    return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC;
1322 	case ERTS_ALC_A_EHEAP:
1323 	    return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC;
1324 	default:
1325 	    return ERTS_ALC_DEFAULT_ENABLED_ACUL;
1326 	}
1327     }
1328     errno = 0;
1329     tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1330     if (errno != 0 || rest == value || tmp < 0 || 100 < tmp)
1331 	bad_value(param, param_end, value);
1332     return (Uint) tmp;
1333 }
1334 
1335 static void
handle_au_arg(struct au_init * auip,char * sub_param,char ** argv,int * ip,int u_switch)1336 handle_au_arg(struct au_init *auip,
1337 	      char* sub_param,
1338 	      char** argv,
1339 	      int* ip,
1340 	      int u_switch)
1341 {
1342     char *param = argv[*ip]+1;
1343 
1344     switch (sub_param[0]) {
1345     case 'a':
1346         if (sub_param[1] == 'c') { /* Migration parameters "ac*" */
1347             UWord value;
1348             UWord* wp;
1349             if (!auip->carrier_migration_allowed && !u_switch)
1350                 goto bad_switch;
1351 
1352             if (has_prefix("acul", sub_param)) {
1353                 value = get_acul_value(auip, sub_param + 4, argv, ip);
1354                 wp = &auip->init.util.acul;
1355             }
1356             else if (has_prefix("acnl", sub_param)) {
1357                 value = get_amount_value(sub_param + 4, argv, ip);
1358                 wp = &auip->init.util.acnl;
1359             }
1360             else if (has_prefix("acfml", sub_param)) {
1361                 value = get_amount_value(sub_param + 5, argv, ip);
1362                 wp = &auip->init.util.acfml;
1363             }
1364             else
1365                 goto bad_switch;
1366 
1367             if (auip->carrier_migration_allowed)
1368                 *wp = value;
1369         }
1370 	else if(has_prefix("asbcst", sub_param)) {
1371 	    auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip);
1372 	}
1373 	else if(has_prefix("as", sub_param)) {
1374 	    char *alg = get_value(sub_param + 2, argv, ip);
1375 	    if (sys_strcmp("bf", alg) == 0) {
1376 		auip->astrat = ERTS_ALC_S_BESTFIT;
1377 		auip->init.bf.ao = 0;
1378 	    }
1379 	    else if (sys_strcmp("aobf", alg) == 0) {
1380 		auip->astrat = ERTS_ALC_S_BESTFIT;
1381 		auip->init.bf.ao = 1;
1382 	    }
1383 	    else if (sys_strcmp("gf", alg) == 0) {
1384 		auip->astrat = ERTS_ALC_S_GOODFIT;
1385 	    }
1386 	    else if (sys_strcmp("af", alg) == 0) {
1387 		auip->astrat = ERTS_ALC_S_AFIT;
1388 	    }
1389 	    else if (sys_strcmp("aoff", alg) == 0) {
1390 		auip->astrat = ERTS_ALC_S_FIRSTFIT;
1391 		auip->init.aoff.crr_order = FF_AOFF;
1392 		auip->init.aoff.blk_order = FF_AOFF;
1393 	    }
1394 	    else if (sys_strcmp("aoffcbf", alg) == 0) {
1395 		auip->astrat = ERTS_ALC_S_FIRSTFIT;
1396 		auip->init.aoff.crr_order = FF_AOFF;
1397 		auip->init.aoff.blk_order = FF_BF;
1398 	    }
1399 	    else if (sys_strcmp("aoffcaobf", alg) == 0) {
1400 		auip->astrat = ERTS_ALC_S_FIRSTFIT;
1401 		auip->init.aoff.crr_order = FF_AOFF;
1402 		auip->init.aoff.blk_order = FF_AOBF;
1403 	    }
1404             else if (sys_strcmp("ageffcaoff", alg) == 0) {
1405                 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1406 		auip->init.aoff.crr_order = FF_AGEFF;
1407 		auip->init.aoff.blk_order = FF_AOFF;
1408             }
1409             else if (sys_strcmp("ageffcbf", alg) == 0) {
1410                 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1411 		auip->init.aoff.crr_order = FF_AGEFF;
1412 		auip->init.aoff.blk_order = FF_BF;
1413             }
1414             else if (sys_strcmp("ageffcaobf", alg) == 0) {
1415                 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1416 		auip->init.aoff.crr_order = FF_AGEFF;
1417 		auip->init.aoff.blk_order = FF_AOBF;
1418             }
1419 	    else {
1420                 if (auip->init.util.alloc_no == ERTS_ALC_A_TEST
1421                     && sys_strcmp("chaosff", alg) == 0) {
1422                     auip->astrat = ERTS_ALC_S_FIRSTFIT;
1423                     auip->init.aoff.crr_order = FF_CHAOS;
1424                     auip->init.aoff.blk_order = FF_CHAOS;
1425                 }
1426                 else {
1427                     bad_value(param, sub_param + 1, alg);
1428                 }
1429 	    }
1430 	    if (!strategy_support_carrier_migration(auip))
1431 		auip->init.util.acul = 0;
1432 	} else if (has_prefix("atags", sub_param)) {
1433             auip->init.util.atags = get_bool_value(sub_param + 5, argv, ip);
1434         }
1435 	else
1436 	    goto bad_switch;
1437 	break;
1438     case 'c': {
1439         if (has_prefix("cp", sub_param)) {
1440             char *param, *param_end, *value;
1441             int cp;
1442             if (!auip->carrier_migration_allowed && !u_switch)
1443                 goto bad_switch;
1444             param = argv[*ip]+1;
1445             param_end = sub_param + 2;
1446             value = get_value(param_end, argv, ip);
1447             if (value[0] == '\0' || value[1] != '\0')
1448                 bad_value(param, param_end, value);
1449             switch (value[0]) {
1450             case 'B': cp = ERTS_ALC_A_BINARY; break;
1451             case 'D': cp = ERTS_ALC_A_STANDARD; break;
1452             case 'E': cp = ERTS_ALC_A_ETS; break;
1453             case 'F': cp = ERTS_ALC_A_FIXED_SIZE; break;
1454             case 'H': cp = ERTS_ALC_A_EHEAP; break;
1455             case 'L': cp = ERTS_ALC_A_LONG_LIVED; break;
1456             case 'R': cp = ERTS_ALC_A_DRIVER; break;
1457             case 'S': cp = ERTS_ALC_A_SHORT_LIVED; break;
1458             case '@': cp = ERTS_ALC_COMMON_CPOOL_IX; break;
1459             case ':': cp = auip->init.util.alloc_no; break;
1460             default:  cp = -1;
1461                 bad_value(param, param_end, value);
1462                 break;
1463             }
1464             if (auip->carrier_migration_allowed)
1465                 auip->init.util.cp = cp;
1466         }
1467         else
1468             goto bad_switch;
1469         break;
1470     }
1471     case 'e': {
1472 	int e = get_bool_value(sub_param + 1, argv, ip);
1473         if (!auip->disable_allowed && !e) {
1474             if (!u_switch)
1475                 bad_value(param, sub_param + 1, "false");
1476 	    else
1477 		ASSERT(auip->enable); /* ignore */
1478         }
1479 	else auip->enable = e;
1480 	break;
1481     }
1482     case 'l':
1483 	if (has_prefix("lmbcs", sub_param)) {
1484 	    auip->default_.lmbcs = 0;
1485 	    auip->init.util.lmbcs = get_kb_value(sub_param + 5, argv, ip);
1486 	}
1487 	else
1488 	    goto bad_switch;
1489 	break;
1490     case 'm':
1491 	if (has_prefix("mbcgs", sub_param)) {
1492 	    auip->init.util.mbcgs = get_amount_value(sub_param + 5, argv, ip);
1493 
1494 	}
1495 	else if (has_prefix("mbsd", sub_param)) {
1496 	    auip->init.gf.mbsd = get_amount_value(sub_param + 4, argv, ip);
1497 	    if (auip->init.gf.mbsd < 1)
1498 		auip->init.gf.mbsd = 1;
1499 	}
1500 	else if (has_prefix("mmbcs", sub_param)) {
1501 	    auip->default_.mmbcs = 0;
1502 	    auip->init.util.mmbcs = get_kb_value(sub_param + 5, argv, ip);
1503 	}
1504 	else if (has_prefix("mmmbc", sub_param)) {
1505 	    auip->default_.mmmbc = 0;
1506 	    auip->init.util.mmmbc = get_amount_value(sub_param + 5, argv, ip);
1507 	}
1508 	else if (has_prefix("mmsbc", sub_param)) {
1509 	    auip->init.util.mmsbc = get_amount_value(sub_param + 5, argv, ip);
1510 	}
1511 	else
1512 	    goto bad_switch;
1513 	break;
1514     case 'r':
1515 	if(has_prefix("rsbcmt", sub_param)) {
1516 	    auip->init.util.rsbcmt = get_amount_value(sub_param + 6, argv, ip);
1517 	    if (auip->init.util.rsbcmt > 100)
1518 		auip->init.util.rsbcmt = 100;
1519 	}
1520 	else if(has_prefix("rsbcst", sub_param)) {
1521 	    auip->init.util.rsbcst = get_amount_value(sub_param + 6, argv, ip);
1522 	    if (auip->init.util.rsbcst > 100)
1523 		auip->init.util.rsbcst = 100;
1524 	}
1525 	else if (has_prefix("rmbcmt", sub_param)) {
1526 	    auip->init.util.rmbcmt = get_amount_value(sub_param + 6, argv, ip);
1527 	    if (auip->init.util.rmbcmt > 100)
1528 		auip->init.util.rmbcmt = 100;
1529 	}
1530 	else if (has_prefix("ramv", sub_param)) {
1531 	    auip->init.util.ramv = get_bool_value(sub_param + 4, argv, ip);
1532 	}
1533 	else
1534 	    goto bad_switch;
1535 	break;
1536     case 's':
1537 	if(has_prefix("sbct", sub_param)) {
1538 	    auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip);
1539 	}
1540 	else if (has_prefix("smbcs", sub_param)) {
1541 	    auip->default_.smbcs = 0;
1542 	    auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip);
1543 	}
1544 	else
1545 	    goto bad_switch;
1546 	break;
1547     case 't': {
1548 	int res = get_bool_value(sub_param+1, argv, ip);
1549 	if (res > 0) {
1550 	    if (!auip->thr_spec_allowed) {
1551 		if (!u_switch)
1552                     bad_value(param, sub_param + 1, "true");
1553 		else
1554 		    ASSERT(!auip->thr_spec); /* ignore */
1555 	    }
1556 	    else
1557 		auip->thr_spec = 1;
1558 	    break;
1559 	}
1560 	else if (res == 0) {
1561 	    auip->thr_spec = 0;
1562 	    auip->init.util.acul = 0;
1563 	    break;
1564 	}
1565 	goto bad_switch;
1566     }
1567     default:
1568     bad_switch:
1569 	bad_param(param, sub_param);
1570     }
1571 }
1572 
1573 static void
handle_args(int * argc,char ** argv,erts_alc_hndl_args_init_t * init)1574 handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
1575 {
1576     struct au_init *aui[] = {
1577 	&init->binary_alloc,
1578 	&init->std_alloc,
1579 	&init->ets_alloc,
1580 	&init->eheap_alloc,
1581 	&init->ll_alloc,
1582 	&init->driver_alloc,
1583 	&init->fix_alloc,
1584 	&init->sl_alloc
1585 	/* test_alloc not affected by +Mea??? or +Mu???  */
1586     };
1587     int aui_sz = (int) sizeof(aui)/sizeof(aui[0]);
1588     char *arg;
1589     char *rest;
1590     int i, j;
1591 
1592     i = 1;
1593 
1594     ASSERT(argc && argv && init);
1595 
1596     while (i < *argc) {
1597 	if(argv[i][0] == '-') {
1598 	    char *param = argv[i]+1;
1599 	    switch (argv[i][1]) {
1600 	    case 'M':
1601 		switch (argv[i][2]) {
1602 		case 'B':
1603 		    handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0);
1604 		    break;
1605                 case 'I':
1606                     if (has_prefix("scs", argv[i]+3)) {
1607 #if HAVE_ERTS_MSEG
1608 			init->mseg.literal_mmap.scs =
1609 #endif
1610 			    get_mb_value(argv[i]+6, argv, &i);
1611 		    }
1612                     else
1613                         handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0);
1614 		    break;
1615                 case 'X':
1616                     if (has_prefix("scs", argv[i]+3)) {
1617                         /* Ignore obsolete */
1618                         (void) get_mb_value(argv[i]+6, argv, &i);
1619                     }
1620                     else
1621                         handle_au_arg(&init->exec_alloc, &argv[i][3], argv, &i, 0);
1622                     break;
1623 		case 'D':
1624 		    handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0);
1625 		    break;
1626 		case 'E':
1627 		    handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i, 0);
1628 		    break;
1629 		case 'F':
1630 		    handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i, 0);
1631 		    break;
1632 		case 'H':
1633 		    handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i, 0);
1634 		    break;
1635 		case 'L':
1636 		    handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i, 0);
1637 		    break;
1638 		case 'M':
1639 		    if (has_prefix("amcbf", argv[i]+3)) {
1640 #if HAVE_ERTS_MSEG
1641 			init->mseg.amcbf =
1642 #endif
1643 			    get_kb_value(argv[i]+8, argv, &i);
1644 		    }
1645 		    else if (has_prefix("rmcbf", argv[i]+3)) {
1646 #if HAVE_ERTS_MSEG
1647 			init->mseg.rmcbf =
1648 #endif
1649 			    get_amount_value(argv[i]+8, argv, &i);
1650 		    }
1651 		    else if (has_prefix("mcs", argv[i]+3)) {
1652 #if HAVE_ERTS_MSEG
1653 			init->mseg.mcs =
1654 #endif
1655 			    get_amount_value(argv[i]+6, argv, &i);
1656 		    }
1657 		    else if (has_prefix("scs", argv[i]+3)) {
1658 #if HAVE_ERTS_MSEG
1659 			init->mseg.dflt_mmap.scs =
1660 #endif
1661 			    get_mb_value(argv[i]+6, argv, &i);
1662 		    }
1663 		    else if (has_prefix("sco", argv[i]+3)) {
1664 #if HAVE_ERTS_MSEG
1665 			init->mseg.dflt_mmap.sco =
1666 #endif
1667 			    get_bool_value(argv[i]+6, argv, &i);
1668 		    }
1669 		    else if (has_prefix("scrpm", argv[i]+3)) {
1670 #if HAVE_ERTS_MSEG
1671 			init->mseg.dflt_mmap.scrpm =
1672 #endif
1673 			    get_bool_value(argv[i]+8, argv, &i);
1674 		    }
1675 		    else if (has_prefix("scrfsd", argv[i]+3)) {
1676 #if HAVE_ERTS_MSEG
1677 			init->mseg.dflt_mmap.scrfsd =
1678 #endif
1679 			    get_amount_value(argv[i]+9, argv, &i);
1680 		    }
1681 		    else {
1682 			bad_param(param, param+2);
1683 		    }
1684 		    break;
1685 		case 'R':
1686 		    handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i, 0);
1687 		    break;
1688 		case 'S':
1689 		    handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i, 0);
1690 		    break;
1691 		case 'T':
1692 		    handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0);
1693 		    break;
1694 		case 'Z':
1695 		    handle_au_arg(&init->test_alloc, &argv[i][3], argv, &i, 0);
1696 		    break;
1697 		case 'Y': { /* sys_alloc */
1698 		    if (has_prefix("tt", param+2)) {
1699 			/* set trim threshold */
1700 			arg = get_value(param+4, argv, &i);
1701 			errno = 0;
1702 			init->trim_threshold = (int) strtol(arg, &rest, 10);
1703 			if (errno != 0
1704 			    || rest == arg
1705 			    || init->trim_threshold < 0
1706 			    || (INT_MAX/1024) < init->trim_threshold) {
1707 			    bad_value(param, param+4, arg);
1708 			}
1709 			VERBOSE(DEBUG_SYSTEM,
1710                                 ("using trim threshold: %d\n",
1711                                  init->trim_threshold));
1712 			init->trim_threshold *= 1024;
1713 		    }
1714 		    else if (has_prefix("tp", param+2)) {
1715 			/* set top pad */
1716 			arg = get_value(param+4, argv, &i);
1717 			errno = 0;
1718 			init->top_pad = (int) strtol(arg, &rest, 10);
1719 			if (errno != 0
1720 			    || rest == arg
1721 			    || init->top_pad < 0
1722 			    || (INT_MAX/1024) < init->top_pad) {
1723 			    bad_value(param, param+4, arg);
1724 			}
1725 			VERBOSE(DEBUG_SYSTEM,
1726                                 ("using top pad: %d\n",init->top_pad));
1727 			init->top_pad *= 1024;
1728 		    }
1729 		    else if (has_prefix("m", param+2)) {
1730 			/* Has been handled by erlexec */
1731 			(void) get_value(param+3, argv, &i);
1732 		    }
1733 		    else if (has_prefix("e", param+2)) {
1734 			arg = get_value(param+3, argv, &i);
1735 			if (sys_strcmp("true", arg) != 0)
1736 			    bad_value(param, param+3, arg);
1737 		    }
1738 		    else
1739 			bad_param(param, param+2);
1740 		    break;
1741 		}
1742 		case 'e':
1743 		    switch (argv[i][3]) {
1744 		    case 'a': {
1745 			int a;
1746 			arg = get_value(argv[i]+4, argv, &i);
1747 			if (sys_strcmp("min", arg) == 0) {
1748 			    for (a = 0; a < aui_sz; a++)
1749 				aui[a]->enable = 0;
1750 			}
1751 			else if (sys_strcmp("max", arg) == 0) {
1752 			    for (a = 0; a < aui_sz; a++)
1753 				aui[a]->enable = 1;
1754 			}
1755 			else if (sys_strcmp("config", arg) == 0) {
1756 			    init->erts_alloc_config = 1;
1757 			}
1758 			else if (sys_strcmp("r9c", arg) == 0
1759 				 || sys_strcmp("r10b", arg) == 0
1760 				 || sys_strcmp("r11b", arg) == 0) {
1761 			    set_default_sl_alloc_opts(&init->sl_alloc);
1762 			    set_default_std_alloc_opts(&init->std_alloc);
1763 			    set_default_ll_alloc_opts(&init->ll_alloc);
1764 			    set_default_temp_alloc_opts(&init->temp_alloc);
1765 			    set_default_eheap_alloc_opts(&init->eheap_alloc);
1766 			    set_default_binary_alloc_opts(&init->binary_alloc);
1767 			    set_default_ets_alloc_opts(&init->ets_alloc);
1768 			    set_default_driver_alloc_opts(&init->driver_alloc);
1769 			    set_default_driver_alloc_opts(&init->fix_alloc);
1770 
1771 			    init->driver_alloc.enable = 0;
1772 			    if (sys_strcmp("r9c", arg) == 0) {
1773 				init->sl_alloc.enable = 0;
1774 				init->std_alloc.enable = 0;
1775 				init->binary_alloc.enable = 0;
1776 				init->ets_alloc.enable = 0;
1777 			    }
1778 
1779 			    for (a = 0; a < aui_sz; a++) {
1780 				aui[a]->thr_spec = 0;
1781 				aui[a]->init.util.acul = 0;
1782 				aui[a]->init.util.ramv = 0;
1783 				aui[a]->init.util.lmbcs = 5*1024*1024;
1784 			    }
1785 			}
1786 			else {
1787 			    bad_param(param, param+3);
1788 			}
1789 			break;
1790 		    }
1791 		    default:
1792 			bad_param(param, param+1);
1793 		    }
1794 		    break;
1795 		case 'i':
1796 		    switch (argv[i][3]) {
1797 		    case 't':
1798 			init->instr.mtrace = get_value(argv[i]+4, argv, &i);
1799 			break;
1800 		    default:
1801 			bad_param(param, param+2);
1802 		    }
1803 		    break;
1804 		case 'l':
1805 		    if (has_prefix("pm", param+2)) {
1806 			arg = get_value(argv[i]+5, argv, &i);
1807 			if (sys_strcmp("all", arg) == 0)
1808 			    lock_all_physical_memory = 1;
1809 			else if (sys_strcmp("no", arg) == 0)
1810 			    lock_all_physical_memory = 0;
1811 			else
1812 			    bad_value(param, param+4, arg);
1813 			break;
1814 		    }
1815 		    bad_param(param, param+2);
1816 		    break;
1817 		case 'u':
1818 		    if (has_prefix("ycs", argv[i]+3)) {
1819 			init->alloc_util.ycs
1820 			    = get_kb_value(argv[i]+6, argv, &i);
1821 		    }
1822 		    else if (has_prefix("mmc", argv[i]+3)) {
1823 			init->alloc_util.mmc
1824 			    = get_amount_value(argv[i]+6, argv, &i);
1825 		    }
1826 		    else if (has_prefix("sac", argv[i]+3)) {
1827 			init->alloc_util.sac
1828 			    = get_bool_value(argv[i]+6, argv, &i);
1829 		    }
1830 		    else {
1831 			int a;
1832 			int start = i;
1833 			char *param = argv[i];
1834 			char *val = i+1 < *argc ? argv[i+1] : NULL;
1835 
1836 			for (a = 0; a < aui_sz; a++) {
1837 			    if (a > 0) {
1838 				ASSERT(i == start || i == start+1);
1839 				argv[start] = param;
1840 				if (i != start)
1841 				    argv[start + 1] = val;
1842 				i = start;
1843 			    }
1844 			    handle_au_arg(aui[a], &argv[i][3], argv, &i, 1);
1845 			}
1846 		    }
1847 		    break;
1848 		default:
1849 		    bad_param(param, param+1);
1850 		}
1851 		break;
1852 	    case '-':
1853 		if (argv[i][2] == '\0') {
1854 		    /* End of system flags reached */
1855 		    if (init->instr.mtrace) {
1856 			while (i < *argc) {
1857 			    if(sys_strcmp(argv[i], "-sname") == 0
1858 			       || sys_strcmp(argv[i], "-name") == 0) {
1859 				if (i + 1 <*argc) {
1860 				    init->instr.nodename = argv[i+1];
1861 				    break;
1862 				}
1863 			    }
1864 			    i++;
1865 			}
1866 		    }
1867 		    goto args_parsed;
1868 		}
1869 		break;
1870 	    default:
1871 		break;
1872 	    }
1873 	}
1874 	i++;
1875     }
1876 
1877  args_parsed:
1878     /* Handled arguments have been marked with NULL. Slide arguments
1879        not handled towards the beginning of argv. */
1880     for (i = 0, j = 0; i < *argc; i++) {
1881 	if (argv[i])
1882 	    argv[j++] = argv[i];
1883     }
1884     *argc = j;
1885 }
1886 
type_no_str(ErtsAlcType_t n)1887 static char *type_no_str(ErtsAlcType_t n)
1888 {
1889 
1890 #if ERTS_ALC_N_MIN != 0
1891     if (n < ERTS_ALC_N_MIN)
1892 	return NULL;
1893 #endif
1894     if (n > ERTS_ALC_N_MAX)
1895 	return NULL;
1896     return (char *) ERTS_ALC_N2TD(n);
1897 }
1898 
1899 #define type_str(T) type_no_str(ERTS_ALC_T2N((T)))
1900 
1901 void
erts_alloc_register_scheduler(void * vesdp)1902 erts_alloc_register_scheduler(void *vesdp)
1903 {
1904     ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
1905     int ix = (int) esdp->no;
1906     int aix;
1907 
1908     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
1909     for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
1910 	ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
1911 	esdp->alloc_data.deallctr[aix] = NULL;
1912 	esdp->alloc_data.pref_ix[aix] = -1;
1913 	if (tspec->enabled) {
1914 	    if (!tspec->dd)
1915 		esdp->alloc_data.pref_ix[aix] = ix;
1916 	    else {
1917 		Allctr_t *allctr = tspec->allctr[ix];
1918 		ASSERT(allctr);
1919 		esdp->alloc_data.deallctr[aix] = allctr;
1920 		esdp->alloc_data.pref_ix[aix] = ix;
1921 	    }
1922 	}
1923     }
1924 }
1925 
1926 void
erts_alloc_scheduler_handle_delayed_dealloc(void * vesdp,int * need_thr_progress,ErtsThrPrgrVal * thr_prgr_p,int * more_work)1927 erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
1928 					    int *need_thr_progress,
1929 					    ErtsThrPrgrVal *thr_prgr_p,
1930 					    int *more_work)
1931 {
1932     ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
1933     int aix;
1934     for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
1935 	Allctr_t *allctr;
1936 	if (esdp)
1937 	    allctr = esdp->alloc_data.deallctr[aix];
1938 	else {
1939 	    ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
1940 	    if (tspec->enabled && tspec->dd)
1941 		allctr = tspec->allctr[0];
1942 	    else
1943 		allctr = NULL;
1944 	}
1945 	if (allctr) {
1946 	    erts_alcu_check_delayed_dealloc(allctr,
1947 					    1,
1948 					    need_thr_progress,
1949 					    thr_prgr_p,
1950 					    more_work);
1951 	}
1952     }
1953 }
1954 
1955 erts_aint32_t
erts_alloc_fix_alloc_shrink(int ix,erts_aint32_t flgs)1956 erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
1957 {
1958     ErtsAllocatorThrSpec_t *tspec;
1959     tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
1960     if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec && tspec->enabled)
1961 	return erts_alcu_fix_alloc_shrink(tspec->allctr[ix], flgs);
1962     if (ix == 0 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
1963 	return erts_alcu_fix_alloc_shrink(
1964 	    erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
1965     return 0;
1966 }
1967 
1968 static void
no_verify(Allctr_t * allctr)1969 no_verify(Allctr_t *allctr)
1970 {
1971 
1972 }
1973 
1974 erts_alloc_verify_func_t
erts_alloc_get_verify_unused_temp_alloc(Allctr_t ** allctr)1975 erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr)
1976 {
1977     if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util
1978 	&& erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) {
1979 	ErtsAllocatorThrSpec_t *tspec;
1980 	int ix = ERTS_ALC_GET_THR_IX();
1981 	tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY];
1982 
1983 	if (ix < tspec->size) {
1984 	    *allctr = tspec->allctr[ix];
1985 	    return erts_alcu_verify_unused;
1986 	}
1987     }
1988 
1989     *allctr = NULL;
1990     return no_verify;
1991 }
1992 
1993 __decl_noreturn void
erts_alc_fatal_error(int error,int func,ErtsAlcType_t n,...)1994 erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
1995 {
1996     char buf[10];
1997     char *t_str;
1998     char *allctr_str;
1999 
2000     ASSERT(n >= ERTS_ALC_N_MIN);
2001     ASSERT(n <= ERTS_ALC_N_MAX);
2002 
2003 
2004     if (n < ERTS_ALC_N_MIN || ERTS_ALC_N_MAX < n)
2005 	allctr_str = "UNKNOWN";
2006     else {
2007 	ErtsAlcType_t a = ERTS_ALC_T2A(ERTS_ALC_N2T(n));
2008 	if (erts_allctrs_info[a].enabled)
2009 	    allctr_str = (char *) ERTS_ALC_A2AD(a);
2010 	else
2011 	    allctr_str = (char *) ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM);
2012     }
2013 
2014     t_str = type_no_str(n);
2015     if (!t_str) {
2016 	erts_snprintf(buf, sizeof(buf), "%d", (int) n);
2017 	t_str = buf;
2018     }
2019 
2020     switch (error) {
2021     case ERTS_ALC_E_NOTSUP: {
2022 	char *op_str;
2023 	switch (func) {
2024 	case ERTS_ALC_O_ALLOC:		op_str = "alloc";	break;
2025 	case ERTS_ALC_O_REALLOC:	op_str = "realloc";	break;
2026 	case ERTS_ALC_O_FREE:		op_str = "free";	break;
2027 	default:			op_str = "UNKNOWN";	break;
2028 	}
2029 	erts_exit(ERTS_ABORT_EXIT,
2030 		 "%s: %s operation not supported (memory type: \"%s\")\n",
2031 		 allctr_str, op_str, t_str);
2032 	break;
2033     }
2034     case ERTS_ALC_E_NOMEM: {
2035 	Uint size;
2036 	va_list argp;
2037 	char *op = func == ERTS_ALC_O_REALLOC ? "reallocate" : "allocate";
2038 
2039 
2040 	va_start(argp, n);
2041 	size = va_arg(argp, Uint);
2042 	va_end(argp);
2043 	erts_exit(ERTS_DUMP_EXIT,
2044 		 "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n",
2045 		 allctr_str, op, size, t_str);
2046 	break;
2047     }
2048     case ERTS_ALC_E_NOALLCTR:
2049 	erts_exit(ERTS_ABORT_EXIT,
2050 		 "erts_alloc: Unknown allocator type: %d\n",
2051 		 ERTS_ALC_T2A(ERTS_ALC_N2T(n)));
2052 	break;
2053     default:
2054 	erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error);
2055 	break;
2056     }
2057 }
2058 
2059 __decl_noreturn void
erts_alloc_enomem(ErtsAlcType_t type,Uint size)2060 erts_alloc_enomem(ErtsAlcType_t type, Uint size)
2061 {
2062     erts_alloc_n_enomem(ERTS_ALC_T2N(type), size);
2063 }
2064 
2065 __decl_noreturn void
erts_alloc_n_enomem(ErtsAlcType_t n,Uint size)2066 erts_alloc_n_enomem(ErtsAlcType_t n, Uint size)
2067 {
2068     erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_ALLOC, n, size);
2069 }
2070 
2071 __decl_noreturn void
erts_realloc_enomem(ErtsAlcType_t type,void * ptr,Uint size)2072 erts_realloc_enomem(ErtsAlcType_t type, void *ptr, Uint size)
2073 {
2074     erts_realloc_n_enomem(ERTS_ALC_T2N(type), ptr, size);
2075 }
2076 
2077 __decl_noreturn void
erts_realloc_n_enomem(ErtsAlcType_t n,void * ptr,Uint size)2078 erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size)
2079 {
2080     erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size);
2081 }
2082 
2083 static ERTS_INLINE UWord
alcu_size(ErtsAlcType_t alloc_no,ErtsAlcUFixInfo_t * fi,int fisz)2084 alcu_size(ErtsAlcType_t alloc_no, ErtsAlcUFixInfo_t *fi, int fisz)
2085 {
2086     UWord res;
2087     int ai;
2088 
2089     if (!erts_allctrs_info[alloc_no].thr_spec) {
2090         AllctrSize_t size;
2091         Allctr_t *allctr;
2092 
2093         allctr = erts_allctrs_info[alloc_no].extra;
2094         erts_alcu_current_size(allctr, &size, fi, fisz);
2095 
2096         return size.blocks;
2097     }
2098 
2099     res = 0;
2100 
2101     /* Thread-specific allocators can migrate carriers across types, so we have
2102      * to visit every allocator type to gather information on blocks that were
2103      * allocated by us. */
2104     for (ai = ERTS_ALC_A_MIN; ai < ERTS_ALC_A_MAX; ai++) {
2105         ErtsAllocatorThrSpec_t *tspec;
2106         Allctr_t *allctr;
2107         int i;
2108 
2109         if (!erts_allctrs_info[ai].thr_spec) {
2110             continue;
2111         }
2112 
2113         tspec = &erts_allctr_thr_spec[ai];
2114         ASSERT(tspec->enabled);
2115 
2116         for (i = tspec->size - 1; i >= 0; i--) {
2117             allctr = tspec->allctr[i];
2118 
2119             if (allctr) {
2120                 AllctrSize_t size;
2121 
2122                 if (ai == alloc_no) {
2123                     erts_alcu_current_size(allctr, &size, fi, fisz);
2124                 } else {
2125                     erts_alcu_foreign_size(allctr, alloc_no, &size);
2126                 }
2127 
2128                 ASSERT(((SWord)size.blocks) >= 0);
2129 
2130                 res += size.blocks;
2131             }
2132         }
2133     }
2134 
2135     return res;
2136 }
2137 
2138 static ERTS_INLINE void
add_fix_values(UWord * ap,UWord * up,ErtsAlcUFixInfo_t * fi,ErtsAlcType_t type)2139 add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type)
2140 {
2141     int ix = ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE;
2142     ASSERT(0 <= ix && ix < ERTS_ALC_NO_FIXED_SIZES);
2143 
2144     *ap += (UWord) fi[ix].allocated;
2145     *up += (UWord) fi[ix].used;
2146 }
2147 
2148 Eterm
erts_memory(fmtfn_t * print_to_p,void * print_to_arg,void * proc,Eterm earg)2149 erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
2150 {
2151 /*
2152  * NOTE! When updating this function, make sure to also update
2153  *       erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl
2154  */
2155 #define ERTS_MEM_NEED_ALL_ALCU (want_tot_or_sys)
2156     struct {
2157 	int total;
2158 	int processes;
2159 	int processes_used;
2160 	int system;
2161 	int atom;
2162 	int atom_used;
2163 	int binary;
2164 	int code;
2165 	int ets;
2166     } want = {0};
2167     struct {
2168 	UWord total;
2169 	UWord processes;
2170 	UWord processes_used;
2171 	UWord system;
2172 	UWord atom;
2173 	UWord atom_used;
2174 	UWord binary;
2175 	UWord code;
2176 	UWord ets;
2177     } size = {0};
2178     Eterm atoms[sizeof(size)/sizeof(UWord)];
2179     UWord *uintps[sizeof(size)/sizeof(UWord)];
2180     Eterm euints[sizeof(size)/sizeof(UWord)];
2181     int want_tot_or_sys;
2182     int length;
2183     Eterm res = THE_NON_VALUE;
2184     ErtsAlcType_t ai;
2185     int only_one_value = 0;
2186     ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}};
2187 
2188     ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
2189 
2190     /* Figure out whats wanted... */
2191 
2192     length = 0;
2193     if (is_non_value(earg)) { /* i.e. wants all */
2194 	want.total = 1;
2195 	atoms[length] = am_total;
2196 	uintps[length++] = &size.total;
2197 
2198 	want.processes = 1;
2199 	atoms[length] = am_processes;
2200 	uintps[length++] = &size.processes;
2201 
2202 	want.processes_used = 1;
2203 	atoms[length] = am_processes_used;
2204 	uintps[length++] = &size.processes_used;
2205 
2206 	want.system = 1;
2207 	atoms[length] = am_system;
2208 	uintps[length++] = &size.system;
2209 
2210 	want.atom = 1;
2211 	atoms[length] = am_atom;
2212 	uintps[length++] = &size.atom;
2213 
2214 	want.atom_used = 1;
2215 	atoms[length] = am_atom_used;
2216 	uintps[length++] = &size.atom_used;
2217 
2218 	want.binary = 1;
2219 	atoms[length] = am_binary;
2220 	uintps[length++] = &size.binary;
2221 
2222 	want.code = 1;
2223 	atoms[length] = am_code;
2224 	uintps[length++] = &size.code;
2225 
2226 	want.ets = 1;
2227 	atoms[length] = am_ets;
2228 	uintps[length++] = &size.ets;
2229     }
2230     else {
2231 	DeclareTmpHeapNoproc(tmp_heap,2);
2232 	Eterm wanted_list;
2233 
2234 	if (is_nil(earg))
2235 	    return NIL;
2236 
2237 	UseTmpHeapNoproc(2);
2238 	if (is_not_atom(earg))
2239 	    wanted_list = earg;
2240 	else {
2241 	    wanted_list = CONS(&tmp_heap[0], earg, NIL);
2242 	    only_one_value = 1;
2243 	}
2244 
2245 	while (is_list(wanted_list)) {
2246 	    switch (CAR(list_val(wanted_list))) {
2247 	    case am_total:
2248 		if (!want.total) {
2249 		    want.total = 1;
2250 		    atoms[length] = am_total;
2251 		    uintps[length++] = &size.total;
2252 		}
2253 		break;
2254 	    case am_processes:
2255 		if (!want.processes) {
2256 		    want.processes = 1;
2257 		    atoms[length] = am_processes;
2258 		    uintps[length++] = &size.processes;
2259 		}
2260 		break;
2261 	    case am_processes_used:
2262 		if (!want.processes_used) {
2263 		    want.processes_used = 1;
2264 		    atoms[length] = am_processes_used;
2265 		    uintps[length++] = &size.processes_used;
2266 		}
2267 		break;
2268 	    case am_system:
2269 		if (!want.system) {
2270 		    want.system = 1;
2271 		    atoms[length] = am_system;
2272 		    uintps[length++] = &size.system;
2273 		}
2274 		break;
2275 	    case am_atom:
2276 		if (!want.atom) {
2277 		    want.atom = 1;
2278 		    atoms[length] = am_atom;
2279 		    uintps[length++] = &size.atom;
2280 		}
2281 		break;
2282 	    case am_atom_used:
2283 		if (!want.atom_used) {
2284 		    want.atom_used = 1;
2285 		    atoms[length] = am_atom_used;
2286 		    uintps[length++] = &size.atom_used;
2287 		}
2288 		break;
2289 	    case am_binary:
2290 		if (!want.binary) {
2291 		    want.binary = 1;
2292 		    atoms[length] = am_binary;
2293 		    uintps[length++] = &size.binary;
2294 		}
2295 		break;
2296 	    case am_code:
2297 		if (!want.code) {
2298 		    want.code = 1;
2299 		    atoms[length] = am_code;
2300 		    uintps[length++] = &size.code;
2301 		}
2302 		break;
2303 	    case am_ets:
2304 		if (!want.ets) {
2305 		    want.ets = 1;
2306 		    atoms[length] = am_ets;
2307 		    uintps[length++] = &size.ets;
2308 		}
2309 		break;
2310 	    default:
2311 		UnUseTmpHeapNoproc(2);
2312 		return am_badarg;
2313 	    }
2314 	    wanted_list = CDR(list_val(wanted_list));
2315 	}
2316 	UnUseTmpHeapNoproc(2);
2317 	if (is_not_nil(wanted_list))
2318 	    return am_badarg;
2319     }
2320 
2321     /* All alloc_util allocators *have* to be enabled, except test_alloc */
2322 
2323     for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
2324 	switch (ai) {
2325 	case ERTS_ALC_A_SYSTEM:
2326         case ERTS_ALC_A_TEST:
2327 	    break;
2328 	default:
2329 	    if (!erts_allctrs_info[ai].enabled
2330 		|| !erts_allctrs_info[ai].alloc_util) {
2331 		return am_notsup;
2332 	    }
2333 	    break;
2334 	}
2335     }
2336 
2337     ASSERT(length <= sizeof(atoms)/sizeof(Eterm));
2338     ASSERT(length <= sizeof(euints)/sizeof(Eterm));
2339     ASSERT(length <= sizeof(uintps)/sizeof(UWord));
2340 
2341 
2342     if (proc) {
2343 	ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN
2344 			   == erts_proc_lc_my_proc_locks(proc));
2345 	/* We'll need locks early in the lock order */
2346 	erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
2347     }
2348 
2349     /* Calculate values needed... */
2350 
2351     want_tot_or_sys = want.total || want.system;
2352 
2353     if (ERTS_MEM_NEED_ALL_ALCU) {
2354 	size.total = 0;
2355 
2356 	for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
2357 	    if (erts_allctrs_info[ai].alloc_util) {
2358 		UWord *save;
2359 		UWord asz;
2360 		switch (ai) {
2361 		case ERTS_ALC_A_TEMPORARY:
2362 		     /*
2363 		      * Often not thread safe and usually never
2364 		      * contain any allocated memory.
2365 		      */
2366 		    continue;
2367                 case ERTS_ALC_A_TEST:
2368                     continue;
2369 		case ERTS_ALC_A_EHEAP:
2370 		    save = &size.processes;
2371 		    break;
2372 		case ERTS_ALC_A_ETS:
2373 		    save = &size.ets;
2374 		    break;
2375 		case ERTS_ALC_A_BINARY:
2376 		    save = &size.binary;
2377 		    break;
2378 		case ERTS_ALC_A_FIXED_SIZE:
2379 		    asz = alcu_size(ai, fi, ERTS_ALC_NO_FIXED_SIZES);
2380 		    size.total += asz;
2381 		    continue;
2382 		default:
2383 		    save = NULL;
2384 		    break;
2385 		}
2386 		asz = alcu_size(ai, NULL, 0);
2387 		if (save)
2388 		    *save = asz;
2389 		size.total += asz;
2390 	    }
2391 	}
2392     }
2393 
2394 
2395 
2396     if (want_tot_or_sys || want.processes || want.processes_used) {
2397 	UWord tmp;
2398 
2399 	if (ERTS_MEM_NEED_ALL_ALCU)
2400 	    tmp = size.processes;
2401 	else {
2402 	    alcu_size(ERTS_ALC_A_FIXED_SIZE,
2403 		      fi, ERTS_ALC_NO_FIXED_SIZES);
2404 	    tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0);
2405 	}
2406 	tmp += erts_ptab_mem_size(&erts_proc);
2407 	tmp += erts_bif_timer_memory_size();
2408 
2409 	size.processes = size.processes_used = tmp;
2410 
2411 	add_fix_values(&size.processes,
2412 		       &size.processes_used,
2413 		       fi,
2414 		       ERTS_ALC_T_PROC);
2415 	add_fix_values(&size.processes,
2416 		       &size.processes_used,
2417 		       fi,
2418 		       ERTS_ALC_T_MONITOR);
2419 	add_fix_values(&size.processes,
2420 		       &size.processes_used,
2421 		       fi,
2422 		       ERTS_ALC_T_LINK);
2423 	add_fix_values(&size.processes,
2424 		       &size.processes_used,
2425 		       fi,
2426 		       ERTS_ALC_T_MSG_REF);
2427 	add_fix_values(&size.processes,
2428 		       &size.processes_used,
2429 		       fi,
2430 		       ERTS_ALC_T_LL_PTIMER);
2431 	add_fix_values(&size.processes,
2432 		       &size.processes_used,
2433 		       fi,
2434 		       ERTS_ALC_T_HL_PTIMER);
2435 	add_fix_values(&size.processes,
2436 		       &size.processes_used,
2437 		       fi,
2438 		       ERTS_ALC_T_BIF_TIMER);
2439     }
2440 
2441     if (want.atom || want.atom_used) {
2442 	Uint reserved_atom_space, atom_space;
2443 	erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space);
2444 	size.atom = size.atom_used = atom_table_sz();
2445 
2446 	if (want.atom)
2447 	    size.atom += reserved_atom_space;
2448 
2449 	if (want.atom_used)
2450 	    size.atom_used += atom_space;
2451     }
2452 
2453     if (!ERTS_MEM_NEED_ALL_ALCU && want.binary)
2454 	size.binary = alcu_size(ERTS_ALC_A_BINARY, NULL, 0);
2455 
2456     if (want.code) {
2457 	size.code = module_table_sz();
2458 	size.code += export_table_sz();
2459 	size.code += export_entries_sz();
2460 	size.code += erts_fun_table_sz();
2461 	size.code += erts_ranges_sz();
2462 	size.code += erts_total_code_size;
2463     }
2464 
2465     if (want.ets) {
2466 	if (!ERTS_MEM_NEED_ALL_ALCU)
2467 	    size.ets = alcu_size(ERTS_ALC_A_ETS, NULL, 0);
2468 	size.ets += erts_get_ets_misc_mem_size();
2469     }
2470 
2471     if (want_tot_or_sys) {
2472         ASSERT(size.total >= size.processes);
2473 	size.system = size.total - size.processes;
2474     }
2475 
2476     if (print_to_p) {
2477 	int i;
2478 	fmtfn_t to = *print_to_p;
2479 	void *arg = print_to_arg;
2480 
2481 	/* Print result... */
2482 	erts_print(to, arg, "=memory\n");
2483 	for (i = 0; i < length; i++)
2484 	    erts_print(to, arg, "%T: %bpu\n", atoms[i], *uintps[i]);
2485     }
2486 
2487     if (proc) {
2488 	/* Build erlang term result... */
2489 	Uint *hp;
2490 	Uint hsz;
2491 
2492 	erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
2493 
2494 	if (only_one_value) {
2495 	    ASSERT(length == 1);
2496 	    hsz = 0;
2497 	    erts_bld_uword(NULL, &hsz, *uintps[0]);
2498 	    hp = hsz ? HAlloc((Process *) proc, hsz) : NULL;
2499 	    res = erts_bld_uword(&hp, NULL, *uintps[0]);
2500 	}
2501 	else {
2502 	    Uint **hpp = NULL;
2503 	    Uint *hszp = &hsz;
2504 	    hsz = 0;
2505 
2506 	    while (1) {
2507 		int i;
2508 		for (i = 0; i < length; i++)
2509 		    euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]);
2510 		res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints);
2511 		if (hpp)
2512 		    break;
2513 		hp = HAlloc((Process *) proc, hsz);
2514 		hpp = &hp;
2515 		hszp = NULL;
2516 	    }
2517 	}
2518     }
2519 
2520     return res;
2521 
2522 #undef ERTS_MEM_NEED_ALL_ALCU
2523 }
2524 
2525 struct aa_values {
2526     Uint arity;
2527     const char *name;
2528     Uint ui[2];
2529 };
2530 
2531 Eterm
erts_allocated_areas(fmtfn_t * print_to_p,void * print_to_arg,void * proc)2532 erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc)
2533 {
2534 #define MAX_AA_VALUES (24)
2535     struct aa_values values[MAX_AA_VALUES];
2536     Eterm res = THE_NON_VALUE;
2537     int i, length;
2538     Uint reserved_atom_space, atom_space;
2539 
2540     if (proc) {
2541 	ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN
2542 			   == erts_proc_lc_my_proc_locks(proc));
2543 
2544 	/* We'll need locks early in the lock order */
2545 	erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
2546     }
2547 
2548     i = 0;
2549 
2550     values[i].arity = 2;
2551     values[i].name = "sys_misc";
2552     values[i].ui[0] = erts_sys_misc_mem_sz();
2553     i++;
2554 
2555     values[i].arity = 2;
2556     values[i].name = "static";
2557     values[i].ui[0] =
2558 	sizeof(ErtsPTab)*2			/* proc & port tables */
2559 	+ erts_timer_wheel_memory_size();	/* Timer wheel */
2560     i++;
2561 
2562     erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space);
2563 
2564     values[i].arity = 3;
2565     values[i].name = "atom_space";
2566     values[i].ui[0] = reserved_atom_space;
2567     values[i].ui[1] = atom_space;
2568     i++;
2569 
2570     values[i].arity = 2;
2571     values[i].name = "atom_table";
2572     values[i].ui[0] = atom_table_sz();
2573     i++;
2574 
2575     values[i].arity = 2;
2576     values[i].name = "module_table";
2577     values[i].ui[0] = module_table_sz();
2578     i++;
2579 
2580     values[i].arity = 2;
2581     values[i].name = "export_table";
2582     values[i].ui[0] = export_table_sz();
2583     i++;
2584 
2585     values[i].arity = 2;
2586     values[i].name = "export_list";
2587     values[i].ui[0] = export_entries_sz();
2588     i++;
2589 
2590     values[i].arity = 2;
2591     values[i].name = "register_table";
2592     values[i].ui[0] = process_reg_sz();
2593     i++;
2594 
2595     values[i].arity = 2;
2596     values[i].name = "fun_table";
2597     values[i].ui[0] = erts_fun_table_sz();
2598     i++;
2599 
2600     values[i].arity = 2;
2601     values[i].name = "module_refs";
2602     values[i].ui[0] = erts_ranges_sz();
2603     i++;
2604 
2605     values[i].arity = 2;
2606     values[i].name = "loaded_code";
2607     values[i].ui[0] = erts_total_code_size;
2608     i++;
2609 
2610     values[i].arity = 2;
2611     values[i].name = "dist_table";
2612     values[i].ui[0] = erts_dist_table_size();
2613     i++;
2614 
2615     values[i].arity = 2;
2616     values[i].name = "node_table";
2617     values[i].ui[0] = erts_node_table_size();
2618     i++;
2619 
2620     values[i].arity = 2;
2621     values[i].name = "bits_bufs_size";
2622     values[i].ui[0] = erts_bits_bufs_size();
2623     i++;
2624 
2625     values[i].arity = 2;
2626     values[i].name = "bif_timer";
2627     values[i].ui[0] = erts_bif_timer_memory_size();
2628     i++;
2629 
2630     values[i].arity = 2;
2631     values[i].name = "process_table";
2632     values[i].ui[0] = erts_ptab_mem_size(&erts_proc);
2633     i++;
2634 
2635     values[i].arity = 2;
2636     values[i].name = "port_table";
2637     values[i].ui[0] = erts_ptab_mem_size(&erts_port);
2638     i++;
2639 
2640     values[i].arity = 2;
2641     values[i].name = "ets_misc";
2642     values[i].ui[0] = erts_get_ets_misc_mem_size();
2643     i++;
2644 
2645     length = i;
2646     ASSERT(length <= MAX_AA_VALUES);
2647 
2648     if (print_to_p) {
2649 	/* Print result... */
2650 	fmtfn_t to = *print_to_p;
2651 	void *arg = print_to_arg;
2652 
2653 	erts_print(to, arg, "=allocated_areas\n");
2654 	for (i = 0; i < length; i++) {
2655 	    switch (values[i].arity) {
2656 	    case 2:
2657 		erts_print(to, arg, "%s: %beu\n",
2658 			   values[i].name, values[i].ui[0]);
2659 		break;
2660 	    case 3:
2661 		erts_print(to, arg, "%s: %beu %beu\n",
2662 			   values[i].name, values[i].ui[0], values[i].ui[1]);
2663 		break;
2664 	    default:
2665 		erts_print(to, arg, "ERROR: internal_error\n");
2666 		ASSERT(0);
2667 		return am_internal_error;
2668 	    }
2669 	}
2670     }
2671 
2672     if (proc) {
2673 	/* Build erlang term result... */
2674 	Eterm tuples[MAX_AA_VALUES];
2675 	Uint *hp;
2676 	Uint **hpp;
2677 	Uint hsz;
2678 	Uint *hszp;
2679 
2680 	erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
2681 
2682 	hpp = NULL;
2683 	hsz = 0;
2684 	hszp = &hsz;
2685 
2686 	while (1) {
2687 	    int i;
2688 	    for (i = 0; i < length; i++) {
2689 		Eterm atom;
2690 		if (hpp)
2691 		    atom = am_atom_put(values[i].name,
2692 				       (int) sys_strlen(values[i].name));
2693 		else
2694 		    atom = am_true;
2695 
2696 		switch (values[i].arity) {
2697 		case 2:
2698 		    tuples[i] = erts_bld_tuple(hpp, hszp, 2,
2699 					       atom,
2700 					       erts_bld_uint(hpp, hszp,
2701 							     values[i].ui[0]));
2702 		    break;
2703 		case 3:
2704 		    tuples[i] = erts_bld_tuple(hpp, hszp, 3,
2705 					       atom,
2706 					       erts_bld_uint(hpp, hszp,
2707 							     values[i].ui[0]),
2708 					       erts_bld_uint(hpp, hszp,
2709 							     values[i].ui[1]));
2710 		    break;
2711 		default:
2712 		    ASSERT(0);
2713 		    return am_internal_error;
2714 		}
2715 	    }
2716 	    res = erts_bld_list(hpp, hszp, length, tuples);
2717 	    if (hpp)
2718 		break;
2719 	    hp = HAlloc((Process *) proc, hsz);
2720 	    hpp = &hp;
2721 	    hszp = NULL;
2722 	}
2723     }
2724 
2725     return res;
2726 #undef MAX_AA_VALUES
2727 }
2728 
2729 Eterm
erts_alloc_util_allocators(void * proc)2730 erts_alloc_util_allocators(void *proc)
2731 {
2732     Eterm res;
2733     Uint *hp;
2734     Uint sz;
2735     int i;
2736     /*
2737      * Currently all allocators except sys_alloc are
2738      * alloc_util allocators.
2739      * Also hide test_alloc which is disabled by default
2740      * and only intended for our own testing.
2741      */
2742     sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2;
2743     ASSERT(sz > 0);
2744     hp = HAlloc((Process *) proc, sz);
2745     res = NIL;
2746     for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) {
2747 	switch (i) {
2748 	case ERTS_ALC_A_SYSTEM:
2749         case ERTS_ALC_A_TEST:
2750 	    break;
2751 	default: {
2752 	    char *alc_str = (char *) ERTS_ALC_A2AD(i);
2753 	    Eterm alc = am_atom_put(alc_str, sys_strlen(alc_str));
2754 	    res = CONS(hp, alc, res);
2755 	    hp += 2;
2756 	    break;
2757 	}
2758 	}
2759     }
2760     return res;
2761 }
2762 
2763 void
erts_allocator_info(fmtfn_t to,void * arg)2764 erts_allocator_info(fmtfn_t to, void *arg)
2765 {
2766     ErtsAlcType_t a;
2767 
2768     ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
2769 
2770     for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
2771 	int ai;
2772 	for (ai = 0; ai == 0 || ai < erts_allctrs_info[a].thr_spec; ai++) {
2773 	    if (erts_allctrs_info[a].thr_spec) {
2774 		if (!erts_allctr_thr_spec[a].allctr[ai])
2775 		    continue;
2776 		erts_print(to, arg, "=allocator:%s[%d]\n",
2777 			   ERTS_ALC_A2AD(a), ai);
2778 	    }
2779 	    else {
2780 		erts_print(to, arg, "=allocator:%s\n", ERTS_ALC_A2AD(a));
2781 	    }
2782 	    if (!erts_allctrs_info[a].enabled)
2783 		erts_print(to, arg, "option e: false\n");
2784 	    else {
2785 		if (erts_allctrs_info[a].alloc_util) {
2786 		    void *as;
2787 		    if (!erts_allctrs_info[a].thr_spec)
2788 			as = erts_allctrs_info[a].extra;
2789 		    else {
2790 			ASSERT(erts_allctr_thr_spec[a].enabled);
2791 			as = erts_allctr_thr_spec[a].allctr[ai];
2792 		    }
2793 		    /* Binary alloc has its own thread safety... */
2794 		    erts_alcu_info(as, 0, 0, &to, arg, NULL, NULL);
2795 		}
2796 		else {
2797 		    switch (a) {
2798 		    case ERTS_ALC_A_SYSTEM: {
2799 			SysAllocStat sas;
2800 			erts_print(to, arg, "option e: true\n");
2801 			erts_print(to, arg, "option m: libc\n");
2802 			sys_alloc_stat(&sas);
2803 			if(sas.trim_threshold >= 0)
2804 			    erts_print(to, arg, "option tt: %d\n", sas.trim_threshold);
2805 			if(sas.top_pad >= 0)
2806 			    erts_print(to, arg, "option tp: %d\n", sas.top_pad);
2807 			break;
2808 		    }
2809 		    default:
2810 			ASSERT(0);
2811 			break;
2812 		    }
2813 		}
2814 	    }
2815 	}
2816     }
2817 
2818 #if HAVE_ERTS_MSEG
2819     {
2820 	struct erts_mmap_info_struct emis;
2821 	int max = (int) erts_no_schedulers;
2822 	int i;
2823 	for (i = 0; i <= max; i++) {
2824 	    erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
2825 	    erts_mseg_info(i, &to, arg, 0, 0, NULL, NULL);
2826 	}
2827 	erts_print(to, arg, "=allocator:erts_mmap.default_mmap\n");
2828 	erts_mmap_info(&erts_dflt_mmapper, &to, arg, NULL, NULL, &emis);
2829 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
2830         erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n");
2831         erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis);
2832 #endif
2833     }
2834 #endif
2835 
2836     erts_print(to, arg, "=allocator:alloc_util\n");
2837     erts_alcu_au_info_options(&to, arg, NULL, NULL);
2838 
2839     erts_print(to, arg, "=allocator:instr\n");
2840 
2841     erts_print(to, arg, "option t: %s\n",
2842 	       erts_mtrace_enabled ? "true" : "false");
2843 
2844 }
2845 
2846 Eterm
erts_allocator_options(void * proc)2847 erts_allocator_options(void *proc)
2848 {
2849 #if HAVE_ERTS_MSEG
2850     int use_mseg = 0;
2851 #endif
2852     Uint sz, *szp, *hp, **hpp;
2853     Eterm res, features, settings;
2854     Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7];
2855     Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7];
2856     int a, length;
2857     SysAllocStat sas;
2858     Uint *endp = NULL;
2859 
2860     sys_alloc_stat(&sas);
2861 
2862     /* First find out the heap size needed ... */
2863     hpp = NULL;
2864     szp = &sz;
2865     sz = 0;
2866 
2867  bld_term:
2868 
2869     length = 0;
2870     features = NIL;
2871     settings = NIL;
2872 
2873     for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
2874 	Eterm tmp = NIL;
2875 	atoms[length] = am_atom_put((char *) ERTS_ALC_A2AD(a),
2876 				    sys_strlen(ERTS_ALC_A2AD(a)));
2877 	if (erts_allctrs_info[a].enabled) {
2878 	    if (erts_allctrs_info[a].alloc_util) {
2879 		Allctr_t *allctr;
2880 #if HAVE_ERTS_MSEG
2881 		use_mseg++;
2882 #endif
2883 		if (erts_allctr_thr_spec[a].enabled)
2884 		    allctr = erts_allctr_thr_spec[a].allctr[0];
2885 		else
2886 		    allctr = erts_allctrs_info[a].extra;
2887 		tmp = erts_alcu_info_options(allctr, NULL, NULL, hpp, szp);
2888 	    }
2889 	    else {
2890 		int l = 0;
2891 		Eterm as[4];
2892 		Eterm ts[4];
2893 
2894 		as[l] = ERTS_MAKE_AM("e");
2895 		ts[l++] = am_true;
2896 
2897 		switch (a) {
2898 		case ERTS_ALC_A_SYSTEM:
2899 		    as[l] = ERTS_MAKE_AM("m");
2900 		    ts[l++] = ERTS_MAKE_AM("libc");
2901 		    if(sas.trim_threshold >= 0) {
2902 			as[l] = ERTS_MAKE_AM("tt");
2903 			ts[l++] = erts_bld_uint(hpp, szp,
2904 						(Uint) sas.trim_threshold);
2905 		    }
2906 		    if(sas.top_pad >= 0) {
2907 			as[l] = ERTS_MAKE_AM("tp");
2908 			ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad);
2909 		    }
2910 		    break;
2911 		default:
2912 		    break;
2913 		}
2914 
2915 		tmp = erts_bld_2tup_list(hpp, szp, l, as, ts);
2916 
2917 	    }
2918 
2919 	}
2920 	else {
2921 	    Eterm atom = ERTS_MAKE_AM("e");
2922 	    Eterm term = am_false;
2923 	    tmp = erts_bld_2tup_list(hpp, szp, 1, &atom, &term);
2924 	}
2925 
2926 	terms[length++] = tmp;
2927 
2928     }
2929 
2930 #if HAVE_ERTS_MSEG
2931     if (use_mseg) {
2932 	atoms[length] = ERTS_MAKE_AM("mseg_alloc");
2933 	terms[length++] = erts_mseg_info_options(0, NULL, NULL, hpp, szp);
2934     }
2935 #endif
2936 
2937     atoms[length] = ERTS_MAKE_AM("alloc_util");
2938     terms[length++] = erts_alcu_au_info_options(NULL, NULL, hpp, szp);
2939 
2940 #if HAVE_ERTS_MMAP
2941     atoms[length] = ERTS_MAKE_AM("erts_mmap");
2942     terms[length++] = erts_mmap_info_options(&erts_dflt_mmapper, NULL, NULL,
2943                                              NULL, hpp, szp);
2944 #endif
2945     {
2946 	Eterm o[1], v[1];
2947 	o[0] = ERTS_MAKE_AM("t");
2948 	v[0] = erts_mtrace_enabled ? am_true : am_false;
2949 
2950 	atoms[length] = ERTS_MAKE_AM("instr");
2951 	terms[length++] = erts_bld_2tup_list(hpp, szp, 1, o, v);
2952     }
2953 
2954     atoms[length] = ERTS_MAKE_AM("lock_physical_memory");
2955     terms[length++] = (lock_all_physical_memory ? am_all : am_no);
2956 
2957     settings = erts_bld_2tup_list(hpp, szp, length, atoms, terms);
2958 
2959     length = 0;
2960 
2961     for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
2962 	if (erts_allctrs_info[a].enabled) {
2963 	    terms[length++] = am_atom_put((char *) ERTS_ALC_A2AD(a),
2964 					  sys_strlen(ERTS_ALC_A2AD(a)));
2965 	}
2966     }
2967 
2968 #if HAVE_ERTS_MSEG
2969     if (use_mseg)
2970 	terms[length++] = ERTS_MAKE_AM("mseg_alloc");
2971 #endif
2972 #if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
2973     terms[length++] = ERTS_MAKE_AM("sys_aligned_alloc");
2974 #endif
2975 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
2976     terms[length++] = ERTS_MAKE_AM("literal_mmap");
2977 #endif
2978     features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
2979 
2980 #if defined(__GLIBC__)
2981     {
2982 	Eterm AM_glibc = ERTS_MAKE_AM("glibc");
2983 	Eterm version;
2984 
2985 	version = erts_bld_cons(hpp,
2986 				szp,
2987 				make_small(__GLIBC__),
2988 #ifdef __GLIBC_MINOR__
2989 				erts_bld_cons(hpp,
2990 					      szp,
2991 					      make_small(__GLIBC_MINOR__),
2992 					      NIL)
2993 #else
2994 				NIL
2995 #endif
2996 	    );
2997 
2998 	res = erts_bld_tuple(hpp, szp, 4,
2999 			     AM_glibc, version, features, settings);
3000     }
3001 
3002 #else /* unknown allocator */
3003 
3004     res = erts_bld_tuple(hpp, szp, 4,
3005 			 am_undefined, NIL, features, settings);
3006 
3007 #endif
3008 
3009     if (szp) {
3010 	/* ... and then build the term */
3011 	hp = HAlloc((Process *) proc, sz);
3012 	endp = hp + sz;
3013 	hpp = &hp;
3014 	szp = NULL;
3015 	goto bld_term;
3016     }
3017 
3018     ASSERT(endp >= hp);
3019     HRelease((Process *) proc, endp, hp);
3020 
3021     return res;
3022 }
3023 
erts_alloc_permanent_cache_aligned(ErtsAlcType_t type,Uint size)3024 void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size)
3025 {
3026     UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1)
3027 #ifdef VALGRIND
3028 				  + sizeof(UWord)
3029 #endif
3030 				 );
3031 
3032 #ifdef VALGRIND
3033     {   /* Link them to avoid Leak_PossiblyLost */
3034 	static UWord* first_in_list = NULL;
3035         *(UWord**)v = first_in_list;
3036 	first_in_list = (UWord*) v;
3037 	v += sizeof(UWord);
3038     }
3039 #endif
3040 
3041     if (v & ERTS_CACHE_LINE_MASK) {
3042 	v = (v & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
3043     }
3044     ASSERT((v & ERTS_CACHE_LINE_MASK) == 0);
3045     return (void*)v;
3046 }
3047 
3048 static void
reply_alloc_info(void * vair)3049 reply_alloc_info(void *vair)
3050 {
3051     ErtsAllocInfoReq *air = (ErtsAllocInfoReq *) vair;
3052     Uint sched_id = erts_get_scheduler_id();
3053     int global_instances = air->req_sched == sched_id;
3054     ErtsProcLocks rp_locks;
3055     Process *rp = air->proc;
3056     Eterm ref_copy = NIL, ai_list, msg = NIL;
3057     Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL;
3058     Eterm **hpp;
3059     Uint sz, *szp;
3060     ErlOffHeap *ohp = NULL;
3061     ErtsMessage *mp = NULL;
3062 #if HAVE_ERTS_MMAP
3063     struct erts_mmap_info_struct mmap_info_dflt;
3064 # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
3065     struct erts_mmap_info_struct mmap_info_literal;
3066 # endif
3067 #endif
3068     int i;
3069     Eterm (*info_func)(Allctr_t *,
3070 		       int,
3071 		       int,
3072 		       fmtfn_t *,
3073 		       void *,
3074 		       Uint **,
3075 		       Uint *) = (air->only_sz
3076 				  ? erts_alcu_sz_info
3077 				  : erts_alcu_info);
3078 
3079     rp_locks = air->req_sched == sched_id ? ERTS_PROC_LOCK_MAIN : 0;
3080 
3081     sz = 0;
3082     hpp = NULL;
3083     szp = &sz;
3084 
3085     while (1) {
3086 
3087 	if (hpp)
3088 	    ref_copy = erts_iref_storage_make_ref(&air->iref,
3089                                                   hpp, ohp, 0);
3090 	else
3091 	    *szp += erts_iref_storage_heap_size(&air->iref);
3092 
3093 	ai_list = NIL;
3094 	for (i = 0; air->allocs[i] != ERTS_ALC_A_INVALID; i++);
3095 	for (i--; i >= 0; i--) {
3096 	    int ai = air->allocs[i];
3097 	    Allctr_t *allctr;
3098 	    Eterm ainfo;
3099 	    Eterm alloc_atom;
3100 	    if (global_instances) {
3101 		switch (ai) {
3102 		case ERTS_ALC_A_SYSTEM: {
3103 		    alloc_atom = erts_bld_atom(hpp, szp, "sys_alloc");
3104 		    ainfo = NIL;
3105 		    if (!air->only_sz) {
3106 			SysAllocStat sas;
3107 			if (hpp)
3108 			    sys_alloc_stat(&sas);
3109 			if (szp) {
3110 			    /* ensure ehough heap */
3111 			    sas.top_pad = INT_MAX;
3112 			    sas.trim_threshold = INT_MAX;
3113 			}
3114 			if (sas.top_pad >= 0) {
3115 			    ainfo = erts_bld_cons(
3116 				hpp, szp,
3117 				erts_bld_tuple(
3118 				    hpp, szp, 2,
3119 				    erts_bld_atom(hpp, szp, "tp"),
3120 				    erts_bld_uint(
3121 					hpp, szp,
3122 					(Uint) sas.top_pad)),
3123 				ainfo);
3124 			}
3125 			if (sas.trim_threshold >= 0) {
3126 			    ainfo = erts_bld_cons(
3127 				hpp, szp,
3128 				erts_bld_tuple(
3129 				    hpp, szp, 2,
3130 				    erts_bld_atom(hpp, szp, "tt"),
3131 				    erts_bld_uint(
3132 					hpp, szp,
3133 					(Uint) sas.trim_threshold)),
3134 				ainfo);
3135 			}
3136 			ainfo = erts_bld_cons(hpp, szp,
3137 					      erts_bld_tuple(
3138 						  hpp, szp, 2,
3139 						  erts_bld_atom(hpp, szp,
3140 								"m"),
3141 						  erts_bld_atom(hpp, szp,
3142 								"libc")),
3143 					      ainfo);
3144 			ainfo = erts_bld_cons(hpp, szp,
3145 					      erts_bld_tuple(
3146 						  hpp, szp, 2,
3147 						  erts_bld_atom(hpp, szp,
3148 								"e"),
3149 						  am_true),
3150 					      ainfo);
3151 			ainfo = erts_bld_tuple(hpp, szp, 2,
3152 					       erts_bld_atom(hpp, szp,
3153 							     "options"),
3154 					       ainfo);
3155 			ainfo = erts_bld_cons(hpp, szp,ainfo,NIL);
3156 		    }
3157 		    ainfo = erts_bld_tuple(hpp, szp, 3,
3158 					   alloc_atom,
3159 					   make_small(0),
3160 					   ainfo);
3161 		    break;
3162 		}
3163 		case ERTS_ALC_INFO_A_ALLOC_UTIL:
3164 		    alloc_atom = erts_bld_atom(hpp, szp, "alloc_util");
3165 		    ainfo = (air->only_sz
3166 			     ? NIL
3167 			     : erts_alcu_au_info_options(NULL, NULL,
3168 							 hpp, szp));
3169 		    ainfo = erts_bld_tuple(hpp, szp, 3,
3170 					   alloc_atom,
3171 					   make_small(0),
3172 					   ainfo);
3173 		    break;
3174                 case ERTS_ALC_INFO_A_ERTS_MMAP:
3175                     alloc_atom = erts_bld_atom(hpp, szp, "erts_mmap");
3176 #if HAVE_ERTS_MMAP
3177                     ainfo = (air->only_sz ? NIL :
3178                              erts_mmap_info(&erts_dflt_mmapper, NULL, NULL,
3179                                             hpp, szp, &mmap_info_dflt));
3180                     ainfo = erts_bld_tuple3(hpp, szp,
3181                                             alloc_atom,
3182                                             erts_bld_atom(hpp,szp,"default_mmap"),
3183                                             ainfo);
3184 #  if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
3185                     ai_list = erts_bld_cons(hpp, szp,
3186                                             ainfo, ai_list);
3187                     ainfo = (air->only_sz ? NIL :
3188                              erts_mmap_info(&erts_literal_mmapper, NULL, NULL,
3189                                             hpp, szp, &mmap_info_literal));
3190                     ainfo = erts_bld_tuple3(hpp, szp,
3191                                             alloc_atom,
3192                                             erts_bld_atom(hpp,szp,"literal_mmap"),
3193                                             ainfo);
3194 #  endif
3195 #else  /* !HAVE_ERTS_MMAP */
3196                     ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
3197                                             am_false);
3198 #endif
3199                     break;
3200 		case ERTS_ALC_INFO_A_MSEG_ALLOC:
3201 		    alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
3202 #if HAVE_ERTS_MSEG
3203 		    ainfo = erts_mseg_info(0, NULL, NULL, hpp != NULL,
3204                                            air->only_sz, hpp, szp);
3205 		    ainfo = erts_bld_tuple3(hpp, szp,
3206                                             alloc_atom,
3207                                             make_small(0),
3208                                             ainfo);
3209 
3210 #else
3211 		    ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
3212                                             am_false);
3213 #endif
3214 		    break;
3215 #ifndef ERTS_ALC_A_EXEC
3216 		case ERTS_ALC_INFO_A_DISABLED_EXEC:
3217 		    alloc_atom = erts_bld_atom(hpp, szp, "exec_alloc");
3218 		    ainfo = erts_bld_tuple2(hpp, szp, alloc_atom, am_false);
3219 		    break;
3220 #endif
3221 		default:
3222 		    alloc_atom = erts_bld_atom(hpp, szp,
3223 					       (char *) ERTS_ALC_A2AD(ai));
3224 		    if (!erts_allctrs_info[ai].enabled)
3225 			ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
3226 					       am_false);
3227 		    else if (erts_allctrs_info[ai].alloc_util) {
3228 			if (erts_allctrs_info[ai].thr_spec)
3229 			    allctr = erts_allctr_thr_spec[ai].allctr[0];
3230 			else
3231 			    allctr = erts_allctrs_info[ai].extra;
3232 			ainfo = info_func(allctr, air->internal, hpp != NULL,
3233 					  NULL, NULL, hpp, szp);
3234 			ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom,
3235 					       make_small(0), ainfo);
3236 		    }
3237 		    else {
3238 			erts_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n",
3239 				 __FILE__, __LINE__);
3240 		    }
3241 		}
3242 		ai_list = erts_bld_cons(hpp, szp,
3243 					ainfo, ai_list);
3244 	    }
3245 	    switch (ai) {
3246 	    case ERTS_ALC_A_SYSTEM:
3247             case ERTS_ALC_INFO_A_ALLOC_UTIL:
3248 	    case ERTS_ALC_INFO_A_ERTS_MMAP:
3249 	    case ERTS_ALC_INFO_A_DISABLED_EXEC:
3250 		break;
3251 	    case ERTS_ALC_INFO_A_MSEG_ALLOC:
3252 #if HAVE_ERTS_MSEG
3253 		alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
3254 		ainfo = erts_mseg_info(sched_id, NULL, NULL,
3255                                        hpp != NULL, air->only_sz, hpp, szp);
3256 		ainfo = erts_bld_tuple(hpp, szp, 3,
3257 				       alloc_atom,
3258 				       make_small(sched_id),
3259 				       ainfo);
3260 		ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list);
3261 #endif
3262 		break;
3263 	    default:
3264 		if (erts_allctrs_info[ai].thr_spec) {
3265 		    alloc_atom = erts_bld_atom(hpp, szp,
3266 					       (char *) ERTS_ALC_A2AD(ai));
3267 		    allctr = erts_allctr_thr_spec[ai].allctr[sched_id];
3268 		    ainfo = info_func(allctr, air->internal, hpp != NULL, NULL,
3269 				      NULL, hpp, szp);
3270 		    ai_list = erts_bld_cons(hpp, szp,
3271 					    erts_bld_tuple(
3272 						hpp, szp,
3273 						3,
3274 						alloc_atom,
3275 						make_small(sched_id),
3276 						ainfo),
3277 					    ai_list);
3278 		}
3279 		break;
3280 	    }
3281 	    msg = erts_bld_tuple(hpp, szp,
3282 				 3,
3283 				 ref_copy,
3284 				 make_small(sched_id),
3285 				 ai_list);
3286 
3287 	}
3288 	if (hpp)
3289 	    break;
3290 
3291 	mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
3292 	hp_start = hp;
3293 	hp_end = hp + sz;
3294 	szp = NULL;
3295 	hpp = &hp;
3296     }
3297 
3298     if (hp != hp_end)
3299 	erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1);
3300 
3301     erts_queue_message(rp, rp_locks, mp, msg, am_system);
3302 
3303     if (air->req_sched == sched_id)
3304 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
3305 
3306     erts_proc_unlock(rp, rp_locks);
3307     erts_proc_dec_refc(rp);
3308 
3309     if (erts_atomic32_dec_read_nob(&air->refc) == 0) {
3310         erts_iref_storage_clean(&air->iref);
3311 	aireq_free(air);
3312     }
3313 }
3314 
3315 int
erts_request_alloc_info(struct process * c_p,Eterm ref,Eterm allocs,int only_sz,int internal)3316 erts_request_alloc_info(struct process *c_p,
3317 			Eterm ref,
3318 			Eterm allocs,
3319 			int only_sz,
3320 			int internal)
3321 {
3322     ErtsAllocInfoReq *air = aireq_alloc();
3323     Eterm req_ai[ERTS_ALC_INFO_A_END] = {0};
3324     Eterm alist;
3325     int airix = 0, ai;
3326 
3327     air->req_sched = erts_get_scheduler_id();
3328 
3329     air->only_sz = only_sz;
3330 
3331     air->internal = internal;
3332 
3333     air->proc = c_p;
3334 
3335     if (is_not_internal_ref(ref))
3336 	return 0;
3337 
3338     erts_iref_storage_save(&air->iref, ref);
3339 
3340     if (is_not_list(allocs))
3341 	return 0;
3342 
3343     alist = allocs;
3344 
3345     while (is_list(alist)) {
3346 	int saved = 0;
3347 	Eterm* consp = list_val(alist);
3348 	Eterm alloc = CAR(consp);
3349 
3350 	for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++)
3351 	    if (erts_is_atom_str(erts_alc_a2ad[ai], alloc, 0))
3352 		goto save_alloc;
3353 	if (erts_is_atom_str("mseg_alloc", alloc, 0)) {
3354 	    ai = ERTS_ALC_INFO_A_MSEG_ALLOC;
3355 	    goto save_alloc;
3356 	}
3357         if (erts_is_atom_str("erts_mmap", alloc, 0)) {
3358             ai = ERTS_ALC_INFO_A_ERTS_MMAP;
3359             goto save_alloc;
3360         }
3361 #ifndef ERTS_ALC_A_EXEC
3362 	if (erts_is_atom_str("exec_alloc", alloc, 0)) {
3363 	    ai = ERTS_ALC_INFO_A_DISABLED_EXEC;
3364 	    goto save_alloc;
3365 	}
3366 #endif
3367 	if (erts_is_atom_str("alloc_util", alloc, 0)) {
3368 	    ai = ERTS_ALC_INFO_A_ALLOC_UTIL;
3369 	save_alloc:
3370 	    if (req_ai[ai])
3371 		return 0;
3372 	    air->allocs[airix++] = ai;
3373 	    req_ai[ai] = 1;
3374 	    saved = 1;
3375 	}
3376 
3377 	if (!saved)
3378 	    return 0;
3379 
3380 	alist = CDR(consp);
3381     }
3382 
3383     if (is_not_nil(alist))
3384 	return 0;
3385 
3386     air->allocs[airix] = ERTS_ALC_A_INVALID;
3387 
3388     erts_atomic32_init_nob(&air->refc,
3389 			       (erts_aint32_t) erts_no_schedulers);
3390 
3391     erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
3392 
3393     if (erts_no_schedulers > 1)
3394 	erts_schedule_multi_misc_aux_work(1,
3395 					  erts_no_schedulers,
3396 					  reply_alloc_info,
3397 					  (void *) air);
3398 
3399     reply_alloc_info((void *) air);
3400 
3401     return 1;
3402 }
3403 
erts_alloc_set_dyn_param(Process * c_p,Eterm tuple)3404 Eterm erts_alloc_set_dyn_param(Process* c_p, Eterm tuple)
3405 {
3406     ErtsAllocatorThrSpec_t *tspec;
3407     ErtsAlcType_t ai;
3408     Allctr_t* allctr;
3409     Eterm* tp;
3410     Eterm res;
3411 
3412     if (!is_tuple_arity(tuple, 3))
3413         goto badarg;
3414 
3415     tp = tuple_val(tuple);
3416 
3417     /*
3418      * Ex: {ets_alloc, sbct, 256000}
3419      */
3420     if (!is_atom(tp[1]) || !is_atom(tp[2]) || !is_integer(tp[3]))
3421         goto badarg;
3422 
3423     for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++)
3424         if (erts_is_atom_str(erts_alc_a2ad[ai], tp[1], 0))
3425             break;
3426 
3427     if (ai > ERTS_ALC_A_MAX)
3428         goto badarg;
3429 
3430     if (!erts_allctrs_info[ai].enabled ||
3431         !erts_allctrs_info[ai].alloc_util) {
3432         return am_notsup;
3433     }
3434 
3435     if (tp[2] == am_sbct) {
3436         Uint sbct;
3437         int i, ok;
3438 
3439         if (!term_to_Uint(tp[3], &sbct))
3440             goto badarg;
3441 
3442         tspec = &erts_allctr_thr_spec[ai];
3443         if (tspec->enabled) {
3444             ok = 0;
3445             for (i = 0; i < tspec->size; i++) {
3446                 allctr = tspec->allctr[i];
3447                 ok |= allctr->try_set_dyn_param(allctr, am_sbct, sbct);
3448             }
3449         }
3450         else {
3451             allctr = erts_allctrs_info[ai].extra;
3452             ok = allctr->try_set_dyn_param(allctr, am_sbct, sbct);
3453         }
3454         return ok ? am_ok : am_notsup;
3455     }
3456     return am_notsup;
3457 
3458 badarg:
3459     ERTS_BIF_PREP_ERROR(res, c_p, EXC_BADARG);
3460     return res;
3461 }
3462 
3463 /*
3464  * The allocator wrapper prelocking stuff below is about the locking order.
3465  * It only affects wrappers (erl_mtrace.c) that keep locks during
3466  * alloc/realloc/free.
3467  *
3468  * Some query functions in erl_alloc_util.c lock the allocator mutex and then
3469  * use erts_printf that in turn may call the sys allocator through the wrappers.
3470  * To avoid breaking locking order these query functions first "pre-locks" all
3471  * allocator wrappers.
3472  */
3473 ErtsAllocatorWrapper_t *erts_allctr_wrappers;
3474 int erts_allctr_wrapper_prelocked = 0;
3475 erts_tsd_key_t erts_allctr_prelock_tsd_key;
3476 
erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t * wrapper)3477 void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper)
3478 {
3479     ASSERT(wrapper->lock && wrapper->unlock);
3480     wrapper->next = erts_allctr_wrappers;
3481     erts_allctr_wrappers = wrapper;
3482 }
3483 
erts_allctr_wrapper_pre_lock(void)3484 void erts_allctr_wrapper_pre_lock(void)
3485 {
3486     if (erts_allctr_wrappers) {
3487 	ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers;
3488 	for ( ; wrapper; wrapper = wrapper->next) {
3489 	    wrapper->lock();
3490 	}
3491 	ASSERT(!erts_allctr_wrapper_prelocked);
3492 	erts_allctr_wrapper_prelocked = 1;
3493 	erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)1);
3494     }
3495 }
3496 
erts_allctr_wrapper_pre_unlock(void)3497 void erts_allctr_wrapper_pre_unlock(void)
3498 {
3499     if (erts_allctr_wrappers) {
3500 	ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers;
3501 
3502 	erts_allctr_wrapper_prelocked = 0;
3503 	erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)0);
3504 	for ( ; wrapper; wrapper = wrapper->next) {
3505 	    wrapper->unlock();
3506 	}
3507     }
3508 }
3509 
3510 
3511 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
3512  * NOTE: erts_alc_test() is only supposed to be used for testing.            *
3513  *                                                                           *
3514  * Keep alloc_SUITE_data/allocator_test.h updated if changes are made        *
3515  * to erts_alc_test()                                                        *
3516 \*                                                                           */
3517 #define ERTS_ALC_TEST_ABORT erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n")
3518 
erts_alc_test(UWord op,UWord a1,UWord a2,UWord a3)3519 UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
3520 {
3521     switch (op >> 8) {
3522     case 0x0:	return erts_alcu_test(op,  a1, a2);
3523     case 0x1:	return erts_gfalc_test(op, a1, a2);
3524     case 0x2:	return erts_bfalc_test(op, a1, a2);
3525     case 0x3:	return erts_afalc_test(op, a1, a2);
3526     case 0x4:	return erts_mseg_test(op,  a1, a2, a3);
3527     case 0x5:	return erts_aoffalc_test(op, a1, a2);
3528     case 0xf:
3529 	switch (op) {
3530 	case 0xf00:
3531 	    if (((Allctr_t *) a1)->thread_safe)
3532 		return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_TEST,
3533 							  (void *) a1,
3534 							  (Uint) a2);
3535 	    else
3536 		return (UWord) erts_alcu_alloc(ERTS_ALC_T_TEST,
3537 						       (void *) a1,
3538 						       (Uint) a2);
3539 	case 0xf01:
3540 	    if (((Allctr_t *) a1)->thread_safe)
3541 		return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_TEST,
3542 							    (void *) a1,
3543 							    (void *) a2,
3544 							    (Uint) a3);
3545 	    else
3546 		return (UWord) erts_alcu_realloc(ERTS_ALC_T_TEST,
3547 							 (void *) a1,
3548 							 (void *) a2,
3549 							 (Uint) a3);
3550 	case 0xf02:
3551 	    if (((Allctr_t *) a1)->thread_safe)
3552 		erts_alcu_free_ts(ERTS_ALC_T_TEST, (void *) a1, (void *) a2);
3553 	    else
3554 		erts_alcu_free(ERTS_ALC_T_TEST, (void *) a1, (void *) a2);
3555 	    return 0;
3556 	case 0xf03: {
3557 	    Allctr_t *allctr;
3558 	    struct au_init init;
3559 
3560 	    SET_DEFAULT_ALLOC_OPTS(&init);
3561 	    init.enable = 1;
3562 	    init.astrat = ERTS_ALC_S_GOODFIT;
3563 	    init.init.util.name_prefix = (char *) a1;
3564 	    init.init.util.alloc_no = ERTS_ALC_A_TEST;
3565 	    init.init.util.alloc_strat = init.astrat;
3566 	    init.init.util.ts = 1;
3567 	    if ((char **) a3) {
3568 		char **argv = (char **) a3;
3569 		int i = 0;
3570 		while (argv[i]) {
3571 		    if (argv[i][0] == '-' && argv[i][1] == 't')
3572 			handle_au_arg(&init, &argv[i][2], argv, &i, 0);
3573 		    else
3574 			return (UWord) NULL;
3575 		    i++;
3576 		}
3577 	    }
3578 
3579 	    switch (init.astrat) {
3580 	    case ERTS_ALC_S_GOODFIT:
3581 		allctr = erts_gfalc_start((GFAllctr_t *)
3582 					  erts_alloc(ERTS_ALC_T_TEST,
3583 						     sizeof(GFAllctr_t)),
3584 					  &init.init.gf,
3585 					  &init.init.util);
3586 		break;
3587 	    case ERTS_ALC_S_BESTFIT:
3588 		allctr = erts_bfalc_start((BFAllctr_t *)
3589 					  erts_alloc(ERTS_ALC_T_TEST,
3590 						     sizeof(BFAllctr_t)),
3591 					  &init.init.bf,
3592 					  &init.init.util);
3593 		break;
3594 	    case ERTS_ALC_S_AFIT:
3595 		allctr = erts_afalc_start((AFAllctr_t *)
3596 					  erts_alloc(ERTS_ALC_T_TEST,
3597 							    sizeof(AFAllctr_t)),
3598 					  &init.init.af,
3599 					  &init.init.util);
3600 		break;
3601 	    case ERTS_ALC_S_FIRSTFIT:
3602 		allctr = erts_aoffalc_start((AOFFAllctr_t *)
3603 					  erts_alloc(ERTS_ALC_T_TEST,
3604 						     sizeof(AOFFAllctr_t)),
3605 					  &init.init.aoff,
3606 					  &init.init.util);
3607 		break;
3608 
3609 	    default:
3610 		ASSERT(0);
3611 		allctr = NULL;
3612 		break;
3613 	    }
3614 
3615 	    return (UWord) allctr;
3616 	}
3617 	case 0xf04:
3618 	    erts_alcu_stop((Allctr_t *) a1);
3619 	    erts_free(ERTS_ALC_T_TEST, (void *) a1);
3620 	    break;
3621 	case 0xf05: return (UWord) 1;
3622 	case 0xf06: return (UWord) ((Allctr_t *) a1)->thread_safe;
3623 #ifdef ETHR_NO_FORKSAFETY
3624 	case 0xf07: return (UWord) 0;
3625 #else
3626 	case 0xf07: return (UWord) ((Allctr_t *) a1)->thread_safe;
3627 #endif
3628 	case 0xf08: {
3629 	    ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_mutex));
3630 	    if (ethr_mutex_init(mtx) != 0)
3631 		ERTS_ALC_TEST_ABORT;
3632 	    return (UWord) mtx;
3633 	}
3634 	case 0xf09: {
3635 	    ethr_mutex *mtx = (ethr_mutex *) a1;
3636 	    if (ethr_mutex_destroy(mtx) != 0)
3637 		ERTS_ALC_TEST_ABORT;
3638 	    erts_free(ERTS_ALC_T_TEST, (void *) mtx);
3639 	    break;
3640 	}
3641 	case 0xf0a:
3642 	    ethr_mutex_lock((ethr_mutex *) a1);
3643 	    break;
3644 	case 0xf0b:
3645 	    ethr_mutex_unlock((ethr_mutex *) a1);
3646 	    break;
3647 	case 0xf0c: {
3648 	    ethr_cond *cnd = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_cond));
3649 	    if (ethr_cond_init(cnd) != 0)
3650 		ERTS_ALC_TEST_ABORT;
3651 	    return (UWord) cnd;
3652 	}
3653 	case 0xf0d: {
3654 	    ethr_cond *cnd = (ethr_cond *) a1;
3655 	    if (ethr_cond_destroy(cnd) != 0)
3656 		ERTS_ALC_TEST_ABORT;
3657 	    erts_free(ERTS_ALC_T_TEST, (void *) cnd);
3658 	    break;
3659 	}
3660 	case 0xf0e:
3661 	    ethr_cond_broadcast((ethr_cond *) a1);
3662 	    break;
3663 	case 0xf0f: {
3664 	    int res;
3665 	    do {
3666 		res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2);
3667 	    } while (res == EINTR);
3668 	    break;
3669 	}
3670 	case 0xf10: {
3671 	    ethr_tid *tid = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_tid));
3672 	    if (ethr_thr_create(tid,
3673 				(void * (*)(void *)) a1,
3674 				(void *) a2,
3675 				NULL) != 0)
3676 		ERTS_ALC_TEST_ABORT;
3677 	    return (UWord) tid;
3678 	}
3679 	case 0xf11: {
3680 	    ethr_tid *tid = (ethr_tid *) a1;
3681 	    if (ethr_thr_join(*tid, NULL) != 0)
3682 		ERTS_ALC_TEST_ABORT;
3683 	    erts_free(ERTS_ALC_T_TEST, (void *) tid);
3684 	    break;
3685 	}
3686 	case 0xf12:
3687 	    ethr_thr_exit((void *) a1);
3688 	    ERTS_ALC_TEST_ABORT;
3689 	    break;
3690 	case 0xf13: return (UWord) 1;
3691 	case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1);
3692 
3693 	case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0;
3694 
3695         case 0xf16: return (UWord) erts_realloc(ERTS_ALC_T_TEST, (void*)a1, (Uint)a2);
3696 
3697 	case 0xf17: {
3698             Uint extra_hdr_sz = UNIT_CEILING((Uint)a1);
3699 	    ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST];
3700 	    Uint offset = ts->allctr[0]->mbc_header_size;
3701 	    void* orig_creating_mbc = ts->allctr[0]->creating_mbc;
3702 	    void* orig_destroying_mbc = ts->allctr[0]->destroying_mbc;
3703 	    void* new_creating_mbc = *(void**)a2; /* inout arg */
3704 	    void* new_destroying_mbc = *(void**)a3; /* inout arg */
3705 	    int i;
3706 
3707 	    for (i=0; i < ts->size; i++) {
3708 		Allctr_t* ap = ts->allctr[i];
3709 		if (ap->mbc_header_size != offset
3710 		    || ap->creating_mbc != orig_creating_mbc
3711 		    || ap->destroying_mbc != orig_destroying_mbc
3712 		    || ap->mbc_list.first != NULL)
3713 		    return -1;
3714 	    }
3715 	    for (i=0; i < ts->size; i++) {
3716 		ts->allctr[i]->mbc_header_size += extra_hdr_sz;
3717 		ts->allctr[i]->creating_mbc = new_creating_mbc;
3718 		ts->allctr[i]->destroying_mbc = new_destroying_mbc;
3719 	    }
3720 	    *(void**)a2 = orig_creating_mbc;
3721 	    *(void**)a3 = orig_destroying_mbc;
3722 	    return offset;
3723 	}
3724 	case 0xf18: {
3725 	    ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST];
3726 	    return ts->allctr[0]->largest_mbc_size;
3727 	}
3728 	default:
3729 	    break;
3730 	}
3731 	return (UWord) 0;
3732     default:
3733 	break;
3734     }
3735 
3736     ASSERT(0);
3737     return ~((UWord) 0);
3738 }
3739 
3740 #ifdef DEBUG
3741 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
3742  * Debug stuff                                                               *
3743 \*                                                                           */
3744 
3745 #if 0
3746 #define PRINT_OPS
3747 #else
3748 #undef PRINT_OPS
3749 #endif
3750 
3751 #ifdef HARD_DEBUG
3752 #define FENCE_SZ		(4*sizeof(UWord))
3753 #else
3754 #define FENCE_SZ		(3*sizeof(UWord))
3755 #endif
3756 
3757 #if defined(ARCH_64)
3758 #define FENCE_PATTERN 0xABCDEF97ABCDEF97
3759 #else
3760 #define FENCE_PATTERN 0xABCDEF97
3761 #endif
3762 
3763 #define TYPE_PATTERN_MASK  ERTS_ALC_N_MASK
3764 #define TYPE_PATTERN_SHIFT 16
3765 
3766 #define FIXED_FENCE_PATTERN_MASK \
3767   (~((UWord) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT)))
3768 #define FIXED_FENCE_PATTERN \
3769   (FENCE_PATTERN & FIXED_FENCE_PATTERN_MASK)
3770 
3771 #define MK_PATTERN(T) \
3772   (FIXED_FENCE_PATTERN | (((T) & TYPE_PATTERN_MASK) << TYPE_PATTERN_SHIFT))
3773 
3774 #define GET_TYPE_OF_PATTERN(P) \
3775   (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK)
3776 
3777 #ifdef HARD_DEBUG
3778 
3779 #define ERL_ALC_HDBG_MAX_MBLK 100000
3780 #define ERTS_ALC_O_CHECK -1
3781 
3782 typedef struct hdbg_mblk_ hdbg_mblk;
3783 struct hdbg_mblk_ {
3784     hdbg_mblk *next;
3785     hdbg_mblk *prev;
3786     void *p;
3787     Uint s;
3788     ErtsAlcType_t n;
3789 };
3790 
3791 static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK];
3792 
3793 static hdbg_mblk *free_hdbg_mblks;
3794 static hdbg_mblk *used_hdbg_mblks;
3795 static erts_mtx_t hdbg_mblk_mtx;
3796 
3797 static void
hdbg_init(void)3798 hdbg_init(void)
3799 {
3800     int i;
3801     for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++)
3802 	hdbg_mblks[i].next = &hdbg_mblks[i+1];
3803     hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL;
3804     free_hdbg_mblks = &hdbg_mblks[0];
3805     used_hdbg_mblks = NULL;
3806     erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug", NIL,
3807         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR);
3808 }
3809 
3810 static void *check_memory_fence(void *ptr,
3811 				Uint *size,
3812 				ErtsAlcType_t n,
3813 				int func);
3814 void erts_hdbg_chk_blks(void);
3815 
3816 void
erts_hdbg_chk_blks(void)3817 erts_hdbg_chk_blks(void)
3818 {
3819     hdbg_mblk *mblk;
3820 
3821     erts_mtx_lock(&hdbg_mblk_mtx);
3822     for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) {
3823 	Uint sz;
3824 	check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK);
3825 	ASSERT(sz == mblk->s);
3826     }
3827     erts_mtx_unlock(&hdbg_mblk_mtx);
3828 }
3829 
3830 static hdbg_mblk *
hdbg_alloc(void * p,Uint s,ErtsAlcType_t n)3831 hdbg_alloc(void *p, Uint s, ErtsAlcType_t n)
3832 {
3833     hdbg_mblk *mblk;
3834 
3835     erts_mtx_lock(&hdbg_mblk_mtx);
3836     mblk = free_hdbg_mblks;
3837     if (!mblk) {
3838 	erts_fprintf(stderr,
3839 		     "Ran out of debug blocks; please increase "
3840 		     "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n",
3841 		     ERL_ALC_HDBG_MAX_MBLK);
3842 	abort();
3843     }
3844     free_hdbg_mblks = mblk->next;
3845 
3846     mblk->p = p;
3847     mblk->s = s;
3848     mblk->n = n;
3849 
3850     mblk->next = used_hdbg_mblks;
3851     mblk->prev = NULL;
3852     if (used_hdbg_mblks)
3853 	used_hdbg_mblks->prev = mblk;
3854     used_hdbg_mblks = mblk;
3855     erts_mtx_unlock(&hdbg_mblk_mtx);
3856     return (void *) mblk;
3857 }
3858 
3859 static void
hdbg_free(hdbg_mblk * mblk)3860 hdbg_free(hdbg_mblk *mblk)
3861 {
3862     erts_mtx_lock(&hdbg_mblk_mtx);
3863     if (mblk->next)
3864 	mblk->next->prev = mblk->prev;
3865     if (mblk->prev)
3866 	mblk->prev->next = mblk->next;
3867     else
3868 	used_hdbg_mblks = mblk->next;
3869 
3870     mblk->next = free_hdbg_mblks;
3871     free_hdbg_mblks = mblk;
3872     erts_mtx_unlock(&hdbg_mblk_mtx);
3873 }
3874 
3875 #endif
3876 
3877 #ifdef  ERTS_ALLOC_UTIL_HARD_DEBUG
3878 static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func);
3879 
check_allocated_block(Uint type,void * blk)3880 void check_allocated_block( Uint type, void *blk)
3881 {
3882     Uint dummy;
3883     check_memory_fence(blk, &dummy, ERTS_ALC_T2N(type), ERTS_ALC_O_FREE);
3884 }
3885 
check_allocators(void)3886 void check_allocators(void)
3887 {
3888     int i;
3889     if (!erts_initialized)
3890 	return;
3891     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; ++i) {
3892 	if (erts_allctrs_info[i].alloc_util) {
3893 	    ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra;
3894 	    Allctr_t *allctr = real_af->extra;
3895 	    Carrier_t *ct;
3896 	if (allctr->thread_safe)
3897 	    erts_mtx_lock(&allctr->mutex);
3898 
3899 	    if (allctr->check_mbc) {
3900 		for (ct = allctr->mbc_list.first; ct; ct = ct->next) {
3901 		    fprintf(stderr,"Checking allocator %d\r\n",i);
3902 		    allctr->check_mbc(allctr,ct);
3903 		}
3904 	    }
3905 	if (allctr->thread_safe)
3906 	    erts_mtx_unlock(&allctr->mutex);
3907 	}
3908     }
3909 }
3910 #endif
3911 
3912 static void *
set_memory_fence(void * ptr,Uint sz,ErtsAlcType_t n)3913 set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n)
3914 {
3915     UWord *ui_ptr;
3916     UWord pattern;
3917 #ifdef HARD_DEBUG
3918     hdbg_mblk **mblkpp;
3919 #endif
3920 
3921     if (!ptr)
3922 	return NULL;
3923 
3924     ui_ptr = (UWord *) ptr;
3925     pattern = MK_PATTERN(n);
3926 
3927 #ifdef HARD_DEBUG
3928     mblkpp = (hdbg_mblk **) ui_ptr++;
3929 #endif
3930 
3931     *(ui_ptr++) = sz;
3932     *(ui_ptr++) = pattern;
3933     sys_memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord));
3934 
3935 #ifdef HARD_DEBUG
3936     *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n);
3937 #endif
3938 
3939     return (void *) ui_ptr;
3940 }
3941 
3942 static void *
check_memory_fence(void * ptr,Uint * size,ErtsAlcType_t n,int func)3943 check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
3944 {
3945     Uint sz;
3946     Uint found_type;
3947     UWord pre_pattern, expected_pattern;
3948     UWord post_pattern;
3949     UWord *ui_ptr;
3950 #ifdef HARD_DEBUG
3951     hdbg_mblk *mblk;
3952 #endif
3953 
3954     if (!ptr)
3955 	return NULL;
3956 
3957     expected_pattern = MK_PATTERN(n);
3958 
3959     ui_ptr = (UWord *) ptr;
3960     pre_pattern = *(--ui_ptr);
3961     *size = sz = *(--ui_ptr);
3962 #ifdef HARD_DEBUG
3963     mblk = (hdbg_mblk *) *(--ui_ptr);
3964 #endif
3965 
3966     found_type = GET_TYPE_OF_PATTERN(pre_pattern);
3967 
3968     if (found_type != n) {
3969         erts_exit(ERTS_ABORT_EXIT, "ERROR: Miss matching allocator types"
3970                   " used in alloc and free\n");
3971     }
3972 
3973     if (pre_pattern != expected_pattern) {
3974 	if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN)
3975 	    erts_exit(ERTS_ABORT_EXIT,
3976 		     "ERROR: Fence at beginning of memory block (p=0x%u) "
3977 		     "clobbered.\n",
3978 		     (UWord) ptr);
3979     }
3980 
3981     sys_memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord));
3982 
3983     if (post_pattern != expected_pattern || pre_pattern != post_pattern) {
3984 	char fbuf[10];
3985 	char obuf[10];
3986 	char *ftype;
3987 	char *otype;
3988 	char *op_str;
3989 
3990 	if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN)
3991 	    erts_exit(ERTS_ABORT_EXIT,
3992 		     "ERROR: Fence at end of memory block (p=0x%u, sz=%u) "
3993 		     "clobbered.\n",
3994 		     (UWord) ptr, (UWord) sz);
3995 	if (found_type != GET_TYPE_OF_PATTERN(post_pattern))
3996 	    erts_exit(ERTS_ABORT_EXIT,
3997 		     "ERROR: Fence around memory block (p=0x%u, sz=%u) "
3998 		     "clobbered.\n",
3999 		     (UWord) ptr, (UWord) sz);
4000 
4001 	ftype = type_no_str(found_type);
4002 	if (!ftype) {
4003 	    erts_snprintf(fbuf, sizeof(fbuf), "%d", (int) found_type);
4004 	    ftype = fbuf;
4005 	}
4006 	otype = type_no_str(n);
4007 	if (!otype) {
4008 	    erts_snprintf(obuf, sizeof(obuf), "%d", (int) n);
4009 	    otype = obuf;
4010 	}
4011 
4012 	switch (func) {
4013 	case ERTS_ALC_O_ALLOC:		op_str = "allocated";	break;
4014 	case ERTS_ALC_O_REALLOC:	op_str = "reallocated";	break;
4015 	case ERTS_ALC_O_FREE:		op_str = "freed";	break;
4016 	default:			op_str = "???";		break;
4017 	}
4018 
4019 	erts_exit(ERTS_ABORT_EXIT,
4020 		 "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\","
4021 		 " but %s as type \"%s\".\n",
4022 		 (UWord) ptr, (UWord) sz, ftype, op_str, otype);
4023     }
4024 
4025 #ifdef HARD_DEBUG
4026     switch (func) {
4027     case ERTS_ALC_O_REALLOC:
4028     case ERTS_ALC_O_FREE:
4029 	hdbg_free(mblk);
4030 	break;
4031     default:
4032 	break;
4033     }
4034 #endif
4035 
4036     return (void *) ui_ptr;
4037 }
4038 
4039 static ErtsAllocatorFunctions_t real_allctrs[ERTS_ALC_A_MAX+1];
4040 
4041 static void *
debug_alloc(ErtsAlcType_t type,void * extra,Uint size)4042 debug_alloc(ErtsAlcType_t type, void *extra, Uint size)
4043 {
4044     ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
4045     ErtsAlcType_t n;
4046     Uint dsize;
4047     void *res;
4048 
4049 #ifdef HARD_DEBUG
4050     erts_hdbg_chk_blks();
4051 #endif
4052 
4053     n = ERTS_ALC_T2N(type);
4054 
4055     ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
4056     dsize = size + FENCE_SZ;
4057     res = (*real_af->alloc)(type, real_af->extra, dsize);
4058 
4059     res = set_memory_fence(res, size, n);
4060 
4061 #ifdef PRINT_OPS
4062     fprintf(stderr, "0x%lx = alloc(%s, %lu)\r\n",
4063 	    (Uint) res, ERTS_ALC_N2TD(n), size);
4064 #endif
4065 
4066     return res;
4067 }
4068 
4069 
4070 static void *
debug_realloc(ErtsAlcType_t type,void * extra,void * ptr,Uint size)4071 debug_realloc(ErtsAlcType_t type, void *extra, void *ptr, Uint size)
4072 {
4073     ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
4074     ErtsAlcType_t n;
4075     Uint dsize;
4076     Uint old_size;
4077     void *dptr;
4078     void *res;
4079 
4080     n = ERTS_ALC_T2N(type);
4081 
4082     ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
4083 
4084     dsize = size + FENCE_SZ;
4085     dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC);
4086 
4087 #ifdef HARD_DEBUG
4088     erts_hdbg_chk_blks();
4089 #endif
4090 
4091     if (ptr && old_size > size)
4092 	sys_memset((void *) (((char *) ptr) + size),
4093 		   0xf,
4094 		   sizeof(Uint) + old_size - size);
4095 
4096     res = (*real_af->realloc)(type, real_af->extra, dptr, dsize);
4097 
4098     res = set_memory_fence(res, size, n);
4099 
4100 #ifdef PRINT_OPS
4101     fprintf(stderr, "0x%lx = realloc(%s, 0x%lx, %lu)\r\n",
4102 	    (Uint) res, ERTS_ALC_N2TD(n), (Uint) ptr, size);
4103 #endif
4104 
4105     return res;
4106 }
4107 
4108 static void
debug_free(ErtsAlcType_t type,void * extra,void * ptr)4109 debug_free(ErtsAlcType_t type, void *extra, void *ptr)
4110 {
4111     ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
4112     ErtsAlcType_t n;
4113     void *dptr;
4114     Uint size;
4115     int free_pattern;
4116 
4117     n = ERTS_ALC_T2N(type);
4118     free_pattern = n;
4119 
4120     ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
4121 
4122     if (!ptr)
4123         return;
4124 
4125     dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE);
4126 
4127 #ifdef ERTS_ALC_A_EXEC
4128 # if defined(__i386__) || defined(__x86_64__)
4129     if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) {
4130         free_pattern = 0x0f; /* Illegal instruction */
4131     }
4132 # endif
4133 #endif
4134     sys_memset((void *) dptr, free_pattern, size + FENCE_SZ);
4135 
4136     (*real_af->free)(type, real_af->extra, dptr);
4137 
4138 #ifdef PRINT_OPS
4139     fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr);
4140 #endif
4141 
4142 #ifdef HARD_DEBUG
4143     erts_hdbg_chk_blks();
4144 #endif
4145 
4146 }
4147 
4148 static Uint
install_debug_functions(void)4149 install_debug_functions(void)
4150 {
4151     int i;
4152     ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
4153 
4154     sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs));
4155 
4156     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
4157 	erts_allctrs[i].alloc	= debug_alloc;
4158 	erts_allctrs[i].realloc	= debug_realloc;
4159 	erts_allctrs[i].free	= debug_free;
4160 	erts_allctrs[i].extra	= (void *) &real_allctrs[i];
4161     }
4162     return FENCE_SZ;
4163 }
4164 
4165 #endif /* #ifdef DEBUG */
4166