1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2002-2020. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21
22 /*
23 * Description: Management of memory allocators.
24 *
25 * Author: Rickard Green
26 */
27
28 #ifdef HAVE_CONFIG_H
29 # include "config.h"
30 #endif
31 #define ERTS_ALLOC_C__
32 #define ERTS_ALC_INTERNAL__
33 #define ERTS_WANT_MEM_MAPPERS
34 #include "sys.h"
35 #define ERL_THREADS_EMU_INTERNAL__
36 #include "erl_threads.h"
37 #include "global.h"
38 #include "erl_db.h"
39 #include "erl_binary.h"
40 #include "erl_bits.h"
41 #include "erl_mtrace.h"
42 #include "erl_mseg.h"
43 #include "erl_monitor_link.h"
44 #include "erl_hl_timer.h"
45 #include "erl_cpu_topology.h"
46 #include "erl_thr_queue.h"
47 #include "erl_nfunc_sched.h"
48 #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
49 #include "erl_check_io.h"
50 #endif
51 #include "erl_bif_unique.h"
52
53 #define GET_ERL_GF_ALLOC_IMPL
54 #include "erl_goodfit_alloc.h"
55 #define GET_ERL_BF_ALLOC_IMPL
56 #include "erl_bestfit_alloc.h"
57 #define GET_ERL_AF_ALLOC_IMPL
58 #include "erl_afit_alloc.h"
59 #define GET_ERL_AOFF_ALLOC_IMPL
60 #include "erl_ao_firstfit_alloc.h"
61
62
63 #if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_AU_MAX_PREF_ALLOC_INSTANCES
64 # error "Too many schedulers; cannot create that many pref alloc instances"
65 #endif
66
67 #define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS
68
69 #if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND)
70 #define AU_ALLOC_DEFAULT_ENABLE(X) 0
71 #else
72 #define AU_ALLOC_DEFAULT_ENABLE(X) (X)
73 #endif
74
75 #define ERTS_ALC_DEFAULT_ENABLED_ACUL 60
76 #define ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC 45
77 #define ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC 85
78
79 #define ERTS_ALC_DEFAULT_ACUL ERTS_ALC_DEFAULT_ENABLED_ACUL
80 #define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC
81 #define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC
82
83
84 #ifdef DEBUG
85 static Uint install_debug_functions(void);
86 #if 0
87 #define HARD_DEBUG
88 #ifdef __GNUC__
89 #warning "* * * * * * * * * * * * * *"
90 #warning "* HARD DEBUG IS ENABLED! *"
91 #warning "* * * * * * * * * * * * * *"
92 #endif
93 #endif
94 #endif
95
96 static int lock_all_physical_memory = 0;
97
98 ErtsAllocatorFunctions_t ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]);
99 ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
100 ErtsAllocatorThrSpec_t ERTS_WRITE_UNLIKELY(erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]);
101
102 #define ERTS_MIN(A, B) ((A) < (B) ? (A) : (B))
103 #define ERTS_MAX(A, B) ((A) > (B) ? (A) : (B))
104
105 typedef union {
106 GFAllctr_t gfa;
107 char align_gfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(GFAllctr_t))];
108 BFAllctr_t bfa;
109 char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))];
110 AFAllctr_t afa;
111 char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))];
112 AOFFAllctr_t aoffa;
113 char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))];
114 } ErtsAllocatorState_t erts_align_attribute(ERTS_CACHE_LINE_SIZE);
115
116 static ErtsAllocatorState_t std_alloc_state;
117 static ErtsAllocatorState_t ll_alloc_state;
118 static ErtsAllocatorState_t sl_alloc_state;
119 static ErtsAllocatorState_t temp_alloc_state;
120 static ErtsAllocatorState_t eheap_alloc_state;
121 static ErtsAllocatorState_t binary_alloc_state;
122 static ErtsAllocatorState_t ets_alloc_state;
123 static ErtsAllocatorState_t driver_alloc_state;
124 static ErtsAllocatorState_t fix_alloc_state;
125 static ErtsAllocatorState_t literal_alloc_state;
126 #ifdef ERTS_ALC_A_EXEC
127 static ErtsAllocatorState_t exec_alloc_state;
128 #endif
129 static ErtsAllocatorState_t test_alloc_state;
130
131 enum {
132 ERTS_ALC_INFO_A_ALLOC_UTIL = ERTS_ALC_A_MAX + 1,
133 ERTS_ALC_INFO_A_MSEG_ALLOC,
134 ERTS_ALC_INFO_A_ERTS_MMAP,
135 ERTS_ALC_INFO_A_DISABLED_EXEC, /* fake a disabled "exec_alloc" */
136 ERTS_ALC_INFO_A_END
137 };
138
139 typedef struct {
140 erts_atomic32_t refc;
141 int only_sz;
142 int internal;
143 Uint req_sched;
144 Process *proc;
145 ErtsIRefStorage iref;
146 int allocs[ERTS_ALC_INFO_A_END - ERTS_ALC_A_MIN + 1];
147 } ErtsAllocInfoReq;
148
149 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq,
150 ErtsAllocInfoReq,
151 5,
152 ERTS_ALC_T_AINFO_REQ)
153
154 ErtsAlcType_t erts_fix_core_allocator_ix;
155
156 struct au_init {
157 int enable;
158 int thr_spec;
159 int disable_allowed;
160 int thr_spec_allowed;
161 int carrier_migration_allowed;
162 ErtsAlcStrat_t astrat;
163 struct {
164 AllctrInit_t util;
165 GFAllctrInit_t gf;
166 BFAllctrInit_t bf;
167 AFAllctrInit_t af;
168 AOFFAllctrInit_t aoff;
169 } init;
170 struct {
171 int mmbcs;
172 int lmbcs;
173 int smbcs;
174 int mmmbc;
175 } default_;
176 };
177
178 #define DEFAULT_ALLCTR_INIT { \
179 ERTS_DEFAULT_ALLCTR_INIT, \
180 ERTS_DEFAULT_GF_ALLCTR_INIT, \
181 ERTS_DEFAULT_BF_ALLCTR_INIT, \
182 ERTS_DEFAULT_AF_ALLCTR_INIT, \
183 ERTS_DEFAULT_AOFF_ALLCTR_INIT \
184 }
185
186 typedef struct {
187 int erts_alloc_config;
188 #if HAVE_ERTS_MSEG
189 ErtsMsegInit_t mseg;
190 #endif
191 int trim_threshold;
192 int top_pad;
193 AlcUInit_t alloc_util;
194 struct {
195 char *mtrace;
196 char *nodename;
197 } instr;
198 struct au_init sl_alloc;
199 struct au_init std_alloc;
200 struct au_init ll_alloc;
201 struct au_init temp_alloc;
202 struct au_init eheap_alloc;
203 struct au_init binary_alloc;
204 struct au_init ets_alloc;
205 struct au_init driver_alloc;
206 struct au_init fix_alloc;
207 struct au_init literal_alloc;
208 struct au_init exec_alloc;
209 struct au_init test_alloc;
210 } erts_alc_hndl_args_init_t;
211
212 #define ERTS_AU_INIT__ {0, 0, 1, 1, 1, \
213 ERTS_ALC_S_GOODFIT, DEFAULT_ALLCTR_INIT, \
214 {1,1,1,1}}
215
216 #define SET_DEFAULT_ALLOC_OPTS(IP) \
217 do { \
218 struct au_init aui__ = ERTS_AU_INIT__; \
219 sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init)); \
220 } while (0)
221
222 static void
set_default_sl_alloc_opts(struct au_init * ip)223 set_default_sl_alloc_opts(struct au_init *ip)
224 {
225 SET_DEFAULT_ALLOC_OPTS(ip);
226 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
227 ip->thr_spec = 1;
228 ip->astrat = ERTS_ALC_S_GOODFIT;
229 ip->init.util.name_prefix = "sl_";
230 ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED;
231 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
232 #ifndef SMALL_MEMORY
233 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
234 #else
235 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
236 #endif
237 ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED;
238 ip->init.util.rsbcst = 80;
239 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
240 }
241
242 static void
set_default_std_alloc_opts(struct au_init * ip)243 set_default_std_alloc_opts(struct au_init *ip)
244 {
245 SET_DEFAULT_ALLOC_OPTS(ip);
246 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
247 ip->thr_spec = 1;
248 ip->astrat = ERTS_ALC_S_BESTFIT;
249 ip->init.util.name_prefix = "std_";
250 ip->init.util.alloc_no = ERTS_ALC_A_STANDARD;
251 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
252 #ifndef SMALL_MEMORY
253 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
254 #else
255 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
256 #endif
257 ip->init.util.ts = ERTS_ALC_MTA_STANDARD;
258 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
259 }
260
261 static void
set_default_ll_alloc_opts(struct au_init * ip)262 set_default_ll_alloc_opts(struct au_init *ip)
263 {
264 SET_DEFAULT_ALLOC_OPTS(ip);
265 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
266 ip->thr_spec = 0;
267 ip->astrat = ERTS_ALC_S_BESTFIT;
268 ip->init.bf.ao = 1;
269 ip->init.util.ramv = 0;
270 ip->init.util.mmsbc = 0;
271 ip->init.util.sbct = ~((UWord) 0);
272 ip->init.util.name_prefix = "ll_";
273 ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED;
274 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
275 #ifndef SMALL_MEMORY
276 ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */
277 #else
278 ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */
279 #endif
280 ip->init.util.ts = ERTS_ALC_MTA_LONG_LIVED;
281 ip->init.util.asbcst = 0;
282 ip->init.util.rsbcst = 0;
283 ip->init.util.rsbcmt = 0;
284 ip->init.util.rmbcmt = 0;
285 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL_LL_ALLOC;
286 }
287
288 static void
set_default_literal_alloc_opts(struct au_init * ip)289 set_default_literal_alloc_opts(struct au_init *ip)
290 {
291 SET_DEFAULT_ALLOC_OPTS(ip);
292 ip->enable = 1;
293 ip->thr_spec = 0;
294 ip->disable_allowed = 0;
295 ip->thr_spec_allowed = 0;
296 ip->carrier_migration_allowed = 0;
297 ip->astrat = ERTS_ALC_S_BESTFIT;
298 ip->init.bf.ao = 1;
299 ip->init.util.ramv = 0;
300 ip->init.util.mmsbc = 0;
301 ip->init.util.sbct = ~((UWord) 0);
302 ip->init.util.name_prefix = "literal_";
303 ip->init.util.alloc_no = ERTS_ALC_A_LITERAL;
304 #ifndef SMALL_MEMORY
305 ip->init.util.mmbcs = 1024*1024; /* Main carrier size */
306 #else
307 ip->init.util.mmbcs = 256*1024; /* Main carrier size */
308 #endif
309 ip->init.util.ts = ERTS_ALC_MTA_LITERAL;
310 ip->init.util.asbcst = 0;
311 ip->init.util.rsbcst = 0;
312 ip->init.util.rsbcmt = 0;
313 ip->init.util.rmbcmt = 0;
314 ip->init.util.acul = 0;
315
316 #if defined(ARCH_32)
317 # if HAVE_ERTS_MSEG
318 ip->init.util.mseg_alloc = &erts_alcu_literal_32_mseg_alloc;
319 ip->init.util.mseg_realloc = &erts_alcu_literal_32_mseg_realloc;
320 ip->init.util.mseg_dealloc = &erts_alcu_literal_32_mseg_dealloc;
321 # endif
322 ip->init.util.sys_alloc = &erts_alcu_literal_32_sys_alloc;
323 ip->init.util.sys_realloc = &erts_alcu_literal_32_sys_realloc;
324 ip->init.util.sys_dealloc = &erts_alcu_literal_32_sys_dealloc;
325 #elif defined(ARCH_64)
326 # ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
327 ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc;
328 ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc;
329 ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc;
330 ip->init.util.mseg_mmapper = &erts_literal_mmapper;
331 # endif
332 #else
333 # error Unknown architecture
334 #endif
335 }
336
337 #ifdef ERTS_ALC_A_EXEC
338 static void
set_default_exec_alloc_opts(struct au_init * ip)339 set_default_exec_alloc_opts(struct au_init *ip)
340 {
341 SET_DEFAULT_ALLOC_OPTS(ip);
342 ip->enable = 1;
343 ip->thr_spec = 0;
344 ip->disable_allowed = 0;
345 ip->thr_spec_allowed = 0;
346 ip->carrier_migration_allowed = 0;
347 ip->astrat = ERTS_ALC_S_BESTFIT;
348 ip->init.bf.ao = 1;
349 ip->init.util.ramv = 0;
350 ip->init.util.mmsbc = 0;
351 ip->init.util.sbct = ~((UWord) 0);
352 ip->init.util.name_prefix = "exec_";
353 ip->init.util.alloc_no = ERTS_ALC_A_EXEC;
354 ip->init.util.mmbcs = 0; /* No main carrier */
355 ip->init.util.ts = ERTS_ALC_MTA_EXEC;
356 ip->init.util.asbcst = 0;
357 ip->init.util.rsbcst = 0;
358 ip->init.util.rsbcmt = 0;
359 ip->init.util.rmbcmt = 0;
360 ip->init.util.acul = 0;
361
362 ip->init.util.mseg_alloc = &erts_alcu_exec_mseg_alloc;
363 ip->init.util.mseg_realloc = &erts_alcu_exec_mseg_realloc;
364 ip->init.util.mseg_dealloc = &erts_alcu_exec_mseg_dealloc;
365 }
366 #endif /* ERTS_ALC_A_EXEC */
367
368 static void
set_default_temp_alloc_opts(struct au_init * ip)369 set_default_temp_alloc_opts(struct au_init *ip)
370 {
371 SET_DEFAULT_ALLOC_OPTS(ip);
372 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
373 ip->thr_spec = 1;
374 ip->disable_allowed = 0;
375 ip->carrier_migration_allowed = 0;
376 ip->astrat = ERTS_ALC_S_AFIT;
377 ip->init.util.name_prefix = "temp_";
378 ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY;
379 #ifndef SMALL_MEMORY
380 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
381 #else
382 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
383 #endif
384 ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY;
385 ip->init.util.rsbcst = 90;
386 ip->init.util.rmbcmt = 100;
387 }
388
389 static void
set_default_eheap_alloc_opts(struct au_init * ip)390 set_default_eheap_alloc_opts(struct au_init *ip)
391 {
392 SET_DEFAULT_ALLOC_OPTS(ip);
393 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
394 ip->thr_spec = 1;
395 ip->astrat = ERTS_ALC_S_GOODFIT;
396 ip->init.util.name_prefix = "eheap_";
397 ip->init.util.alloc_no = ERTS_ALC_A_EHEAP;
398 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
399 #ifndef SMALL_MEMORY
400 ip->init.util.mmbcs = 512*1024; /* Main carrier size */
401 #else
402 ip->init.util.mmbcs = 256*1024; /* Main carrier size */
403 #endif
404 ip->init.util.ts = ERTS_ALC_MTA_EHEAP;
405 ip->init.util.rsbcst = 50;
406 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC;
407 }
408
409 static void
set_default_binary_alloc_opts(struct au_init * ip)410 set_default_binary_alloc_opts(struct au_init *ip)
411 {
412 SET_DEFAULT_ALLOC_OPTS(ip);
413 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
414 ip->thr_spec = 1;
415 ip->astrat = ERTS_ALC_S_BESTFIT;
416 ip->init.util.name_prefix = "binary_";
417 ip->init.util.alloc_no = ERTS_ALC_A_BINARY;
418 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
419 #ifndef SMALL_MEMORY
420 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
421 #else
422 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
423 #endif
424 ip->init.util.ts = ERTS_ALC_MTA_BINARY;
425 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
426 ip->init.util.atags = 1;
427 }
428
429 static void
set_default_ets_alloc_opts(struct au_init * ip)430 set_default_ets_alloc_opts(struct au_init *ip)
431 {
432 SET_DEFAULT_ALLOC_OPTS(ip);
433 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
434 ip->thr_spec = 1;
435 ip->astrat = ERTS_ALC_S_BESTFIT;
436 ip->init.util.name_prefix = "ets_";
437 ip->init.util.alloc_no = ERTS_ALC_A_ETS;
438 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
439 #ifndef SMALL_MEMORY
440 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
441 #else
442 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
443 #endif
444 ip->init.util.ts = ERTS_ALC_MTA_ETS;
445 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
446 }
447
448 static void
set_default_driver_alloc_opts(struct au_init * ip)449 set_default_driver_alloc_opts(struct au_init *ip)
450 {
451 SET_DEFAULT_ALLOC_OPTS(ip);
452 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
453 ip->thr_spec = 1;
454 ip->astrat = ERTS_ALC_S_BESTFIT;
455 ip->init.util.name_prefix = "driver_";
456 ip->init.util.alloc_no = ERTS_ALC_A_DRIVER;
457 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
458 #ifndef SMALL_MEMORY
459 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
460 #else
461 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
462 #endif
463 ip->init.util.ts = ERTS_ALC_MTA_DRIVER;
464 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
465 ip->init.util.atags = 1;
466 }
467
468 static void
set_default_fix_alloc_opts(struct au_init * ip,size_t * fix_type_sizes)469 set_default_fix_alloc_opts(struct au_init *ip,
470 size_t *fix_type_sizes)
471 {
472 SET_DEFAULT_ALLOC_OPTS(ip);
473 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
474 ip->thr_spec = 1;
475 ip->astrat = ERTS_ALC_S_BESTFIT;
476 ip->init.bf.ao = 1;
477 ip->init.util.name_prefix = "fix_";
478 ip->init.util.fix_type_size = fix_type_sizes;
479 ip->init.util.alloc_no = ERTS_ALC_A_FIXED_SIZE;
480 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
481 #ifndef SMALL_MEMORY
482 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
483 #else
484 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
485 #endif
486 ip->init.util.ts = ERTS_ALC_MTA_FIXED_SIZE;
487 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
488 }
489
490 static void
set_default_test_alloc_opts(struct au_init * ip)491 set_default_test_alloc_opts(struct au_init *ip)
492 {
493 SET_DEFAULT_ALLOC_OPTS(ip);
494 ip->enable = 0; /* Disabled by default */
495 ip->thr_spec = -1 * erts_no_schedulers;
496 ip->astrat = ERTS_ALC_S_FIRSTFIT;
497 ip->init.aoff.crr_order = FF_AOFF;
498 ip->init.aoff.blk_order = FF_BF;
499 ip->init.util.name_prefix = "test_";
500 ip->init.util.alloc_no = ERTS_ALC_A_TEST;
501 ip->init.util.cp = ERTS_ALC_A_TEST;
502 ip->init.util.mmbcs = 0; /* Main carrier size */
503 ip->init.util.ts = ERTS_ALC_MTA_TEST;
504 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
505 ip->init.util.atags = 1;
506
507 /* Use a constant minimal MBC size */
508 #if ERTS_SA_MB_CARRIERS
509 ip->init.util.smbcs = ERTS_SACRR_UNIT_SZ;
510 ip->init.util.lmbcs = ERTS_SACRR_UNIT_SZ;
511 ip->init.util.sbct = ERTS_SACRR_UNIT_SZ;
512 #else
513 ip->init.util.smbcs = 1 << 12;
514 ip->init.util.lmbcs = 1 << 12;
515 ip->init.util.sbct = 1 << 12;
516 #endif
517 }
518
519
520
521 static void
adjust_tpref(struct au_init * ip,int no_sched)522 adjust_tpref(struct au_init *ip, int no_sched)
523 {
524 if (ip->thr_spec) {
525 ip->thr_spec = no_sched;
526 ip->thr_spec *= -1; /* thread preferred */
527
528 /* If default ... */
529
530 /* ... shrink main multi-block carrier size */
531 if (ip->default_.mmbcs)
532 ip->init.util.mmbcs /= ERTS_MIN(4, no_sched);
533 /* ... shrink largest multi-block carrier size */
534 if (ip->default_.lmbcs)
535 ip->init.util.lmbcs /= ERTS_MIN(2, no_sched);
536 /* ... shrink smallest multi-block carrier size */
537 if (ip->default_.smbcs)
538 ip->init.util.smbcs /= ERTS_MIN(4, no_sched);
539 }
540 }
541
542
543 static void handle_args(int *, char **, erts_alc_hndl_args_init_t *);
544
545 static void
546 set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu);
547
548 static void
549 start_au_allocator(ErtsAlcType_t alctr_n,
550 struct au_init *init,
551 ErtsAllocatorState_t *state);
552
553 static void
refuse_af_strategy(struct au_init * init)554 refuse_af_strategy(struct au_init *init)
555 {
556 if (init->astrat == ERTS_ALC_S_AFIT)
557 init->astrat = ERTS_ALC_S_GOODFIT;
558 }
559
560 #ifdef HARD_DEBUG
561 static void hdbg_init(void);
562 #endif
563
adjust_fix_alloc_sizes(UWord extra_block_size)564 static void adjust_fix_alloc_sizes(UWord extra_block_size)
565 {
566
567 if (extra_block_size && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled) {
568 int j;
569
570 if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec) {
571 int i;
572 ErtsAllocatorThrSpec_t* tspec;
573
574 tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
575 ASSERT(tspec->enabled);
576
577 for (i=0; i < tspec->size; i++) {
578 Allctr_t* allctr = tspec->allctr[i];
579 for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) {
580 size_t size = allctr->fix[j].type_size;
581 size = MAX(size + extra_block_size,
582 sizeof(ErtsAllctrDDBlock_t));
583 allctr->fix[j].type_size = size;
584 }
585 }
586 }
587 else
588 {
589 Allctr_t* allctr = erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra;
590 for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) {
591 size_t size = allctr->fix[j].type_size;
592 size = MAX(size + extra_block_size,
593 sizeof(ErtsAllctrDDBlock_t));
594 allctr->fix[j].type_size = size;
595 }
596 }
597 }
598 }
599
600 static ERTS_INLINE int
strategy_support_carrier_migration(struct au_init * auip)601 strategy_support_carrier_migration(struct au_init *auip)
602 {
603 /*
604 * Currently only aoff* and ageff* support carrier
605 * migration, i.e, type AOFIRSTFIT.
606 */
607 return auip->astrat == ERTS_ALC_S_FIRSTFIT;
608 }
609
610 static ERTS_INLINE void
adjust_carrier_migration_support(struct au_init * auip)611 adjust_carrier_migration_support(struct au_init *auip)
612 {
613 if (auip->init.util.acul) {
614 auip->thr_spec = -1; /* Need thread preferred */
615
616 /*
617 * If strategy cannot handle carrier migration,
618 * default to a strategy that can...
619 */
620 if (!strategy_support_carrier_migration(auip)) {
621 /* Default to aoffcbf */
622 auip->astrat = ERTS_ALC_S_FIRSTFIT;
623 auip->init.aoff.crr_order = FF_AOFF;
624 auip->init.aoff.blk_order = FF_BF;
625 }
626 }
627 }
628
629 void
erts_alloc_init(int * argc,char ** argv,ErtsAllocInitOpts * eaiop)630 erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
631 {
632 UWord extra_block_size = 0;
633 int i, ncpu;
634 erts_alc_hndl_args_init_t init = {
635 0,
636 #if HAVE_ERTS_MSEG
637 ERTS_MSEG_INIT_DEFAULT_INITIALIZER,
638 #endif
639 ERTS_DEFAULT_TRIM_THRESHOLD,
640 ERTS_DEFAULT_TOP_PAD,
641 ERTS_DEFAULT_ALCU_INIT,
642 };
643 size_t fix_type_sizes[ERTS_ALC_NO_FIXED_SIZES] = {0};
644
645 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)]
646 = sizeof(Process);
647 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR)]
648 = sizeof(ErtsMonitorDataHeap);
649 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LINK)]
650 = sizeof(ErtsLinkData);
651 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
652 = sizeof(ErtsDrvSelectDataState);
653 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_SEL_D_STATE)]
654 = sizeof(ErtsNifSelectDataState);
655 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)]
656 = sizeof(ErtsMessageRef);
657 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)]
658 = sizeof(ErtsThrQElement_t);
659 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LL_PTIMER)]
660 = erts_timer_type_size(ERTS_ALC_T_LL_PTIMER);
661 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_HL_PTIMER)]
662 = erts_timer_type_size(ERTS_ALC_T_HL_PTIMER);
663 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)]
664 = erts_timer_type_size(ERTS_ALC_T_BIF_TIMER);
665 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)]
666 = sizeof(NifExportTrace);
667 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)]
668 = sizeof(ErtsNSchedMagicRefTableEntry);
669 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MINDIRECTION)]
670 = ERTS_MAGIC_BIN_UNALIGNED_SIZE(sizeof(ErtsMagicIndirectionWord));
671
672 #ifdef HARD_DEBUG
673 hdbg_init();
674 #endif
675
676 lock_all_physical_memory = 0;
677
678 ncpu = eaiop->ncpu;
679 if (ncpu < 1)
680 ncpu = 1;
681
682 erts_tsd_key_create(&erts_allctr_prelock_tsd_key,
683 "erts_allctr_prelock_tsd_key");
684
685 erts_sys_alloc_init();
686 erts_init_utils_mem();
687
688 set_default_sl_alloc_opts(&init.sl_alloc);
689 set_default_std_alloc_opts(&init.std_alloc);
690 set_default_ll_alloc_opts(&init.ll_alloc);
691 set_default_temp_alloc_opts(&init.temp_alloc);
692 set_default_eheap_alloc_opts(&init.eheap_alloc);
693 set_default_binary_alloc_opts(&init.binary_alloc);
694 set_default_ets_alloc_opts(&init.ets_alloc);
695 set_default_driver_alloc_opts(&init.driver_alloc);
696 set_default_fix_alloc_opts(&init.fix_alloc,
697 fix_type_sizes);
698 set_default_literal_alloc_opts(&init.literal_alloc);
699 #ifdef ERTS_ALC_A_EXEC
700 set_default_exec_alloc_opts(&init.exec_alloc);
701 #endif
702 set_default_test_alloc_opts(&init.test_alloc);
703
704 if (argc && argv)
705 handle_args(argc, argv, &init);
706
707 if (lock_all_physical_memory) {
708 #ifdef HAVE_MLOCKALL
709 errno = 0;
710 if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) {
711 int err = errno;
712 char *errstr = err ? strerror(err) : "unknown";
713 erts_exit(1, "Failed to lock physical memory: %s (%d)\n",
714 errstr, err);
715 }
716 #else
717 erts_exit(1, "Failed to lock physical memory: Not supported\n");
718 #endif
719 }
720
721
722 /* Make adjustments for carrier migration support */
723 init.temp_alloc.init.util.acul = 0;
724 adjust_carrier_migration_support(&init.sl_alloc);
725 adjust_carrier_migration_support(&init.std_alloc);
726 adjust_carrier_migration_support(&init.ll_alloc);
727 adjust_carrier_migration_support(&init.eheap_alloc);
728 adjust_carrier_migration_support(&init.binary_alloc);
729 adjust_carrier_migration_support(&init.ets_alloc);
730 adjust_carrier_migration_support(&init.driver_alloc);
731 adjust_carrier_migration_support(&init.fix_alloc);
732 adjust_carrier_migration_support(&init.literal_alloc);
733 #ifdef ERTS_ALC_A_EXEC
734 adjust_carrier_migration_support(&init.exec_alloc);
735 #endif
736
737 if (init.erts_alloc_config) {
738 /* Adjust flags that erts_alloc_config won't like */
739
740 /* No thread specific instances */
741 init.temp_alloc.thr_spec = 0;
742 init.sl_alloc.thr_spec = 0;
743 init.std_alloc.thr_spec = 0;
744 init.ll_alloc.thr_spec = 0;
745 init.eheap_alloc.thr_spec = 0;
746 init.binary_alloc.thr_spec = 0;
747 init.ets_alloc.thr_spec = 0;
748 init.driver_alloc.thr_spec = 0;
749 init.fix_alloc.thr_spec = 0;
750 init.literal_alloc.thr_spec = 0;
751 #ifdef ERTS_ALC_A_EXEC
752 init.exec_alloc.thr_spec = 0;
753 #endif
754
755 /* No carrier migration */
756 init.temp_alloc.init.util.acul = 0;
757 init.sl_alloc.init.util.acul = 0;
758 init.std_alloc.init.util.acul = 0;
759 init.ll_alloc.init.util.acul = 0;
760 init.eheap_alloc.init.util.acul = 0;
761 init.binary_alloc.init.util.acul = 0;
762 init.ets_alloc.init.util.acul = 0;
763 init.driver_alloc.init.util.acul = 0;
764 init.fix_alloc.init.util.acul = 0;
765 init.literal_alloc.init.util.acul = 0;
766 #ifdef ERTS_ALC_A_EXEC
767 init.exec_alloc.init.util.acul = 0;
768 #endif
769 }
770
771 /* Only temp_alloc can use thread specific interface */
772 if (init.temp_alloc.thr_spec)
773 init.temp_alloc.thr_spec = erts_no_schedulers;
774
775 /* Others must use thread preferred interface */
776 adjust_tpref(&init.sl_alloc, erts_no_schedulers);
777 adjust_tpref(&init.std_alloc, erts_no_schedulers);
778 adjust_tpref(&init.ll_alloc, erts_no_schedulers);
779 adjust_tpref(&init.eheap_alloc, erts_no_schedulers);
780 adjust_tpref(&init.binary_alloc, erts_no_schedulers);
781 adjust_tpref(&init.ets_alloc, erts_no_schedulers);
782 adjust_tpref(&init.driver_alloc, erts_no_schedulers);
783 adjust_tpref(&init.fix_alloc, erts_no_schedulers);
784 adjust_tpref(&init.literal_alloc, erts_no_schedulers);
785 #ifdef ERTS_ALC_A_EXEC
786 adjust_tpref(&init.exec_alloc, erts_no_schedulers);
787 #endif
788
789
790 /*
791 * The following allocators cannot be run with afit strategy.
792 * Make sure they don't...
793 */
794 refuse_af_strategy(&init.sl_alloc);
795 refuse_af_strategy(&init.std_alloc);
796 refuse_af_strategy(&init.ll_alloc);
797 refuse_af_strategy(&init.eheap_alloc);
798 refuse_af_strategy(&init.binary_alloc);
799 refuse_af_strategy(&init.ets_alloc);
800 refuse_af_strategy(&init.driver_alloc);
801 refuse_af_strategy(&init.fix_alloc);
802 refuse_af_strategy(&init.literal_alloc);
803 #ifdef ERTS_ALC_A_EXEC
804 refuse_af_strategy(&init.exec_alloc);
805 #endif
806
807 if (!init.temp_alloc.thr_spec)
808 refuse_af_strategy(&init.temp_alloc);
809
810 erts_mtrace_pre_init();
811 #if HAVE_ERTS_MSEG
812 init.mseg.nos = erts_no_schedulers;
813 erts_mseg_init(&init.mseg);
814 #endif
815
816 erts_alcu_init(&init.alloc_util);
817 erts_afalc_init();
818 erts_bfalc_init();
819 erts_gfalc_init();
820 erts_aoffalc_init();
821
822 for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
823 erts_allctrs[i].alloc = NULL;
824 erts_allctrs[i].realloc = NULL;
825 erts_allctrs[i].free = NULL;
826 erts_allctrs[i].extra = NULL;
827 erts_allctrs_info[i].alloc_util = 0;
828 erts_allctrs_info[i].enabled = 0;
829 erts_allctrs_info[i].thr_spec = 0;
830 erts_allctrs_info[i].extra = NULL;
831 }
832
833 erts_allctrs[ERTS_ALC_A_SYSTEM].alloc = erts_sys_alloc;
834 erts_allctrs[ERTS_ALC_A_SYSTEM].realloc = erts_sys_realloc;
835 erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free;
836 erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1;
837
838 set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu);
839 set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu);
840 set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu);
841 set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu);
842 set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, ncpu);
843 set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc, ncpu);
844 set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu);
845 set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
846 set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
847 set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu);
848 #ifdef ERTS_ALC_A_EXEC
849 set_au_allocator(ERTS_ALC_A_EXEC, &init.exec_alloc, ncpu);
850 #endif
851 set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu);
852
853 for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
854 if (!erts_allctrs[i].alloc)
855 erts_exit(ERTS_ABORT_EXIT,
856 "Missing alloc function for %s\n", ERTS_ALC_A2AD(i));
857 if (!erts_allctrs[i].realloc)
858 erts_exit(ERTS_ABORT_EXIT,
859 "Missing realloc function for %s\n", ERTS_ALC_A2AD(i));
860 if (!erts_allctrs[i].free)
861 erts_exit(ERTS_ABORT_EXIT,
862 "Missing free function for %s\n", ERTS_ALC_A2AD(i));
863 }
864
865 sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold);
866 sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad);
867
868 erts_mtrace_init(init.instr.mtrace, init.instr.nodename);
869
870 start_au_allocator(ERTS_ALC_A_TEMPORARY,
871 &init.temp_alloc,
872 &temp_alloc_state);
873
874 start_au_allocator(ERTS_ALC_A_SHORT_LIVED,
875 &init.sl_alloc,
876 &sl_alloc_state);
877
878 start_au_allocator(ERTS_ALC_A_STANDARD,
879 &init.std_alloc,
880 &std_alloc_state);
881
882 start_au_allocator(ERTS_ALC_A_LONG_LIVED,
883 &init.ll_alloc,
884 &ll_alloc_state);
885 start_au_allocator(ERTS_ALC_A_EHEAP,
886 &init.eheap_alloc,
887 &eheap_alloc_state);
888
889 start_au_allocator(ERTS_ALC_A_BINARY,
890 &init.binary_alloc,
891 &binary_alloc_state);
892
893 start_au_allocator(ERTS_ALC_A_ETS,
894 &init.ets_alloc,
895 &ets_alloc_state);
896
897 start_au_allocator(ERTS_ALC_A_DRIVER,
898 &init.driver_alloc,
899 &driver_alloc_state);
900
901 start_au_allocator(ERTS_ALC_A_FIXED_SIZE,
902 &init.fix_alloc,
903 &fix_alloc_state);
904 start_au_allocator(ERTS_ALC_A_LITERAL,
905 &init.literal_alloc,
906 &literal_alloc_state);
907 #ifdef ERTS_ALC_A_EXEC
908 start_au_allocator(ERTS_ALC_A_EXEC,
909 &init.exec_alloc,
910 &exec_alloc_state);
911 #endif
912 start_au_allocator(ERTS_ALC_A_TEST,
913 &init.test_alloc,
914 &test_alloc_state);
915
916 erts_mtrace_install_wrapper_functions();
917
918 init_aireq_alloc();
919
920 #ifdef DEBUG
921 extra_block_size += install_debug_functions();
922 #endif
923 adjust_fix_alloc_sizes(extra_block_size);
924 }
925
926 void
erts_alloc_late_init(void)927 erts_alloc_late_init(void)
928 {
929
930 }
931
932 static void *
erts_realloc_fixed_size(ErtsAlcType_t type,void * extra,void * p,Uint size)933 erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size)
934 {
935 erts_exit(ERTS_ABORT_EXIT,
936 "Attempt to reallocate a block of the fixed size type %s\n",
937 ERTS_ALC_T2TD(type));
938 }
939
940
941 static void
set_au_allocator(ErtsAlcType_t alctr_n,struct au_init * init,int ncpu)942 set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
943 {
944 ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
945 ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
946 ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
947
948 /*
949 * Some allocators are forced on if halfword heap is used.
950 */
951 if (init->init.util.force)
952 init->enable = 1;
953
954 tspec->enabled = 0;
955 tspec->dd = 0;
956 tspec->aix = alctr_n;
957 tspec->size = 0;
958 ai->thr_spec = 0;
959
960 if (!init->enable) {
961 af->alloc = erts_sys_alloc;
962 af->realloc = erts_sys_realloc;
963 af->free = erts_sys_free;
964 af->extra = NULL;
965 ai->alloc_util = 0;
966 ai->enabled = 0;
967 ai->extra = NULL;
968 return;
969 }
970
971 if (init->thr_spec) {
972 if (init->thr_spec > 0) {
973 af->alloc = erts_alcu_alloc_thr_spec;
974 if (init->init.util.fix_type_size)
975 af->realloc = erts_realloc_fixed_size;
976 else if (init->init.util.ramv)
977 af->realloc = erts_alcu_realloc_mv_thr_spec;
978 else
979 af->realloc = erts_alcu_realloc_thr_spec;
980 af->free = erts_alcu_free_thr_spec;
981 }
982 else {
983 af->alloc = erts_alcu_alloc_thr_pref;
984 if (init->init.util.fix_type_size)
985 af->realloc = erts_realloc_fixed_size;
986 else if (init->init.util.ramv)
987 af->realloc = erts_alcu_realloc_mv_thr_pref;
988 else
989 af->realloc = erts_alcu_realloc_thr_pref;
990 af->free = erts_alcu_free_thr_pref;
991 tspec->dd = 1;
992 }
993
994 tspec->enabled = 1;
995 tspec->size = abs(init->thr_spec) + 1;
996
997 ai->thr_spec = tspec->size;
998 }
999 else
1000 if (init->init.util.ts) {
1001 af->alloc = erts_alcu_alloc_ts;
1002 if (init->init.util.fix_type_size)
1003 af->realloc = erts_realloc_fixed_size;
1004 else if (init->init.util.ramv)
1005 af->realloc = erts_alcu_realloc_mv_ts;
1006 else
1007 af->realloc = erts_alcu_realloc_ts;
1008 af->free = erts_alcu_free_ts;
1009 }
1010 else
1011 {
1012 erts_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n",
1013 init->init.util.name_prefix);
1014 }
1015 af->extra = NULL;
1016 ai->alloc_util = 1;
1017 ai->enabled = 1;
1018 }
1019
1020 static void
start_au_allocator(ErtsAlcType_t alctr_n,struct au_init * init,ErtsAllocatorState_t * state)1021 start_au_allocator(ErtsAlcType_t alctr_n,
1022 struct au_init *init,
1023 ErtsAllocatorState_t *state)
1024 {
1025 int i;
1026 int size = 1;
1027 void *as0;
1028 ErtsAlcStrat_t astrat;
1029 ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
1030 ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
1031 ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
1032 ErtsAlcFixList_t *fix_lists = NULL;
1033 size_t fix_list_size = 0;
1034
1035 if (!init->enable)
1036 return;
1037
1038 if (init->thr_spec) {
1039 char *states = erts_sys_alloc(0,
1040 NULL,
1041 ((sizeof(Allctr_t *)
1042 * (tspec->size + 1))
1043 + (sizeof(ErtsAllocatorState_t)
1044 * tspec->size)
1045 + ERTS_CACHE_LINE_SIZE - 1));
1046 if (!states)
1047 erts_exit(ERTS_ABORT_EXIT,
1048 "Failed to allocate allocator states for %salloc\n",
1049 init->init.util.name_prefix);
1050 tspec->allctr = (Allctr_t **) states;
1051 states += sizeof(Allctr_t *) * (tspec->size + 1);
1052 states = ((((UWord) states) & ERTS_CACHE_LINE_MASK)
1053 ? (char *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK)
1054 + ERTS_CACHE_LINE_SIZE)
1055 : (char *) states);
1056 tspec->allctr[0] = (Allctr_t *) state;
1057 size = tspec->size;
1058 for (i = 1; i < size; i++)
1059 tspec->allctr[i] = (Allctr_t *)
1060 &((ErtsAllocatorState_t *) states)[i-1];
1061 }
1062
1063 if (init->init.util.fix_type_size) {
1064 size_t tot_fix_list_size;
1065 fix_list_size = sizeof(ErtsAlcFixList_t)*ERTS_ALC_NO_FIXED_SIZES;
1066 fix_list_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(fix_list_size);
1067 tot_fix_list_size = fix_list_size;
1068 if (init->thr_spec)
1069 tot_fix_list_size *= tspec->size;
1070 fix_lists = erts_sys_alloc(0,
1071 NULL,
1072 (tot_fix_list_size
1073 + ERTS_CACHE_LINE_SIZE - 1));
1074 if (!fix_lists)
1075 erts_exit(ERTS_ABORT_EXIT,
1076 "Failed to allocate fix lists for %salloc\n",
1077 init->init.util.name_prefix);
1078
1079 if (((UWord) fix_lists) & ERTS_CACHE_LINE_MASK)
1080 fix_lists = ((ErtsAlcFixList_t *)
1081 ((((UWord) fix_lists) & ~ERTS_CACHE_LINE_MASK)
1082 + ERTS_CACHE_LINE_SIZE));
1083 }
1084
1085 for (i = 0; i < size; i++) {
1086 Allctr_t *as;
1087 astrat = init->astrat;
1088
1089 if (!init->thr_spec)
1090 as0 = state;
1091 else {
1092 as0 = (void *) tspec->allctr[i];
1093 if (!as0)
1094 continue;
1095 if (init->thr_spec < 0) {
1096 init->init.util.ts = i == 0;
1097 init->init.util.tspec = 0;
1098 init->init.util.tpref = -1*init->thr_spec + 1;
1099 }
1100 else {
1101 if (i != 0)
1102 init->init.util.ts = 0;
1103 else {
1104 if (astrat == ERTS_ALC_S_AFIT)
1105 astrat = ERTS_ALC_S_GOODFIT;
1106 init->init.util.ts = 1;
1107 }
1108 init->init.util.tspec = init->thr_spec + 1;
1109 init->init.util.tpref = 0;
1110 }
1111 }
1112
1113 if (fix_lists) {
1114 init->init.util.fix = fix_lists;
1115 fix_lists = ((ErtsAlcFixList_t *)
1116 (((char *) fix_lists) + fix_list_size));
1117 }
1118
1119 init->init.util.alloc_strat = astrat;
1120 init->init.util.ix = i;
1121
1122 switch (astrat) {
1123 case ERTS_ALC_S_GOODFIT:
1124 as = erts_gfalc_start((GFAllctr_t *) as0,
1125 &init->init.gf,
1126 &init->init.util);
1127 break;
1128 case ERTS_ALC_S_BESTFIT:
1129 as = erts_bfalc_start((BFAllctr_t *) as0,
1130 &init->init.bf,
1131 &init->init.util);
1132 break;
1133 case ERTS_ALC_S_AFIT:
1134 as = erts_afalc_start((AFAllctr_t *) as0,
1135 &init->init.af,
1136 &init->init.util);
1137 break;
1138 case ERTS_ALC_S_FIRSTFIT:
1139 as = erts_aoffalc_start((AOFFAllctr_t *) as0,
1140 &init->init.aoff,
1141 &init->init.util);
1142 break;
1143
1144 default:
1145 as = NULL;
1146 ASSERT(0);
1147 }
1148
1149 if (!as)
1150 erts_exit(ERTS_ABORT_EXIT,
1151 "Failed to start %salloc\n", init->init.util.name_prefix);
1152
1153 ASSERT(as == (void *) as0);
1154 af->extra = as;
1155 }
1156
1157 if (init->thr_spec)
1158 af->extra = tspec;
1159 ai->extra = af->extra;
1160 }
1161
1162
bad_param(char * param_start,char * param_end)1163 static void bad_param(char *param_start, char *param_end)
1164 {
1165 size_t len = param_end - param_start;
1166 char param[100];
1167 if (len > 99)
1168 len = 99;
1169 sys_memcpy((void *) param, (void *) param_start, len);
1170 param[len] = '\0';
1171 erts_fprintf(stderr, "bad \"%s\" parameter\n", param);
1172 erts_usage();
1173 }
1174
bad_value(char * param_start,char * param_end,char * value)1175 static void bad_value(char *param_start, char *param_end, char *value)
1176 {
1177 size_t len = param_end - param_start;
1178 char param[100];
1179 if (len > 99)
1180 len = 99;
1181 sys_memcpy((void *) param, (void *) param_start, len);
1182 param[len] = '\0';
1183 erts_fprintf(stderr, "bad \"%s\" value: %s\n", param, value);
1184 erts_usage();
1185 }
1186
1187 /* Get arg marks argument as handled by
1188 putting NULL in argv */
1189 static char *
get_value(char * rest,char ** argv,int * ip)1190 get_value(char* rest, char** argv, int* ip)
1191 {
1192 char *param = argv[*ip]+1;
1193 argv[*ip] = NULL;
1194 if (*rest == '\0') {
1195 char *next = argv[*ip + 1];
1196 if (next[0] == '-'
1197 && next[1] == '-'
1198 && next[2] == '\0') {
1199 bad_value(param, rest, "");
1200 }
1201 (*ip)++;
1202 argv[*ip] = NULL;
1203 return next;
1204 }
1205 return rest;
1206 }
1207
1208 static ERTS_INLINE int
has_prefix(const char * prefix,const char * string)1209 has_prefix(const char *prefix, const char *string)
1210 {
1211 int i;
1212 for (i = 0; prefix[i]; i++)
1213 if (prefix[i] != string[i])
1214 return 0;
1215 return 1;
1216 }
1217
1218 static int
get_bool_value(char * param_end,char ** argv,int * ip)1219 get_bool_value(char *param_end, char** argv, int* ip)
1220 {
1221 char *param = argv[*ip]+1;
1222 char *value = get_value(param_end, argv, ip);
1223 if (sys_strcmp(value, "true") == 0)
1224 return 1;
1225 else if (sys_strcmp(value, "false") == 0)
1226 return 0;
1227 else
1228 bad_value(param, param_end, value);
1229 return -1;
1230 }
1231
kb_to_bytes(Sint kb,Uint * bytes)1232 static Uint kb_to_bytes(Sint kb, Uint *bytes)
1233 {
1234 const Uint max = ((~((Uint) 0))/1024) + 1;
1235
1236 if (kb < 0 || (Uint)kb > max)
1237 return 0;
1238 if ((Uint)kb == max)
1239 *bytes = ~((Uint) 0);
1240 else
1241 *bytes = ((Uint) kb)*1024;
1242 return 1;
1243 }
1244
1245 static Uint
get_kb_value(char * param_end,char ** argv,int * ip)1246 get_kb_value(char *param_end, char** argv, int* ip)
1247 {
1248 Sint tmp;
1249 Uint bytes = 0;
1250 char *rest;
1251 char *param = argv[*ip]+1;
1252 char *value = get_value(param_end, argv, ip);
1253 errno = 0;
1254 tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1255 if (errno != 0 || rest == value || !kb_to_bytes(tmp, &bytes))
1256 bad_value(param, param_end, value);
1257 return bytes;
1258 }
1259
1260 static UWord
get_mb_value(char * param_end,char ** argv,int * ip)1261 get_mb_value(char *param_end, char** argv, int* ip)
1262 {
1263 SWord tmp;
1264 UWord max = ((~((UWord) 0))/(1024*1024)) + 1;
1265 char *rest;
1266 char *param = argv[*ip]+1;
1267 char *value = get_value(param_end, argv, ip);
1268 errno = 0;
1269 tmp = (SWord) ErtsStrToSint(value, &rest, 10);
1270 if (errno != 0 || rest == value || tmp < 0 || max < ((UWord) tmp))
1271 bad_value(param, param_end, value);
1272 if (max == (UWord) tmp)
1273 return ~((UWord) 0);
1274 else
1275 return ((UWord) tmp)*1024*1024;
1276 }
1277
1278
1279 #if 0
1280 static Uint
1281 get_byte_value(char *param_end, char** argv, int* ip)
1282 {
1283 Sint tmp;
1284 char *rest;
1285 char *param = argv[*ip]+1;
1286 char *value = get_value(param_end, argv, ip);
1287 errno = 0;
1288 tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1289 if (errno != 0 || rest == value || tmp < 0)
1290 bad_value(param, param_end, value);
1291 return (Uint) tmp;
1292 }
1293 #endif
1294
1295 static Uint
get_amount_value(char * param_end,char ** argv,int * ip)1296 get_amount_value(char *param_end, char** argv, int* ip)
1297 {
1298 Sint tmp;
1299 char *rest;
1300 char *param = argv[*ip]+1;
1301 char *value = get_value(param_end, argv, ip);
1302 errno = 0;
1303 tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1304 if (errno != 0 || rest == value || tmp < 0)
1305 bad_value(param, param_end, value);
1306 return (Uint) tmp;
1307 }
1308
1309 static Uint
get_acul_value(struct au_init * auip,char * param_end,char ** argv,int * ip)1310 get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip)
1311 {
1312 Sint tmp;
1313 char *rest;
1314 char *param = argv[*ip]+1;
1315 char *value = get_value(param_end, argv, ip);
1316 if (sys_strcmp(value, "de") == 0) {
1317 switch (auip->init.util.alloc_no) {
1318 case ERTS_ALC_A_LONG_LIVED:
1319 return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC;
1320 case ERTS_ALC_A_EHEAP:
1321 return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC;
1322 default:
1323 return ERTS_ALC_DEFAULT_ENABLED_ACUL;
1324 }
1325 }
1326 errno = 0;
1327 tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1328 if (errno != 0 || rest == value || tmp < 0 || 100 < tmp)
1329 bad_value(param, param_end, value);
1330 return (Uint) tmp;
1331 }
1332
1333 static void
handle_au_arg(struct au_init * auip,char * sub_param,char ** argv,int * ip,int u_switch)1334 handle_au_arg(struct au_init *auip,
1335 char* sub_param,
1336 char** argv,
1337 int* ip,
1338 int u_switch)
1339 {
1340 char *param = argv[*ip]+1;
1341
1342 switch (sub_param[0]) {
1343 case 'a':
1344 if (sub_param[1] == 'c') { /* Migration parameters "ac*" */
1345 UWord value;
1346 UWord* wp;
1347 if (!auip->carrier_migration_allowed && !u_switch)
1348 goto bad_switch;
1349
1350 if (has_prefix("acul", sub_param)) {
1351 value = get_acul_value(auip, sub_param + 4, argv, ip);
1352 wp = &auip->init.util.acul;
1353 }
1354 else if (has_prefix("acnl", sub_param)) {
1355 value = get_amount_value(sub_param + 4, argv, ip);
1356 wp = &auip->init.util.acnl;
1357 }
1358 else if (has_prefix("acfml", sub_param)) {
1359 value = get_amount_value(sub_param + 5, argv, ip);
1360 wp = &auip->init.util.acfml;
1361 }
1362 else
1363 goto bad_switch;
1364
1365 if (auip->carrier_migration_allowed)
1366 *wp = value;
1367 }
1368 else if(has_prefix("asbcst", sub_param)) {
1369 auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip);
1370 }
1371 else if(has_prefix("as", sub_param)) {
1372 char *alg = get_value(sub_param + 2, argv, ip);
1373 if (sys_strcmp("bf", alg) == 0) {
1374 auip->astrat = ERTS_ALC_S_BESTFIT;
1375 auip->init.bf.ao = 0;
1376 }
1377 else if (sys_strcmp("aobf", alg) == 0) {
1378 auip->astrat = ERTS_ALC_S_BESTFIT;
1379 auip->init.bf.ao = 1;
1380 }
1381 else if (sys_strcmp("gf", alg) == 0) {
1382 auip->astrat = ERTS_ALC_S_GOODFIT;
1383 }
1384 else if (sys_strcmp("af", alg) == 0) {
1385 auip->astrat = ERTS_ALC_S_AFIT;
1386 }
1387 else if (sys_strcmp("aoff", alg) == 0) {
1388 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1389 auip->init.aoff.crr_order = FF_AOFF;
1390 auip->init.aoff.blk_order = FF_AOFF;
1391 }
1392 else if (sys_strcmp("aoffcbf", alg) == 0) {
1393 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1394 auip->init.aoff.crr_order = FF_AOFF;
1395 auip->init.aoff.blk_order = FF_BF;
1396 }
1397 else if (sys_strcmp("aoffcaobf", alg) == 0) {
1398 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1399 auip->init.aoff.crr_order = FF_AOFF;
1400 auip->init.aoff.blk_order = FF_AOBF;
1401 }
1402 else if (sys_strcmp("ageffcaoff", alg) == 0) {
1403 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1404 auip->init.aoff.crr_order = FF_AGEFF;
1405 auip->init.aoff.blk_order = FF_AOFF;
1406 }
1407 else if (sys_strcmp("ageffcbf", alg) == 0) {
1408 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1409 auip->init.aoff.crr_order = FF_AGEFF;
1410 auip->init.aoff.blk_order = FF_BF;
1411 }
1412 else if (sys_strcmp("ageffcaobf", alg) == 0) {
1413 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1414 auip->init.aoff.crr_order = FF_AGEFF;
1415 auip->init.aoff.blk_order = FF_AOBF;
1416 }
1417 else {
1418 if (auip->init.util.alloc_no == ERTS_ALC_A_TEST
1419 && sys_strcmp("chaosff", alg) == 0) {
1420 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1421 auip->init.aoff.crr_order = FF_CHAOS;
1422 auip->init.aoff.blk_order = FF_CHAOS;
1423 }
1424 else {
1425 bad_value(param, sub_param + 1, alg);
1426 }
1427 }
1428 if (!strategy_support_carrier_migration(auip))
1429 auip->init.util.acul = 0;
1430 } else if (has_prefix("atags", sub_param)) {
1431 auip->init.util.atags = get_bool_value(sub_param + 5, argv, ip);
1432 }
1433 else
1434 goto bad_switch;
1435 break;
1436 case 'c': {
1437 if (has_prefix("cp", sub_param)) {
1438 char *param, *param_end, *value;
1439 int cp;
1440 if (!auip->carrier_migration_allowed && !u_switch)
1441 goto bad_switch;
1442 param = argv[*ip]+1;
1443 param_end = sub_param + 2;
1444 value = get_value(param_end, argv, ip);
1445 if (value[0] == '\0' || value[1] != '\0')
1446 bad_value(param, param_end, value);
1447 switch (value[0]) {
1448 case 'B': cp = ERTS_ALC_A_BINARY; break;
1449 case 'D': cp = ERTS_ALC_A_STANDARD; break;
1450 case 'E': cp = ERTS_ALC_A_ETS; break;
1451 case 'F': cp = ERTS_ALC_A_FIXED_SIZE; break;
1452 case 'H': cp = ERTS_ALC_A_EHEAP; break;
1453 case 'L': cp = ERTS_ALC_A_LONG_LIVED; break;
1454 case 'R': cp = ERTS_ALC_A_DRIVER; break;
1455 case 'S': cp = ERTS_ALC_A_SHORT_LIVED; break;
1456 case '@': cp = ERTS_ALC_COMMON_CPOOL_IX; break;
1457 case ':': cp = auip->init.util.alloc_no; break;
1458 default: cp = -1;
1459 bad_value(param, param_end, value);
1460 break;
1461 }
1462 if (auip->carrier_migration_allowed)
1463 auip->init.util.cp = cp;
1464 }
1465 else
1466 goto bad_switch;
1467 break;
1468 }
1469 case 'e': {
1470 int e = get_bool_value(sub_param + 1, argv, ip);
1471 if (!auip->disable_allowed && !e) {
1472 if (!u_switch)
1473 bad_value(param, sub_param + 1, "false");
1474 else
1475 ASSERT(auip->enable); /* ignore */
1476 }
1477 else auip->enable = e;
1478 break;
1479 }
1480 case 'l':
1481 if (has_prefix("lmbcs", sub_param)) {
1482 auip->default_.lmbcs = 0;
1483 auip->init.util.lmbcs = get_kb_value(sub_param + 5, argv, ip);
1484 }
1485 else
1486 goto bad_switch;
1487 break;
1488 case 'm':
1489 if (has_prefix("mbcgs", sub_param)) {
1490 auip->init.util.mbcgs = get_amount_value(sub_param + 5, argv, ip);
1491
1492 }
1493 else if (has_prefix("mbsd", sub_param)) {
1494 auip->init.gf.mbsd = get_amount_value(sub_param + 4, argv, ip);
1495 if (auip->init.gf.mbsd < 1)
1496 auip->init.gf.mbsd = 1;
1497 }
1498 else if (has_prefix("mmbcs", sub_param)) {
1499 auip->default_.mmbcs = 0;
1500 auip->init.util.mmbcs = get_kb_value(sub_param + 5, argv, ip);
1501 }
1502 else if (has_prefix("mmmbc", sub_param)) {
1503 auip->default_.mmmbc = 0;
1504 auip->init.util.mmmbc = get_amount_value(sub_param + 5, argv, ip);
1505 }
1506 else if (has_prefix("mmsbc", sub_param)) {
1507 auip->init.util.mmsbc = get_amount_value(sub_param + 5, argv, ip);
1508 }
1509 else
1510 goto bad_switch;
1511 break;
1512 case 'r':
1513 if(has_prefix("rsbcmt", sub_param)) {
1514 auip->init.util.rsbcmt = get_amount_value(sub_param + 6, argv, ip);
1515 if (auip->init.util.rsbcmt > 100)
1516 auip->init.util.rsbcmt = 100;
1517 }
1518 else if(has_prefix("rsbcst", sub_param)) {
1519 auip->init.util.rsbcst = get_amount_value(sub_param + 6, argv, ip);
1520 if (auip->init.util.rsbcst > 100)
1521 auip->init.util.rsbcst = 100;
1522 }
1523 else if (has_prefix("rmbcmt", sub_param)) {
1524 auip->init.util.rmbcmt = get_amount_value(sub_param + 6, argv, ip);
1525 if (auip->init.util.rmbcmt > 100)
1526 auip->init.util.rmbcmt = 100;
1527 }
1528 else if (has_prefix("ramv", sub_param)) {
1529 auip->init.util.ramv = get_bool_value(sub_param + 4, argv, ip);
1530 }
1531 else
1532 goto bad_switch;
1533 break;
1534 case 's':
1535 if(has_prefix("sbct", sub_param)) {
1536 auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip);
1537 }
1538 else if (has_prefix("smbcs", sub_param)) {
1539 auip->default_.smbcs = 0;
1540 auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip);
1541 }
1542 else
1543 goto bad_switch;
1544 break;
1545 case 't': {
1546 int res = get_bool_value(sub_param+1, argv, ip);
1547 if (res > 0) {
1548 if (!auip->thr_spec_allowed) {
1549 if (!u_switch)
1550 bad_value(param, sub_param + 1, "true");
1551 else
1552 ASSERT(!auip->thr_spec); /* ignore */
1553 }
1554 else
1555 auip->thr_spec = 1;
1556 break;
1557 }
1558 else if (res == 0) {
1559 auip->thr_spec = 0;
1560 auip->init.util.acul = 0;
1561 break;
1562 }
1563 goto bad_switch;
1564 }
1565 default:
1566 bad_switch:
1567 bad_param(param, sub_param);
1568 }
1569 }
1570
1571 static void
handle_args(int * argc,char ** argv,erts_alc_hndl_args_init_t * init)1572 handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
1573 {
1574 struct au_init *aui[] = {
1575 &init->binary_alloc,
1576 &init->std_alloc,
1577 &init->ets_alloc,
1578 &init->eheap_alloc,
1579 &init->ll_alloc,
1580 &init->driver_alloc,
1581 &init->fix_alloc,
1582 &init->sl_alloc
1583 /* test_alloc not affected by +Mea??? or +Mu??? */
1584 };
1585 int aui_sz = (int) sizeof(aui)/sizeof(aui[0]);
1586 char *arg;
1587 char *rest;
1588 int i, j;
1589
1590 i = 1;
1591
1592 ASSERT(argc && argv && init);
1593
1594 while (i < *argc) {
1595 if(argv[i][0] == '-') {
1596 char *param = argv[i]+1;
1597 switch (argv[i][1]) {
1598 case 'M':
1599 switch (argv[i][2]) {
1600 case 'B':
1601 handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0);
1602 break;
1603 case 'I':
1604 if (has_prefix("scs", argv[i]+3)) {
1605 #if HAVE_ERTS_MSEG
1606 init->mseg.literal_mmap.scs =
1607 #endif
1608 get_mb_value(argv[i]+6, argv, &i);
1609 }
1610 else
1611 handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0);
1612 break;
1613 case 'X':
1614 if (has_prefix("scs", argv[i]+3)) {
1615 /* Ignore obsolete */
1616 (void) get_mb_value(argv[i]+6, argv, &i);
1617 }
1618 else
1619 handle_au_arg(&init->exec_alloc, &argv[i][3], argv, &i, 0);
1620 break;
1621 case 'D':
1622 handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0);
1623 break;
1624 case 'E':
1625 handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i, 0);
1626 break;
1627 case 'F':
1628 handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i, 0);
1629 break;
1630 case 'H':
1631 handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i, 0);
1632 break;
1633 case 'L':
1634 handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i, 0);
1635 break;
1636 case 'M':
1637 if (has_prefix("amcbf", argv[i]+3)) {
1638 #if HAVE_ERTS_MSEG
1639 init->mseg.amcbf =
1640 #endif
1641 get_kb_value(argv[i]+8, argv, &i);
1642 }
1643 else if (has_prefix("rmcbf", argv[i]+3)) {
1644 #if HAVE_ERTS_MSEG
1645 init->mseg.rmcbf =
1646 #endif
1647 get_amount_value(argv[i]+8, argv, &i);
1648 }
1649 else if (has_prefix("mcs", argv[i]+3)) {
1650 #if HAVE_ERTS_MSEG
1651 init->mseg.mcs =
1652 #endif
1653 get_amount_value(argv[i]+6, argv, &i);
1654 }
1655 else if (has_prefix("scs", argv[i]+3)) {
1656 #if HAVE_ERTS_MSEG
1657 init->mseg.dflt_mmap.scs =
1658 #endif
1659 get_mb_value(argv[i]+6, argv, &i);
1660 }
1661 else if (has_prefix("sco", argv[i]+3)) {
1662 #if HAVE_ERTS_MSEG
1663 init->mseg.dflt_mmap.sco =
1664 #endif
1665 get_bool_value(argv[i]+6, argv, &i);
1666 }
1667 else if (has_prefix("scrpm", argv[i]+3)) {
1668 #if HAVE_ERTS_MSEG
1669 init->mseg.dflt_mmap.scrpm =
1670 #endif
1671 get_bool_value(argv[i]+8, argv, &i);
1672 }
1673 else if (has_prefix("scrfsd", argv[i]+3)) {
1674 #if HAVE_ERTS_MSEG
1675 init->mseg.dflt_mmap.scrfsd =
1676 #endif
1677 get_amount_value(argv[i]+9, argv, &i);
1678 }
1679 else {
1680 bad_param(param, param+2);
1681 }
1682 break;
1683 case 'R':
1684 handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i, 0);
1685 break;
1686 case 'S':
1687 handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i, 0);
1688 break;
1689 case 'T':
1690 handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0);
1691 break;
1692 case 'Z':
1693 handle_au_arg(&init->test_alloc, &argv[i][3], argv, &i, 0);
1694 break;
1695 case 'Y': { /* sys_alloc */
1696 if (has_prefix("tt", param+2)) {
1697 /* set trim threshold */
1698 arg = get_value(param+4, argv, &i);
1699 errno = 0;
1700 init->trim_threshold = (int) strtol(arg, &rest, 10);
1701 if (errno != 0
1702 || rest == arg
1703 || init->trim_threshold < 0
1704 || (INT_MAX/1024) < init->trim_threshold) {
1705 bad_value(param, param+4, arg);
1706 }
1707 VERBOSE(DEBUG_SYSTEM,
1708 ("using trim threshold: %d\n",
1709 init->trim_threshold));
1710 init->trim_threshold *= 1024;
1711 }
1712 else if (has_prefix("tp", param+2)) {
1713 /* set top pad */
1714 arg = get_value(param+4, argv, &i);
1715 errno = 0;
1716 init->top_pad = (int) strtol(arg, &rest, 10);
1717 if (errno != 0
1718 || rest == arg
1719 || init->top_pad < 0
1720 || (INT_MAX/1024) < init->top_pad) {
1721 bad_value(param, param+4, arg);
1722 }
1723 VERBOSE(DEBUG_SYSTEM,
1724 ("using top pad: %d\n",init->top_pad));
1725 init->top_pad *= 1024;
1726 }
1727 else if (has_prefix("m", param+2)) {
1728 /* Has been handled by erlexec */
1729 (void) get_value(param+3, argv, &i);
1730 }
1731 else if (has_prefix("e", param+2)) {
1732 arg = get_value(param+3, argv, &i);
1733 if (sys_strcmp("true", arg) != 0)
1734 bad_value(param, param+3, arg);
1735 }
1736 else
1737 bad_param(param, param+2);
1738 break;
1739 }
1740 case 'e':
1741 switch (argv[i][3]) {
1742 case 'a': {
1743 int a;
1744 arg = get_value(argv[i]+4, argv, &i);
1745 if (sys_strcmp("min", arg) == 0) {
1746 for (a = 0; a < aui_sz; a++)
1747 aui[a]->enable = 0;
1748 }
1749 else if (sys_strcmp("max", arg) == 0) {
1750 for (a = 0; a < aui_sz; a++)
1751 aui[a]->enable = 1;
1752 }
1753 else if (sys_strcmp("config", arg) == 0) {
1754 init->erts_alloc_config = 1;
1755 }
1756 else if (sys_strcmp("r9c", arg) == 0
1757 || sys_strcmp("r10b", arg) == 0
1758 || sys_strcmp("r11b", arg) == 0) {
1759 set_default_sl_alloc_opts(&init->sl_alloc);
1760 set_default_std_alloc_opts(&init->std_alloc);
1761 set_default_ll_alloc_opts(&init->ll_alloc);
1762 set_default_temp_alloc_opts(&init->temp_alloc);
1763 set_default_eheap_alloc_opts(&init->eheap_alloc);
1764 set_default_binary_alloc_opts(&init->binary_alloc);
1765 set_default_ets_alloc_opts(&init->ets_alloc);
1766 set_default_driver_alloc_opts(&init->driver_alloc);
1767 set_default_driver_alloc_opts(&init->fix_alloc);
1768
1769 init->driver_alloc.enable = 0;
1770 if (sys_strcmp("r9c", arg) == 0) {
1771 init->sl_alloc.enable = 0;
1772 init->std_alloc.enable = 0;
1773 init->binary_alloc.enable = 0;
1774 init->ets_alloc.enable = 0;
1775 }
1776
1777 for (a = 0; a < aui_sz; a++) {
1778 aui[a]->thr_spec = 0;
1779 aui[a]->init.util.acul = 0;
1780 aui[a]->init.util.ramv = 0;
1781 aui[a]->init.util.lmbcs = 5*1024*1024;
1782 }
1783 }
1784 else {
1785 bad_param(param, param+3);
1786 }
1787 break;
1788 }
1789 default:
1790 bad_param(param, param+1);
1791 }
1792 break;
1793 case 'i':
1794 switch (argv[i][3]) {
1795 case 't':
1796 init->instr.mtrace = get_value(argv[i]+4, argv, &i);
1797 break;
1798 default:
1799 bad_param(param, param+2);
1800 }
1801 break;
1802 case 'l':
1803 if (has_prefix("pm", param+2)) {
1804 arg = get_value(argv[i]+5, argv, &i);
1805 if (sys_strcmp("all", arg) == 0)
1806 lock_all_physical_memory = 1;
1807 else if (sys_strcmp("no", arg) == 0)
1808 lock_all_physical_memory = 0;
1809 else
1810 bad_value(param, param+4, arg);
1811 break;
1812 }
1813 bad_param(param, param+2);
1814 break;
1815 case 'u':
1816 if (has_prefix("ycs", argv[i]+3)) {
1817 init->alloc_util.ycs
1818 = get_kb_value(argv[i]+6, argv, &i);
1819 }
1820 else if (has_prefix("mmc", argv[i]+3)) {
1821 init->alloc_util.mmc
1822 = get_amount_value(argv[i]+6, argv, &i);
1823 }
1824 else if (has_prefix("sac", argv[i]+3)) {
1825 init->alloc_util.sac
1826 = get_bool_value(argv[i]+6, argv, &i);
1827 }
1828 else {
1829 int a;
1830 int start = i;
1831 char *param = argv[i];
1832 char *val = i+1 < *argc ? argv[i+1] : NULL;
1833
1834 for (a = 0; a < aui_sz; a++) {
1835 if (a > 0) {
1836 ASSERT(i == start || i == start+1);
1837 argv[start] = param;
1838 if (i != start)
1839 argv[start + 1] = val;
1840 i = start;
1841 }
1842 handle_au_arg(aui[a], &argv[i][3], argv, &i, 1);
1843 }
1844 }
1845 break;
1846 default:
1847 bad_param(param, param+1);
1848 }
1849 break;
1850 case '-':
1851 if (argv[i][2] == '\0') {
1852 /* End of system flags reached */
1853 if (init->instr.mtrace) {
1854 while (i < *argc) {
1855 if(sys_strcmp(argv[i], "-sname") == 0
1856 || sys_strcmp(argv[i], "-name") == 0) {
1857 if (i + 1 <*argc) {
1858 init->instr.nodename = argv[i+1];
1859 break;
1860 }
1861 }
1862 i++;
1863 }
1864 }
1865 goto args_parsed;
1866 }
1867 break;
1868 default:
1869 break;
1870 }
1871 }
1872 i++;
1873 }
1874
1875 args_parsed:
1876 /* Handled arguments have been marked with NULL. Slide arguments
1877 not handled towards the beginning of argv. */
1878 for (i = 0, j = 0; i < *argc; i++) {
1879 if (argv[i])
1880 argv[j++] = argv[i];
1881 }
1882 *argc = j;
1883 }
1884
type_no_str(ErtsAlcType_t n)1885 static char *type_no_str(ErtsAlcType_t n)
1886 {
1887
1888 #if ERTS_ALC_N_MIN != 0
1889 if (n < ERTS_ALC_N_MIN)
1890 return NULL;
1891 #endif
1892 if (n > ERTS_ALC_N_MAX)
1893 return NULL;
1894 return (char *) ERTS_ALC_N2TD(n);
1895 }
1896
1897 #define type_str(T) type_no_str(ERTS_ALC_T2N((T)))
1898
1899 void
erts_alloc_register_scheduler(void * vesdp)1900 erts_alloc_register_scheduler(void *vesdp)
1901 {
1902 ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
1903 int ix = (int) esdp->no;
1904 int aix;
1905
1906 ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
1907 for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
1908 ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
1909 esdp->alloc_data.deallctr[aix] = NULL;
1910 esdp->alloc_data.pref_ix[aix] = -1;
1911 if (tspec->enabled) {
1912 if (!tspec->dd)
1913 esdp->alloc_data.pref_ix[aix] = ix;
1914 else {
1915 Allctr_t *allctr = tspec->allctr[ix];
1916 ASSERT(allctr);
1917 esdp->alloc_data.deallctr[aix] = allctr;
1918 esdp->alloc_data.pref_ix[aix] = ix;
1919 }
1920 }
1921 }
1922 }
1923
1924 void
erts_alloc_scheduler_handle_delayed_dealloc(void * vesdp,int * need_thr_progress,ErtsThrPrgrVal * thr_prgr_p,int * more_work)1925 erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
1926 int *need_thr_progress,
1927 ErtsThrPrgrVal *thr_prgr_p,
1928 int *more_work)
1929 {
1930 ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
1931 int aix;
1932 for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
1933 Allctr_t *allctr;
1934 if (esdp)
1935 allctr = esdp->alloc_data.deallctr[aix];
1936 else {
1937 ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
1938 if (tspec->enabled && tspec->dd)
1939 allctr = tspec->allctr[0];
1940 else
1941 allctr = NULL;
1942 }
1943 if (allctr) {
1944 erts_alcu_check_delayed_dealloc(allctr,
1945 1,
1946 need_thr_progress,
1947 thr_prgr_p,
1948 more_work);
1949 }
1950 }
1951 }
1952
1953 erts_aint32_t
erts_alloc_fix_alloc_shrink(int ix,erts_aint32_t flgs)1954 erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
1955 {
1956 ErtsAllocatorThrSpec_t *tspec;
1957 tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
1958 if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec && tspec->enabled)
1959 return erts_alcu_fix_alloc_shrink(tspec->allctr[ix], flgs);
1960 if (ix == 0 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
1961 return erts_alcu_fix_alloc_shrink(
1962 erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
1963 return 0;
1964 }
1965
1966 static void
no_verify(Allctr_t * allctr)1967 no_verify(Allctr_t *allctr)
1968 {
1969
1970 }
1971
1972 erts_alloc_verify_func_t
erts_alloc_get_verify_unused_temp_alloc(Allctr_t ** allctr)1973 erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr)
1974 {
1975 if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util
1976 && erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) {
1977 ErtsAllocatorThrSpec_t *tspec;
1978 int ix = ERTS_ALC_GET_THR_IX();
1979 tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY];
1980
1981 if (ix < tspec->size) {
1982 *allctr = tspec->allctr[ix];
1983 return erts_alcu_verify_unused;
1984 }
1985 }
1986
1987 *allctr = NULL;
1988 return no_verify;
1989 }
1990
1991 __decl_noreturn void
erts_alc_fatal_error(int error,int func,ErtsAlcType_t n,...)1992 erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
1993 {
1994 char buf[10];
1995 char *t_str;
1996 char *allctr_str;
1997
1998 ASSERT(n >= ERTS_ALC_N_MIN);
1999 ASSERT(n <= ERTS_ALC_N_MAX);
2000
2001
2002 if (n < ERTS_ALC_N_MIN || ERTS_ALC_N_MAX < n)
2003 allctr_str = "UNKNOWN";
2004 else {
2005 ErtsAlcType_t a = ERTS_ALC_T2A(ERTS_ALC_N2T(n));
2006 if (erts_allctrs_info[a].enabled)
2007 allctr_str = (char *) ERTS_ALC_A2AD(a);
2008 else
2009 allctr_str = (char *) ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM);
2010 }
2011
2012 t_str = type_no_str(n);
2013 if (!t_str) {
2014 erts_snprintf(buf, sizeof(buf), "%d", (int) n);
2015 t_str = buf;
2016 }
2017
2018 switch (error) {
2019 case ERTS_ALC_E_NOTSUP: {
2020 char *op_str;
2021 switch (func) {
2022 case ERTS_ALC_O_ALLOC: op_str = "alloc"; break;
2023 case ERTS_ALC_O_REALLOC: op_str = "realloc"; break;
2024 case ERTS_ALC_O_FREE: op_str = "free"; break;
2025 default: op_str = "UNKNOWN"; break;
2026 }
2027 erts_exit(ERTS_ABORT_EXIT,
2028 "%s: %s operation not supported (memory type: \"%s\")\n",
2029 allctr_str, op_str, t_str);
2030 break;
2031 }
2032 case ERTS_ALC_E_NOMEM: {
2033 Uint size;
2034 va_list argp;
2035 char *op = func == ERTS_ALC_O_REALLOC ? "reallocate" : "allocate";
2036
2037
2038 va_start(argp, n);
2039 size = va_arg(argp, Uint);
2040 va_end(argp);
2041 erts_exit(ERTS_DUMP_EXIT,
2042 "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n",
2043 allctr_str, op, size, t_str);
2044 break;
2045 }
2046 case ERTS_ALC_E_NOALLCTR:
2047 erts_exit(ERTS_ABORT_EXIT,
2048 "erts_alloc: Unknown allocator type: %d\n",
2049 ERTS_ALC_T2A(ERTS_ALC_N2T(n)));
2050 break;
2051 default:
2052 erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error);
2053 break;
2054 }
2055 }
2056
2057 __decl_noreturn void
erts_alloc_enomem(ErtsAlcType_t type,Uint size)2058 erts_alloc_enomem(ErtsAlcType_t type, Uint size)
2059 {
2060 erts_alloc_n_enomem(ERTS_ALC_T2N(type), size);
2061 }
2062
2063 __decl_noreturn void
erts_alloc_n_enomem(ErtsAlcType_t n,Uint size)2064 erts_alloc_n_enomem(ErtsAlcType_t n, Uint size)
2065 {
2066 erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_ALLOC, n, size);
2067 }
2068
2069 __decl_noreturn void
erts_realloc_enomem(ErtsAlcType_t type,void * ptr,Uint size)2070 erts_realloc_enomem(ErtsAlcType_t type, void *ptr, Uint size)
2071 {
2072 erts_realloc_n_enomem(ERTS_ALC_T2N(type), ptr, size);
2073 }
2074
2075 __decl_noreturn void
erts_realloc_n_enomem(ErtsAlcType_t n,void * ptr,Uint size)2076 erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size)
2077 {
2078 erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size);
2079 }
2080
2081 static ERTS_INLINE UWord
alcu_size(ErtsAlcType_t alloc_no,ErtsAlcUFixInfo_t * fi,int fisz)2082 alcu_size(ErtsAlcType_t alloc_no, ErtsAlcUFixInfo_t *fi, int fisz)
2083 {
2084 UWord res;
2085 int ai;
2086
2087 if (!erts_allctrs_info[alloc_no].thr_spec) {
2088 AllctrSize_t size;
2089 Allctr_t *allctr;
2090
2091 allctr = erts_allctrs_info[alloc_no].extra;
2092 erts_alcu_current_size(allctr, &size, fi, fisz);
2093
2094 return size.blocks;
2095 }
2096
2097 res = 0;
2098
2099 /* Thread-specific allocators can migrate carriers across types, so we have
2100 * to visit every allocator type to gather information on blocks that were
2101 * allocated by us. */
2102 for (ai = ERTS_ALC_A_MIN; ai < ERTS_ALC_A_MAX; ai++) {
2103 ErtsAllocatorThrSpec_t *tspec;
2104 Allctr_t *allctr;
2105 int i;
2106
2107 if (!erts_allctrs_info[ai].thr_spec) {
2108 continue;
2109 }
2110
2111 tspec = &erts_allctr_thr_spec[ai];
2112 ASSERT(tspec->enabled);
2113
2114 for (i = tspec->size - 1; i >= 0; i--) {
2115 allctr = tspec->allctr[i];
2116
2117 if (allctr) {
2118 AllctrSize_t size;
2119
2120 if (ai == alloc_no) {
2121 erts_alcu_current_size(allctr, &size, fi, fisz);
2122 } else {
2123 erts_alcu_foreign_size(allctr, alloc_no, &size);
2124 }
2125
2126 ASSERT(((SWord)size.blocks) >= 0);
2127
2128 res += size.blocks;
2129 }
2130 }
2131 }
2132
2133 return res;
2134 }
2135
2136 static ERTS_INLINE void
add_fix_values(UWord * ap,UWord * up,ErtsAlcUFixInfo_t * fi,ErtsAlcType_t type)2137 add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type)
2138 {
2139 int ix = ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE;
2140 ASSERT(0 <= ix && ix < ERTS_ALC_NO_FIXED_SIZES);
2141
2142 *ap += (UWord) fi[ix].allocated;
2143 *up += (UWord) fi[ix].used;
2144 }
2145
2146 Eterm
erts_memory(fmtfn_t * print_to_p,void * print_to_arg,void * proc,Eterm earg)2147 erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
2148 {
2149 /*
2150 * NOTE! When updating this function, make sure to also update
2151 * erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl
2152 */
2153 #define ERTS_MEM_NEED_ALL_ALCU (want_tot_or_sys)
2154 struct {
2155 int total;
2156 int processes;
2157 int processes_used;
2158 int system;
2159 int atom;
2160 int atom_used;
2161 int binary;
2162 int code;
2163 int ets;
2164 } want = {0};
2165 struct {
2166 UWord total;
2167 UWord processes;
2168 UWord processes_used;
2169 UWord system;
2170 UWord atom;
2171 UWord atom_used;
2172 UWord binary;
2173 UWord code;
2174 UWord ets;
2175 } size = {0};
2176 Eterm atoms[sizeof(size)/sizeof(UWord)];
2177 UWord *uintps[sizeof(size)/sizeof(UWord)];
2178 Eterm euints[sizeof(size)/sizeof(UWord)];
2179 int want_tot_or_sys;
2180 int length;
2181 Eterm res = THE_NON_VALUE;
2182 ErtsAlcType_t ai;
2183 int only_one_value = 0;
2184 ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}};
2185
2186 ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
2187
2188 /* Figure out whats wanted... */
2189
2190 length = 0;
2191 if (is_non_value(earg)) { /* i.e. wants all */
2192 want.total = 1;
2193 atoms[length] = am_total;
2194 uintps[length++] = &size.total;
2195
2196 want.processes = 1;
2197 atoms[length] = am_processes;
2198 uintps[length++] = &size.processes;
2199
2200 want.processes_used = 1;
2201 atoms[length] = am_processes_used;
2202 uintps[length++] = &size.processes_used;
2203
2204 want.system = 1;
2205 atoms[length] = am_system;
2206 uintps[length++] = &size.system;
2207
2208 want.atom = 1;
2209 atoms[length] = am_atom;
2210 uintps[length++] = &size.atom;
2211
2212 want.atom_used = 1;
2213 atoms[length] = am_atom_used;
2214 uintps[length++] = &size.atom_used;
2215
2216 want.binary = 1;
2217 atoms[length] = am_binary;
2218 uintps[length++] = &size.binary;
2219
2220 want.code = 1;
2221 atoms[length] = am_code;
2222 uintps[length++] = &size.code;
2223
2224 want.ets = 1;
2225 atoms[length] = am_ets;
2226 uintps[length++] = &size.ets;
2227 }
2228 else {
2229 DeclareTmpHeapNoproc(tmp_heap,2);
2230 Eterm wanted_list;
2231
2232 if (is_nil(earg))
2233 return NIL;
2234
2235 UseTmpHeapNoproc(2);
2236 if (is_not_atom(earg))
2237 wanted_list = earg;
2238 else {
2239 wanted_list = CONS(&tmp_heap[0], earg, NIL);
2240 only_one_value = 1;
2241 }
2242
2243 while (is_list(wanted_list)) {
2244 switch (CAR(list_val(wanted_list))) {
2245 case am_total:
2246 if (!want.total) {
2247 want.total = 1;
2248 atoms[length] = am_total;
2249 uintps[length++] = &size.total;
2250 }
2251 break;
2252 case am_processes:
2253 if (!want.processes) {
2254 want.processes = 1;
2255 atoms[length] = am_processes;
2256 uintps[length++] = &size.processes;
2257 }
2258 break;
2259 case am_processes_used:
2260 if (!want.processes_used) {
2261 want.processes_used = 1;
2262 atoms[length] = am_processes_used;
2263 uintps[length++] = &size.processes_used;
2264 }
2265 break;
2266 case am_system:
2267 if (!want.system) {
2268 want.system = 1;
2269 atoms[length] = am_system;
2270 uintps[length++] = &size.system;
2271 }
2272 break;
2273 case am_atom:
2274 if (!want.atom) {
2275 want.atom = 1;
2276 atoms[length] = am_atom;
2277 uintps[length++] = &size.atom;
2278 }
2279 break;
2280 case am_atom_used:
2281 if (!want.atom_used) {
2282 want.atom_used = 1;
2283 atoms[length] = am_atom_used;
2284 uintps[length++] = &size.atom_used;
2285 }
2286 break;
2287 case am_binary:
2288 if (!want.binary) {
2289 want.binary = 1;
2290 atoms[length] = am_binary;
2291 uintps[length++] = &size.binary;
2292 }
2293 break;
2294 case am_code:
2295 if (!want.code) {
2296 want.code = 1;
2297 atoms[length] = am_code;
2298 uintps[length++] = &size.code;
2299 }
2300 break;
2301 case am_ets:
2302 if (!want.ets) {
2303 want.ets = 1;
2304 atoms[length] = am_ets;
2305 uintps[length++] = &size.ets;
2306 }
2307 break;
2308 default:
2309 UnUseTmpHeapNoproc(2);
2310 return am_badarg;
2311 }
2312 wanted_list = CDR(list_val(wanted_list));
2313 }
2314 UnUseTmpHeapNoproc(2);
2315 if (is_not_nil(wanted_list))
2316 return am_badarg;
2317 }
2318
2319 /* All alloc_util allocators *have* to be enabled, except test_alloc */
2320
2321 for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
2322 switch (ai) {
2323 case ERTS_ALC_A_SYSTEM:
2324 case ERTS_ALC_A_TEST:
2325 break;
2326 default:
2327 if (!erts_allctrs_info[ai].enabled
2328 || !erts_allctrs_info[ai].alloc_util) {
2329 return am_notsup;
2330 }
2331 break;
2332 }
2333 }
2334
2335 ASSERT(length <= sizeof(atoms)/sizeof(Eterm));
2336 ASSERT(length <= sizeof(euints)/sizeof(Eterm));
2337 ASSERT(length <= sizeof(uintps)/sizeof(UWord));
2338
2339
2340 if (proc) {
2341 ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN
2342 == erts_proc_lc_my_proc_locks(proc));
2343 /* We'll need locks early in the lock order */
2344 erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
2345 }
2346
2347 /* Calculate values needed... */
2348
2349 want_tot_or_sys = want.total || want.system;
2350
2351 if (ERTS_MEM_NEED_ALL_ALCU) {
2352 size.total = 0;
2353
2354 for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
2355 if (erts_allctrs_info[ai].alloc_util) {
2356 UWord *save;
2357 UWord asz;
2358 switch (ai) {
2359 case ERTS_ALC_A_TEMPORARY:
2360 /*
2361 * Often not thread safe and usually never
2362 * contain any allocated memory.
2363 */
2364 continue;
2365 case ERTS_ALC_A_TEST:
2366 continue;
2367 case ERTS_ALC_A_EHEAP:
2368 save = &size.processes;
2369 break;
2370 case ERTS_ALC_A_ETS:
2371 save = &size.ets;
2372 break;
2373 case ERTS_ALC_A_BINARY:
2374 save = &size.binary;
2375 break;
2376 case ERTS_ALC_A_FIXED_SIZE:
2377 asz = alcu_size(ai, fi, ERTS_ALC_NO_FIXED_SIZES);
2378 size.total += asz;
2379 continue;
2380 default:
2381 save = NULL;
2382 break;
2383 }
2384 asz = alcu_size(ai, NULL, 0);
2385 if (save)
2386 *save = asz;
2387 size.total += asz;
2388 }
2389 }
2390 }
2391
2392
2393
2394 if (want_tot_or_sys || want.processes || want.processes_used) {
2395 UWord tmp;
2396
2397 if (ERTS_MEM_NEED_ALL_ALCU)
2398 tmp = size.processes;
2399 else {
2400 alcu_size(ERTS_ALC_A_FIXED_SIZE,
2401 fi, ERTS_ALC_NO_FIXED_SIZES);
2402 tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0);
2403 }
2404 tmp += erts_ptab_mem_size(&erts_proc);
2405 tmp += erts_bif_timer_memory_size();
2406
2407 size.processes = size.processes_used = tmp;
2408
2409 add_fix_values(&size.processes,
2410 &size.processes_used,
2411 fi,
2412 ERTS_ALC_T_PROC);
2413 add_fix_values(&size.processes,
2414 &size.processes_used,
2415 fi,
2416 ERTS_ALC_T_MONITOR);
2417 add_fix_values(&size.processes,
2418 &size.processes_used,
2419 fi,
2420 ERTS_ALC_T_LINK);
2421 add_fix_values(&size.processes,
2422 &size.processes_used,
2423 fi,
2424 ERTS_ALC_T_MSG_REF);
2425 add_fix_values(&size.processes,
2426 &size.processes_used,
2427 fi,
2428 ERTS_ALC_T_LL_PTIMER);
2429 add_fix_values(&size.processes,
2430 &size.processes_used,
2431 fi,
2432 ERTS_ALC_T_HL_PTIMER);
2433 add_fix_values(&size.processes,
2434 &size.processes_used,
2435 fi,
2436 ERTS_ALC_T_BIF_TIMER);
2437 add_fix_values(&size.processes,
2438 &size.processes_used,
2439 fi,
2440 ERTS_ALC_T_NIF_EXP_TRACE);
2441 }
2442
2443 if (want.atom || want.atom_used) {
2444 Uint reserved_atom_space, atom_space;
2445 erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space);
2446 size.atom = size.atom_used = atom_table_sz();
2447
2448 if (want.atom)
2449 size.atom += reserved_atom_space;
2450
2451 if (want.atom_used)
2452 size.atom_used += atom_space;
2453 }
2454
2455 if (!ERTS_MEM_NEED_ALL_ALCU && want.binary)
2456 size.binary = alcu_size(ERTS_ALC_A_BINARY, NULL, 0);
2457
2458 if (want.code) {
2459 size.code = module_table_sz();
2460 size.code += export_table_sz();
2461 size.code += export_entries_sz();
2462 size.code += erts_fun_table_sz();
2463 size.code += erts_ranges_sz();
2464 size.code += erts_total_code_size;
2465 }
2466
2467 if (want.ets) {
2468 if (!ERTS_MEM_NEED_ALL_ALCU)
2469 size.ets = alcu_size(ERTS_ALC_A_ETS, NULL, 0);
2470 size.ets += erts_get_ets_misc_mem_size();
2471 }
2472
2473 if (want_tot_or_sys) {
2474 ASSERT(size.total >= size.processes);
2475 size.system = size.total - size.processes;
2476 }
2477
2478 if (print_to_p) {
2479 int i;
2480 fmtfn_t to = *print_to_p;
2481 void *arg = print_to_arg;
2482
2483 /* Print result... */
2484 erts_print(to, arg, "=memory\n");
2485 for (i = 0; i < length; i++)
2486 erts_print(to, arg, "%T: %bpu\n", atoms[i], *uintps[i]);
2487 }
2488
2489 if (proc) {
2490 /* Build erlang term result... */
2491 Uint *hp;
2492 Uint hsz;
2493
2494 erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
2495
2496 if (only_one_value) {
2497 ASSERT(length == 1);
2498 hsz = 0;
2499 erts_bld_uword(NULL, &hsz, *uintps[0]);
2500 hp = hsz ? HAlloc((Process *) proc, hsz) : NULL;
2501 res = erts_bld_uword(&hp, NULL, *uintps[0]);
2502 }
2503 else {
2504 Uint **hpp = NULL;
2505 Uint *hszp = &hsz;
2506 hsz = 0;
2507
2508 while (1) {
2509 int i;
2510 for (i = 0; i < length; i++)
2511 euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]);
2512 res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints);
2513 if (hpp)
2514 break;
2515 hp = HAlloc((Process *) proc, hsz);
2516 hpp = &hp;
2517 hszp = NULL;
2518 }
2519 }
2520 }
2521
2522 return res;
2523
2524 #undef ERTS_MEM_NEED_ALL_ALCU
2525 }
2526
2527 struct aa_values {
2528 Uint arity;
2529 const char *name;
2530 Uint ui[2];
2531 };
2532
2533 Eterm
erts_allocated_areas(fmtfn_t * print_to_p,void * print_to_arg,void * proc)2534 erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc)
2535 {
2536 #define MAX_AA_VALUES (24)
2537 struct aa_values values[MAX_AA_VALUES];
2538 Eterm res = THE_NON_VALUE;
2539 int i, length;
2540 Uint reserved_atom_space, atom_space;
2541
2542 if (proc) {
2543 ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN
2544 == erts_proc_lc_my_proc_locks(proc));
2545
2546 /* We'll need locks early in the lock order */
2547 erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
2548 }
2549
2550 i = 0;
2551
2552 values[i].arity = 2;
2553 values[i].name = "sys_misc";
2554 values[i].ui[0] = erts_sys_misc_mem_sz();
2555 i++;
2556
2557 values[i].arity = 2;
2558 values[i].name = "static";
2559 values[i].ui[0] =
2560 sizeof(ErtsPTab)*2 /* proc & port tables */
2561 + erts_timer_wheel_memory_size(); /* Timer wheel */
2562 i++;
2563
2564 erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space);
2565
2566 values[i].arity = 3;
2567 values[i].name = "atom_space";
2568 values[i].ui[0] = reserved_atom_space;
2569 values[i].ui[1] = atom_space;
2570 i++;
2571
2572 values[i].arity = 2;
2573 values[i].name = "atom_table";
2574 values[i].ui[0] = atom_table_sz();
2575 i++;
2576
2577 values[i].arity = 2;
2578 values[i].name = "module_table";
2579 values[i].ui[0] = module_table_sz();
2580 i++;
2581
2582 values[i].arity = 2;
2583 values[i].name = "export_table";
2584 values[i].ui[0] = export_table_sz();
2585 i++;
2586
2587 values[i].arity = 2;
2588 values[i].name = "export_list";
2589 values[i].ui[0] = export_entries_sz();
2590 i++;
2591
2592 values[i].arity = 2;
2593 values[i].name = "register_table";
2594 values[i].ui[0] = process_reg_sz();
2595 i++;
2596
2597 values[i].arity = 2;
2598 values[i].name = "fun_table";
2599 values[i].ui[0] = erts_fun_table_sz();
2600 i++;
2601
2602 values[i].arity = 2;
2603 values[i].name = "module_refs";
2604 values[i].ui[0] = erts_ranges_sz();
2605 i++;
2606
2607 values[i].arity = 2;
2608 values[i].name = "loaded_code";
2609 values[i].ui[0] = erts_total_code_size;
2610 i++;
2611
2612 values[i].arity = 2;
2613 values[i].name = "dist_table";
2614 values[i].ui[0] = erts_dist_table_size();
2615 i++;
2616
2617 values[i].arity = 2;
2618 values[i].name = "node_table";
2619 values[i].ui[0] = erts_node_table_size();
2620 i++;
2621
2622 values[i].arity = 2;
2623 values[i].name = "bits_bufs_size";
2624 values[i].ui[0] = erts_bits_bufs_size();
2625 i++;
2626
2627 values[i].arity = 2;
2628 values[i].name = "bif_timer";
2629 values[i].ui[0] = erts_bif_timer_memory_size();
2630 i++;
2631
2632 values[i].arity = 2;
2633 values[i].name = "process_table";
2634 values[i].ui[0] = erts_ptab_mem_size(&erts_proc);
2635 i++;
2636
2637 values[i].arity = 2;
2638 values[i].name = "port_table";
2639 values[i].ui[0] = erts_ptab_mem_size(&erts_port);
2640 i++;
2641
2642 values[i].arity = 2;
2643 values[i].name = "ets_misc";
2644 values[i].ui[0] = erts_get_ets_misc_mem_size();
2645 i++;
2646
2647 length = i;
2648 ASSERT(length <= MAX_AA_VALUES);
2649
2650 if (print_to_p) {
2651 /* Print result... */
2652 fmtfn_t to = *print_to_p;
2653 void *arg = print_to_arg;
2654
2655 erts_print(to, arg, "=allocated_areas\n");
2656 for (i = 0; i < length; i++) {
2657 switch (values[i].arity) {
2658 case 2:
2659 erts_print(to, arg, "%s: %beu\n",
2660 values[i].name, values[i].ui[0]);
2661 break;
2662 case 3:
2663 erts_print(to, arg, "%s: %beu %beu\n",
2664 values[i].name, values[i].ui[0], values[i].ui[1]);
2665 break;
2666 default:
2667 erts_print(to, arg, "ERROR: internal_error\n");
2668 ASSERT(0);
2669 return am_internal_error;
2670 }
2671 }
2672 }
2673
2674 if (proc) {
2675 /* Build erlang term result... */
2676 Eterm tuples[MAX_AA_VALUES];
2677 Uint *hp;
2678 Uint **hpp;
2679 Uint hsz;
2680 Uint *hszp;
2681
2682 erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
2683
2684 hpp = NULL;
2685 hsz = 0;
2686 hszp = &hsz;
2687
2688 while (1) {
2689 int i;
2690 for (i = 0; i < length; i++) {
2691 Eterm atom;
2692 if (hpp)
2693 atom = am_atom_put(values[i].name,
2694 (int) sys_strlen(values[i].name));
2695 else
2696 atom = am_true;
2697
2698 switch (values[i].arity) {
2699 case 2:
2700 tuples[i] = erts_bld_tuple(hpp, hszp, 2,
2701 atom,
2702 erts_bld_uint(hpp, hszp,
2703 values[i].ui[0]));
2704 break;
2705 case 3:
2706 tuples[i] = erts_bld_tuple(hpp, hszp, 3,
2707 atom,
2708 erts_bld_uint(hpp, hszp,
2709 values[i].ui[0]),
2710 erts_bld_uint(hpp, hszp,
2711 values[i].ui[1]));
2712 break;
2713 default:
2714 ASSERT(0);
2715 return am_internal_error;
2716 }
2717 }
2718 res = erts_bld_list(hpp, hszp, length, tuples);
2719 if (hpp)
2720 break;
2721 hp = HAlloc((Process *) proc, hsz);
2722 hpp = &hp;
2723 hszp = NULL;
2724 }
2725 }
2726
2727 return res;
2728 #undef MAX_AA_VALUES
2729 }
2730
2731 Eterm
erts_alloc_util_allocators(void * proc)2732 erts_alloc_util_allocators(void *proc)
2733 {
2734 Eterm res;
2735 Uint *hp;
2736 Uint sz;
2737 int i;
2738 /*
2739 * Currently all allocators except sys_alloc are
2740 * alloc_util allocators.
2741 * Also hide test_alloc which is disabled by default
2742 * and only intended for our own testing.
2743 */
2744 sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2;
2745 ASSERT(sz > 0);
2746 hp = HAlloc((Process *) proc, sz);
2747 res = NIL;
2748 for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) {
2749 switch (i) {
2750 case ERTS_ALC_A_SYSTEM:
2751 case ERTS_ALC_A_TEST:
2752 break;
2753 default: {
2754 char *alc_str = (char *) ERTS_ALC_A2AD(i);
2755 Eterm alc = am_atom_put(alc_str, sys_strlen(alc_str));
2756 res = CONS(hp, alc, res);
2757 hp += 2;
2758 break;
2759 }
2760 }
2761 }
2762 return res;
2763 }
2764
2765 void
erts_allocator_info(fmtfn_t to,void * arg)2766 erts_allocator_info(fmtfn_t to, void *arg)
2767 {
2768 ErtsAlcType_t a;
2769
2770 ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
2771
2772 for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
2773 int ai;
2774 for (ai = 0; ai == 0 || ai < erts_allctrs_info[a].thr_spec; ai++) {
2775 if (erts_allctrs_info[a].thr_spec) {
2776 if (!erts_allctr_thr_spec[a].allctr[ai])
2777 continue;
2778 erts_print(to, arg, "=allocator:%s[%d]\n",
2779 ERTS_ALC_A2AD(a), ai);
2780 }
2781 else {
2782 erts_print(to, arg, "=allocator:%s\n", ERTS_ALC_A2AD(a));
2783 }
2784 if (!erts_allctrs_info[a].enabled)
2785 erts_print(to, arg, "option e: false\n");
2786 else {
2787 if (erts_allctrs_info[a].alloc_util) {
2788 void *as;
2789 if (!erts_allctrs_info[a].thr_spec)
2790 as = erts_allctrs_info[a].extra;
2791 else {
2792 ASSERT(erts_allctr_thr_spec[a].enabled);
2793 as = erts_allctr_thr_spec[a].allctr[ai];
2794 }
2795 /* Binary alloc has its own thread safety... */
2796 erts_alcu_info(as, 0, 0, &to, arg, NULL, NULL);
2797 }
2798 else {
2799 switch (a) {
2800 case ERTS_ALC_A_SYSTEM: {
2801 SysAllocStat sas;
2802 erts_print(to, arg, "option e: true\n");
2803 erts_print(to, arg, "option m: libc\n");
2804 sys_alloc_stat(&sas);
2805 if(sas.trim_threshold >= 0)
2806 erts_print(to, arg, "option tt: %d\n", sas.trim_threshold);
2807 if(sas.top_pad >= 0)
2808 erts_print(to, arg, "option tp: %d\n", sas.top_pad);
2809 break;
2810 }
2811 default:
2812 ASSERT(0);
2813 break;
2814 }
2815 }
2816 }
2817 }
2818 }
2819
2820 #if HAVE_ERTS_MSEG
2821 {
2822 struct erts_mmap_info_struct emis;
2823 int max = (int) erts_no_schedulers;
2824 int i;
2825 for (i = 0; i <= max; i++) {
2826 erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
2827 erts_mseg_info(i, &to, arg, 0, 0, NULL, NULL);
2828 }
2829 erts_print(to, arg, "=allocator:erts_mmap.default_mmap\n");
2830 erts_mmap_info(&erts_dflt_mmapper, &to, arg, NULL, NULL, &emis);
2831 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
2832 erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n");
2833 erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis);
2834 #endif
2835 }
2836 #endif
2837
2838 erts_print(to, arg, "=allocator:alloc_util\n");
2839 erts_alcu_au_info_options(&to, arg, NULL, NULL);
2840
2841 erts_print(to, arg, "=allocator:instr\n");
2842
2843 erts_print(to, arg, "option t: %s\n",
2844 erts_mtrace_enabled ? "true" : "false");
2845
2846 }
2847
2848 Eterm
erts_allocator_options(void * proc)2849 erts_allocator_options(void *proc)
2850 {
2851 #if HAVE_ERTS_MSEG
2852 int use_mseg = 0;
2853 #endif
2854 Uint sz, *szp, *hp, **hpp;
2855 Eterm res, features, settings;
2856 Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7];
2857 Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7];
2858 int a, length;
2859 SysAllocStat sas;
2860 Uint *endp = NULL;
2861
2862 sys_alloc_stat(&sas);
2863
2864 /* First find out the heap size needed ... */
2865 hpp = NULL;
2866 szp = &sz;
2867 sz = 0;
2868
2869 bld_term:
2870
2871 length = 0;
2872 features = NIL;
2873 settings = NIL;
2874
2875 for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
2876 Eterm tmp = NIL;
2877 atoms[length] = am_atom_put((char *) ERTS_ALC_A2AD(a),
2878 sys_strlen(ERTS_ALC_A2AD(a)));
2879 if (erts_allctrs_info[a].enabled) {
2880 if (erts_allctrs_info[a].alloc_util) {
2881 Allctr_t *allctr;
2882 #if HAVE_ERTS_MSEG
2883 use_mseg++;
2884 #endif
2885 if (erts_allctr_thr_spec[a].enabled)
2886 allctr = erts_allctr_thr_spec[a].allctr[0];
2887 else
2888 allctr = erts_allctrs_info[a].extra;
2889 tmp = erts_alcu_info_options(allctr, NULL, NULL, hpp, szp);
2890 }
2891 else {
2892 int l = 0;
2893 Eterm as[4];
2894 Eterm ts[4];
2895
2896 as[l] = ERTS_MAKE_AM("e");
2897 ts[l++] = am_true;
2898
2899 switch (a) {
2900 case ERTS_ALC_A_SYSTEM:
2901 as[l] = ERTS_MAKE_AM("m");
2902 ts[l++] = ERTS_MAKE_AM("libc");
2903 if(sas.trim_threshold >= 0) {
2904 as[l] = ERTS_MAKE_AM("tt");
2905 ts[l++] = erts_bld_uint(hpp, szp,
2906 (Uint) sas.trim_threshold);
2907 }
2908 if(sas.top_pad >= 0) {
2909 as[l] = ERTS_MAKE_AM("tp");
2910 ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad);
2911 }
2912 break;
2913 default:
2914 break;
2915 }
2916
2917 tmp = erts_bld_2tup_list(hpp, szp, l, as, ts);
2918
2919 }
2920
2921 }
2922 else {
2923 Eterm atom = ERTS_MAKE_AM("e");
2924 Eterm term = am_false;
2925 tmp = erts_bld_2tup_list(hpp, szp, 1, &atom, &term);
2926 }
2927
2928 terms[length++] = tmp;
2929
2930 }
2931
2932 #if HAVE_ERTS_MSEG
2933 if (use_mseg) {
2934 atoms[length] = ERTS_MAKE_AM("mseg_alloc");
2935 terms[length++] = erts_mseg_info_options(0, NULL, NULL, hpp, szp);
2936 }
2937 #endif
2938
2939 atoms[length] = ERTS_MAKE_AM("alloc_util");
2940 terms[length++] = erts_alcu_au_info_options(NULL, NULL, hpp, szp);
2941
2942 #if HAVE_ERTS_MMAP
2943 atoms[length] = ERTS_MAKE_AM("erts_mmap");
2944 terms[length++] = erts_mmap_info_options(&erts_dflt_mmapper, NULL, NULL,
2945 NULL, hpp, szp);
2946 #endif
2947 {
2948 Eterm o[1], v[1];
2949 o[0] = ERTS_MAKE_AM("t");
2950 v[0] = erts_mtrace_enabled ? am_true : am_false;
2951
2952 atoms[length] = ERTS_MAKE_AM("instr");
2953 terms[length++] = erts_bld_2tup_list(hpp, szp, 1, o, v);
2954 }
2955
2956 atoms[length] = ERTS_MAKE_AM("lock_physical_memory");
2957 terms[length++] = (lock_all_physical_memory ? am_all : am_no);
2958
2959 settings = erts_bld_2tup_list(hpp, szp, length, atoms, terms);
2960
2961 length = 0;
2962
2963 for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
2964 if (erts_allctrs_info[a].enabled) {
2965 terms[length++] = am_atom_put((char *) ERTS_ALC_A2AD(a),
2966 sys_strlen(ERTS_ALC_A2AD(a)));
2967 }
2968 }
2969
2970 #if HAVE_ERTS_MSEG
2971 if (use_mseg)
2972 terms[length++] = ERTS_MAKE_AM("mseg_alloc");
2973 #endif
2974 #if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
2975 terms[length++] = ERTS_MAKE_AM("sys_aligned_alloc");
2976 #endif
2977 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
2978 terms[length++] = ERTS_MAKE_AM("literal_mmap");
2979 #endif
2980 features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
2981
2982 #if defined(__GLIBC__)
2983 {
2984 Eterm AM_glibc = ERTS_MAKE_AM("glibc");
2985 Eterm version;
2986
2987 version = erts_bld_cons(hpp,
2988 szp,
2989 make_small(__GLIBC__),
2990 #ifdef __GLIBC_MINOR__
2991 erts_bld_cons(hpp,
2992 szp,
2993 make_small(__GLIBC_MINOR__),
2994 NIL)
2995 #else
2996 NIL
2997 #endif
2998 );
2999
3000 res = erts_bld_tuple(hpp, szp, 4,
3001 AM_glibc, version, features, settings);
3002 }
3003
3004 #else /* unknown allocator */
3005
3006 res = erts_bld_tuple(hpp, szp, 4,
3007 am_undefined, NIL, features, settings);
3008
3009 #endif
3010
3011 if (szp) {
3012 /* ... and then build the term */
3013 hp = HAlloc((Process *) proc, sz);
3014 endp = hp + sz;
3015 hpp = &hp;
3016 szp = NULL;
3017 goto bld_term;
3018 }
3019
3020 ASSERT(endp >= hp);
3021 HRelease((Process *) proc, endp, hp);
3022
3023 return res;
3024 }
3025
erts_alloc_permanent_cache_aligned(ErtsAlcType_t type,Uint size)3026 void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size)
3027 {
3028 UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1)
3029 #ifdef VALGRIND
3030 + sizeof(UWord)
3031 #endif
3032 );
3033
3034 #ifdef VALGRIND
3035 { /* Link them to avoid Leak_PossiblyLost */
3036 static UWord* first_in_list = NULL;
3037 *(UWord**)v = first_in_list;
3038 first_in_list = (UWord*) v;
3039 v += sizeof(UWord);
3040 }
3041 #endif
3042
3043 if (v & ERTS_CACHE_LINE_MASK) {
3044 v = (v & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
3045 }
3046 ASSERT((v & ERTS_CACHE_LINE_MASK) == 0);
3047 return (void*)v;
3048 }
3049
3050 static void
reply_alloc_info(void * vair)3051 reply_alloc_info(void *vair)
3052 {
3053 ErtsAllocInfoReq *air = (ErtsAllocInfoReq *) vair;
3054 Uint sched_id = erts_get_scheduler_id();
3055 int global_instances = air->req_sched == sched_id;
3056 ErtsProcLocks rp_locks;
3057 Process *rp = air->proc;
3058 Eterm ref_copy = NIL, ai_list, msg = NIL;
3059 Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL;
3060 Eterm **hpp;
3061 Uint sz, *szp;
3062 ErlOffHeap *ohp = NULL;
3063 ErtsMessage *mp = NULL;
3064 #if HAVE_ERTS_MMAP
3065 struct erts_mmap_info_struct mmap_info_dflt;
3066 # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
3067 struct erts_mmap_info_struct mmap_info_literal;
3068 # endif
3069 #endif
3070 int i;
3071 Eterm (*info_func)(Allctr_t *,
3072 int,
3073 int,
3074 fmtfn_t *,
3075 void *,
3076 Uint **,
3077 Uint *) = (air->only_sz
3078 ? erts_alcu_sz_info
3079 : erts_alcu_info);
3080
3081 rp_locks = air->req_sched == sched_id ? ERTS_PROC_LOCK_MAIN : 0;
3082
3083 sz = 0;
3084 hpp = NULL;
3085 szp = &sz;
3086
3087 while (1) {
3088
3089 if (hpp)
3090 ref_copy = erts_iref_storage_make_ref(&air->iref,
3091 hpp, ohp, 0);
3092 else
3093 *szp += erts_iref_storage_heap_size(&air->iref);
3094
3095 ai_list = NIL;
3096 for (i = 0; air->allocs[i] != ERTS_ALC_A_INVALID; i++);
3097 for (i--; i >= 0; i--) {
3098 int ai = air->allocs[i];
3099 Allctr_t *allctr;
3100 Eterm ainfo;
3101 Eterm alloc_atom;
3102 if (global_instances) {
3103 switch (ai) {
3104 case ERTS_ALC_A_SYSTEM: {
3105 alloc_atom = erts_bld_atom(hpp, szp, "sys_alloc");
3106 ainfo = NIL;
3107 if (!air->only_sz) {
3108 SysAllocStat sas;
3109 if (hpp)
3110 sys_alloc_stat(&sas);
3111 if (szp) {
3112 /* ensure ehough heap */
3113 sas.top_pad = INT_MAX;
3114 sas.trim_threshold = INT_MAX;
3115 }
3116 if (sas.top_pad >= 0) {
3117 ainfo = erts_bld_cons(
3118 hpp, szp,
3119 erts_bld_tuple(
3120 hpp, szp, 2,
3121 erts_bld_atom(hpp, szp, "tp"),
3122 erts_bld_uint(
3123 hpp, szp,
3124 (Uint) sas.top_pad)),
3125 ainfo);
3126 }
3127 if (sas.trim_threshold >= 0) {
3128 ainfo = erts_bld_cons(
3129 hpp, szp,
3130 erts_bld_tuple(
3131 hpp, szp, 2,
3132 erts_bld_atom(hpp, szp, "tt"),
3133 erts_bld_uint(
3134 hpp, szp,
3135 (Uint) sas.trim_threshold)),
3136 ainfo);
3137 }
3138 ainfo = erts_bld_cons(hpp, szp,
3139 erts_bld_tuple(
3140 hpp, szp, 2,
3141 erts_bld_atom(hpp, szp,
3142 "m"),
3143 erts_bld_atom(hpp, szp,
3144 "libc")),
3145 ainfo);
3146 ainfo = erts_bld_cons(hpp, szp,
3147 erts_bld_tuple(
3148 hpp, szp, 2,
3149 erts_bld_atom(hpp, szp,
3150 "e"),
3151 am_true),
3152 ainfo);
3153 ainfo = erts_bld_tuple(hpp, szp, 2,
3154 erts_bld_atom(hpp, szp,
3155 "options"),
3156 ainfo);
3157 ainfo = erts_bld_cons(hpp, szp,ainfo,NIL);
3158 }
3159 ainfo = erts_bld_tuple(hpp, szp, 3,
3160 alloc_atom,
3161 make_small(0),
3162 ainfo);
3163 break;
3164 }
3165 case ERTS_ALC_INFO_A_ALLOC_UTIL:
3166 alloc_atom = erts_bld_atom(hpp, szp, "alloc_util");
3167 ainfo = (air->only_sz
3168 ? NIL
3169 : erts_alcu_au_info_options(NULL, NULL,
3170 hpp, szp));
3171 ainfo = erts_bld_tuple(hpp, szp, 3,
3172 alloc_atom,
3173 make_small(0),
3174 ainfo);
3175 break;
3176 case ERTS_ALC_INFO_A_ERTS_MMAP:
3177 alloc_atom = erts_bld_atom(hpp, szp, "erts_mmap");
3178 #if HAVE_ERTS_MMAP
3179 ainfo = (air->only_sz ? NIL :
3180 erts_mmap_info(&erts_dflt_mmapper, NULL, NULL,
3181 hpp, szp, &mmap_info_dflt));
3182 ainfo = erts_bld_tuple3(hpp, szp,
3183 alloc_atom,
3184 erts_bld_atom(hpp,szp,"default_mmap"),
3185 ainfo);
3186 # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
3187 ai_list = erts_bld_cons(hpp, szp,
3188 ainfo, ai_list);
3189 ainfo = (air->only_sz ? NIL :
3190 erts_mmap_info(&erts_literal_mmapper, NULL, NULL,
3191 hpp, szp, &mmap_info_literal));
3192 ainfo = erts_bld_tuple3(hpp, szp,
3193 alloc_atom,
3194 erts_bld_atom(hpp,szp,"literal_mmap"),
3195 ainfo);
3196 # endif
3197 #else /* !HAVE_ERTS_MMAP */
3198 ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
3199 am_false);
3200 #endif
3201 break;
3202 case ERTS_ALC_INFO_A_MSEG_ALLOC:
3203 alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
3204 #if HAVE_ERTS_MSEG
3205 ainfo = erts_mseg_info(0, NULL, NULL, hpp != NULL,
3206 air->only_sz, hpp, szp);
3207 ainfo = erts_bld_tuple3(hpp, szp,
3208 alloc_atom,
3209 make_small(0),
3210 ainfo);
3211
3212 #else
3213 ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
3214 am_false);
3215 #endif
3216 break;
3217 #ifndef ERTS_ALC_A_EXEC
3218 case ERTS_ALC_INFO_A_DISABLED_EXEC:
3219 alloc_atom = erts_bld_atom(hpp, szp, "exec_alloc");
3220 ainfo = erts_bld_tuple2(hpp, szp, alloc_atom, am_false);
3221 break;
3222 #endif
3223 default:
3224 alloc_atom = erts_bld_atom(hpp, szp,
3225 (char *) ERTS_ALC_A2AD(ai));
3226 if (!erts_allctrs_info[ai].enabled)
3227 ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
3228 am_false);
3229 else if (erts_allctrs_info[ai].alloc_util) {
3230 if (erts_allctrs_info[ai].thr_spec)
3231 allctr = erts_allctr_thr_spec[ai].allctr[0];
3232 else
3233 allctr = erts_allctrs_info[ai].extra;
3234 ainfo = info_func(allctr, air->internal, hpp != NULL,
3235 NULL, NULL, hpp, szp);
3236 ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom,
3237 make_small(0), ainfo);
3238 }
3239 else {
3240 erts_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n",
3241 __FILE__, __LINE__);
3242 }
3243 }
3244 ai_list = erts_bld_cons(hpp, szp,
3245 ainfo, ai_list);
3246 }
3247 switch (ai) {
3248 case ERTS_ALC_A_SYSTEM:
3249 case ERTS_ALC_INFO_A_ALLOC_UTIL:
3250 case ERTS_ALC_INFO_A_ERTS_MMAP:
3251 case ERTS_ALC_INFO_A_DISABLED_EXEC:
3252 break;
3253 case ERTS_ALC_INFO_A_MSEG_ALLOC:
3254 #if HAVE_ERTS_MSEG
3255 alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
3256 ainfo = erts_mseg_info(sched_id, NULL, NULL,
3257 hpp != NULL, air->only_sz, hpp, szp);
3258 ainfo = erts_bld_tuple(hpp, szp, 3,
3259 alloc_atom,
3260 make_small(sched_id),
3261 ainfo);
3262 ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list);
3263 #endif
3264 break;
3265 default:
3266 if (erts_allctrs_info[ai].thr_spec) {
3267 alloc_atom = erts_bld_atom(hpp, szp,
3268 (char *) ERTS_ALC_A2AD(ai));
3269 allctr = erts_allctr_thr_spec[ai].allctr[sched_id];
3270 ainfo = info_func(allctr, air->internal, hpp != NULL, NULL,
3271 NULL, hpp, szp);
3272 ai_list = erts_bld_cons(hpp, szp,
3273 erts_bld_tuple(
3274 hpp, szp,
3275 3,
3276 alloc_atom,
3277 make_small(sched_id),
3278 ainfo),
3279 ai_list);
3280 }
3281 break;
3282 }
3283 msg = erts_bld_tuple(hpp, szp,
3284 3,
3285 ref_copy,
3286 make_small(sched_id),
3287 ai_list);
3288
3289 }
3290 if (hpp)
3291 break;
3292
3293 mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
3294 hp_start = hp;
3295 hp_end = hp + sz;
3296 szp = NULL;
3297 hpp = &hp;
3298 }
3299
3300 if (hp != hp_end)
3301 erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1);
3302
3303 erts_queue_message(rp, rp_locks, mp, msg, am_system);
3304
3305 if (air->req_sched == sched_id)
3306 rp_locks &= ~ERTS_PROC_LOCK_MAIN;
3307
3308 erts_proc_unlock(rp, rp_locks);
3309 erts_proc_dec_refc(rp);
3310
3311 if (erts_atomic32_dec_read_nob(&air->refc) == 0) {
3312 erts_iref_storage_clean(&air->iref);
3313 aireq_free(air);
3314 }
3315 }
3316
3317 int
erts_request_alloc_info(struct process * c_p,Eterm ref,Eterm allocs,int only_sz,int internal)3318 erts_request_alloc_info(struct process *c_p,
3319 Eterm ref,
3320 Eterm allocs,
3321 int only_sz,
3322 int internal)
3323 {
3324 ErtsAllocInfoReq *air = aireq_alloc();
3325 Eterm req_ai[ERTS_ALC_INFO_A_END] = {0};
3326 Eterm alist;
3327 int airix = 0, ai;
3328
3329 air->req_sched = erts_get_scheduler_id();
3330
3331 air->only_sz = only_sz;
3332
3333 air->internal = internal;
3334
3335 air->proc = c_p;
3336
3337 if (is_not_internal_ref(ref))
3338 return 0;
3339
3340 erts_iref_storage_save(&air->iref, ref);
3341
3342 if (is_not_list(allocs))
3343 return 0;
3344
3345 alist = allocs;
3346
3347 while (is_list(alist)) {
3348 int saved = 0;
3349 Eterm* consp = list_val(alist);
3350 Eterm alloc = CAR(consp);
3351
3352 for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++)
3353 if (erts_is_atom_str(erts_alc_a2ad[ai], alloc, 0))
3354 goto save_alloc;
3355 if (erts_is_atom_str("mseg_alloc", alloc, 0)) {
3356 ai = ERTS_ALC_INFO_A_MSEG_ALLOC;
3357 goto save_alloc;
3358 }
3359 if (erts_is_atom_str("erts_mmap", alloc, 0)) {
3360 ai = ERTS_ALC_INFO_A_ERTS_MMAP;
3361 goto save_alloc;
3362 }
3363 #ifndef ERTS_ALC_A_EXEC
3364 if (erts_is_atom_str("exec_alloc", alloc, 0)) {
3365 ai = ERTS_ALC_INFO_A_DISABLED_EXEC;
3366 goto save_alloc;
3367 }
3368 #endif
3369 if (erts_is_atom_str("alloc_util", alloc, 0)) {
3370 ai = ERTS_ALC_INFO_A_ALLOC_UTIL;
3371 save_alloc:
3372 if (req_ai[ai])
3373 return 0;
3374 air->allocs[airix++] = ai;
3375 req_ai[ai] = 1;
3376 saved = 1;
3377 }
3378
3379 if (!saved)
3380 return 0;
3381
3382 alist = CDR(consp);
3383 }
3384
3385 if (is_not_nil(alist))
3386 return 0;
3387
3388 air->allocs[airix] = ERTS_ALC_A_INVALID;
3389
3390 erts_atomic32_init_nob(&air->refc,
3391 (erts_aint32_t) erts_no_schedulers);
3392
3393 erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
3394
3395 if (erts_no_schedulers > 1)
3396 erts_schedule_multi_misc_aux_work(1,
3397 erts_no_schedulers,
3398 reply_alloc_info,
3399 (void *) air);
3400
3401 reply_alloc_info((void *) air);
3402
3403 return 1;
3404 }
3405
erts_alloc_set_dyn_param(Process * c_p,Eterm tuple)3406 Eterm erts_alloc_set_dyn_param(Process* c_p, Eterm tuple)
3407 {
3408 ErtsAllocatorThrSpec_t *tspec;
3409 ErtsAlcType_t ai;
3410 Allctr_t* allctr;
3411 Eterm* tp;
3412 Eterm res;
3413
3414 if (!is_tuple_arity(tuple, 3))
3415 goto badarg;
3416
3417 tp = tuple_val(tuple);
3418
3419 /*
3420 * Ex: {ets_alloc, sbct, 256000}
3421 */
3422 if (!is_atom(tp[1]) || !is_atom(tp[2]) || !is_integer(tp[3]))
3423 goto badarg;
3424
3425 for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++)
3426 if (erts_is_atom_str(erts_alc_a2ad[ai], tp[1], 0))
3427 break;
3428
3429 if (ai > ERTS_ALC_A_MAX)
3430 goto badarg;
3431
3432 if (!erts_allctrs_info[ai].enabled ||
3433 !erts_allctrs_info[ai].alloc_util) {
3434 return am_notsup;
3435 }
3436
3437 if (tp[2] == am_sbct) {
3438 Uint sbct;
3439 int i, ok;
3440
3441 if (!term_to_Uint(tp[3], &sbct))
3442 goto badarg;
3443
3444 tspec = &erts_allctr_thr_spec[ai];
3445 if (tspec->enabled) {
3446 ok = 0;
3447 for (i = 0; i < tspec->size; i++) {
3448 allctr = tspec->allctr[i];
3449 ok |= allctr->try_set_dyn_param(allctr, am_sbct, sbct);
3450 }
3451 }
3452 else {
3453 allctr = erts_allctrs_info[ai].extra;
3454 ok = allctr->try_set_dyn_param(allctr, am_sbct, sbct);
3455 }
3456 return ok ? am_ok : am_notsup;
3457 }
3458 return am_notsup;
3459
3460 badarg:
3461 ERTS_BIF_PREP_ERROR(res, c_p, EXC_BADARG);
3462 return res;
3463 }
3464
3465 /*
3466 * The allocator wrapper prelocking stuff below is about the locking order.
3467 * It only affects wrappers (erl_mtrace.c) that keep locks during
3468 * alloc/realloc/free.
3469 *
3470 * Some query functions in erl_alloc_util.c lock the allocator mutex and then
3471 * use erts_printf that in turn may call the sys allocator through the wrappers.
3472 * To avoid breaking locking order these query functions first "pre-locks" all
3473 * allocator wrappers.
3474 */
3475 ErtsAllocatorWrapper_t *erts_allctr_wrappers;
3476 int erts_allctr_wrapper_prelocked = 0;
3477 erts_tsd_key_t erts_allctr_prelock_tsd_key;
3478
erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t * wrapper)3479 void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper)
3480 {
3481 ASSERT(wrapper->lock && wrapper->unlock);
3482 wrapper->next = erts_allctr_wrappers;
3483 erts_allctr_wrappers = wrapper;
3484 }
3485
erts_allctr_wrapper_pre_lock(void)3486 void erts_allctr_wrapper_pre_lock(void)
3487 {
3488 if (erts_allctr_wrappers) {
3489 ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers;
3490 for ( ; wrapper; wrapper = wrapper->next) {
3491 wrapper->lock();
3492 }
3493 ASSERT(!erts_allctr_wrapper_prelocked);
3494 erts_allctr_wrapper_prelocked = 1;
3495 erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)1);
3496 }
3497 }
3498
erts_allctr_wrapper_pre_unlock(void)3499 void erts_allctr_wrapper_pre_unlock(void)
3500 {
3501 if (erts_allctr_wrappers) {
3502 ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers;
3503
3504 erts_allctr_wrapper_prelocked = 0;
3505 erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)0);
3506 for ( ; wrapper; wrapper = wrapper->next) {
3507 wrapper->unlock();
3508 }
3509 }
3510 }
3511
3512
3513 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
3514 * NOTE: erts_alc_test() is only supposed to be used for testing. *
3515 * *
3516 * Keep alloc_SUITE_data/allocator_test.h updated if changes are made *
3517 * to erts_alc_test() *
3518 \* */
3519 #define ERTS_ALC_TEST_ABORT erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n")
3520
erts_alc_test(UWord op,UWord a1,UWord a2,UWord a3)3521 UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
3522 {
3523 switch (op >> 8) {
3524 case 0x0: return erts_alcu_test(op, a1, a2);
3525 case 0x1: return erts_gfalc_test(op, a1, a2);
3526 case 0x2: return erts_bfalc_test(op, a1, a2);
3527 case 0x3: return erts_afalc_test(op, a1, a2);
3528 case 0x4: return erts_mseg_test(op, a1, a2, a3);
3529 case 0x5: return erts_aoffalc_test(op, a1, a2);
3530 case 0xf:
3531 switch (op) {
3532 case 0xf00:
3533 if (((Allctr_t *) a1)->thread_safe)
3534 return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_TEST,
3535 (void *) a1,
3536 (Uint) a2);
3537 else
3538 return (UWord) erts_alcu_alloc(ERTS_ALC_T_TEST,
3539 (void *) a1,
3540 (Uint) a2);
3541 case 0xf01:
3542 if (((Allctr_t *) a1)->thread_safe)
3543 return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_TEST,
3544 (void *) a1,
3545 (void *) a2,
3546 (Uint) a3);
3547 else
3548 return (UWord) erts_alcu_realloc(ERTS_ALC_T_TEST,
3549 (void *) a1,
3550 (void *) a2,
3551 (Uint) a3);
3552 case 0xf02:
3553 if (((Allctr_t *) a1)->thread_safe)
3554 erts_alcu_free_ts(ERTS_ALC_T_TEST, (void *) a1, (void *) a2);
3555 else
3556 erts_alcu_free(ERTS_ALC_T_TEST, (void *) a1, (void *) a2);
3557 return 0;
3558 case 0xf03: {
3559 Allctr_t *allctr;
3560 struct au_init init;
3561
3562 SET_DEFAULT_ALLOC_OPTS(&init);
3563 init.enable = 1;
3564 init.astrat = ERTS_ALC_S_GOODFIT;
3565 init.init.util.name_prefix = (char *) a1;
3566 init.init.util.alloc_no = ERTS_ALC_A_TEST;
3567 init.init.util.alloc_strat = init.astrat;
3568 init.init.util.ts = 1;
3569 if ((char **) a3) {
3570 char **argv = (char **) a3;
3571 int i = 0;
3572 while (argv[i]) {
3573 if (argv[i][0] == '-' && argv[i][1] == 't')
3574 handle_au_arg(&init, &argv[i][2], argv, &i, 0);
3575 else
3576 return (UWord) NULL;
3577 i++;
3578 }
3579 }
3580
3581 switch (init.astrat) {
3582 case ERTS_ALC_S_GOODFIT:
3583 allctr = erts_gfalc_start((GFAllctr_t *)
3584 erts_alloc(ERTS_ALC_T_TEST,
3585 sizeof(GFAllctr_t)),
3586 &init.init.gf,
3587 &init.init.util);
3588 break;
3589 case ERTS_ALC_S_BESTFIT:
3590 allctr = erts_bfalc_start((BFAllctr_t *)
3591 erts_alloc(ERTS_ALC_T_TEST,
3592 sizeof(BFAllctr_t)),
3593 &init.init.bf,
3594 &init.init.util);
3595 break;
3596 case ERTS_ALC_S_AFIT:
3597 allctr = erts_afalc_start((AFAllctr_t *)
3598 erts_alloc(ERTS_ALC_T_TEST,
3599 sizeof(AFAllctr_t)),
3600 &init.init.af,
3601 &init.init.util);
3602 break;
3603 case ERTS_ALC_S_FIRSTFIT:
3604 allctr = erts_aoffalc_start((AOFFAllctr_t *)
3605 erts_alloc(ERTS_ALC_T_TEST,
3606 sizeof(AOFFAllctr_t)),
3607 &init.init.aoff,
3608 &init.init.util);
3609 break;
3610
3611 default:
3612 ASSERT(0);
3613 allctr = NULL;
3614 break;
3615 }
3616
3617 return (UWord) allctr;
3618 }
3619 case 0xf04:
3620 erts_alcu_stop((Allctr_t *) a1);
3621 erts_free(ERTS_ALC_T_TEST, (void *) a1);
3622 break;
3623 case 0xf05: return (UWord) 1;
3624 case 0xf06: return (UWord) ((Allctr_t *) a1)->thread_safe;
3625 #ifdef ETHR_NO_FORKSAFETY
3626 case 0xf07: return (UWord) 0;
3627 #else
3628 case 0xf07: return (UWord) ((Allctr_t *) a1)->thread_safe;
3629 #endif
3630 case 0xf08: {
3631 ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_mutex));
3632 if (ethr_mutex_init(mtx) != 0)
3633 ERTS_ALC_TEST_ABORT;
3634 return (UWord) mtx;
3635 }
3636 case 0xf09: {
3637 ethr_mutex *mtx = (ethr_mutex *) a1;
3638 if (ethr_mutex_destroy(mtx) != 0)
3639 ERTS_ALC_TEST_ABORT;
3640 erts_free(ERTS_ALC_T_TEST, (void *) mtx);
3641 break;
3642 }
3643 case 0xf0a:
3644 ethr_mutex_lock((ethr_mutex *) a1);
3645 break;
3646 case 0xf0b:
3647 ethr_mutex_unlock((ethr_mutex *) a1);
3648 break;
3649 case 0xf0c: {
3650 ethr_cond *cnd = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_cond));
3651 if (ethr_cond_init(cnd) != 0)
3652 ERTS_ALC_TEST_ABORT;
3653 return (UWord) cnd;
3654 }
3655 case 0xf0d: {
3656 ethr_cond *cnd = (ethr_cond *) a1;
3657 if (ethr_cond_destroy(cnd) != 0)
3658 ERTS_ALC_TEST_ABORT;
3659 erts_free(ERTS_ALC_T_TEST, (void *) cnd);
3660 break;
3661 }
3662 case 0xf0e:
3663 ethr_cond_broadcast((ethr_cond *) a1);
3664 break;
3665 case 0xf0f: {
3666 int res;
3667 do {
3668 res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2);
3669 } while (res == EINTR);
3670 break;
3671 }
3672 case 0xf10: {
3673 ethr_tid *tid = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_tid));
3674 if (ethr_thr_create(tid,
3675 (void * (*)(void *)) a1,
3676 (void *) a2,
3677 NULL) != 0)
3678 ERTS_ALC_TEST_ABORT;
3679 return (UWord) tid;
3680 }
3681 case 0xf11: {
3682 ethr_tid *tid = (ethr_tid *) a1;
3683 if (ethr_thr_join(*tid, NULL) != 0)
3684 ERTS_ALC_TEST_ABORT;
3685 erts_free(ERTS_ALC_T_TEST, (void *) tid);
3686 break;
3687 }
3688 case 0xf12:
3689 ethr_thr_exit((void *) a1);
3690 ERTS_ALC_TEST_ABORT;
3691 break;
3692 case 0xf13: return (UWord) 1;
3693 case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1);
3694
3695 case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0;
3696
3697 case 0xf16: return (UWord) erts_realloc(ERTS_ALC_T_TEST, (void*)a1, (Uint)a2);
3698
3699 case 0xf17: {
3700 Uint extra_hdr_sz = UNIT_CEILING((Uint)a1);
3701 ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST];
3702 Uint offset = ts->allctr[0]->mbc_header_size;
3703 void* orig_creating_mbc = ts->allctr[0]->creating_mbc;
3704 void* orig_destroying_mbc = ts->allctr[0]->destroying_mbc;
3705 void* new_creating_mbc = *(void**)a2; /* inout arg */
3706 void* new_destroying_mbc = *(void**)a3; /* inout arg */
3707 int i;
3708
3709 for (i=0; i < ts->size; i++) {
3710 Allctr_t* ap = ts->allctr[i];
3711 if (ap->mbc_header_size != offset
3712 || ap->creating_mbc != orig_creating_mbc
3713 || ap->destroying_mbc != orig_destroying_mbc
3714 || ap->mbc_list.first != NULL)
3715 return -1;
3716 }
3717 for (i=0; i < ts->size; i++) {
3718 ts->allctr[i]->mbc_header_size += extra_hdr_sz;
3719 ts->allctr[i]->creating_mbc = new_creating_mbc;
3720 ts->allctr[i]->destroying_mbc = new_destroying_mbc;
3721 }
3722 *(void**)a2 = orig_creating_mbc;
3723 *(void**)a3 = orig_destroying_mbc;
3724 return offset;
3725 }
3726 case 0xf18: {
3727 ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST];
3728 return ts->allctr[0]->largest_mbc_size;
3729 }
3730 default:
3731 break;
3732 }
3733 return (UWord) 0;
3734 default:
3735 break;
3736 }
3737
3738 ASSERT(0);
3739 return ~((UWord) 0);
3740 }
3741
3742 #ifdef DEBUG
3743 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
3744 * Debug stuff *
3745 \* */
3746
3747 #if 0
3748 #define PRINT_OPS
3749 #else
3750 #undef PRINT_OPS
3751 #endif
3752
3753 #ifdef HARD_DEBUG
3754 #define FENCE_SZ (4*sizeof(UWord))
3755 #else
3756 #define FENCE_SZ (3*sizeof(UWord))
3757 #endif
3758
3759 #if defined(ARCH_64)
3760 #define FENCE_PATTERN 0xABCDEF97ABCDEF97
3761 #else
3762 #define FENCE_PATTERN 0xABCDEF97
3763 #endif
3764
3765 #define TYPE_PATTERN_MASK ERTS_ALC_N_MASK
3766 #define TYPE_PATTERN_SHIFT 16
3767
3768 #define FIXED_FENCE_PATTERN_MASK \
3769 (~((UWord) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT)))
3770 #define FIXED_FENCE_PATTERN \
3771 (FENCE_PATTERN & FIXED_FENCE_PATTERN_MASK)
3772
3773 #define MK_PATTERN(T) \
3774 (FIXED_FENCE_PATTERN | (((T) & TYPE_PATTERN_MASK) << TYPE_PATTERN_SHIFT))
3775
3776 #define GET_TYPE_OF_PATTERN(P) \
3777 (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK)
3778
3779 #ifdef HARD_DEBUG
3780
3781 #define ERL_ALC_HDBG_MAX_MBLK 100000
3782 #define ERTS_ALC_O_CHECK -1
3783
3784 typedef struct hdbg_mblk_ hdbg_mblk;
3785 struct hdbg_mblk_ {
3786 hdbg_mblk *next;
3787 hdbg_mblk *prev;
3788 void *p;
3789 Uint s;
3790 ErtsAlcType_t n;
3791 };
3792
3793 static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK];
3794
3795 static hdbg_mblk *free_hdbg_mblks;
3796 static hdbg_mblk *used_hdbg_mblks;
3797 static erts_mtx_t hdbg_mblk_mtx;
3798
3799 static void
hdbg_init(void)3800 hdbg_init(void)
3801 {
3802 int i;
3803 for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++)
3804 hdbg_mblks[i].next = &hdbg_mblks[i+1];
3805 hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL;
3806 free_hdbg_mblks = &hdbg_mblks[0];
3807 used_hdbg_mblks = NULL;
3808 erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug", NIL,
3809 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR);
3810 }
3811
3812 static void *check_memory_fence(void *ptr,
3813 Uint *size,
3814 ErtsAlcType_t n,
3815 int func);
3816 void erts_hdbg_chk_blks(void);
3817
3818 void
erts_hdbg_chk_blks(void)3819 erts_hdbg_chk_blks(void)
3820 {
3821 hdbg_mblk *mblk;
3822
3823 erts_mtx_lock(&hdbg_mblk_mtx);
3824 for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) {
3825 Uint sz;
3826 check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK);
3827 ASSERT(sz == mblk->s);
3828 }
3829 erts_mtx_unlock(&hdbg_mblk_mtx);
3830 }
3831
3832 static hdbg_mblk *
hdbg_alloc(void * p,Uint s,ErtsAlcType_t n)3833 hdbg_alloc(void *p, Uint s, ErtsAlcType_t n)
3834 {
3835 hdbg_mblk *mblk;
3836
3837 erts_mtx_lock(&hdbg_mblk_mtx);
3838 mblk = free_hdbg_mblks;
3839 if (!mblk) {
3840 erts_fprintf(stderr,
3841 "Ran out of debug blocks; please increase "
3842 "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n",
3843 ERL_ALC_HDBG_MAX_MBLK);
3844 abort();
3845 }
3846 free_hdbg_mblks = mblk->next;
3847
3848 mblk->p = p;
3849 mblk->s = s;
3850 mblk->n = n;
3851
3852 mblk->next = used_hdbg_mblks;
3853 mblk->prev = NULL;
3854 if (used_hdbg_mblks)
3855 used_hdbg_mblks->prev = mblk;
3856 used_hdbg_mblks = mblk;
3857 erts_mtx_unlock(&hdbg_mblk_mtx);
3858 return (void *) mblk;
3859 }
3860
3861 static void
hdbg_free(hdbg_mblk * mblk)3862 hdbg_free(hdbg_mblk *mblk)
3863 {
3864 erts_mtx_lock(&hdbg_mblk_mtx);
3865 if (mblk->next)
3866 mblk->next->prev = mblk->prev;
3867 if (mblk->prev)
3868 mblk->prev->next = mblk->next;
3869 else
3870 used_hdbg_mblks = mblk->next;
3871
3872 mblk->next = free_hdbg_mblks;
3873 free_hdbg_mblks = mblk;
3874 erts_mtx_unlock(&hdbg_mblk_mtx);
3875 }
3876
3877 #endif
3878
3879 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
3880 static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func);
3881
check_allocated_block(Uint type,void * blk)3882 void check_allocated_block( Uint type, void *blk)
3883 {
3884 Uint dummy;
3885 check_memory_fence(blk, &dummy, ERTS_ALC_T2N(type), ERTS_ALC_O_FREE);
3886 }
3887
check_allocators(void)3888 void check_allocators(void)
3889 {
3890 int i;
3891 if (!erts_initialized)
3892 return;
3893 for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; ++i) {
3894 if (erts_allctrs_info[i].alloc_util) {
3895 ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra;
3896 Allctr_t *allctr = real_af->extra;
3897 Carrier_t *ct;
3898 if (allctr->thread_safe)
3899 erts_mtx_lock(&allctr->mutex);
3900
3901 if (allctr->check_mbc) {
3902 for (ct = allctr->mbc_list.first; ct; ct = ct->next) {
3903 fprintf(stderr,"Checking allocator %d\r\n",i);
3904 allctr->check_mbc(allctr,ct);
3905 }
3906 }
3907 if (allctr->thread_safe)
3908 erts_mtx_unlock(&allctr->mutex);
3909 }
3910 }
3911 }
3912 #endif
3913
3914 static void *
set_memory_fence(void * ptr,Uint sz,ErtsAlcType_t n)3915 set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n)
3916 {
3917 UWord *ui_ptr;
3918 UWord pattern;
3919 #ifdef HARD_DEBUG
3920 hdbg_mblk **mblkpp;
3921 #endif
3922
3923 if (!ptr)
3924 return NULL;
3925
3926 ui_ptr = (UWord *) ptr;
3927 pattern = MK_PATTERN(n);
3928
3929 #ifdef HARD_DEBUG
3930 mblkpp = (hdbg_mblk **) ui_ptr++;
3931 #endif
3932
3933 *(ui_ptr++) = sz;
3934 *(ui_ptr++) = pattern;
3935 sys_memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord));
3936
3937 #ifdef HARD_DEBUG
3938 *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n);
3939 #endif
3940
3941 return (void *) ui_ptr;
3942 }
3943
3944 static void *
check_memory_fence(void * ptr,Uint * size,ErtsAlcType_t n,int func)3945 check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
3946 {
3947 Uint sz;
3948 Uint found_type;
3949 UWord pre_pattern, expected_pattern;
3950 UWord post_pattern;
3951 UWord *ui_ptr;
3952 #ifdef HARD_DEBUG
3953 hdbg_mblk *mblk;
3954 #endif
3955
3956 if (!ptr)
3957 return NULL;
3958
3959 expected_pattern = MK_PATTERN(n);
3960
3961 ui_ptr = (UWord *) ptr;
3962 pre_pattern = *(--ui_ptr);
3963 *size = sz = *(--ui_ptr);
3964 #ifdef HARD_DEBUG
3965 mblk = (hdbg_mblk *) *(--ui_ptr);
3966 #endif
3967
3968 found_type = GET_TYPE_OF_PATTERN(pre_pattern);
3969
3970 if (found_type != n) {
3971 erts_exit(ERTS_ABORT_EXIT, "ERROR: Miss matching allocator types"
3972 " used in alloc and free\n");
3973 }
3974
3975 if (pre_pattern != expected_pattern) {
3976 if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN)
3977 erts_exit(ERTS_ABORT_EXIT,
3978 "ERROR: Fence at beginning of memory block (p=0x%u) "
3979 "clobbered.\n",
3980 (UWord) ptr);
3981 }
3982
3983 sys_memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord));
3984
3985 if (post_pattern != expected_pattern || pre_pattern != post_pattern) {
3986 char fbuf[10];
3987 char obuf[10];
3988 char *ftype;
3989 char *otype;
3990 char *op_str;
3991
3992 if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN)
3993 erts_exit(ERTS_ABORT_EXIT,
3994 "ERROR: Fence at end of memory block (p=0x%u, sz=%u) "
3995 "clobbered.\n",
3996 (UWord) ptr, (UWord) sz);
3997 if (found_type != GET_TYPE_OF_PATTERN(post_pattern))
3998 erts_exit(ERTS_ABORT_EXIT,
3999 "ERROR: Fence around memory block (p=0x%u, sz=%u) "
4000 "clobbered.\n",
4001 (UWord) ptr, (UWord) sz);
4002
4003 ftype = type_no_str(found_type);
4004 if (!ftype) {
4005 erts_snprintf(fbuf, sizeof(fbuf), "%d", (int) found_type);
4006 ftype = fbuf;
4007 }
4008 otype = type_no_str(n);
4009 if (!otype) {
4010 erts_snprintf(obuf, sizeof(obuf), "%d", (int) n);
4011 otype = obuf;
4012 }
4013
4014 switch (func) {
4015 case ERTS_ALC_O_ALLOC: op_str = "allocated"; break;
4016 case ERTS_ALC_O_REALLOC: op_str = "reallocated"; break;
4017 case ERTS_ALC_O_FREE: op_str = "freed"; break;
4018 default: op_str = "???"; break;
4019 }
4020
4021 erts_exit(ERTS_ABORT_EXIT,
4022 "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\","
4023 " but %s as type \"%s\".\n",
4024 (UWord) ptr, (UWord) sz, ftype, op_str, otype);
4025 }
4026
4027 #ifdef HARD_DEBUG
4028 switch (func) {
4029 case ERTS_ALC_O_REALLOC:
4030 case ERTS_ALC_O_FREE:
4031 hdbg_free(mblk);
4032 break;
4033 default:
4034 break;
4035 }
4036 #endif
4037
4038 return (void *) ui_ptr;
4039 }
4040
4041 static ErtsAllocatorFunctions_t real_allctrs[ERTS_ALC_A_MAX+1];
4042
4043 static void *
debug_alloc(ErtsAlcType_t type,void * extra,Uint size)4044 debug_alloc(ErtsAlcType_t type, void *extra, Uint size)
4045 {
4046 ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
4047 ErtsAlcType_t n;
4048 Uint dsize;
4049 void *res;
4050
4051 #ifdef HARD_DEBUG
4052 erts_hdbg_chk_blks();
4053 #endif
4054
4055 n = ERTS_ALC_T2N(type);
4056
4057 ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
4058 dsize = size + FENCE_SZ;
4059 res = (*real_af->alloc)(type, real_af->extra, dsize);
4060
4061 res = set_memory_fence(res, size, n);
4062
4063 #ifdef PRINT_OPS
4064 fprintf(stderr, "0x%lx = alloc(%s, %lu)\r\n",
4065 (Uint) res, ERTS_ALC_N2TD(n), size);
4066 #endif
4067
4068 return res;
4069 }
4070
4071
4072 static void *
debug_realloc(ErtsAlcType_t type,void * extra,void * ptr,Uint size)4073 debug_realloc(ErtsAlcType_t type, void *extra, void *ptr, Uint size)
4074 {
4075 ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
4076 ErtsAlcType_t n;
4077 Uint dsize;
4078 Uint old_size;
4079 void *dptr;
4080 void *res;
4081
4082 n = ERTS_ALC_T2N(type);
4083
4084 ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
4085
4086 dsize = size + FENCE_SZ;
4087 dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC);
4088
4089 #ifdef HARD_DEBUG
4090 erts_hdbg_chk_blks();
4091 #endif
4092
4093 if (ptr && old_size > size)
4094 sys_memset((void *) (((char *) ptr) + size),
4095 0xf,
4096 sizeof(Uint) + old_size - size);
4097
4098 res = (*real_af->realloc)(type, real_af->extra, dptr, dsize);
4099
4100 res = set_memory_fence(res, size, n);
4101
4102 #ifdef PRINT_OPS
4103 fprintf(stderr, "0x%lx = realloc(%s, 0x%lx, %lu)\r\n",
4104 (Uint) res, ERTS_ALC_N2TD(n), (Uint) ptr, size);
4105 #endif
4106
4107 return res;
4108 }
4109
4110 static void
debug_free(ErtsAlcType_t type,void * extra,void * ptr)4111 debug_free(ErtsAlcType_t type, void *extra, void *ptr)
4112 {
4113 ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
4114 ErtsAlcType_t n;
4115 void *dptr;
4116 Uint size;
4117 int free_pattern;
4118
4119 n = ERTS_ALC_T2N(type);
4120 free_pattern = n;
4121
4122 ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
4123
4124 if (!ptr)
4125 return;
4126
4127 dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE);
4128
4129 #ifdef ERTS_ALC_A_EXEC
4130 # if defined(__i386__) || defined(__x86_64__)
4131 if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) {
4132 free_pattern = 0x0f; /* Illegal instruction */
4133 }
4134 # endif
4135 #endif
4136 sys_memset((void *) dptr, free_pattern, size + FENCE_SZ);
4137
4138 (*real_af->free)(type, real_af->extra, dptr);
4139
4140 #ifdef PRINT_OPS
4141 fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr);
4142 #endif
4143
4144 #ifdef HARD_DEBUG
4145 erts_hdbg_chk_blks();
4146 #endif
4147
4148 }
4149
4150 static Uint
install_debug_functions(void)4151 install_debug_functions(void)
4152 {
4153 int i;
4154 ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
4155
4156 sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs));
4157
4158 for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
4159 erts_allctrs[i].alloc = debug_alloc;
4160 erts_allctrs[i].realloc = debug_realloc;
4161 erts_allctrs[i].free = debug_free;
4162 erts_allctrs[i].extra = (void *) &real_allctrs[i];
4163 }
4164 return FENCE_SZ;
4165 }
4166
4167 #endif /* #ifdef DEBUG */
4168