1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2002-2020. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21
22 /*
23 * Description: Management of memory allocators.
24 *
25 * Author: Rickard Green
26 */
27
28 #ifdef HAVE_CONFIG_H
29 # include "config.h"
30 #endif
31 #define ERTS_ALLOC_C__
32 #define ERTS_ALC_INTERNAL__
33 #define ERTS_WANT_MEM_MAPPERS
34 #include "sys.h"
35 #define ERL_THREADS_EMU_INTERNAL__
36 #include "erl_threads.h"
37 #include "global.h"
38 #include "erl_db.h"
39 #include "erl_binary.h"
40 #include "erl_bits.h"
41 #include "erl_mtrace.h"
42 #include "erl_mseg.h"
43 #include "erl_monitor_link.h"
44 #include "erl_hl_timer.h"
45 #include "erl_cpu_topology.h"
46 #include "erl_thr_queue.h"
47 #include "erl_nfunc_sched.h"
48 #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
49 #include "erl_check_io.h"
50 #endif
51 #include "erl_bif_unique.h"
52
53 #define GET_ERL_GF_ALLOC_IMPL
54 #include "erl_goodfit_alloc.h"
55 #define GET_ERL_BF_ALLOC_IMPL
56 #include "erl_bestfit_alloc.h"
57 #define GET_ERL_AF_ALLOC_IMPL
58 #include "erl_afit_alloc.h"
59 #define GET_ERL_AOFF_ALLOC_IMPL
60 #include "erl_ao_firstfit_alloc.h"
61
62
63 #if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_AU_MAX_PREF_ALLOC_INSTANCES
64 # error "Too many schedulers; cannot create that many pref alloc instances"
65 #endif
66
67 #define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS
68
69 #if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND) || defined(ADDRESS_SANITIZER)
70 #define AU_ALLOC_DEFAULT_ENABLE(X) 0
71 #else
72 #define AU_ALLOC_DEFAULT_ENABLE(X) (X)
73 #endif
74
75 #define ERTS_ALC_DEFAULT_ENABLED_ACUL 60
76 #define ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC 45
77 #define ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC 85
78
79 #define ERTS_ALC_DEFAULT_ACUL ERTS_ALC_DEFAULT_ENABLED_ACUL
80 #define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC
81 #define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC
82
83
84 #ifdef DEBUG
85 static Uint install_debug_functions(void);
86 #if 0
87 #define HARD_DEBUG
88 #ifdef __GNUC__
89 #warning "* * * * * * * * * * * * * *"
90 #warning "* HARD DEBUG IS ENABLED! *"
91 #warning "* * * * * * * * * * * * * *"
92 #endif
93 #endif
94 #endif
95
96 static int lock_all_physical_memory = 0;
97
98 ErtsAllocatorFunctions_t ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]);
99 ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
100 ErtsAllocatorThrSpec_t ERTS_WRITE_UNLIKELY(erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]);
101
102 #define ERTS_MIN(A, B) ((A) < (B) ? (A) : (B))
103 #define ERTS_MAX(A, B) ((A) > (B) ? (A) : (B))
104
105 typedef union {
106 GFAllctr_t gfa;
107 char align_gfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(GFAllctr_t))];
108 BFAllctr_t bfa;
109 char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))];
110 AFAllctr_t afa;
111 char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))];
112 AOFFAllctr_t aoffa;
113 char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))];
114 } ErtsAllocatorState_t erts_align_attribute(ERTS_CACHE_LINE_SIZE);
115
116 static ErtsAllocatorState_t std_alloc_state;
117 static ErtsAllocatorState_t ll_alloc_state;
118 static ErtsAllocatorState_t sl_alloc_state;
119 static ErtsAllocatorState_t temp_alloc_state;
120 static ErtsAllocatorState_t eheap_alloc_state;
121 static ErtsAllocatorState_t binary_alloc_state;
122 static ErtsAllocatorState_t ets_alloc_state;
123 static ErtsAllocatorState_t driver_alloc_state;
124 static ErtsAllocatorState_t fix_alloc_state;
125 static ErtsAllocatorState_t literal_alloc_state;
126 #ifdef ERTS_ALC_A_EXEC
127 static ErtsAllocatorState_t exec_alloc_state;
128 #endif
129 static ErtsAllocatorState_t test_alloc_state;
130
131 enum {
132 ERTS_ALC_INFO_A_ALLOC_UTIL = ERTS_ALC_A_MAX + 1,
133 ERTS_ALC_INFO_A_MSEG_ALLOC,
134 ERTS_ALC_INFO_A_ERTS_MMAP,
135 ERTS_ALC_INFO_A_DISABLED_EXEC, /* fake a disabled "exec_alloc" */
136 ERTS_ALC_INFO_A_END
137 };
138
139 typedef struct {
140 erts_atomic32_t refc;
141 int only_sz;
142 int internal;
143 Uint req_sched;
144 Process *proc;
145 ErtsIRefStorage iref;
146 int allocs[ERTS_ALC_INFO_A_END - ERTS_ALC_A_MIN + 1];
147 } ErtsAllocInfoReq;
148
149 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq,
150 ErtsAllocInfoReq,
151 5,
152 ERTS_ALC_T_AINFO_REQ)
153
154 ErtsAlcType_t erts_fix_core_allocator_ix;
155
156 struct au_init {
157 int enable;
158 int thr_spec;
159 int disable_allowed;
160 int thr_spec_allowed;
161 int carrier_migration_allowed;
162 ErtsAlcStrat_t astrat;
163 struct {
164 AllctrInit_t util;
165 GFAllctrInit_t gf;
166 BFAllctrInit_t bf;
167 AFAllctrInit_t af;
168 AOFFAllctrInit_t aoff;
169 } init;
170 struct {
171 int mmbcs;
172 int lmbcs;
173 int smbcs;
174 int mmmbc;
175 } default_;
176 };
177
178 #define DEFAULT_ALLCTR_INIT { \
179 ERTS_DEFAULT_ALLCTR_INIT, \
180 ERTS_DEFAULT_GF_ALLCTR_INIT, \
181 ERTS_DEFAULT_BF_ALLCTR_INIT, \
182 ERTS_DEFAULT_AF_ALLCTR_INIT, \
183 ERTS_DEFAULT_AOFF_ALLCTR_INIT \
184 }
185
186 typedef struct {
187 int erts_alloc_config;
188 #if HAVE_ERTS_MSEG
189 ErtsMsegInit_t mseg;
190 #endif
191 int trim_threshold;
192 int top_pad;
193 AlcUInit_t alloc_util;
194 struct {
195 char *mtrace;
196 char *nodename;
197 } instr;
198 struct au_init sl_alloc;
199 struct au_init std_alloc;
200 struct au_init ll_alloc;
201 struct au_init temp_alloc;
202 struct au_init eheap_alloc;
203 struct au_init binary_alloc;
204 struct au_init ets_alloc;
205 struct au_init driver_alloc;
206 struct au_init fix_alloc;
207 struct au_init literal_alloc;
208 struct au_init exec_alloc;
209 struct au_init test_alloc;
210 } erts_alc_hndl_args_init_t;
211
212 #define ERTS_AU_INIT__ {0, 0, 1, 1, 1, \
213 ERTS_ALC_S_GOODFIT, DEFAULT_ALLCTR_INIT, \
214 {1,1,1,1}}
215
216 #define SET_DEFAULT_ALLOC_OPTS(IP) \
217 do { \
218 struct au_init aui__ = ERTS_AU_INIT__; \
219 sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init)); \
220 } while (0)
221
222 static void
set_default_sl_alloc_opts(struct au_init * ip)223 set_default_sl_alloc_opts(struct au_init *ip)
224 {
225 SET_DEFAULT_ALLOC_OPTS(ip);
226 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
227 ip->thr_spec = 1;
228 ip->astrat = ERTS_ALC_S_GOODFIT;
229 ip->init.util.name_prefix = "sl_";
230 ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED;
231 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
232 #ifndef SMALL_MEMORY
233 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
234 #else
235 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
236 #endif
237 ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED;
238 ip->init.util.rsbcst = 80;
239 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
240 }
241
242 static void
set_default_std_alloc_opts(struct au_init * ip)243 set_default_std_alloc_opts(struct au_init *ip)
244 {
245 SET_DEFAULT_ALLOC_OPTS(ip);
246 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
247 ip->thr_spec = 1;
248 ip->astrat = ERTS_ALC_S_BESTFIT;
249 ip->init.util.name_prefix = "std_";
250 ip->init.util.alloc_no = ERTS_ALC_A_STANDARD;
251 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
252 #ifndef SMALL_MEMORY
253 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
254 #else
255 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
256 #endif
257 ip->init.util.ts = ERTS_ALC_MTA_STANDARD;
258 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
259 }
260
261 static void
set_default_ll_alloc_opts(struct au_init * ip)262 set_default_ll_alloc_opts(struct au_init *ip)
263 {
264 SET_DEFAULT_ALLOC_OPTS(ip);
265 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
266 ip->thr_spec = 0;
267 ip->astrat = ERTS_ALC_S_BESTFIT;
268 ip->init.bf.ao = 1;
269 ip->init.util.ramv = 0;
270 ip->init.util.mmsbc = 0;
271 ip->init.util.sbct = ~((UWord) 0);
272 ip->init.util.name_prefix = "ll_";
273 ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED;
274 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
275 #ifndef SMALL_MEMORY
276 ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */
277 #else
278 ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */
279 #endif
280 ip->init.util.ts = ERTS_ALC_MTA_LONG_LIVED;
281 ip->init.util.asbcst = 0;
282 ip->init.util.rsbcst = 0;
283 ip->init.util.rsbcmt = 0;
284 ip->init.util.rmbcmt = 0;
285 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL_LL_ALLOC;
286 }
287
288 static void
set_default_literal_alloc_opts(struct au_init * ip)289 set_default_literal_alloc_opts(struct au_init *ip)
290 {
291 SET_DEFAULT_ALLOC_OPTS(ip);
292 #ifdef ADDRESS_SANITIZER
293 ip->enable = 0;
294 #else
295 ip->enable = 1;
296 #endif
297 ip->thr_spec = 0;
298 ip->disable_allowed = 0;
299 ip->thr_spec_allowed = 0;
300 ip->carrier_migration_allowed = 0;
301 ip->astrat = ERTS_ALC_S_BESTFIT;
302 ip->init.bf.ao = 1;
303 ip->init.util.ramv = 0;
304 ip->init.util.mmsbc = 0;
305 ip->init.util.sbct = ~((UWord) 0);
306 ip->init.util.name_prefix = "literal_";
307 ip->init.util.alloc_no = ERTS_ALC_A_LITERAL;
308 #ifndef SMALL_MEMORY
309 ip->init.util.mmbcs = 1024*1024; /* Main carrier size */
310 #else
311 ip->init.util.mmbcs = 256*1024; /* Main carrier size */
312 #endif
313 ip->init.util.ts = ERTS_ALC_MTA_LITERAL;
314 ip->init.util.asbcst = 0;
315 ip->init.util.rsbcst = 0;
316 ip->init.util.rsbcmt = 0;
317 ip->init.util.rmbcmt = 0;
318 ip->init.util.acul = 0;
319
320 #if defined(ARCH_32)
321 # if HAVE_ERTS_MSEG
322 ip->init.util.mseg_alloc = &erts_alcu_literal_32_mseg_alloc;
323 ip->init.util.mseg_realloc = &erts_alcu_literal_32_mseg_realloc;
324 ip->init.util.mseg_dealloc = &erts_alcu_literal_32_mseg_dealloc;
325 # endif
326 ip->init.util.sys_alloc = &erts_alcu_literal_32_sys_alloc;
327 ip->init.util.sys_realloc = &erts_alcu_literal_32_sys_realloc;
328 ip->init.util.sys_dealloc = &erts_alcu_literal_32_sys_dealloc;
329 #elif defined(ARCH_64)
330 # ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
331 ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc;
332 ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc;
333 ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc;
334 ip->init.util.mseg_mmapper = &erts_literal_mmapper;
335 # endif
336 #else
337 # error Unknown architecture
338 #endif
339 }
340
341 #ifdef ERTS_ALC_A_EXEC
342 static void
set_default_exec_alloc_opts(struct au_init * ip)343 set_default_exec_alloc_opts(struct au_init *ip)
344 {
345 SET_DEFAULT_ALLOC_OPTS(ip);
346 ip->enable = 1;
347 ip->thr_spec = 0;
348 ip->disable_allowed = 0;
349 ip->thr_spec_allowed = 0;
350 ip->carrier_migration_allowed = 0;
351 ip->astrat = ERTS_ALC_S_BESTFIT;
352 ip->init.bf.ao = 1;
353 ip->init.util.ramv = 0;
354 ip->init.util.mmsbc = 0;
355 ip->init.util.sbct = ~((UWord) 0);
356 ip->init.util.name_prefix = "exec_";
357 ip->init.util.alloc_no = ERTS_ALC_A_EXEC;
358 ip->init.util.mmbcs = 0; /* No main carrier */
359 ip->init.util.ts = ERTS_ALC_MTA_EXEC;
360 ip->init.util.asbcst = 0;
361 ip->init.util.rsbcst = 0;
362 ip->init.util.rsbcmt = 0;
363 ip->init.util.rmbcmt = 0;
364 ip->init.util.acul = 0;
365
366 ip->init.util.mseg_alloc = &erts_alcu_exec_mseg_alloc;
367 ip->init.util.mseg_realloc = &erts_alcu_exec_mseg_realloc;
368 ip->init.util.mseg_dealloc = &erts_alcu_exec_mseg_dealloc;
369 }
370 #endif /* ERTS_ALC_A_EXEC */
371
372 static void
set_default_temp_alloc_opts(struct au_init * ip)373 set_default_temp_alloc_opts(struct au_init *ip)
374 {
375 SET_DEFAULT_ALLOC_OPTS(ip);
376 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
377 ip->thr_spec = 1;
378 ip->disable_allowed = 0;
379 ip->carrier_migration_allowed = 0;
380 ip->astrat = ERTS_ALC_S_AFIT;
381 ip->init.util.name_prefix = "temp_";
382 ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY;
383 #ifndef SMALL_MEMORY
384 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
385 #else
386 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
387 #endif
388 ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY;
389 ip->init.util.rsbcst = 90;
390 ip->init.util.rmbcmt = 100;
391 }
392
393 static void
set_default_eheap_alloc_opts(struct au_init * ip)394 set_default_eheap_alloc_opts(struct au_init *ip)
395 {
396 SET_DEFAULT_ALLOC_OPTS(ip);
397 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
398 ip->thr_spec = 1;
399 ip->astrat = ERTS_ALC_S_GOODFIT;
400 ip->init.util.name_prefix = "eheap_";
401 ip->init.util.alloc_no = ERTS_ALC_A_EHEAP;
402 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
403 #ifndef SMALL_MEMORY
404 ip->init.util.mmbcs = 512*1024; /* Main carrier size */
405 #else
406 ip->init.util.mmbcs = 256*1024; /* Main carrier size */
407 #endif
408 ip->init.util.ts = ERTS_ALC_MTA_EHEAP;
409 ip->init.util.rsbcst = 50;
410 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC;
411 }
412
413 static void
set_default_binary_alloc_opts(struct au_init * ip)414 set_default_binary_alloc_opts(struct au_init *ip)
415 {
416 SET_DEFAULT_ALLOC_OPTS(ip);
417 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
418 ip->thr_spec = 1;
419 ip->astrat = ERTS_ALC_S_BESTFIT;
420 ip->init.util.name_prefix = "binary_";
421 ip->init.util.alloc_no = ERTS_ALC_A_BINARY;
422 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
423 #ifndef SMALL_MEMORY
424 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
425 #else
426 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
427 #endif
428 ip->init.util.ts = ERTS_ALC_MTA_BINARY;
429 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
430 ip->init.util.atags = 1;
431 }
432
433 static void
set_default_ets_alloc_opts(struct au_init * ip)434 set_default_ets_alloc_opts(struct au_init *ip)
435 {
436 SET_DEFAULT_ALLOC_OPTS(ip);
437 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
438 ip->thr_spec = 1;
439 ip->astrat = ERTS_ALC_S_BESTFIT;
440 ip->init.util.name_prefix = "ets_";
441 ip->init.util.alloc_no = ERTS_ALC_A_ETS;
442 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
443 #ifndef SMALL_MEMORY
444 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
445 #else
446 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
447 #endif
448 ip->init.util.ts = ERTS_ALC_MTA_ETS;
449 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
450 }
451
452 static void
set_default_driver_alloc_opts(struct au_init * ip)453 set_default_driver_alloc_opts(struct au_init *ip)
454 {
455 SET_DEFAULT_ALLOC_OPTS(ip);
456 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
457 ip->thr_spec = 1;
458 ip->astrat = ERTS_ALC_S_BESTFIT;
459 ip->init.util.name_prefix = "driver_";
460 ip->init.util.alloc_no = ERTS_ALC_A_DRIVER;
461 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
462 #ifndef SMALL_MEMORY
463 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
464 #else
465 ip->init.util.mmbcs = 32*1024; /* Main carrier size */
466 #endif
467 ip->init.util.ts = ERTS_ALC_MTA_DRIVER;
468 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
469 ip->init.util.atags = 1;
470 }
471
472 static void
set_default_fix_alloc_opts(struct au_init * ip,size_t * fix_type_sizes)473 set_default_fix_alloc_opts(struct au_init *ip,
474 size_t *fix_type_sizes)
475 {
476 SET_DEFAULT_ALLOC_OPTS(ip);
477 ip->enable = AU_ALLOC_DEFAULT_ENABLE(1);
478 ip->thr_spec = 1;
479 ip->astrat = ERTS_ALC_S_BESTFIT;
480 ip->init.bf.ao = 1;
481 ip->init.util.name_prefix = "fix_";
482 ip->init.util.fix_type_size = fix_type_sizes;
483 ip->init.util.alloc_no = ERTS_ALC_A_FIXED_SIZE;
484 ip->init.util.cp = ERTS_ALC_COMMON_CPOOL_IX;
485 #ifndef SMALL_MEMORY
486 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
487 #else
488 ip->init.util.mmbcs = 128*1024; /* Main carrier size */
489 #endif
490 ip->init.util.ts = ERTS_ALC_MTA_FIXED_SIZE;
491 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
492 }
493
494 static void
set_default_test_alloc_opts(struct au_init * ip)495 set_default_test_alloc_opts(struct au_init *ip)
496 {
497 SET_DEFAULT_ALLOC_OPTS(ip);
498 ip->enable = 0; /* Disabled by default */
499 ip->thr_spec = -1 * erts_no_schedulers;
500 ip->astrat = ERTS_ALC_S_FIRSTFIT;
501 ip->init.aoff.crr_order = FF_AOFF;
502 ip->init.aoff.blk_order = FF_BF;
503 ip->init.util.name_prefix = "test_";
504 ip->init.util.alloc_no = ERTS_ALC_A_TEST;
505 ip->init.util.cp = ERTS_ALC_A_TEST;
506 ip->init.util.mmbcs = 0; /* Main carrier size */
507 ip->init.util.ts = ERTS_ALC_MTA_TEST;
508 ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
509 ip->init.util.atags = 1;
510
511 /* Use a constant minimal MBC size */
512 #if ERTS_SA_MB_CARRIERS
513 ip->init.util.smbcs = ERTS_SACRR_UNIT_SZ;
514 ip->init.util.lmbcs = ERTS_SACRR_UNIT_SZ;
515 ip->init.util.sbct = ERTS_SACRR_UNIT_SZ;
516 #else
517 ip->init.util.smbcs = 1 << 12;
518 ip->init.util.lmbcs = 1 << 12;
519 ip->init.util.sbct = 1 << 12;
520 #endif
521 }
522
523
524
525 static void
adjust_tpref(struct au_init * ip,int no_sched)526 adjust_tpref(struct au_init *ip, int no_sched)
527 {
528 if (ip->thr_spec) {
529 ip->thr_spec = no_sched;
530 ip->thr_spec *= -1; /* thread preferred */
531
532 /* If default ... */
533
534 /* ... shrink main multi-block carrier size */
535 if (ip->default_.mmbcs)
536 ip->init.util.mmbcs /= ERTS_MIN(4, no_sched);
537 /* ... shrink largest multi-block carrier size */
538 if (ip->default_.lmbcs)
539 ip->init.util.lmbcs /= ERTS_MIN(2, no_sched);
540 /* ... shrink smallest multi-block carrier size */
541 if (ip->default_.smbcs)
542 ip->init.util.smbcs /= ERTS_MIN(4, no_sched);
543 }
544 }
545
546
547 static void handle_args(int *, char **, erts_alc_hndl_args_init_t *);
548
549 static void
550 set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu);
551
552 static void
553 start_au_allocator(ErtsAlcType_t alctr_n,
554 struct au_init *init,
555 ErtsAllocatorState_t *state);
556
557 static void
refuse_af_strategy(struct au_init * init)558 refuse_af_strategy(struct au_init *init)
559 {
560 if (init->astrat == ERTS_ALC_S_AFIT)
561 init->astrat = ERTS_ALC_S_GOODFIT;
562 }
563
564 #ifdef HARD_DEBUG
565 static void hdbg_init(void);
566 #endif
567
adjust_fix_alloc_sizes(UWord extra_block_size)568 static void adjust_fix_alloc_sizes(UWord extra_block_size)
569 {
570
571 if (extra_block_size && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled) {
572 int j;
573
574 if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec) {
575 int i;
576 ErtsAllocatorThrSpec_t* tspec;
577
578 tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
579 ASSERT(tspec->enabled);
580
581 for (i=0; i < tspec->size; i++) {
582 Allctr_t* allctr = tspec->allctr[i];
583 for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) {
584 size_t size = allctr->fix[j].type_size;
585 size = MAX(size + extra_block_size,
586 sizeof(ErtsAllctrDDBlock_t));
587 allctr->fix[j].type_size = size;
588 }
589 }
590 }
591 else
592 {
593 Allctr_t* allctr = erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra;
594 for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) {
595 size_t size = allctr->fix[j].type_size;
596 size = MAX(size + extra_block_size,
597 sizeof(ErtsAllctrDDBlock_t));
598 allctr->fix[j].type_size = size;
599 }
600 }
601 }
602 }
603
604 static ERTS_INLINE int
strategy_support_carrier_migration(struct au_init * auip)605 strategy_support_carrier_migration(struct au_init *auip)
606 {
607 /*
608 * Currently only aoff* and ageff* support carrier
609 * migration, i.e, type AOFIRSTFIT.
610 */
611 return auip->astrat == ERTS_ALC_S_FIRSTFIT;
612 }
613
614 static ERTS_INLINE void
adjust_carrier_migration_support(struct au_init * auip)615 adjust_carrier_migration_support(struct au_init *auip)
616 {
617 if (auip->init.util.acul) {
618 auip->thr_spec = -1; /* Need thread preferred */
619
620 /*
621 * If strategy cannot handle carrier migration,
622 * default to a strategy that can...
623 */
624 if (!strategy_support_carrier_migration(auip)) {
625 /* Default to aoffcbf */
626 auip->astrat = ERTS_ALC_S_FIRSTFIT;
627 auip->init.aoff.crr_order = FF_AOFF;
628 auip->init.aoff.blk_order = FF_BF;
629 }
630 }
631 }
632
633 void
erts_alloc_init(int * argc,char ** argv,ErtsAllocInitOpts * eaiop)634 erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
635 {
636 UWord extra_block_size = 0;
637 int i, ncpu;
638 erts_alc_hndl_args_init_t init = {
639 0,
640 #if HAVE_ERTS_MSEG
641 ERTS_MSEG_INIT_DEFAULT_INITIALIZER,
642 #endif
643 ERTS_DEFAULT_TRIM_THRESHOLD,
644 ERTS_DEFAULT_TOP_PAD,
645 ERTS_DEFAULT_ALCU_INIT,
646 };
647 size_t fix_type_sizes[ERTS_ALC_NO_FIXED_SIZES] = {0};
648
649 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)]
650 = sizeof(Process);
651 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR)]
652 = sizeof(ErtsMonitorDataHeap);
653 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LINK)]
654 = sizeof(ErtsILink);
655 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
656 = sizeof(ErtsDrvSelectDataState);
657 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_SEL_D_STATE)]
658 = sizeof(ErtsNifSelectDataState);
659 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)]
660 = sizeof(ErtsMessageRef);
661 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)]
662 = sizeof(ErtsThrQElement_t);
663 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LL_PTIMER)]
664 = erts_timer_type_size(ERTS_ALC_T_LL_PTIMER);
665 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_HL_PTIMER)]
666 = erts_timer_type_size(ERTS_ALC_T_HL_PTIMER);
667 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)]
668 = erts_timer_type_size(ERTS_ALC_T_BIF_TIMER);
669 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)]
670 = sizeof(ErtsNSchedMagicRefTableEntry);
671 fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MINDIRECTION)]
672 = ERTS_MAGIC_BIN_UNALIGNED_SIZE(sizeof(ErtsMagicIndirectionWord));
673
674 #ifdef HARD_DEBUG
675 hdbg_init();
676 #endif
677
678 lock_all_physical_memory = 0;
679
680 ncpu = eaiop->ncpu;
681 if (ncpu < 1)
682 ncpu = 1;
683
684 erts_tsd_key_create(&erts_allctr_prelock_tsd_key,
685 "erts_allctr_prelock_tsd_key");
686
687 erts_sys_alloc_init();
688 erts_init_utils_mem();
689
690 set_default_sl_alloc_opts(&init.sl_alloc);
691 set_default_std_alloc_opts(&init.std_alloc);
692 set_default_ll_alloc_opts(&init.ll_alloc);
693 set_default_temp_alloc_opts(&init.temp_alloc);
694 set_default_eheap_alloc_opts(&init.eheap_alloc);
695 set_default_binary_alloc_opts(&init.binary_alloc);
696 set_default_ets_alloc_opts(&init.ets_alloc);
697 set_default_driver_alloc_opts(&init.driver_alloc);
698 set_default_fix_alloc_opts(&init.fix_alloc,
699 fix_type_sizes);
700 set_default_literal_alloc_opts(&init.literal_alloc);
701 #ifdef ERTS_ALC_A_EXEC
702 set_default_exec_alloc_opts(&init.exec_alloc);
703 #endif
704 set_default_test_alloc_opts(&init.test_alloc);
705
706 if (argc && argv)
707 handle_args(argc, argv, &init);
708
709 if (lock_all_physical_memory) {
710 #ifdef HAVE_MLOCKALL
711 errno = 0;
712 if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) {
713 int err = errno;
714 char *errstr = err ? strerror(err) : "unknown";
715 erts_exit(1, "Failed to lock physical memory: %s (%d)\n",
716 errstr, err);
717 }
718 #else
719 erts_exit(1, "Failed to lock physical memory: Not supported\n");
720 #endif
721 }
722
723
724 /* Make adjustments for carrier migration support */
725 init.temp_alloc.init.util.acul = 0;
726 adjust_carrier_migration_support(&init.sl_alloc);
727 adjust_carrier_migration_support(&init.std_alloc);
728 adjust_carrier_migration_support(&init.ll_alloc);
729 adjust_carrier_migration_support(&init.eheap_alloc);
730 adjust_carrier_migration_support(&init.binary_alloc);
731 adjust_carrier_migration_support(&init.ets_alloc);
732 adjust_carrier_migration_support(&init.driver_alloc);
733 adjust_carrier_migration_support(&init.fix_alloc);
734 adjust_carrier_migration_support(&init.literal_alloc);
735 #ifdef ERTS_ALC_A_EXEC
736 adjust_carrier_migration_support(&init.exec_alloc);
737 #endif
738
739 if (init.erts_alloc_config) {
740 /* Adjust flags that erts_alloc_config won't like */
741
742 /* No thread specific instances */
743 init.temp_alloc.thr_spec = 0;
744 init.sl_alloc.thr_spec = 0;
745 init.std_alloc.thr_spec = 0;
746 init.ll_alloc.thr_spec = 0;
747 init.eheap_alloc.thr_spec = 0;
748 init.binary_alloc.thr_spec = 0;
749 init.ets_alloc.thr_spec = 0;
750 init.driver_alloc.thr_spec = 0;
751 init.fix_alloc.thr_spec = 0;
752 init.literal_alloc.thr_spec = 0;
753 #ifdef ERTS_ALC_A_EXEC
754 init.exec_alloc.thr_spec = 0;
755 #endif
756
757 /* No carrier migration */
758 init.temp_alloc.init.util.acul = 0;
759 init.sl_alloc.init.util.acul = 0;
760 init.std_alloc.init.util.acul = 0;
761 init.ll_alloc.init.util.acul = 0;
762 init.eheap_alloc.init.util.acul = 0;
763 init.binary_alloc.init.util.acul = 0;
764 init.ets_alloc.init.util.acul = 0;
765 init.driver_alloc.init.util.acul = 0;
766 init.fix_alloc.init.util.acul = 0;
767 init.literal_alloc.init.util.acul = 0;
768 #ifdef ERTS_ALC_A_EXEC
769 init.exec_alloc.init.util.acul = 0;
770 #endif
771 }
772
773 /* Only temp_alloc can use thread specific interface */
774 if (init.temp_alloc.thr_spec)
775 init.temp_alloc.thr_spec = erts_no_schedulers;
776
777 /* Others must use thread preferred interface */
778 adjust_tpref(&init.sl_alloc, erts_no_schedulers);
779 adjust_tpref(&init.std_alloc, erts_no_schedulers);
780 adjust_tpref(&init.ll_alloc, erts_no_schedulers);
781 adjust_tpref(&init.eheap_alloc, erts_no_schedulers);
782 adjust_tpref(&init.binary_alloc, erts_no_schedulers);
783 adjust_tpref(&init.ets_alloc, erts_no_schedulers);
784 adjust_tpref(&init.driver_alloc, erts_no_schedulers);
785 adjust_tpref(&init.fix_alloc, erts_no_schedulers);
786 adjust_tpref(&init.literal_alloc, erts_no_schedulers);
787 #ifdef ERTS_ALC_A_EXEC
788 adjust_tpref(&init.exec_alloc, erts_no_schedulers);
789 #endif
790
791
792 /*
793 * The following allocators cannot be run with afit strategy.
794 * Make sure they don't...
795 */
796 refuse_af_strategy(&init.sl_alloc);
797 refuse_af_strategy(&init.std_alloc);
798 refuse_af_strategy(&init.ll_alloc);
799 refuse_af_strategy(&init.eheap_alloc);
800 refuse_af_strategy(&init.binary_alloc);
801 refuse_af_strategy(&init.ets_alloc);
802 refuse_af_strategy(&init.driver_alloc);
803 refuse_af_strategy(&init.fix_alloc);
804 refuse_af_strategy(&init.literal_alloc);
805 #ifdef ERTS_ALC_A_EXEC
806 refuse_af_strategy(&init.exec_alloc);
807 #endif
808
809 if (!init.temp_alloc.thr_spec)
810 refuse_af_strategy(&init.temp_alloc);
811
812 erts_mtrace_pre_init();
813 #if HAVE_ERTS_MSEG
814 init.mseg.nos = erts_no_schedulers;
815 erts_mseg_init(&init.mseg);
816 #endif
817
818 erts_alcu_init(&init.alloc_util);
819 erts_afalc_init();
820 erts_bfalc_init();
821 erts_gfalc_init();
822 erts_aoffalc_init();
823
824 for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
825 erts_allctrs[i].alloc = NULL;
826 erts_allctrs[i].realloc = NULL;
827 erts_allctrs[i].free = NULL;
828 erts_allctrs[i].extra = NULL;
829 erts_allctrs_info[i].alloc_util = 0;
830 erts_allctrs_info[i].enabled = 0;
831 erts_allctrs_info[i].thr_spec = 0;
832 erts_allctrs_info[i].extra = NULL;
833 }
834
835 erts_allctrs[ERTS_ALC_A_SYSTEM].alloc = erts_sys_alloc;
836 erts_allctrs[ERTS_ALC_A_SYSTEM].realloc = erts_sys_realloc;
837 erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free;
838 erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1;
839
840 set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu);
841 set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu);
842 set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu);
843 set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu);
844 set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, ncpu);
845 set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc, ncpu);
846 set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu);
847 set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
848 set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
849 set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu);
850 #ifdef ERTS_ALC_A_EXEC
851 set_au_allocator(ERTS_ALC_A_EXEC, &init.exec_alloc, ncpu);
852 #endif
853 set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu);
854
855 for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
856 if (!erts_allctrs[i].alloc)
857 erts_exit(ERTS_ABORT_EXIT,
858 "Missing alloc function for %s\n", ERTS_ALC_A2AD(i));
859 if (!erts_allctrs[i].realloc)
860 erts_exit(ERTS_ABORT_EXIT,
861 "Missing realloc function for %s\n", ERTS_ALC_A2AD(i));
862 if (!erts_allctrs[i].free)
863 erts_exit(ERTS_ABORT_EXIT,
864 "Missing free function for %s\n", ERTS_ALC_A2AD(i));
865 }
866
867 sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold);
868 sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad);
869
870 erts_mtrace_init(init.instr.mtrace, init.instr.nodename);
871
872 start_au_allocator(ERTS_ALC_A_TEMPORARY,
873 &init.temp_alloc,
874 &temp_alloc_state);
875
876 start_au_allocator(ERTS_ALC_A_SHORT_LIVED,
877 &init.sl_alloc,
878 &sl_alloc_state);
879
880 start_au_allocator(ERTS_ALC_A_STANDARD,
881 &init.std_alloc,
882 &std_alloc_state);
883
884 start_au_allocator(ERTS_ALC_A_LONG_LIVED,
885 &init.ll_alloc,
886 &ll_alloc_state);
887 start_au_allocator(ERTS_ALC_A_EHEAP,
888 &init.eheap_alloc,
889 &eheap_alloc_state);
890
891 start_au_allocator(ERTS_ALC_A_BINARY,
892 &init.binary_alloc,
893 &binary_alloc_state);
894
895 start_au_allocator(ERTS_ALC_A_ETS,
896 &init.ets_alloc,
897 &ets_alloc_state);
898
899 start_au_allocator(ERTS_ALC_A_DRIVER,
900 &init.driver_alloc,
901 &driver_alloc_state);
902
903 start_au_allocator(ERTS_ALC_A_FIXED_SIZE,
904 &init.fix_alloc,
905 &fix_alloc_state);
906 start_au_allocator(ERTS_ALC_A_LITERAL,
907 &init.literal_alloc,
908 &literal_alloc_state);
909 #ifdef ERTS_ALC_A_EXEC
910 start_au_allocator(ERTS_ALC_A_EXEC,
911 &init.exec_alloc,
912 &exec_alloc_state);
913 #endif
914 start_au_allocator(ERTS_ALC_A_TEST,
915 &init.test_alloc,
916 &test_alloc_state);
917
918 erts_mtrace_install_wrapper_functions();
919
920 init_aireq_alloc();
921
922 #ifdef DEBUG
923 extra_block_size += install_debug_functions();
924 #endif
925 adjust_fix_alloc_sizes(extra_block_size);
926 }
927
928 void
erts_alloc_late_init(void)929 erts_alloc_late_init(void)
930 {
931
932 }
933
934 static void *
erts_realloc_fixed_size(ErtsAlcType_t type,void * extra,void * p,Uint size)935 erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size)
936 {
937 erts_exit(ERTS_ABORT_EXIT,
938 "Attempt to reallocate a block of the fixed size type %s\n",
939 ERTS_ALC_T2TD(type));
940 }
941
942
943 static void
set_au_allocator(ErtsAlcType_t alctr_n,struct au_init * init,int ncpu)944 set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
945 {
946 ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
947 ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
948 ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
949
950 /*
951 * Some allocators are forced on if halfword heap is used.
952 */
953 if (init->init.util.force)
954 init->enable = 1;
955
956 tspec->enabled = 0;
957 tspec->dd = 0;
958 tspec->aix = alctr_n;
959 tspec->size = 0;
960 ai->thr_spec = 0;
961
962 if (!init->enable) {
963 af->alloc = erts_sys_alloc;
964 af->realloc = erts_sys_realloc;
965 af->free = erts_sys_free;
966 af->extra = NULL;
967 ai->alloc_util = 0;
968 ai->enabled = 0;
969 ai->extra = NULL;
970 return;
971 }
972
973 if (init->thr_spec) {
974 if (init->thr_spec > 0) {
975 af->alloc = erts_alcu_alloc_thr_spec;
976 if (init->init.util.fix_type_size)
977 af->realloc = erts_realloc_fixed_size;
978 else if (init->init.util.ramv)
979 af->realloc = erts_alcu_realloc_mv_thr_spec;
980 else
981 af->realloc = erts_alcu_realloc_thr_spec;
982 af->free = erts_alcu_free_thr_spec;
983 }
984 else {
985 af->alloc = erts_alcu_alloc_thr_pref;
986 if (init->init.util.fix_type_size)
987 af->realloc = erts_realloc_fixed_size;
988 else if (init->init.util.ramv)
989 af->realloc = erts_alcu_realloc_mv_thr_pref;
990 else
991 af->realloc = erts_alcu_realloc_thr_pref;
992 af->free = erts_alcu_free_thr_pref;
993 tspec->dd = 1;
994 }
995
996 tspec->enabled = 1;
997 tspec->size = abs(init->thr_spec) + 1;
998
999 ai->thr_spec = tspec->size;
1000 }
1001 else
1002 if (init->init.util.ts) {
1003 af->alloc = erts_alcu_alloc_ts;
1004 if (init->init.util.fix_type_size)
1005 af->realloc = erts_realloc_fixed_size;
1006 else if (init->init.util.ramv)
1007 af->realloc = erts_alcu_realloc_mv_ts;
1008 else
1009 af->realloc = erts_alcu_realloc_ts;
1010 af->free = erts_alcu_free_ts;
1011 }
1012 else
1013 {
1014 erts_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n",
1015 init->init.util.name_prefix);
1016 }
1017 af->extra = NULL;
1018 ai->alloc_util = 1;
1019 ai->enabled = 1;
1020 }
1021
1022 static void
start_au_allocator(ErtsAlcType_t alctr_n,struct au_init * init,ErtsAllocatorState_t * state)1023 start_au_allocator(ErtsAlcType_t alctr_n,
1024 struct au_init *init,
1025 ErtsAllocatorState_t *state)
1026 {
1027 int i;
1028 int size = 1;
1029 void *as0;
1030 ErtsAlcStrat_t astrat;
1031 ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
1032 ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
1033 ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
1034 ErtsAlcFixList_t *fix_lists = NULL;
1035 size_t fix_list_size = 0;
1036
1037 if (!init->enable)
1038 return;
1039
1040 if (init->thr_spec) {
1041 char *states = erts_sys_alloc(0,
1042 NULL,
1043 ((sizeof(Allctr_t *)
1044 * (tspec->size + 1))
1045 + (sizeof(ErtsAllocatorState_t)
1046 * tspec->size)
1047 + ERTS_CACHE_LINE_SIZE - 1));
1048 if (!states)
1049 erts_exit(ERTS_ABORT_EXIT,
1050 "Failed to allocate allocator states for %salloc\n",
1051 init->init.util.name_prefix);
1052 tspec->allctr = (Allctr_t **) states;
1053 states += sizeof(Allctr_t *) * (tspec->size + 1);
1054 states = ((((UWord) states) & ERTS_CACHE_LINE_MASK)
1055 ? (char *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK)
1056 + ERTS_CACHE_LINE_SIZE)
1057 : (char *) states);
1058 tspec->allctr[0] = (Allctr_t *) state;
1059 size = tspec->size;
1060 for (i = 1; i < size; i++)
1061 tspec->allctr[i] = (Allctr_t *)
1062 &((ErtsAllocatorState_t *) states)[i-1];
1063 }
1064
1065 if (init->init.util.fix_type_size) {
1066 size_t tot_fix_list_size;
1067 fix_list_size = sizeof(ErtsAlcFixList_t)*ERTS_ALC_NO_FIXED_SIZES;
1068 fix_list_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(fix_list_size);
1069 tot_fix_list_size = fix_list_size;
1070 if (init->thr_spec)
1071 tot_fix_list_size *= tspec->size;
1072 fix_lists = erts_sys_alloc(0,
1073 NULL,
1074 (tot_fix_list_size
1075 + ERTS_CACHE_LINE_SIZE - 1));
1076 if (!fix_lists)
1077 erts_exit(ERTS_ABORT_EXIT,
1078 "Failed to allocate fix lists for %salloc\n",
1079 init->init.util.name_prefix);
1080
1081 if (((UWord) fix_lists) & ERTS_CACHE_LINE_MASK)
1082 fix_lists = ((ErtsAlcFixList_t *)
1083 ((((UWord) fix_lists) & ~ERTS_CACHE_LINE_MASK)
1084 + ERTS_CACHE_LINE_SIZE));
1085 }
1086
1087 for (i = 0; i < size; i++) {
1088 Allctr_t *as;
1089 astrat = init->astrat;
1090
1091 if (!init->thr_spec)
1092 as0 = state;
1093 else {
1094 as0 = (void *) tspec->allctr[i];
1095 if (!as0)
1096 continue;
1097 if (init->thr_spec < 0) {
1098 init->init.util.ts = i == 0;
1099 init->init.util.tspec = 0;
1100 init->init.util.tpref = -1*init->thr_spec + 1;
1101 }
1102 else {
1103 if (i != 0)
1104 init->init.util.ts = 0;
1105 else {
1106 if (astrat == ERTS_ALC_S_AFIT)
1107 astrat = ERTS_ALC_S_GOODFIT;
1108 init->init.util.ts = 1;
1109 }
1110 init->init.util.tspec = init->thr_spec + 1;
1111 init->init.util.tpref = 0;
1112 }
1113 }
1114
1115 if (fix_lists) {
1116 init->init.util.fix = fix_lists;
1117 fix_lists = ((ErtsAlcFixList_t *)
1118 (((char *) fix_lists) + fix_list_size));
1119 }
1120
1121 init->init.util.alloc_strat = astrat;
1122 init->init.util.ix = i;
1123
1124 switch (astrat) {
1125 case ERTS_ALC_S_GOODFIT:
1126 as = erts_gfalc_start((GFAllctr_t *) as0,
1127 &init->init.gf,
1128 &init->init.util);
1129 break;
1130 case ERTS_ALC_S_BESTFIT:
1131 as = erts_bfalc_start((BFAllctr_t *) as0,
1132 &init->init.bf,
1133 &init->init.util);
1134 break;
1135 case ERTS_ALC_S_AFIT:
1136 as = erts_afalc_start((AFAllctr_t *) as0,
1137 &init->init.af,
1138 &init->init.util);
1139 break;
1140 case ERTS_ALC_S_FIRSTFIT:
1141 as = erts_aoffalc_start((AOFFAllctr_t *) as0,
1142 &init->init.aoff,
1143 &init->init.util);
1144 break;
1145
1146 default:
1147 as = NULL;
1148 ASSERT(0);
1149 }
1150
1151 if (!as)
1152 erts_exit(ERTS_ABORT_EXIT,
1153 "Failed to start %salloc\n", init->init.util.name_prefix);
1154
1155 ASSERT(as == (void *) as0);
1156 af->extra = as;
1157 }
1158
1159 if (init->thr_spec)
1160 af->extra = tspec;
1161 ai->extra = af->extra;
1162 }
1163
1164
bad_param(char * param_start,char * param_end)1165 static void bad_param(char *param_start, char *param_end)
1166 {
1167 size_t len = param_end - param_start;
1168 char param[100];
1169 if (len > 99)
1170 len = 99;
1171 sys_memcpy((void *) param, (void *) param_start, len);
1172 param[len] = '\0';
1173 erts_fprintf(stderr, "bad \"%s\" parameter\n", param);
1174 erts_usage();
1175 }
1176
bad_value(char * param_start,char * param_end,char * value)1177 static void bad_value(char *param_start, char *param_end, char *value)
1178 {
1179 size_t len = param_end - param_start;
1180 char param[100];
1181 if (len > 99)
1182 len = 99;
1183 sys_memcpy((void *) param, (void *) param_start, len);
1184 param[len] = '\0';
1185 erts_fprintf(stderr, "bad \"%s\" value: %s\n", param, value);
1186 erts_usage();
1187 }
1188
1189 /* Get arg marks argument as handled by
1190 putting NULL in argv */
1191 static char *
get_value(char * rest,char ** argv,int * ip)1192 get_value(char* rest, char** argv, int* ip)
1193 {
1194 char *param = argv[*ip]+1;
1195 argv[*ip] = NULL;
1196 if (*rest == '\0') {
1197 char *next = argv[*ip + 1];
1198 if (next[0] == '-'
1199 && next[1] == '-'
1200 && next[2] == '\0') {
1201 bad_value(param, rest, "");
1202 }
1203 (*ip)++;
1204 argv[*ip] = NULL;
1205 return next;
1206 }
1207 return rest;
1208 }
1209
1210 static ERTS_INLINE int
has_prefix(const char * prefix,const char * string)1211 has_prefix(const char *prefix, const char *string)
1212 {
1213 int i;
1214 for (i = 0; prefix[i]; i++)
1215 if (prefix[i] != string[i])
1216 return 0;
1217 return 1;
1218 }
1219
1220 static int
get_bool_value(char * param_end,char ** argv,int * ip)1221 get_bool_value(char *param_end, char** argv, int* ip)
1222 {
1223 char *param = argv[*ip]+1;
1224 char *value = get_value(param_end, argv, ip);
1225 if (sys_strcmp(value, "true") == 0)
1226 return 1;
1227 else if (sys_strcmp(value, "false") == 0)
1228 return 0;
1229 else
1230 bad_value(param, param_end, value);
1231 return -1;
1232 }
1233
kb_to_bytes(Sint kb,Uint * bytes)1234 static Uint kb_to_bytes(Sint kb, Uint *bytes)
1235 {
1236 const Uint max = ((~((Uint) 0))/1024) + 1;
1237
1238 if (kb < 0 || (Uint)kb > max)
1239 return 0;
1240 if ((Uint)kb == max)
1241 *bytes = ~((Uint) 0);
1242 else
1243 *bytes = ((Uint) kb)*1024;
1244 return 1;
1245 }
1246
1247 static Uint
get_kb_value(char * param_end,char ** argv,int * ip)1248 get_kb_value(char *param_end, char** argv, int* ip)
1249 {
1250 Sint tmp;
1251 Uint bytes = 0;
1252 char *rest;
1253 char *param = argv[*ip]+1;
1254 char *value = get_value(param_end, argv, ip);
1255 errno = 0;
1256 tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1257 if (errno != 0 || rest == value || !kb_to_bytes(tmp, &bytes))
1258 bad_value(param, param_end, value);
1259 return bytes;
1260 }
1261
1262 static UWord
get_mb_value(char * param_end,char ** argv,int * ip)1263 get_mb_value(char *param_end, char** argv, int* ip)
1264 {
1265 SWord tmp;
1266 UWord max = ((~((UWord) 0))/(1024*1024)) + 1;
1267 char *rest;
1268 char *param = argv[*ip]+1;
1269 char *value = get_value(param_end, argv, ip);
1270 errno = 0;
1271 tmp = (SWord) ErtsStrToSint(value, &rest, 10);
1272 if (errno != 0 || rest == value || tmp < 0 || max < ((UWord) tmp))
1273 bad_value(param, param_end, value);
1274 if (max == (UWord) tmp)
1275 return ~((UWord) 0);
1276 else
1277 return ((UWord) tmp)*1024*1024;
1278 }
1279
1280
1281 #if 0
1282 static Uint
1283 get_byte_value(char *param_end, char** argv, int* ip)
1284 {
1285 Sint tmp;
1286 char *rest;
1287 char *param = argv[*ip]+1;
1288 char *value = get_value(param_end, argv, ip);
1289 errno = 0;
1290 tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1291 if (errno != 0 || rest == value || tmp < 0)
1292 bad_value(param, param_end, value);
1293 return (Uint) tmp;
1294 }
1295 #endif
1296
1297 static Uint
get_amount_value(char * param_end,char ** argv,int * ip)1298 get_amount_value(char *param_end, char** argv, int* ip)
1299 {
1300 Sint tmp;
1301 char *rest;
1302 char *param = argv[*ip]+1;
1303 char *value = get_value(param_end, argv, ip);
1304 errno = 0;
1305 tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1306 if (errno != 0 || rest == value || tmp < 0)
1307 bad_value(param, param_end, value);
1308 return (Uint) tmp;
1309 }
1310
1311 static Uint
get_acul_value(struct au_init * auip,char * param_end,char ** argv,int * ip)1312 get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip)
1313 {
1314 Sint tmp;
1315 char *rest;
1316 char *param = argv[*ip]+1;
1317 char *value = get_value(param_end, argv, ip);
1318 if (sys_strcmp(value, "de") == 0) {
1319 switch (auip->init.util.alloc_no) {
1320 case ERTS_ALC_A_LONG_LIVED:
1321 return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC;
1322 case ERTS_ALC_A_EHEAP:
1323 return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC;
1324 default:
1325 return ERTS_ALC_DEFAULT_ENABLED_ACUL;
1326 }
1327 }
1328 errno = 0;
1329 tmp = (Sint) ErtsStrToSint(value, &rest, 10);
1330 if (errno != 0 || rest == value || tmp < 0 || 100 < tmp)
1331 bad_value(param, param_end, value);
1332 return (Uint) tmp;
1333 }
1334
1335 static void
handle_au_arg(struct au_init * auip,char * sub_param,char ** argv,int * ip,int u_switch)1336 handle_au_arg(struct au_init *auip,
1337 char* sub_param,
1338 char** argv,
1339 int* ip,
1340 int u_switch)
1341 {
1342 char *param = argv[*ip]+1;
1343
1344 switch (sub_param[0]) {
1345 case 'a':
1346 if (sub_param[1] == 'c') { /* Migration parameters "ac*" */
1347 UWord value;
1348 UWord* wp;
1349 if (!auip->carrier_migration_allowed && !u_switch)
1350 goto bad_switch;
1351
1352 if (has_prefix("acul", sub_param)) {
1353 value = get_acul_value(auip, sub_param + 4, argv, ip);
1354 wp = &auip->init.util.acul;
1355 }
1356 else if (has_prefix("acnl", sub_param)) {
1357 value = get_amount_value(sub_param + 4, argv, ip);
1358 wp = &auip->init.util.acnl;
1359 }
1360 else if (has_prefix("acfml", sub_param)) {
1361 value = get_amount_value(sub_param + 5, argv, ip);
1362 wp = &auip->init.util.acfml;
1363 }
1364 else
1365 goto bad_switch;
1366
1367 if (auip->carrier_migration_allowed)
1368 *wp = value;
1369 }
1370 else if(has_prefix("asbcst", sub_param)) {
1371 auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip);
1372 }
1373 else if(has_prefix("as", sub_param)) {
1374 char *alg = get_value(sub_param + 2, argv, ip);
1375 if (sys_strcmp("bf", alg) == 0) {
1376 auip->astrat = ERTS_ALC_S_BESTFIT;
1377 auip->init.bf.ao = 0;
1378 }
1379 else if (sys_strcmp("aobf", alg) == 0) {
1380 auip->astrat = ERTS_ALC_S_BESTFIT;
1381 auip->init.bf.ao = 1;
1382 }
1383 else if (sys_strcmp("gf", alg) == 0) {
1384 auip->astrat = ERTS_ALC_S_GOODFIT;
1385 }
1386 else if (sys_strcmp("af", alg) == 0) {
1387 auip->astrat = ERTS_ALC_S_AFIT;
1388 }
1389 else if (sys_strcmp("aoff", alg) == 0) {
1390 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1391 auip->init.aoff.crr_order = FF_AOFF;
1392 auip->init.aoff.blk_order = FF_AOFF;
1393 }
1394 else if (sys_strcmp("aoffcbf", alg) == 0) {
1395 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1396 auip->init.aoff.crr_order = FF_AOFF;
1397 auip->init.aoff.blk_order = FF_BF;
1398 }
1399 else if (sys_strcmp("aoffcaobf", alg) == 0) {
1400 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1401 auip->init.aoff.crr_order = FF_AOFF;
1402 auip->init.aoff.blk_order = FF_AOBF;
1403 }
1404 else if (sys_strcmp("ageffcaoff", alg) == 0) {
1405 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1406 auip->init.aoff.crr_order = FF_AGEFF;
1407 auip->init.aoff.blk_order = FF_AOFF;
1408 }
1409 else if (sys_strcmp("ageffcbf", alg) == 0) {
1410 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1411 auip->init.aoff.crr_order = FF_AGEFF;
1412 auip->init.aoff.blk_order = FF_BF;
1413 }
1414 else if (sys_strcmp("ageffcaobf", alg) == 0) {
1415 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1416 auip->init.aoff.crr_order = FF_AGEFF;
1417 auip->init.aoff.blk_order = FF_AOBF;
1418 }
1419 else {
1420 if (auip->init.util.alloc_no == ERTS_ALC_A_TEST
1421 && sys_strcmp("chaosff", alg) == 0) {
1422 auip->astrat = ERTS_ALC_S_FIRSTFIT;
1423 auip->init.aoff.crr_order = FF_CHAOS;
1424 auip->init.aoff.blk_order = FF_CHAOS;
1425 }
1426 else {
1427 bad_value(param, sub_param + 1, alg);
1428 }
1429 }
1430 if (!strategy_support_carrier_migration(auip))
1431 auip->init.util.acul = 0;
1432 } else if (has_prefix("atags", sub_param)) {
1433 auip->init.util.atags = get_bool_value(sub_param + 5, argv, ip);
1434 }
1435 else
1436 goto bad_switch;
1437 break;
1438 case 'c': {
1439 if (has_prefix("cp", sub_param)) {
1440 char *param, *param_end, *value;
1441 int cp;
1442 if (!auip->carrier_migration_allowed && !u_switch)
1443 goto bad_switch;
1444 param = argv[*ip]+1;
1445 param_end = sub_param + 2;
1446 value = get_value(param_end, argv, ip);
1447 if (value[0] == '\0' || value[1] != '\0')
1448 bad_value(param, param_end, value);
1449 switch (value[0]) {
1450 case 'B': cp = ERTS_ALC_A_BINARY; break;
1451 case 'D': cp = ERTS_ALC_A_STANDARD; break;
1452 case 'E': cp = ERTS_ALC_A_ETS; break;
1453 case 'F': cp = ERTS_ALC_A_FIXED_SIZE; break;
1454 case 'H': cp = ERTS_ALC_A_EHEAP; break;
1455 case 'L': cp = ERTS_ALC_A_LONG_LIVED; break;
1456 case 'R': cp = ERTS_ALC_A_DRIVER; break;
1457 case 'S': cp = ERTS_ALC_A_SHORT_LIVED; break;
1458 case '@': cp = ERTS_ALC_COMMON_CPOOL_IX; break;
1459 case ':': cp = auip->init.util.alloc_no; break;
1460 default: cp = -1;
1461 bad_value(param, param_end, value);
1462 break;
1463 }
1464 if (auip->carrier_migration_allowed)
1465 auip->init.util.cp = cp;
1466 }
1467 else
1468 goto bad_switch;
1469 break;
1470 }
1471 case 'e': {
1472 int e = get_bool_value(sub_param + 1, argv, ip);
1473 if (!auip->disable_allowed && !e) {
1474 if (!u_switch)
1475 bad_value(param, sub_param + 1, "false");
1476 else
1477 ASSERT(auip->enable); /* ignore */
1478 }
1479 else auip->enable = e;
1480 break;
1481 }
1482 case 'l':
1483 if (has_prefix("lmbcs", sub_param)) {
1484 auip->default_.lmbcs = 0;
1485 auip->init.util.lmbcs = get_kb_value(sub_param + 5, argv, ip);
1486 }
1487 else
1488 goto bad_switch;
1489 break;
1490 case 'm':
1491 if (has_prefix("mbcgs", sub_param)) {
1492 auip->init.util.mbcgs = get_amount_value(sub_param + 5, argv, ip);
1493
1494 }
1495 else if (has_prefix("mbsd", sub_param)) {
1496 auip->init.gf.mbsd = get_amount_value(sub_param + 4, argv, ip);
1497 if (auip->init.gf.mbsd < 1)
1498 auip->init.gf.mbsd = 1;
1499 }
1500 else if (has_prefix("mmbcs", sub_param)) {
1501 auip->default_.mmbcs = 0;
1502 auip->init.util.mmbcs = get_kb_value(sub_param + 5, argv, ip);
1503 }
1504 else if (has_prefix("mmmbc", sub_param)) {
1505 auip->default_.mmmbc = 0;
1506 auip->init.util.mmmbc = get_amount_value(sub_param + 5, argv, ip);
1507 }
1508 else if (has_prefix("mmsbc", sub_param)) {
1509 auip->init.util.mmsbc = get_amount_value(sub_param + 5, argv, ip);
1510 }
1511 else
1512 goto bad_switch;
1513 break;
1514 case 'r':
1515 if(has_prefix("rsbcmt", sub_param)) {
1516 auip->init.util.rsbcmt = get_amount_value(sub_param + 6, argv, ip);
1517 if (auip->init.util.rsbcmt > 100)
1518 auip->init.util.rsbcmt = 100;
1519 }
1520 else if(has_prefix("rsbcst", sub_param)) {
1521 auip->init.util.rsbcst = get_amount_value(sub_param + 6, argv, ip);
1522 if (auip->init.util.rsbcst > 100)
1523 auip->init.util.rsbcst = 100;
1524 }
1525 else if (has_prefix("rmbcmt", sub_param)) {
1526 auip->init.util.rmbcmt = get_amount_value(sub_param + 6, argv, ip);
1527 if (auip->init.util.rmbcmt > 100)
1528 auip->init.util.rmbcmt = 100;
1529 }
1530 else if (has_prefix("ramv", sub_param)) {
1531 auip->init.util.ramv = get_bool_value(sub_param + 4, argv, ip);
1532 }
1533 else
1534 goto bad_switch;
1535 break;
1536 case 's':
1537 if(has_prefix("sbct", sub_param)) {
1538 auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip);
1539 }
1540 else if (has_prefix("smbcs", sub_param)) {
1541 auip->default_.smbcs = 0;
1542 auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip);
1543 }
1544 else
1545 goto bad_switch;
1546 break;
1547 case 't': {
1548 int res = get_bool_value(sub_param+1, argv, ip);
1549 if (res > 0) {
1550 if (!auip->thr_spec_allowed) {
1551 if (!u_switch)
1552 bad_value(param, sub_param + 1, "true");
1553 else
1554 ASSERT(!auip->thr_spec); /* ignore */
1555 }
1556 else
1557 auip->thr_spec = 1;
1558 break;
1559 }
1560 else if (res == 0) {
1561 auip->thr_spec = 0;
1562 auip->init.util.acul = 0;
1563 break;
1564 }
1565 goto bad_switch;
1566 }
1567 default:
1568 bad_switch:
1569 bad_param(param, sub_param);
1570 }
1571 }
1572
1573 static void
handle_args(int * argc,char ** argv,erts_alc_hndl_args_init_t * init)1574 handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
1575 {
1576 struct au_init *aui[] = {
1577 &init->binary_alloc,
1578 &init->std_alloc,
1579 &init->ets_alloc,
1580 &init->eheap_alloc,
1581 &init->ll_alloc,
1582 &init->driver_alloc,
1583 &init->fix_alloc,
1584 &init->sl_alloc
1585 /* test_alloc not affected by +Mea??? or +Mu??? */
1586 };
1587 int aui_sz = (int) sizeof(aui)/sizeof(aui[0]);
1588 char *arg;
1589 char *rest;
1590 int i, j;
1591
1592 i = 1;
1593
1594 ASSERT(argc && argv && init);
1595
1596 while (i < *argc) {
1597 if(argv[i][0] == '-') {
1598 char *param = argv[i]+1;
1599 switch (argv[i][1]) {
1600 case 'M':
1601 switch (argv[i][2]) {
1602 case 'B':
1603 handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0);
1604 break;
1605 case 'I':
1606 if (has_prefix("scs", argv[i]+3)) {
1607 #if HAVE_ERTS_MSEG
1608 init->mseg.literal_mmap.scs =
1609 #endif
1610 get_mb_value(argv[i]+6, argv, &i);
1611 }
1612 else
1613 handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0);
1614 break;
1615 case 'X':
1616 if (has_prefix("scs", argv[i]+3)) {
1617 /* Ignore obsolete */
1618 (void) get_mb_value(argv[i]+6, argv, &i);
1619 }
1620 else
1621 handle_au_arg(&init->exec_alloc, &argv[i][3], argv, &i, 0);
1622 break;
1623 case 'D':
1624 handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0);
1625 break;
1626 case 'E':
1627 handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i, 0);
1628 break;
1629 case 'F':
1630 handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i, 0);
1631 break;
1632 case 'H':
1633 handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i, 0);
1634 break;
1635 case 'L':
1636 handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i, 0);
1637 break;
1638 case 'M':
1639 if (has_prefix("amcbf", argv[i]+3)) {
1640 #if HAVE_ERTS_MSEG
1641 init->mseg.amcbf =
1642 #endif
1643 get_kb_value(argv[i]+8, argv, &i);
1644 }
1645 else if (has_prefix("rmcbf", argv[i]+3)) {
1646 #if HAVE_ERTS_MSEG
1647 init->mseg.rmcbf =
1648 #endif
1649 get_amount_value(argv[i]+8, argv, &i);
1650 }
1651 else if (has_prefix("mcs", argv[i]+3)) {
1652 #if HAVE_ERTS_MSEG
1653 init->mseg.mcs =
1654 #endif
1655 get_amount_value(argv[i]+6, argv, &i);
1656 }
1657 else if (has_prefix("scs", argv[i]+3)) {
1658 #if HAVE_ERTS_MSEG
1659 init->mseg.dflt_mmap.scs =
1660 #endif
1661 get_mb_value(argv[i]+6, argv, &i);
1662 }
1663 else if (has_prefix("sco", argv[i]+3)) {
1664 #if HAVE_ERTS_MSEG
1665 init->mseg.dflt_mmap.sco =
1666 #endif
1667 get_bool_value(argv[i]+6, argv, &i);
1668 }
1669 else if (has_prefix("scrpm", argv[i]+3)) {
1670 #if HAVE_ERTS_MSEG
1671 init->mseg.dflt_mmap.scrpm =
1672 #endif
1673 get_bool_value(argv[i]+8, argv, &i);
1674 }
1675 else if (has_prefix("scrfsd", argv[i]+3)) {
1676 #if HAVE_ERTS_MSEG
1677 init->mseg.dflt_mmap.scrfsd =
1678 #endif
1679 get_amount_value(argv[i]+9, argv, &i);
1680 }
1681 else {
1682 bad_param(param, param+2);
1683 }
1684 break;
1685 case 'R':
1686 handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i, 0);
1687 break;
1688 case 'S':
1689 handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i, 0);
1690 break;
1691 case 'T':
1692 handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0);
1693 break;
1694 case 'Z':
1695 handle_au_arg(&init->test_alloc, &argv[i][3], argv, &i, 0);
1696 break;
1697 case 'Y': { /* sys_alloc */
1698 if (has_prefix("tt", param+2)) {
1699 /* set trim threshold */
1700 arg = get_value(param+4, argv, &i);
1701 errno = 0;
1702 init->trim_threshold = (int) strtol(arg, &rest, 10);
1703 if (errno != 0
1704 || rest == arg
1705 || init->trim_threshold < 0
1706 || (INT_MAX/1024) < init->trim_threshold) {
1707 bad_value(param, param+4, arg);
1708 }
1709 VERBOSE(DEBUG_SYSTEM,
1710 ("using trim threshold: %d\n",
1711 init->trim_threshold));
1712 init->trim_threshold *= 1024;
1713 }
1714 else if (has_prefix("tp", param+2)) {
1715 /* set top pad */
1716 arg = get_value(param+4, argv, &i);
1717 errno = 0;
1718 init->top_pad = (int) strtol(arg, &rest, 10);
1719 if (errno != 0
1720 || rest == arg
1721 || init->top_pad < 0
1722 || (INT_MAX/1024) < init->top_pad) {
1723 bad_value(param, param+4, arg);
1724 }
1725 VERBOSE(DEBUG_SYSTEM,
1726 ("using top pad: %d\n",init->top_pad));
1727 init->top_pad *= 1024;
1728 }
1729 else if (has_prefix("m", param+2)) {
1730 /* Has been handled by erlexec */
1731 (void) get_value(param+3, argv, &i);
1732 }
1733 else if (has_prefix("e", param+2)) {
1734 arg = get_value(param+3, argv, &i);
1735 if (sys_strcmp("true", arg) != 0)
1736 bad_value(param, param+3, arg);
1737 }
1738 else
1739 bad_param(param, param+2);
1740 break;
1741 }
1742 case 'e':
1743 switch (argv[i][3]) {
1744 case 'a': {
1745 int a;
1746 arg = get_value(argv[i]+4, argv, &i);
1747 if (sys_strcmp("min", arg) == 0) {
1748 for (a = 0; a < aui_sz; a++)
1749 aui[a]->enable = 0;
1750 }
1751 else if (sys_strcmp("max", arg) == 0) {
1752 for (a = 0; a < aui_sz; a++)
1753 aui[a]->enable = 1;
1754 }
1755 else if (sys_strcmp("config", arg) == 0) {
1756 init->erts_alloc_config = 1;
1757 }
1758 else if (sys_strcmp("r9c", arg) == 0
1759 || sys_strcmp("r10b", arg) == 0
1760 || sys_strcmp("r11b", arg) == 0) {
1761 set_default_sl_alloc_opts(&init->sl_alloc);
1762 set_default_std_alloc_opts(&init->std_alloc);
1763 set_default_ll_alloc_opts(&init->ll_alloc);
1764 set_default_temp_alloc_opts(&init->temp_alloc);
1765 set_default_eheap_alloc_opts(&init->eheap_alloc);
1766 set_default_binary_alloc_opts(&init->binary_alloc);
1767 set_default_ets_alloc_opts(&init->ets_alloc);
1768 set_default_driver_alloc_opts(&init->driver_alloc);
1769 set_default_driver_alloc_opts(&init->fix_alloc);
1770
1771 init->driver_alloc.enable = 0;
1772 if (sys_strcmp("r9c", arg) == 0) {
1773 init->sl_alloc.enable = 0;
1774 init->std_alloc.enable = 0;
1775 init->binary_alloc.enable = 0;
1776 init->ets_alloc.enable = 0;
1777 }
1778
1779 for (a = 0; a < aui_sz; a++) {
1780 aui[a]->thr_spec = 0;
1781 aui[a]->init.util.acul = 0;
1782 aui[a]->init.util.ramv = 0;
1783 aui[a]->init.util.lmbcs = 5*1024*1024;
1784 }
1785 }
1786 else {
1787 bad_param(param, param+3);
1788 }
1789 break;
1790 }
1791 default:
1792 bad_param(param, param+1);
1793 }
1794 break;
1795 case 'i':
1796 switch (argv[i][3]) {
1797 case 't':
1798 init->instr.mtrace = get_value(argv[i]+4, argv, &i);
1799 break;
1800 default:
1801 bad_param(param, param+2);
1802 }
1803 break;
1804 case 'l':
1805 if (has_prefix("pm", param+2)) {
1806 arg = get_value(argv[i]+5, argv, &i);
1807 if (sys_strcmp("all", arg) == 0)
1808 lock_all_physical_memory = 1;
1809 else if (sys_strcmp("no", arg) == 0)
1810 lock_all_physical_memory = 0;
1811 else
1812 bad_value(param, param+4, arg);
1813 break;
1814 }
1815 bad_param(param, param+2);
1816 break;
1817 case 'u':
1818 if (has_prefix("ycs", argv[i]+3)) {
1819 init->alloc_util.ycs
1820 = get_kb_value(argv[i]+6, argv, &i);
1821 }
1822 else if (has_prefix("mmc", argv[i]+3)) {
1823 init->alloc_util.mmc
1824 = get_amount_value(argv[i]+6, argv, &i);
1825 }
1826 else if (has_prefix("sac", argv[i]+3)) {
1827 init->alloc_util.sac
1828 = get_bool_value(argv[i]+6, argv, &i);
1829 }
1830 else {
1831 int a;
1832 int start = i;
1833 char *param = argv[i];
1834 char *val = i+1 < *argc ? argv[i+1] : NULL;
1835
1836 for (a = 0; a < aui_sz; a++) {
1837 if (a > 0) {
1838 ASSERT(i == start || i == start+1);
1839 argv[start] = param;
1840 if (i != start)
1841 argv[start + 1] = val;
1842 i = start;
1843 }
1844 handle_au_arg(aui[a], &argv[i][3], argv, &i, 1);
1845 }
1846 }
1847 break;
1848 default:
1849 bad_param(param, param+1);
1850 }
1851 break;
1852 case '-':
1853 if (argv[i][2] == '\0') {
1854 /* End of system flags reached */
1855 if (init->instr.mtrace) {
1856 while (i < *argc) {
1857 if(sys_strcmp(argv[i], "-sname") == 0
1858 || sys_strcmp(argv[i], "-name") == 0) {
1859 if (i + 1 <*argc) {
1860 init->instr.nodename = argv[i+1];
1861 break;
1862 }
1863 }
1864 i++;
1865 }
1866 }
1867 goto args_parsed;
1868 }
1869 break;
1870 default:
1871 break;
1872 }
1873 }
1874 i++;
1875 }
1876
1877 args_parsed:
1878 /* Handled arguments have been marked with NULL. Slide arguments
1879 not handled towards the beginning of argv. */
1880 for (i = 0, j = 0; i < *argc; i++) {
1881 if (argv[i])
1882 argv[j++] = argv[i];
1883 }
1884 *argc = j;
1885 }
1886
type_no_str(ErtsAlcType_t n)1887 static char *type_no_str(ErtsAlcType_t n)
1888 {
1889
1890 #if ERTS_ALC_N_MIN != 0
1891 if (n < ERTS_ALC_N_MIN)
1892 return NULL;
1893 #endif
1894 if (n > ERTS_ALC_N_MAX)
1895 return NULL;
1896 return (char *) ERTS_ALC_N2TD(n);
1897 }
1898
1899 #define type_str(T) type_no_str(ERTS_ALC_T2N((T)))
1900
1901 void
erts_alloc_register_scheduler(void * vesdp)1902 erts_alloc_register_scheduler(void *vesdp)
1903 {
1904 ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
1905 int ix = (int) esdp->no;
1906 int aix;
1907
1908 ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
1909 for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
1910 ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
1911 esdp->alloc_data.deallctr[aix] = NULL;
1912 esdp->alloc_data.pref_ix[aix] = -1;
1913 if (tspec->enabled) {
1914 if (!tspec->dd)
1915 esdp->alloc_data.pref_ix[aix] = ix;
1916 else {
1917 Allctr_t *allctr = tspec->allctr[ix];
1918 ASSERT(allctr);
1919 esdp->alloc_data.deallctr[aix] = allctr;
1920 esdp->alloc_data.pref_ix[aix] = ix;
1921 }
1922 }
1923 }
1924 }
1925
1926 void
erts_alloc_scheduler_handle_delayed_dealloc(void * vesdp,int * need_thr_progress,ErtsThrPrgrVal * thr_prgr_p,int * more_work)1927 erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
1928 int *need_thr_progress,
1929 ErtsThrPrgrVal *thr_prgr_p,
1930 int *more_work)
1931 {
1932 ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
1933 int aix;
1934 for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
1935 Allctr_t *allctr;
1936 if (esdp)
1937 allctr = esdp->alloc_data.deallctr[aix];
1938 else {
1939 ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
1940 if (tspec->enabled && tspec->dd)
1941 allctr = tspec->allctr[0];
1942 else
1943 allctr = NULL;
1944 }
1945 if (allctr) {
1946 erts_alcu_check_delayed_dealloc(allctr,
1947 1,
1948 need_thr_progress,
1949 thr_prgr_p,
1950 more_work);
1951 }
1952 }
1953 }
1954
1955 erts_aint32_t
erts_alloc_fix_alloc_shrink(int ix,erts_aint32_t flgs)1956 erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
1957 {
1958 ErtsAllocatorThrSpec_t *tspec;
1959 tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
1960 if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec && tspec->enabled)
1961 return erts_alcu_fix_alloc_shrink(tspec->allctr[ix], flgs);
1962 if (ix == 0 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
1963 return erts_alcu_fix_alloc_shrink(
1964 erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
1965 return 0;
1966 }
1967
1968 static void
no_verify(Allctr_t * allctr)1969 no_verify(Allctr_t *allctr)
1970 {
1971
1972 }
1973
1974 erts_alloc_verify_func_t
erts_alloc_get_verify_unused_temp_alloc(Allctr_t ** allctr)1975 erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr)
1976 {
1977 if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util
1978 && erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) {
1979 ErtsAllocatorThrSpec_t *tspec;
1980 int ix = ERTS_ALC_GET_THR_IX();
1981 tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY];
1982
1983 if (ix < tspec->size) {
1984 *allctr = tspec->allctr[ix];
1985 return erts_alcu_verify_unused;
1986 }
1987 }
1988
1989 *allctr = NULL;
1990 return no_verify;
1991 }
1992
1993 __decl_noreturn void
erts_alc_fatal_error(int error,int func,ErtsAlcType_t n,...)1994 erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...)
1995 {
1996 char buf[10];
1997 char *t_str;
1998 char *allctr_str;
1999
2000 ASSERT(n >= ERTS_ALC_N_MIN);
2001 ASSERT(n <= ERTS_ALC_N_MAX);
2002
2003
2004 if (n < ERTS_ALC_N_MIN || ERTS_ALC_N_MAX < n)
2005 allctr_str = "UNKNOWN";
2006 else {
2007 ErtsAlcType_t a = ERTS_ALC_T2A(ERTS_ALC_N2T(n));
2008 if (erts_allctrs_info[a].enabled)
2009 allctr_str = (char *) ERTS_ALC_A2AD(a);
2010 else
2011 allctr_str = (char *) ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM);
2012 }
2013
2014 t_str = type_no_str(n);
2015 if (!t_str) {
2016 erts_snprintf(buf, sizeof(buf), "%d", (int) n);
2017 t_str = buf;
2018 }
2019
2020 switch (error) {
2021 case ERTS_ALC_E_NOTSUP: {
2022 char *op_str;
2023 switch (func) {
2024 case ERTS_ALC_O_ALLOC: op_str = "alloc"; break;
2025 case ERTS_ALC_O_REALLOC: op_str = "realloc"; break;
2026 case ERTS_ALC_O_FREE: op_str = "free"; break;
2027 default: op_str = "UNKNOWN"; break;
2028 }
2029 erts_exit(ERTS_ABORT_EXIT,
2030 "%s: %s operation not supported (memory type: \"%s\")\n",
2031 allctr_str, op_str, t_str);
2032 break;
2033 }
2034 case ERTS_ALC_E_NOMEM: {
2035 Uint size;
2036 va_list argp;
2037 char *op = func == ERTS_ALC_O_REALLOC ? "reallocate" : "allocate";
2038
2039
2040 va_start(argp, n);
2041 size = va_arg(argp, Uint);
2042 va_end(argp);
2043 erts_exit(ERTS_DUMP_EXIT,
2044 "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n",
2045 allctr_str, op, size, t_str);
2046 break;
2047 }
2048 case ERTS_ALC_E_NOALLCTR:
2049 erts_exit(ERTS_ABORT_EXIT,
2050 "erts_alloc: Unknown allocator type: %d\n",
2051 ERTS_ALC_T2A(ERTS_ALC_N2T(n)));
2052 break;
2053 default:
2054 erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error);
2055 break;
2056 }
2057 }
2058
2059 __decl_noreturn void
erts_alloc_enomem(ErtsAlcType_t type,Uint size)2060 erts_alloc_enomem(ErtsAlcType_t type, Uint size)
2061 {
2062 erts_alloc_n_enomem(ERTS_ALC_T2N(type), size);
2063 }
2064
2065 __decl_noreturn void
erts_alloc_n_enomem(ErtsAlcType_t n,Uint size)2066 erts_alloc_n_enomem(ErtsAlcType_t n, Uint size)
2067 {
2068 erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_ALLOC, n, size);
2069 }
2070
2071 __decl_noreturn void
erts_realloc_enomem(ErtsAlcType_t type,void * ptr,Uint size)2072 erts_realloc_enomem(ErtsAlcType_t type, void *ptr, Uint size)
2073 {
2074 erts_realloc_n_enomem(ERTS_ALC_T2N(type), ptr, size);
2075 }
2076
2077 __decl_noreturn void
erts_realloc_n_enomem(ErtsAlcType_t n,void * ptr,Uint size)2078 erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size)
2079 {
2080 erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size);
2081 }
2082
2083 static ERTS_INLINE UWord
alcu_size(ErtsAlcType_t alloc_no,ErtsAlcUFixInfo_t * fi,int fisz)2084 alcu_size(ErtsAlcType_t alloc_no, ErtsAlcUFixInfo_t *fi, int fisz)
2085 {
2086 UWord res;
2087 int ai;
2088
2089 if (!erts_allctrs_info[alloc_no].thr_spec) {
2090 AllctrSize_t size;
2091 Allctr_t *allctr;
2092
2093 allctr = erts_allctrs_info[alloc_no].extra;
2094 erts_alcu_current_size(allctr, &size, fi, fisz);
2095
2096 return size.blocks;
2097 }
2098
2099 res = 0;
2100
2101 /* Thread-specific allocators can migrate carriers across types, so we have
2102 * to visit every allocator type to gather information on blocks that were
2103 * allocated by us. */
2104 for (ai = ERTS_ALC_A_MIN; ai < ERTS_ALC_A_MAX; ai++) {
2105 ErtsAllocatorThrSpec_t *tspec;
2106 Allctr_t *allctr;
2107 int i;
2108
2109 if (!erts_allctrs_info[ai].thr_spec) {
2110 continue;
2111 }
2112
2113 tspec = &erts_allctr_thr_spec[ai];
2114 ASSERT(tspec->enabled);
2115
2116 for (i = tspec->size - 1; i >= 0; i--) {
2117 allctr = tspec->allctr[i];
2118
2119 if (allctr) {
2120 AllctrSize_t size;
2121
2122 if (ai == alloc_no) {
2123 erts_alcu_current_size(allctr, &size, fi, fisz);
2124 } else {
2125 erts_alcu_foreign_size(allctr, alloc_no, &size);
2126 }
2127
2128 ASSERT(((SWord)size.blocks) >= 0);
2129
2130 res += size.blocks;
2131 }
2132 }
2133 }
2134
2135 return res;
2136 }
2137
2138 static ERTS_INLINE void
add_fix_values(UWord * ap,UWord * up,ErtsAlcUFixInfo_t * fi,ErtsAlcType_t type)2139 add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type)
2140 {
2141 int ix = ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE;
2142 ASSERT(0 <= ix && ix < ERTS_ALC_NO_FIXED_SIZES);
2143
2144 *ap += (UWord) fi[ix].allocated;
2145 *up += (UWord) fi[ix].used;
2146 }
2147
2148 Eterm
erts_memory(fmtfn_t * print_to_p,void * print_to_arg,void * proc,Eterm earg)2149 erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
2150 {
2151 /*
2152 * NOTE! When updating this function, make sure to also update
2153 * erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl
2154 */
2155 #define ERTS_MEM_NEED_ALL_ALCU (want_tot_or_sys)
2156 struct {
2157 int total;
2158 int processes;
2159 int processes_used;
2160 int system;
2161 int atom;
2162 int atom_used;
2163 int binary;
2164 int code;
2165 int ets;
2166 } want = {0};
2167 struct {
2168 UWord total;
2169 UWord processes;
2170 UWord processes_used;
2171 UWord system;
2172 UWord atom;
2173 UWord atom_used;
2174 UWord binary;
2175 UWord code;
2176 UWord ets;
2177 } size = {0};
2178 Eterm atoms[sizeof(size)/sizeof(UWord)];
2179 UWord *uintps[sizeof(size)/sizeof(UWord)];
2180 Eterm euints[sizeof(size)/sizeof(UWord)];
2181 int want_tot_or_sys;
2182 int length;
2183 Eterm res = THE_NON_VALUE;
2184 ErtsAlcType_t ai;
2185 int only_one_value = 0;
2186 ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}};
2187
2188 ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
2189
2190 /* Figure out whats wanted... */
2191
2192 length = 0;
2193 if (is_non_value(earg)) { /* i.e. wants all */
2194 want.total = 1;
2195 atoms[length] = am_total;
2196 uintps[length++] = &size.total;
2197
2198 want.processes = 1;
2199 atoms[length] = am_processes;
2200 uintps[length++] = &size.processes;
2201
2202 want.processes_used = 1;
2203 atoms[length] = am_processes_used;
2204 uintps[length++] = &size.processes_used;
2205
2206 want.system = 1;
2207 atoms[length] = am_system;
2208 uintps[length++] = &size.system;
2209
2210 want.atom = 1;
2211 atoms[length] = am_atom;
2212 uintps[length++] = &size.atom;
2213
2214 want.atom_used = 1;
2215 atoms[length] = am_atom_used;
2216 uintps[length++] = &size.atom_used;
2217
2218 want.binary = 1;
2219 atoms[length] = am_binary;
2220 uintps[length++] = &size.binary;
2221
2222 want.code = 1;
2223 atoms[length] = am_code;
2224 uintps[length++] = &size.code;
2225
2226 want.ets = 1;
2227 atoms[length] = am_ets;
2228 uintps[length++] = &size.ets;
2229 }
2230 else {
2231 DeclareTmpHeapNoproc(tmp_heap,2);
2232 Eterm wanted_list;
2233
2234 if (is_nil(earg))
2235 return NIL;
2236
2237 UseTmpHeapNoproc(2);
2238 if (is_not_atom(earg))
2239 wanted_list = earg;
2240 else {
2241 wanted_list = CONS(&tmp_heap[0], earg, NIL);
2242 only_one_value = 1;
2243 }
2244
2245 while (is_list(wanted_list)) {
2246 switch (CAR(list_val(wanted_list))) {
2247 case am_total:
2248 if (!want.total) {
2249 want.total = 1;
2250 atoms[length] = am_total;
2251 uintps[length++] = &size.total;
2252 }
2253 break;
2254 case am_processes:
2255 if (!want.processes) {
2256 want.processes = 1;
2257 atoms[length] = am_processes;
2258 uintps[length++] = &size.processes;
2259 }
2260 break;
2261 case am_processes_used:
2262 if (!want.processes_used) {
2263 want.processes_used = 1;
2264 atoms[length] = am_processes_used;
2265 uintps[length++] = &size.processes_used;
2266 }
2267 break;
2268 case am_system:
2269 if (!want.system) {
2270 want.system = 1;
2271 atoms[length] = am_system;
2272 uintps[length++] = &size.system;
2273 }
2274 break;
2275 case am_atom:
2276 if (!want.atom) {
2277 want.atom = 1;
2278 atoms[length] = am_atom;
2279 uintps[length++] = &size.atom;
2280 }
2281 break;
2282 case am_atom_used:
2283 if (!want.atom_used) {
2284 want.atom_used = 1;
2285 atoms[length] = am_atom_used;
2286 uintps[length++] = &size.atom_used;
2287 }
2288 break;
2289 case am_binary:
2290 if (!want.binary) {
2291 want.binary = 1;
2292 atoms[length] = am_binary;
2293 uintps[length++] = &size.binary;
2294 }
2295 break;
2296 case am_code:
2297 if (!want.code) {
2298 want.code = 1;
2299 atoms[length] = am_code;
2300 uintps[length++] = &size.code;
2301 }
2302 break;
2303 case am_ets:
2304 if (!want.ets) {
2305 want.ets = 1;
2306 atoms[length] = am_ets;
2307 uintps[length++] = &size.ets;
2308 }
2309 break;
2310 default:
2311 UnUseTmpHeapNoproc(2);
2312 return am_badarg;
2313 }
2314 wanted_list = CDR(list_val(wanted_list));
2315 }
2316 UnUseTmpHeapNoproc(2);
2317 if (is_not_nil(wanted_list))
2318 return am_badarg;
2319 }
2320
2321 /* All alloc_util allocators *have* to be enabled, except test_alloc */
2322
2323 for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
2324 switch (ai) {
2325 case ERTS_ALC_A_SYSTEM:
2326 case ERTS_ALC_A_TEST:
2327 break;
2328 default:
2329 if (!erts_allctrs_info[ai].enabled
2330 || !erts_allctrs_info[ai].alloc_util) {
2331 return am_notsup;
2332 }
2333 break;
2334 }
2335 }
2336
2337 ASSERT(length <= sizeof(atoms)/sizeof(Eterm));
2338 ASSERT(length <= sizeof(euints)/sizeof(Eterm));
2339 ASSERT(length <= sizeof(uintps)/sizeof(UWord));
2340
2341
2342 if (proc) {
2343 ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN
2344 == erts_proc_lc_my_proc_locks(proc));
2345 /* We'll need locks early in the lock order */
2346 erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
2347 }
2348
2349 /* Calculate values needed... */
2350
2351 want_tot_or_sys = want.total || want.system;
2352
2353 if (ERTS_MEM_NEED_ALL_ALCU) {
2354 size.total = 0;
2355
2356 for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
2357 if (erts_allctrs_info[ai].alloc_util) {
2358 UWord *save;
2359 UWord asz;
2360 switch (ai) {
2361 case ERTS_ALC_A_TEMPORARY:
2362 /*
2363 * Often not thread safe and usually never
2364 * contain any allocated memory.
2365 */
2366 continue;
2367 case ERTS_ALC_A_TEST:
2368 continue;
2369 case ERTS_ALC_A_EHEAP:
2370 save = &size.processes;
2371 break;
2372 case ERTS_ALC_A_ETS:
2373 save = &size.ets;
2374 break;
2375 case ERTS_ALC_A_BINARY:
2376 save = &size.binary;
2377 break;
2378 case ERTS_ALC_A_FIXED_SIZE:
2379 asz = alcu_size(ai, fi, ERTS_ALC_NO_FIXED_SIZES);
2380 size.total += asz;
2381 continue;
2382 default:
2383 save = NULL;
2384 break;
2385 }
2386 asz = alcu_size(ai, NULL, 0);
2387 if (save)
2388 *save = asz;
2389 size.total += asz;
2390 }
2391 }
2392 }
2393
2394
2395
2396 if (want_tot_or_sys || want.processes || want.processes_used) {
2397 UWord tmp;
2398
2399 if (ERTS_MEM_NEED_ALL_ALCU)
2400 tmp = size.processes;
2401 else {
2402 alcu_size(ERTS_ALC_A_FIXED_SIZE,
2403 fi, ERTS_ALC_NO_FIXED_SIZES);
2404 tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0);
2405 }
2406 tmp += erts_ptab_mem_size(&erts_proc);
2407 tmp += erts_bif_timer_memory_size();
2408
2409 size.processes = size.processes_used = tmp;
2410
2411 add_fix_values(&size.processes,
2412 &size.processes_used,
2413 fi,
2414 ERTS_ALC_T_PROC);
2415 add_fix_values(&size.processes,
2416 &size.processes_used,
2417 fi,
2418 ERTS_ALC_T_MONITOR);
2419 add_fix_values(&size.processes,
2420 &size.processes_used,
2421 fi,
2422 ERTS_ALC_T_LINK);
2423 add_fix_values(&size.processes,
2424 &size.processes_used,
2425 fi,
2426 ERTS_ALC_T_MSG_REF);
2427 add_fix_values(&size.processes,
2428 &size.processes_used,
2429 fi,
2430 ERTS_ALC_T_LL_PTIMER);
2431 add_fix_values(&size.processes,
2432 &size.processes_used,
2433 fi,
2434 ERTS_ALC_T_HL_PTIMER);
2435 add_fix_values(&size.processes,
2436 &size.processes_used,
2437 fi,
2438 ERTS_ALC_T_BIF_TIMER);
2439 }
2440
2441 if (want.atom || want.atom_used) {
2442 Uint reserved_atom_space, atom_space;
2443 erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space);
2444 size.atom = size.atom_used = atom_table_sz();
2445
2446 if (want.atom)
2447 size.atom += reserved_atom_space;
2448
2449 if (want.atom_used)
2450 size.atom_used += atom_space;
2451 }
2452
2453 if (!ERTS_MEM_NEED_ALL_ALCU && want.binary)
2454 size.binary = alcu_size(ERTS_ALC_A_BINARY, NULL, 0);
2455
2456 if (want.code) {
2457 size.code = module_table_sz();
2458 size.code += export_table_sz();
2459 size.code += export_entries_sz();
2460 size.code += erts_fun_table_sz();
2461 size.code += erts_ranges_sz();
2462 size.code += erts_total_code_size;
2463 }
2464
2465 if (want.ets) {
2466 if (!ERTS_MEM_NEED_ALL_ALCU)
2467 size.ets = alcu_size(ERTS_ALC_A_ETS, NULL, 0);
2468 size.ets += erts_get_ets_misc_mem_size();
2469 }
2470
2471 if (want_tot_or_sys) {
2472 ASSERT(size.total >= size.processes);
2473 size.system = size.total - size.processes;
2474 }
2475
2476 if (print_to_p) {
2477 int i;
2478 fmtfn_t to = *print_to_p;
2479 void *arg = print_to_arg;
2480
2481 /* Print result... */
2482 erts_print(to, arg, "=memory\n");
2483 for (i = 0; i < length; i++)
2484 erts_print(to, arg, "%T: %bpu\n", atoms[i], *uintps[i]);
2485 }
2486
2487 if (proc) {
2488 /* Build erlang term result... */
2489 Uint *hp;
2490 Uint hsz;
2491
2492 erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
2493
2494 if (only_one_value) {
2495 ASSERT(length == 1);
2496 hsz = 0;
2497 erts_bld_uword(NULL, &hsz, *uintps[0]);
2498 hp = hsz ? HAlloc((Process *) proc, hsz) : NULL;
2499 res = erts_bld_uword(&hp, NULL, *uintps[0]);
2500 }
2501 else {
2502 Uint **hpp = NULL;
2503 Uint *hszp = &hsz;
2504 hsz = 0;
2505
2506 while (1) {
2507 int i;
2508 for (i = 0; i < length; i++)
2509 euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]);
2510 res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints);
2511 if (hpp)
2512 break;
2513 hp = HAlloc((Process *) proc, hsz);
2514 hpp = &hp;
2515 hszp = NULL;
2516 }
2517 }
2518 }
2519
2520 return res;
2521
2522 #undef ERTS_MEM_NEED_ALL_ALCU
2523 }
2524
2525 struct aa_values {
2526 Uint arity;
2527 const char *name;
2528 Uint ui[2];
2529 };
2530
2531 Eterm
erts_allocated_areas(fmtfn_t * print_to_p,void * print_to_arg,void * proc)2532 erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc)
2533 {
2534 #define MAX_AA_VALUES (24)
2535 struct aa_values values[MAX_AA_VALUES];
2536 Eterm res = THE_NON_VALUE;
2537 int i, length;
2538 Uint reserved_atom_space, atom_space;
2539
2540 if (proc) {
2541 ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN
2542 == erts_proc_lc_my_proc_locks(proc));
2543
2544 /* We'll need locks early in the lock order */
2545 erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
2546 }
2547
2548 i = 0;
2549
2550 values[i].arity = 2;
2551 values[i].name = "sys_misc";
2552 values[i].ui[0] = erts_sys_misc_mem_sz();
2553 i++;
2554
2555 values[i].arity = 2;
2556 values[i].name = "static";
2557 values[i].ui[0] =
2558 sizeof(ErtsPTab)*2 /* proc & port tables */
2559 + erts_timer_wheel_memory_size(); /* Timer wheel */
2560 i++;
2561
2562 erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space);
2563
2564 values[i].arity = 3;
2565 values[i].name = "atom_space";
2566 values[i].ui[0] = reserved_atom_space;
2567 values[i].ui[1] = atom_space;
2568 i++;
2569
2570 values[i].arity = 2;
2571 values[i].name = "atom_table";
2572 values[i].ui[0] = atom_table_sz();
2573 i++;
2574
2575 values[i].arity = 2;
2576 values[i].name = "module_table";
2577 values[i].ui[0] = module_table_sz();
2578 i++;
2579
2580 values[i].arity = 2;
2581 values[i].name = "export_table";
2582 values[i].ui[0] = export_table_sz();
2583 i++;
2584
2585 values[i].arity = 2;
2586 values[i].name = "export_list";
2587 values[i].ui[0] = export_entries_sz();
2588 i++;
2589
2590 values[i].arity = 2;
2591 values[i].name = "register_table";
2592 values[i].ui[0] = process_reg_sz();
2593 i++;
2594
2595 values[i].arity = 2;
2596 values[i].name = "fun_table";
2597 values[i].ui[0] = erts_fun_table_sz();
2598 i++;
2599
2600 values[i].arity = 2;
2601 values[i].name = "module_refs";
2602 values[i].ui[0] = erts_ranges_sz();
2603 i++;
2604
2605 values[i].arity = 2;
2606 values[i].name = "loaded_code";
2607 values[i].ui[0] = erts_total_code_size;
2608 i++;
2609
2610 values[i].arity = 2;
2611 values[i].name = "dist_table";
2612 values[i].ui[0] = erts_dist_table_size();
2613 i++;
2614
2615 values[i].arity = 2;
2616 values[i].name = "node_table";
2617 values[i].ui[0] = erts_node_table_size();
2618 i++;
2619
2620 values[i].arity = 2;
2621 values[i].name = "bits_bufs_size";
2622 values[i].ui[0] = erts_bits_bufs_size();
2623 i++;
2624
2625 values[i].arity = 2;
2626 values[i].name = "bif_timer";
2627 values[i].ui[0] = erts_bif_timer_memory_size();
2628 i++;
2629
2630 values[i].arity = 2;
2631 values[i].name = "process_table";
2632 values[i].ui[0] = erts_ptab_mem_size(&erts_proc);
2633 i++;
2634
2635 values[i].arity = 2;
2636 values[i].name = "port_table";
2637 values[i].ui[0] = erts_ptab_mem_size(&erts_port);
2638 i++;
2639
2640 values[i].arity = 2;
2641 values[i].name = "ets_misc";
2642 values[i].ui[0] = erts_get_ets_misc_mem_size();
2643 i++;
2644
2645 length = i;
2646 ASSERT(length <= MAX_AA_VALUES);
2647
2648 if (print_to_p) {
2649 /* Print result... */
2650 fmtfn_t to = *print_to_p;
2651 void *arg = print_to_arg;
2652
2653 erts_print(to, arg, "=allocated_areas\n");
2654 for (i = 0; i < length; i++) {
2655 switch (values[i].arity) {
2656 case 2:
2657 erts_print(to, arg, "%s: %beu\n",
2658 values[i].name, values[i].ui[0]);
2659 break;
2660 case 3:
2661 erts_print(to, arg, "%s: %beu %beu\n",
2662 values[i].name, values[i].ui[0], values[i].ui[1]);
2663 break;
2664 default:
2665 erts_print(to, arg, "ERROR: internal_error\n");
2666 ASSERT(0);
2667 return am_internal_error;
2668 }
2669 }
2670 }
2671
2672 if (proc) {
2673 /* Build erlang term result... */
2674 Eterm tuples[MAX_AA_VALUES];
2675 Uint *hp;
2676 Uint **hpp;
2677 Uint hsz;
2678 Uint *hszp;
2679
2680 erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
2681
2682 hpp = NULL;
2683 hsz = 0;
2684 hszp = &hsz;
2685
2686 while (1) {
2687 int i;
2688 for (i = 0; i < length; i++) {
2689 Eterm atom;
2690 if (hpp)
2691 atom = am_atom_put(values[i].name,
2692 (int) sys_strlen(values[i].name));
2693 else
2694 atom = am_true;
2695
2696 switch (values[i].arity) {
2697 case 2:
2698 tuples[i] = erts_bld_tuple(hpp, hszp, 2,
2699 atom,
2700 erts_bld_uint(hpp, hszp,
2701 values[i].ui[0]));
2702 break;
2703 case 3:
2704 tuples[i] = erts_bld_tuple(hpp, hszp, 3,
2705 atom,
2706 erts_bld_uint(hpp, hszp,
2707 values[i].ui[0]),
2708 erts_bld_uint(hpp, hszp,
2709 values[i].ui[1]));
2710 break;
2711 default:
2712 ASSERT(0);
2713 return am_internal_error;
2714 }
2715 }
2716 res = erts_bld_list(hpp, hszp, length, tuples);
2717 if (hpp)
2718 break;
2719 hp = HAlloc((Process *) proc, hsz);
2720 hpp = &hp;
2721 hszp = NULL;
2722 }
2723 }
2724
2725 return res;
2726 #undef MAX_AA_VALUES
2727 }
2728
2729 Eterm
erts_alloc_util_allocators(void * proc)2730 erts_alloc_util_allocators(void *proc)
2731 {
2732 Eterm res;
2733 Uint *hp;
2734 Uint sz;
2735 int i;
2736 /*
2737 * Currently all allocators except sys_alloc are
2738 * alloc_util allocators.
2739 * Also hide test_alloc which is disabled by default
2740 * and only intended for our own testing.
2741 */
2742 sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2;
2743 ASSERT(sz > 0);
2744 hp = HAlloc((Process *) proc, sz);
2745 res = NIL;
2746 for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) {
2747 switch (i) {
2748 case ERTS_ALC_A_SYSTEM:
2749 case ERTS_ALC_A_TEST:
2750 break;
2751 default: {
2752 char *alc_str = (char *) ERTS_ALC_A2AD(i);
2753 Eterm alc = am_atom_put(alc_str, sys_strlen(alc_str));
2754 res = CONS(hp, alc, res);
2755 hp += 2;
2756 break;
2757 }
2758 }
2759 }
2760 return res;
2761 }
2762
2763 void
erts_allocator_info(fmtfn_t to,void * arg)2764 erts_allocator_info(fmtfn_t to, void *arg)
2765 {
2766 ErtsAlcType_t a;
2767
2768 ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
2769
2770 for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
2771 int ai;
2772 for (ai = 0; ai == 0 || ai < erts_allctrs_info[a].thr_spec; ai++) {
2773 if (erts_allctrs_info[a].thr_spec) {
2774 if (!erts_allctr_thr_spec[a].allctr[ai])
2775 continue;
2776 erts_print(to, arg, "=allocator:%s[%d]\n",
2777 ERTS_ALC_A2AD(a), ai);
2778 }
2779 else {
2780 erts_print(to, arg, "=allocator:%s\n", ERTS_ALC_A2AD(a));
2781 }
2782 if (!erts_allctrs_info[a].enabled)
2783 erts_print(to, arg, "option e: false\n");
2784 else {
2785 if (erts_allctrs_info[a].alloc_util) {
2786 void *as;
2787 if (!erts_allctrs_info[a].thr_spec)
2788 as = erts_allctrs_info[a].extra;
2789 else {
2790 ASSERT(erts_allctr_thr_spec[a].enabled);
2791 as = erts_allctr_thr_spec[a].allctr[ai];
2792 }
2793 /* Binary alloc has its own thread safety... */
2794 erts_alcu_info(as, 0, 0, &to, arg, NULL, NULL);
2795 }
2796 else {
2797 switch (a) {
2798 case ERTS_ALC_A_SYSTEM: {
2799 SysAllocStat sas;
2800 erts_print(to, arg, "option e: true\n");
2801 erts_print(to, arg, "option m: libc\n");
2802 sys_alloc_stat(&sas);
2803 if(sas.trim_threshold >= 0)
2804 erts_print(to, arg, "option tt: %d\n", sas.trim_threshold);
2805 if(sas.top_pad >= 0)
2806 erts_print(to, arg, "option tp: %d\n", sas.top_pad);
2807 break;
2808 }
2809 default:
2810 ASSERT(0);
2811 break;
2812 }
2813 }
2814 }
2815 }
2816 }
2817
2818 #if HAVE_ERTS_MSEG
2819 {
2820 struct erts_mmap_info_struct emis;
2821 int max = (int) erts_no_schedulers;
2822 int i;
2823 for (i = 0; i <= max; i++) {
2824 erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
2825 erts_mseg_info(i, &to, arg, 0, 0, NULL, NULL);
2826 }
2827 erts_print(to, arg, "=allocator:erts_mmap.default_mmap\n");
2828 erts_mmap_info(&erts_dflt_mmapper, &to, arg, NULL, NULL, &emis);
2829 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
2830 erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n");
2831 erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis);
2832 #endif
2833 }
2834 #endif
2835
2836 erts_print(to, arg, "=allocator:alloc_util\n");
2837 erts_alcu_au_info_options(&to, arg, NULL, NULL);
2838
2839 erts_print(to, arg, "=allocator:instr\n");
2840
2841 erts_print(to, arg, "option t: %s\n",
2842 erts_mtrace_enabled ? "true" : "false");
2843
2844 }
2845
2846 Eterm
erts_allocator_options(void * proc)2847 erts_allocator_options(void *proc)
2848 {
2849 #if HAVE_ERTS_MSEG
2850 int use_mseg = 0;
2851 #endif
2852 Uint sz, *szp, *hp, **hpp;
2853 Eterm res, features, settings;
2854 Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7];
2855 Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7];
2856 int a, length;
2857 SysAllocStat sas;
2858 Uint *endp = NULL;
2859
2860 sys_alloc_stat(&sas);
2861
2862 /* First find out the heap size needed ... */
2863 hpp = NULL;
2864 szp = &sz;
2865 sz = 0;
2866
2867 bld_term:
2868
2869 length = 0;
2870 features = NIL;
2871 settings = NIL;
2872
2873 for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
2874 Eterm tmp = NIL;
2875 atoms[length] = am_atom_put((char *) ERTS_ALC_A2AD(a),
2876 sys_strlen(ERTS_ALC_A2AD(a)));
2877 if (erts_allctrs_info[a].enabled) {
2878 if (erts_allctrs_info[a].alloc_util) {
2879 Allctr_t *allctr;
2880 #if HAVE_ERTS_MSEG
2881 use_mseg++;
2882 #endif
2883 if (erts_allctr_thr_spec[a].enabled)
2884 allctr = erts_allctr_thr_spec[a].allctr[0];
2885 else
2886 allctr = erts_allctrs_info[a].extra;
2887 tmp = erts_alcu_info_options(allctr, NULL, NULL, hpp, szp);
2888 }
2889 else {
2890 int l = 0;
2891 Eterm as[4];
2892 Eterm ts[4];
2893
2894 as[l] = ERTS_MAKE_AM("e");
2895 ts[l++] = am_true;
2896
2897 switch (a) {
2898 case ERTS_ALC_A_SYSTEM:
2899 as[l] = ERTS_MAKE_AM("m");
2900 ts[l++] = ERTS_MAKE_AM("libc");
2901 if(sas.trim_threshold >= 0) {
2902 as[l] = ERTS_MAKE_AM("tt");
2903 ts[l++] = erts_bld_uint(hpp, szp,
2904 (Uint) sas.trim_threshold);
2905 }
2906 if(sas.top_pad >= 0) {
2907 as[l] = ERTS_MAKE_AM("tp");
2908 ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad);
2909 }
2910 break;
2911 default:
2912 break;
2913 }
2914
2915 tmp = erts_bld_2tup_list(hpp, szp, l, as, ts);
2916
2917 }
2918
2919 }
2920 else {
2921 Eterm atom = ERTS_MAKE_AM("e");
2922 Eterm term = am_false;
2923 tmp = erts_bld_2tup_list(hpp, szp, 1, &atom, &term);
2924 }
2925
2926 terms[length++] = tmp;
2927
2928 }
2929
2930 #if HAVE_ERTS_MSEG
2931 if (use_mseg) {
2932 atoms[length] = ERTS_MAKE_AM("mseg_alloc");
2933 terms[length++] = erts_mseg_info_options(0, NULL, NULL, hpp, szp);
2934 }
2935 #endif
2936
2937 atoms[length] = ERTS_MAKE_AM("alloc_util");
2938 terms[length++] = erts_alcu_au_info_options(NULL, NULL, hpp, szp);
2939
2940 #if HAVE_ERTS_MMAP
2941 atoms[length] = ERTS_MAKE_AM("erts_mmap");
2942 terms[length++] = erts_mmap_info_options(&erts_dflt_mmapper, NULL, NULL,
2943 NULL, hpp, szp);
2944 #endif
2945 {
2946 Eterm o[1], v[1];
2947 o[0] = ERTS_MAKE_AM("t");
2948 v[0] = erts_mtrace_enabled ? am_true : am_false;
2949
2950 atoms[length] = ERTS_MAKE_AM("instr");
2951 terms[length++] = erts_bld_2tup_list(hpp, szp, 1, o, v);
2952 }
2953
2954 atoms[length] = ERTS_MAKE_AM("lock_physical_memory");
2955 terms[length++] = (lock_all_physical_memory ? am_all : am_no);
2956
2957 settings = erts_bld_2tup_list(hpp, szp, length, atoms, terms);
2958
2959 length = 0;
2960
2961 for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
2962 if (erts_allctrs_info[a].enabled) {
2963 terms[length++] = am_atom_put((char *) ERTS_ALC_A2AD(a),
2964 sys_strlen(ERTS_ALC_A2AD(a)));
2965 }
2966 }
2967
2968 #if HAVE_ERTS_MSEG
2969 if (use_mseg)
2970 terms[length++] = ERTS_MAKE_AM("mseg_alloc");
2971 #endif
2972 #if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
2973 terms[length++] = ERTS_MAKE_AM("sys_aligned_alloc");
2974 #endif
2975 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
2976 terms[length++] = ERTS_MAKE_AM("literal_mmap");
2977 #endif
2978 features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
2979
2980 #if defined(__GLIBC__)
2981 {
2982 Eterm AM_glibc = ERTS_MAKE_AM("glibc");
2983 Eterm version;
2984
2985 version = erts_bld_cons(hpp,
2986 szp,
2987 make_small(__GLIBC__),
2988 #ifdef __GLIBC_MINOR__
2989 erts_bld_cons(hpp,
2990 szp,
2991 make_small(__GLIBC_MINOR__),
2992 NIL)
2993 #else
2994 NIL
2995 #endif
2996 );
2997
2998 res = erts_bld_tuple(hpp, szp, 4,
2999 AM_glibc, version, features, settings);
3000 }
3001
3002 #else /* unknown allocator */
3003
3004 res = erts_bld_tuple(hpp, szp, 4,
3005 am_undefined, NIL, features, settings);
3006
3007 #endif
3008
3009 if (szp) {
3010 /* ... and then build the term */
3011 hp = HAlloc((Process *) proc, sz);
3012 endp = hp + sz;
3013 hpp = &hp;
3014 szp = NULL;
3015 goto bld_term;
3016 }
3017
3018 ASSERT(endp >= hp);
3019 HRelease((Process *) proc, endp, hp);
3020
3021 return res;
3022 }
3023
erts_alloc_permanent_cache_aligned(ErtsAlcType_t type,Uint size)3024 void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size)
3025 {
3026 UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1)
3027 #ifdef VALGRIND
3028 + sizeof(UWord)
3029 #endif
3030 );
3031
3032 #ifdef VALGRIND
3033 { /* Link them to avoid Leak_PossiblyLost */
3034 static UWord* first_in_list = NULL;
3035 *(UWord**)v = first_in_list;
3036 first_in_list = (UWord*) v;
3037 v += sizeof(UWord);
3038 }
3039 #endif
3040
3041 if (v & ERTS_CACHE_LINE_MASK) {
3042 v = (v & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
3043 }
3044 ASSERT((v & ERTS_CACHE_LINE_MASK) == 0);
3045 return (void*)v;
3046 }
3047
3048 static void
reply_alloc_info(void * vair)3049 reply_alloc_info(void *vair)
3050 {
3051 ErtsAllocInfoReq *air = (ErtsAllocInfoReq *) vair;
3052 Uint sched_id = erts_get_scheduler_id();
3053 int global_instances = air->req_sched == sched_id;
3054 ErtsProcLocks rp_locks;
3055 Process *rp = air->proc;
3056 Eterm ref_copy = NIL, ai_list, msg = NIL;
3057 Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL;
3058 Eterm **hpp;
3059 Uint sz, *szp;
3060 ErlOffHeap *ohp = NULL;
3061 ErtsMessage *mp = NULL;
3062 #if HAVE_ERTS_MMAP
3063 struct erts_mmap_info_struct mmap_info_dflt;
3064 # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
3065 struct erts_mmap_info_struct mmap_info_literal;
3066 # endif
3067 #endif
3068 int i;
3069 Eterm (*info_func)(Allctr_t *,
3070 int,
3071 int,
3072 fmtfn_t *,
3073 void *,
3074 Uint **,
3075 Uint *) = (air->only_sz
3076 ? erts_alcu_sz_info
3077 : erts_alcu_info);
3078
3079 rp_locks = air->req_sched == sched_id ? ERTS_PROC_LOCK_MAIN : 0;
3080
3081 sz = 0;
3082 hpp = NULL;
3083 szp = &sz;
3084
3085 while (1) {
3086
3087 if (hpp)
3088 ref_copy = erts_iref_storage_make_ref(&air->iref,
3089 hpp, ohp, 0);
3090 else
3091 *szp += erts_iref_storage_heap_size(&air->iref);
3092
3093 ai_list = NIL;
3094 for (i = 0; air->allocs[i] != ERTS_ALC_A_INVALID; i++);
3095 for (i--; i >= 0; i--) {
3096 int ai = air->allocs[i];
3097 Allctr_t *allctr;
3098 Eterm ainfo;
3099 Eterm alloc_atom;
3100 if (global_instances) {
3101 switch (ai) {
3102 case ERTS_ALC_A_SYSTEM: {
3103 alloc_atom = erts_bld_atom(hpp, szp, "sys_alloc");
3104 ainfo = NIL;
3105 if (!air->only_sz) {
3106 SysAllocStat sas;
3107 if (hpp)
3108 sys_alloc_stat(&sas);
3109 if (szp) {
3110 /* ensure ehough heap */
3111 sas.top_pad = INT_MAX;
3112 sas.trim_threshold = INT_MAX;
3113 }
3114 if (sas.top_pad >= 0) {
3115 ainfo = erts_bld_cons(
3116 hpp, szp,
3117 erts_bld_tuple(
3118 hpp, szp, 2,
3119 erts_bld_atom(hpp, szp, "tp"),
3120 erts_bld_uint(
3121 hpp, szp,
3122 (Uint) sas.top_pad)),
3123 ainfo);
3124 }
3125 if (sas.trim_threshold >= 0) {
3126 ainfo = erts_bld_cons(
3127 hpp, szp,
3128 erts_bld_tuple(
3129 hpp, szp, 2,
3130 erts_bld_atom(hpp, szp, "tt"),
3131 erts_bld_uint(
3132 hpp, szp,
3133 (Uint) sas.trim_threshold)),
3134 ainfo);
3135 }
3136 ainfo = erts_bld_cons(hpp, szp,
3137 erts_bld_tuple(
3138 hpp, szp, 2,
3139 erts_bld_atom(hpp, szp,
3140 "m"),
3141 erts_bld_atom(hpp, szp,
3142 "libc")),
3143 ainfo);
3144 ainfo = erts_bld_cons(hpp, szp,
3145 erts_bld_tuple(
3146 hpp, szp, 2,
3147 erts_bld_atom(hpp, szp,
3148 "e"),
3149 am_true),
3150 ainfo);
3151 ainfo = erts_bld_tuple(hpp, szp, 2,
3152 erts_bld_atom(hpp, szp,
3153 "options"),
3154 ainfo);
3155 ainfo = erts_bld_cons(hpp, szp,ainfo,NIL);
3156 }
3157 ainfo = erts_bld_tuple(hpp, szp, 3,
3158 alloc_atom,
3159 make_small(0),
3160 ainfo);
3161 break;
3162 }
3163 case ERTS_ALC_INFO_A_ALLOC_UTIL:
3164 alloc_atom = erts_bld_atom(hpp, szp, "alloc_util");
3165 ainfo = (air->only_sz
3166 ? NIL
3167 : erts_alcu_au_info_options(NULL, NULL,
3168 hpp, szp));
3169 ainfo = erts_bld_tuple(hpp, szp, 3,
3170 alloc_atom,
3171 make_small(0),
3172 ainfo);
3173 break;
3174 case ERTS_ALC_INFO_A_ERTS_MMAP:
3175 alloc_atom = erts_bld_atom(hpp, szp, "erts_mmap");
3176 #if HAVE_ERTS_MMAP
3177 ainfo = (air->only_sz ? NIL :
3178 erts_mmap_info(&erts_dflt_mmapper, NULL, NULL,
3179 hpp, szp, &mmap_info_dflt));
3180 ainfo = erts_bld_tuple3(hpp, szp,
3181 alloc_atom,
3182 erts_bld_atom(hpp,szp,"default_mmap"),
3183 ainfo);
3184 # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
3185 ai_list = erts_bld_cons(hpp, szp,
3186 ainfo, ai_list);
3187 ainfo = (air->only_sz ? NIL :
3188 erts_mmap_info(&erts_literal_mmapper, NULL, NULL,
3189 hpp, szp, &mmap_info_literal));
3190 ainfo = erts_bld_tuple3(hpp, szp,
3191 alloc_atom,
3192 erts_bld_atom(hpp,szp,"literal_mmap"),
3193 ainfo);
3194 # endif
3195 #else /* !HAVE_ERTS_MMAP */
3196 ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
3197 am_false);
3198 #endif
3199 break;
3200 case ERTS_ALC_INFO_A_MSEG_ALLOC:
3201 alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
3202 #if HAVE_ERTS_MSEG
3203 ainfo = erts_mseg_info(0, NULL, NULL, hpp != NULL,
3204 air->only_sz, hpp, szp);
3205 ainfo = erts_bld_tuple3(hpp, szp,
3206 alloc_atom,
3207 make_small(0),
3208 ainfo);
3209
3210 #else
3211 ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
3212 am_false);
3213 #endif
3214 break;
3215 #ifndef ERTS_ALC_A_EXEC
3216 case ERTS_ALC_INFO_A_DISABLED_EXEC:
3217 alloc_atom = erts_bld_atom(hpp, szp, "exec_alloc");
3218 ainfo = erts_bld_tuple2(hpp, szp, alloc_atom, am_false);
3219 break;
3220 #endif
3221 default:
3222 alloc_atom = erts_bld_atom(hpp, szp,
3223 (char *) ERTS_ALC_A2AD(ai));
3224 if (!erts_allctrs_info[ai].enabled)
3225 ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
3226 am_false);
3227 else if (erts_allctrs_info[ai].alloc_util) {
3228 if (erts_allctrs_info[ai].thr_spec)
3229 allctr = erts_allctr_thr_spec[ai].allctr[0];
3230 else
3231 allctr = erts_allctrs_info[ai].extra;
3232 ainfo = info_func(allctr, air->internal, hpp != NULL,
3233 NULL, NULL, hpp, szp);
3234 ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom,
3235 make_small(0), ainfo);
3236 }
3237 else {
3238 erts_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n",
3239 __FILE__, __LINE__);
3240 }
3241 }
3242 ai_list = erts_bld_cons(hpp, szp,
3243 ainfo, ai_list);
3244 }
3245 switch (ai) {
3246 case ERTS_ALC_A_SYSTEM:
3247 case ERTS_ALC_INFO_A_ALLOC_UTIL:
3248 case ERTS_ALC_INFO_A_ERTS_MMAP:
3249 case ERTS_ALC_INFO_A_DISABLED_EXEC:
3250 break;
3251 case ERTS_ALC_INFO_A_MSEG_ALLOC:
3252 #if HAVE_ERTS_MSEG
3253 alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
3254 ainfo = erts_mseg_info(sched_id, NULL, NULL,
3255 hpp != NULL, air->only_sz, hpp, szp);
3256 ainfo = erts_bld_tuple(hpp, szp, 3,
3257 alloc_atom,
3258 make_small(sched_id),
3259 ainfo);
3260 ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list);
3261 #endif
3262 break;
3263 default:
3264 if (erts_allctrs_info[ai].thr_spec) {
3265 alloc_atom = erts_bld_atom(hpp, szp,
3266 (char *) ERTS_ALC_A2AD(ai));
3267 allctr = erts_allctr_thr_spec[ai].allctr[sched_id];
3268 ainfo = info_func(allctr, air->internal, hpp != NULL, NULL,
3269 NULL, hpp, szp);
3270 ai_list = erts_bld_cons(hpp, szp,
3271 erts_bld_tuple(
3272 hpp, szp,
3273 3,
3274 alloc_atom,
3275 make_small(sched_id),
3276 ainfo),
3277 ai_list);
3278 }
3279 break;
3280 }
3281 msg = erts_bld_tuple(hpp, szp,
3282 3,
3283 ref_copy,
3284 make_small(sched_id),
3285 ai_list);
3286
3287 }
3288 if (hpp)
3289 break;
3290
3291 mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
3292 hp_start = hp;
3293 hp_end = hp + sz;
3294 szp = NULL;
3295 hpp = &hp;
3296 }
3297
3298 if (hp != hp_end)
3299 erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1);
3300
3301 erts_queue_message(rp, rp_locks, mp, msg, am_system);
3302
3303 if (air->req_sched == sched_id)
3304 rp_locks &= ~ERTS_PROC_LOCK_MAIN;
3305
3306 erts_proc_unlock(rp, rp_locks);
3307 erts_proc_dec_refc(rp);
3308
3309 if (erts_atomic32_dec_read_nob(&air->refc) == 0) {
3310 erts_iref_storage_clean(&air->iref);
3311 aireq_free(air);
3312 }
3313 }
3314
3315 int
erts_request_alloc_info(struct process * c_p,Eterm ref,Eterm allocs,int only_sz,int internal)3316 erts_request_alloc_info(struct process *c_p,
3317 Eterm ref,
3318 Eterm allocs,
3319 int only_sz,
3320 int internal)
3321 {
3322 ErtsAllocInfoReq *air = aireq_alloc();
3323 Eterm req_ai[ERTS_ALC_INFO_A_END] = {0};
3324 Eterm alist;
3325 int airix = 0, ai;
3326
3327 air->req_sched = erts_get_scheduler_id();
3328
3329 air->only_sz = only_sz;
3330
3331 air->internal = internal;
3332
3333 air->proc = c_p;
3334
3335 if (is_not_internal_ref(ref))
3336 return 0;
3337
3338 erts_iref_storage_save(&air->iref, ref);
3339
3340 if (is_not_list(allocs))
3341 return 0;
3342
3343 alist = allocs;
3344
3345 while (is_list(alist)) {
3346 int saved = 0;
3347 Eterm* consp = list_val(alist);
3348 Eterm alloc = CAR(consp);
3349
3350 for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++)
3351 if (erts_is_atom_str(erts_alc_a2ad[ai], alloc, 0))
3352 goto save_alloc;
3353 if (erts_is_atom_str("mseg_alloc", alloc, 0)) {
3354 ai = ERTS_ALC_INFO_A_MSEG_ALLOC;
3355 goto save_alloc;
3356 }
3357 if (erts_is_atom_str("erts_mmap", alloc, 0)) {
3358 ai = ERTS_ALC_INFO_A_ERTS_MMAP;
3359 goto save_alloc;
3360 }
3361 #ifndef ERTS_ALC_A_EXEC
3362 if (erts_is_atom_str("exec_alloc", alloc, 0)) {
3363 ai = ERTS_ALC_INFO_A_DISABLED_EXEC;
3364 goto save_alloc;
3365 }
3366 #endif
3367 if (erts_is_atom_str("alloc_util", alloc, 0)) {
3368 ai = ERTS_ALC_INFO_A_ALLOC_UTIL;
3369 save_alloc:
3370 if (req_ai[ai])
3371 return 0;
3372 air->allocs[airix++] = ai;
3373 req_ai[ai] = 1;
3374 saved = 1;
3375 }
3376
3377 if (!saved)
3378 return 0;
3379
3380 alist = CDR(consp);
3381 }
3382
3383 if (is_not_nil(alist))
3384 return 0;
3385
3386 air->allocs[airix] = ERTS_ALC_A_INVALID;
3387
3388 erts_atomic32_init_nob(&air->refc,
3389 (erts_aint32_t) erts_no_schedulers);
3390
3391 erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
3392
3393 if (erts_no_schedulers > 1)
3394 erts_schedule_multi_misc_aux_work(1,
3395 erts_no_schedulers,
3396 reply_alloc_info,
3397 (void *) air);
3398
3399 reply_alloc_info((void *) air);
3400
3401 return 1;
3402 }
3403
erts_alloc_set_dyn_param(Process * c_p,Eterm tuple)3404 Eterm erts_alloc_set_dyn_param(Process* c_p, Eterm tuple)
3405 {
3406 ErtsAllocatorThrSpec_t *tspec;
3407 ErtsAlcType_t ai;
3408 Allctr_t* allctr;
3409 Eterm* tp;
3410 Eterm res;
3411
3412 if (!is_tuple_arity(tuple, 3))
3413 goto badarg;
3414
3415 tp = tuple_val(tuple);
3416
3417 /*
3418 * Ex: {ets_alloc, sbct, 256000}
3419 */
3420 if (!is_atom(tp[1]) || !is_atom(tp[2]) || !is_integer(tp[3]))
3421 goto badarg;
3422
3423 for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++)
3424 if (erts_is_atom_str(erts_alc_a2ad[ai], tp[1], 0))
3425 break;
3426
3427 if (ai > ERTS_ALC_A_MAX)
3428 goto badarg;
3429
3430 if (!erts_allctrs_info[ai].enabled ||
3431 !erts_allctrs_info[ai].alloc_util) {
3432 return am_notsup;
3433 }
3434
3435 if (tp[2] == am_sbct) {
3436 Uint sbct;
3437 int i, ok;
3438
3439 if (!term_to_Uint(tp[3], &sbct))
3440 goto badarg;
3441
3442 tspec = &erts_allctr_thr_spec[ai];
3443 if (tspec->enabled) {
3444 ok = 0;
3445 for (i = 0; i < tspec->size; i++) {
3446 allctr = tspec->allctr[i];
3447 ok |= allctr->try_set_dyn_param(allctr, am_sbct, sbct);
3448 }
3449 }
3450 else {
3451 allctr = erts_allctrs_info[ai].extra;
3452 ok = allctr->try_set_dyn_param(allctr, am_sbct, sbct);
3453 }
3454 return ok ? am_ok : am_notsup;
3455 }
3456 return am_notsup;
3457
3458 badarg:
3459 ERTS_BIF_PREP_ERROR(res, c_p, EXC_BADARG);
3460 return res;
3461 }
3462
3463 /*
3464 * The allocator wrapper prelocking stuff below is about the locking order.
3465 * It only affects wrappers (erl_mtrace.c) that keep locks during
3466 * alloc/realloc/free.
3467 *
3468 * Some query functions in erl_alloc_util.c lock the allocator mutex and then
3469 * use erts_printf that in turn may call the sys allocator through the wrappers.
3470 * To avoid breaking locking order these query functions first "pre-locks" all
3471 * allocator wrappers.
3472 */
3473 ErtsAllocatorWrapper_t *erts_allctr_wrappers;
3474 int erts_allctr_wrapper_prelocked = 0;
3475 erts_tsd_key_t erts_allctr_prelock_tsd_key;
3476
erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t * wrapper)3477 void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper)
3478 {
3479 ASSERT(wrapper->lock && wrapper->unlock);
3480 wrapper->next = erts_allctr_wrappers;
3481 erts_allctr_wrappers = wrapper;
3482 }
3483
erts_allctr_wrapper_pre_lock(void)3484 void erts_allctr_wrapper_pre_lock(void)
3485 {
3486 if (erts_allctr_wrappers) {
3487 ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers;
3488 for ( ; wrapper; wrapper = wrapper->next) {
3489 wrapper->lock();
3490 }
3491 ASSERT(!erts_allctr_wrapper_prelocked);
3492 erts_allctr_wrapper_prelocked = 1;
3493 erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)1);
3494 }
3495 }
3496
erts_allctr_wrapper_pre_unlock(void)3497 void erts_allctr_wrapper_pre_unlock(void)
3498 {
3499 if (erts_allctr_wrappers) {
3500 ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers;
3501
3502 erts_allctr_wrapper_prelocked = 0;
3503 erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)0);
3504 for ( ; wrapper; wrapper = wrapper->next) {
3505 wrapper->unlock();
3506 }
3507 }
3508 }
3509
3510
3511 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
3512 * NOTE: erts_alc_test() is only supposed to be used for testing. *
3513 * *
3514 * Keep alloc_SUITE_data/allocator_test.h updated if changes are made *
3515 * to erts_alc_test() *
3516 \* */
3517 #define ERTS_ALC_TEST_ABORT erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n")
3518
erts_alc_test(UWord op,UWord a1,UWord a2,UWord a3)3519 UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
3520 {
3521 switch (op >> 8) {
3522 case 0x0: return erts_alcu_test(op, a1, a2);
3523 case 0x1: return erts_gfalc_test(op, a1, a2);
3524 case 0x2: return erts_bfalc_test(op, a1, a2);
3525 case 0x3: return erts_afalc_test(op, a1, a2);
3526 case 0x4: return erts_mseg_test(op, a1, a2, a3);
3527 case 0x5: return erts_aoffalc_test(op, a1, a2);
3528 case 0xf:
3529 switch (op) {
3530 case 0xf00:
3531 if (((Allctr_t *) a1)->thread_safe)
3532 return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_TEST,
3533 (void *) a1,
3534 (Uint) a2);
3535 else
3536 return (UWord) erts_alcu_alloc(ERTS_ALC_T_TEST,
3537 (void *) a1,
3538 (Uint) a2);
3539 case 0xf01:
3540 if (((Allctr_t *) a1)->thread_safe)
3541 return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_TEST,
3542 (void *) a1,
3543 (void *) a2,
3544 (Uint) a3);
3545 else
3546 return (UWord) erts_alcu_realloc(ERTS_ALC_T_TEST,
3547 (void *) a1,
3548 (void *) a2,
3549 (Uint) a3);
3550 case 0xf02:
3551 if (((Allctr_t *) a1)->thread_safe)
3552 erts_alcu_free_ts(ERTS_ALC_T_TEST, (void *) a1, (void *) a2);
3553 else
3554 erts_alcu_free(ERTS_ALC_T_TEST, (void *) a1, (void *) a2);
3555 return 0;
3556 case 0xf03: {
3557 Allctr_t *allctr;
3558 struct au_init init;
3559
3560 SET_DEFAULT_ALLOC_OPTS(&init);
3561 init.enable = 1;
3562 init.astrat = ERTS_ALC_S_GOODFIT;
3563 init.init.util.name_prefix = (char *) a1;
3564 init.init.util.alloc_no = ERTS_ALC_A_TEST;
3565 init.init.util.alloc_strat = init.astrat;
3566 init.init.util.ts = 1;
3567 if ((char **) a3) {
3568 char **argv = (char **) a3;
3569 int i = 0;
3570 while (argv[i]) {
3571 if (argv[i][0] == '-' && argv[i][1] == 't')
3572 handle_au_arg(&init, &argv[i][2], argv, &i, 0);
3573 else
3574 return (UWord) NULL;
3575 i++;
3576 }
3577 }
3578
3579 switch (init.astrat) {
3580 case ERTS_ALC_S_GOODFIT:
3581 allctr = erts_gfalc_start((GFAllctr_t *)
3582 erts_alloc(ERTS_ALC_T_TEST,
3583 sizeof(GFAllctr_t)),
3584 &init.init.gf,
3585 &init.init.util);
3586 break;
3587 case ERTS_ALC_S_BESTFIT:
3588 allctr = erts_bfalc_start((BFAllctr_t *)
3589 erts_alloc(ERTS_ALC_T_TEST,
3590 sizeof(BFAllctr_t)),
3591 &init.init.bf,
3592 &init.init.util);
3593 break;
3594 case ERTS_ALC_S_AFIT:
3595 allctr = erts_afalc_start((AFAllctr_t *)
3596 erts_alloc(ERTS_ALC_T_TEST,
3597 sizeof(AFAllctr_t)),
3598 &init.init.af,
3599 &init.init.util);
3600 break;
3601 case ERTS_ALC_S_FIRSTFIT:
3602 allctr = erts_aoffalc_start((AOFFAllctr_t *)
3603 erts_alloc(ERTS_ALC_T_TEST,
3604 sizeof(AOFFAllctr_t)),
3605 &init.init.aoff,
3606 &init.init.util);
3607 break;
3608
3609 default:
3610 ASSERT(0);
3611 allctr = NULL;
3612 break;
3613 }
3614
3615 return (UWord) allctr;
3616 }
3617 case 0xf04:
3618 erts_alcu_stop((Allctr_t *) a1);
3619 erts_free(ERTS_ALC_T_TEST, (void *) a1);
3620 break;
3621 case 0xf05: return (UWord) 1;
3622 case 0xf06: return (UWord) ((Allctr_t *) a1)->thread_safe;
3623 #ifdef ETHR_NO_FORKSAFETY
3624 case 0xf07: return (UWord) 0;
3625 #else
3626 case 0xf07: return (UWord) ((Allctr_t *) a1)->thread_safe;
3627 #endif
3628 case 0xf08: {
3629 ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_mutex));
3630 if (ethr_mutex_init(mtx) != 0)
3631 ERTS_ALC_TEST_ABORT;
3632 return (UWord) mtx;
3633 }
3634 case 0xf09: {
3635 ethr_mutex *mtx = (ethr_mutex *) a1;
3636 if (ethr_mutex_destroy(mtx) != 0)
3637 ERTS_ALC_TEST_ABORT;
3638 erts_free(ERTS_ALC_T_TEST, (void *) mtx);
3639 break;
3640 }
3641 case 0xf0a:
3642 ethr_mutex_lock((ethr_mutex *) a1);
3643 break;
3644 case 0xf0b:
3645 ethr_mutex_unlock((ethr_mutex *) a1);
3646 break;
3647 case 0xf0c: {
3648 ethr_cond *cnd = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_cond));
3649 if (ethr_cond_init(cnd) != 0)
3650 ERTS_ALC_TEST_ABORT;
3651 return (UWord) cnd;
3652 }
3653 case 0xf0d: {
3654 ethr_cond *cnd = (ethr_cond *) a1;
3655 if (ethr_cond_destroy(cnd) != 0)
3656 ERTS_ALC_TEST_ABORT;
3657 erts_free(ERTS_ALC_T_TEST, (void *) cnd);
3658 break;
3659 }
3660 case 0xf0e:
3661 ethr_cond_broadcast((ethr_cond *) a1);
3662 break;
3663 case 0xf0f: {
3664 int res;
3665 do {
3666 res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2);
3667 } while (res == EINTR);
3668 break;
3669 }
3670 case 0xf10: {
3671 ethr_tid *tid = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_tid));
3672 if (ethr_thr_create(tid,
3673 (void * (*)(void *)) a1,
3674 (void *) a2,
3675 NULL) != 0)
3676 ERTS_ALC_TEST_ABORT;
3677 return (UWord) tid;
3678 }
3679 case 0xf11: {
3680 ethr_tid *tid = (ethr_tid *) a1;
3681 if (ethr_thr_join(*tid, NULL) != 0)
3682 ERTS_ALC_TEST_ABORT;
3683 erts_free(ERTS_ALC_T_TEST, (void *) tid);
3684 break;
3685 }
3686 case 0xf12:
3687 ethr_thr_exit((void *) a1);
3688 ERTS_ALC_TEST_ABORT;
3689 break;
3690 case 0xf13: return (UWord) 1;
3691 case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1);
3692
3693 case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0;
3694
3695 case 0xf16: return (UWord) erts_realloc(ERTS_ALC_T_TEST, (void*)a1, (Uint)a2);
3696
3697 case 0xf17: {
3698 Uint extra_hdr_sz = UNIT_CEILING((Uint)a1);
3699 ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST];
3700 Uint offset = ts->allctr[0]->mbc_header_size;
3701 void* orig_creating_mbc = ts->allctr[0]->creating_mbc;
3702 void* orig_destroying_mbc = ts->allctr[0]->destroying_mbc;
3703 void* new_creating_mbc = *(void**)a2; /* inout arg */
3704 void* new_destroying_mbc = *(void**)a3; /* inout arg */
3705 int i;
3706
3707 for (i=0; i < ts->size; i++) {
3708 Allctr_t* ap = ts->allctr[i];
3709 if (ap->mbc_header_size != offset
3710 || ap->creating_mbc != orig_creating_mbc
3711 || ap->destroying_mbc != orig_destroying_mbc
3712 || ap->mbc_list.first != NULL)
3713 return -1;
3714 }
3715 for (i=0; i < ts->size; i++) {
3716 ts->allctr[i]->mbc_header_size += extra_hdr_sz;
3717 ts->allctr[i]->creating_mbc = new_creating_mbc;
3718 ts->allctr[i]->destroying_mbc = new_destroying_mbc;
3719 }
3720 *(void**)a2 = orig_creating_mbc;
3721 *(void**)a3 = orig_destroying_mbc;
3722 return offset;
3723 }
3724 case 0xf18: {
3725 ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST];
3726 return ts->allctr[0]->largest_mbc_size;
3727 }
3728 default:
3729 break;
3730 }
3731 return (UWord) 0;
3732 default:
3733 break;
3734 }
3735
3736 ASSERT(0);
3737 return ~((UWord) 0);
3738 }
3739
3740 #ifdef DEBUG
3741 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
3742 * Debug stuff *
3743 \* */
3744
3745 #if 0
3746 #define PRINT_OPS
3747 #else
3748 #undef PRINT_OPS
3749 #endif
3750
3751 #ifdef HARD_DEBUG
3752 #define FENCE_SZ (4*sizeof(UWord))
3753 #else
3754 #define FENCE_SZ (3*sizeof(UWord))
3755 #endif
3756
3757 #if defined(ARCH_64)
3758 #define FENCE_PATTERN 0xABCDEF97ABCDEF97
3759 #else
3760 #define FENCE_PATTERN 0xABCDEF97
3761 #endif
3762
3763 #define TYPE_PATTERN_MASK ERTS_ALC_N_MASK
3764 #define TYPE_PATTERN_SHIFT 16
3765
3766 #define FIXED_FENCE_PATTERN_MASK \
3767 (~((UWord) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT)))
3768 #define FIXED_FENCE_PATTERN \
3769 (FENCE_PATTERN & FIXED_FENCE_PATTERN_MASK)
3770
3771 #define MK_PATTERN(T) \
3772 (FIXED_FENCE_PATTERN | (((T) & TYPE_PATTERN_MASK) << TYPE_PATTERN_SHIFT))
3773
3774 #define GET_TYPE_OF_PATTERN(P) \
3775 (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK)
3776
3777 #ifdef HARD_DEBUG
3778
3779 #define ERL_ALC_HDBG_MAX_MBLK 100000
3780 #define ERTS_ALC_O_CHECK -1
3781
3782 typedef struct hdbg_mblk_ hdbg_mblk;
3783 struct hdbg_mblk_ {
3784 hdbg_mblk *next;
3785 hdbg_mblk *prev;
3786 void *p;
3787 Uint s;
3788 ErtsAlcType_t n;
3789 };
3790
3791 static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK];
3792
3793 static hdbg_mblk *free_hdbg_mblks;
3794 static hdbg_mblk *used_hdbg_mblks;
3795 static erts_mtx_t hdbg_mblk_mtx;
3796
3797 static void
hdbg_init(void)3798 hdbg_init(void)
3799 {
3800 int i;
3801 for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++)
3802 hdbg_mblks[i].next = &hdbg_mblks[i+1];
3803 hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL;
3804 free_hdbg_mblks = &hdbg_mblks[0];
3805 used_hdbg_mblks = NULL;
3806 erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug", NIL,
3807 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR);
3808 }
3809
3810 static void *check_memory_fence(void *ptr,
3811 Uint *size,
3812 ErtsAlcType_t n,
3813 int func);
3814 void erts_hdbg_chk_blks(void);
3815
3816 void
erts_hdbg_chk_blks(void)3817 erts_hdbg_chk_blks(void)
3818 {
3819 hdbg_mblk *mblk;
3820
3821 erts_mtx_lock(&hdbg_mblk_mtx);
3822 for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) {
3823 Uint sz;
3824 check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK);
3825 ASSERT(sz == mblk->s);
3826 }
3827 erts_mtx_unlock(&hdbg_mblk_mtx);
3828 }
3829
3830 static hdbg_mblk *
hdbg_alloc(void * p,Uint s,ErtsAlcType_t n)3831 hdbg_alloc(void *p, Uint s, ErtsAlcType_t n)
3832 {
3833 hdbg_mblk *mblk;
3834
3835 erts_mtx_lock(&hdbg_mblk_mtx);
3836 mblk = free_hdbg_mblks;
3837 if (!mblk) {
3838 erts_fprintf(stderr,
3839 "Ran out of debug blocks; please increase "
3840 "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n",
3841 ERL_ALC_HDBG_MAX_MBLK);
3842 abort();
3843 }
3844 free_hdbg_mblks = mblk->next;
3845
3846 mblk->p = p;
3847 mblk->s = s;
3848 mblk->n = n;
3849
3850 mblk->next = used_hdbg_mblks;
3851 mblk->prev = NULL;
3852 if (used_hdbg_mblks)
3853 used_hdbg_mblks->prev = mblk;
3854 used_hdbg_mblks = mblk;
3855 erts_mtx_unlock(&hdbg_mblk_mtx);
3856 return (void *) mblk;
3857 }
3858
3859 static void
hdbg_free(hdbg_mblk * mblk)3860 hdbg_free(hdbg_mblk *mblk)
3861 {
3862 erts_mtx_lock(&hdbg_mblk_mtx);
3863 if (mblk->next)
3864 mblk->next->prev = mblk->prev;
3865 if (mblk->prev)
3866 mblk->prev->next = mblk->next;
3867 else
3868 used_hdbg_mblks = mblk->next;
3869
3870 mblk->next = free_hdbg_mblks;
3871 free_hdbg_mblks = mblk;
3872 erts_mtx_unlock(&hdbg_mblk_mtx);
3873 }
3874
3875 #endif
3876
3877 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
3878 static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func);
3879
check_allocated_block(Uint type,void * blk)3880 void check_allocated_block( Uint type, void *blk)
3881 {
3882 Uint dummy;
3883 check_memory_fence(blk, &dummy, ERTS_ALC_T2N(type), ERTS_ALC_O_FREE);
3884 }
3885
check_allocators(void)3886 void check_allocators(void)
3887 {
3888 int i;
3889 if (!erts_initialized)
3890 return;
3891 for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; ++i) {
3892 if (erts_allctrs_info[i].alloc_util) {
3893 ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra;
3894 Allctr_t *allctr = real_af->extra;
3895 Carrier_t *ct;
3896 if (allctr->thread_safe)
3897 erts_mtx_lock(&allctr->mutex);
3898
3899 if (allctr->check_mbc) {
3900 for (ct = allctr->mbc_list.first; ct; ct = ct->next) {
3901 fprintf(stderr,"Checking allocator %d\r\n",i);
3902 allctr->check_mbc(allctr,ct);
3903 }
3904 }
3905 if (allctr->thread_safe)
3906 erts_mtx_unlock(&allctr->mutex);
3907 }
3908 }
3909 }
3910 #endif
3911
3912 static void *
set_memory_fence(void * ptr,Uint sz,ErtsAlcType_t n)3913 set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n)
3914 {
3915 UWord *ui_ptr;
3916 UWord pattern;
3917 #ifdef HARD_DEBUG
3918 hdbg_mblk **mblkpp;
3919 #endif
3920
3921 if (!ptr)
3922 return NULL;
3923
3924 ui_ptr = (UWord *) ptr;
3925 pattern = MK_PATTERN(n);
3926
3927 #ifdef HARD_DEBUG
3928 mblkpp = (hdbg_mblk **) ui_ptr++;
3929 #endif
3930
3931 *(ui_ptr++) = sz;
3932 *(ui_ptr++) = pattern;
3933 sys_memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord));
3934
3935 #ifdef HARD_DEBUG
3936 *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n);
3937 #endif
3938
3939 return (void *) ui_ptr;
3940 }
3941
3942 static void *
check_memory_fence(void * ptr,Uint * size,ErtsAlcType_t n,int func)3943 check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
3944 {
3945 Uint sz;
3946 Uint found_type;
3947 UWord pre_pattern, expected_pattern;
3948 UWord post_pattern;
3949 UWord *ui_ptr;
3950 #ifdef HARD_DEBUG
3951 hdbg_mblk *mblk;
3952 #endif
3953
3954 if (!ptr)
3955 return NULL;
3956
3957 expected_pattern = MK_PATTERN(n);
3958
3959 ui_ptr = (UWord *) ptr;
3960 pre_pattern = *(--ui_ptr);
3961 *size = sz = *(--ui_ptr);
3962 #ifdef HARD_DEBUG
3963 mblk = (hdbg_mblk *) *(--ui_ptr);
3964 #endif
3965
3966 found_type = GET_TYPE_OF_PATTERN(pre_pattern);
3967
3968 if (found_type != n) {
3969 erts_exit(ERTS_ABORT_EXIT, "ERROR: Miss matching allocator types"
3970 " used in alloc and free\n");
3971 }
3972
3973 if (pre_pattern != expected_pattern) {
3974 if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN)
3975 erts_exit(ERTS_ABORT_EXIT,
3976 "ERROR: Fence at beginning of memory block (p=0x%u) "
3977 "clobbered.\n",
3978 (UWord) ptr);
3979 }
3980
3981 sys_memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord));
3982
3983 if (post_pattern != expected_pattern || pre_pattern != post_pattern) {
3984 char fbuf[10];
3985 char obuf[10];
3986 char *ftype;
3987 char *otype;
3988 char *op_str;
3989
3990 if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN)
3991 erts_exit(ERTS_ABORT_EXIT,
3992 "ERROR: Fence at end of memory block (p=0x%u, sz=%u) "
3993 "clobbered.\n",
3994 (UWord) ptr, (UWord) sz);
3995 if (found_type != GET_TYPE_OF_PATTERN(post_pattern))
3996 erts_exit(ERTS_ABORT_EXIT,
3997 "ERROR: Fence around memory block (p=0x%u, sz=%u) "
3998 "clobbered.\n",
3999 (UWord) ptr, (UWord) sz);
4000
4001 ftype = type_no_str(found_type);
4002 if (!ftype) {
4003 erts_snprintf(fbuf, sizeof(fbuf), "%d", (int) found_type);
4004 ftype = fbuf;
4005 }
4006 otype = type_no_str(n);
4007 if (!otype) {
4008 erts_snprintf(obuf, sizeof(obuf), "%d", (int) n);
4009 otype = obuf;
4010 }
4011
4012 switch (func) {
4013 case ERTS_ALC_O_ALLOC: op_str = "allocated"; break;
4014 case ERTS_ALC_O_REALLOC: op_str = "reallocated"; break;
4015 case ERTS_ALC_O_FREE: op_str = "freed"; break;
4016 default: op_str = "???"; break;
4017 }
4018
4019 erts_exit(ERTS_ABORT_EXIT,
4020 "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\","
4021 " but %s as type \"%s\".\n",
4022 (UWord) ptr, (UWord) sz, ftype, op_str, otype);
4023 }
4024
4025 #ifdef HARD_DEBUG
4026 switch (func) {
4027 case ERTS_ALC_O_REALLOC:
4028 case ERTS_ALC_O_FREE:
4029 hdbg_free(mblk);
4030 break;
4031 default:
4032 break;
4033 }
4034 #endif
4035
4036 return (void *) ui_ptr;
4037 }
4038
4039 static ErtsAllocatorFunctions_t real_allctrs[ERTS_ALC_A_MAX+1];
4040
4041 static void *
debug_alloc(ErtsAlcType_t type,void * extra,Uint size)4042 debug_alloc(ErtsAlcType_t type, void *extra, Uint size)
4043 {
4044 ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
4045 ErtsAlcType_t n;
4046 Uint dsize;
4047 void *res;
4048
4049 #ifdef HARD_DEBUG
4050 erts_hdbg_chk_blks();
4051 #endif
4052
4053 n = ERTS_ALC_T2N(type);
4054
4055 ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
4056 dsize = size + FENCE_SZ;
4057 res = (*real_af->alloc)(type, real_af->extra, dsize);
4058
4059 res = set_memory_fence(res, size, n);
4060
4061 #ifdef PRINT_OPS
4062 fprintf(stderr, "0x%lx = alloc(%s, %lu)\r\n",
4063 (Uint) res, ERTS_ALC_N2TD(n), size);
4064 #endif
4065
4066 return res;
4067 }
4068
4069
4070 static void *
debug_realloc(ErtsAlcType_t type,void * extra,void * ptr,Uint size)4071 debug_realloc(ErtsAlcType_t type, void *extra, void *ptr, Uint size)
4072 {
4073 ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
4074 ErtsAlcType_t n;
4075 Uint dsize;
4076 Uint old_size;
4077 void *dptr;
4078 void *res;
4079
4080 n = ERTS_ALC_T2N(type);
4081
4082 ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
4083
4084 dsize = size + FENCE_SZ;
4085 dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC);
4086
4087 #ifdef HARD_DEBUG
4088 erts_hdbg_chk_blks();
4089 #endif
4090
4091 if (ptr && old_size > size)
4092 sys_memset((void *) (((char *) ptr) + size),
4093 0xf,
4094 sizeof(Uint) + old_size - size);
4095
4096 res = (*real_af->realloc)(type, real_af->extra, dptr, dsize);
4097
4098 res = set_memory_fence(res, size, n);
4099
4100 #ifdef PRINT_OPS
4101 fprintf(stderr, "0x%lx = realloc(%s, 0x%lx, %lu)\r\n",
4102 (Uint) res, ERTS_ALC_N2TD(n), (Uint) ptr, size);
4103 #endif
4104
4105 return res;
4106 }
4107
4108 static void
debug_free(ErtsAlcType_t type,void * extra,void * ptr)4109 debug_free(ErtsAlcType_t type, void *extra, void *ptr)
4110 {
4111 ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
4112 ErtsAlcType_t n;
4113 void *dptr;
4114 Uint size;
4115 int free_pattern;
4116
4117 n = ERTS_ALC_T2N(type);
4118 free_pattern = n;
4119
4120 ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
4121
4122 if (!ptr)
4123 return;
4124
4125 dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE);
4126
4127 #ifdef ERTS_ALC_A_EXEC
4128 # if defined(__i386__) || defined(__x86_64__)
4129 if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) {
4130 free_pattern = 0x0f; /* Illegal instruction */
4131 }
4132 # endif
4133 #endif
4134 sys_memset((void *) dptr, free_pattern, size + FENCE_SZ);
4135
4136 (*real_af->free)(type, real_af->extra, dptr);
4137
4138 #ifdef PRINT_OPS
4139 fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr);
4140 #endif
4141
4142 #ifdef HARD_DEBUG
4143 erts_hdbg_chk_blks();
4144 #endif
4145
4146 }
4147
4148 static Uint
install_debug_functions(void)4149 install_debug_functions(void)
4150 {
4151 int i;
4152 ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
4153
4154 sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs));
4155
4156 for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
4157 erts_allctrs[i].alloc = debug_alloc;
4158 erts_allctrs[i].realloc = debug_realloc;
4159 erts_allctrs[i].free = debug_free;
4160 erts_allctrs[i].extra = (void *) &real_allctrs[i];
4161 }
4162 return FENCE_SZ;
4163 }
4164
4165 #endif /* #ifdef DEBUG */
4166