1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2002-2018. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20 #ifdef HAVE_CONFIG_H
21 # include "config.h"
22 #endif
23
24 #define ERL_WANT_GC_INTERNALS__
25
26 #include "sys.h"
27 #include "erl_vm.h"
28 #include "global.h"
29 #include "erl_process.h"
30 #include "erl_db.h"
31 #include "beam_catches.h"
32 #include "erl_binary.h"
33 #include "erl_bits.h"
34 #include "erl_map.h"
35 #include "error.h"
36 #include "big.h"
37 #include "erl_gc.h"
38 #ifdef HIPE
39 #include "hipe_stack.h"
40 #include "hipe_mode_switch.h"
41 #endif
42 #include "dtrace-wrapper.h"
43 #include "erl_bif_unique.h"
44 #include "dist.h"
45 #include "erl_nfunc_sched.h"
46 #include "erl_proc_sig_queue.h"
47
48 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
49 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
50 #define ERTS_INACT_WR_PB_LEAVE_LIMIT 10
51 #define ERTS_INACT_WR_PB_LEAVE_PERCENTAGE 10
52
53 #if defined(DEBUG) || 0
54 #define ERTS_GC_DEBUG
55 #else
56 #undef ERTS_GC_DEBUG
57 #endif
58 #ifdef ERTS_GC_DEBUG
59 # define ERTS_GC_ASSERT ASSERT
60 #else
61 # define ERTS_GC_ASSERT(B) ((void) 1)
62 #endif
63
64 #if defined(DEBUG) && 0
65 # define HARDDEBUG 1
66 #endif
67
68 /*
69 * Returns number of elements in an array.
70 */
71 #define ALENGTH(a) (sizeof(a)/sizeof(a[0]))
72
73 # define STACK_SZ_ON_HEAP(p) ((p)->hend - (p)->stop)
74 # define OverRunCheck(P) \
75 if ((P)->stop < (P)->htop) { \
76 erts_fprintf(stderr, "hend=%p\n", (p)->hend); \
77 erts_fprintf(stderr, "stop=%p\n", (p)->stop); \
78 erts_fprintf(stderr, "htop=%p\n", (p)->htop); \
79 erts_fprintf(stderr, "heap=%p\n", (p)->heap); \
80 erts_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \
81 __FILE__,__LINE__,(P)->common.id); \
82 }
83
84 #ifdef DEBUG
85 #define ErtsGcQuickSanityCheck(P) \
86 do { \
87 ASSERT((P)->heap < (P)->hend); \
88 ASSERT((p)->abandoned_heap || (P)->heap_sz == (P)->hend - (P)->heap); \
89 ASSERT((P)->heap <= (P)->htop && (P)->htop <= (P)->hend); \
90 ASSERT((P)->heap <= (P)->stop && (P)->stop <= (P)->hend); \
91 ASSERT((p)->abandoned_heap || ((P)->heap <= (P)->high_water && (P)->high_water <= (P)->hend)); \
92 OverRunCheck((P)); \
93 } while (0)
94 #else
95 #define ErtsGcQuickSanityCheck(P) \
96 do { \
97 OverRunCheck((P)); \
98 } while (0)
99 #endif
100 /*
101 * This structure describes the rootset for the GC.
102 */
103 typedef struct roots {
104 Eterm* v; /* Pointers to vectors with terms to GC
105 * (e.g. the stack).
106 */
107 Uint sz; /* Size of each vector. */
108 } Roots;
109
110 typedef struct {
111 Roots def[32]; /* Default storage. */
112 Roots* roots; /* Pointer to root set array. */
113 Uint size; /* Storage size. */
114 int num_roots; /* Number of root arrays. */
115 } Rootset;
116
117 static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
118 static void cleanup_rootset(Rootset *rootset);
119 static Eterm *full_sweep_heaps(Process *p,
120 ErlHeapFragment *live_hf_end,
121 int hibernate,
122 Eterm *n_heap, Eterm* n_htop,
123 char *oh, Uint oh_size,
124 Eterm *objv, int nobj);
125 static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
126 int need, Eterm* objv, int nobj, int fcalls,
127 Uint max_young_gen_usage);
128 static int major_collection(Process* p, ErlHeapFragment *live_hf_end,
129 int need, Eterm* objv, int nobj,
130 Uint ygen_usage, Uint *recl);
131 static int minor_collection(Process* p, ErlHeapFragment *live_hf_end,
132 int need, Eterm* objv, int nobj,
133 Uint ygen_usage, Uint *recl);
134 static void do_minor(Process *p, ErlHeapFragment *live_hf_end,
135 char *mature, Uint mature_size,
136 Uint new_sz, Eterm* objv, int nobj);
137 static Eterm *sweep_new_heap(Eterm *n_hp, Eterm *n_htop,
138 char* old_heap, Uint old_heap_size);
139 static Eterm *sweep_heaps(Eterm *n_hp, Eterm *n_htop,
140 char* old_heap, Uint old_heap_size);
141 static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop,
142 char* old_heap, Uint old_heap_size,
143 char* src, Uint src_size);
144 static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
145 char* src, Uint src_size);
146 static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end,
147 Eterm* htop);
148 static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
149 static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
150 static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
151 static void sweep_off_heap(Process *p, int fullsweep);
152 static void offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
153 static void offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
154 static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
155 Eterm* objv, int nobj);
156 static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size);
157 static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size);
158 static int reached_max_heap_size(Process *p, Uint total_heap_size,
159 Uint extra_heap_size, Uint extra_old_heap_size);
160 static void init_gc_info(ErtsGCInfo *gcip);
161 static Uint64 next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz);
162
163 #ifdef HARDDEBUG
164 static void disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop);
165 static void disallow_heap_frag_ref_in_old_heap(Process* p);
166 #endif
167
168 #if defined(ARCH_64)
169 # define MAX_HEAP_SIZES 154
170 #else
171 # define MAX_HEAP_SIZES 59
172 #endif
173
174 static Sint heap_sizes[MAX_HEAP_SIZES]; /* Suitable heap sizes. */
175 static int num_heap_sizes; /* Number of heap sizes. */
176
177 Uint erts_test_long_gc_sleep; /* Only used for testing... */
178
179 typedef struct {
180 Process *proc;
181 Eterm ref;
182 Eterm ref_heap[ERTS_REF_THING_SIZE];
183 Uint req_sched;
184 erts_atomic32_t refc;
185 } ErtsGCInfoReq;
186
187 static struct {
188 erts_mtx_t mtx;
189 ErtsGCInfo info;
190 } dirty_gc;
191
move_msgs_to_heap(Process * c_p)192 static void move_msgs_to_heap(Process *c_p)
193 {
194 Uint64 pre_oh, post_oh;
195
196 pre_oh = c_p->off_heap.overhead;
197 erts_proc_sig_move_msgs_to_heap(c_p);
198 post_oh = c_p->off_heap.overhead;
199
200 if (pre_oh != post_oh) {
201 /* Got new binaries; update bin vheap size... */
202 c_p->bin_vheap_sz = next_vheap_size(c_p, post_oh,
203 c_p->bin_vheap_sz);
204 }
205 }
206
207 static ERTS_INLINE int
gc_cost(Uint gc_moved_live_words,Uint resize_moved_words)208 gc_cost(Uint gc_moved_live_words, Uint resize_moved_words)
209 {
210 Sint reds;
211
212 reds = gc_moved_live_words/10;
213 reds += resize_moved_words/100;
214 if (reds < 1)
215 return 1;
216 if (reds > INT_MAX)
217 return INT_MAX;
218 return (int) reds;
219 }
220
221 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq,
222 ErtsGCInfoReq,
223 5,
224 ERTS_ALC_T_GC_INFO_REQ)
225
226 /*
227 * Initialize GC global data.
228 */
229 void
erts_init_gc(void)230 erts_init_gc(void)
231 {
232 int i = 0, ix;
233 Sint max_heap_size = 0;
234
235 ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word));
236 ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word));
237 ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header));
238 ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size));
239 ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size));
240 ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size));
241 ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next));
242 ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next));
243 ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next));
244
245 erts_test_long_gc_sleep = 0;
246
247 /*
248 * Heap sizes start growing in a Fibonacci sequence.
249 *
250 * Fib growth is not really ok for really large heaps, for
251 * example is fib(35) == 14meg, whereas fib(36) == 24meg;
252 * we really don't want that growth when the heaps are that big.
253 */
254
255 /* Growth stage 1 - Fibonacci + 1*/
256 /* 12,38 will hit size 233, the old default */
257
258 heap_sizes[0] = 12;
259 heap_sizes[1] = 38;
260
261 for(i = 2; i < 23; i++) {
262 /* one extra word for block header */
263 heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-2] + 1;
264 }
265
266
267 /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words]
268 * for 64 bit we want max_heap_size to be MAX(52bit) / 8 [words]
269 */
270
271 max_heap_size = sizeof(Eterm) < 8 ? (Sint)((~(Uint)0)/(sizeof(Eterm))) :
272 (Sint)(((Uint64)1 << 53)/sizeof(Eterm));
273
274 /* Growth stage 2 - 20% growth */
275 /* At 1.3 mega words heap, we start to slow down. */
276 for (i = 23; i < ALENGTH(heap_sizes); i++) {
277 heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-1]/5;
278 if ((heap_sizes[i] < 0) || heap_sizes[i] > max_heap_size) {
279 /* Size turned negative. Discard this last size. */
280 i--;
281 break;
282 }
283 }
284 num_heap_sizes = i;
285
286 for (ix = 0; ix < erts_no_schedulers; ix++) {
287 ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
288 init_gc_info(&esdp->gc_info);
289 }
290
291 erts_mtx_init(&dirty_gc.mtx, "dirty_gc_info", NIL,
292 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
293 init_gc_info(&dirty_gc.info);
294
295 init_gcireq_alloc();
296 }
297
298 /*
299 * Find the next heap size equal to or greater than the given size (if offset == 0).
300 *
301 * If offset is 1, the next higher heap size is returned (always greater than size).
302 */
303 Uint
erts_next_heap_size(Uint size,Uint offset)304 erts_next_heap_size(Uint size, Uint offset)
305 {
306 if (size < heap_sizes[0]) {
307 return heap_sizes[0];
308 } else {
309 Sint* low = heap_sizes;
310 Sint* high = heap_sizes + num_heap_sizes;
311 Sint* mid;
312
313 while (low < high) {
314 mid = low + (high-low) / 2;
315 if (size < mid[0]) {
316 high = mid;
317 } else if (size == mid[0]) {
318 ASSERT(mid+offset-heap_sizes < num_heap_sizes);
319 return mid[offset];
320 } else if (size < mid[1]) {
321 ASSERT(mid[0] < size && size <= mid[1]);
322 ASSERT(mid+offset-heap_sizes < num_heap_sizes);
323 return mid[offset+1];
324 } else {
325 low = mid + 1;
326 }
327 }
328 erts_exit(ERTS_ERROR_EXIT, "no next heap size found: %beu, offset %beu\n", size, offset);
329 }
330 return 0;
331 }
332 /*
333 * Return the next heap size to use. Make sure we never return
334 * a smaller heap size than the minimum heap size for the process.
335 * (Use of the erlang:hibernate/3 BIF could have shrinked the
336 * heap below the minimum heap size.)
337 */
338 static Uint
next_heap_size(Process * p,Uint size,Uint offset)339 next_heap_size(Process* p, Uint size, Uint offset)
340 {
341 size = erts_next_heap_size(size, offset);
342 return size < p->min_heap_size ? p->min_heap_size : size;
343 }
344
345 Eterm
erts_heap_sizes(Process * p)346 erts_heap_sizes(Process* p)
347 {
348 int i;
349 int n = 0;
350 int big = 0;
351 Eterm res = NIL;
352 Eterm* hp;
353 Eterm* bigp;
354
355 for (i = num_heap_sizes-1; i >= 0; i--) {
356 n += 2;
357 if (!IS_SSMALL(heap_sizes[i])) {
358 big += BIG_UINT_HEAP_SIZE;
359 }
360 }
361
362 /*
363 * We store all big numbers first on the heap, followed
364 * by all the cons cells.
365 */
366 bigp = HAlloc(p, n+big);
367 hp = bigp+big;
368 for (i = num_heap_sizes-1; i >= 0; i--) {
369 Eterm num;
370 Sint sz = heap_sizes[i];
371
372 if (IS_SSMALL(sz)) {
373 num = make_small(sz);
374 } else {
375 num = uint_to_big(sz, bigp);
376 bigp += BIG_UINT_HEAP_SIZE;
377 }
378 res = CONS(hp, num, res);
379 hp += 2;
380 }
381 return res;
382 }
383
384 void
erts_offset_heap(Eterm * hp,Uint sz,Sint offs,Eterm * low,Eterm * high)385 erts_offset_heap(Eterm* hp, Uint sz, Sint offs, Eterm* low, Eterm* high)
386 {
387 offset_heap(hp, sz, offs, (char*) low, ((char *)high)-((char *)low));
388 }
389
390 void
erts_offset_heap_ptr(Eterm * hp,Uint sz,Sint offs,Eterm * low,Eterm * high)391 erts_offset_heap_ptr(Eterm* hp, Uint sz, Sint offs,
392 Eterm* low, Eterm* high)
393 {
394 offset_heap_ptr(hp, sz, offs, (char *) low, ((char *)high)-((char *)low));
395 }
396
397
398 #define ptr_within(ptr, low, high) ((ptr) < (high) && (ptr) >= (low))
399
400 void
erts_offset_off_heap(ErlOffHeap * ohp,Sint offs,Eterm * low,Eterm * high)401 erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high)
402 {
403 if (ohp->first && ptr_within((Eterm *)ohp->first, low, high)) {
404 Eterm** uptr = (Eterm**) (void *) &ohp->first;
405 *uptr += offs;
406 }
407 }
408 #undef ptr_within
409
410 Eterm
erts_gc_after_bif_call_lhf(Process * p,ErlHeapFragment * live_hf_end,Eterm result,Eterm * regs,Uint arity)411 erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
412 Eterm result, Eterm* regs, Uint arity)
413 {
414 int cost;
415
416 if (p->flags & (F_HIBERNATE_SCHED|F_HIPE_RECV_LOCKED)) {
417 /*
418 * We just hibernated. We do *not* want to mess
419 * up the hibernation by an ordinary GC...
420 *
421 * OR
422 *
423 * We left a receive in HiPE with message
424 * queue lock locked, and we do not want to
425 * do a GC with message queue locked...
426 */
427 return result;
428 }
429
430 if (!p->mbuf) {
431 /* Must have GC:d in BIF call... invalidate live_hf_end */
432 live_hf_end = ERTS_INVALID_HFRAG_PTR;
433 }
434
435 if (is_non_value(result)) {
436 if (p->freason == TRAP) {
437 #ifdef HIPE
438 if (regs == NULL) {
439 regs = erts_proc_sched_data(p)->x_reg_array;
440 }
441 #endif
442 cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls, 0);
443 } else {
444 cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls, 0);
445 }
446 } else {
447 Eterm val[1];
448
449 val[0] = result;
450 cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls, 0);
451 result = val[0];
452 }
453 BUMP_REDS(p, cost);
454
455 return result;
456 }
457
458 Eterm
erts_gc_after_bif_call(Process * p,Eterm result,Eterm * regs,Uint arity)459 erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity)
460 {
461 return erts_gc_after_bif_call_lhf(p, ERTS_INVALID_HFRAG_PTR,
462 result, regs, arity);
463 }
464
reset_active_writer(Process * p)465 static ERTS_INLINE void reset_active_writer(Process *p)
466 {
467 struct erl_off_heap_header* ptr;
468 ptr = MSO(p).first;
469 while (ptr) {
470 if (ptr->thing_word == HEADER_PROC_BIN) {
471 ProcBin *pbp = (ProcBin*) ptr;
472 pbp->flags &= ~PB_ACTIVE_WRITER;
473 }
474 ptr = ptr->next;
475 }
476 }
477
478 #define ERTS_DELAY_GC_EXTRA_FREE 40
479 #define ERTS_ABANDON_HEAP_COST 10
480
481 static int
delay_garbage_collection(Process * p,ErlHeapFragment * live_hf_end,int need,int fcalls)482 delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int fcalls)
483 {
484 ErlHeapFragment *hfrag;
485 Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop;
486 Eterm *stop, *hend;
487 Uint hsz, ssz;
488 int reds_left;
489
490 ERTS_HOLE_CHECK(p);
491
492 if ((p->flags & F_DISABLE_GC)
493 && p->live_hf_end == ERTS_INVALID_HFRAG_PTR) {
494 /*
495 * A BIF yielded with disabled GC. Remember
496 * heap fragments created by the BIF until we
497 * do next GC.
498 */
499 p->live_hf_end = live_hf_end;
500 }
501
502 if (need == 0) {
503 if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
504 ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)));
505 goto force_reschedule;
506 }
507 return 1;
508 }
509 /*
510 * Satisfy need in a heap fragment...
511 */
512 ASSERT(need > 0);
513
514 orig_heap = p->heap;
515 orig_hend = p->hend;
516 orig_htop = p->htop;
517 orig_stop = p->stop;
518
519 ssz = orig_hend - orig_stop;
520 hsz = ssz + need + ERTS_DELAY_GC_EXTRA_FREE;
521
522 hfrag = new_message_buffer(hsz);
523 p->heap = p->htop = &hfrag->mem[0];
524 p->hend = hend = &hfrag->mem[hsz];
525 p->stop = stop = hend - ssz;
526 sys_memcpy((void *) stop, (void *) orig_stop, ssz * sizeof(Eterm));
527
528 if (p->abandoned_heap) {
529 /*
530 * Active heap already in a fragment; adjust it and
531 * save it into mbuf list...
532 */
533 ErlHeapFragment *hfrag = ((ErlHeapFragment *)
534 (((char *) orig_heap)
535 - offsetof(ErlHeapFragment, mem)));
536 Uint used = orig_htop - orig_heap;
537 hfrag->used_size = used;
538 p->mbuf_sz += used;
539 ASSERT(hfrag->used_size <= hfrag->alloc_size);
540 ASSERT(!hfrag->off_heap.first && !hfrag->off_heap.overhead);
541 hfrag->next = p->mbuf;
542 p->mbuf = hfrag;
543 }
544 else {
545 /* Do not leave a hole in the abandoned heap... */
546 if (orig_htop < orig_hend) {
547 *orig_htop = make_pos_bignum_header(orig_hend-orig_htop-1);
548 if (orig_htop + 1 < orig_hend) {
549 orig_hend[-1] = (Uint) (orig_htop - orig_heap);
550 p->flags |= F_ABANDONED_HEAP_USE;
551 }
552 }
553 p->abandoned_heap = orig_heap;
554 }
555
556 #ifdef CHECK_FOR_HOLES
557 p->last_htop = p->htop;
558 p->heap_hfrag = hfrag;
559 #endif
560
561 force_reschedule:
562
563 /* Make sure that we do a proper GC as soon as possible... */
564 p->flags |= F_FORCE_GC;
565 reds_left = ERTS_REDS_LEFT(p, fcalls);
566 ASSERT(CONTEXT_REDS - reds_left >= erts_proc_sched_data(p)->virtual_reds);
567
568 if (reds_left > ERTS_ABANDON_HEAP_COST) {
569 int vreds = reds_left - ERTS_ABANDON_HEAP_COST;
570 erts_proc_sched_data((p))->virtual_reds += vreds;
571 }
572
573 ERTS_CHK_MBUF_SZ(p);
574
575 ASSERT(CONTEXT_REDS >= erts_proc_sched_data(p)->virtual_reds);
576 return reds_left;
577 }
578
579 static ERTS_FORCE_INLINE Uint
young_gen_usage(Process * p)580 young_gen_usage(Process *p)
581 {
582 Uint hsz;
583 Eterm *aheap;
584
585 ERTS_CHK_MBUF_SZ(p);
586
587 hsz = p->mbuf_sz;
588
589 if (p->flags & F_ON_HEAP_MSGQ) {
590 ERTS_FOREACH_SIG_PRIVQS(
591 p, mp,
592 {
593 /*
594 * We leave not yet decoded distribution messages
595 * as they are in the queue since it is not
596 * possible to determine a maximum size until
597 * actual decoding. However, we use their estimated
598 * size when calculating need, and by this making
599 * it more likely that they will fit on the heap
600 * when actually decoded.
601 *
602 * We however ignore off heap messages...
603 */
604 if (ERTS_SIG_IS_MSG(mp)
605 && mp->data.attached
606 && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
607 hsz += erts_msg_attached_data_size(mp);
608 }
609 });
610 }
611
612 hsz += p->htop - p->heap;
613 aheap = p->abandoned_heap;
614 if (aheap) {
615 /* used in orig heap */
616 if (p->flags & F_ABANDONED_HEAP_USE)
617 hsz += aheap[p->heap_sz-1];
618 else
619 hsz += p->heap_sz;
620 }
621 return hsz;
622 }
623
624 #define ERTS_GET_ORIG_HEAP(Proc, Heap, HTop) \
625 do { \
626 Eterm *aheap__ = (Proc)->abandoned_heap; \
627 if (!aheap__) { \
628 (Heap) = (Proc)->heap; \
629 (HTop) = (Proc)->htop; \
630 } \
631 else { \
632 (Heap) = aheap__; \
633 if ((Proc)->flags & F_ABANDONED_HEAP_USE) \
634 (HTop) = aheap__ + aheap__[(Proc)->heap_sz-1]; \
635 else \
636 (HTop) = aheap__ + (Proc)->heap_sz; \
637 } \
638 } while (0)
639
640
641 static ERTS_INLINE void
check_for_possibly_long_gc(Process * p,Uint ygen_usage)642 check_for_possibly_long_gc(Process *p, Uint ygen_usage)
643 {
644 int major;
645 Uint sz;
646
647 major = (p->flags & F_NEED_FULLSWEEP) || GEN_GCS(p) >= MAX_GEN_GCS(p);
648
649 sz = ygen_usage;
650 sz += p->hend - p->stop;
651 if (p->flags & F_ON_HEAP_MSGQ)
652 sz += erts_proc_sig_privqs_len(p);
653 if (major)
654 sz += p->old_htop - p->old_heap;
655
656 if (sz >= ERTS_POTENTIALLY_LONG_GC_HSIZE) {
657 ASSERT(!(p->flags & (F_DISABLE_GC|F_DELAY_GC)));
658 p->flags |= major ? F_DIRTY_MAJOR_GC : F_DIRTY_MINOR_GC;
659 erts_schedule_dirty_sys_execution(p);
660 }
661 }
662
663
664 /*
665 * Garbage collect a process.
666 *
667 * p: Pointer to the process structure.
668 * need: Number of Eterm words needed on the heap.
669 * objv: Array of terms to add to rootset; that is to preserve.
670 * nobj: Number of objects in objv.
671 */
672 static int
garbage_collect(Process * p,ErlHeapFragment * live_hf_end,int need,Eterm * objv,int nobj,int fcalls,Uint max_young_gen_usage)673 garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
674 int need, Eterm* objv, int nobj, int fcalls,
675 Uint max_young_gen_usage)
676 {
677 Uint reclaimed_now = 0;
678 Uint ygen_usage;
679 Eterm gc_trace_end_tag;
680 int reds;
681 ErtsMonotonicTime start_time;
682 ErtsSchedulerData *esdp = erts_proc_sched_data(p);
683 erts_aint32_t state;
684 ERTS_MSACC_PUSH_STATE();
685 #ifdef USE_VM_PROBES
686 DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
687 #endif
688
689 ERTS_UNDEF(start_time, 0);
690 ERTS_CHK_MBUF_SZ(p);
691
692 ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= esdp->virtual_reds);
693
694 state = erts_atomic32_read_nob(&p->state);
695
696 if ((p->flags & (F_DISABLE_GC|F_DELAY_GC)) || state & ERTS_PSFLG_EXITING) {
697 delay_gc_before_start:
698 return delay_garbage_collection(p, live_hf_end, need, fcalls);
699 }
700
701 ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p);
702
703 if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
704 check_for_possibly_long_gc(p, ygen_usage);
705 if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))
706 goto delay_gc_before_start;
707 }
708
709 if (p->abandoned_heap)
710 live_hf_end = ERTS_INVALID_HFRAG_PTR;
711 else if (p->live_hf_end != ERTS_INVALID_HFRAG_PTR)
712 live_hf_end = p->live_hf_end;
713
714 ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_GC);
715
716 erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
717 if (erts_system_monitor_long_gc != 0)
718 start_time = erts_get_monotonic_time(esdp);
719
720 ERTS_CHK_OFFHEAP(p);
721
722 ErtsGcQuickSanityCheck(p);
723
724 #ifdef USE_VM_PROBES
725 *pidbuf = '\0';
726 if (DTRACE_ENABLED(gc_major_start)
727 || DTRACE_ENABLED(gc_major_end)
728 || DTRACE_ENABLED(gc_minor_start)
729 || DTRACE_ENABLED(gc_minor_end)) {
730 dtrace_proc_str(p, pidbuf);
731 }
732 #endif
733 /*
734 * Test which type of GC to do.
735 */
736
737 if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) {
738 if (IS_TRACED_FL(p, F_TRACE_GC)) {
739 trace_gc(p, am_gc_minor_start, need, THE_NON_VALUE);
740 }
741 DTRACE2(gc_minor_start, pidbuf, need);
742 reds = minor_collection(p, live_hf_end, need, objv, nobj,
743 ygen_usage, &reclaimed_now);
744 DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
745 if (reds == -1) {
746 if (IS_TRACED_FL(p, F_TRACE_GC)) {
747 trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE);
748 }
749 if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
750 p->flags |= F_NEED_FULLSWEEP;
751 check_for_possibly_long_gc(p, ygen_usage);
752 if (p->flags & F_DIRTY_MAJOR_GC)
753 goto delay_gc_after_start;
754 }
755 goto do_major_collection;
756 }
757 if (ERTS_SCHEDULER_IS_DIRTY(esdp))
758 p->flags &= ~F_DIRTY_MINOR_GC;
759 gc_trace_end_tag = am_gc_minor_end;
760 } else {
761 do_major_collection:
762 ERTS_MSACC_SET_STATE_CACHED_X(ERTS_MSACC_STATE_GC_FULL);
763 if (IS_TRACED_FL(p, F_TRACE_GC)) {
764 trace_gc(p, am_gc_major_start, need, THE_NON_VALUE);
765 }
766 DTRACE2(gc_major_start, pidbuf, need);
767 reds = major_collection(p, live_hf_end, need, objv, nobj,
768 ygen_usage, &reclaimed_now);
769 if (ERTS_SCHEDULER_IS_DIRTY(esdp))
770 p->flags &= ~(F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC);
771 DTRACE2(gc_major_end, pidbuf, reclaimed_now);
772 gc_trace_end_tag = am_gc_major_end;
773 ERTS_MSACC_SET_STATE_CACHED_X(ERTS_MSACC_STATE_GC);
774 }
775
776 reset_active_writer(p);
777
778 /*
779 * Finish.
780 */
781
782 ERTS_CHK_OFFHEAP(p);
783
784 ErtsGcQuickSanityCheck(p);
785
786 /* Max heap size has been reached and the process was configured
787 to be killed, so we kill it and set it in a delayed garbage
788 collecting state. There should be no gc_end trace or
789 long_gc/large_gc triggers when this happens as process was
790 killed before a GC could be done. */
791 if (reds == -2) {
792 int res;
793
794 erts_set_self_exiting(p, am_killed);
795
796 delay_gc_after_start:
797 /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so
798 we have to remove it after the signal is sent */
799 erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
800
801 /* We have to make sure that we have space for need on the heap */
802 res = delay_garbage_collection(p, live_hf_end, need, fcalls);
803 ERTS_MSACC_POP_STATE();
804 return res;
805 }
806
807 erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
808
809 if (IS_TRACED_FL(p, F_TRACE_GC)) {
810 trace_gc(p, gc_trace_end_tag, reclaimed_now, THE_NON_VALUE);
811 }
812
813 if (erts_system_monitor_long_gc != 0) {
814 ErtsMonotonicTime end_time;
815 Uint gc_time;
816 if (erts_test_long_gc_sleep)
817 while (0 != erts_milli_sleep(erts_test_long_gc_sleep));
818 end_time = erts_get_monotonic_time(esdp);
819 gc_time = (Uint) ERTS_MONOTONIC_TO_MSEC(end_time - start_time);
820 if (gc_time && gc_time > erts_system_monitor_long_gc) {
821 monitor_long_gc(p, gc_time);
822 }
823 }
824 if (erts_system_monitor_large_heap != 0) {
825 Uint size = HEAP_SIZE(p);
826 size += OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0;
827 if (size >= erts_system_monitor_large_heap)
828 monitor_large_heap(p);
829 }
830
831 if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
832 erts_mtx_lock(&dirty_gc.mtx);
833 dirty_gc.info.garbage_cols++;
834 dirty_gc.info.reclaimed += reclaimed_now;
835 erts_mtx_unlock(&dirty_gc.mtx);
836 }
837 else
838 {
839 esdp->gc_info.garbage_cols++;
840 esdp->gc_info.reclaimed += reclaimed_now;
841 }
842
843 FLAGS(p) &= ~(F_FORCE_GC|F_HIBERNATED);
844 p->live_hf_end = ERTS_INVALID_HFRAG_PTR;
845
846 ERTS_MSACC_POP_STATE();
847
848 #ifdef CHECK_FOR_HOLES
849 /*
850 * We intentionally do not rescan the areas copied by the GC.
851 * We trust the GC not to leave any holes.
852 */
853 p->last_htop = p->htop;
854 p->last_mbuf = 0;
855 #endif
856
857 #ifdef DEBUG
858 /*
859 * The scanning for pointers from the old_heap into the new_heap or
860 * heap fragments turned out to be costly, so we remember how far we
861 * have scanned this time and will start scanning there next time.
862 * (We will not detect wild writes into the old heap, or modifications
863 * of the old heap in-between garbage collections.)
864 */
865 p->last_old_htop = p->old_htop;
866 #endif
867
868 ASSERT(!p->mbuf);
869 ASSERT(!ERTS_IS_GC_DESIRED(p));
870 ASSERT(need <= HEAP_LIMIT(p) - HEAP_TOP(p));
871
872 return reds;
873 }
874
875 int
erts_garbage_collect_nobump(Process * p,int need,Eterm * objv,int nobj,int fcalls)876 erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls)
877 {
878 int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls, 0);
879 int reds_left = ERTS_REDS_LEFT(p, fcalls);
880 if (reds > reds_left)
881 reds = reds_left;
882 ASSERT(CONTEXT_REDS - (reds_left - reds) >= erts_proc_sched_data(p)->virtual_reds);
883 return reds;
884 }
885
886 void
erts_garbage_collect(Process * p,int need,Eterm * objv,int nobj)887 erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
888 {
889 int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls, 0);
890 BUMP_REDS(p, reds);
891 ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p)
892 >= erts_proc_sched_data(p)->virtual_reds);
893 }
894
895
896 /*
897 * Place all living data on a the new heap; deallocate any old heap.
898 * Meant to be used by hibernate/3.
899 */
900 static int
garbage_collect_hibernate(Process * p,int check_long_gc)901 garbage_collect_hibernate(Process* p, int check_long_gc)
902 {
903 Uint heap_size;
904 Eterm* heap;
905 Eterm* htop;
906 Uint actual_size;
907 char* area;
908 Uint area_size;
909 Sint offs;
910 int reds;
911
912 if (p->flags & F_DISABLE_GC)
913 ERTS_INTERNAL_ERROR("GC disabled");
914
915 if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)))
916 p->flags &= ~(F_DIRTY_GC_HIBERNATE|F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC);
917 else if (check_long_gc) {
918 Uint flags = p->flags;
919 p->flags |= F_NEED_FULLSWEEP;
920 check_for_possibly_long_gc(p, (p->htop - p->heap) + p->mbuf_sz);
921 if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
922 p->flags = flags|F_DIRTY_GC_HIBERNATE;
923 return 1;
924 }
925 p->flags = flags;
926 }
927 /*
928 * Preliminaries.
929 */
930 erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
931 ErtsGcQuickSanityCheck(p);
932 ASSERT(p->stop == p->hend); /* Stack must be empty. */
933
934 /*
935 * Do it.
936 */
937
938 heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz;
939
940 heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP,
941 sizeof(Eterm)*heap_size);
942 htop = heap;
943
944 htop = full_sweep_heaps(p,
945 ERTS_INVALID_HFRAG_PTR,
946 1,
947 heap,
948 htop,
949 (char *) p->old_heap,
950 (char *) p->old_htop - (char *) p->old_heap,
951 p->arg_reg,
952 p->arity);
953
954 #ifdef HARDDEBUG
955 disallow_heap_frag_ref_in_heap(p, heap, htop);
956 #endif
957
958 erts_deallocate_young_generation(p);
959
960 p->heap = heap;
961 p->high_water = htop;
962 p->htop = htop;
963 p->hend = p->heap + heap_size;
964 p->stop = p->hend;
965 p->heap_sz = heap_size;
966
967 heap_size = actual_size = p->htop - p->heap;
968 if (heap_size == 0) {
969 heap_size = 1; /* We want a heap... */
970 }
971
972 FLAGS(p) &= ~F_FORCE_GC;
973 p->live_hf_end = ERTS_INVALID_HFRAG_PTR;
974
975 /*
976 * Move the heap to its final destination.
977 *
978 * IMPORTANT: We have garbage collected to a temporary heap and
979 * then copy the result to a newly allocated heap of exact size.
980 * This is intentional and important! Garbage collecting as usual
981 * and then shrinking the heap by reallocating it caused serious
982 * fragmentation problems when large amounts of processes were
983 * hibernated.
984 */
985
986 ASSERT(p->hend - p->stop == 0); /* Empty stack */
987 ASSERT(actual_size < p->heap_sz);
988
989 heap = ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*heap_size);
990 sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm));
991 ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm));
992
993 p->stop = p->hend = heap + heap_size;
994
995 offs = heap - p->heap;
996 area = (char *) p->heap;
997 area_size = ((char *) p->htop) - area;
998 offset_heap(heap, actual_size, offs, area, area_size);
999 p->high_water = heap + (p->high_water - p->heap);
1000 offset_rootset(p, offs, area, area_size, p->arg_reg, p->arity);
1001 p->htop = heap + actual_size;
1002 p->heap = heap;
1003 p->heap_sz = heap_size;
1004
1005
1006 #ifdef CHECK_FOR_HOLES
1007 p->last_htop = p->htop;
1008 p->last_mbuf = 0;
1009 #endif
1010 #ifdef DEBUG
1011 p->last_old_htop = NULL;
1012 #endif
1013
1014 /*
1015 * Finishing.
1016 */
1017
1018 ErtsGcQuickSanityCheck(p);
1019
1020 p->flags |= F_HIBERNATED;
1021
1022 erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
1023
1024 reds = gc_cost(actual_size, actual_size);
1025 return reds;
1026 }
1027
1028 void
erts_garbage_collect_hibernate(Process * p)1029 erts_garbage_collect_hibernate(Process* p)
1030 {
1031 int reds = garbage_collect_hibernate(p, 1);
1032 BUMP_REDS(p, reds);
1033 }
1034
1035 /*
1036 * HiPE native code stack scanning procedures:
1037 * - fullsweep_nstack()
1038 * - gensweep_nstack()
1039 * - offset_nstack()
1040 * - sweep_literals_nstack()
1041 */
1042 #if defined(HIPE)
1043
1044 #define GENSWEEP_NSTACK(p,old_htop,n_htop) \
1045 do { \
1046 Eterm *tmp_old_htop = old_htop; \
1047 Eterm *tmp_n_htop = n_htop; \
1048 gensweep_nstack((p), &tmp_old_htop, &tmp_n_htop); \
1049 old_htop = tmp_old_htop; \
1050 n_htop = tmp_n_htop; \
1051 } while(0)
1052
1053 /*
1054 * offset_nstack() can ignore the descriptor-based traversal the other
1055 * nstack procedures use and simply call offset_heap_ptr() instead.
1056 * This relies on two facts:
1057 * 1. The only live non-Erlang terms on an nstack are return addresses,
1058 * and they will be skipped thanks to the low/high range check.
1059 * 2. Dead values, even if mistaken for pointers into the low/high area,
1060 * can be offset safely since they won't be dereferenced.
1061 *
1062 * XXX: WARNING: If HiPE starts storing other non-Erlang values on the
1063 * nstack, such as floats, then this will have to be changed.
1064 */
offset_nstack(Process * p,Sint offs,char * area,Uint area_size)1065 static ERTS_INLINE void offset_nstack(Process* p, Sint offs,
1066 char* area, Uint area_size)
1067 {
1068 if (p->hipe.nstack) {
1069 ASSERT(p->hipe.nsp && p->hipe.nstend);
1070 offset_heap_ptr(hipe_nstack_start(p), hipe_nstack_used(p),
1071 offs, area, area_size);
1072 }
1073 else {
1074 ASSERT(!p->hipe.nsp && !p->hipe.nstend);
1075 }
1076 }
1077
1078 #else /* !HIPE */
1079
1080 #define fullsweep_nstack(p,n_htop) (n_htop)
1081 #define GENSWEEP_NSTACK(p,old_htop,n_htop) do{}while(0)
1082 #define offset_nstack(p,offs,area,area_size) do{}while(0)
1083 #define sweep_literals_nstack(p,old_htop,area,area_size) (old_htop)
1084
1085 #endif /* HIPE */
1086
1087 int
erts_garbage_collect_literals(Process * p,Eterm * literals,Uint byte_lit_size,struct erl_off_heap_header * oh,int fcalls)1088 erts_garbage_collect_literals(Process* p, Eterm* literals,
1089 Uint byte_lit_size,
1090 struct erl_off_heap_header* oh,
1091 int fcalls)
1092 {
1093 Uint lit_size = byte_lit_size / sizeof(Eterm);
1094 Uint old_heap_size;
1095 Eterm* temp_lit;
1096 Sint offs;
1097 Rootset rootset; /* Rootset for GC (stack, dictionary, etc). */
1098 Roots* roots;
1099 char* area;
1100 Uint area_size;
1101 Eterm* old_htop;
1102 Uint n;
1103 Uint ygen_usage = 0;
1104 struct erl_off_heap_header** prev = NULL;
1105 Sint64 reds;
1106 int hibernated = !!(p->flags & F_HIBERNATED);
1107
1108 if (p->flags & (F_DISABLE_GC|F_DELAY_GC))
1109 ERTS_INTERNAL_ERROR("GC disabled");
1110
1111 /*
1112 * First an ordinary major collection...
1113 */
1114
1115 p->flags |= F_NEED_FULLSWEEP;
1116
1117 if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)))
1118 p->flags &= ~F_DIRTY_CLA;
1119 else {
1120 Uint size = byte_lit_size/sizeof(Uint);
1121 ygen_usage = young_gen_usage(p);
1122 if (hibernated)
1123 size = size*2 + 3*ygen_usage;
1124 else
1125 size = size + 2*ygen_usage;
1126 check_for_possibly_long_gc(p, size);
1127 if (p->flags & F_DIRTY_MAJOR_GC) {
1128 p->flags |= F_DIRTY_CLA;
1129 return 10;
1130 }
1131 }
1132
1133 reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0,
1134 p->arg_reg, p->arity, fcalls,
1135 ygen_usage);
1136 if (ERTS_PROC_IS_EXITING(p)) {
1137 return 0;
1138 }
1139
1140 ASSERT(!(p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)));
1141
1142 if (MAX_HEAP_SIZE_GET(p)) {
1143 Uint new_heap_size;
1144 Uint old_heap_size;
1145 Uint total_heap_size;
1146
1147 new_heap_size = HEAP_END(p) - HEAP_START(p);
1148 old_heap_size = erts_next_heap_size(lit_size, 0);
1149 total_heap_size = new_heap_size + old_heap_size;
1150 if (MAX_HEAP_SIZE_GET(p) < total_heap_size &&
1151 reached_max_heap_size(p, total_heap_size,
1152 new_heap_size, old_heap_size)) {
1153 erts_set_self_exiting(p, am_killed);
1154 return 0;
1155 }
1156 }
1157
1158 /*
1159 * Set GC state.
1160 */
1161 erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
1162
1163 /*
1164 * Just did a major collection (which has discarded the old heap),
1165 * so that we don't have to cope with pointer to literals on the
1166 * old heap. We will now allocate an old heap to contain the literals.
1167 */
1168
1169 ASSERT(p->old_heap == 0); /* Must NOT have an old heap yet. */
1170 old_heap_size = erts_next_heap_size(lit_size, 0);
1171 p->old_heap = p->old_htop = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_OLD_HEAP,
1172 sizeof(Eterm)*old_heap_size);
1173 p->old_hend = p->old_heap + old_heap_size;
1174
1175 /*
1176 * We soon want to garbage collect the literals. But since a GC is
1177 * destructive (MOVED markers are written), we must copy the literals
1178 * to a temporary area and change all references to literals.
1179 */
1180 temp_lit = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, byte_lit_size);
1181 sys_memcpy(temp_lit, literals, byte_lit_size);
1182 offs = temp_lit - literals;
1183 offset_heap(temp_lit, lit_size, offs, (char *) literals, byte_lit_size);
1184 offset_heap(p->heap, p->htop - p->heap, offs, (char *) literals, byte_lit_size);
1185 offset_rootset(p, offs, (char *) literals, byte_lit_size, p->arg_reg, p->arity);
1186 if (oh) {
1187 oh = (struct erl_off_heap_header *) ((Eterm *)(void *) oh + offs);
1188 }
1189
1190 /*
1191 * Now the literals are placed in memory that is safe to write into,
1192 * so now we GC the literals into the old heap. First we go through the
1193 * rootset.
1194 */
1195
1196 area = (char *) temp_lit;
1197 area_size = byte_lit_size;
1198 n = setup_rootset(p, p->arg_reg, p->arity, &rootset);
1199 roots = rootset.roots;
1200 old_htop = sweep_literals_nstack(p, p->old_htop, area, area_size);
1201 while (n--) {
1202 Eterm* g_ptr = roots->v;
1203 Uint g_sz = roots->sz;
1204 Eterm* ptr;
1205 Eterm val;
1206
1207 roots++;
1208
1209 for ( ; g_sz--; g_ptr++) {
1210 Eterm gval = *g_ptr;
1211
1212 switch (primary_tag(gval)) {
1213 case TAG_PRIMARY_BOXED:
1214 ptr = boxed_val(gval);
1215 val = *ptr;
1216 if (IS_MOVED_BOXED(val)) {
1217 ASSERT(is_boxed(val));
1218 *g_ptr = val;
1219 } else if (ErtsInArea(ptr, area, area_size)) {
1220 move_boxed(ptr,val,&old_htop,g_ptr);
1221 }
1222 break;
1223 case TAG_PRIMARY_LIST:
1224 ptr = list_val(gval);
1225 val = *ptr;
1226 if (IS_MOVED_CONS(val)) { /* Moved */
1227 *g_ptr = ptr[1];
1228 } else if (ErtsInArea(ptr, area, area_size)) {
1229 move_cons(ptr,val,&old_htop,g_ptr);
1230 }
1231 break;
1232 default:
1233 break;
1234 }
1235 }
1236 }
1237 ASSERT(p->old_htop <= old_htop && old_htop <= p->old_hend);
1238 cleanup_rootset(&rootset);
1239
1240 /*
1241 * Now all references in the rootset to the literals have been updated.
1242 * Now we'll have to go through all heaps updating all other references.
1243 */
1244
1245 old_htop = sweep_literals_to_old_heap(p->heap, p->htop, old_htop, area, area_size);
1246 old_htop = sweep_literal_area(p->old_heap, old_htop,
1247 (char *) p->old_heap, sizeof(Eterm)*old_heap_size,
1248 area, area_size);
1249 ASSERT(p->old_htop <= old_htop && old_htop <= p->old_hend);
1250 p->old_htop = old_htop;
1251
1252 /*
1253 * Prepare to sweep off-heap objects. Since all MSOs on the new
1254 * heap must be come before MSOs on the old heap, find the end of
1255 * current MSO list and use that as a starting point.
1256 */
1257
1258 if (oh) {
1259 prev = &MSO(p).first;
1260 while (*prev) {
1261 prev = &(*prev)->next;
1262 }
1263 }
1264
1265 /*
1266 * Sweep through all off-heap objects in the temporary literal area.
1267 */
1268
1269 while (oh) {
1270 if (IS_MOVED_BOXED(oh->thing_word)) {
1271 struct erl_off_heap_header* ptr;
1272
1273 /*
1274 * This off-heap object has been copied to the heap.
1275 * We must increment its reference count and
1276 * link it into the MSO list for the process.
1277 */
1278
1279 ptr = (struct erl_off_heap_header*) boxed_val(oh->thing_word);
1280 switch (thing_subtag(ptr->thing_word)) {
1281 case REFC_BINARY_SUBTAG:
1282 {
1283 Binary* bptr = ((ProcBin*)ptr)->val;
1284 erts_refc_inc(&bptr->intern.refc, 1);
1285 break;
1286 }
1287 case FUN_SUBTAG:
1288 {
1289 ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe;
1290 erts_refc_inc(&fe->refc, 1);
1291 break;
1292 }
1293 case REF_SUBTAG:
1294 {
1295 ErtsMagicBinary *bptr;
1296 ASSERT(is_magic_ref_thing(ptr));
1297 bptr = ((ErtsMRefThing *) ptr)->mb;
1298 erts_refc_inc(&bptr->intern.refc, 1);
1299 break;
1300 }
1301 default:
1302 {
1303 ExternalThing *etp;
1304 ASSERT(is_external_header(ptr->thing_word));
1305 etp = (ExternalThing *) ptr;
1306 erts_refc_inc(&etp->node->refc, 1);
1307 break;
1308 }
1309 }
1310 *prev = ptr;
1311 prev = &ptr->next;
1312 }
1313 oh = oh->next;
1314 }
1315
1316 if (prev) {
1317 *prev = NULL;
1318 }
1319
1320 /*
1321 * We no longer need this temporary area.
1322 */
1323 erts_free(ERTS_ALC_T_TMP, (void *) temp_lit);
1324
1325 /*
1326 * Restore status.
1327 */
1328 erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
1329
1330 reds += (Sint64) gc_cost((p->htop - p->heap) + byte_lit_size/sizeof(Uint), 0);
1331
1332 if (hibernated) {
1333 /* Restore the process into hibernated state... */
1334 reds += garbage_collect_hibernate(p, 0);
1335 }
1336
1337 if (reds > INT_MAX)
1338 return INT_MAX;
1339 return (int) reds;
1340 }
1341
1342 static int
minor_collection(Process * p,ErlHeapFragment * live_hf_end,int need,Eterm * objv,int nobj,Uint ygen_usage,Uint * recl)1343 minor_collection(Process* p, ErlHeapFragment *live_hf_end,
1344 int need, Eterm* objv, int nobj,
1345 Uint ygen_usage, Uint *recl)
1346 {
1347 Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap;
1348 Uint mature_size = p->high_water - mature;
1349 Uint size_before = ygen_usage;
1350
1351 /*
1352 * Check if we have gone past the max heap size limit
1353 */
1354
1355 if (MAX_HEAP_SIZE_GET(p)) {
1356 Uint heap_size = size_before,
1357 /* Note that we also count the un-allocated area
1358 in between the stack and heap */
1359 stack_size = HEAP_END(p) - HEAP_TOP(p),
1360 extra_heap_size,
1361 extra_old_heap_size = 0;
1362
1363 /* Add potential old heap size */
1364 if (OLD_HEAP(p) == NULL && mature_size != 0) {
1365 extra_old_heap_size = erts_next_heap_size(size_before, 1);
1366 heap_size += extra_old_heap_size;
1367 } else if (OLD_HEAP(p))
1368 heap_size += OLD_HEND(p) - OLD_HEAP(p);
1369
1370 /* Add potential new young heap size */
1371 extra_heap_size = next_heap_size(p, stack_size + size_before, 0);
1372 heap_size += extra_heap_size;
1373
1374 if (heap_size > MAX_HEAP_SIZE_GET(p))
1375 if (reached_max_heap_size(p, heap_size, extra_heap_size, extra_old_heap_size))
1376 return -2;
1377 }
1378
1379 /*
1380 * Allocate an old heap if we don't have one and if we'll need one.
1381 */
1382
1383 if (OLD_HEAP(p) == NULL && mature_size != 0) {
1384 Eterm* n_old;
1385
1386 /* Note: We choose a larger heap size than strictly needed,
1387 * which seems to reduce the number of fullsweeps.
1388 * This improved Estone by more than 1200 estones on my computer
1389 * (Ultra Sparc 10).
1390 */
1391 Uint new_sz = erts_next_heap_size(size_before, 1);
1392
1393 /* Create new, empty old_heap */
1394 n_old = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_OLD_HEAP,
1395 sizeof(Eterm)*new_sz);
1396
1397 OLD_HEND(p) = n_old + new_sz;
1398 OLD_HEAP(p) = OLD_HTOP(p) = n_old;
1399 }
1400
1401 /*
1402 * Do a minor collection if there is an old heap and if it
1403 * is large enough.
1404 */
1405
1406 if (OLD_HEAP(p) &&
1407 ((mature_size <= OLD_HEND(p) - OLD_HTOP(p)) &&
1408 ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
1409 Eterm *prev_old_htop;
1410 Uint stack_size, size_after, adjust_size, need_after, new_sz, new_mature;
1411
1412 stack_size = p->hend - p->stop;
1413 new_sz = stack_size + size_before;
1414 new_sz = next_heap_size(p, new_sz, 0);
1415
1416 prev_old_htop = p->old_htop;
1417 do_minor(p, live_hf_end, (char *) mature, mature_size*sizeof(Eterm),
1418 new_sz, objv, nobj);
1419
1420 if (p->flags & F_ON_HEAP_MSGQ)
1421 move_msgs_to_heap(p);
1422
1423 new_mature = p->old_htop - prev_old_htop;
1424
1425 size_after = new_mature;
1426 size_after += HEAP_TOP(p) - HEAP_START(p) + p->mbuf_sz;
1427 *recl += (size_before - size_after);
1428
1429 ErtsGcQuickSanityCheck(p);
1430
1431 GEN_GCS(p)++;
1432 need_after = ((HEAP_TOP(p) - HEAP_START(p))
1433 + need
1434 + stack_size);
1435
1436 /*
1437 * Excessively large heaps should be shrunk, but
1438 * don't even bother on reasonable small heaps.
1439 *
1440 * The reason for this is that after tenuring, we often
1441 * use a really small portion of new heap, therefore, unless
1442 * the heap size is substantial, we don't want to shrink.
1443 */
1444
1445 adjust_size = 0;
1446
1447 if ((HEAP_SIZE(p) > 3000) && (4 * need_after < HEAP_SIZE(p)) &&
1448 ((HEAP_SIZE(p) > 8000) ||
1449 (HEAP_SIZE(p) > (OLD_HEND(p) - OLD_HEAP(p))))) {
1450 Uint wanted = 3 * need_after;
1451 Uint old_heap_sz = OLD_HEND(p) - OLD_HEAP(p);
1452
1453 /*
1454 * Additional test to make sure we don't make the heap too small
1455 * compared to the size of the older generation heap.
1456 */
1457 if (wanted*9 < old_heap_sz) {
1458 Uint new_wanted = old_heap_sz / 8;
1459 if (new_wanted > wanted) {
1460 wanted = new_wanted;
1461 }
1462 }
1463
1464 wanted = wanted < MIN_HEAP_SIZE(p) ? MIN_HEAP_SIZE(p)
1465 : next_heap_size(p, wanted, 0);
1466 if (wanted < HEAP_SIZE(p)) {
1467 shrink_new_heap(p, wanted, objv, nobj);
1468 adjust_size = p->htop - p->heap;
1469 }
1470
1471 }
1472 else if (need_after > HEAP_SIZE(p)) {
1473 grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj);
1474 adjust_size = p->htop - p->heap;
1475 }
1476 /*else: The heap size turned out to be just right. We are done. */
1477
1478 ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
1479
1480 /* The heap usage during GC should be larger than what we end up
1481 after a GC, even if we grow it. If this assertion is not true
1482 we have to check size in grow_new_heap and potentially kill the
1483 process from there */
1484 ASSERT(!MAX_HEAP_SIZE_GET(p) ||
1485 !(MAX_HEAP_SIZE_FLAGS_GET(p) & MAX_HEAP_SIZE_KILL) ||
1486 MAX_HEAP_SIZE_GET(p) > (young_gen_usage(p) +
1487 (OLD_HEND(p) - OLD_HEAP(p)) +
1488 (HEAP_END(p) - HEAP_TOP(p))));
1489
1490 return gc_cost(size_after, adjust_size);
1491 }
1492
1493 /*
1494 * Not enough room for a minor collection. Must force a major collection.
1495 */
1496 return -1;
1497 }
1498
1499 static void
do_minor(Process * p,ErlHeapFragment * live_hf_end,char * mature,Uint mature_size,Uint new_sz,Eterm * objv,int nobj)1500 do_minor(Process *p, ErlHeapFragment *live_hf_end,
1501 char *mature, Uint mature_size,
1502 Uint new_sz, Eterm* objv, int nobj)
1503 {
1504 Rootset rootset; /* Rootset for GC (stack, dictionary, etc). */
1505 Roots* roots;
1506 Eterm* n_htop;
1507 Uint n;
1508 Eterm* ptr;
1509 Eterm val;
1510 Eterm gval;
1511 Eterm* old_htop = OLD_HTOP(p);
1512 Eterm* n_heap;
1513 char* oh = (char *) OLD_HEAP(p);
1514 Uint oh_size = (char *) OLD_HTOP(p) - oh;
1515
1516 VERBOSE(DEBUG_SHCOPY, ("[pid=%T] MINOR GC: %p %p %p %p\n", p->common.id,
1517 HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)));
1518
1519 n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
1520 sizeof(Eterm)*new_sz);
1521
1522 n = setup_rootset(p, objv, nobj, &rootset);
1523 roots = rootset.roots;
1524
1525 /*
1526 * All allocations done. Start defile heap with move markers.
1527 * A crash dump due to allocation failure above will see a healthy heap.
1528 */
1529
1530 if (live_hf_end != ERTS_INVALID_HFRAG_PTR) {
1531 /*
1532 * Move heap frags that we know are completely live
1533 * directly into the new young heap generation.
1534 */
1535 n_htop = collect_live_heap_frags(p, live_hf_end, n_htop);
1536 }
1537
1538 GENSWEEP_NSTACK(p, old_htop, n_htop);
1539 while (n--) {
1540 Eterm* g_ptr = roots->v;
1541 Uint g_sz = roots->sz;
1542
1543 roots++;
1544 for ( ; g_sz--; g_ptr++) {
1545 gval = *g_ptr;
1546
1547 switch (primary_tag(gval)) {
1548
1549 case TAG_PRIMARY_BOXED: {
1550 ptr = boxed_val(gval);
1551 val = *ptr;
1552 if (IS_MOVED_BOXED(val)) {
1553 ASSERT(is_boxed(val));
1554 *g_ptr = val;
1555 } else if (ErtsInArea(ptr, mature, mature_size)) {
1556 move_boxed(ptr,val,&old_htop,g_ptr);
1557 } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
1558 move_boxed(ptr,val,&n_htop,g_ptr);
1559 }
1560 break;
1561 }
1562
1563 case TAG_PRIMARY_LIST: {
1564 ptr = list_val(gval);
1565 val = *ptr;
1566 if (IS_MOVED_CONS(val)) { /* Moved */
1567 *g_ptr = ptr[1];
1568 } else if (ErtsInArea(ptr, mature, mature_size)) {
1569 move_cons(ptr,val,&old_htop,g_ptr);
1570 } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
1571 move_cons(ptr,val,&n_htop,g_ptr);
1572 }
1573 break;
1574 }
1575 default:
1576 break;
1577 }
1578 }
1579 }
1580
1581 cleanup_rootset(&rootset);
1582
1583 /*
1584 * Now all references in the rootset point to the new heap. However,
1585 * most references on the new heap point to the old heap so the next stage
1586 * is to scan through the new heap evacuating data from the old heap
1587 * until all is changed.
1588 */
1589
1590 if (mature_size == 0) {
1591 n_htop = sweep_new_heap(n_heap, n_htop, oh, oh_size);
1592 } else {
1593 Eterm* n_hp = n_heap;
1594 Eterm* ptr;
1595 Eterm val;
1596 Eterm gval;
1597
1598 while (n_hp != n_htop) {
1599 ASSERT(n_hp < n_htop);
1600 gval = *n_hp;
1601 switch (primary_tag(gval)) {
1602 case TAG_PRIMARY_BOXED: {
1603 ptr = boxed_val(gval);
1604 val = *ptr;
1605 if (IS_MOVED_BOXED(val)) {
1606 ASSERT(is_boxed(val));
1607 *n_hp++ = val;
1608 } else if (ErtsInArea(ptr, mature, mature_size)) {
1609 move_boxed(ptr,val,&old_htop,n_hp++);
1610 } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
1611 move_boxed(ptr,val,&n_htop,n_hp++);
1612 } else {
1613 n_hp++;
1614 }
1615 break;
1616 }
1617 case TAG_PRIMARY_LIST: {
1618 ptr = list_val(gval);
1619 val = *ptr;
1620 if (IS_MOVED_CONS(val)) {
1621 *n_hp++ = ptr[1];
1622 } else if (ErtsInArea(ptr, mature, mature_size)) {
1623 move_cons(ptr,val,&old_htop,n_hp++);
1624 } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
1625 move_cons(ptr,val,&n_htop,n_hp++);
1626 } else {
1627 n_hp++;
1628 }
1629 break;
1630 }
1631 case TAG_PRIMARY_HEADER: {
1632 if (!header_is_thing(gval))
1633 n_hp++;
1634 else {
1635 if (header_is_bin_matchstate(gval)) {
1636 ErlBinMatchState *ms = (ErlBinMatchState*) n_hp;
1637 ErlBinMatchBuffer *mb = &(ms->mb);
1638 Eterm* origptr = &(mb->orig);
1639 ptr = boxed_val(*origptr);
1640 val = *ptr;
1641 if (IS_MOVED_BOXED(val)) {
1642 *origptr = val;
1643 mb->base = binary_bytes(val);
1644 } else if (ErtsInArea(ptr, mature, mature_size)) {
1645 move_boxed(ptr,val,&old_htop,origptr);
1646 mb->base = binary_bytes(mb->orig);
1647 } else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) {
1648 move_boxed(ptr,val,&n_htop,origptr);
1649 mb->base = binary_bytes(mb->orig);
1650 }
1651 }
1652 n_hp += (thing_arityval(gval)+1);
1653 }
1654 break;
1655 }
1656 default:
1657 n_hp++;
1658 break;
1659 }
1660 }
1661 }
1662
1663 /*
1664 * And also if we have been tenuring, references on the second generation
1665 * may point to the old (soon to be deleted) new_heap.
1666 */
1667
1668 if (OLD_HTOP(p) < old_htop)
1669 old_htop = sweep_new_heap(OLD_HTOP(p), old_htop, oh, oh_size);
1670 OLD_HTOP(p) = old_htop;
1671 HIGH_WATER(p) = n_htop;
1672
1673 if (MSO(p).first) {
1674 sweep_off_heap(p, 0);
1675 }
1676
1677 #ifdef HARDDEBUG
1678 /*
1679 * Go through the old_heap before, and try to find references from the old_heap
1680 * into the old new_heap that has just been evacuated and is about to be freed
1681 * (as well as looking for reference into heap fragments, of course).
1682 */
1683 disallow_heap_frag_ref_in_old_heap(p);
1684 #endif
1685
1686 /* Copy stack to end of new heap */
1687 n = p->hend - p->stop;
1688 sys_memcpy(n_heap + new_sz - n, p->stop, n * sizeof(Eterm));
1689 p->stop = n_heap + new_sz - n;
1690
1691 #ifdef USE_VM_PROBES
1692 if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) {
1693 DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
1694
1695 dtrace_proc_str(p, pidbuf);
1696 DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
1697 }
1698 #endif
1699
1700 #ifdef HARDDEBUG
1701 disallow_heap_frag_ref_in_heap(p, n_heap, n_htop);
1702 #endif
1703
1704 erts_deallocate_young_generation(p);
1705
1706 HEAP_START(p) = n_heap;
1707 HEAP_TOP(p) = n_htop;
1708 HEAP_SIZE(p) = new_sz;
1709 HEAP_END(p) = n_heap + new_sz;
1710 }
1711
1712 /*
1713 * Major collection. DISCARD the old heap.
1714 */
1715
1716 static int
major_collection(Process * p,ErlHeapFragment * live_hf_end,int need,Eterm * objv,int nobj,Uint ygen_usage,Uint * recl)1717 major_collection(Process* p, ErlHeapFragment *live_hf_end,
1718 int need, Eterm* objv, int nobj,
1719 Uint ygen_usage, Uint *recl)
1720 {
1721 Uint size_before, size_after, stack_size;
1722 Eterm* n_heap;
1723 Eterm* n_htop;
1724 char* oh = (char *) OLD_HEAP(p);
1725 Uint oh_size = (char *) OLD_HTOP(p) - oh;
1726 Uint new_sz, stk_sz;
1727 int adjusted;
1728
1729 VERBOSE(DEBUG_SHCOPY, ("[pid=%T] MAJOR GC: %p %p %p %p\n", p->common.id,
1730 HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p)));
1731
1732 /*
1733 * Do a fullsweep GC. First figure out the size of the heap
1734 * to receive all live data.
1735 */
1736
1737 size_before = ygen_usage;
1738 size_before += p->old_htop - p->old_heap;
1739 stack_size = p->hend - p->stop;
1740
1741 new_sz = stack_size + size_before;
1742 new_sz = next_heap_size(p, new_sz, 0);
1743
1744 /*
1745 * Should we grow although we don't actually need to?
1746 */
1747
1748 if (new_sz == HEAP_SIZE(p) && FLAGS(p) & F_HEAP_GROW) {
1749 new_sz = next_heap_size(p, HEAP_SIZE(p), 1);
1750 }
1751
1752
1753 if (MAX_HEAP_SIZE_GET(p)) {
1754 Uint heap_size = size_before;
1755
1756 /* Add unused space in old heap */
1757 heap_size += OLD_HEND(p) - OLD_HTOP(p);
1758
1759 /* Add stack + unused space in young heap */
1760 heap_size += HEAP_END(p) - HEAP_TOP(p);
1761
1762 /* Add size of new young heap */
1763 heap_size += new_sz;
1764
1765 if (MAX_HEAP_SIZE_GET(p) < heap_size)
1766 if (reached_max_heap_size(p, heap_size, new_sz, 0))
1767 return -2;
1768 }
1769
1770 FLAGS(p) &= ~(F_HEAP_GROW|F_NEED_FULLSWEEP);
1771 n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
1772 sizeof(Eterm)*new_sz);
1773
1774 n_htop = full_sweep_heaps(p, live_hf_end, 0, n_heap, n_htop, oh, oh_size,
1775 objv, nobj);
1776
1777 /* Move the stack to the end of the heap */
1778 stk_sz = HEAP_END(p) - p->stop;
1779 sys_memcpy(n_heap + new_sz - stk_sz, p->stop, stk_sz * sizeof(Eterm));
1780 p->stop = n_heap + new_sz - stk_sz;
1781
1782 #ifdef USE_VM_PROBES
1783 if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) {
1784 DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
1785
1786 dtrace_proc_str(p, pidbuf);
1787 DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
1788 }
1789 #endif
1790
1791 #ifdef HARDDEBUG
1792 disallow_heap_frag_ref_in_heap(p, n_heap, n_htop);
1793 #endif
1794
1795 erts_deallocate_young_generation(p);
1796
1797 HEAP_START(p) = n_heap;
1798 HEAP_TOP(p) = n_htop;
1799 HEAP_SIZE(p) = new_sz;
1800 HEAP_END(p) = n_heap + new_sz;
1801 GEN_GCS(p) = 0;
1802
1803 HIGH_WATER(p) = HEAP_TOP(p);
1804
1805 if (p->flags & F_ON_HEAP_MSGQ)
1806 move_msgs_to_heap(p);
1807
1808 ErtsGcQuickSanityCheck(p);
1809
1810 size_after = HEAP_TOP(p) - HEAP_START(p) + p->mbuf_sz;
1811 *recl += size_before - size_after;
1812
1813 adjusted = adjust_after_fullsweep(p, need, objv, nobj);
1814
1815 ErtsGcQuickSanityCheck(p);
1816
1817 return gc_cost(size_after, adjusted ? size_after : 0);
1818 }
1819
1820 static Eterm *
full_sweep_heaps(Process * p,ErlHeapFragment * live_hf_end,int hibernate,Eterm * n_heap,Eterm * n_htop,char * oh,Uint oh_size,Eterm * objv,int nobj)1821 full_sweep_heaps(Process *p,
1822 ErlHeapFragment *live_hf_end,
1823 int hibernate,
1824 Eterm *n_heap, Eterm* n_htop,
1825 char *oh, Uint oh_size,
1826 Eterm *objv, int nobj)
1827 {
1828 Rootset rootset;
1829 Roots *roots;
1830 Uint n;
1831
1832 /*
1833 * Copy all top-level terms directly referenced by the rootset to
1834 * the new new_heap.
1835 */
1836
1837 n = setup_rootset(p, objv, nobj, &rootset);
1838
1839 /*
1840 * All allocations done. Start defile heap with move markers.
1841 * A crash dump due to allocation failure above will see a healthy heap.
1842 */
1843
1844 if (live_hf_end != ERTS_INVALID_HFRAG_PTR) {
1845 /*
1846 * Move heap frags that we know are completely live
1847 * directly into the heap.
1848 */
1849 n_htop = collect_live_heap_frags(p, live_hf_end, n_htop);
1850 }
1851
1852 #ifdef HIPE
1853 if (hibernate)
1854 hipe_empty_nstack(p);
1855 else
1856 n_htop = fullsweep_nstack(p, n_htop);
1857 #endif
1858
1859 roots = rootset.roots;
1860 while (n--) {
1861 Eterm* g_ptr = roots->v;
1862 Eterm g_sz = roots->sz;
1863
1864 roots++;
1865 for ( ; g_sz--; g_ptr++) {
1866 Eterm* ptr;
1867 Eterm val;
1868 Eterm gval = *g_ptr;
1869
1870 switch (primary_tag(gval)) {
1871
1872 case TAG_PRIMARY_BOXED: {
1873 ptr = boxed_val(gval);
1874 val = *ptr;
1875 if (IS_MOVED_BOXED(val)) {
1876 ASSERT(is_boxed(val));
1877 *g_ptr = val;
1878 } else if (!erts_is_literal(gval, ptr)) {
1879 move_boxed(ptr,val,&n_htop,g_ptr);
1880 }
1881 break;
1882 }
1883
1884 case TAG_PRIMARY_LIST: {
1885 ptr = list_val(gval);
1886 val = *ptr;
1887 if (IS_MOVED_CONS(val)) {
1888 *g_ptr = ptr[1];
1889 } else if (!erts_is_literal(gval, ptr)) {
1890 move_cons(ptr,val,&n_htop,g_ptr);
1891 }
1892 break;
1893 }
1894
1895 default:
1896 break;
1897 }
1898 }
1899 }
1900
1901 cleanup_rootset(&rootset);
1902
1903 /*
1904 * Now all references on the stack point to the new heap. However,
1905 * most references on the new heap point to the old heap so the next stage
1906 * is to scan through the new heap evacuating data from the old heap
1907 * until all is copied.
1908 */
1909
1910 n_htop = sweep_heaps(n_heap, n_htop, oh, oh_size);
1911
1912 if (MSO(p).first) {
1913 sweep_off_heap(p, 1);
1914 }
1915
1916 if (OLD_HEAP(p) != NULL) {
1917 ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP,
1918 OLD_HEAP(p),
1919 (OLD_HEND(p) - OLD_HEAP(p)) * sizeof(Eterm));
1920 OLD_HEAP(p) = OLD_HTOP(p) = OLD_HEND(p) = NULL;
1921 }
1922
1923 return n_htop;
1924 }
1925
1926 static int
adjust_after_fullsweep(Process * p,int need,Eterm * objv,int nobj)1927 adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
1928 {
1929 int adjusted = 0;
1930 Uint wanted, sz, need_after;
1931 Uint stack_size = STACK_SZ_ON_HEAP(p);
1932
1933 /*
1934 * Resize the heap if needed.
1935 */
1936
1937 need_after = (HEAP_TOP(p) - HEAP_START(p)) + need + stack_size;
1938 if (HEAP_SIZE(p) < need_after) {
1939 /* Too small - grow to match requested need */
1940 sz = next_heap_size(p, need_after, 0);
1941 grow_new_heap(p, sz, objv, nobj);
1942 adjusted = 1;
1943 } else if (3 * HEAP_SIZE(p) < 4 * need_after){
1944 /* Need more than 75% of current, postpone to next GC.*/
1945 FLAGS(p) |= F_HEAP_GROW;
1946 } else if (4 * need_after < HEAP_SIZE(p) && HEAP_SIZE(p) > H_MIN_SIZE){
1947 /* We need less than 25% of the current heap, shrink.*/
1948 /* XXX - This is how it was done in the old GC:
1949 wanted = 4 * need_after;
1950 I think this is better as fullsweep is used mainly on
1951 small memory systems, but I could be wrong... */
1952 wanted = 2 * need_after;
1953
1954 sz = wanted < p->min_heap_size ? p->min_heap_size
1955 : next_heap_size(p, wanted, 0);
1956
1957 if (sz < HEAP_SIZE(p)) {
1958 shrink_new_heap(p, sz, objv, nobj);
1959 adjusted = 1;
1960 }
1961 }
1962 return adjusted;
1963 }
1964
1965 void
erts_deallocate_young_generation(Process * c_p)1966 erts_deallocate_young_generation(Process *c_p)
1967 {
1968 Eterm *orig_heap;
1969
1970 if (!c_p->abandoned_heap) {
1971 orig_heap = c_p->heap;
1972 ASSERT(!(c_p->flags & F_ABANDONED_HEAP_USE));
1973 }
1974 else {
1975 ErlHeapFragment *hfrag;
1976
1977 orig_heap = c_p->abandoned_heap;
1978 c_p->abandoned_heap = NULL;
1979 c_p->flags &= ~F_ABANDONED_HEAP_USE;
1980
1981 /*
1982 * Temporary heap located in heap fragment
1983 * only referred to by 'c_p->heap'. Add it to
1984 * 'c_p->mbuf' list and deallocate it as any
1985 * other heap fragment...
1986 */
1987 hfrag = ((ErlHeapFragment *)
1988 (((char *) c_p->heap)
1989 - offsetof(ErlHeapFragment, mem)));
1990
1991 ASSERT(!hfrag->off_heap.first);
1992 ASSERT(!hfrag->off_heap.overhead);
1993 ASSERT(!hfrag->next);
1994 ASSERT(c_p->htop - c_p->heap <= hfrag->alloc_size);
1995
1996 hfrag->next = c_p->mbuf;
1997 c_p->mbuf = hfrag;
1998 }
1999
2000 if (c_p->mbuf) {
2001 free_message_buffer(c_p->mbuf);
2002 c_p->mbuf = NULL;
2003 }
2004 if (c_p->msg_frag) {
2005 erts_cleanup_messages(c_p->msg_frag);
2006 c_p->msg_frag = NULL;
2007 }
2008 c_p->mbuf_sz = 0;
2009
2010 ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
2011 orig_heap,
2012 c_p->heap_sz * sizeof(Eterm));
2013 }
2014
2015 #ifdef HARDDEBUG
2016
2017 /*
2018 * Routines to verify that we don't have pointer into heap fragments from
2019 * that are not allowed to have them.
2020 *
2021 * For performance reasons, we use _unchecked_list_val(), _unchecked_boxed_val(),
2022 * and so on to avoid a function call.
2023 */
2024
2025 static void
disallow_heap_frag_ref_in_heap(Process * p,Eterm * heap,Eterm * htop)2026 disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop)
2027 {
2028 Eterm* hp;
2029 Uint heap_size;
2030
2031 if (p->mbuf == 0) {
2032 return;
2033 }
2034
2035 heap_size = (htop - heap)*sizeof(Eterm);
2036
2037 hp = heap;
2038 while (hp < htop) {
2039 ErlHeapFragment* qb;
2040 Eterm* ptr;
2041 Eterm val;
2042
2043 val = *hp++;
2044 switch (primary_tag(val)) {
2045 case TAG_PRIMARY_BOXED:
2046 ptr = _unchecked_boxed_val(val);
2047 if (!ErtsInArea(ptr, heap, heap_size)) {
2048 for (qb = MBUF(p); qb != NULL; qb = qb->next) {
2049 if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
2050 abort();
2051 }
2052 }
2053 }
2054 break;
2055 case TAG_PRIMARY_LIST:
2056 ptr = _unchecked_list_val(val);
2057 if (!ErtsInArea(ptr, heap, heap_size)) {
2058 for (qb = MBUF(p); qb != NULL; qb = qb->next) {
2059 if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
2060 abort();
2061 }
2062 }
2063 }
2064 break;
2065 case TAG_PRIMARY_HEADER:
2066 if (header_is_thing(val)) {
2067 hp += _unchecked_thing_arityval(val);
2068 }
2069 break;
2070 }
2071 }
2072 }
2073
2074 static void
disallow_heap_frag_ref_in_old_heap(Process * p)2075 disallow_heap_frag_ref_in_old_heap(Process* p)
2076 {
2077 Eterm* hp;
2078 Eterm* htop;
2079 Eterm* old_heap;
2080 Uint old_heap_size;
2081 Eterm* new_heap;
2082 Uint new_heap_size;
2083
2084 htop = p->old_htop;
2085 old_heap = p->old_heap;
2086 old_heap_size = (htop - old_heap)*sizeof(Eterm);
2087 new_heap = p->heap;
2088 new_heap_size = (p->htop - new_heap)*sizeof(Eterm);
2089
2090 ASSERT(!p->last_old_htop
2091 || (old_heap <= p->last_old_htop && p->last_old_htop <= htop));
2092 hp = p->last_old_htop ? p->last_old_htop : old_heap;
2093 while (hp < htop) {
2094 ErlHeapFragment* qb;
2095 Eterm* ptr;
2096 Eterm val;
2097
2098 val = *hp++;
2099 switch (primary_tag(val)) {
2100 case TAG_PRIMARY_BOXED:
2101 ptr = (Eterm *) val;
2102 if (!ErtsInArea(ptr, old_heap, old_heap_size)) {
2103 if (ErtsInArea(ptr, new_heap, new_heap_size)) {
2104 abort();
2105 }
2106 for (qb = MBUF(p); qb != NULL; qb = qb->next) {
2107 if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
2108 abort();
2109 }
2110 }
2111 }
2112 break;
2113 case TAG_PRIMARY_LIST:
2114 ptr = (Eterm *) val;
2115 if (!ErtsInArea(ptr, old_heap, old_heap_size)) {
2116 if (ErtsInArea(ptr, new_heap, new_heap_size)) {
2117 abort();
2118 }
2119 for (qb = MBUF(p); qb != NULL; qb = qb->next) {
2120 if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
2121 abort();
2122 }
2123 }
2124 }
2125 break;
2126 case TAG_PRIMARY_HEADER:
2127 if (header_is_thing(val)) {
2128 hp += _unchecked_thing_arityval(val);
2129 if (!ErtsInArea(hp, old_heap, old_heap_size+1)) {
2130 abort();
2131 }
2132 }
2133 break;
2134 }
2135 }
2136 }
2137 #endif
2138
2139 typedef enum {
2140 ErtsSweepNewHeap,
2141 ErtsSweepHeaps,
2142 ErtsSweepLiteralArea
2143 } ErtsSweepType;
2144
2145 static ERTS_FORCE_INLINE Eterm *
sweep(Eterm * n_hp,Eterm * n_htop,ErtsSweepType type,char * oh,Uint ohsz,char * src,Uint src_size)2146 sweep(Eterm *n_hp, Eterm *n_htop,
2147 ErtsSweepType type,
2148 char *oh, Uint ohsz,
2149 char *src, Uint src_size)
2150 {
2151 Eterm* ptr;
2152 Eterm val;
2153 Eterm gval;
2154
2155 #undef ERTS_IS_IN_SWEEP_AREA
2156
2157 #define ERTS_IS_IN_SWEEP_AREA(TPtr, Ptr) \
2158 (type == ErtsSweepHeaps \
2159 ? !erts_is_literal((TPtr), (Ptr)) \
2160 : (type == ErtsSweepNewHeap \
2161 ? ErtsInYoungGen((TPtr), (Ptr), oh, ohsz) \
2162 : ErtsInArea((Ptr), src, src_size)))
2163
2164 while (n_hp != n_htop) {
2165 ASSERT(n_hp < n_htop);
2166 gval = *n_hp;
2167 switch (primary_tag(gval)) {
2168 case TAG_PRIMARY_BOXED: {
2169 ptr = boxed_val(gval);
2170 val = *ptr;
2171 if (IS_MOVED_BOXED(val)) {
2172 ASSERT(is_boxed(val));
2173 *n_hp++ = val;
2174 } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
2175 move_boxed(ptr,val,&n_htop,n_hp++);
2176 } else {
2177 n_hp++;
2178 }
2179 break;
2180 }
2181 case TAG_PRIMARY_LIST: {
2182 ptr = list_val(gval);
2183 val = *ptr;
2184 if (IS_MOVED_CONS(val)) {
2185 *n_hp++ = ptr[1];
2186 } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
2187 move_cons(ptr,val,&n_htop,n_hp++);
2188 } else {
2189 n_hp++;
2190 }
2191 break;
2192 }
2193 case TAG_PRIMARY_HEADER: {
2194 if (!header_is_thing(gval)) {
2195 n_hp++;
2196 } else {
2197 if (header_is_bin_matchstate(gval)) {
2198 ErlBinMatchState *ms = (ErlBinMatchState*) n_hp;
2199 ErlBinMatchBuffer *mb = &(ms->mb);
2200 Eterm* origptr;
2201 origptr = &(mb->orig);
2202 ptr = boxed_val(*origptr);
2203 val = *ptr;
2204 if (IS_MOVED_BOXED(val)) {
2205 *origptr = val;
2206 mb->base = binary_bytes(*origptr);
2207 } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) {
2208 move_boxed(ptr,val,&n_htop,origptr);
2209 mb->base = binary_bytes(*origptr);
2210 }
2211 }
2212 n_hp += (thing_arityval(gval)+1);
2213 }
2214 break;
2215 }
2216 default:
2217 n_hp++;
2218 break;
2219 }
2220 }
2221 return n_htop;
2222 #undef ERTS_IS_IN_SWEEP_AREA
2223 }
2224
2225 static Eterm *
sweep_new_heap(Eterm * n_hp,Eterm * n_htop,char * old_heap,Uint old_heap_size)2226 sweep_new_heap(Eterm *n_hp, Eterm *n_htop, char* old_heap, Uint old_heap_size)
2227 {
2228 return sweep(n_hp, n_htop,
2229 ErtsSweepNewHeap,
2230 old_heap, old_heap_size,
2231 NULL, 0);
2232 }
2233
2234 static Eterm *
sweep_heaps(Eterm * n_hp,Eterm * n_htop,char * old_heap,Uint old_heap_size)2235 sweep_heaps(Eterm *n_hp, Eterm *n_htop, char* old_heap, Uint old_heap_size)
2236 {
2237 return sweep(n_hp, n_htop,
2238 ErtsSweepHeaps,
2239 old_heap, old_heap_size,
2240 NULL, 0);
2241 }
2242
2243 static Eterm *
sweep_literal_area(Eterm * n_hp,Eterm * n_htop,char * old_heap,Uint old_heap_size,char * src,Uint src_size)2244 sweep_literal_area(Eterm *n_hp, Eterm *n_htop,
2245 char* old_heap, Uint old_heap_size,
2246 char* src, Uint src_size)
2247 {
2248 return sweep(n_hp, n_htop,
2249 ErtsSweepLiteralArea,
2250 old_heap, old_heap_size,
2251 src, src_size);
2252 }
2253
2254 static Eterm*
sweep_literals_to_old_heap(Eterm * heap_ptr,Eterm * heap_end,Eterm * htop,char * src,Uint src_size)2255 sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
2256 char* src, Uint src_size)
2257 {
2258 while (heap_ptr < heap_end) {
2259 Eterm* ptr;
2260 Eterm val;
2261 Eterm gval = *heap_ptr;
2262
2263 switch (primary_tag(gval)) {
2264 case TAG_PRIMARY_BOXED: {
2265 ptr = boxed_val(gval);
2266 val = *ptr;
2267 if (IS_MOVED_BOXED(val)) {
2268 ASSERT(is_boxed(val));
2269 *heap_ptr++ = val;
2270 } else if (ErtsInArea(ptr, src, src_size)) {
2271 move_boxed(ptr,val,&htop,heap_ptr++);
2272 } else {
2273 heap_ptr++;
2274 }
2275 break;
2276 }
2277 case TAG_PRIMARY_LIST: {
2278 ptr = list_val(gval);
2279 val = *ptr;
2280 if (IS_MOVED_CONS(val)) {
2281 *heap_ptr++ = ptr[1];
2282 } else if (ErtsInArea(ptr, src, src_size)) {
2283 move_cons(ptr,val,&htop,heap_ptr++);
2284 } else {
2285 heap_ptr++;
2286 }
2287 break;
2288 }
2289 case TAG_PRIMARY_HEADER: {
2290 if (!header_is_thing(gval)) {
2291 heap_ptr++;
2292 } else {
2293 if (header_is_bin_matchstate(gval)) {
2294 ErlBinMatchState *ms = (ErlBinMatchState*) heap_ptr;
2295 ErlBinMatchBuffer *mb = &(ms->mb);
2296 Eterm* origptr;
2297 origptr = &(mb->orig);
2298 ptr = boxed_val(*origptr);
2299 val = *ptr;
2300 if (IS_MOVED_BOXED(val)) {
2301 *origptr = val;
2302 mb->base = binary_bytes(*origptr);
2303 } else if (ErtsInArea(ptr, src, src_size)) {
2304 move_boxed(ptr,val,&htop,origptr);
2305 mb->base = binary_bytes(*origptr);
2306 }
2307 }
2308 heap_ptr += (thing_arityval(gval)+1);
2309 }
2310 break;
2311 }
2312 default:
2313 heap_ptr++;
2314 break;
2315 }
2316 }
2317 return htop;
2318 }
2319
2320 /*
2321 * Move an area (heap fragment) by sweeping over it and set move markers.
2322 */
2323 static Eterm*
move_one_area(Eterm * n_htop,char * src,Uint src_size)2324 move_one_area(Eterm* n_htop, char* src, Uint src_size)
2325 {
2326 Eterm* ptr = (Eterm*) src;
2327 Eterm* end = ptr + src_size/sizeof(Eterm);
2328 Eterm dummy_ref;
2329
2330 while (ptr != end) {
2331 Eterm val;
2332 ASSERT(ptr < end);
2333 val = *ptr;
2334 ASSERT(val != ERTS_HOLE_MARKER);
2335 if (is_header(val)) {
2336 ASSERT(ptr + header_arity(val) < end);
2337 ptr = move_boxed(ptr, val, &n_htop, &dummy_ref);
2338 }
2339 else { /* must be a cons cell */
2340 ASSERT(ptr+1 < end);
2341 move_cons(ptr, val, &n_htop, &dummy_ref);
2342 ptr += 2;
2343 }
2344 }
2345
2346 return n_htop;
2347 }
2348
2349 /*
2350 * Collect heap fragments and check that they point in the correct direction.
2351 */
2352
2353 static Eterm*
collect_live_heap_frags(Process * p,ErlHeapFragment * live_hf_end,Eterm * n_htop)2354 collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, Eterm* n_htop)
2355 {
2356 ErlHeapFragment* qb;
2357 char* frag_begin;
2358 Uint frag_size;
2359
2360 /*
2361 * Move the heap fragments to the new heap. Note that no GC is done on
2362 * the heap fragments. Any garbage will thus be moved as well and survive
2363 * until next GC.
2364 */
2365 qb = MBUF(p);
2366 while (qb != live_hf_end) {
2367 ASSERT(!qb->off_heap.first); /* process fragments use the MSO(p) list */
2368 frag_size = qb->used_size * sizeof(Eterm);
2369 if (frag_size != 0) {
2370 frag_begin = (char *) qb->mem;
2371 n_htop = move_one_area(n_htop, frag_begin, frag_size);
2372 }
2373 qb = qb->next;
2374 }
2375 return n_htop;
2376 }
2377
2378 void
erts_copy_one_frag(Eterm ** hpp,ErlOffHeap * off_heap,ErlHeapFragment * bp,Eterm * refs,int nrefs)2379 erts_copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
2380 ErlHeapFragment *bp, Eterm *refs, int nrefs)
2381 {
2382 Uint sz;
2383 int i;
2384 Sint offs;
2385 struct erl_off_heap_header* oh;
2386 Eterm *fhp, *hp;
2387
2388 OH_OVERHEAD(off_heap, bp->off_heap.overhead);
2389 sz = bp->used_size;
2390
2391 fhp = bp->mem;
2392 hp = *hpp;
2393 offs = hp - fhp;
2394
2395 oh = NULL;
2396 while (sz--) {
2397 Uint cpy_sz;
2398 Eterm val = *fhp++;
2399
2400 switch (primary_tag(val)) {
2401 case TAG_PRIMARY_IMMED1:
2402 *hp++ = val;
2403 break;
2404 case TAG_PRIMARY_LIST:
2405 if (erts_is_literal(val,list_val(val))) {
2406 *hp++ = val;
2407 } else {
2408 *hp++ = offset_ptr(val, offs);
2409 }
2410 break;
2411 case TAG_PRIMARY_BOXED:
2412 if (erts_is_literal(val,boxed_val(val))) {
2413 *hp++ = val;
2414 } else {
2415 *hp++ = offset_ptr(val, offs);
2416 }
2417 break;
2418 case TAG_PRIMARY_HEADER:
2419 *hp++ = val;
2420 switch (val & _HEADER_SUBTAG_MASK) {
2421 case ARITYVAL_SUBTAG:
2422 break;
2423 case REF_SUBTAG:
2424 if (is_ordinary_ref_thing(fhp - 1))
2425 goto the_default;
2426 case REFC_BINARY_SUBTAG:
2427 case FUN_SUBTAG:
2428 case EXTERNAL_PID_SUBTAG:
2429 case EXTERNAL_PORT_SUBTAG:
2430 case EXTERNAL_REF_SUBTAG:
2431 oh = (struct erl_off_heap_header*) (hp-1);
2432 cpy_sz = thing_arityval(val);
2433 goto cpy_words;
2434 default:
2435 the_default:
2436 cpy_sz = header_arity(val);
2437
2438 cpy_words:
2439 ASSERT(sz >= cpy_sz);
2440 sz -= cpy_sz;
2441 sys_memcpy(hp, fhp, cpy_sz * sizeof(Eterm));
2442 hp += cpy_sz;
2443 fhp += cpy_sz;
2444 if (oh) {
2445 /* Add to offheap list */
2446 oh->next = off_heap->first;
2447 off_heap->first = oh;
2448 ASSERT(*hpp <= (Eterm*)oh);
2449 ASSERT(hp > (Eterm*)oh);
2450 oh = NULL;
2451 }
2452 break;
2453 }
2454 break;
2455 }
2456 }
2457
2458 ASSERT(bp->used_size == hp - *hpp);
2459 *hpp = hp;
2460
2461 for (i = 0; i < nrefs; i++) {
2462 if (is_not_immed(refs[i]) && !erts_is_literal(refs[i],ptr_val(refs[i])))
2463 refs[i] = offset_ptr(refs[i], offs);
2464 }
2465 bp->off_heap.first = NULL;
2466 }
2467
2468 static Uint
setup_rootset(Process * p,Eterm * objv,int nobj,Rootset * rootset)2469 setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
2470 {
2471 /*
2472 * NOTE!
2473 * Remember to update offset_rootset() when changing
2474 * this function.
2475 */
2476 Roots* roots;
2477 Uint n;
2478
2479 n = 0;
2480 roots = rootset->roots = rootset->def;
2481 rootset->size = ALENGTH(rootset->def);
2482
2483 roots[n].v = p->stop;
2484 roots[n].sz = STACK_START(p) - p->stop;
2485 ++n;
2486
2487 if (p->dictionary != NULL) {
2488 roots[n].v = ERTS_PD_START(p->dictionary);
2489 roots[n].sz = ERTS_PD_SIZE(p->dictionary);
2490 ++n;
2491 }
2492 if (nobj > 0) {
2493 roots[n].v = objv;
2494 roots[n].sz = nobj;
2495 ++n;
2496 }
2497
2498 ASSERT((is_nil(p->seq_trace_token) ||
2499 is_tuple(follow_moved(p->seq_trace_token, (Eterm) 0)) ||
2500 is_atom(p->seq_trace_token)));
2501 if (is_not_immed(p->seq_trace_token)) {
2502 roots[n].v = &p->seq_trace_token;
2503 roots[n].sz = 1;
2504 n++;
2505 }
2506 #ifdef USE_VM_PROBES
2507 if (is_not_immed(p->dt_utag)) {
2508 roots[n].v = &p->dt_utag;
2509 roots[n].sz = 1;
2510 n++;
2511 }
2512 #endif
2513 ASSERT(IS_TRACER_VALID(ERTS_TRACER(p)));
2514
2515 ASSERT(is_pid(follow_moved(p->group_leader, (Eterm) 0)));
2516 if (is_not_immed(p->group_leader)) {
2517 roots[n].v = &p->group_leader;
2518 roots[n].sz = 1;
2519 n++;
2520 }
2521
2522 /*
2523 * The process may be garbage-collected while it is terminating.
2524 * (fvalue contains the EXIT reason and ftrace the saved stack trace.)
2525 */
2526 if (is_not_immed(p->fvalue)) {
2527 roots[n].v = &p->fvalue;
2528 roots[n].sz = 1;
2529 n++;
2530 }
2531 if (is_not_immed(p->ftrace)) {
2532 roots[n].v = &p->ftrace;
2533 roots[n].sz = 1;
2534 n++;
2535 }
2536
2537 /*
2538 * If a NIF or BIF has saved arguments, they need to be added
2539 */
2540 if (erts_setup_nif_export_rootset(p, &roots[n].v, &roots[n].sz))
2541 n++;
2542
2543 ASSERT(n <= rootset->size);
2544
2545 switch (p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) {
2546 case F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG:
2547 (void) erts_move_messages_off_heap(p);
2548 case F_OFF_HEAP_MSGQ:
2549 break;
2550 case F_OFF_HEAP_MSGQ_CHNG:
2551 case 0: {
2552 Sint len;
2553 /*
2554 * We do not have off heap message queue enabled, i.e. we
2555 * need to add signal queues to rootset...
2556 */
2557
2558 len = erts_proc_sig_privqs_len(p);
2559
2560 /* Ensure large enough rootset... */
2561 if (n + len > rootset->size) {
2562 Uint new_size = n + len;
2563 ERTS_GC_ASSERT(roots == rootset->def);
2564 roots = erts_alloc(ERTS_ALC_T_ROOTSET,
2565 new_size*sizeof(Roots));
2566 sys_memcpy(roots, rootset->def, n*sizeof(Roots));
2567 rootset->size = new_size;
2568 }
2569
2570 ERTS_FOREACH_SIG_PRIVQS(
2571 p, mp,
2572 {
2573 if (ERTS_SIG_IS_INTERNAL_MSG(mp) && !mp->data.attached) {
2574 /*
2575 * Message may refer data on heap;
2576 * add it to rootset...
2577 */
2578 roots[n].v = mp->m;
2579 roots[n].sz = ERL_MESSAGE_REF_ARRAY_SZ;
2580 n++;
2581 }
2582 });
2583 break;
2584 }
2585 }
2586
2587 ASSERT(rootset->size >= n);
2588
2589 rootset->roots = roots;
2590 rootset->num_roots = n;
2591 return n;
2592 }
2593
2594 static
cleanup_rootset(Rootset * rootset)2595 void cleanup_rootset(Rootset* rootset)
2596 {
2597 if (rootset->roots != rootset->def) {
2598 erts_free(ERTS_ALC_T_ROOTSET, rootset->roots);
2599 }
2600 }
2601
2602 static void
grow_new_heap(Process * p,Uint new_sz,Eterm * objv,int nobj)2603 grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj)
2604 {
2605 Eterm* new_heap;
2606 Uint heap_size = HEAP_TOP(p) - HEAP_START(p);
2607 Uint stack_size = p->hend - p->stop;
2608 Sint offs;
2609
2610 ASSERT(HEAP_SIZE(p) < new_sz);
2611 new_heap = (Eterm *) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP,
2612 (void *) HEAP_START(p),
2613 sizeof(Eterm)*(HEAP_SIZE(p)),
2614 sizeof(Eterm)*new_sz);
2615
2616 if ((offs = new_heap - HEAP_START(p)) == 0) { /* No move. */
2617 HEAP_END(p) = new_heap + new_sz;
2618 sys_memmove(p->hend - stack_size, p->stop, stack_size * sizeof(Eterm));
2619 p->stop = p->hend - stack_size;
2620 } else {
2621 char* area = (char *) HEAP_START(p);
2622 Uint area_size = (char *) HEAP_TOP(p) - area;
2623 Eterm* prev_stop = p->stop;
2624
2625 offset_heap(new_heap, heap_size, offs, area, area_size);
2626
2627 HIGH_WATER(p) = new_heap + (HIGH_WATER(p) - HEAP_START(p));
2628
2629 HEAP_END(p) = new_heap + new_sz;
2630 prev_stop = new_heap + (p->stop - p->heap);
2631 p->stop = p->hend - stack_size;
2632 sys_memmove(p->stop, prev_stop, stack_size * sizeof(Eterm));
2633
2634 offset_rootset(p, offs, area, area_size, objv, nobj);
2635 HEAP_TOP(p) = new_heap + heap_size;
2636 HEAP_START(p) = new_heap;
2637 }
2638
2639 #ifdef USE_VM_PROBES
2640 if (DTRACE_ENABLED(process_heap_grow)) {
2641 DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
2642
2643 dtrace_proc_str(p, pidbuf);
2644 DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
2645 }
2646 #endif
2647
2648 HEAP_SIZE(p) = new_sz;
2649 }
2650
2651 static void
shrink_new_heap(Process * p,Uint new_sz,Eterm * objv,int nobj)2652 shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj)
2653 {
2654 Eterm* new_heap;
2655 Uint heap_size = HEAP_TOP(p) - HEAP_START(p);
2656 Sint offs;
2657 Uint stack_size = p->hend - p->stop;
2658
2659 ASSERT(new_sz < p->heap_sz);
2660 sys_memmove(p->heap + new_sz - stack_size, p->stop, stack_size *
2661 sizeof(Eterm));
2662 new_heap = (Eterm *) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP,
2663 (void*)p->heap,
2664 sizeof(Eterm)*(HEAP_SIZE(p)),
2665 sizeof(Eterm)*new_sz);
2666 p->hend = new_heap + new_sz;
2667 p->stop = p->hend - stack_size;
2668
2669 if ((offs = new_heap - HEAP_START(p)) != 0) {
2670 char* area = (char *) HEAP_START(p);
2671 Uint area_size = (char *) HEAP_TOP(p) - area;
2672
2673 /*
2674 * Normally, we don't expect a shrunk heap to move, but you never
2675 * know on some strange embedded systems... Or when using purify.
2676 */
2677
2678 offset_heap(new_heap, heap_size, offs, area, area_size);
2679
2680 HIGH_WATER(p) = new_heap + (HIGH_WATER(p) - HEAP_START(p));
2681 offset_rootset(p, offs, area, area_size, objv, nobj);
2682 HEAP_TOP(p) = new_heap + heap_size;
2683 HEAP_START(p) = new_heap;
2684 }
2685
2686 #ifdef USE_VM_PROBES
2687 if (DTRACE_ENABLED(process_heap_shrink)) {
2688 DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
2689
2690 dtrace_proc_str(p, pidbuf);
2691 DTRACE3(process_heap_shrink, pidbuf, HEAP_SIZE(p), new_sz);
2692 }
2693 #endif
2694
2695 HEAP_SIZE(p) = new_sz;
2696 }
2697
2698 static Uint64
do_next_vheap_size(Uint64 vheap,Uint64 vheap_sz)2699 do_next_vheap_size(Uint64 vheap, Uint64 vheap_sz) {
2700
2701 /* grow
2702 *
2703 * vheap_sz ======================
2704 *
2705 * vheap 75% + grow
2706 * ----------------------
2707 *
2708 * vheap 25 - 75% same
2709 * ----------------------
2710 *
2711 * vheap ~ - 25% shrink
2712 *
2713 * ----------------------
2714 */
2715
2716 if ((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
2717 Uint64 new_vheap_sz = vheap_sz;
2718
2719 while((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
2720 /* the golden ratio = 1.618 */
2721 new_vheap_sz = (Uint64) vheap_sz * 1.618;
2722 if (new_vheap_sz < vheap_sz ) {
2723 return vheap_sz;
2724 }
2725 vheap_sz = new_vheap_sz;
2726 }
2727
2728 return vheap_sz;
2729 }
2730
2731 if (vheap < (Uint64) (vheap_sz/4)) {
2732 return (vheap_sz >> 1);
2733 }
2734
2735 return vheap_sz;
2736
2737 }
2738
2739 static Uint64
next_vheap_size(Process * p,Uint64 vheap,Uint64 vheap_sz)2740 next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz) {
2741 Uint64 new_vheap_sz = do_next_vheap_size(vheap, vheap_sz);
2742 return new_vheap_sz < p->min_vheap_size ? p->min_vheap_size : new_vheap_sz;
2743 }
2744
2745 struct shrink_cand_data {
2746 struct erl_off_heap_header* new_candidates;
2747 struct erl_off_heap_header* new_candidates_end;
2748 struct erl_off_heap_header* old_candidates;
2749 Uint no_of_candidates;
2750 Uint no_of_active;
2751 };
2752
2753 static ERTS_INLINE void
link_live_proc_bin(struct shrink_cand_data * shrink,struct erl_off_heap_header *** prevppp,struct erl_off_heap_header ** currpp,int new_heap)2754 link_live_proc_bin(struct shrink_cand_data *shrink,
2755 struct erl_off_heap_header*** prevppp,
2756 struct erl_off_heap_header** currpp,
2757 int new_heap)
2758 {
2759 ProcBin *pbp = (ProcBin*) *currpp;
2760 ASSERT(**prevppp == *currpp);
2761
2762 *currpp = pbp->next;
2763 if (pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) {
2764 ASSERT(((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
2765 == (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
2766 || ((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
2767 == PB_IS_WRITABLE));
2768
2769
2770 if (pbp->flags & PB_ACTIVE_WRITER) {
2771 shrink->no_of_active++;
2772 }
2773 else { /* inactive */
2774 Uint unused = pbp->val->orig_size - pbp->size;
2775 /* Our allocators are 8 byte aligned, i.e., shrinking with
2776 less than 8 bytes will have no real effect */
2777 if (unused >= 8) { /* A shrink candidate; save in candidate list */
2778 **prevppp = pbp->next;
2779 if (new_heap) {
2780 if (!shrink->new_candidates)
2781 shrink->new_candidates_end = (struct erl_off_heap_header*)pbp;
2782 pbp->next = shrink->new_candidates;
2783 shrink->new_candidates = (struct erl_off_heap_header*)pbp;
2784 }
2785 else {
2786 pbp->next = shrink->old_candidates;
2787 shrink->old_candidates = (struct erl_off_heap_header*)pbp;
2788 }
2789 shrink->no_of_candidates++;
2790 return;
2791 }
2792 }
2793 }
2794
2795 /* Not a shrink candidate; keep in original mso list */
2796 *prevppp = &pbp->next;
2797 }
2798
2799 #ifdef ERTS_MAGIC_REF_THING_HEADER
2800 /*
2801 * ERTS_MAGIC_REF_THING_HEADER only defined when there
2802 * is a size difference between magic and ordinary references...
2803 */
2804 # define ERTS_USED_MAGIC_REF_THING_HEADER__ ERTS_MAGIC_REF_THING_HEADER
2805 #else
2806 # define ERTS_USED_MAGIC_REF_THING_HEADER__ ERTS_REF_THING_HEADER
2807 #endif
2808
2809
2810 static void
sweep_off_heap(Process * p,int fullsweep)2811 sweep_off_heap(Process *p, int fullsweep)
2812 {
2813 struct shrink_cand_data shrink = {0};
2814 struct erl_off_heap_header* ptr;
2815 struct erl_off_heap_header** prev;
2816 char* oheap = NULL;
2817 Uint oheap_sz = 0;
2818 Uint64 bin_vheap = 0;
2819 #ifdef DEBUG
2820 int seen_mature = 0;
2821 #endif
2822
2823 if (fullsweep == 0) {
2824 oheap = (char *) OLD_HEAP(p);
2825 oheap_sz = (char *) OLD_HEND(p) - oheap;
2826 }
2827
2828 BIN_OLD_VHEAP(p) = 0;
2829
2830 prev = &MSO(p).first;
2831 ptr = MSO(p).first;
2832
2833 /* First part of the list will reside on the (old) new-heap.
2834 * Keep if moved, otherwise deref.
2835 */
2836 while (ptr) {
2837 if (IS_MOVED_BOXED(ptr->thing_word)) {
2838 ASSERT(!ErtsInArea(ptr, oheap, oheap_sz));
2839 *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word);
2840 ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
2841 switch (ptr->thing_word) {
2842 case HEADER_PROC_BIN: {
2843 int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz);
2844 ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1)));
2845 if (to_new_heap) {
2846 bin_vheap += ptr->size / sizeof(Eterm);
2847 } else {
2848 BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
2849 }
2850 link_live_proc_bin(&shrink, &prev, &ptr, to_new_heap);
2851 break;
2852 }
2853 case ERTS_USED_MAGIC_REF_THING_HEADER__: {
2854 Uint size;
2855 int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz);
2856 ASSERT(is_magic_ref_thing(ptr));
2857 ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1)));
2858 size = (Uint) ((ErtsMRefThing *) ptr)->mb->orig_size;
2859 if (to_new_heap)
2860 bin_vheap += size / sizeof(Eterm);
2861 else
2862 BIN_OLD_VHEAP(p) += size / sizeof(Eterm); /* for binary gc (words)*/
2863 /* fall through... */
2864 }
2865 default:
2866 prev = &ptr->next;
2867 ptr = ptr->next;
2868 }
2869 }
2870 else if (ErtsInArea(ptr, oheap, oheap_sz))
2871 break; /* and let old-heap loop continue */
2872 else {
2873 /* garbage */
2874 switch (thing_subtag(ptr->thing_word)) {
2875 case REFC_BINARY_SUBTAG:
2876 {
2877 Binary* bptr = ((ProcBin*)ptr)->val;
2878 erts_bin_release(bptr);
2879 break;
2880 }
2881 case FUN_SUBTAG:
2882 {
2883 ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe;
2884 if (erts_refc_dectest(&fe->refc, 0) == 0) {
2885 erts_erase_fun_entry(fe);
2886 }
2887 break;
2888 }
2889 case REF_SUBTAG:
2890 {
2891 ErtsMagicBinary *bptr;
2892 ASSERT(is_magic_ref_thing(ptr));
2893 bptr = ((ErtsMRefThing *) ptr)->mb;
2894 erts_bin_release((Binary *) bptr);
2895 break;
2896 }
2897 default:
2898 ASSERT(is_external_header(ptr->thing_word));
2899 erts_deref_node_entry(((ExternalThing*)ptr)->node);
2900 }
2901 *prev = ptr = ptr->next;
2902 }
2903 }
2904
2905 /* The rest of the list resides on old-heap, and we just did a
2906 * generational collection - keep objects in list.
2907 */
2908 while (ptr) {
2909 ASSERT(ErtsInArea(ptr, oheap, oheap_sz));
2910 ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
2911 switch (ptr->thing_word) {
2912 case HEADER_PROC_BIN:
2913 BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
2914 link_live_proc_bin(&shrink, &prev, &ptr, 0);
2915 break;
2916 case ERTS_USED_MAGIC_REF_THING_HEADER__:
2917 ASSERT(is_magic_ref_thing(ptr));
2918 BIN_OLD_VHEAP(p) +=
2919 (((Uint) ((ErtsMRefThing *) ptr)->mb->orig_size)
2920 / sizeof(Eterm)); /* for binary gc (words)*/
2921 /* fall through... */
2922 default:
2923 ASSERT(is_fun_header(ptr->thing_word) ||
2924 is_external_header(ptr->thing_word)
2925 || is_magic_ref_thing(ptr));
2926 prev = &ptr->next;
2927 ptr = ptr->next;
2928 break;
2929 }
2930 }
2931
2932 if (fullsweep) {
2933 BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p) + MSO(p).overhead, BIN_OLD_VHEAP_SZ(p));
2934 }
2935 BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
2936 MSO(p).overhead = bin_vheap;
2937
2938 /*
2939 * If we got any shrink candidates, check them out.
2940 */
2941
2942 if (shrink.no_of_candidates) {
2943 ProcBin *candlist[] = { (ProcBin*)shrink.new_candidates,
2944 (ProcBin*)shrink.old_candidates };
2945 Uint leave_unused = 0;
2946 int i;
2947
2948 if (shrink.no_of_active == 0) {
2949 if (shrink.no_of_candidates <= ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT)
2950 leave_unused = ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE;
2951 else if (shrink.no_of_candidates <= ERTS_INACT_WR_PB_LEAVE_LIMIT)
2952 leave_unused = ERTS_INACT_WR_PB_LEAVE_PERCENTAGE;
2953 }
2954
2955 for (i = 0; i < sizeof(candlist)/sizeof(candlist[0]); i++) {
2956 ProcBin* pb;
2957 for (pb = candlist[i]; pb; pb = (ProcBin*)pb->next) {
2958 Uint new_size = pb->size;
2959
2960 if (leave_unused) {
2961 new_size += (new_size * 100) / leave_unused;
2962 /* Our allocators are 8 byte aligned, i.e., shrinking with
2963 less than 8 bytes will have no real effect */
2964 if (new_size + 8 >= pb->val->orig_size)
2965 continue;
2966 }
2967
2968 pb->val = erts_bin_realloc(pb->val, new_size);
2969 pb->bytes = (byte *) pb->val->orig_bytes;
2970 }
2971 }
2972
2973
2974 /*
2975 * We now potentially have the mso list divided into three lists:
2976 * - shrink candidates on new heap (inactive writable with unused data)
2977 * - shrink candidates on old heap (inactive writable with unused data)
2978 * - other binaries (read only + active writable ...) + funs and externals
2979 *
2980 * Put them back together: new candidates -> other -> old candidates
2981 * This order will ensure that the list only refers from new
2982 * generation to old and never from old to new *which is important*.
2983 */
2984 if (shrink.new_candidates) {
2985 if (prev == &MSO(p).first) /* empty other binaries list */
2986 prev = &shrink.new_candidates_end->next;
2987 else
2988 shrink.new_candidates_end->next = MSO(p).first;
2989 MSO(p).first = shrink.new_candidates;
2990 }
2991 }
2992 *prev = shrink.old_candidates;
2993 }
2994
2995 /*
2996 * Offset pointers into the heap (not stack).
2997 */
2998
2999 static void
offset_heap(Eterm * hp,Uint sz,Sint offs,char * area,Uint area_size)3000 offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
3001 {
3002 while (sz--) {
3003 Eterm val = *hp;
3004 switch (primary_tag(val)) {
3005 case TAG_PRIMARY_LIST:
3006 case TAG_PRIMARY_BOXED:
3007 if (ErtsInArea(ptr_val(val), area, area_size)) {
3008 *hp = offset_ptr(val, offs);
3009 }
3010 hp++;
3011 break;
3012 case TAG_PRIMARY_HEADER: {
3013 Uint tari;
3014
3015 if (header_is_transparent(val)) {
3016 hp++;
3017 continue;
3018 }
3019 tari = thing_arityval(val);
3020 switch (thing_subtag(val)) {
3021 case REF_SUBTAG:
3022 if (is_ordinary_ref_thing(hp))
3023 break;
3024 case REFC_BINARY_SUBTAG:
3025 case FUN_SUBTAG:
3026 case EXTERNAL_PID_SUBTAG:
3027 case EXTERNAL_PORT_SUBTAG:
3028 case EXTERNAL_REF_SUBTAG:
3029 {
3030 struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp;
3031
3032 if (ErtsInArea(oh->next, area, area_size)) {
3033 Eterm** uptr = (Eterm **) (void *) &oh->next;
3034 *uptr += offs; /* Patch the mso chain */
3035 }
3036 }
3037 break;
3038 case BIN_MATCHSTATE_SUBTAG:
3039 {
3040 ErlBinMatchState *ms = (ErlBinMatchState*) hp;
3041 ErlBinMatchBuffer *mb = &(ms->mb);
3042 if (ErtsInArea(ptr_val(mb->orig), area, area_size)) {
3043 mb->orig = offset_ptr(mb->orig, offs);
3044 mb->base = binary_bytes(mb->orig);
3045 }
3046 }
3047 break;
3048 }
3049 sz -= tari;
3050 hp += tari + 1;
3051 break;
3052 }
3053 default:
3054 hp++;
3055 continue;
3056 }
3057 }
3058 }
3059
3060 /*
3061 * Offset pointers to heap from stack.
3062 */
3063
3064 static void
offset_heap_ptr(Eterm * hp,Uint sz,Sint offs,char * area,Uint area_size)3065 offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
3066 {
3067 while (sz--) {
3068 Eterm val = *hp;
3069 switch (primary_tag(val)) {
3070 case TAG_PRIMARY_LIST:
3071 case TAG_PRIMARY_BOXED:
3072 if (ErtsInArea(ptr_val(val), area, area_size)) {
3073 *hp = offset_ptr(val, offs);
3074 }
3075 hp++;
3076 break;
3077 default:
3078 hp++;
3079 break;
3080 }
3081 }
3082 }
3083
3084 static void
offset_off_heap(Process * p,Sint offs,char * area,Uint area_size)3085 offset_off_heap(Process* p, Sint offs, char* area, Uint area_size)
3086 {
3087 if (MSO(p).first && ErtsInArea((Eterm *)MSO(p).first, area, area_size)) {
3088 Eterm** uptr = (Eterm**) (void *) &MSO(p).first;
3089 *uptr += offs;
3090 }
3091 }
3092
3093 #ifndef USE_VM_PROBES
3094 #define ERTS_OFFSET_DT_UTAG(MP, A, ASZ, O)
3095 #else
3096 #define ERTS_OFFSET_DT_UTAG(MP, A, ASZ, O) \
3097 do { \
3098 Eterm utag__ = ERL_MESSAGE_DT_UTAG((MP)); \
3099 if (is_boxed(utag__) && ErtsInArea(ptr_val(utag__), (A), (ASZ))) { \
3100 ERL_MESSAGE_DT_UTAG((MP)) = offset_ptr(utag__, (O)); \
3101 } \
3102 } while (0)
3103 #endif
3104
3105 static ERTS_INLINE void
offset_message(ErtsMessage * mp,Sint offs,char * area,Uint area_size)3106 offset_message(ErtsMessage *mp, Sint offs, char* area, Uint area_size)
3107 {
3108 Eterm mesg = ERL_MESSAGE_TERM(mp);
3109 if (ERTS_SIG_IS_MSG_TAG(mesg)) {
3110 if (ERTS_SIG_IS_INTERNAL_MSG_TAG(mesg)) {
3111 switch (primary_tag(mesg)) {
3112 case TAG_PRIMARY_LIST:
3113 case TAG_PRIMARY_BOXED:
3114 if (ErtsInArea(ptr_val(mesg), area, area_size)) {
3115 ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs);
3116 }
3117 break;
3118 }
3119 }
3120 mesg = ERL_MESSAGE_TOKEN(mp);
3121 if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
3122 ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs);
3123 }
3124
3125 ERTS_OFFSET_DT_UTAG(mp, area, area_size, offs);
3126
3127 ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) ||
3128 is_tuple(ERL_MESSAGE_TOKEN(mp)) ||
3129 is_atom(ERL_MESSAGE_TOKEN(mp))));
3130 }
3131 }
3132
3133 /*
3134 * Offset pointers in message queue.
3135 */
3136 static void
offset_mqueue(Process * p,Sint offs,char * area,Uint area_size)3137 offset_mqueue(Process *p, Sint offs, char* area, Uint area_size)
3138 {
3139 if ((p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) != F_OFF_HEAP_MSGQ)
3140 ERTS_FOREACH_SIG_PRIVQS(p, mp, offset_message(mp, offs, area, area_size));
3141 }
3142
3143 static void ERTS_INLINE
offset_one_rootset(Process * p,Sint offs,char * area,Uint area_size,Eterm * objv,int nobj)3144 offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
3145 Eterm* objv, int nobj)
3146 {
3147 Eterm *v;
3148 Uint sz;
3149 if (p->dictionary) {
3150 offset_heap(ERTS_PD_START(p->dictionary),
3151 ERTS_PD_SIZE(p->dictionary),
3152 offs, area, area_size);
3153 }
3154
3155 offset_heap_ptr(&p->fvalue, 1, offs, area, area_size);
3156 offset_heap_ptr(&p->ftrace, 1, offs, area, area_size);
3157 offset_heap_ptr(&p->seq_trace_token, 1, offs, area, area_size);
3158 #ifdef USE_VM_PROBES
3159 offset_heap_ptr(&p->dt_utag, 1, offs, area, area_size);
3160 #endif
3161 offset_heap_ptr(&p->group_leader, 1, offs, area, area_size);
3162 offset_mqueue(p, offs, area, area_size);
3163 offset_heap_ptr(p->stop, (STACK_START(p) - p->stop), offs, area, area_size);
3164 offset_nstack(p, offs, area, area_size);
3165 if (nobj > 0) {
3166 offset_heap_ptr(objv, nobj, offs, area, area_size);
3167 }
3168 offset_off_heap(p, offs, area, area_size);
3169 if (erts_setup_nif_export_rootset(p, &v, &sz))
3170 offset_heap_ptr(v, sz, offs, area, area_size);
3171 }
3172
3173 static void
offset_rootset(Process * p,Sint offs,char * area,Uint area_size,Eterm * objv,int nobj)3174 offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
3175 Eterm* objv, int nobj)
3176 {
3177 offset_one_rootset(p, offs, area, area_size, objv, nobj);
3178 }
3179
3180 static void
init_gc_info(ErtsGCInfo * gcip)3181 init_gc_info(ErtsGCInfo *gcip)
3182 {
3183 gcip->reclaimed = 0;
3184 gcip->garbage_cols = 0;
3185 }
3186
3187 static void
reply_gc_info(void * vgcirp)3188 reply_gc_info(void *vgcirp)
3189 {
3190 Uint64 reclaimed = 0, garbage_cols = 0;
3191 ErtsSchedulerData *esdp = erts_get_scheduler_data();
3192 ErtsGCInfoReq *gcirp = (ErtsGCInfoReq *) vgcirp;
3193 ErtsProcLocks rp_locks = (gcirp->req_sched == esdp->no
3194 ? ERTS_PROC_LOCK_MAIN
3195 : 0);
3196 Process *rp = gcirp->proc;
3197 Eterm ref_copy = NIL, msg;
3198 Eterm *hp = NULL;
3199 Eterm **hpp;
3200 Uint sz, *szp;
3201 ErlOffHeap *ohp = NULL;
3202 ErtsMessage *mp = NULL;
3203
3204 ASSERT(esdp);
3205
3206 reclaimed = esdp->gc_info.reclaimed;
3207 garbage_cols = esdp->gc_info.garbage_cols;
3208 /*
3209 * Add dirty schedulers info on requesting
3210 * schedulers info
3211 */
3212 if (gcirp->req_sched == esdp->no) {
3213 erts_mtx_lock(&dirty_gc.mtx);
3214 reclaimed += dirty_gc.info.reclaimed;
3215 garbage_cols += dirty_gc.info.garbage_cols;
3216 erts_mtx_unlock(&dirty_gc.mtx);
3217 }
3218
3219 sz = 0;
3220 hpp = NULL;
3221 szp = &sz;
3222
3223 while (1) {
3224 if (hpp)
3225 ref_copy = STORE_NC(hpp, ohp, gcirp->ref);
3226 else
3227 *szp += ERTS_REF_THING_SIZE;
3228
3229 msg = erts_bld_tuple(hpp, szp, 3,
3230 make_small(esdp->no),
3231 erts_bld_uint64(hpp, szp, garbage_cols),
3232 erts_bld_uint64(hpp, szp, reclaimed));
3233
3234 msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
3235 if (hpp)
3236 break;
3237
3238 mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
3239
3240 szp = NULL;
3241 hpp = &hp;
3242 }
3243
3244 erts_queue_message(rp, rp_locks, mp, msg, am_system);
3245
3246 if (gcirp->req_sched == esdp->no)
3247 rp_locks &= ~ERTS_PROC_LOCK_MAIN;
3248
3249 if (rp_locks)
3250 erts_proc_unlock(rp, rp_locks);
3251
3252 erts_proc_dec_refc(rp);
3253
3254 if (erts_atomic32_dec_read_nob(&gcirp->refc) == 0)
3255 gcireq_free(vgcirp);
3256 }
3257
erts_sub_binary_to_heap_binary(Eterm * ptr,Eterm ** hpp,Eterm * orig)3258 Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig) {
3259 Eterm *htop = *hpp;
3260 Eterm gval;
3261 ErlSubBin *sb = (ErlSubBin *)ptr;
3262 ErlHeapBin *hb = (ErlHeapBin *)htop;
3263 Eterm *real_bin;
3264 byte *bs;
3265
3266 real_bin = binary_val(follow_moved(sb->orig, (Eterm)0));
3267
3268 if (*real_bin == HEADER_PROC_BIN) {
3269 bs = ((ProcBin *) real_bin)->bytes + sb->offs;
3270 } else {
3271 bs = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + sb->offs;
3272 }
3273
3274 hb->thing_word = header_heap_bin(sb->size);
3275 hb->size = sb->size;
3276 sys_memcpy((byte *)hb->data, bs, sb->size);
3277
3278 gval = make_boxed(htop);
3279 *orig = gval;
3280 *ptr = gval;
3281
3282 ptr += ERL_SUB_BIN_SIZE;
3283 htop += heap_bin_size(sb->size);
3284
3285 *hpp = htop;
3286 return ptr;
3287 }
3288
3289
3290 Eterm
erts_gc_info_request(Process * c_p)3291 erts_gc_info_request(Process *c_p)
3292 {
3293 ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
3294 Eterm ref;
3295 ErtsGCInfoReq *gcirp;
3296 Eterm *hp;
3297
3298 gcirp = gcireq_alloc();
3299 ref = erts_make_ref(c_p);
3300 hp = &gcirp->ref_heap[0];
3301
3302 gcirp->proc = c_p;
3303 gcirp->ref = STORE_NC(&hp, NULL, ref);
3304 gcirp->req_sched = esdp->no;
3305 erts_atomic32_init_nob(&gcirp->refc,
3306 (erts_aint32_t) erts_no_schedulers);
3307
3308 erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
3309
3310 if (erts_no_schedulers > 1)
3311 erts_schedule_multi_misc_aux_work(1,
3312 erts_no_schedulers,
3313 reply_gc_info,
3314 (void *) gcirp);
3315
3316 reply_gc_info((void *) gcirp);
3317
3318 return ref;
3319 }
3320
3321 Eterm
erts_process_gc_info(Process * p,Uint * sizep,Eterm ** hpp,Uint extra_heap_block,Uint extra_old_heap_block_size)3322 erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp,
3323 Uint extra_heap_block,
3324 Uint extra_old_heap_block_size)
3325 {
3326 ERTS_DECL_AM(bin_vheap_size);
3327 ERTS_DECL_AM(bin_vheap_block_size);
3328 ERTS_DECL_AM(bin_old_vheap_size);
3329 ERTS_DECL_AM(bin_old_vheap_block_size);
3330 Eterm tags[] = {
3331 /* If you increase the number of elements here, make sure to update
3332 any call sites as they may have stack allocations that depend
3333 on the number of elements here. */
3334 am_old_heap_block_size,
3335 am_heap_block_size,
3336 am_mbuf_size,
3337 am_recent_size,
3338 am_stack_size,
3339 am_old_heap_size,
3340 am_heap_size,
3341 AM_bin_vheap_size,
3342 AM_bin_vheap_block_size,
3343 AM_bin_old_vheap_size,
3344 AM_bin_old_vheap_block_size
3345 };
3346 UWord values[] = {
3347 OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) + extra_old_heap_block_size
3348 : extra_old_heap_block_size,
3349 HEAP_SIZE(p) + extra_heap_block,
3350 MBUF_SIZE(p),
3351 HIGH_WATER(p) - HEAP_START(p),
3352 STACK_START(p) - p->stop,
3353 OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0,
3354 HEAP_TOP(p) - HEAP_START(p),
3355 MSO(p).overhead,
3356 BIN_VHEAP_SZ(p),
3357 BIN_OLD_VHEAP(p),
3358 BIN_OLD_VHEAP_SZ(p)
3359 };
3360
3361 Eterm res = THE_NON_VALUE;
3362
3363 ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(*tags));
3364 ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == ERTS_PROCESS_GC_INFO_MAX_TERMS);
3365
3366 if (p->abandoned_heap) {
3367 Eterm *htop, *heap;
3368 ERTS_GET_ORIG_HEAP(p, heap, htop);
3369 values[3] = HIGH_WATER(p) - heap;
3370 values[6] = htop - heap;
3371 }
3372
3373 if (p->flags & F_ON_HEAP_MSGQ) {
3374 /* If on heap messages in the internal queue are counted
3375 as being part of the heap, so we have to add them to the
3376 am_mbuf_size value. process_info(total_heap_size) should
3377 be the same as adding old_heap_block_size + heap_block_size
3378 + mbuf_size.
3379 */
3380 ERTS_FOREACH_SIG_PRIVQS(
3381 p, mp,
3382 {
3383 if (ERTS_SIG_IS_MSG(mp)
3384 && mp->data.attached
3385 && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
3386 values[2] += erts_msg_attached_data_size(mp);
3387 }
3388 });
3389 }
3390
3391 res = erts_bld_atom_uword_2tup_list(hpp,
3392 sizep,
3393 sizeof(values)/sizeof(*values),
3394 tags,
3395 values);
3396
3397 return res;
3398 }
3399
3400 static int
reached_max_heap_size(Process * p,Uint total_heap_size,Uint extra_heap_size,Uint extra_old_heap_size)3401 reached_max_heap_size(Process *p, Uint total_heap_size,
3402 Uint extra_heap_size, Uint extra_old_heap_size)
3403 {
3404 Uint max_heap_flags = MAX_HEAP_SIZE_FLAGS_GET(p);
3405 if (IS_TRACED_FL(p, F_TRACE_GC) ||
3406 max_heap_flags & MAX_HEAP_SIZE_LOG) {
3407 Eterm msg;
3408 Uint size = 0;
3409 Eterm *o_hp , *hp;
3410 erts_process_gc_info(p, &size, NULL, extra_heap_size,
3411 extra_old_heap_size);
3412 o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, size * sizeof(Eterm));
3413 msg = erts_process_gc_info(p, NULL, &hp, extra_heap_size,
3414 extra_old_heap_size);
3415
3416 if (max_heap_flags & MAX_HEAP_SIZE_LOG) {
3417 int alive = erts_is_alive;
3418 erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
3419 Eterm *o_hp, *hp, args = NIL;
3420
3421 /* Build the format message */
3422 erts_dsprintf(dsbufp, " Process: ~p ");
3423 if (alive)
3424 erts_dsprintf(dsbufp, "on node ~p");
3425 erts_dsprintf(dsbufp, "~n Context: maximum heap size reached~n");
3426 erts_dsprintf(dsbufp, " Max Heap Size: ~p~n");
3427 erts_dsprintf(dsbufp, " Total Heap Size: ~p~n");
3428 erts_dsprintf(dsbufp, " Kill: ~p~n");
3429 erts_dsprintf(dsbufp, " Error Logger: ~p~n");
3430 erts_dsprintf(dsbufp, " GC Info: ~p~n");
3431
3432 /* Build the args in reverse order */
3433 o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, 2*(alive ? 7 : 6) * sizeof(Eterm));
3434 args = CONS(hp, msg, args); hp += 2;
3435 args = CONS(hp, am_true, args); hp += 2;
3436 args = CONS(hp, (max_heap_flags & MAX_HEAP_SIZE_KILL ? am_true : am_false), args); hp += 2;
3437 args = CONS(hp, make_small(total_heap_size), args); hp += 2;
3438 args = CONS(hp, make_small(MAX_HEAP_SIZE_GET(p)), args); hp += 2;
3439 if (alive) {
3440 args = CONS(hp, erts_this_node->sysname, args); hp += 2;
3441 }
3442 args = CONS(hp, p->common.id, args); hp += 2;
3443
3444 erts_send_error_term_to_logger(p->group_leader, dsbufp, args);
3445 erts_free(ERTS_ALC_T_TMP, o_hp);
3446 }
3447
3448 if (IS_TRACED_FL(p, F_TRACE_GC))
3449 trace_gc(p, am_gc_max_heap_size, 0, msg);
3450
3451 erts_free(ERTS_ALC_T_TMP, o_hp);
3452 }
3453 /* returns true if we should kill the process */
3454 return max_heap_flags & MAX_HEAP_SIZE_KILL;
3455 }
3456
3457 Eterm
erts_max_heap_size_map(Sint max_heap_size,Uint max_heap_flags,Eterm ** hpp,Uint * sz)3458 erts_max_heap_size_map(Sint max_heap_size, Uint max_heap_flags,
3459 Eterm **hpp, Uint *sz)
3460 {
3461 if (!hpp) {
3462 *sz += ERTS_MAX_HEAP_SIZE_MAP_SZ;
3463 return THE_NON_VALUE;
3464 } else {
3465 Eterm *hp = *hpp;
3466 Eterm keys = TUPLE3(hp, am_error_logger, am_kill, am_size);
3467 flatmap_t *mp;
3468 hp += 4;
3469 mp = (flatmap_t*) hp;
3470 mp->thing_word = MAP_HEADER_FLATMAP;
3471 mp->size = 3;
3472 mp->keys = keys;
3473 hp += MAP_HEADER_FLATMAP_SZ;
3474 *hp++ = max_heap_flags & MAX_HEAP_SIZE_LOG ? am_true : am_false;
3475 *hp++ = max_heap_flags & MAX_HEAP_SIZE_KILL ? am_true : am_false;
3476 *hp++ = make_small(max_heap_size);
3477 *hpp = hp;
3478 return make_flatmap(mp);
3479 }
3480 }
3481
3482 int
erts_max_heap_size(Eterm arg,Uint * max_heap_size,Uint * max_heap_flags)3483 erts_max_heap_size(Eterm arg, Uint *max_heap_size, Uint *max_heap_flags)
3484 {
3485 Sint sz;
3486 *max_heap_flags = H_MAX_FLAGS;
3487 if (is_small(arg)) {
3488 sz = signed_val(arg);
3489 *max_heap_flags = H_MAX_FLAGS;
3490 } else if (is_map(arg)) {
3491 const Eterm *size = erts_maps_get(am_size, arg);
3492 const Eterm *kill = erts_maps_get(am_kill, arg);
3493 const Eterm *log = erts_maps_get(am_error_logger, arg);
3494 if (size && is_small(*size)) {
3495 sz = signed_val(*size);
3496 } else {
3497 /* size is mandatory */
3498 return 0;
3499 }
3500 if (kill) {
3501 if (*kill == am_true)
3502 *max_heap_flags |= MAX_HEAP_SIZE_KILL;
3503 else if (*kill == am_false)
3504 *max_heap_flags &= ~MAX_HEAP_SIZE_KILL;
3505 else
3506 return 0;
3507 }
3508 if (log) {
3509 if (*log == am_true)
3510 *max_heap_flags |= MAX_HEAP_SIZE_LOG;
3511 else if (*log == am_false)
3512 *max_heap_flags &= ~MAX_HEAP_SIZE_LOG;
3513 else
3514 return 0;
3515 }
3516 } else
3517 return 0;
3518 if (sz < 0)
3519 return 0;
3520 *max_heap_size = sz;
3521 return 1;
3522 }
3523
3524 #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
3525
3526 int
erts_dbg_within_proc(Eterm * ptr,Process * p,Eterm * real_htop)3527 erts_dbg_within_proc(Eterm *ptr, Process *p, Eterm *real_htop)
3528 {
3529 ErlHeapFragment* bp;
3530 ErtsMessage* mp;
3531 Eterm *htop, *heap;
3532
3533 if (p->abandoned_heap) {
3534 ERTS_GET_ORIG_HEAP(p, heap, htop);
3535 if (heap <= ptr && ptr < htop)
3536 return 1;
3537 }
3538
3539 heap = p->heap;
3540 htop = real_htop ? real_htop : HEAP_TOP(p);
3541
3542 if (OLD_HEAP(p) && (OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p))) {
3543 return 1;
3544 }
3545 if (heap <= ptr && ptr < htop) {
3546 return 1;
3547 }
3548
3549 mp = p->msg_frag;
3550 bp = p->mbuf;
3551
3552 if (bp)
3553 goto search_heap_frags;
3554
3555 while (mp) {
3556
3557 bp = erts_message_to_heap_frag(mp);
3558 mp = mp->next;
3559
3560 search_heap_frags:
3561
3562 while (bp) {
3563 if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) {
3564 return 1;
3565 }
3566 bp = bp->next;
3567 }
3568 }
3569
3570 return 0;
3571 }
3572
3573 #endif
3574
3575 #ifdef ERTS_OFFHEAP_DEBUG
3576
3577 #define ERTS_CHK_OFFHEAP_ASSERT(EXP) \
3578 do { \
3579 if (!(EXP)) \
3580 erts_exit(ERTS_ABORT_EXIT, \
3581 "%s:%d: Assertion failed: %s\n", \
3582 __FILE__, __LINE__, #EXP); \
3583 } while (0)
3584
3585
3586 #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
3587 # define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31)
3588 #endif
3589
3590 void
erts_check_off_heap2(Process * p,Eterm * htop)3591 erts_check_off_heap2(Process *p, Eterm *htop)
3592 {
3593 Eterm *oheap = (Eterm *) OLD_HEAP(p);
3594 Eterm *ohtop = (Eterm *) OLD_HTOP(p);
3595 int old;
3596 union erl_off_heap_ptr u;
3597
3598 old = 0;
3599 for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) {
3600 erts_aint_t refc;
3601 switch (thing_subtag(u.hdr->thing_word)) {
3602 case REFC_BINARY_SUBTAG:
3603 refc = erts_refc_read(&u.pb->val->intern.refc, 1);
3604 break;
3605 case FUN_SUBTAG:
3606 refc = erts_refc_read(&u.fun->fe->refc, 1);
3607 break;
3608 case EXTERNAL_PID_SUBTAG:
3609 case EXTERNAL_PORT_SUBTAG:
3610 case EXTERNAL_REF_SUBTAG:
3611 refc = erts_refc_read(&u.ext->node->refc, 1);
3612 break;
3613 case REF_SUBTAG:
3614 ASSERT(is_magic_ref_thing(u.hdr));
3615 refc = erts_refc_read(&u.mref->mb->intern.refc, 1);
3616 break;
3617 default:
3618 ASSERT(!"erts_check_off_heap2: Invalid thing_word");
3619 }
3620 ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
3621 #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
3622 ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_OFFHEAP_VISITED_BIT));
3623 u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT;
3624 #endif
3625 if (old) {
3626 ERTS_CHK_OFFHEAP_ASSERT(oheap <= u.ep && u.ep < ohtop);
3627 }
3628 else if (oheap <= u.ep && u.ep < ohtop)
3629 old = 1;
3630 else {
3631 ERTS_CHK_OFFHEAP_ASSERT(erts_dbg_within_proc(u.ep, p, htop));
3632 }
3633 }
3634
3635 #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
3636 for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next)
3637 u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT;
3638 #endif
3639 }
3640
3641 void
erts_check_off_heap(Process * p)3642 erts_check_off_heap(Process *p)
3643 {
3644 erts_check_off_heap2(p, NULL);
3645 }
3646
3647 #endif
3648