1 /*
2  *  Memory allocation handling.
3  */
4 
5 #include "duk_internal.h"
6 
7 /*
8  *  Allocate memory with garbage collection.
9  */
10 
11 /* Slow path: voluntary GC triggered, first alloc attempt failed, or zero size. */
duk__heap_mem_alloc_slowpath(duk_heap * heap,duk_size_t size)12 DUK_LOCAL DUK_NOINLINE_PERF DUK_COLD void *duk__heap_mem_alloc_slowpath(duk_heap *heap, duk_size_t size) {
13 	void *res;
14 	duk_small_int_t i;
15 
16 	DUK_ASSERT(heap != NULL);
17 	DUK_ASSERT(heap->alloc_func != NULL);
18 	DUK_ASSERT_DISABLE(size >= 0);
19 
20 	if (size == 0) {
21 		DUK_D(DUK_DPRINT("zero size alloc in slow path, return NULL"));
22 		return NULL;
23 	}
24 
25 	DUK_D(DUK_DPRINT("first alloc attempt failed or voluntary GC limit reached, attempt to gc and retry"));
26 
27 #if 0
28 	/*
29 	 *  If GC is already running there is no point in attempting a GC
30 	 *  because it will be skipped.  This could be checked for explicitly,
31 	 *  but it isn't actually needed: the loop below will eventually
32 	 *  fail resulting in a NULL.
33 	 */
34 
35 	if (heap->ms_prevent_count != 0) {
36 		DUK_D(DUK_DPRINT("duk_heap_mem_alloc() failed, gc in progress (gc skipped), alloc size %ld", (long) size));
37 		return NULL;
38 	}
39 #endif
40 
41 	/*
42 	 *  Retry with several GC attempts.  Initial attempts are made without
43 	 *  emergency mode; later attempts use emergency mode which minimizes
44 	 *  memory allocations forcibly.
45 	 */
46 
47 	for (i = 0; i < DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_LIMIT; i++) {
48 		duk_small_uint_t flags;
49 
50 		flags = 0;
51 		if (i >= DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_EMERGENCY_LIMIT - 1) {
52 			flags |= DUK_MS_FLAG_EMERGENCY;
53 		}
54 
55 		duk_heap_mark_and_sweep(heap, flags);
56 
57 		DUK_ASSERT(size > 0);
58 		res = heap->alloc_func(heap->heap_udata, size);
59 		if (res != NULL) {
60 			DUK_D(DUK_DPRINT("duk_heap_mem_alloc() succeeded after gc (pass %ld), alloc size %ld",
61 			                 (long) (i + 1), (long) size));
62 			return res;
63 		}
64 	}
65 
66 	DUK_D(DUK_DPRINT("duk_heap_mem_alloc() failed even after gc, alloc size %ld", (long) size));
67 	return NULL;
68 }
69 
duk_heap_mem_alloc(duk_heap * heap,duk_size_t size)70 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_alloc(duk_heap *heap, duk_size_t size) {
71 	void *res;
72 
73 	DUK_ASSERT(heap != NULL);
74 	DUK_ASSERT(heap->alloc_func != NULL);
75 	DUK_ASSERT_DISABLE(size >= 0);
76 
77 #if defined(DUK_USE_VOLUNTARY_GC)
78 	/* Voluntary periodic GC (if enabled). */
79 	if (DUK_UNLIKELY(--(heap)->ms_trigger_counter < 0)) {
80 		goto slowpath;
81 	}
82 #endif
83 
84 #if defined(DUK_USE_GC_TORTURE)
85 	/* Simulate alloc failure on every alloc, except when mark-and-sweep
86 	 * is running.
87 	 */
88 	if (heap->ms_prevent_count == 0) {
89 		DUK_DDD(DUK_DDDPRINT("gc torture enabled, pretend that first alloc attempt fails"));
90 		res = NULL;
91 		DUK_UNREF(res);
92 		goto slowpath;
93 	}
94 #endif
95 
96 	/* Zero-size allocation should happen very rarely (if at all), so
97 	 * don't check zero size on NULL; handle it in the slow path
98 	 * instead.  This reduces size of inlined code.
99 	 */
100 	res = heap->alloc_func(heap->heap_udata, size);
101 	if (DUK_LIKELY(res != NULL)) {
102 		return res;
103 	}
104 
105  slowpath:
106 
107 	if (size == 0) {
108 		DUK_D(DUK_DPRINT("first alloc attempt returned NULL for zero size alloc, use slow path to deal with it"));
109 	} else {
110 		DUK_D(DUK_DPRINT("first alloc attempt failed, attempt to gc and retry"));
111 	}
112 	return duk__heap_mem_alloc_slowpath(heap, size);
113 }
114 
duk_heap_mem_alloc_zeroed(duk_heap * heap,duk_size_t size)115 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_alloc_zeroed(duk_heap *heap, duk_size_t size) {
116 	void *res;
117 
118 	DUK_ASSERT(heap != NULL);
119 	DUK_ASSERT(heap->alloc_func != NULL);
120 	DUK_ASSERT_DISABLE(size >= 0);
121 
122 	res = DUK_ALLOC(heap, size);
123 	if (DUK_LIKELY(res != NULL)) {
124 		duk_memzero(res, size);
125 	}
126 	return res;
127 }
128 
duk_heap_mem_alloc_checked(duk_hthread * thr,duk_size_t size)129 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_alloc_checked(duk_hthread *thr, duk_size_t size) {
130 	void *res;
131 
132 	DUK_ASSERT(thr != NULL);
133 	DUK_ASSERT(thr->heap != NULL);
134 	DUK_ASSERT(thr->heap->alloc_func != NULL);
135 
136 	res = duk_heap_mem_alloc(thr->heap, size);
137 	if (DUK_LIKELY(res != NULL)) {
138 		return res;
139 	} else if (size == 0) {
140 		DUK_ASSERT(res == NULL);
141 		return res;
142 	}
143 	DUK_ERROR_ALLOC_FAILED(thr);
144 	DUK_WO_NORETURN(return NULL;);
145 }
146 
duk_heap_mem_alloc_checked_zeroed(duk_hthread * thr,duk_size_t size)147 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_alloc_checked_zeroed(duk_hthread *thr, duk_size_t size) {
148 	void *res;
149 
150 	DUK_ASSERT(thr != NULL);
151 	DUK_ASSERT(thr->heap != NULL);
152 	DUK_ASSERT(thr->heap->alloc_func != NULL);
153 
154 	res = duk_heap_mem_alloc(thr->heap, size);
155 	if (DUK_LIKELY(res != NULL)) {
156 		duk_memzero(res, size);
157 		return res;
158 	} else if (size == 0) {
159 		DUK_ASSERT(res == NULL);
160 		return res;
161 	}
162 	DUK_ERROR_ALLOC_FAILED(thr);
163 	DUK_WO_NORETURN(return NULL;);
164 }
165 
166 /*
167  *  Reallocate memory with garbage collection.
168  */
169 
170 /* Slow path: voluntary GC triggered, first realloc attempt failed, or zero size. */
duk__heap_mem_realloc_slowpath(duk_heap * heap,void * ptr,duk_size_t newsize)171 DUK_LOCAL DUK_NOINLINE_PERF DUK_COLD void *duk__heap_mem_realloc_slowpath(duk_heap *heap, void *ptr, duk_size_t newsize) {
172 	void *res;
173 	duk_small_int_t i;
174 
175 	DUK_ASSERT(heap != NULL);
176 	DUK_ASSERT(heap->realloc_func != NULL);
177 	/* ptr may be NULL */
178 	DUK_ASSERT_DISABLE(newsize >= 0);
179 
180 	/* Newsize was 0 and realloc() returned NULL, this has the semantics
181 	 * of free(oldptr), i.e. memory was successfully freed.
182 	 */
183 	if (newsize == 0) {
184 		DUK_D(DUK_DPRINT("zero size realloc in slow path, return NULL"));
185 		return NULL;
186 	}
187 
188 	DUK_D(DUK_DPRINT("first realloc attempt failed, attempt to gc and retry"));
189 
190 #if 0
191 	/*
192 	 *  Avoid a GC if GC is already running.  See duk_heap_mem_alloc().
193 	 */
194 
195 	if (heap->ms_prevent_count != 0) {
196 		DUK_D(DUK_DPRINT("duk_heap_mem_realloc() failed, gc in progress (gc skipped), alloc size %ld", (long) newsize));
197 		return NULL;
198 	}
199 #endif
200 
201 	/*
202 	 *  Retry with several GC attempts.  Initial attempts are made without
203 	 *  emergency mode; later attempts use emergency mode which minimizes
204 	 *  memory allocations forcibly.
205 	 */
206 
207 	for (i = 0; i < DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_LIMIT; i++) {
208 		duk_small_uint_t flags;
209 
210 		flags = 0;
211 		if (i >= DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_EMERGENCY_LIMIT - 1) {
212 			flags |= DUK_MS_FLAG_EMERGENCY;
213 		}
214 
215 		duk_heap_mark_and_sweep(heap, flags);
216 
217 		DUK_ASSERT(newsize > 0);
218 		res = heap->realloc_func(heap->heap_udata, ptr, newsize);
219 		if (res || newsize == 0) {
220 			DUK_D(DUK_DPRINT("duk_heap_mem_realloc() succeeded after gc (pass %ld), alloc size %ld",
221 			                 (long) (i + 1), (long) newsize));
222 			return res;
223 		}
224 	}
225 
226 	DUK_D(DUK_DPRINT("duk_heap_mem_realloc() failed even after gc, alloc size %ld", (long) newsize));
227 	return NULL;
228 }
229 
duk_heap_mem_realloc(duk_heap * heap,void * ptr,duk_size_t newsize)230 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_realloc(duk_heap *heap, void *ptr, duk_size_t newsize) {
231 	void *res;
232 
233 	DUK_ASSERT(heap != NULL);
234 	DUK_ASSERT(heap->realloc_func != NULL);
235 	/* ptr may be NULL */
236 	DUK_ASSERT_DISABLE(newsize >= 0);
237 
238 #if defined(DUK_USE_VOLUNTARY_GC)
239 	/* Voluntary periodic GC (if enabled). */
240 	if (DUK_UNLIKELY(--(heap)->ms_trigger_counter < 0)) {
241 		goto slowpath;
242 	}
243 #endif
244 
245 #if defined(DUK_USE_GC_TORTURE)
246 	/* Simulate alloc failure on every realloc, except when mark-and-sweep
247 	 * is running.
248 	 */
249 	if (heap->ms_prevent_count == 0) {
250 		DUK_DDD(DUK_DDDPRINT("gc torture enabled, pretend that first realloc attempt fails"));
251 		res = NULL;
252 		DUK_UNREF(res);
253 		goto slowpath;
254 	}
255 #endif
256 
257 	res = heap->realloc_func(heap->heap_udata, ptr, newsize);
258 	if (DUK_LIKELY(res != NULL)) {
259 		return res;
260 	}
261 
262  slowpath:
263 
264 	if (newsize == 0) {
265 		DUK_D(DUK_DPRINT("first realloc attempt returned NULL for zero size realloc, use slow path to deal with it"));
266 	} else {
267 		DUK_D(DUK_DPRINT("first realloc attempt failed, attempt to gc and retry"));
268 	}
269 	return duk__heap_mem_realloc_slowpath(heap, ptr, newsize);
270 }
271 
272 /*
273  *  Reallocate memory with garbage collection, using a callback to provide
274  *  the current allocated pointer.  This variant is used when a mark-and-sweep
275  *  (e.g. finalizers) might change the original pointer.
276  */
277 
278 /* Slow path: voluntary GC triggered, first realloc attempt failed, or zero size. */
duk__heap_mem_realloc_indirect_slowpath(duk_heap * heap,duk_mem_getptr cb,void * ud,duk_size_t newsize)279 DUK_LOCAL DUK_NOINLINE_PERF DUK_COLD void *duk__heap_mem_realloc_indirect_slowpath(duk_heap *heap, duk_mem_getptr cb, void *ud, duk_size_t newsize) {
280 	void *res;
281 	duk_small_int_t i;
282 
283 	DUK_ASSERT(heap != NULL);
284 	DUK_ASSERT(heap->realloc_func != NULL);
285 	DUK_ASSERT_DISABLE(newsize >= 0);
286 
287 	if (newsize == 0) {
288 		DUK_D(DUK_DPRINT("zero size indirect realloc in slow path, return NULL"));
289 		return NULL;
290 	}
291 
292 	DUK_D(DUK_DPRINT("first indirect realloc attempt failed, attempt to gc and retry"));
293 
294 #if 0
295 	/*
296 	 *  Avoid a GC if GC is already running.  See duk_heap_mem_alloc().
297 	 */
298 
299 	if (heap->ms_prevent_count != 0) {
300 		DUK_D(DUK_DPRINT("duk_heap_mem_realloc_indirect() failed, gc in progress (gc skipped), alloc size %ld", (long) newsize));
301 		return NULL;
302 	}
303 #endif
304 
305 	/*
306 	 *  Retry with several GC attempts.  Initial attempts are made without
307 	 *  emergency mode; later attempts use emergency mode which minimizes
308 	 *  memory allocations forcibly.
309 	 */
310 
311 	for (i = 0; i < DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_LIMIT; i++) {
312 		duk_small_uint_t flags;
313 
314 #if defined(DUK_USE_DEBUG)
315 		void *ptr_pre;
316 		void *ptr_post;
317 #endif
318 
319 #if defined(DUK_USE_DEBUG)
320 		ptr_pre = cb(heap, ud);
321 #endif
322 		flags = 0;
323 		if (i >= DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_EMERGENCY_LIMIT - 1) {
324 			flags |= DUK_MS_FLAG_EMERGENCY;
325 		}
326 
327 		duk_heap_mark_and_sweep(heap, flags);
328 #if defined(DUK_USE_DEBUG)
329 		ptr_post = cb(heap, ud);
330 		if (ptr_pre != ptr_post) {
331 			DUK_DD(DUK_DDPRINT("realloc base pointer changed by mark-and-sweep: %p -> %p",
332 			                   (void *) ptr_pre, (void *) ptr_post));
333 		}
334 #endif
335 
336 		/* Note: key issue here is to re-lookup the base pointer on every attempt.
337 		 * The pointer being reallocated may change after every mark-and-sweep.
338 		 */
339 
340 		DUK_ASSERT(newsize > 0);
341 		res = heap->realloc_func(heap->heap_udata, cb(heap, ud), newsize);
342 		if (res || newsize == 0) {
343 			DUK_D(DUK_DPRINT("duk_heap_mem_realloc_indirect() succeeded after gc (pass %ld), alloc size %ld",
344 			                 (long) (i + 1), (long) newsize));
345 			return res;
346 		}
347 	}
348 
349 	DUK_D(DUK_DPRINT("duk_heap_mem_realloc_indirect() failed even after gc, alloc size %ld", (long) newsize));
350 	return NULL;
351 }
352 
duk_heap_mem_realloc_indirect(duk_heap * heap,duk_mem_getptr cb,void * ud,duk_size_t newsize)353 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_realloc_indirect(duk_heap *heap, duk_mem_getptr cb, void *ud, duk_size_t newsize) {
354 	void *res;
355 
356 	DUK_ASSERT(heap != NULL);
357 	DUK_ASSERT(heap->realloc_func != NULL);
358 	DUK_ASSERT_DISABLE(newsize >= 0);
359 
360 #if defined(DUK_USE_VOLUNTARY_GC)
361 	/* Voluntary periodic GC (if enabled). */
362 	if (DUK_UNLIKELY(--(heap)->ms_trigger_counter < 0)) {
363 		goto slowpath;
364 	}
365 #endif
366 
367 #if defined(DUK_USE_GC_TORTURE)
368 	/* Simulate alloc failure on every realloc, except when mark-and-sweep
369 	 * is running.
370 	 */
371 	if (heap->ms_prevent_count == 0) {
372 		DUK_DDD(DUK_DDDPRINT("gc torture enabled, pretend that first indirect realloc attempt fails"));
373 		res = NULL;
374 		DUK_UNREF(res);
375 		goto slowpath;
376 	}
377 #endif
378 
379 	res = heap->realloc_func(heap->heap_udata, cb(heap, ud), newsize);
380 	if (DUK_LIKELY(res != NULL)) {
381 		return res;
382 	}
383 
384  slowpath:
385 
386 	if (newsize == 0) {
387 		DUK_D(DUK_DPRINT("first indirect realloc attempt returned NULL for zero size realloc, use slow path to deal with it"));
388 	} else {
389 		DUK_D(DUK_DPRINT("first indirect realloc attempt failed, attempt to gc and retry"));
390 	}
391 	return duk__heap_mem_realloc_indirect_slowpath(heap, cb, ud, newsize);
392 }
393 
394 /*
395  *  Free memory
396  */
397 
duk_heap_mem_free(duk_heap * heap,void * ptr)398 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void duk_heap_mem_free(duk_heap *heap, void *ptr) {
399 	DUK_ASSERT(heap != NULL);
400 	DUK_ASSERT(heap->free_func != NULL);
401 	/* ptr may be NULL */
402 
403 	/* Must behave like a no-op with NULL and any pointer returned from
404 	 * malloc/realloc with zero size.
405 	 */
406 	heap->free_func(heap->heap_udata, ptr);
407 
408 	/* Never perform a GC (even voluntary) in a memory free, otherwise
409 	 * all call sites doing frees would need to deal with the side effects.
410 	 * No need to update voluntary GC counter either.
411 	 */
412 }
413