1 /*
2 * Memory allocation handling.
3 */
4
5 #include "duk_internal.h"
6
7 /*
8 * Allocate memory with garbage collection.
9 */
10
11 /* Slow path: voluntary GC triggered, first alloc attempt failed, or zero size. */
duk__heap_mem_alloc_slowpath(duk_heap * heap,duk_size_t size)12 DUK_LOCAL DUK_NOINLINE_PERF DUK_COLD void *duk__heap_mem_alloc_slowpath(duk_heap *heap, duk_size_t size) {
13 void *res;
14 duk_small_int_t i;
15
16 DUK_ASSERT(heap != NULL);
17 DUK_ASSERT(heap->alloc_func != NULL);
18 DUK_ASSERT_DISABLE(size >= 0);
19
20 if (size == 0) {
21 DUK_D(DUK_DPRINT("zero size alloc in slow path, return NULL"));
22 return NULL;
23 }
24
25 DUK_D(DUK_DPRINT("first alloc attempt failed or voluntary GC limit reached, attempt to gc and retry"));
26
27 #if 0
28 /*
29 * If GC is already running there is no point in attempting a GC
30 * because it will be skipped. This could be checked for explicitly,
31 * but it isn't actually needed: the loop below will eventually
32 * fail resulting in a NULL.
33 */
34
35 if (heap->ms_prevent_count != 0) {
36 DUK_D(DUK_DPRINT("duk_heap_mem_alloc() failed, gc in progress (gc skipped), alloc size %ld", (long) size));
37 return NULL;
38 }
39 #endif
40
41 /*
42 * Retry with several GC attempts. Initial attempts are made without
43 * emergency mode; later attempts use emergency mode which minimizes
44 * memory allocations forcibly.
45 */
46
47 for (i = 0; i < DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_LIMIT; i++) {
48 duk_small_uint_t flags;
49
50 flags = 0;
51 if (i >= DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_EMERGENCY_LIMIT - 1) {
52 flags |= DUK_MS_FLAG_EMERGENCY;
53 }
54
55 duk_heap_mark_and_sweep(heap, flags);
56
57 DUK_ASSERT(size > 0);
58 res = heap->alloc_func(heap->heap_udata, size);
59 if (res != NULL) {
60 DUK_D(DUK_DPRINT("duk_heap_mem_alloc() succeeded after gc (pass %ld), alloc size %ld",
61 (long) (i + 1), (long) size));
62 return res;
63 }
64 }
65
66 DUK_D(DUK_DPRINT("duk_heap_mem_alloc() failed even after gc, alloc size %ld", (long) size));
67 return NULL;
68 }
69
duk_heap_mem_alloc(duk_heap * heap,duk_size_t size)70 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_alloc(duk_heap *heap, duk_size_t size) {
71 void *res;
72
73 DUK_ASSERT(heap != NULL);
74 DUK_ASSERT(heap->alloc_func != NULL);
75 DUK_ASSERT_DISABLE(size >= 0);
76
77 #if defined(DUK_USE_VOLUNTARY_GC)
78 /* Voluntary periodic GC (if enabled). */
79 if (DUK_UNLIKELY(--(heap)->ms_trigger_counter < 0)) {
80 goto slowpath;
81 }
82 #endif
83
84 #if defined(DUK_USE_GC_TORTURE)
85 /* Simulate alloc failure on every alloc, except when mark-and-sweep
86 * is running.
87 */
88 if (heap->ms_prevent_count == 0) {
89 DUK_DDD(DUK_DDDPRINT("gc torture enabled, pretend that first alloc attempt fails"));
90 res = NULL;
91 DUK_UNREF(res);
92 goto slowpath;
93 }
94 #endif
95
96 /* Zero-size allocation should happen very rarely (if at all), so
97 * don't check zero size on NULL; handle it in the slow path
98 * instead. This reduces size of inlined code.
99 */
100 res = heap->alloc_func(heap->heap_udata, size);
101 if (DUK_LIKELY(res != NULL)) {
102 return res;
103 }
104
105 slowpath:
106
107 if (size == 0) {
108 DUK_D(DUK_DPRINT("first alloc attempt returned NULL for zero size alloc, use slow path to deal with it"));
109 } else {
110 DUK_D(DUK_DPRINT("first alloc attempt failed, attempt to gc and retry"));
111 }
112 return duk__heap_mem_alloc_slowpath(heap, size);
113 }
114
duk_heap_mem_alloc_zeroed(duk_heap * heap,duk_size_t size)115 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_alloc_zeroed(duk_heap *heap, duk_size_t size) {
116 void *res;
117
118 DUK_ASSERT(heap != NULL);
119 DUK_ASSERT(heap->alloc_func != NULL);
120 DUK_ASSERT_DISABLE(size >= 0);
121
122 res = DUK_ALLOC(heap, size);
123 if (DUK_LIKELY(res != NULL)) {
124 duk_memzero(res, size);
125 }
126 return res;
127 }
128
duk_heap_mem_alloc_checked(duk_hthread * thr,duk_size_t size)129 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_alloc_checked(duk_hthread *thr, duk_size_t size) {
130 void *res;
131
132 DUK_ASSERT(thr != NULL);
133 DUK_ASSERT(thr->heap != NULL);
134 DUK_ASSERT(thr->heap->alloc_func != NULL);
135
136 res = duk_heap_mem_alloc(thr->heap, size);
137 if (DUK_LIKELY(res != NULL)) {
138 return res;
139 } else if (size == 0) {
140 DUK_ASSERT(res == NULL);
141 return res;
142 }
143 DUK_ERROR_ALLOC_FAILED(thr);
144 DUK_WO_NORETURN(return NULL;);
145 }
146
duk_heap_mem_alloc_checked_zeroed(duk_hthread * thr,duk_size_t size)147 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_alloc_checked_zeroed(duk_hthread *thr, duk_size_t size) {
148 void *res;
149
150 DUK_ASSERT(thr != NULL);
151 DUK_ASSERT(thr->heap != NULL);
152 DUK_ASSERT(thr->heap->alloc_func != NULL);
153
154 res = duk_heap_mem_alloc(thr->heap, size);
155 if (DUK_LIKELY(res != NULL)) {
156 duk_memzero(res, size);
157 return res;
158 } else if (size == 0) {
159 DUK_ASSERT(res == NULL);
160 return res;
161 }
162 DUK_ERROR_ALLOC_FAILED(thr);
163 DUK_WO_NORETURN(return NULL;);
164 }
165
166 /*
167 * Reallocate memory with garbage collection.
168 */
169
170 /* Slow path: voluntary GC triggered, first realloc attempt failed, or zero size. */
duk__heap_mem_realloc_slowpath(duk_heap * heap,void * ptr,duk_size_t newsize)171 DUK_LOCAL DUK_NOINLINE_PERF DUK_COLD void *duk__heap_mem_realloc_slowpath(duk_heap *heap, void *ptr, duk_size_t newsize) {
172 void *res;
173 duk_small_int_t i;
174
175 DUK_ASSERT(heap != NULL);
176 DUK_ASSERT(heap->realloc_func != NULL);
177 /* ptr may be NULL */
178 DUK_ASSERT_DISABLE(newsize >= 0);
179
180 /* Newsize was 0 and realloc() returned NULL, this has the semantics
181 * of free(oldptr), i.e. memory was successfully freed.
182 */
183 if (newsize == 0) {
184 DUK_D(DUK_DPRINT("zero size realloc in slow path, return NULL"));
185 return NULL;
186 }
187
188 DUK_D(DUK_DPRINT("first realloc attempt failed, attempt to gc and retry"));
189
190 #if 0
191 /*
192 * Avoid a GC if GC is already running. See duk_heap_mem_alloc().
193 */
194
195 if (heap->ms_prevent_count != 0) {
196 DUK_D(DUK_DPRINT("duk_heap_mem_realloc() failed, gc in progress (gc skipped), alloc size %ld", (long) newsize));
197 return NULL;
198 }
199 #endif
200
201 /*
202 * Retry with several GC attempts. Initial attempts are made without
203 * emergency mode; later attempts use emergency mode which minimizes
204 * memory allocations forcibly.
205 */
206
207 for (i = 0; i < DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_LIMIT; i++) {
208 duk_small_uint_t flags;
209
210 flags = 0;
211 if (i >= DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_EMERGENCY_LIMIT - 1) {
212 flags |= DUK_MS_FLAG_EMERGENCY;
213 }
214
215 duk_heap_mark_and_sweep(heap, flags);
216
217 DUK_ASSERT(newsize > 0);
218 res = heap->realloc_func(heap->heap_udata, ptr, newsize);
219 if (res || newsize == 0) {
220 DUK_D(DUK_DPRINT("duk_heap_mem_realloc() succeeded after gc (pass %ld), alloc size %ld",
221 (long) (i + 1), (long) newsize));
222 return res;
223 }
224 }
225
226 DUK_D(DUK_DPRINT("duk_heap_mem_realloc() failed even after gc, alloc size %ld", (long) newsize));
227 return NULL;
228 }
229
duk_heap_mem_realloc(duk_heap * heap,void * ptr,duk_size_t newsize)230 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_realloc(duk_heap *heap, void *ptr, duk_size_t newsize) {
231 void *res;
232
233 DUK_ASSERT(heap != NULL);
234 DUK_ASSERT(heap->realloc_func != NULL);
235 /* ptr may be NULL */
236 DUK_ASSERT_DISABLE(newsize >= 0);
237
238 #if defined(DUK_USE_VOLUNTARY_GC)
239 /* Voluntary periodic GC (if enabled). */
240 if (DUK_UNLIKELY(--(heap)->ms_trigger_counter < 0)) {
241 goto slowpath;
242 }
243 #endif
244
245 #if defined(DUK_USE_GC_TORTURE)
246 /* Simulate alloc failure on every realloc, except when mark-and-sweep
247 * is running.
248 */
249 if (heap->ms_prevent_count == 0) {
250 DUK_DDD(DUK_DDDPRINT("gc torture enabled, pretend that first realloc attempt fails"));
251 res = NULL;
252 DUK_UNREF(res);
253 goto slowpath;
254 }
255 #endif
256
257 res = heap->realloc_func(heap->heap_udata, ptr, newsize);
258 if (DUK_LIKELY(res != NULL)) {
259 return res;
260 }
261
262 slowpath:
263
264 if (newsize == 0) {
265 DUK_D(DUK_DPRINT("first realloc attempt returned NULL for zero size realloc, use slow path to deal with it"));
266 } else {
267 DUK_D(DUK_DPRINT("first realloc attempt failed, attempt to gc and retry"));
268 }
269 return duk__heap_mem_realloc_slowpath(heap, ptr, newsize);
270 }
271
272 /*
273 * Reallocate memory with garbage collection, using a callback to provide
274 * the current allocated pointer. This variant is used when a mark-and-sweep
275 * (e.g. finalizers) might change the original pointer.
276 */
277
278 /* Slow path: voluntary GC triggered, first realloc attempt failed, or zero size. */
duk__heap_mem_realloc_indirect_slowpath(duk_heap * heap,duk_mem_getptr cb,void * ud,duk_size_t newsize)279 DUK_LOCAL DUK_NOINLINE_PERF DUK_COLD void *duk__heap_mem_realloc_indirect_slowpath(duk_heap *heap, duk_mem_getptr cb, void *ud, duk_size_t newsize) {
280 void *res;
281 duk_small_int_t i;
282
283 DUK_ASSERT(heap != NULL);
284 DUK_ASSERT(heap->realloc_func != NULL);
285 DUK_ASSERT_DISABLE(newsize >= 0);
286
287 if (newsize == 0) {
288 DUK_D(DUK_DPRINT("zero size indirect realloc in slow path, return NULL"));
289 return NULL;
290 }
291
292 DUK_D(DUK_DPRINT("first indirect realloc attempt failed, attempt to gc and retry"));
293
294 #if 0
295 /*
296 * Avoid a GC if GC is already running. See duk_heap_mem_alloc().
297 */
298
299 if (heap->ms_prevent_count != 0) {
300 DUK_D(DUK_DPRINT("duk_heap_mem_realloc_indirect() failed, gc in progress (gc skipped), alloc size %ld", (long) newsize));
301 return NULL;
302 }
303 #endif
304
305 /*
306 * Retry with several GC attempts. Initial attempts are made without
307 * emergency mode; later attempts use emergency mode which minimizes
308 * memory allocations forcibly.
309 */
310
311 for (i = 0; i < DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_LIMIT; i++) {
312 duk_small_uint_t flags;
313
314 #if defined(DUK_USE_DEBUG)
315 void *ptr_pre;
316 void *ptr_post;
317 #endif
318
319 #if defined(DUK_USE_DEBUG)
320 ptr_pre = cb(heap, ud);
321 #endif
322 flags = 0;
323 if (i >= DUK_HEAP_ALLOC_FAIL_MARKANDSWEEP_EMERGENCY_LIMIT - 1) {
324 flags |= DUK_MS_FLAG_EMERGENCY;
325 }
326
327 duk_heap_mark_and_sweep(heap, flags);
328 #if defined(DUK_USE_DEBUG)
329 ptr_post = cb(heap, ud);
330 if (ptr_pre != ptr_post) {
331 DUK_DD(DUK_DDPRINT("realloc base pointer changed by mark-and-sweep: %p -> %p",
332 (void *) ptr_pre, (void *) ptr_post));
333 }
334 #endif
335
336 /* Note: key issue here is to re-lookup the base pointer on every attempt.
337 * The pointer being reallocated may change after every mark-and-sweep.
338 */
339
340 DUK_ASSERT(newsize > 0);
341 res = heap->realloc_func(heap->heap_udata, cb(heap, ud), newsize);
342 if (res || newsize == 0) {
343 DUK_D(DUK_DPRINT("duk_heap_mem_realloc_indirect() succeeded after gc (pass %ld), alloc size %ld",
344 (long) (i + 1), (long) newsize));
345 return res;
346 }
347 }
348
349 DUK_D(DUK_DPRINT("duk_heap_mem_realloc_indirect() failed even after gc, alloc size %ld", (long) newsize));
350 return NULL;
351 }
352
duk_heap_mem_realloc_indirect(duk_heap * heap,duk_mem_getptr cb,void * ud,duk_size_t newsize)353 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void *duk_heap_mem_realloc_indirect(duk_heap *heap, duk_mem_getptr cb, void *ud, duk_size_t newsize) {
354 void *res;
355
356 DUK_ASSERT(heap != NULL);
357 DUK_ASSERT(heap->realloc_func != NULL);
358 DUK_ASSERT_DISABLE(newsize >= 0);
359
360 #if defined(DUK_USE_VOLUNTARY_GC)
361 /* Voluntary periodic GC (if enabled). */
362 if (DUK_UNLIKELY(--(heap)->ms_trigger_counter < 0)) {
363 goto slowpath;
364 }
365 #endif
366
367 #if defined(DUK_USE_GC_TORTURE)
368 /* Simulate alloc failure on every realloc, except when mark-and-sweep
369 * is running.
370 */
371 if (heap->ms_prevent_count == 0) {
372 DUK_DDD(DUK_DDDPRINT("gc torture enabled, pretend that first indirect realloc attempt fails"));
373 res = NULL;
374 DUK_UNREF(res);
375 goto slowpath;
376 }
377 #endif
378
379 res = heap->realloc_func(heap->heap_udata, cb(heap, ud), newsize);
380 if (DUK_LIKELY(res != NULL)) {
381 return res;
382 }
383
384 slowpath:
385
386 if (newsize == 0) {
387 DUK_D(DUK_DPRINT("first indirect realloc attempt returned NULL for zero size realloc, use slow path to deal with it"));
388 } else {
389 DUK_D(DUK_DPRINT("first indirect realloc attempt failed, attempt to gc and retry"));
390 }
391 return duk__heap_mem_realloc_indirect_slowpath(heap, cb, ud, newsize);
392 }
393
394 /*
395 * Free memory
396 */
397
duk_heap_mem_free(duk_heap * heap,void * ptr)398 DUK_INTERNAL DUK_INLINE_PERF DUK_HOT void duk_heap_mem_free(duk_heap *heap, void *ptr) {
399 DUK_ASSERT(heap != NULL);
400 DUK_ASSERT(heap->free_func != NULL);
401 /* ptr may be NULL */
402
403 /* Must behave like a no-op with NULL and any pointer returned from
404 * malloc/realloc with zero size.
405 */
406 heap->free_func(heap->heap_udata, ptr);
407
408 /* Never perform a GC (even voluntary) in a memory free, otherwise
409 * all call sites doing frees would need to deal with the side effects.
410 * No need to update voluntary GC counter either.
411 */
412 }
413