1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 /*
8 * [SMDOC] GC Scheduling
9 *
10 * GC Scheduling Overview
11 * ======================
12 *
13 * Scheduling GC's in SpiderMonkey/Firefox is tremendously complicated because
14 * of the large number of subtle, cross-cutting, and widely dispersed factors
15 * that must be taken into account. A summary of some of the more important
16 * factors follows.
17 *
18 * Cost factors:
19 *
20 * * GC too soon and we'll revisit an object graph almost identical to the
21 * one we just visited; since we are unlikely to find new garbage, the
22 * traversal will be largely overhead. We rely heavily on external factors
23 * to signal us that we are likely to find lots of garbage: e.g. "a tab
24 * just got closed".
25 *
26 * * GC too late and we'll run out of memory to allocate (e.g. Out-Of-Memory,
27 * hereafter simply abbreviated to OOM). If this happens inside
28 * SpiderMonkey we may be able to recover, but most embedder allocations
29 * will simply crash on OOM, even if the GC has plenty of free memory it
30 * could surrender.
31 *
32 * * Memory fragmentation: if we fill the process with GC allocations, a
33 * request for a large block of contiguous memory may fail because no
34 * contiguous block is free, despite having enough memory available to
35 * service the request.
36 *
37 * * Management overhead: if our GC heap becomes large, we create extra
38 * overhead when managing the GC's structures, even if the allocations are
39 * mostly unused.
40 *
41 * Heap Management Factors:
42 *
43 * * GC memory: The GC has its own allocator that it uses to make fixed size
44 * allocations for GC managed things. In cases where the GC thing requires
45 * larger or variable sized memory to implement itself, it is responsible
46 * for using the system heap.
47 *
48 * * C Heap Memory: Rather than allowing for large or variable allocations,
49 * the SpiderMonkey GC allows GC things to hold pointers to C heap memory.
50 * It is the responsibility of the thing to free this memory with a custom
51 * finalizer (with the sole exception of NativeObject, which knows about
52 * slots and elements for performance reasons). C heap memory has different
53 * performance and overhead tradeoffs than GC internal memory, which need
54 * to be considered with scheduling a GC.
55 *
56 * Application Factors:
57 *
58 * * Most applications allocate heavily at startup, then enter a processing
59 * stage where memory utilization remains roughly fixed with a slower
60 * allocation rate. This is not always the case, however, so while we may
61 * optimize for this pattern, we must be able to handle arbitrary
62 * allocation patterns.
63 *
64 * Other factors:
65 *
66 * * Other memory: This is memory allocated outside the purview of the GC.
67 * Data mapped by the system for code libraries, data allocated by those
68 * libraries, data in the JSRuntime that is used to manage the engine,
69 * memory used by the embedding that is not attached to a GC thing, memory
70 * used by unrelated processes running on the hardware that use space we
71 * could otherwise use for allocation, etc. While we don't have to manage
72 * it, we do have to take it into account when scheduling since it affects
73 * when we will OOM.
74 *
75 * * Physical Reality: All real machines have limits on the number of bits
76 * that they are physically able to store. While modern operating systems
77 * can generally make additional space available with swapping, at some
78 * point there are simply no more bits to allocate. There is also the
79 * factor of address space limitations, particularly on 32bit machines.
80 *
81 * * Platform Factors: Each OS makes use of wildly different memory
82 * management techniques. These differences result in different performance
83 * tradeoffs, different fragmentation patterns, and different hard limits
84 * on the amount of physical and/or virtual memory that we can use before
85 * OOMing.
86 *
87 *
88 * Reasons for scheduling GC
89 * -------------------------
90 *
91 * While code generally takes the above factors into account in only an ad-hoc
92 * fashion, the API forces the user to pick a "reason" for the GC. We have a
93 * bunch of JS::GCReason reasons in GCAPI.h. These fall into a few categories
94 * that generally coincide with one or more of the above factors.
95 *
96 * Embedding reasons:
97 *
98 * 1) Do a GC now because the embedding knows something useful about the
99 * zone's memory retention state. These are GCReasons like LOAD_END,
100 * PAGE_HIDE, SET_NEW_DOCUMENT, DOM_UTILS. Mostly, Gecko uses these to
101 * indicate that a significant fraction of the scheduled zone's memory is
102 * probably reclaimable.
103 *
104 * 2) Do some known amount of GC work now because the embedding knows now is
105 * a good time to do a long, unblockable operation of a known duration.
106 * These are INTER_SLICE_GC and REFRESH_FRAME.
107 *
108 * Correctness reasons:
109 *
110 * 3) Do a GC now because correctness depends on some GC property. For
111 * example, CC_WAITING is where the embedding requires the mark bits
112 * to be set correct. Also, EVICT_NURSERY where we need to work on the
113 * tenured heap.
114 *
115 * 4) Do a GC because we are shutting down: e.g. SHUTDOWN_CC or DESTROY_*.
116 *
117 * 5) Do a GC because a compartment was accessed between GC slices when we
118 * would have otherwise discarded it. We have to do a second GC to clean
119 * it up: e.g. COMPARTMENT_REVIVED.
120 *
121 * Emergency Reasons:
122 *
123 * 6) Do an all-zones, non-incremental GC now because the embedding knows it
124 * cannot wait: e.g. MEM_PRESSURE.
125 *
126 * 7) OOM when fetching a new Chunk results in a LAST_DITCH GC.
127 *
128 * Heap Size Limitation Reasons:
129 *
130 * 8) Do an incremental, zonal GC with reason MAYBEGC when we discover that
131 * the gc's allocated size is approaching the current trigger. This is
132 * called MAYBEGC because we make this check in the MaybeGC function.
133 * MaybeGC gets called at the top of the main event loop. Normally, it is
134 * expected that this callback will keep the heap size limited. It is
135 * relatively inexpensive, because it is invoked with no JS running and
136 * thus few stack roots to scan. For this reason, the GC's "trigger" bytes
137 * is less than the GC's "max" bytes as used by the trigger below.
138 *
139 * 9) Do an incremental, zonal GC with reason MAYBEGC when we go to allocate
140 * a new GC thing and find that the GC heap size has grown beyond the
141 * configured maximum (JSGC_MAX_BYTES). We trigger this GC by returning
142 * nullptr and then calling maybeGC at the top level of the allocator.
143 * This is then guaranteed to fail the "size greater than trigger" check
144 * above, since trigger is always less than max. After performing the GC,
145 * the allocator unconditionally returns nullptr to force an OOM exception
146 * is raised by the script.
147 *
148 * Note that this differs from a LAST_DITCH GC where we actually run out
149 * of memory (i.e., a call to a system allocator fails) when trying to
150 * allocate. Unlike above, LAST_DITCH GC only happens when we are really
151 * out of memory, not just when we cross an arbitrary trigger; despite
152 * this, it may still return an allocation at the end and allow the script
153 * to continue, if the LAST_DITCH GC was able to free up enough memory.
154 *
155 * 10) Do a GC under reason ALLOC_TRIGGER when we are over the GC heap trigger
156 * limit, but in the allocator rather than in a random call to maybeGC.
157 * This occurs if we allocate too much before returning to the event loop
158 * and calling maybeGC; this is extremely common in benchmarks and
159 * long-running Worker computations. Note that this uses a wildly
160 * different mechanism from the above in that it sets the interrupt flag
161 * and does the GC at the next loop head, before the next alloc, or
162 * maybeGC. The reason for this is that this check is made after the
163 * allocation and we cannot GC with an uninitialized thing in the heap.
164 *
165 * 11) Do an incremental, zonal GC with reason TOO_MUCH_MALLOC when the total
166 * amount of malloced memory is greater than the malloc trigger limit for the
167 * zone.
168 *
169 *
170 * Size Limitation Triggers Explanation
171 * ------------------------------------
172 *
173 * The GC internally is entirely unaware of the context of the execution of
174 * the mutator. It sees only:
175 *
176 * A) Allocated size: this is the amount of memory currently requested by the
177 * mutator. This quantity is monotonically increasing: i.e. the allocation
178 * rate is always >= 0. It is also easy for the system to track.
179 *
180 * B) Retained size: this is the amount of memory that the mutator can
181 * currently reach. Said another way, it is the size of the heap
182 * immediately after a GC (modulo background sweeping). This size is very
183 * costly to know exactly and also extremely hard to estimate with any
184 * fidelity.
185 *
186 * For reference, a common allocated vs. retained graph might look like:
187 *
188 * | ** **
189 * | ** ** * **
190 * | ** * ** * **
191 * | * ** * ** * **
192 * | ** ** * ** * **
193 * s| * * ** ** + + **
194 * i| * * * + + + + +
195 * z| * * * + + + + +
196 * e| * **+
197 * | * +
198 * | * +
199 * | * +
200 * | * +
201 * | * +
202 * |*+
203 * +--------------------------------------------------
204 * time
205 * *** = allocated
206 * +++ = retained
207 *
208 * Note that this is a bit of a simplification
209 * because in reality we track malloc and GC heap
210 * sizes separately and have a different level of
211 * granularity and accuracy on each heap.
212 *
213 * This presents some obvious implications for Mark-and-Sweep collectors.
214 * Namely:
215 * -> t[marking] ~= size[retained]
216 * -> t[sweeping] ~= size[allocated] - size[retained]
217 *
218 * In a non-incremental collector, maintaining low latency and high
219 * responsiveness requires that total GC times be as low as possible. Thus,
220 * in order to stay responsive when we did not have a fully incremental
221 * collector, our GC triggers were focused on minimizing collection time.
222 * Furthermore, since size[retained] is not under control of the GC, all the
223 * GC could do to control collection times was reduce sweep times by
224 * minimizing size[allocated], per the equation above.
225 *
226 * The result of the above is GC triggers that focus on size[allocated] to
227 * the exclusion of other important factors and default heuristics that are
228 * not optimal for a fully incremental collector. On the other hand, this is
229 * not all bad: minimizing size[allocated] also minimizes the chance of OOM
230 * and sweeping remains one of the hardest areas to further incrementalize.
231 *
232 * EAGER_ALLOC_TRIGGER
233 * -------------------
234 * Occurs when we return to the event loop and find our heap is getting
235 * largish, but before t[marking] OR t[sweeping] is too large for a
236 * responsive non-incremental GC. This is intended to be the common case
237 * in normal web applications: e.g. we just finished an event handler and
238 * the few objects we allocated when computing the new whatzitz have
239 * pushed us slightly over the limit. After this GC we rescale the new
240 * EAGER_ALLOC_TRIGGER trigger to 150% of size[retained] so that our
241 * non-incremental GC times will always be proportional to this size
242 * rather than being dominated by sweeping.
243 *
244 * As a concession to mutators that allocate heavily during their startup
245 * phase, we have a highFrequencyGCMode that ups the growth rate to 300%
246 * of the current size[retained] so that we'll do fewer longer GCs at the
247 * end of the mutator startup rather than more, smaller GCs.
248 *
249 * Assumptions:
250 * -> Responsiveness is proportional to t[marking] + t[sweeping].
251 * -> size[retained] is proportional only to GC allocations.
252 *
253 * ALLOC_TRIGGER (non-incremental)
254 * -------------------------------
255 * If we do not return to the event loop before getting all the way to our
256 * gc trigger bytes then MAYBEGC will never fire. To avoid OOMing, we
257 * succeed the current allocation and set the script interrupt so that we
258 * will (hopefully) do a GC before we overflow our max and have to raise
259 * an OOM exception for the script.
260 *
261 * Assumptions:
262 * -> Common web scripts will return to the event loop before using
263 * 10% of the current triggerBytes worth of GC memory.
264 *
265 * ALLOC_TRIGGER (incremental)
266 * ---------------------------
267 * In practice the above trigger is rough: if a website is just on the
268 * cusp, sometimes it will trigger a non-incremental GC moments before
269 * returning to the event loop, where it could have done an incremental
270 * GC. Thus, we recently added an incremental version of the above with a
271 * substantially lower threshold, so that we have a soft limit here. If
272 * IGC can collect faster than the allocator generates garbage, even if
273 * the allocator does not return to the event loop frequently, we should
274 * not have to fall back to a non-incremental GC.
275 *
276 * INCREMENTAL_TOO_SLOW
277 * --------------------
278 * Do a full, non-incremental GC if we overflow ALLOC_TRIGGER during an
279 * incremental GC. When in the middle of an incremental GC, we suppress
280 * our other triggers, so we need a way to backstop the IGC if the
281 * mutator allocates faster than the IGC can clean things up.
282 *
283 * TOO_MUCH_MALLOC
284 * ---------------
285 * Performs a GC before size[allocated] - size[retained] gets too large
286 * for non-incremental sweeping to be fast in the case that we have
287 * significantly more malloc allocation than GC allocation. This is meant
288 * to complement MAYBEGC triggers. We track this by counting malloced
289 * bytes; the counter gets reset at every GC since we do not always have a
290 * size at the time we call free. Because of this, the malloc heuristic
291 * is, unfortunately, not usefully able to augment our other GC heap
292 * triggers and is limited to this singular heuristic.
293 *
294 * Assumptions:
295 * -> EITHER size[allocated_by_malloc] ~= size[allocated_by_GC]
296 * OR time[sweeping] ~= size[allocated_by_malloc]
297 * -> size[retained] @ t0 ~= size[retained] @ t1
298 * i.e. That the mutator is in steady-state operation.
299 *
300 * LAST_DITCH_GC
301 * -------------
302 * Does a GC because we are out of memory.
303 *
304 * Assumptions:
305 * -> size[retained] < size[available_memory]
306 */
307
308 #ifndef gc_Scheduling_h
309 #define gc_Scheduling_h
310
311 #include "mozilla/Atomics.h"
312 #include "mozilla/DebugOnly.h"
313
314 #include "gc/GCEnum.h"
315 #include "js/AllocPolicy.h"
316 #include "js/GCAPI.h"
317 #include "js/HashTable.h"
318 #include "js/HeapAPI.h"
319 #include "js/SliceBudget.h"
320 #include "threading/ProtectedData.h"
321
322 namespace js {
323
324 class AutoLockGC;
325 class ZoneAllocator;
326 class ZoneAllocPolicy;
327
328 namespace gc {
329
330 struct Cell;
331
332 /*
333 * Default settings for tuning the GC. Some of these can be set at runtime,
334 * This list is not complete, some tuning parameters are not listed here.
335 *
336 * If you change the values here, please also consider changing them in
337 * modules/libpref/init/all.js where they are duplicated for the Firefox
338 * preferences.
339 */
340 namespace TuningDefaults {
341
342 /* JSGC_ALLOCATION_THRESHOLD */
343 static const size_t GCZoneAllocThresholdBase = 27 * 1024 * 1024;
344
345 /*
346 * JSGC_MIN_NURSERY_BYTES
347 *
348 * With some testing (Bug 1532838) we increased this to 256K from 192K
349 * which improves performance. We should try to reduce this for background
350 * tabs.
351 */
352 static const size_t GCMinNurseryBytes = 256 * 1024;
353
354 /*
355 * JSGC_SMALL_HEAP_INCREMENTAL_LIMIT
356 *
357 * This must be greater than 1.3 to maintain performance on splay-latency.
358 */
359 static const double SmallHeapIncrementalLimit = 1.40;
360
361 /* JSGC_LARGE_HEAP_INCREMENTAL_LIMIT */
362 static const double LargeHeapIncrementalLimit = 1.10;
363
364 /* JSGC_ZONE_ALLOC_DELAY_KB */
365 static const size_t ZoneAllocDelayBytes = 1024 * 1024;
366
367 /* JSGC_HIGH_FREQUENCY_TIME_LIMIT */
368 static const auto HighFrequencyThreshold = 1; // in seconds
369
370 /* JSGC_SMALL_HEAP_SIZE_MAX */
371 static const size_t SmallHeapSizeMaxBytes = 100 * 1024 * 1024;
372
373 /* JSGC_LARGE_HEAP_SIZE_MIN */
374 static const size_t LargeHeapSizeMinBytes = 500 * 1024 * 1024;
375
376 /* JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH */
377 static const double HighFrequencySmallHeapGrowth = 3.0;
378
379 /* JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH */
380 static const double HighFrequencyLargeHeapGrowth = 1.5;
381
382 /* JSGC_LOW_FREQUENCY_HEAP_GROWTH */
383 static const double LowFrequencyHeapGrowth = 1.5;
384
385 /* JSGC_MIN_EMPTY_CHUNK_COUNT */
386 static const uint32_t MinEmptyChunkCount = 1;
387
388 /* JSGC_MAX_EMPTY_CHUNK_COUNT */
389 static const uint32_t MaxEmptyChunkCount = 30;
390
391 /* JSGC_SLICE_TIME_BUDGET_MS */
392 static const int64_t DefaultTimeBudgetMS = SliceBudget::UnlimitedTimeBudget;
393
394 /* JSGC_MODE */
395 static const JSGCMode Mode = JSGC_MODE_ZONE_INCREMENTAL;
396
397 /* JSGC_COMPACTING_ENABLED */
398 static const bool CompactingEnabled = true;
399
400 /* JSGC_INCREMENTAL_WEAKMAP_ENABLED */
401 static const bool IncrementalWeakMapMarkingEnabled = true;
402
403 /* JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION */
404 static const uint32_t NurseryFreeThresholdForIdleCollection = ChunkSize / 4;
405
406 /* JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION_PERCENT */
407 static const double NurseryFreeThresholdForIdleCollectionFraction = 0.25;
408
409 /* JSGC_PRETENURE_THRESHOLD */
410 static const double PretenureThreshold = 0.6;
411
412 /* JSGC_PRETENURE_GROUP_THRESHOLD */
413 static const double PretenureGroupThreshold = 3000;
414
415 /* JSGC_MIN_LAST_DITCH_GC_PERIOD */
416 static const auto MinLastDitchGCPeriod = 60; // in seconds
417
418 /* JSGC_MALLOC_THRESHOLD_BASE */
419 static const size_t MallocThresholdBase = 38 * 1024 * 1024;
420
421 /* JSGC_MALLOC_GROWTH_FACTOR */
422 static const double MallocGrowthFactor = 1.5;
423
424 } // namespace TuningDefaults
425
426 /*
427 * Encapsulates all of the GC tunables. These are effectively constant and
428 * should only be modified by setParameter.
429 */
430 class GCSchedulingTunables {
431 /*
432 * JSGC_MAX_BYTES
433 *
434 * Maximum nominal heap before last ditch GC.
435 */
436 UnprotectedData<size_t> gcMaxBytes_;
437
438 /*
439 * JSGC_MIN_NURSERY_BYTES
440 * JSGC_MAX_NURSERY_BYTES
441 *
442 * Minimum and maximum nursery size for each runtime.
443 */
444 MainThreadData<size_t> gcMinNurseryBytes_;
445 MainThreadData<size_t> gcMaxNurseryBytes_;
446
447 /*
448 * JSGC_ALLOCATION_THRESHOLD
449 *
450 * The base value used to compute zone->threshold.bytes(). When
451 * gcHeapSize.bytes() exceeds threshold.bytes() for a zone, the zone may be
452 * scheduled for a GC, depending on the exact circumstances.
453 */
454 MainThreadOrGCTaskData<size_t> gcZoneAllocThresholdBase_;
455
456 /*
457 * JSGC_SMALL_HEAP_INCREMENTAL_LIMIT
458 *
459 * Multiple of threshold.bytes() which triggers a non-incremental GC.
460 */
461 UnprotectedData<double> smallHeapIncrementalLimit_;
462
463 /*
464 * JSGC_LARGE_HEAP_INCREMENTAL_LIMIT
465 *
466 * Multiple of threshold.bytes() which triggers a non-incremental GC.
467 */
468 UnprotectedData<double> largeHeapIncrementalLimit_;
469
470 /*
471 * Number of bytes to allocate between incremental slices in GCs triggered by
472 * the zone allocation threshold.
473 *
474 * This value does not have a JSGCParamKey parameter yet.
475 */
476 UnprotectedData<size_t> zoneAllocDelayBytes_;
477
478 /*
479 * JSGC_HIGH_FREQUENCY_TIME_LIMIT
480 *
481 * We enter high-frequency mode if we GC a twice within this many
482 * microseconds.
483 */
484 MainThreadOrGCTaskData<mozilla::TimeDuration> highFrequencyThreshold_;
485
486 /*
487 * JSGC_SMALL_HEAP_SIZE_MAX
488 * JSGC_LARGE_HEAP_SIZE_MIN
489 * JSGC_HIGH_FREQUENCY_SMALL_HEAP_GROWTH
490 * JSGC_HIGH_FREQUENCY_LARGE_HEAP_GROWTH
491 *
492 * When in the |highFrequencyGC| mode, these parameterize the per-zone
493 * "HeapGrowthFactor" computation.
494 */
495 MainThreadOrGCTaskData<size_t> smallHeapSizeMaxBytes_;
496 MainThreadOrGCTaskData<size_t> largeHeapSizeMinBytes_;
497 MainThreadOrGCTaskData<double> highFrequencySmallHeapGrowth_;
498 MainThreadOrGCTaskData<double> highFrequencyLargeHeapGrowth_;
499
500 /*
501 * JSGC_LOW_FREQUENCY_HEAP_GROWTH
502 *
503 * When not in |highFrequencyGC| mode, this is the global (stored per-zone)
504 * "HeapGrowthFactor".
505 */
506 MainThreadOrGCTaskData<double> lowFrequencyHeapGrowth_;
507
508 /*
509 * JSGC_MIN_EMPTY_CHUNK_COUNT
510 * JSGC_MAX_EMPTY_CHUNK_COUNT
511 *
512 * Controls the number of empty chunks reserved for future allocation.
513 */
514 UnprotectedData<uint32_t> minEmptyChunkCount_;
515 UnprotectedData<uint32_t> maxEmptyChunkCount_;
516
517 /*
518 * JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION
519 * JSGC_NURSERY_FREE_THRESHOLD_FOR_IDLE_COLLECTION_FRACTION
520 *
521 * Attempt to run a minor GC in the idle time if the free space falls
522 * below this threshold. The absolute threshold is used when the nursery is
523 * large and the percentage when it is small. See Nursery::shouldCollect()
524 */
525 UnprotectedData<uint32_t> nurseryFreeThresholdForIdleCollection_;
526 UnprotectedData<double> nurseryFreeThresholdForIdleCollectionFraction_;
527
528 /*
529 * JSGC_PRETENURE_THRESHOLD
530 *
531 * Fraction of objects tenured to trigger pretenuring (between 0 and 1). If
532 * this fraction is met, the GC proceeds to calculate which objects will be
533 * tenured. If this is 1.0f (actually if it is not < 1.0f) then pretenuring
534 * is disabled.
535 */
536 UnprotectedData<double> pretenureThreshold_;
537
538 /*
539 * JSGC_PRETENURE_GROUP_THRESHOLD
540 *
541 * During a single nursery collection, if this many objects from the same
542 * object group are tenured, then that group will be pretenured.
543 */
544 UnprotectedData<uint32_t> pretenureGroupThreshold_;
545
546 /*
547 * JSGC_MIN_LAST_DITCH_GC_PERIOD
548 *
549 * Last ditch GC is skipped if allocation failure occurs less than this many
550 * seconds from the previous one.
551 */
552 MainThreadData<mozilla::TimeDuration> minLastDitchGCPeriod_;
553
554 /*
555 * JSGC_MALLOC_THRESHOLD_BASE
556 *
557 * The base value used to compute the GC trigger for malloc allocated memory.
558 */
559 MainThreadOrGCTaskData<size_t> mallocThresholdBase_;
560
561 /*
562 * JSGC_MALLOC_GROWTH_FACTOR
563 *
564 * Malloc memory growth factor.
565 */
566 MainThreadOrGCTaskData<double> mallocGrowthFactor_;
567
568 public:
569 GCSchedulingTunables();
570
gcMaxBytes()571 size_t gcMaxBytes() const { return gcMaxBytes_; }
gcMinNurseryBytes()572 size_t gcMinNurseryBytes() const { return gcMinNurseryBytes_; }
gcMaxNurseryBytes()573 size_t gcMaxNurseryBytes() const { return gcMaxNurseryBytes_; }
gcZoneAllocThresholdBase()574 size_t gcZoneAllocThresholdBase() const { return gcZoneAllocThresholdBase_; }
smallHeapIncrementalLimit()575 double smallHeapIncrementalLimit() const {
576 return smallHeapIncrementalLimit_;
577 }
largeHeapIncrementalLimit()578 double largeHeapIncrementalLimit() const {
579 return largeHeapIncrementalLimit_;
580 }
zoneAllocDelayBytes()581 size_t zoneAllocDelayBytes() const { return zoneAllocDelayBytes_; }
highFrequencyThreshold()582 const mozilla::TimeDuration& highFrequencyThreshold() const {
583 return highFrequencyThreshold_;
584 }
smallHeapSizeMaxBytes()585 size_t smallHeapSizeMaxBytes() const { return smallHeapSizeMaxBytes_; }
largeHeapSizeMinBytes()586 size_t largeHeapSizeMinBytes() const { return largeHeapSizeMinBytes_; }
highFrequencySmallHeapGrowth()587 double highFrequencySmallHeapGrowth() const {
588 return highFrequencySmallHeapGrowth_;
589 }
highFrequencyLargeHeapGrowth()590 double highFrequencyLargeHeapGrowth() const {
591 return highFrequencyLargeHeapGrowth_;
592 }
lowFrequencyHeapGrowth()593 double lowFrequencyHeapGrowth() const { return lowFrequencyHeapGrowth_; }
minEmptyChunkCount(const AutoLockGC &)594 unsigned minEmptyChunkCount(const AutoLockGC&) const {
595 return minEmptyChunkCount_;
596 }
maxEmptyChunkCount()597 unsigned maxEmptyChunkCount() const { return maxEmptyChunkCount_; }
nurseryFreeThresholdForIdleCollection()598 uint32_t nurseryFreeThresholdForIdleCollection() const {
599 return nurseryFreeThresholdForIdleCollection_;
600 }
nurseryFreeThresholdForIdleCollectionFraction()601 double nurseryFreeThresholdForIdleCollectionFraction() const {
602 return nurseryFreeThresholdForIdleCollectionFraction_;
603 }
604
attemptPretenuring()605 bool attemptPretenuring() const { return pretenureThreshold_ < 1.0; }
pretenureThreshold()606 double pretenureThreshold() const { return pretenureThreshold_; }
pretenureGroupThreshold()607 uint32_t pretenureGroupThreshold() const { return pretenureGroupThreshold_; }
608
minLastDitchGCPeriod()609 mozilla::TimeDuration minLastDitchGCPeriod() const {
610 return minLastDitchGCPeriod_;
611 }
612
mallocThresholdBase()613 size_t mallocThresholdBase() const { return mallocThresholdBase_; }
mallocGrowthFactor()614 double mallocGrowthFactor() const { return mallocGrowthFactor_; }
615
616 MOZ_MUST_USE bool setParameter(JSGCParamKey key, uint32_t value,
617 const AutoLockGC& lock);
618 void resetParameter(JSGCParamKey key, const AutoLockGC& lock);
619
620 private:
621 void setSmallHeapSizeMaxBytes(size_t value);
622 void setLargeHeapSizeMinBytes(size_t value);
623 void setHighFrequencySmallHeapGrowth(double value);
624 void setHighFrequencyLargeHeapGrowth(double value);
625 void setLowFrequencyHeapGrowth(double value);
626 void setMinEmptyChunkCount(uint32_t value);
627 void setMaxEmptyChunkCount(uint32_t value);
628 };
629
630 class GCSchedulingState {
631 /*
632 * Influences how we schedule and run GC's in several subtle ways. The most
633 * important factor is in how it controls the "HeapGrowthFactor". The
634 * growth factor is a measure of how large (as a percentage of the last GC)
635 * the heap is allowed to grow before we try to schedule another GC.
636 */
637 MainThreadOrGCTaskData<bool> inHighFrequencyGCMode_;
638
639 public:
640 /*
641 * Influences the GC thresholds for the atoms zone to discourage collection of
642 * this zone during page load.
643 */
644 MainThreadOrGCTaskData<bool> inPageLoad;
645
GCSchedulingState()646 GCSchedulingState() : inHighFrequencyGCMode_(false) {}
647
inHighFrequencyGCMode()648 bool inHighFrequencyGCMode() const { return inHighFrequencyGCMode_; }
649
updateHighFrequencyMode(const mozilla::TimeStamp & lastGCTime,const mozilla::TimeStamp & currentTime,const GCSchedulingTunables & tunables)650 void updateHighFrequencyMode(const mozilla::TimeStamp& lastGCTime,
651 const mozilla::TimeStamp& currentTime,
652 const GCSchedulingTunables& tunables) {
653 #ifndef JS_MORE_DETERMINISTIC
654 inHighFrequencyGCMode_ =
655 !lastGCTime.IsNull() &&
656 lastGCTime + tunables.highFrequencyThreshold() > currentTime;
657 #endif
658 }
659 };
660
661 enum class TriggerKind { None, Incremental, NonIncremental };
662
663 struct TriggerResult {
664 TriggerKind kind;
665 size_t usedBytes;
666 size_t thresholdBytes;
667 };
668
669 using AtomicByteCount = mozilla::Atomic<size_t, mozilla::ReleaseAcquire>;
670
671 /*
672 * Tracks the size of allocated data. This is used for both GC and malloc data.
673 * It automatically maintains the memory usage relationship between parent and
674 * child instances, i.e. between those in a GCRuntime and its Zones.
675 */
676 class HeapSize {
677 /*
678 * An instance that contains our parent's heap usage, or null if this is the
679 * top-level usage container.
680 */
681 HeapSize* const parent_;
682
683 /*
684 * The number of bytes in use. For GC heaps this is approximate to the nearest
685 * ArenaSize. It is atomic because it is updated by both the active and GC
686 * helper threads.
687 */
688 AtomicByteCount bytes_;
689
690 /*
691 * The number of bytes retained after the last collection. This is updated
692 * dynamically during incremental GC. It does not include allocations that
693 * happen during a GC.
694 */
695 AtomicByteCount retainedBytes_;
696
697 public:
HeapSize(HeapSize * parent)698 explicit HeapSize(HeapSize* parent) : parent_(parent), bytes_(0) {}
699
bytes()700 size_t bytes() const { return bytes_; }
retainedBytes()701 size_t retainedBytes() const { return retainedBytes_; }
702
updateOnGCStart()703 void updateOnGCStart() { retainedBytes_ = size_t(bytes_); }
704
addGCArena()705 void addGCArena() { addBytes(ArenaSize); }
removeGCArena()706 void removeGCArena() {
707 MOZ_ASSERT(retainedBytes_ >= ArenaSize);
708 removeBytes(ArenaSize, true /* only sweeping removes arenas */);
709 }
710
addBytes(size_t nbytes)711 void addBytes(size_t nbytes) {
712 mozilla::DebugOnly<size_t> initialBytes(bytes_);
713 MOZ_ASSERT(initialBytes + nbytes > initialBytes);
714 bytes_ += nbytes;
715 if (parent_) {
716 parent_->addBytes(nbytes);
717 }
718 }
removeBytes(size_t nbytes,bool wasSwept)719 void removeBytes(size_t nbytes, bool wasSwept) {
720 if (wasSwept) {
721 // TODO: We would like to assert that retainedBytes_ >= nbytes is here but
722 // we can't do that yet, so clamp the result to zero.
723 retainedBytes_ = nbytes <= retainedBytes_ ? retainedBytes_ - nbytes : 0;
724 }
725 MOZ_ASSERT(bytes_ >= nbytes);
726 bytes_ -= nbytes;
727 if (parent_) {
728 parent_->removeBytes(nbytes, wasSwept);
729 }
730 }
731
732 /* Pair to adoptArenas. Adopts the attendant usage statistics. */
adopt(HeapSize & source)733 void adopt(HeapSize& source) {
734 // Skip retainedBytes_: we never adopt zones that are currently being
735 // collected.
736 bytes_ += source.bytes_;
737 source.retainedBytes_ = 0;
738 source.bytes_ = 0;
739 }
740 };
741
742 // Heap size thresholds used to trigger GC. This is an abstract base class for
743 // GC heap and malloc thresholds defined below.
744 class HeapThreshold {
745 protected:
HeapThreshold()746 HeapThreshold()
747 : startBytes_(SIZE_MAX),
748 incrementalLimitBytes_(SIZE_MAX),
749 sliceBytes_(SIZE_MAX) {}
750
751 // The threshold at which to start a new incremental collection.
752 //
753 // TODO: This is currently read off-thread during parsing, but at some point
754 // we should be able to make this MainThreadData<>.
755 AtomicByteCount startBytes_;
756
757 // The threshold at which start a new non-incremental collection or finish an
758 // ongoing collection non-incrementally.
759 size_t incrementalLimitBytes_;
760
761 // The threshold at which to trigger a slice during an ongoing incremental
762 // collection.
763 size_t sliceBytes_;
764
765 public:
startBytes()766 size_t startBytes() const { return startBytes_; }
sliceBytes()767 size_t sliceBytes() const { return sliceBytes_; }
incrementalLimitBytes()768 size_t incrementalLimitBytes() const { return incrementalLimitBytes_; }
769 double eagerAllocTrigger(bool highFrequencyGC) const;
770
771 void setSliceThreshold(ZoneAllocator* zone, const HeapSize& heapSize,
772 const GCSchedulingTunables& tunables);
clearSliceThreshold()773 void clearSliceThreshold() { sliceBytes_ = SIZE_MAX; }
hasSliceThreshold()774 bool hasSliceThreshold() const { return sliceBytes_ != SIZE_MAX; }
775
776 protected:
777 void setIncrementalLimitFromStartBytes(size_t retainedBytes,
778 const GCSchedulingTunables& tunables);
779 };
780
781 // A heap threshold that is based on a multiple of the retained size after the
782 // last collection adjusted based on collection frequency and retained
783 // size. This is used to determine when to do a zone GC based on GC heap size.
784 class GCHeapThreshold : public HeapThreshold {
785 public:
786 void updateStartThreshold(size_t lastBytes, JSGCInvocationKind gckind,
787 const GCSchedulingTunables& tunables,
788 const GCSchedulingState& state, bool isAtomsZone,
789 const AutoLockGC& lock);
790
791 private:
792 static double computeZoneHeapGrowthFactorForHeapSize(
793 size_t lastBytes, const GCSchedulingTunables& tunables,
794 const GCSchedulingState& state);
795 static size_t computeZoneTriggerBytes(double growthFactor, size_t lastBytes,
796 JSGCInvocationKind gckind,
797 const GCSchedulingTunables& tunables,
798 const AutoLockGC& lock);
799 };
800
801 // A heap threshold that is calculated as a constant multiple of the retained
802 // size after the last collection. This is used to determines when to do a zone
803 // GC based on malloc data.
804 class MallocHeapThreshold : public HeapThreshold {
805 public:
806 void updateStartThreshold(size_t lastBytes,
807 const GCSchedulingTunables& tunables,
808 const AutoLockGC& lock);
809
810 private:
811 static size_t computeZoneTriggerBytes(double growthFactor, size_t lastBytes,
812 size_t baseBytes,
813 const AutoLockGC& lock);
814 };
815
816 // A fixed threshold that's used to determine when we need to do a zone GC based
817 // on allocated JIT code.
818 class JitHeapThreshold : public HeapThreshold {
819 public:
JitHeapThreshold(size_t bytes)820 explicit JitHeapThreshold(size_t bytes) { startBytes_ = bytes; }
821 };
822
823 struct SharedMemoryUse {
SharedMemoryUseSharedMemoryUse824 explicit SharedMemoryUse(MemoryUse use) : count(0), nbytes(0) {
825 #ifdef DEBUG
826 this->use = use;
827 #endif
828 }
829
830 size_t count;
831 size_t nbytes;
832 #ifdef DEBUG
833 MemoryUse use;
834 #endif
835 };
836
837 // A map which tracks shared memory uses (shared in the sense that an allocation
838 // can be referenced by more than one GC thing in a zone). This allows us to
839 // only account for the memory once.
840 using SharedMemoryMap =
841 HashMap<void*, SharedMemoryUse, DefaultHasher<void*>, SystemAllocPolicy>;
842
843 #ifdef DEBUG
844
845 // Counts memory associated with GC things in a zone.
846 //
847 // This records details of the cell (or non-cell pointer) the memory allocation
848 // is associated with to check the correctness of the information provided. This
849 // is not present in opt builds.
850 class MemoryTracker {
851 public:
852 MemoryTracker();
853 void fixupAfterMovingGC();
854 void checkEmptyOnDestroy();
855
856 void adopt(MemoryTracker& other);
857
858 // Track memory by associated GC thing pointer.
859 void trackGCMemory(Cell* cell, size_t nbytes, MemoryUse use);
860 void untrackGCMemory(Cell* cell, size_t nbytes, MemoryUse use);
861 void swapGCMemory(Cell* a, Cell* b, MemoryUse use);
862
863 // Track memory by associated non-GC thing pointer.
864 void registerNonGCMemory(void* ptr, MemoryUse use);
865 void unregisterNonGCMemory(void* ptr, MemoryUse use);
866 void moveNonGCMemory(void* dst, void* src, MemoryUse use);
867 void incNonGCMemory(void* ptr, size_t nbytes, MemoryUse use);
868 void decNonGCMemory(void* ptr, size_t nbytes, MemoryUse use);
869
870 private:
871 template <typename Ptr>
872 struct Key {
873 Key(Ptr* ptr, MemoryUse use);
874 Ptr* ptr() const;
875 MemoryUse use() const;
876
877 private:
878 # ifdef JS_64BIT
879 // Pack this into a single word on 64 bit platforms.
880 uintptr_t ptr_ : 56;
881 uintptr_t use_ : 8;
882 # else
883 uintptr_t ptr_ : 32;
884 uintptr_t use_ : 8;
885 # endif
886 };
887
888 template <typename Ptr>
889 struct Hasher {
890 using KeyT = Key<Ptr>;
891 using Lookup = KeyT;
892 static HashNumber hash(const Lookup& l);
893 static bool match(const KeyT& key, const Lookup& l);
894 static void rekey(KeyT& k, const KeyT& newKey);
895 };
896
897 template <typename Ptr>
898 using Map = HashMap<Key<Ptr>, size_t, Hasher<Ptr>, SystemAllocPolicy>;
899 using GCMap = Map<Cell>;
900 using NonGCMap = Map<void>;
901
902 static bool isGCMemoryUse(MemoryUse use);
903 static bool isNonGCMemoryUse(MemoryUse use);
904 static bool allowMultipleAssociations(MemoryUse use);
905
906 size_t getAndRemoveEntry(const Key<Cell>& key, LockGuard<Mutex>& lock);
907
908 Mutex mutex;
909
910 // Map containing the allocated size associated with (cell, use) pairs.
911 GCMap gcMap;
912
913 // Map containing the allocated size associated (non-cell pointer, use) pairs.
914 NonGCMap nonGCMap;
915 };
916
917 #endif // DEBUG
918
LinearInterpolate(double x,double x0,double y0,double x1,double y1)919 static inline double LinearInterpolate(double x, double x0, double y0,
920 double x1, double y1) {
921 MOZ_ASSERT(x0 < x1);
922
923 if (x < x0) {
924 return y0;
925 }
926
927 if (x < x1) {
928 return y0 + (y1 - y0) * ((x - x0) / (x1 - x0));
929 }
930
931 return y1;
932 }
933
934 } // namespace gc
935 } // namespace js
936
937 #endif // gc_Scheduling_h
938