1*404b540aSrobert /* "Bag-of-pages" zone garbage collector for the GNU compiler.
2*404b540aSrobert Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005
3*404b540aSrobert Free Software Foundation, Inc.
4*404b540aSrobert
5*404b540aSrobert Contributed by Richard Henderson (rth@redhat.com) and Daniel Berlin
6*404b540aSrobert (dberlin@dberlin.org). Rewritten by Daniel Jacobowitz
7*404b540aSrobert <dan@codesourcery.com>.
8*404b540aSrobert
9*404b540aSrobert This file is part of GCC.
10*404b540aSrobert
11*404b540aSrobert GCC is free software; you can redistribute it and/or modify it under
12*404b540aSrobert the terms of the GNU General Public License as published by the Free
13*404b540aSrobert Software Foundation; either version 2, or (at your option) any later
14*404b540aSrobert version.
15*404b540aSrobert
16*404b540aSrobert GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17*404b540aSrobert WARRANTY; without even the implied warranty of MERCHANTABILITY or
18*404b540aSrobert FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19*404b540aSrobert for more details.
20*404b540aSrobert
21*404b540aSrobert You should have received a copy of the GNU General Public License
22*404b540aSrobert along with GCC; see the file COPYING. If not, write to the Free
23*404b540aSrobert Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
24*404b540aSrobert 02110-1301, USA. */
25*404b540aSrobert
26*404b540aSrobert #include "config.h"
27*404b540aSrobert #include "system.h"
28*404b540aSrobert #include "coretypes.h"
29*404b540aSrobert #include "tm.h"
30*404b540aSrobert #include "tree.h"
31*404b540aSrobert #include "rtl.h"
32*404b540aSrobert #include "tm_p.h"
33*404b540aSrobert #include "toplev.h"
34*404b540aSrobert #include "varray.h"
35*404b540aSrobert #include "flags.h"
36*404b540aSrobert #include "ggc.h"
37*404b540aSrobert #include "timevar.h"
38*404b540aSrobert #include "params.h"
39*404b540aSrobert #include "bitmap.h"
40*404b540aSrobert
41*404b540aSrobert #ifdef ENABLE_VALGRIND_CHECKING
42*404b540aSrobert # ifdef HAVE_VALGRIND_MEMCHECK_H
43*404b540aSrobert # include <valgrind/memcheck.h>
44*404b540aSrobert # elif defined HAVE_MEMCHECK_H
45*404b540aSrobert # include <memcheck.h>
46*404b540aSrobert # else
47*404b540aSrobert # include <valgrind.h>
48*404b540aSrobert # endif
49*404b540aSrobert #else
50*404b540aSrobert /* Avoid #ifdef:s when we can help it. */
51*404b540aSrobert #define VALGRIND_DISCARD(x)
52*404b540aSrobert #define VALGRIND_MALLOCLIKE_BLOCK(w,x,y,z)
53*404b540aSrobert #define VALGRIND_FREELIKE_BLOCK(x,y)
54*404b540aSrobert #endif
55*404b540aSrobert
56*404b540aSrobert /* Prefer MAP_ANON(YMOUS) to /dev/zero, since we don't need to keep a
57*404b540aSrobert file open. Prefer either to valloc. */
58*404b540aSrobert #ifdef HAVE_MMAP_ANON
59*404b540aSrobert # undef HAVE_MMAP_DEV_ZERO
60*404b540aSrobert
61*404b540aSrobert # include <sys/mman.h>
62*404b540aSrobert # ifndef MAP_FAILED
63*404b540aSrobert # define MAP_FAILED -1
64*404b540aSrobert # endif
65*404b540aSrobert # if !defined (MAP_ANONYMOUS) && defined (MAP_ANON)
66*404b540aSrobert # define MAP_ANONYMOUS MAP_ANON
67*404b540aSrobert # endif
68*404b540aSrobert # define USING_MMAP
69*404b540aSrobert #endif
70*404b540aSrobert
71*404b540aSrobert #ifdef HAVE_MMAP_DEV_ZERO
72*404b540aSrobert # include <sys/mman.h>
73*404b540aSrobert # ifndef MAP_FAILED
74*404b540aSrobert # define MAP_FAILED -1
75*404b540aSrobert # endif
76*404b540aSrobert # define USING_MMAP
77*404b540aSrobert #endif
78*404b540aSrobert
79*404b540aSrobert #ifndef USING_MMAP
80*404b540aSrobert #error Zone collector requires mmap
81*404b540aSrobert #endif
82*404b540aSrobert
83*404b540aSrobert #if (GCC_VERSION < 3001)
84*404b540aSrobert #define prefetch(X) ((void) X)
85*404b540aSrobert #define prefetchw(X) ((void) X)
86*404b540aSrobert #else
87*404b540aSrobert #define prefetch(X) __builtin_prefetch (X)
88*404b540aSrobert #define prefetchw(X) __builtin_prefetch (X, 1, 3)
89*404b540aSrobert #endif
90*404b540aSrobert
91*404b540aSrobert /* FUTURE NOTES:
92*404b540aSrobert
93*404b540aSrobert If we track inter-zone pointers, we can mark single zones at a
94*404b540aSrobert time.
95*404b540aSrobert
96*404b540aSrobert If we have a zone where we guarantee no inter-zone pointers, we
97*404b540aSrobert could mark that zone separately.
98*404b540aSrobert
99*404b540aSrobert The garbage zone should not be marked, and we should return 1 in
100*404b540aSrobert ggc_set_mark for any object in the garbage zone, which cuts off
101*404b540aSrobert marking quickly. */
102*404b540aSrobert
103*404b540aSrobert /* Strategy:
104*404b540aSrobert
105*404b540aSrobert This garbage-collecting allocator segregates objects into zones.
106*404b540aSrobert It also segregates objects into "large" and "small" bins. Large
107*404b540aSrobert objects are greater than page size.
108*404b540aSrobert
109*404b540aSrobert Pages for small objects are broken up into chunks. The page has
110*404b540aSrobert a bitmap which marks the start position of each chunk (whether
111*404b540aSrobert allocated or free). Free chunks are on one of the zone's free
112*404b540aSrobert lists and contain a pointer to the next free chunk. Chunks in
113*404b540aSrobert most of the free lists have a fixed size determined by the
114*404b540aSrobert free list. Chunks in the "other" sized free list have their size
115*404b540aSrobert stored right after their chain pointer.
116*404b540aSrobert
117*404b540aSrobert Empty pages (of all sizes) are kept on a single page cache list,
118*404b540aSrobert and are considered first when new pages are required; they are
119*404b540aSrobert deallocated at the start of the next collection if they haven't
120*404b540aSrobert been recycled by then. The free page list is currently per-zone. */
121*404b540aSrobert
122*404b540aSrobert /* Define GGC_DEBUG_LEVEL to print debugging information.
123*404b540aSrobert 0: No debugging output.
124*404b540aSrobert 1: GC statistics only.
125*404b540aSrobert 2: Page-entry allocations/deallocations as well.
126*404b540aSrobert 3: Object allocations as well.
127*404b540aSrobert 4: Object marks as well. */
128*404b540aSrobert #define GGC_DEBUG_LEVEL (0)
129*404b540aSrobert
130*404b540aSrobert #ifndef HOST_BITS_PER_PTR
131*404b540aSrobert #define HOST_BITS_PER_PTR HOST_BITS_PER_LONG
132*404b540aSrobert #endif
133*404b540aSrobert
134*404b540aSrobert /* This structure manages small free chunks. The SIZE field is only
135*404b540aSrobert initialized if the chunk is in the "other" sized free list. Large
136*404b540aSrobert chunks are allocated one at a time to their own page, and so don't
137*404b540aSrobert come in here. */
138*404b540aSrobert
139*404b540aSrobert struct alloc_chunk {
140*404b540aSrobert struct alloc_chunk *next_free;
141*404b540aSrobert unsigned int size;
142*404b540aSrobert };
143*404b540aSrobert
144*404b540aSrobert /* The size of the fixed-size portion of a small page descriptor. */
145*404b540aSrobert #define PAGE_OVERHEAD (offsetof (struct small_page_entry, alloc_bits))
146*404b540aSrobert
147*404b540aSrobert /* The collector's idea of the page size. This must be a power of two
148*404b540aSrobert no larger than the system page size, because pages must be aligned
149*404b540aSrobert to this amount and are tracked at this granularity in the page
150*404b540aSrobert table. We choose a size at compile time for efficiency.
151*404b540aSrobert
152*404b540aSrobert We could make a better guess at compile time if PAGE_SIZE is a
153*404b540aSrobert constant in system headers, and PAGE_SHIFT is defined... */
154*404b540aSrobert #define GGC_PAGE_SIZE 4096
155*404b540aSrobert #define GGC_PAGE_MASK (GGC_PAGE_SIZE - 1)
156*404b540aSrobert #define GGC_PAGE_SHIFT 12
157*404b540aSrobert
158*404b540aSrobert #if 0
159*404b540aSrobert /* Alternative definitions which use the runtime page size. */
160*404b540aSrobert #define GGC_PAGE_SIZE G.pagesize
161*404b540aSrobert #define GGC_PAGE_MASK G.page_mask
162*404b540aSrobert #define GGC_PAGE_SHIFT G.lg_pagesize
163*404b540aSrobert #endif
164*404b540aSrobert
165*404b540aSrobert /* The size of a small page managed by the garbage collector. This
166*404b540aSrobert must currently be GGC_PAGE_SIZE, but with a few changes could
167*404b540aSrobert be any multiple of it to reduce certain kinds of overhead. */
168*404b540aSrobert #define SMALL_PAGE_SIZE GGC_PAGE_SIZE
169*404b540aSrobert
170*404b540aSrobert /* Free bin information. These numbers may be in need of re-tuning.
171*404b540aSrobert In general, decreasing the number of free bins would seem to
172*404b540aSrobert increase the time it takes to allocate... */
173*404b540aSrobert
174*404b540aSrobert /* FIXME: We can't use anything but MAX_ALIGNMENT for the bin size
175*404b540aSrobert today. */
176*404b540aSrobert
177*404b540aSrobert #define NUM_FREE_BINS 64
178*404b540aSrobert #define FREE_BIN_DELTA MAX_ALIGNMENT
179*404b540aSrobert #define SIZE_BIN_DOWN(SIZE) ((SIZE) / FREE_BIN_DELTA)
180*404b540aSrobert
181*404b540aSrobert /* Allocation and marking parameters. */
182*404b540aSrobert
183*404b540aSrobert /* The smallest allocatable unit to keep track of. */
184*404b540aSrobert #define BYTES_PER_ALLOC_BIT MAX_ALIGNMENT
185*404b540aSrobert
186*404b540aSrobert /* The smallest markable unit. If we require each allocated object
187*404b540aSrobert to contain at least two allocatable units, we can use half as many
188*404b540aSrobert bits for the mark bitmap. But this adds considerable complexity
189*404b540aSrobert to sweeping. */
190*404b540aSrobert #define BYTES_PER_MARK_BIT BYTES_PER_ALLOC_BIT
191*404b540aSrobert
192*404b540aSrobert #define BYTES_PER_MARK_WORD (8 * BYTES_PER_MARK_BIT * sizeof (mark_type))
193*404b540aSrobert
194*404b540aSrobert /* We use this structure to determine the alignment required for
195*404b540aSrobert allocations.
196*404b540aSrobert
197*404b540aSrobert There are several things wrong with this estimation of alignment.
198*404b540aSrobert
199*404b540aSrobert The maximum alignment for a structure is often less than the
200*404b540aSrobert maximum alignment for a basic data type; for instance, on some
201*404b540aSrobert targets long long must be aligned to sizeof (int) in a structure
202*404b540aSrobert and sizeof (long long) in a variable. i386-linux is one example;
203*404b540aSrobert Darwin is another (sometimes, depending on the compiler in use).
204*404b540aSrobert
205*404b540aSrobert Also, long double is not included. Nothing in GCC uses long
206*404b540aSrobert double, so we assume that this is OK. On powerpc-darwin, adding
207*404b540aSrobert long double would bring the maximum alignment up to 16 bytes,
208*404b540aSrobert and until we need long double (or to vectorize compiler operations)
209*404b540aSrobert that's painfully wasteful. This will need to change, some day. */
210*404b540aSrobert
211*404b540aSrobert struct max_alignment {
212*404b540aSrobert char c;
213*404b540aSrobert union {
214*404b540aSrobert HOST_WIDEST_INT i;
215*404b540aSrobert double d;
216*404b540aSrobert } u;
217*404b540aSrobert };
218*404b540aSrobert
219*404b540aSrobert /* The biggest alignment required. */
220*404b540aSrobert
221*404b540aSrobert #define MAX_ALIGNMENT (offsetof (struct max_alignment, u))
222*404b540aSrobert
223*404b540aSrobert /* Compute the smallest multiple of F that is >= X. */
224*404b540aSrobert
225*404b540aSrobert #define ROUND_UP(x, f) (CEIL (x, f) * (f))
226*404b540aSrobert
227*404b540aSrobert /* Types to use for the allocation and mark bitmaps. It might be
228*404b540aSrobert a good idea to add ffsl to libiberty and use unsigned long
229*404b540aSrobert instead; that could speed us up where long is wider than int. */
230*404b540aSrobert
231*404b540aSrobert typedef unsigned int alloc_type;
232*404b540aSrobert typedef unsigned int mark_type;
233*404b540aSrobert #define alloc_ffs(x) ffs(x)
234*404b540aSrobert
235*404b540aSrobert /* A page_entry records the status of an allocation page. This is the
236*404b540aSrobert common data between all three kinds of pages - small, large, and
237*404b540aSrobert PCH. */
238*404b540aSrobert typedef struct page_entry
239*404b540aSrobert {
240*404b540aSrobert /* The address at which the memory is allocated. */
241*404b540aSrobert char *page;
242*404b540aSrobert
243*404b540aSrobert /* The zone that this page entry belongs to. */
244*404b540aSrobert struct alloc_zone *zone;
245*404b540aSrobert
246*404b540aSrobert #ifdef GATHER_STATISTICS
247*404b540aSrobert /* How many collections we've survived. */
248*404b540aSrobert size_t survived;
249*404b540aSrobert #endif
250*404b540aSrobert
251*404b540aSrobert /* Does this page contain small objects, or one large object? */
252*404b540aSrobert bool large_p;
253*404b540aSrobert
254*404b540aSrobert /* Is this page part of the loaded PCH? */
255*404b540aSrobert bool pch_p;
256*404b540aSrobert } page_entry;
257*404b540aSrobert
258*404b540aSrobert /* Additional data needed for small pages. */
259*404b540aSrobert struct small_page_entry
260*404b540aSrobert {
261*404b540aSrobert struct page_entry common;
262*404b540aSrobert
263*404b540aSrobert /* The next small page entry, or NULL if this is the last. */
264*404b540aSrobert struct small_page_entry *next;
265*404b540aSrobert
266*404b540aSrobert /* If currently marking this zone, a pointer to the mark bits
267*404b540aSrobert for this page. If we aren't currently marking this zone,
268*404b540aSrobert this pointer may be stale (pointing to freed memory). */
269*404b540aSrobert mark_type *mark_bits;
270*404b540aSrobert
271*404b540aSrobert /* The allocation bitmap. This array extends far enough to have
272*404b540aSrobert one bit for every BYTES_PER_ALLOC_BIT bytes in the page. */
273*404b540aSrobert alloc_type alloc_bits[1];
274*404b540aSrobert };
275*404b540aSrobert
276*404b540aSrobert /* Additional data needed for large pages. */
277*404b540aSrobert struct large_page_entry
278*404b540aSrobert {
279*404b540aSrobert struct page_entry common;
280*404b540aSrobert
281*404b540aSrobert /* The next large page entry, or NULL if this is the last. */
282*404b540aSrobert struct large_page_entry *next;
283*404b540aSrobert
284*404b540aSrobert /* The number of bytes allocated, not including the page entry. */
285*404b540aSrobert size_t bytes;
286*404b540aSrobert
287*404b540aSrobert /* The previous page in the list, so that we can unlink this one. */
288*404b540aSrobert struct large_page_entry *prev;
289*404b540aSrobert
290*404b540aSrobert /* During marking, is this object marked? */
291*404b540aSrobert bool mark_p;
292*404b540aSrobert };
293*404b540aSrobert
294*404b540aSrobert /* A two-level tree is used to look up the page-entry for a given
295*404b540aSrobert pointer. Two chunks of the pointer's bits are extracted to index
296*404b540aSrobert the first and second levels of the tree, as follows:
297*404b540aSrobert
298*404b540aSrobert HOST_PAGE_SIZE_BITS
299*404b540aSrobert 32 | |
300*404b540aSrobert msb +----------------+----+------+------+ lsb
301*404b540aSrobert | | |
302*404b540aSrobert PAGE_L1_BITS |
303*404b540aSrobert | |
304*404b540aSrobert PAGE_L2_BITS
305*404b540aSrobert
306*404b540aSrobert The bottommost HOST_PAGE_SIZE_BITS are ignored, since page-entry
307*404b540aSrobert pages are aligned on system page boundaries. The next most
308*404b540aSrobert significant PAGE_L2_BITS and PAGE_L1_BITS are the second and first
309*404b540aSrobert index values in the lookup table, respectively.
310*404b540aSrobert
311*404b540aSrobert For 32-bit architectures and the settings below, there are no
312*404b540aSrobert leftover bits. For architectures with wider pointers, the lookup
313*404b540aSrobert tree points to a list of pages, which must be scanned to find the
314*404b540aSrobert correct one. */
315*404b540aSrobert
316*404b540aSrobert #define PAGE_L1_BITS (8)
317*404b540aSrobert #define PAGE_L2_BITS (32 - PAGE_L1_BITS - GGC_PAGE_SHIFT)
318*404b540aSrobert #define PAGE_L1_SIZE ((size_t) 1 << PAGE_L1_BITS)
319*404b540aSrobert #define PAGE_L2_SIZE ((size_t) 1 << PAGE_L2_BITS)
320*404b540aSrobert
321*404b540aSrobert #define LOOKUP_L1(p) \
322*404b540aSrobert (((size_t) (p) >> (32 - PAGE_L1_BITS)) & ((1 << PAGE_L1_BITS) - 1))
323*404b540aSrobert
324*404b540aSrobert #define LOOKUP_L2(p) \
325*404b540aSrobert (((size_t) (p) >> GGC_PAGE_SHIFT) & ((1 << PAGE_L2_BITS) - 1))
326*404b540aSrobert
327*404b540aSrobert #if HOST_BITS_PER_PTR <= 32
328*404b540aSrobert
329*404b540aSrobert /* On 32-bit hosts, we use a two level page table, as pictured above. */
330*404b540aSrobert typedef page_entry **page_table[PAGE_L1_SIZE];
331*404b540aSrobert
332*404b540aSrobert #else
333*404b540aSrobert
334*404b540aSrobert /* On 64-bit hosts, we use the same two level page tables plus a linked
335*404b540aSrobert list that disambiguates the top 32-bits. There will almost always be
336*404b540aSrobert exactly one entry in the list. */
337*404b540aSrobert typedef struct page_table_chain
338*404b540aSrobert {
339*404b540aSrobert struct page_table_chain *next;
340*404b540aSrobert size_t high_bits;
341*404b540aSrobert page_entry **table[PAGE_L1_SIZE];
342*404b540aSrobert } *page_table;
343*404b540aSrobert
344*404b540aSrobert #endif
345*404b540aSrobert
346*404b540aSrobert /* The global variables. */
347*404b540aSrobert static struct globals
348*404b540aSrobert {
349*404b540aSrobert /* The linked list of zones. */
350*404b540aSrobert struct alloc_zone *zones;
351*404b540aSrobert
352*404b540aSrobert /* Lookup table for associating allocation pages with object addresses. */
353*404b540aSrobert page_table lookup;
354*404b540aSrobert
355*404b540aSrobert /* The system's page size, and related constants. */
356*404b540aSrobert size_t pagesize;
357*404b540aSrobert size_t lg_pagesize;
358*404b540aSrobert size_t page_mask;
359*404b540aSrobert
360*404b540aSrobert /* The size to allocate for a small page entry. This includes
361*404b540aSrobert the size of the structure and the size of the allocation
362*404b540aSrobert bitmap. */
363*404b540aSrobert size_t small_page_overhead;
364*404b540aSrobert
365*404b540aSrobert #if defined (HAVE_MMAP_DEV_ZERO)
366*404b540aSrobert /* A file descriptor open to /dev/zero for reading. */
367*404b540aSrobert int dev_zero_fd;
368*404b540aSrobert #endif
369*404b540aSrobert
370*404b540aSrobert /* Allocate pages in chunks of this size, to throttle calls to memory
371*404b540aSrobert allocation routines. The first page is used, the rest go onto the
372*404b540aSrobert free list. */
373*404b540aSrobert size_t quire_size;
374*404b540aSrobert
375*404b540aSrobert /* The file descriptor for debugging output. */
376*404b540aSrobert FILE *debug_file;
377*404b540aSrobert } G;
378*404b540aSrobert
379*404b540aSrobert /* A zone allocation structure. There is one of these for every
380*404b540aSrobert distinct allocation zone. */
381*404b540aSrobert struct alloc_zone
382*404b540aSrobert {
383*404b540aSrobert /* The most recent free chunk is saved here, instead of in the linked
384*404b540aSrobert free list, to decrease list manipulation. It is most likely that we
385*404b540aSrobert will want this one. */
386*404b540aSrobert char *cached_free;
387*404b540aSrobert size_t cached_free_size;
388*404b540aSrobert
389*404b540aSrobert /* Linked lists of free storage. Slots 1 ... NUM_FREE_BINS have chunks of size
390*404b540aSrobert FREE_BIN_DELTA. All other chunks are in slot 0. */
391*404b540aSrobert struct alloc_chunk *free_chunks[NUM_FREE_BINS + 1];
392*404b540aSrobert
393*404b540aSrobert /* The highest bin index which might be non-empty. It may turn out
394*404b540aSrobert to be empty, in which case we have to search downwards. */
395*404b540aSrobert size_t high_free_bin;
396*404b540aSrobert
397*404b540aSrobert /* Bytes currently allocated in this zone. */
398*404b540aSrobert size_t allocated;
399*404b540aSrobert
400*404b540aSrobert /* Linked list of the small pages in this zone. */
401*404b540aSrobert struct small_page_entry *pages;
402*404b540aSrobert
403*404b540aSrobert /* Doubly linked list of large pages in this zone. */
404*404b540aSrobert struct large_page_entry *large_pages;
405*404b540aSrobert
406*404b540aSrobert /* If we are currently marking this zone, a pointer to the mark bits. */
407*404b540aSrobert mark_type *mark_bits;
408*404b540aSrobert
409*404b540aSrobert /* Name of the zone. */
410*404b540aSrobert const char *name;
411*404b540aSrobert
412*404b540aSrobert /* The number of small pages currently allocated in this zone. */
413*404b540aSrobert size_t n_small_pages;
414*404b540aSrobert
415*404b540aSrobert /* Bytes allocated at the end of the last collection. */
416*404b540aSrobert size_t allocated_last_gc;
417*404b540aSrobert
418*404b540aSrobert /* Total amount of memory mapped. */
419*404b540aSrobert size_t bytes_mapped;
420*404b540aSrobert
421*404b540aSrobert /* A cache of free system pages. */
422*404b540aSrobert struct small_page_entry *free_pages;
423*404b540aSrobert
424*404b540aSrobert /* Next zone in the linked list of zones. */
425*404b540aSrobert struct alloc_zone *next_zone;
426*404b540aSrobert
427*404b540aSrobert /* True if this zone was collected during this collection. */
428*404b540aSrobert bool was_collected;
429*404b540aSrobert
430*404b540aSrobert /* True if this zone should be destroyed after the next collection. */
431*404b540aSrobert bool dead;
432*404b540aSrobert
433*404b540aSrobert #ifdef GATHER_STATISTICS
434*404b540aSrobert struct
435*404b540aSrobert {
436*404b540aSrobert /* Total memory allocated with ggc_alloc. */
437*404b540aSrobert unsigned long long total_allocated;
438*404b540aSrobert /* Total overhead for memory to be allocated with ggc_alloc. */
439*404b540aSrobert unsigned long long total_overhead;
440*404b540aSrobert
441*404b540aSrobert /* Total allocations and overhead for sizes less than 32, 64 and 128.
442*404b540aSrobert These sizes are interesting because they are typical cache line
443*404b540aSrobert sizes. */
444*404b540aSrobert
445*404b540aSrobert unsigned long long total_allocated_under32;
446*404b540aSrobert unsigned long long total_overhead_under32;
447*404b540aSrobert
448*404b540aSrobert unsigned long long total_allocated_under64;
449*404b540aSrobert unsigned long long total_overhead_under64;
450*404b540aSrobert
451*404b540aSrobert unsigned long long total_allocated_under128;
452*404b540aSrobert unsigned long long total_overhead_under128;
453*404b540aSrobert } stats;
454*404b540aSrobert #endif
455*404b540aSrobert } main_zone;
456*404b540aSrobert
457*404b540aSrobert /* Some default zones. */
458*404b540aSrobert struct alloc_zone rtl_zone;
459*404b540aSrobert struct alloc_zone tree_zone;
460*404b540aSrobert struct alloc_zone tree_id_zone;
461*404b540aSrobert
462*404b540aSrobert /* The PCH zone does not need a normal zone structure, and it does
463*404b540aSrobert not live on the linked list of zones. */
464*404b540aSrobert struct pch_zone
465*404b540aSrobert {
466*404b540aSrobert /* The start of the PCH zone. NULL if there is none. */
467*404b540aSrobert char *page;
468*404b540aSrobert
469*404b540aSrobert /* The end of the PCH zone. NULL if there is none. */
470*404b540aSrobert char *end;
471*404b540aSrobert
472*404b540aSrobert /* The size of the PCH zone. 0 if there is none. */
473*404b540aSrobert size_t bytes;
474*404b540aSrobert
475*404b540aSrobert /* The allocation bitmap for the PCH zone. */
476*404b540aSrobert alloc_type *alloc_bits;
477*404b540aSrobert
478*404b540aSrobert /* If we are currently marking, the mark bitmap for the PCH zone.
479*404b540aSrobert When it is first read in, we could avoid marking the PCH,
480*404b540aSrobert because it will not contain any pointers to GC memory outside
481*404b540aSrobert of the PCH; however, the PCH is currently mapped as writable,
482*404b540aSrobert so we must mark it in case new pointers are added. */
483*404b540aSrobert mark_type *mark_bits;
484*404b540aSrobert } pch_zone;
485*404b540aSrobert
486*404b540aSrobert #ifdef USING_MMAP
487*404b540aSrobert static char *alloc_anon (char *, size_t, struct alloc_zone *);
488*404b540aSrobert #endif
489*404b540aSrobert static struct small_page_entry * alloc_small_page (struct alloc_zone *);
490*404b540aSrobert static struct large_page_entry * alloc_large_page (size_t, struct alloc_zone *);
491*404b540aSrobert static void free_chunk (char *, size_t, struct alloc_zone *);
492*404b540aSrobert static void free_small_page (struct small_page_entry *);
493*404b540aSrobert static void free_large_page (struct large_page_entry *);
494*404b540aSrobert static void release_pages (struct alloc_zone *);
495*404b540aSrobert static void sweep_pages (struct alloc_zone *);
496*404b540aSrobert static bool ggc_collect_1 (struct alloc_zone *, bool);
497*404b540aSrobert static void new_ggc_zone_1 (struct alloc_zone *, const char *);
498*404b540aSrobert
499*404b540aSrobert /* Traverse the page table and find the entry for a page.
500*404b540aSrobert Die (probably) if the object wasn't allocated via GC. */
501*404b540aSrobert
502*404b540aSrobert static inline page_entry *
lookup_page_table_entry(const void * p)503*404b540aSrobert lookup_page_table_entry (const void *p)
504*404b540aSrobert {
505*404b540aSrobert page_entry ***base;
506*404b540aSrobert size_t L1, L2;
507*404b540aSrobert
508*404b540aSrobert #if HOST_BITS_PER_PTR <= 32
509*404b540aSrobert base = &G.lookup[0];
510*404b540aSrobert #else
511*404b540aSrobert page_table table = G.lookup;
512*404b540aSrobert size_t high_bits = (size_t) p & ~ (size_t) 0xffffffff;
513*404b540aSrobert while (table->high_bits != high_bits)
514*404b540aSrobert table = table->next;
515*404b540aSrobert base = &table->table[0];
516*404b540aSrobert #endif
517*404b540aSrobert
518*404b540aSrobert /* Extract the level 1 and 2 indices. */
519*404b540aSrobert L1 = LOOKUP_L1 (p);
520*404b540aSrobert L2 = LOOKUP_L2 (p);
521*404b540aSrobert
522*404b540aSrobert return base[L1][L2];
523*404b540aSrobert }
524*404b540aSrobert
525*404b540aSrobert /* Set the page table entry for the page that starts at P. If ENTRY
526*404b540aSrobert is NULL, clear the entry. */
527*404b540aSrobert
528*404b540aSrobert static void
set_page_table_entry(void * p,page_entry * entry)529*404b540aSrobert set_page_table_entry (void *p, page_entry *entry)
530*404b540aSrobert {
531*404b540aSrobert page_entry ***base;
532*404b540aSrobert size_t L1, L2;
533*404b540aSrobert
534*404b540aSrobert #if HOST_BITS_PER_PTR <= 32
535*404b540aSrobert base = &G.lookup[0];
536*404b540aSrobert #else
537*404b540aSrobert page_table table;
538*404b540aSrobert size_t high_bits = (size_t) p & ~ (size_t) 0xffffffff;
539*404b540aSrobert for (table = G.lookup; table; table = table->next)
540*404b540aSrobert if (table->high_bits == high_bits)
541*404b540aSrobert goto found;
542*404b540aSrobert
543*404b540aSrobert /* Not found -- allocate a new table. */
544*404b540aSrobert table = xcalloc (1, sizeof(*table));
545*404b540aSrobert table->next = G.lookup;
546*404b540aSrobert table->high_bits = high_bits;
547*404b540aSrobert G.lookup = table;
548*404b540aSrobert found:
549*404b540aSrobert base = &table->table[0];
550*404b540aSrobert #endif
551*404b540aSrobert
552*404b540aSrobert /* Extract the level 1 and 2 indices. */
553*404b540aSrobert L1 = LOOKUP_L1 (p);
554*404b540aSrobert L2 = LOOKUP_L2 (p);
555*404b540aSrobert
556*404b540aSrobert if (base[L1] == NULL)
557*404b540aSrobert base[L1] = xcalloc (PAGE_L2_SIZE, sizeof (page_entry *));
558*404b540aSrobert
559*404b540aSrobert base[L1][L2] = entry;
560*404b540aSrobert }
561*404b540aSrobert
562*404b540aSrobert /* Find the page table entry associated with OBJECT. */
563*404b540aSrobert
564*404b540aSrobert static inline struct page_entry *
zone_get_object_page(const void * object)565*404b540aSrobert zone_get_object_page (const void *object)
566*404b540aSrobert {
567*404b540aSrobert return lookup_page_table_entry (object);
568*404b540aSrobert }
569*404b540aSrobert
570*404b540aSrobert /* Find which element of the alloc_bits array OBJECT should be
571*404b540aSrobert recorded in. */
572*404b540aSrobert static inline unsigned int
zone_get_object_alloc_word(const void * object)573*404b540aSrobert zone_get_object_alloc_word (const void *object)
574*404b540aSrobert {
575*404b540aSrobert return (((size_t) object & (GGC_PAGE_SIZE - 1))
576*404b540aSrobert / (8 * sizeof (alloc_type) * BYTES_PER_ALLOC_BIT));
577*404b540aSrobert }
578*404b540aSrobert
579*404b540aSrobert /* Find which bit of the appropriate word in the alloc_bits array
580*404b540aSrobert OBJECT should be recorded in. */
581*404b540aSrobert static inline unsigned int
zone_get_object_alloc_bit(const void * object)582*404b540aSrobert zone_get_object_alloc_bit (const void *object)
583*404b540aSrobert {
584*404b540aSrobert return (((size_t) object / BYTES_PER_ALLOC_BIT)
585*404b540aSrobert % (8 * sizeof (alloc_type)));
586*404b540aSrobert }
587*404b540aSrobert
588*404b540aSrobert /* Find which element of the mark_bits array OBJECT should be recorded
589*404b540aSrobert in. */
590*404b540aSrobert static inline unsigned int
zone_get_object_mark_word(const void * object)591*404b540aSrobert zone_get_object_mark_word (const void *object)
592*404b540aSrobert {
593*404b540aSrobert return (((size_t) object & (GGC_PAGE_SIZE - 1))
594*404b540aSrobert / (8 * sizeof (mark_type) * BYTES_PER_MARK_BIT));
595*404b540aSrobert }
596*404b540aSrobert
597*404b540aSrobert /* Find which bit of the appropriate word in the mark_bits array
598*404b540aSrobert OBJECT should be recorded in. */
599*404b540aSrobert static inline unsigned int
zone_get_object_mark_bit(const void * object)600*404b540aSrobert zone_get_object_mark_bit (const void *object)
601*404b540aSrobert {
602*404b540aSrobert return (((size_t) object / BYTES_PER_MARK_BIT)
603*404b540aSrobert % (8 * sizeof (mark_type)));
604*404b540aSrobert }
605*404b540aSrobert
606*404b540aSrobert /* Set the allocation bit corresponding to OBJECT in its page's
607*404b540aSrobert bitmap. Used to split this object from the preceding one. */
608*404b540aSrobert static inline void
zone_set_object_alloc_bit(const void * object)609*404b540aSrobert zone_set_object_alloc_bit (const void *object)
610*404b540aSrobert {
611*404b540aSrobert struct small_page_entry *page
612*404b540aSrobert = (struct small_page_entry *) zone_get_object_page (object);
613*404b540aSrobert unsigned int start_word = zone_get_object_alloc_word (object);
614*404b540aSrobert unsigned int start_bit = zone_get_object_alloc_bit (object);
615*404b540aSrobert
616*404b540aSrobert page->alloc_bits[start_word] |= 1L << start_bit;
617*404b540aSrobert }
618*404b540aSrobert
619*404b540aSrobert /* Clear the allocation bit corresponding to OBJECT in PAGE's
620*404b540aSrobert bitmap. Used to coalesce this object with the preceding
621*404b540aSrobert one. */
622*404b540aSrobert static inline void
zone_clear_object_alloc_bit(struct small_page_entry * page,const void * object)623*404b540aSrobert zone_clear_object_alloc_bit (struct small_page_entry *page,
624*404b540aSrobert const void *object)
625*404b540aSrobert {
626*404b540aSrobert unsigned int start_word = zone_get_object_alloc_word (object);
627*404b540aSrobert unsigned int start_bit = zone_get_object_alloc_bit (object);
628*404b540aSrobert
629*404b540aSrobert /* Would xor be quicker? */
630*404b540aSrobert page->alloc_bits[start_word] &= ~(1L << start_bit);
631*404b540aSrobert }
632*404b540aSrobert
633*404b540aSrobert /* Find the size of the object which starts at START_WORD and
634*404b540aSrobert START_BIT in ALLOC_BITS, which is at most MAX_SIZE bytes.
635*404b540aSrobert Helper function for ggc_get_size and zone_find_object_size. */
636*404b540aSrobert
637*404b540aSrobert static inline size_t
zone_object_size_1(alloc_type * alloc_bits,size_t start_word,size_t start_bit,size_t max_size)638*404b540aSrobert zone_object_size_1 (alloc_type *alloc_bits,
639*404b540aSrobert size_t start_word, size_t start_bit,
640*404b540aSrobert size_t max_size)
641*404b540aSrobert {
642*404b540aSrobert size_t size;
643*404b540aSrobert alloc_type alloc_word;
644*404b540aSrobert int indx;
645*404b540aSrobert
646*404b540aSrobert /* Load the first word. */
647*404b540aSrobert alloc_word = alloc_bits[start_word++];
648*404b540aSrobert
649*404b540aSrobert /* If that was the last bit in this word, we'll want to continue
650*404b540aSrobert with the next word. Otherwise, handle the rest of this word. */
651*404b540aSrobert if (start_bit)
652*404b540aSrobert {
653*404b540aSrobert indx = alloc_ffs (alloc_word >> start_bit);
654*404b540aSrobert if (indx)
655*404b540aSrobert /* indx is 1-based. We started at the bit after the object's
656*404b540aSrobert start, but we also ended at the bit after the object's end.
657*404b540aSrobert It cancels out. */
658*404b540aSrobert return indx * BYTES_PER_ALLOC_BIT;
659*404b540aSrobert
660*404b540aSrobert /* The extra 1 accounts for the starting unit, before start_bit. */
661*404b540aSrobert size = (sizeof (alloc_type) * 8 - start_bit + 1) * BYTES_PER_ALLOC_BIT;
662*404b540aSrobert
663*404b540aSrobert if (size >= max_size)
664*404b540aSrobert return max_size;
665*404b540aSrobert
666*404b540aSrobert alloc_word = alloc_bits[start_word++];
667*404b540aSrobert }
668*404b540aSrobert else
669*404b540aSrobert size = BYTES_PER_ALLOC_BIT;
670*404b540aSrobert
671*404b540aSrobert while (alloc_word == 0)
672*404b540aSrobert {
673*404b540aSrobert size += sizeof (alloc_type) * 8 * BYTES_PER_ALLOC_BIT;
674*404b540aSrobert if (size >= max_size)
675*404b540aSrobert return max_size;
676*404b540aSrobert alloc_word = alloc_bits[start_word++];
677*404b540aSrobert }
678*404b540aSrobert
679*404b540aSrobert indx = alloc_ffs (alloc_word);
680*404b540aSrobert return size + (indx - 1) * BYTES_PER_ALLOC_BIT;
681*404b540aSrobert }
682*404b540aSrobert
683*404b540aSrobert /* Find the size of OBJECT on small page PAGE. */
684*404b540aSrobert
685*404b540aSrobert static inline size_t
zone_find_object_size(struct small_page_entry * page,const void * object)686*404b540aSrobert zone_find_object_size (struct small_page_entry *page,
687*404b540aSrobert const void *object)
688*404b540aSrobert {
689*404b540aSrobert const char *object_midptr = (const char *) object + BYTES_PER_ALLOC_BIT;
690*404b540aSrobert unsigned int start_word = zone_get_object_alloc_word (object_midptr);
691*404b540aSrobert unsigned int start_bit = zone_get_object_alloc_bit (object_midptr);
692*404b540aSrobert size_t max_size = (page->common.page + SMALL_PAGE_SIZE
693*404b540aSrobert - (char *) object);
694*404b540aSrobert
695*404b540aSrobert return zone_object_size_1 (page->alloc_bits, start_word, start_bit,
696*404b540aSrobert max_size);
697*404b540aSrobert }
698*404b540aSrobert
699*404b540aSrobert /* Allocate the mark bits for every zone, and set the pointers on each
700*404b540aSrobert page. */
701*404b540aSrobert static void
zone_allocate_marks(void)702*404b540aSrobert zone_allocate_marks (void)
703*404b540aSrobert {
704*404b540aSrobert struct alloc_zone *zone;
705*404b540aSrobert
706*404b540aSrobert for (zone = G.zones; zone; zone = zone->next_zone)
707*404b540aSrobert {
708*404b540aSrobert struct small_page_entry *page;
709*404b540aSrobert mark_type *cur_marks;
710*404b540aSrobert size_t mark_words, mark_words_per_page;
711*404b540aSrobert #ifdef ENABLE_CHECKING
712*404b540aSrobert size_t n = 0;
713*404b540aSrobert #endif
714*404b540aSrobert
715*404b540aSrobert mark_words_per_page
716*404b540aSrobert = (GGC_PAGE_SIZE + BYTES_PER_MARK_WORD - 1) / BYTES_PER_MARK_WORD;
717*404b540aSrobert mark_words = zone->n_small_pages * mark_words_per_page;
718*404b540aSrobert zone->mark_bits = (mark_type *) xcalloc (sizeof (mark_type),
719*404b540aSrobert mark_words);
720*404b540aSrobert cur_marks = zone->mark_bits;
721*404b540aSrobert for (page = zone->pages; page; page = page->next)
722*404b540aSrobert {
723*404b540aSrobert page->mark_bits = cur_marks;
724*404b540aSrobert cur_marks += mark_words_per_page;
725*404b540aSrobert #ifdef ENABLE_CHECKING
726*404b540aSrobert n++;
727*404b540aSrobert #endif
728*404b540aSrobert }
729*404b540aSrobert #ifdef ENABLE_CHECKING
730*404b540aSrobert gcc_assert (n == zone->n_small_pages);
731*404b540aSrobert #endif
732*404b540aSrobert }
733*404b540aSrobert
734*404b540aSrobert /* We don't collect the PCH zone, but we do have to mark it
735*404b540aSrobert (for now). */
736*404b540aSrobert if (pch_zone.bytes)
737*404b540aSrobert pch_zone.mark_bits
738*404b540aSrobert = (mark_type *) xcalloc (sizeof (mark_type),
739*404b540aSrobert CEIL (pch_zone.bytes, BYTES_PER_MARK_WORD));
740*404b540aSrobert }
741*404b540aSrobert
742*404b540aSrobert /* After marking and sweeping, release the memory used for mark bits. */
743*404b540aSrobert static void
zone_free_marks(void)744*404b540aSrobert zone_free_marks (void)
745*404b540aSrobert {
746*404b540aSrobert struct alloc_zone *zone;
747*404b540aSrobert
748*404b540aSrobert for (zone = G.zones; zone; zone = zone->next_zone)
749*404b540aSrobert if (zone->mark_bits)
750*404b540aSrobert {
751*404b540aSrobert free (zone->mark_bits);
752*404b540aSrobert zone->mark_bits = NULL;
753*404b540aSrobert }
754*404b540aSrobert
755*404b540aSrobert if (pch_zone.bytes)
756*404b540aSrobert {
757*404b540aSrobert free (pch_zone.mark_bits);
758*404b540aSrobert pch_zone.mark_bits = NULL;
759*404b540aSrobert }
760*404b540aSrobert }
761*404b540aSrobert
762*404b540aSrobert #ifdef USING_MMAP
763*404b540aSrobert /* Allocate SIZE bytes of anonymous memory, preferably near PREF,
764*404b540aSrobert (if non-null). The ifdef structure here is intended to cause a
765*404b540aSrobert compile error unless exactly one of the HAVE_* is defined. */
766*404b540aSrobert
767*404b540aSrobert static inline char *
alloc_anon(char * pref ATTRIBUTE_UNUSED,size_t size,struct alloc_zone * zone)768*404b540aSrobert alloc_anon (char *pref ATTRIBUTE_UNUSED, size_t size, struct alloc_zone *zone)
769*404b540aSrobert {
770*404b540aSrobert #ifdef HAVE_MMAP_ANON
771*404b540aSrobert char *page = (char *) mmap (pref, size, PROT_READ | PROT_WRITE,
772*404b540aSrobert MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
773*404b540aSrobert #endif
774*404b540aSrobert #ifdef HAVE_MMAP_DEV_ZERO
775*404b540aSrobert char *page = (char *) mmap (pref, size, PROT_READ | PROT_WRITE,
776*404b540aSrobert MAP_PRIVATE, G.dev_zero_fd, 0);
777*404b540aSrobert #endif
778*404b540aSrobert
779*404b540aSrobert if (page == (char *) MAP_FAILED)
780*404b540aSrobert {
781*404b540aSrobert perror ("virtual memory exhausted");
782*404b540aSrobert exit (FATAL_EXIT_CODE);
783*404b540aSrobert }
784*404b540aSrobert
785*404b540aSrobert /* Remember that we allocated this memory. */
786*404b540aSrobert zone->bytes_mapped += size;
787*404b540aSrobert
788*404b540aSrobert /* Pretend we don't have access to the allocated pages. We'll enable
789*404b540aSrobert access to smaller pieces of the area in ggc_alloc. Discard the
790*404b540aSrobert handle to avoid handle leak. */
791*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (page, size));
792*404b540aSrobert
793*404b540aSrobert return page;
794*404b540aSrobert }
795*404b540aSrobert #endif
796*404b540aSrobert
797*404b540aSrobert /* Allocate a new page for allocating small objects in ZONE, and
798*404b540aSrobert return an entry for it. */
799*404b540aSrobert
800*404b540aSrobert static struct small_page_entry *
alloc_small_page(struct alloc_zone * zone)801*404b540aSrobert alloc_small_page (struct alloc_zone *zone)
802*404b540aSrobert {
803*404b540aSrobert struct small_page_entry *entry;
804*404b540aSrobert
805*404b540aSrobert /* Check the list of free pages for one we can use. */
806*404b540aSrobert entry = zone->free_pages;
807*404b540aSrobert if (entry != NULL)
808*404b540aSrobert {
809*404b540aSrobert /* Recycle the allocated memory from this page ... */
810*404b540aSrobert zone->free_pages = entry->next;
811*404b540aSrobert }
812*404b540aSrobert else
813*404b540aSrobert {
814*404b540aSrobert /* We want just one page. Allocate a bunch of them and put the
815*404b540aSrobert extras on the freelist. (Can only do this optimization with
816*404b540aSrobert mmap for backing store.) */
817*404b540aSrobert struct small_page_entry *e, *f = zone->free_pages;
818*404b540aSrobert int i;
819*404b540aSrobert char *page;
820*404b540aSrobert
821*404b540aSrobert page = alloc_anon (NULL, GGC_PAGE_SIZE * G.quire_size, zone);
822*404b540aSrobert
823*404b540aSrobert /* This loop counts down so that the chain will be in ascending
824*404b540aSrobert memory order. */
825*404b540aSrobert for (i = G.quire_size - 1; i >= 1; i--)
826*404b540aSrobert {
827*404b540aSrobert e = xcalloc (1, G.small_page_overhead);
828*404b540aSrobert e->common.page = page + (i << GGC_PAGE_SHIFT);
829*404b540aSrobert e->common.zone = zone;
830*404b540aSrobert e->next = f;
831*404b540aSrobert f = e;
832*404b540aSrobert set_page_table_entry (e->common.page, &e->common);
833*404b540aSrobert }
834*404b540aSrobert
835*404b540aSrobert zone->free_pages = f;
836*404b540aSrobert
837*404b540aSrobert entry = xcalloc (1, G.small_page_overhead);
838*404b540aSrobert entry->common.page = page;
839*404b540aSrobert entry->common.zone = zone;
840*404b540aSrobert set_page_table_entry (page, &entry->common);
841*404b540aSrobert }
842*404b540aSrobert
843*404b540aSrobert zone->n_small_pages++;
844*404b540aSrobert
845*404b540aSrobert if (GGC_DEBUG_LEVEL >= 2)
846*404b540aSrobert fprintf (G.debug_file,
847*404b540aSrobert "Allocating %s page at %p, data %p-%p\n",
848*404b540aSrobert entry->common.zone->name, (PTR) entry, entry->common.page,
849*404b540aSrobert entry->common.page + SMALL_PAGE_SIZE - 1);
850*404b540aSrobert
851*404b540aSrobert return entry;
852*404b540aSrobert }
853*404b540aSrobert
854*404b540aSrobert /* Allocate a large page of size SIZE in ZONE. */
855*404b540aSrobert
856*404b540aSrobert static struct large_page_entry *
alloc_large_page(size_t size,struct alloc_zone * zone)857*404b540aSrobert alloc_large_page (size_t size, struct alloc_zone *zone)
858*404b540aSrobert {
859*404b540aSrobert struct large_page_entry *entry;
860*404b540aSrobert char *page;
861*404b540aSrobert size_t needed_size;
862*404b540aSrobert
863*404b540aSrobert needed_size = size + sizeof (struct large_page_entry);
864*404b540aSrobert page = xmalloc (needed_size);
865*404b540aSrobert
866*404b540aSrobert entry = (struct large_page_entry *) page;
867*404b540aSrobert
868*404b540aSrobert entry->next = NULL;
869*404b540aSrobert entry->common.page = page + sizeof (struct large_page_entry);
870*404b540aSrobert entry->common.large_p = true;
871*404b540aSrobert entry->common.pch_p = false;
872*404b540aSrobert entry->common.zone = zone;
873*404b540aSrobert #ifdef GATHER_STATISTICS
874*404b540aSrobert entry->common.survived = 0;
875*404b540aSrobert #endif
876*404b540aSrobert entry->mark_p = false;
877*404b540aSrobert entry->bytes = size;
878*404b540aSrobert entry->prev = NULL;
879*404b540aSrobert
880*404b540aSrobert set_page_table_entry (entry->common.page, &entry->common);
881*404b540aSrobert
882*404b540aSrobert if (GGC_DEBUG_LEVEL >= 2)
883*404b540aSrobert fprintf (G.debug_file,
884*404b540aSrobert "Allocating %s large page at %p, data %p-%p\n",
885*404b540aSrobert entry->common.zone->name, (PTR) entry, entry->common.page,
886*404b540aSrobert entry->common.page + SMALL_PAGE_SIZE - 1);
887*404b540aSrobert
888*404b540aSrobert return entry;
889*404b540aSrobert }
890*404b540aSrobert
891*404b540aSrobert
892*404b540aSrobert /* For a page that is no longer needed, put it on the free page list. */
893*404b540aSrobert
894*404b540aSrobert static inline void
free_small_page(struct small_page_entry * entry)895*404b540aSrobert free_small_page (struct small_page_entry *entry)
896*404b540aSrobert {
897*404b540aSrobert if (GGC_DEBUG_LEVEL >= 2)
898*404b540aSrobert fprintf (G.debug_file,
899*404b540aSrobert "Deallocating %s page at %p, data %p-%p\n",
900*404b540aSrobert entry->common.zone->name, (PTR) entry,
901*404b540aSrobert entry->common.page, entry->common.page + SMALL_PAGE_SIZE - 1);
902*404b540aSrobert
903*404b540aSrobert gcc_assert (!entry->common.large_p);
904*404b540aSrobert
905*404b540aSrobert /* Mark the page as inaccessible. Discard the handle to
906*404b540aSrobert avoid handle leak. */
907*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (entry->common.page,
908*404b540aSrobert SMALL_PAGE_SIZE));
909*404b540aSrobert
910*404b540aSrobert entry->next = entry->common.zone->free_pages;
911*404b540aSrobert entry->common.zone->free_pages = entry;
912*404b540aSrobert entry->common.zone->n_small_pages--;
913*404b540aSrobert }
914*404b540aSrobert
915*404b540aSrobert /* Release a large page that is no longer needed. */
916*404b540aSrobert
917*404b540aSrobert static inline void
free_large_page(struct large_page_entry * entry)918*404b540aSrobert free_large_page (struct large_page_entry *entry)
919*404b540aSrobert {
920*404b540aSrobert if (GGC_DEBUG_LEVEL >= 2)
921*404b540aSrobert fprintf (G.debug_file,
922*404b540aSrobert "Deallocating %s page at %p, data %p-%p\n",
923*404b540aSrobert entry->common.zone->name, (PTR) entry,
924*404b540aSrobert entry->common.page, entry->common.page + SMALL_PAGE_SIZE - 1);
925*404b540aSrobert
926*404b540aSrobert gcc_assert (entry->common.large_p);
927*404b540aSrobert
928*404b540aSrobert set_page_table_entry (entry->common.page, NULL);
929*404b540aSrobert free (entry);
930*404b540aSrobert }
931*404b540aSrobert
932*404b540aSrobert /* Release the free page cache to the system. */
933*404b540aSrobert
934*404b540aSrobert static void
release_pages(struct alloc_zone * zone)935*404b540aSrobert release_pages (struct alloc_zone *zone)
936*404b540aSrobert {
937*404b540aSrobert #ifdef USING_MMAP
938*404b540aSrobert struct small_page_entry *p, *next;
939*404b540aSrobert char *start;
940*404b540aSrobert size_t len;
941*404b540aSrobert
942*404b540aSrobert /* Gather up adjacent pages so they are unmapped together. */
943*404b540aSrobert p = zone->free_pages;
944*404b540aSrobert
945*404b540aSrobert while (p)
946*404b540aSrobert {
947*404b540aSrobert start = p->common.page;
948*404b540aSrobert next = p->next;
949*404b540aSrobert len = SMALL_PAGE_SIZE;
950*404b540aSrobert set_page_table_entry (p->common.page, NULL);
951*404b540aSrobert p = next;
952*404b540aSrobert
953*404b540aSrobert while (p && p->common.page == start + len)
954*404b540aSrobert {
955*404b540aSrobert next = p->next;
956*404b540aSrobert len += SMALL_PAGE_SIZE;
957*404b540aSrobert set_page_table_entry (p->common.page, NULL);
958*404b540aSrobert p = next;
959*404b540aSrobert }
960*404b540aSrobert
961*404b540aSrobert munmap (start, len);
962*404b540aSrobert zone->bytes_mapped -= len;
963*404b540aSrobert }
964*404b540aSrobert
965*404b540aSrobert zone->free_pages = NULL;
966*404b540aSrobert #endif
967*404b540aSrobert }
968*404b540aSrobert
969*404b540aSrobert /* Place the block at PTR of size SIZE on the free list for ZONE. */
970*404b540aSrobert
971*404b540aSrobert static inline void
free_chunk(char * ptr,size_t size,struct alloc_zone * zone)972*404b540aSrobert free_chunk (char *ptr, size_t size, struct alloc_zone *zone)
973*404b540aSrobert {
974*404b540aSrobert struct alloc_chunk *chunk = (struct alloc_chunk *) ptr;
975*404b540aSrobert size_t bin = 0;
976*404b540aSrobert
977*404b540aSrobert bin = SIZE_BIN_DOWN (size);
978*404b540aSrobert gcc_assert (bin != 0);
979*404b540aSrobert if (bin > NUM_FREE_BINS)
980*404b540aSrobert {
981*404b540aSrobert bin = 0;
982*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (chunk, sizeof (struct alloc_chunk)));
983*404b540aSrobert chunk->size = size;
984*404b540aSrobert chunk->next_free = zone->free_chunks[bin];
985*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (ptr + sizeof (struct alloc_chunk),
986*404b540aSrobert size - sizeof (struct alloc_chunk)));
987*404b540aSrobert }
988*404b540aSrobert else
989*404b540aSrobert {
990*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (chunk, sizeof (struct alloc_chunk *)));
991*404b540aSrobert chunk->next_free = zone->free_chunks[bin];
992*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (ptr + sizeof (struct alloc_chunk *),
993*404b540aSrobert size - sizeof (struct alloc_chunk *)));
994*404b540aSrobert }
995*404b540aSrobert
996*404b540aSrobert zone->free_chunks[bin] = chunk;
997*404b540aSrobert if (bin > zone->high_free_bin)
998*404b540aSrobert zone->high_free_bin = bin;
999*404b540aSrobert if (GGC_DEBUG_LEVEL >= 3)
1000*404b540aSrobert fprintf (G.debug_file, "Deallocating object, chunk=%p\n", (void *)chunk);
1001*404b540aSrobert }
1002*404b540aSrobert
1003*404b540aSrobert /* Allocate a chunk of memory of at least ORIG_SIZE bytes, in ZONE. */
1004*404b540aSrobert
1005*404b540aSrobert void *
ggc_alloc_zone_stat(size_t orig_size,struct alloc_zone * zone MEM_STAT_DECL)1006*404b540aSrobert ggc_alloc_zone_stat (size_t orig_size, struct alloc_zone *zone
1007*404b540aSrobert MEM_STAT_DECL)
1008*404b540aSrobert {
1009*404b540aSrobert size_t bin;
1010*404b540aSrobert size_t csize;
1011*404b540aSrobert struct small_page_entry *entry;
1012*404b540aSrobert struct alloc_chunk *chunk, **pp;
1013*404b540aSrobert void *result;
1014*404b540aSrobert size_t size = orig_size;
1015*404b540aSrobert
1016*404b540aSrobert /* Make sure that zero-sized allocations get a unique and freeable
1017*404b540aSrobert pointer. */
1018*404b540aSrobert if (size == 0)
1019*404b540aSrobert size = MAX_ALIGNMENT;
1020*404b540aSrobert else
1021*404b540aSrobert size = (size + MAX_ALIGNMENT - 1) & -MAX_ALIGNMENT;
1022*404b540aSrobert
1023*404b540aSrobert /* Try to allocate the object from several different sources. Each
1024*404b540aSrobert of these cases is responsible for setting RESULT and SIZE to
1025*404b540aSrobert describe the allocated block, before jumping to FOUND. If a
1026*404b540aSrobert chunk is split, the allocate bit for the new chunk should also be
1027*404b540aSrobert set.
1028*404b540aSrobert
1029*404b540aSrobert Large objects are handled specially. However, they'll just fail
1030*404b540aSrobert the next couple of conditions, so we can wait to check for them
1031*404b540aSrobert below. The large object case is relatively rare (< 1%), so this
1032*404b540aSrobert is a win. */
1033*404b540aSrobert
1034*404b540aSrobert /* First try to split the last chunk we allocated. For best
1035*404b540aSrobert fragmentation behavior it would be better to look for a
1036*404b540aSrobert free bin of the appropriate size for a small object. However,
1037*404b540aSrobert we're unlikely (1% - 7%) to find one, and this gives better
1038*404b540aSrobert locality behavior anyway. This case handles the lion's share
1039*404b540aSrobert of all calls to this function. */
1040*404b540aSrobert if (size <= zone->cached_free_size)
1041*404b540aSrobert {
1042*404b540aSrobert result = zone->cached_free;
1043*404b540aSrobert
1044*404b540aSrobert zone->cached_free_size -= size;
1045*404b540aSrobert if (zone->cached_free_size)
1046*404b540aSrobert {
1047*404b540aSrobert zone->cached_free += size;
1048*404b540aSrobert zone_set_object_alloc_bit (zone->cached_free);
1049*404b540aSrobert }
1050*404b540aSrobert
1051*404b540aSrobert goto found;
1052*404b540aSrobert }
1053*404b540aSrobert
1054*404b540aSrobert /* Next, try to find a free bin of the exactly correct size. */
1055*404b540aSrobert
1056*404b540aSrobert /* We want to round SIZE up, rather than down, but we know it's
1057*404b540aSrobert already aligned to at least FREE_BIN_DELTA, so we can just
1058*404b540aSrobert shift. */
1059*404b540aSrobert bin = SIZE_BIN_DOWN (size);
1060*404b540aSrobert
1061*404b540aSrobert if (bin <= NUM_FREE_BINS
1062*404b540aSrobert && (chunk = zone->free_chunks[bin]) != NULL)
1063*404b540aSrobert {
1064*404b540aSrobert /* We have a chunk of the right size. Pull it off the free list
1065*404b540aSrobert and use it. */
1066*404b540aSrobert
1067*404b540aSrobert zone->free_chunks[bin] = chunk->next_free;
1068*404b540aSrobert
1069*404b540aSrobert /* NOTE: SIZE is only guaranteed to be right if MAX_ALIGNMENT
1070*404b540aSrobert == FREE_BIN_DELTA. */
1071*404b540aSrobert result = chunk;
1072*404b540aSrobert
1073*404b540aSrobert /* The allocation bits are already set correctly. HIGH_FREE_BIN
1074*404b540aSrobert may now be wrong, if this was the last chunk in the high bin.
1075*404b540aSrobert Rather than fixing it up now, wait until we need to search
1076*404b540aSrobert the free bins. */
1077*404b540aSrobert
1078*404b540aSrobert goto found;
1079*404b540aSrobert }
1080*404b540aSrobert
1081*404b540aSrobert /* Next, if there wasn't a chunk of the ideal size, look for a chunk
1082*404b540aSrobert to split. We can find one in the too-big bin, or in the largest
1083*404b540aSrobert sized bin with a chunk in it. Try the largest normal-sized bin
1084*404b540aSrobert first. */
1085*404b540aSrobert
1086*404b540aSrobert if (zone->high_free_bin > bin)
1087*404b540aSrobert {
1088*404b540aSrobert /* Find the highest numbered free bin. It will be at or below
1089*404b540aSrobert the watermark. */
1090*404b540aSrobert while (zone->high_free_bin > bin
1091*404b540aSrobert && zone->free_chunks[zone->high_free_bin] == NULL)
1092*404b540aSrobert zone->high_free_bin--;
1093*404b540aSrobert
1094*404b540aSrobert if (zone->high_free_bin > bin)
1095*404b540aSrobert {
1096*404b540aSrobert size_t tbin = zone->high_free_bin;
1097*404b540aSrobert chunk = zone->free_chunks[tbin];
1098*404b540aSrobert
1099*404b540aSrobert /* Remove the chunk from its previous bin. */
1100*404b540aSrobert zone->free_chunks[tbin] = chunk->next_free;
1101*404b540aSrobert
1102*404b540aSrobert result = (char *) chunk;
1103*404b540aSrobert
1104*404b540aSrobert /* Save the rest of the chunk for future allocation. */
1105*404b540aSrobert if (zone->cached_free_size)
1106*404b540aSrobert free_chunk (zone->cached_free, zone->cached_free_size, zone);
1107*404b540aSrobert
1108*404b540aSrobert chunk = (struct alloc_chunk *) ((char *) result + size);
1109*404b540aSrobert zone->cached_free = (char *) chunk;
1110*404b540aSrobert zone->cached_free_size = (tbin - bin) * FREE_BIN_DELTA;
1111*404b540aSrobert
1112*404b540aSrobert /* Mark the new free chunk as an object, so that we can
1113*404b540aSrobert find the size of the newly allocated object. */
1114*404b540aSrobert zone_set_object_alloc_bit (chunk);
1115*404b540aSrobert
1116*404b540aSrobert /* HIGH_FREE_BIN may now be wrong, if this was the last
1117*404b540aSrobert chunk in the high bin. Rather than fixing it up now,
1118*404b540aSrobert wait until we need to search the free bins. */
1119*404b540aSrobert
1120*404b540aSrobert goto found;
1121*404b540aSrobert }
1122*404b540aSrobert }
1123*404b540aSrobert
1124*404b540aSrobert /* Failing that, look through the "other" bucket for a chunk
1125*404b540aSrobert that is large enough. */
1126*404b540aSrobert pp = &(zone->free_chunks[0]);
1127*404b540aSrobert chunk = *pp;
1128*404b540aSrobert while (chunk && chunk->size < size)
1129*404b540aSrobert {
1130*404b540aSrobert pp = &chunk->next_free;
1131*404b540aSrobert chunk = *pp;
1132*404b540aSrobert }
1133*404b540aSrobert
1134*404b540aSrobert if (chunk)
1135*404b540aSrobert {
1136*404b540aSrobert /* Remove the chunk from its previous bin. */
1137*404b540aSrobert *pp = chunk->next_free;
1138*404b540aSrobert
1139*404b540aSrobert result = (char *) chunk;
1140*404b540aSrobert
1141*404b540aSrobert /* Save the rest of the chunk for future allocation, if there's any
1142*404b540aSrobert left over. */
1143*404b540aSrobert csize = chunk->size;
1144*404b540aSrobert if (csize > size)
1145*404b540aSrobert {
1146*404b540aSrobert if (zone->cached_free_size)
1147*404b540aSrobert free_chunk (zone->cached_free, zone->cached_free_size, zone);
1148*404b540aSrobert
1149*404b540aSrobert chunk = (struct alloc_chunk *) ((char *) result + size);
1150*404b540aSrobert zone->cached_free = (char *) chunk;
1151*404b540aSrobert zone->cached_free_size = csize - size;
1152*404b540aSrobert
1153*404b540aSrobert /* Mark the new free chunk as an object. */
1154*404b540aSrobert zone_set_object_alloc_bit (chunk);
1155*404b540aSrobert }
1156*404b540aSrobert
1157*404b540aSrobert goto found;
1158*404b540aSrobert }
1159*404b540aSrobert
1160*404b540aSrobert /* Handle large allocations. We could choose any threshold between
1161*404b540aSrobert GGC_PAGE_SIZE - sizeof (struct large_page_entry) and
1162*404b540aSrobert GGC_PAGE_SIZE. It can't be smaller, because then it wouldn't
1163*404b540aSrobert be guaranteed to have a unique entry in the lookup table. Large
1164*404b540aSrobert allocations will always fall through to here. */
1165*404b540aSrobert if (size > GGC_PAGE_SIZE)
1166*404b540aSrobert {
1167*404b540aSrobert struct large_page_entry *entry = alloc_large_page (size, zone);
1168*404b540aSrobert
1169*404b540aSrobert #ifdef GATHER_STATISTICS
1170*404b540aSrobert entry->common.survived = 0;
1171*404b540aSrobert #endif
1172*404b540aSrobert
1173*404b540aSrobert entry->next = zone->large_pages;
1174*404b540aSrobert if (zone->large_pages)
1175*404b540aSrobert zone->large_pages->prev = entry;
1176*404b540aSrobert zone->large_pages = entry;
1177*404b540aSrobert
1178*404b540aSrobert result = entry->common.page;
1179*404b540aSrobert
1180*404b540aSrobert goto found;
1181*404b540aSrobert }
1182*404b540aSrobert
1183*404b540aSrobert /* Failing everything above, allocate a new small page. */
1184*404b540aSrobert
1185*404b540aSrobert entry = alloc_small_page (zone);
1186*404b540aSrobert entry->next = zone->pages;
1187*404b540aSrobert zone->pages = entry;
1188*404b540aSrobert
1189*404b540aSrobert /* Mark the first chunk in the new page. */
1190*404b540aSrobert entry->alloc_bits[0] = 1;
1191*404b540aSrobert
1192*404b540aSrobert result = entry->common.page;
1193*404b540aSrobert if (size < SMALL_PAGE_SIZE)
1194*404b540aSrobert {
1195*404b540aSrobert if (zone->cached_free_size)
1196*404b540aSrobert free_chunk (zone->cached_free, zone->cached_free_size, zone);
1197*404b540aSrobert
1198*404b540aSrobert zone->cached_free = (char *) result + size;
1199*404b540aSrobert zone->cached_free_size = SMALL_PAGE_SIZE - size;
1200*404b540aSrobert
1201*404b540aSrobert /* Mark the new free chunk as an object. */
1202*404b540aSrobert zone_set_object_alloc_bit (zone->cached_free);
1203*404b540aSrobert }
1204*404b540aSrobert
1205*404b540aSrobert found:
1206*404b540aSrobert
1207*404b540aSrobert /* We could save TYPE in the chunk, but we don't use that for
1208*404b540aSrobert anything yet. If we wanted to, we could do it by adding it
1209*404b540aSrobert either before the beginning of the chunk or after its end,
1210*404b540aSrobert and adjusting the size and pointer appropriately. */
1211*404b540aSrobert
1212*404b540aSrobert /* We'll probably write to this after we return. */
1213*404b540aSrobert prefetchw (result);
1214*404b540aSrobert
1215*404b540aSrobert #ifdef ENABLE_GC_CHECKING
1216*404b540aSrobert /* `Poison' the entire allocated object. */
1217*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (result, size));
1218*404b540aSrobert memset (result, 0xaf, size);
1219*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (result + orig_size,
1220*404b540aSrobert size - orig_size));
1221*404b540aSrobert #endif
1222*404b540aSrobert
1223*404b540aSrobert /* Tell Valgrind that the memory is there, but its content isn't
1224*404b540aSrobert defined. The bytes at the end of the object are still marked
1225*404b540aSrobert unaccessible. */
1226*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (result, orig_size));
1227*404b540aSrobert
1228*404b540aSrobert /* Keep track of how many bytes are being allocated. This
1229*404b540aSrobert information is used in deciding when to collect. */
1230*404b540aSrobert zone->allocated += size;
1231*404b540aSrobert
1232*404b540aSrobert timevar_ggc_mem_total += size;
1233*404b540aSrobert
1234*404b540aSrobert #ifdef GATHER_STATISTICS
1235*404b540aSrobert ggc_record_overhead (orig_size, size - orig_size, result PASS_MEM_STAT);
1236*404b540aSrobert
1237*404b540aSrobert {
1238*404b540aSrobert size_t object_size = size;
1239*404b540aSrobert size_t overhead = object_size - orig_size;
1240*404b540aSrobert
1241*404b540aSrobert zone->stats.total_overhead += overhead;
1242*404b540aSrobert zone->stats.total_allocated += object_size;
1243*404b540aSrobert
1244*404b540aSrobert if (orig_size <= 32)
1245*404b540aSrobert {
1246*404b540aSrobert zone->stats.total_overhead_under32 += overhead;
1247*404b540aSrobert zone->stats.total_allocated_under32 += object_size;
1248*404b540aSrobert }
1249*404b540aSrobert if (orig_size <= 64)
1250*404b540aSrobert {
1251*404b540aSrobert zone->stats.total_overhead_under64 += overhead;
1252*404b540aSrobert zone->stats.total_allocated_under64 += object_size;
1253*404b540aSrobert }
1254*404b540aSrobert if (orig_size <= 128)
1255*404b540aSrobert {
1256*404b540aSrobert zone->stats.total_overhead_under128 += overhead;
1257*404b540aSrobert zone->stats.total_allocated_under128 += object_size;
1258*404b540aSrobert }
1259*404b540aSrobert }
1260*404b540aSrobert #endif
1261*404b540aSrobert
1262*404b540aSrobert if (GGC_DEBUG_LEVEL >= 3)
1263*404b540aSrobert fprintf (G.debug_file, "Allocating object, size=%lu at %p\n",
1264*404b540aSrobert (unsigned long) size, result);
1265*404b540aSrobert
1266*404b540aSrobert return result;
1267*404b540aSrobert }
1268*404b540aSrobert
1269*404b540aSrobert /* Allocate a SIZE of chunk memory of GTE type, into an appropriate zone
1270*404b540aSrobert for that type. */
1271*404b540aSrobert
1272*404b540aSrobert void *
ggc_alloc_typed_stat(enum gt_types_enum gte,size_t size MEM_STAT_DECL)1273*404b540aSrobert ggc_alloc_typed_stat (enum gt_types_enum gte, size_t size
1274*404b540aSrobert MEM_STAT_DECL)
1275*404b540aSrobert {
1276*404b540aSrobert switch (gte)
1277*404b540aSrobert {
1278*404b540aSrobert case gt_ggc_e_14lang_tree_node:
1279*404b540aSrobert return ggc_alloc_zone_pass_stat (size, &tree_zone);
1280*404b540aSrobert
1281*404b540aSrobert case gt_ggc_e_7rtx_def:
1282*404b540aSrobert return ggc_alloc_zone_pass_stat (size, &rtl_zone);
1283*404b540aSrobert
1284*404b540aSrobert case gt_ggc_e_9rtvec_def:
1285*404b540aSrobert return ggc_alloc_zone_pass_stat (size, &rtl_zone);
1286*404b540aSrobert
1287*404b540aSrobert default:
1288*404b540aSrobert return ggc_alloc_zone_pass_stat (size, &main_zone);
1289*404b540aSrobert }
1290*404b540aSrobert }
1291*404b540aSrobert
1292*404b540aSrobert /* Normal ggc_alloc simply allocates into the main zone. */
1293*404b540aSrobert
1294*404b540aSrobert void *
ggc_alloc_stat(size_t size MEM_STAT_DECL)1295*404b540aSrobert ggc_alloc_stat (size_t size MEM_STAT_DECL)
1296*404b540aSrobert {
1297*404b540aSrobert return ggc_alloc_zone_pass_stat (size, &main_zone);
1298*404b540aSrobert }
1299*404b540aSrobert
1300*404b540aSrobert /* Poison the chunk. */
1301*404b540aSrobert #ifdef ENABLE_GC_CHECKING
1302*404b540aSrobert #define poison_region(PTR, SIZE) \
1303*404b540aSrobert memset ((PTR), 0xa5, (SIZE))
1304*404b540aSrobert #else
1305*404b540aSrobert #define poison_region(PTR, SIZE)
1306*404b540aSrobert #endif
1307*404b540aSrobert
1308*404b540aSrobert /* Free the object at P. */
1309*404b540aSrobert
1310*404b540aSrobert void
ggc_free(void * p)1311*404b540aSrobert ggc_free (void *p)
1312*404b540aSrobert {
1313*404b540aSrobert struct page_entry *page;
1314*404b540aSrobert
1315*404b540aSrobert #ifdef GATHER_STATISTICS
1316*404b540aSrobert ggc_free_overhead (p);
1317*404b540aSrobert #endif
1318*404b540aSrobert
1319*404b540aSrobert poison_region (p, ggc_get_size (p));
1320*404b540aSrobert
1321*404b540aSrobert page = zone_get_object_page (p);
1322*404b540aSrobert
1323*404b540aSrobert if (page->large_p)
1324*404b540aSrobert {
1325*404b540aSrobert struct large_page_entry *large_page
1326*404b540aSrobert = (struct large_page_entry *) page;
1327*404b540aSrobert
1328*404b540aSrobert /* Remove the page from the linked list. */
1329*404b540aSrobert if (large_page->prev)
1330*404b540aSrobert large_page->prev->next = large_page->next;
1331*404b540aSrobert else
1332*404b540aSrobert {
1333*404b540aSrobert gcc_assert (large_page->common.zone->large_pages == large_page);
1334*404b540aSrobert large_page->common.zone->large_pages = large_page->next;
1335*404b540aSrobert }
1336*404b540aSrobert if (large_page->next)
1337*404b540aSrobert large_page->next->prev = large_page->prev;
1338*404b540aSrobert
1339*404b540aSrobert large_page->common.zone->allocated -= large_page->bytes;
1340*404b540aSrobert
1341*404b540aSrobert /* Release the memory associated with this object. */
1342*404b540aSrobert free_large_page (large_page);
1343*404b540aSrobert }
1344*404b540aSrobert else if (page->pch_p)
1345*404b540aSrobert /* Don't do anything. We won't allocate a new object from the
1346*404b540aSrobert PCH zone so there's no point in releasing anything. */
1347*404b540aSrobert ;
1348*404b540aSrobert else
1349*404b540aSrobert {
1350*404b540aSrobert size_t size = ggc_get_size (p);
1351*404b540aSrobert
1352*404b540aSrobert page->zone->allocated -= size;
1353*404b540aSrobert
1354*404b540aSrobert /* Add the chunk to the free list. We don't bother with coalescing,
1355*404b540aSrobert since we are likely to want a chunk of this size again. */
1356*404b540aSrobert free_chunk (p, size, page->zone);
1357*404b540aSrobert }
1358*404b540aSrobert }
1359*404b540aSrobert
1360*404b540aSrobert /* If P is not marked, mark it and return false. Otherwise return true.
1361*404b540aSrobert P must have been allocated by the GC allocator; it mustn't point to
1362*404b540aSrobert static objects, stack variables, or memory allocated with malloc. */
1363*404b540aSrobert
1364*404b540aSrobert int
ggc_set_mark(const void * p)1365*404b540aSrobert ggc_set_mark (const void *p)
1366*404b540aSrobert {
1367*404b540aSrobert struct page_entry *page;
1368*404b540aSrobert const char *ptr = (const char *) p;
1369*404b540aSrobert
1370*404b540aSrobert page = zone_get_object_page (p);
1371*404b540aSrobert
1372*404b540aSrobert if (page->pch_p)
1373*404b540aSrobert {
1374*404b540aSrobert size_t mark_word, mark_bit, offset;
1375*404b540aSrobert offset = (ptr - pch_zone.page) / BYTES_PER_MARK_BIT;
1376*404b540aSrobert mark_word = offset / (8 * sizeof (mark_type));
1377*404b540aSrobert mark_bit = offset % (8 * sizeof (mark_type));
1378*404b540aSrobert
1379*404b540aSrobert if (pch_zone.mark_bits[mark_word] & (1 << mark_bit))
1380*404b540aSrobert return 1;
1381*404b540aSrobert pch_zone.mark_bits[mark_word] |= (1 << mark_bit);
1382*404b540aSrobert }
1383*404b540aSrobert else if (page->large_p)
1384*404b540aSrobert {
1385*404b540aSrobert struct large_page_entry *large_page
1386*404b540aSrobert = (struct large_page_entry *) page;
1387*404b540aSrobert
1388*404b540aSrobert if (large_page->mark_p)
1389*404b540aSrobert return 1;
1390*404b540aSrobert large_page->mark_p = true;
1391*404b540aSrobert }
1392*404b540aSrobert else
1393*404b540aSrobert {
1394*404b540aSrobert struct small_page_entry *small_page
1395*404b540aSrobert = (struct small_page_entry *) page;
1396*404b540aSrobert
1397*404b540aSrobert if (small_page->mark_bits[zone_get_object_mark_word (p)]
1398*404b540aSrobert & (1 << zone_get_object_mark_bit (p)))
1399*404b540aSrobert return 1;
1400*404b540aSrobert small_page->mark_bits[zone_get_object_mark_word (p)]
1401*404b540aSrobert |= (1 << zone_get_object_mark_bit (p));
1402*404b540aSrobert }
1403*404b540aSrobert
1404*404b540aSrobert if (GGC_DEBUG_LEVEL >= 4)
1405*404b540aSrobert fprintf (G.debug_file, "Marking %p\n", p);
1406*404b540aSrobert
1407*404b540aSrobert return 0;
1408*404b540aSrobert }
1409*404b540aSrobert
1410*404b540aSrobert /* Return 1 if P has been marked, zero otherwise.
1411*404b540aSrobert P must have been allocated by the GC allocator; it mustn't point to
1412*404b540aSrobert static objects, stack variables, or memory allocated with malloc. */
1413*404b540aSrobert
1414*404b540aSrobert int
ggc_marked_p(const void * p)1415*404b540aSrobert ggc_marked_p (const void *p)
1416*404b540aSrobert {
1417*404b540aSrobert struct page_entry *page;
1418*404b540aSrobert const char *ptr = p;
1419*404b540aSrobert
1420*404b540aSrobert page = zone_get_object_page (p);
1421*404b540aSrobert
1422*404b540aSrobert if (page->pch_p)
1423*404b540aSrobert {
1424*404b540aSrobert size_t mark_word, mark_bit, offset;
1425*404b540aSrobert offset = (ptr - pch_zone.page) / BYTES_PER_MARK_BIT;
1426*404b540aSrobert mark_word = offset / (8 * sizeof (mark_type));
1427*404b540aSrobert mark_bit = offset % (8 * sizeof (mark_type));
1428*404b540aSrobert
1429*404b540aSrobert return (pch_zone.mark_bits[mark_word] & (1 << mark_bit)) != 0;
1430*404b540aSrobert }
1431*404b540aSrobert
1432*404b540aSrobert if (page->large_p)
1433*404b540aSrobert {
1434*404b540aSrobert struct large_page_entry *large_page
1435*404b540aSrobert = (struct large_page_entry *) page;
1436*404b540aSrobert
1437*404b540aSrobert return large_page->mark_p;
1438*404b540aSrobert }
1439*404b540aSrobert else
1440*404b540aSrobert {
1441*404b540aSrobert struct small_page_entry *small_page
1442*404b540aSrobert = (struct small_page_entry *) page;
1443*404b540aSrobert
1444*404b540aSrobert return 0 != (small_page->mark_bits[zone_get_object_mark_word (p)]
1445*404b540aSrobert & (1 << zone_get_object_mark_bit (p)));
1446*404b540aSrobert }
1447*404b540aSrobert }
1448*404b540aSrobert
1449*404b540aSrobert /* Return the size of the gc-able object P. */
1450*404b540aSrobert
1451*404b540aSrobert size_t
ggc_get_size(const void * p)1452*404b540aSrobert ggc_get_size (const void *p)
1453*404b540aSrobert {
1454*404b540aSrobert struct page_entry *page;
1455*404b540aSrobert const char *ptr = (const char *) p;
1456*404b540aSrobert
1457*404b540aSrobert page = zone_get_object_page (p);
1458*404b540aSrobert
1459*404b540aSrobert if (page->pch_p)
1460*404b540aSrobert {
1461*404b540aSrobert size_t alloc_word, alloc_bit, offset, max_size;
1462*404b540aSrobert offset = (ptr - pch_zone.page) / BYTES_PER_ALLOC_BIT + 1;
1463*404b540aSrobert alloc_word = offset / (8 * sizeof (alloc_type));
1464*404b540aSrobert alloc_bit = offset % (8 * sizeof (alloc_type));
1465*404b540aSrobert max_size = pch_zone.bytes - (ptr - pch_zone.page);
1466*404b540aSrobert return zone_object_size_1 (pch_zone.alloc_bits, alloc_word, alloc_bit,
1467*404b540aSrobert max_size);
1468*404b540aSrobert }
1469*404b540aSrobert
1470*404b540aSrobert if (page->large_p)
1471*404b540aSrobert return ((struct large_page_entry *)page)->bytes;
1472*404b540aSrobert else
1473*404b540aSrobert return zone_find_object_size ((struct small_page_entry *) page, p);
1474*404b540aSrobert }
1475*404b540aSrobert
1476*404b540aSrobert /* Initialize the ggc-zone-mmap allocator. */
1477*404b540aSrobert void
init_ggc(void)1478*404b540aSrobert init_ggc (void)
1479*404b540aSrobert {
1480*404b540aSrobert /* The allocation size must be greater than BYTES_PER_MARK_BIT, and
1481*404b540aSrobert a multiple of both BYTES_PER_ALLOC_BIT and FREE_BIN_DELTA, for
1482*404b540aSrobert the current assumptions to hold. */
1483*404b540aSrobert
1484*404b540aSrobert gcc_assert (FREE_BIN_DELTA == MAX_ALIGNMENT);
1485*404b540aSrobert
1486*404b540aSrobert /* Set up the main zone by hand. */
1487*404b540aSrobert main_zone.name = "Main zone";
1488*404b540aSrobert G.zones = &main_zone;
1489*404b540aSrobert
1490*404b540aSrobert /* Allocate the default zones. */
1491*404b540aSrobert new_ggc_zone_1 (&rtl_zone, "RTL zone");
1492*404b540aSrobert new_ggc_zone_1 (&tree_zone, "Tree zone");
1493*404b540aSrobert new_ggc_zone_1 (&tree_id_zone, "Tree identifier zone");
1494*404b540aSrobert
1495*404b540aSrobert G.pagesize = getpagesize();
1496*404b540aSrobert G.lg_pagesize = exact_log2 (G.pagesize);
1497*404b540aSrobert G.page_mask = ~(G.pagesize - 1);
1498*404b540aSrobert
1499*404b540aSrobert /* Require the system page size to be a multiple of GGC_PAGE_SIZE. */
1500*404b540aSrobert gcc_assert ((G.pagesize & (GGC_PAGE_SIZE - 1)) == 0);
1501*404b540aSrobert
1502*404b540aSrobert /* Allocate 16 system pages at a time. */
1503*404b540aSrobert G.quire_size = 16 * G.pagesize / GGC_PAGE_SIZE;
1504*404b540aSrobert
1505*404b540aSrobert /* Calculate the size of the allocation bitmap and other overhead. */
1506*404b540aSrobert /* Right now we allocate bits for the page header and bitmap. These
1507*404b540aSrobert are wasted, but a little tricky to eliminate. */
1508*404b540aSrobert G.small_page_overhead
1509*404b540aSrobert = PAGE_OVERHEAD + (GGC_PAGE_SIZE / BYTES_PER_ALLOC_BIT / 8);
1510*404b540aSrobert /* G.small_page_overhead = ROUND_UP (G.small_page_overhead, MAX_ALIGNMENT); */
1511*404b540aSrobert
1512*404b540aSrobert #ifdef HAVE_MMAP_DEV_ZERO
1513*404b540aSrobert G.dev_zero_fd = open ("/dev/zero", O_RDONLY);
1514*404b540aSrobert gcc_assert (G.dev_zero_fd != -1);
1515*404b540aSrobert #endif
1516*404b540aSrobert
1517*404b540aSrobert #if 0
1518*404b540aSrobert G.debug_file = fopen ("ggc-mmap.debug", "w");
1519*404b540aSrobert setlinebuf (G.debug_file);
1520*404b540aSrobert #else
1521*404b540aSrobert G.debug_file = stdout;
1522*404b540aSrobert #endif
1523*404b540aSrobert
1524*404b540aSrobert #ifdef USING_MMAP
1525*404b540aSrobert /* StunOS has an amazing off-by-one error for the first mmap allocation
1526*404b540aSrobert after fiddling with RLIMIT_STACK. The result, as hard as it is to
1527*404b540aSrobert believe, is an unaligned page allocation, which would cause us to
1528*404b540aSrobert hork badly if we tried to use it. */
1529*404b540aSrobert {
1530*404b540aSrobert char *p = alloc_anon (NULL, G.pagesize, &main_zone);
1531*404b540aSrobert struct small_page_entry *e;
1532*404b540aSrobert if ((size_t)p & (G.pagesize - 1))
1533*404b540aSrobert {
1534*404b540aSrobert /* How losing. Discard this one and try another. If we still
1535*404b540aSrobert can't get something useful, give up. */
1536*404b540aSrobert
1537*404b540aSrobert p = alloc_anon (NULL, G.pagesize, &main_zone);
1538*404b540aSrobert gcc_assert (!((size_t)p & (G.pagesize - 1)));
1539*404b540aSrobert }
1540*404b540aSrobert
1541*404b540aSrobert if (GGC_PAGE_SIZE == G.pagesize)
1542*404b540aSrobert {
1543*404b540aSrobert /* We have a good page, might as well hold onto it... */
1544*404b540aSrobert e = xcalloc (1, G.small_page_overhead);
1545*404b540aSrobert e->common.page = p;
1546*404b540aSrobert e->common.zone = &main_zone;
1547*404b540aSrobert e->next = main_zone.free_pages;
1548*404b540aSrobert set_page_table_entry (e->common.page, &e->common);
1549*404b540aSrobert main_zone.free_pages = e;
1550*404b540aSrobert }
1551*404b540aSrobert else
1552*404b540aSrobert {
1553*404b540aSrobert munmap (p, G.pagesize);
1554*404b540aSrobert }
1555*404b540aSrobert }
1556*404b540aSrobert #endif
1557*404b540aSrobert }
1558*404b540aSrobert
1559*404b540aSrobert /* Start a new GGC zone. */
1560*404b540aSrobert
1561*404b540aSrobert static void
new_ggc_zone_1(struct alloc_zone * new_zone,const char * name)1562*404b540aSrobert new_ggc_zone_1 (struct alloc_zone *new_zone, const char * name)
1563*404b540aSrobert {
1564*404b540aSrobert new_zone->name = name;
1565*404b540aSrobert new_zone->next_zone = G.zones->next_zone;
1566*404b540aSrobert G.zones->next_zone = new_zone;
1567*404b540aSrobert }
1568*404b540aSrobert
1569*404b540aSrobert struct alloc_zone *
new_ggc_zone(const char * name)1570*404b540aSrobert new_ggc_zone (const char * name)
1571*404b540aSrobert {
1572*404b540aSrobert struct alloc_zone *new_zone = xcalloc (1, sizeof (struct alloc_zone));
1573*404b540aSrobert new_ggc_zone_1 (new_zone, name);
1574*404b540aSrobert return new_zone;
1575*404b540aSrobert }
1576*404b540aSrobert
1577*404b540aSrobert /* Destroy a GGC zone. */
1578*404b540aSrobert void
destroy_ggc_zone(struct alloc_zone * dead_zone)1579*404b540aSrobert destroy_ggc_zone (struct alloc_zone * dead_zone)
1580*404b540aSrobert {
1581*404b540aSrobert struct alloc_zone *z;
1582*404b540aSrobert
1583*404b540aSrobert for (z = G.zones; z && z->next_zone != dead_zone; z = z->next_zone)
1584*404b540aSrobert /* Just find that zone. */
1585*404b540aSrobert continue;
1586*404b540aSrobert
1587*404b540aSrobert /* We should have found the zone in the list. Anything else is fatal. */
1588*404b540aSrobert gcc_assert (z);
1589*404b540aSrobert
1590*404b540aSrobert /* z is dead, baby. z is dead. */
1591*404b540aSrobert z->dead = true;
1592*404b540aSrobert }
1593*404b540aSrobert
1594*404b540aSrobert /* Free all empty pages and objects within a page for a given zone */
1595*404b540aSrobert
1596*404b540aSrobert static void
sweep_pages(struct alloc_zone * zone)1597*404b540aSrobert sweep_pages (struct alloc_zone *zone)
1598*404b540aSrobert {
1599*404b540aSrobert struct large_page_entry **lpp, *lp, *lnext;
1600*404b540aSrobert struct small_page_entry **spp, *sp, *snext;
1601*404b540aSrobert char *last_free;
1602*404b540aSrobert size_t allocated = 0;
1603*404b540aSrobert bool nomarksinpage;
1604*404b540aSrobert
1605*404b540aSrobert /* First, reset the free_chunks lists, since we are going to
1606*404b540aSrobert re-free free chunks in hopes of coalescing them into large chunks. */
1607*404b540aSrobert memset (zone->free_chunks, 0, sizeof (zone->free_chunks));
1608*404b540aSrobert zone->high_free_bin = 0;
1609*404b540aSrobert zone->cached_free = NULL;
1610*404b540aSrobert zone->cached_free_size = 0;
1611*404b540aSrobert
1612*404b540aSrobert /* Large pages are all or none affairs. Either they are completely
1613*404b540aSrobert empty, or they are completely full. */
1614*404b540aSrobert lpp = &zone->large_pages;
1615*404b540aSrobert for (lp = zone->large_pages; lp != NULL; lp = lnext)
1616*404b540aSrobert {
1617*404b540aSrobert gcc_assert (lp->common.large_p);
1618*404b540aSrobert
1619*404b540aSrobert lnext = lp->next;
1620*404b540aSrobert
1621*404b540aSrobert #ifdef GATHER_STATISTICS
1622*404b540aSrobert /* This page has now survived another collection. */
1623*404b540aSrobert lp->common.survived++;
1624*404b540aSrobert #endif
1625*404b540aSrobert
1626*404b540aSrobert if (lp->mark_p)
1627*404b540aSrobert {
1628*404b540aSrobert lp->mark_p = false;
1629*404b540aSrobert allocated += lp->bytes;
1630*404b540aSrobert lpp = &lp->next;
1631*404b540aSrobert }
1632*404b540aSrobert else
1633*404b540aSrobert {
1634*404b540aSrobert *lpp = lnext;
1635*404b540aSrobert #ifdef ENABLE_GC_CHECKING
1636*404b540aSrobert /* Poison the page. */
1637*404b540aSrobert memset (lp->common.page, 0xb5, SMALL_PAGE_SIZE);
1638*404b540aSrobert #endif
1639*404b540aSrobert if (lp->prev)
1640*404b540aSrobert lp->prev->next = lp->next;
1641*404b540aSrobert if (lp->next)
1642*404b540aSrobert lp->next->prev = lp->prev;
1643*404b540aSrobert free_large_page (lp);
1644*404b540aSrobert }
1645*404b540aSrobert }
1646*404b540aSrobert
1647*404b540aSrobert spp = &zone->pages;
1648*404b540aSrobert for (sp = zone->pages; sp != NULL; sp = snext)
1649*404b540aSrobert {
1650*404b540aSrobert char *object, *last_object;
1651*404b540aSrobert char *end;
1652*404b540aSrobert alloc_type *alloc_word_p;
1653*404b540aSrobert mark_type *mark_word_p;
1654*404b540aSrobert
1655*404b540aSrobert gcc_assert (!sp->common.large_p);
1656*404b540aSrobert
1657*404b540aSrobert snext = sp->next;
1658*404b540aSrobert
1659*404b540aSrobert #ifdef GATHER_STATISTICS
1660*404b540aSrobert /* This page has now survived another collection. */
1661*404b540aSrobert sp->common.survived++;
1662*404b540aSrobert #endif
1663*404b540aSrobert
1664*404b540aSrobert /* Step through all chunks, consolidate those that are free and
1665*404b540aSrobert insert them into the free lists. Note that consolidation
1666*404b540aSrobert slows down collection slightly. */
1667*404b540aSrobert
1668*404b540aSrobert last_object = object = sp->common.page;
1669*404b540aSrobert end = sp->common.page + SMALL_PAGE_SIZE;
1670*404b540aSrobert last_free = NULL;
1671*404b540aSrobert nomarksinpage = true;
1672*404b540aSrobert mark_word_p = sp->mark_bits;
1673*404b540aSrobert alloc_word_p = sp->alloc_bits;
1674*404b540aSrobert
1675*404b540aSrobert gcc_assert (BYTES_PER_ALLOC_BIT == BYTES_PER_MARK_BIT);
1676*404b540aSrobert
1677*404b540aSrobert object = sp->common.page;
1678*404b540aSrobert do
1679*404b540aSrobert {
1680*404b540aSrobert unsigned int i, n;
1681*404b540aSrobert alloc_type alloc_word;
1682*404b540aSrobert mark_type mark_word;
1683*404b540aSrobert
1684*404b540aSrobert alloc_word = *alloc_word_p++;
1685*404b540aSrobert mark_word = *mark_word_p++;
1686*404b540aSrobert
1687*404b540aSrobert if (mark_word)
1688*404b540aSrobert nomarksinpage = false;
1689*404b540aSrobert
1690*404b540aSrobert /* There ought to be some way to do this without looping... */
1691*404b540aSrobert i = 0;
1692*404b540aSrobert while ((n = alloc_ffs (alloc_word)) != 0)
1693*404b540aSrobert {
1694*404b540aSrobert /* Extend the current state for n - 1 bits. We can't
1695*404b540aSrobert shift alloc_word by n, even though it isn't used in the
1696*404b540aSrobert loop, in case only the highest bit was set. */
1697*404b540aSrobert alloc_word >>= n - 1;
1698*404b540aSrobert mark_word >>= n - 1;
1699*404b540aSrobert object += BYTES_PER_MARK_BIT * (n - 1);
1700*404b540aSrobert
1701*404b540aSrobert if (mark_word & 1)
1702*404b540aSrobert {
1703*404b540aSrobert if (last_free)
1704*404b540aSrobert {
1705*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (last_free,
1706*404b540aSrobert object
1707*404b540aSrobert - last_free));
1708*404b540aSrobert poison_region (last_free, object - last_free);
1709*404b540aSrobert free_chunk (last_free, object - last_free, zone);
1710*404b540aSrobert last_free = NULL;
1711*404b540aSrobert }
1712*404b540aSrobert else
1713*404b540aSrobert allocated += object - last_object;
1714*404b540aSrobert last_object = object;
1715*404b540aSrobert }
1716*404b540aSrobert else
1717*404b540aSrobert {
1718*404b540aSrobert if (last_free == NULL)
1719*404b540aSrobert {
1720*404b540aSrobert last_free = object;
1721*404b540aSrobert allocated += object - last_object;
1722*404b540aSrobert }
1723*404b540aSrobert else
1724*404b540aSrobert zone_clear_object_alloc_bit (sp, object);
1725*404b540aSrobert }
1726*404b540aSrobert
1727*404b540aSrobert /* Shift to just after the alloc bit we handled. */
1728*404b540aSrobert alloc_word >>= 1;
1729*404b540aSrobert mark_word >>= 1;
1730*404b540aSrobert object += BYTES_PER_MARK_BIT;
1731*404b540aSrobert
1732*404b540aSrobert i += n;
1733*404b540aSrobert }
1734*404b540aSrobert
1735*404b540aSrobert object += BYTES_PER_MARK_BIT * (8 * sizeof (alloc_type) - i);
1736*404b540aSrobert }
1737*404b540aSrobert while (object < end);
1738*404b540aSrobert
1739*404b540aSrobert if (nomarksinpage)
1740*404b540aSrobert {
1741*404b540aSrobert *spp = snext;
1742*404b540aSrobert #ifdef ENABLE_GC_CHECKING
1743*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (sp->common.page, SMALL_PAGE_SIZE));
1744*404b540aSrobert /* Poison the page. */
1745*404b540aSrobert memset (sp->common.page, 0xb5, SMALL_PAGE_SIZE);
1746*404b540aSrobert #endif
1747*404b540aSrobert free_small_page (sp);
1748*404b540aSrobert continue;
1749*404b540aSrobert }
1750*404b540aSrobert else if (last_free)
1751*404b540aSrobert {
1752*404b540aSrobert VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (last_free,
1753*404b540aSrobert object - last_free));
1754*404b540aSrobert poison_region (last_free, object - last_free);
1755*404b540aSrobert free_chunk (last_free, object - last_free, zone);
1756*404b540aSrobert }
1757*404b540aSrobert else
1758*404b540aSrobert allocated += object - last_object;
1759*404b540aSrobert
1760*404b540aSrobert spp = &sp->next;
1761*404b540aSrobert }
1762*404b540aSrobert
1763*404b540aSrobert zone->allocated = allocated;
1764*404b540aSrobert }
1765*404b540aSrobert
1766*404b540aSrobert /* mark-and-sweep routine for collecting a single zone. NEED_MARKING
1767*404b540aSrobert is true if we need to mark before sweeping, false if some other
1768*404b540aSrobert zone collection has already performed marking for us. Returns true
1769*404b540aSrobert if we collected, false otherwise. */
1770*404b540aSrobert
1771*404b540aSrobert static bool
ggc_collect_1(struct alloc_zone * zone,bool need_marking)1772*404b540aSrobert ggc_collect_1 (struct alloc_zone *zone, bool need_marking)
1773*404b540aSrobert {
1774*404b540aSrobert #if 0
1775*404b540aSrobert /* */
1776*404b540aSrobert {
1777*404b540aSrobert int i;
1778*404b540aSrobert for (i = 0; i < NUM_FREE_BINS + 1; i++)
1779*404b540aSrobert {
1780*404b540aSrobert struct alloc_chunk *chunk;
1781*404b540aSrobert int n, tot;
1782*404b540aSrobert
1783*404b540aSrobert n = 0;
1784*404b540aSrobert tot = 0;
1785*404b540aSrobert chunk = zone->free_chunks[i];
1786*404b540aSrobert while (chunk)
1787*404b540aSrobert {
1788*404b540aSrobert n++;
1789*404b540aSrobert tot += chunk->size;
1790*404b540aSrobert chunk = chunk->next_free;
1791*404b540aSrobert }
1792*404b540aSrobert fprintf (stderr, "Bin %d: %d free chunks (%d bytes)\n",
1793*404b540aSrobert i, n, tot);
1794*404b540aSrobert }
1795*404b540aSrobert }
1796*404b540aSrobert /* */
1797*404b540aSrobert #endif
1798*404b540aSrobert
1799*404b540aSrobert if (!quiet_flag)
1800*404b540aSrobert fprintf (stderr, " {%s GC %luk -> ",
1801*404b540aSrobert zone->name, (unsigned long) zone->allocated / 1024);
1802*404b540aSrobert
1803*404b540aSrobert /* Zero the total allocated bytes. This will be recalculated in the
1804*404b540aSrobert sweep phase. */
1805*404b540aSrobert zone->allocated = 0;
1806*404b540aSrobert
1807*404b540aSrobert /* Release the pages we freed the last time we collected, but didn't
1808*404b540aSrobert reuse in the interim. */
1809*404b540aSrobert release_pages (zone);
1810*404b540aSrobert
1811*404b540aSrobert if (need_marking)
1812*404b540aSrobert {
1813*404b540aSrobert zone_allocate_marks ();
1814*404b540aSrobert ggc_mark_roots ();
1815*404b540aSrobert #ifdef GATHER_STATISTICS
1816*404b540aSrobert ggc_prune_overhead_list ();
1817*404b540aSrobert #endif
1818*404b540aSrobert }
1819*404b540aSrobert
1820*404b540aSrobert sweep_pages (zone);
1821*404b540aSrobert zone->was_collected = true;
1822*404b540aSrobert zone->allocated_last_gc = zone->allocated;
1823*404b540aSrobert
1824*404b540aSrobert if (!quiet_flag)
1825*404b540aSrobert fprintf (stderr, "%luk}", (unsigned long) zone->allocated / 1024);
1826*404b540aSrobert return true;
1827*404b540aSrobert }
1828*404b540aSrobert
1829*404b540aSrobert #ifdef GATHER_STATISTICS
1830*404b540aSrobert /* Calculate the average page survival rate in terms of number of
1831*404b540aSrobert collections. */
1832*404b540aSrobert
1833*404b540aSrobert static float
calculate_average_page_survival(struct alloc_zone * zone)1834*404b540aSrobert calculate_average_page_survival (struct alloc_zone *zone)
1835*404b540aSrobert {
1836*404b540aSrobert float count = 0.0;
1837*404b540aSrobert float survival = 0.0;
1838*404b540aSrobert struct small_page_entry *p;
1839*404b540aSrobert struct large_page_entry *lp;
1840*404b540aSrobert for (p = zone->pages; p; p = p->next)
1841*404b540aSrobert {
1842*404b540aSrobert count += 1.0;
1843*404b540aSrobert survival += p->common.survived;
1844*404b540aSrobert }
1845*404b540aSrobert for (lp = zone->large_pages; lp; lp = lp->next)
1846*404b540aSrobert {
1847*404b540aSrobert count += 1.0;
1848*404b540aSrobert survival += lp->common.survived;
1849*404b540aSrobert }
1850*404b540aSrobert return survival/count;
1851*404b540aSrobert }
1852*404b540aSrobert #endif
1853*404b540aSrobert
1854*404b540aSrobert /* Top level collection routine. */
1855*404b540aSrobert
1856*404b540aSrobert void
ggc_collect(void)1857*404b540aSrobert ggc_collect (void)
1858*404b540aSrobert {
1859*404b540aSrobert struct alloc_zone *zone;
1860*404b540aSrobert bool marked = false;
1861*404b540aSrobert
1862*404b540aSrobert timevar_push (TV_GC);
1863*404b540aSrobert
1864*404b540aSrobert if (!ggc_force_collect)
1865*404b540aSrobert {
1866*404b540aSrobert float allocated_last_gc = 0, allocated = 0, min_expand;
1867*404b540aSrobert
1868*404b540aSrobert for (zone = G.zones; zone; zone = zone->next_zone)
1869*404b540aSrobert {
1870*404b540aSrobert allocated_last_gc += zone->allocated_last_gc;
1871*404b540aSrobert allocated += zone->allocated;
1872*404b540aSrobert }
1873*404b540aSrobert
1874*404b540aSrobert allocated_last_gc =
1875*404b540aSrobert MAX (allocated_last_gc,
1876*404b540aSrobert (size_t) PARAM_VALUE (GGC_MIN_HEAPSIZE) * 1024);
1877*404b540aSrobert min_expand = allocated_last_gc * PARAM_VALUE (GGC_MIN_EXPAND) / 100;
1878*404b540aSrobert
1879*404b540aSrobert if (allocated < allocated_last_gc + min_expand)
1880*404b540aSrobert {
1881*404b540aSrobert timevar_pop (TV_GC);
1882*404b540aSrobert return;
1883*404b540aSrobert }
1884*404b540aSrobert }
1885*404b540aSrobert
1886*404b540aSrobert /* Start by possibly collecting the main zone. */
1887*404b540aSrobert main_zone.was_collected = false;
1888*404b540aSrobert marked |= ggc_collect_1 (&main_zone, true);
1889*404b540aSrobert
1890*404b540aSrobert /* In order to keep the number of collections down, we don't
1891*404b540aSrobert collect other zones unless we are collecting the main zone. This
1892*404b540aSrobert gives us roughly the same number of collections as we used to
1893*404b540aSrobert have with the old gc. The number of collection is important
1894*404b540aSrobert because our main slowdown (according to profiling) is now in
1895*404b540aSrobert marking. So if we mark twice as often as we used to, we'll be
1896*404b540aSrobert twice as slow. Hopefully we'll avoid this cost when we mark
1897*404b540aSrobert zone-at-a-time. */
1898*404b540aSrobert /* NOTE drow/2004-07-28: We now always collect the main zone, but
1899*404b540aSrobert keep this code in case the heuristics are further refined. */
1900*404b540aSrobert
1901*404b540aSrobert if (main_zone.was_collected)
1902*404b540aSrobert {
1903*404b540aSrobert struct alloc_zone *zone;
1904*404b540aSrobert
1905*404b540aSrobert for (zone = main_zone.next_zone; zone; zone = zone->next_zone)
1906*404b540aSrobert {
1907*404b540aSrobert zone->was_collected = false;
1908*404b540aSrobert marked |= ggc_collect_1 (zone, !marked);
1909*404b540aSrobert }
1910*404b540aSrobert }
1911*404b540aSrobert
1912*404b540aSrobert #ifdef GATHER_STATISTICS
1913*404b540aSrobert /* Print page survival stats, if someone wants them. */
1914*404b540aSrobert if (GGC_DEBUG_LEVEL >= 2)
1915*404b540aSrobert {
1916*404b540aSrobert for (zone = G.zones; zone; zone = zone->next_zone)
1917*404b540aSrobert {
1918*404b540aSrobert if (zone->was_collected)
1919*404b540aSrobert {
1920*404b540aSrobert float f = calculate_average_page_survival (zone);
1921*404b540aSrobert printf ("Average page survival in zone `%s' is %f\n",
1922*404b540aSrobert zone->name, f);
1923*404b540aSrobert }
1924*404b540aSrobert }
1925*404b540aSrobert }
1926*404b540aSrobert #endif
1927*404b540aSrobert
1928*404b540aSrobert if (marked)
1929*404b540aSrobert zone_free_marks ();
1930*404b540aSrobert
1931*404b540aSrobert /* Free dead zones. */
1932*404b540aSrobert for (zone = G.zones; zone && zone->next_zone; zone = zone->next_zone)
1933*404b540aSrobert {
1934*404b540aSrobert if (zone->next_zone->dead)
1935*404b540aSrobert {
1936*404b540aSrobert struct alloc_zone *dead_zone = zone->next_zone;
1937*404b540aSrobert
1938*404b540aSrobert printf ("Zone `%s' is dead and will be freed.\n", dead_zone->name);
1939*404b540aSrobert
1940*404b540aSrobert /* The zone must be empty. */
1941*404b540aSrobert gcc_assert (!dead_zone->allocated);
1942*404b540aSrobert
1943*404b540aSrobert /* Unchain the dead zone, release all its pages and free it. */
1944*404b540aSrobert zone->next_zone = zone->next_zone->next_zone;
1945*404b540aSrobert release_pages (dead_zone);
1946*404b540aSrobert free (dead_zone);
1947*404b540aSrobert }
1948*404b540aSrobert }
1949*404b540aSrobert
1950*404b540aSrobert timevar_pop (TV_GC);
1951*404b540aSrobert }
1952*404b540aSrobert
1953*404b540aSrobert /* Print allocation statistics. */
1954*404b540aSrobert #define SCALE(x) ((unsigned long) ((x) < 1024*10 \
1955*404b540aSrobert ? (x) \
1956*404b540aSrobert : ((x) < 1024*1024*10 \
1957*404b540aSrobert ? (x) / 1024 \
1958*404b540aSrobert : (x) / (1024*1024))))
1959*404b540aSrobert #define LABEL(x) ((x) < 1024*10 ? ' ' : ((x) < 1024*1024*10 ? 'k' : 'M'))
1960*404b540aSrobert
1961*404b540aSrobert void
ggc_print_statistics(void)1962*404b540aSrobert ggc_print_statistics (void)
1963*404b540aSrobert {
1964*404b540aSrobert struct alloc_zone *zone;
1965*404b540aSrobert struct ggc_statistics stats;
1966*404b540aSrobert size_t total_overhead = 0, total_allocated = 0, total_bytes_mapped = 0;
1967*404b540aSrobert size_t pte_overhead, i;
1968*404b540aSrobert
1969*404b540aSrobert /* Clear the statistics. */
1970*404b540aSrobert memset (&stats, 0, sizeof (stats));
1971*404b540aSrobert
1972*404b540aSrobert /* Make sure collection will really occur. */
1973*404b540aSrobert ggc_force_collect = true;
1974*404b540aSrobert
1975*404b540aSrobert /* Collect and print the statistics common across collectors. */
1976*404b540aSrobert ggc_print_common_statistics (stderr, &stats);
1977*404b540aSrobert
1978*404b540aSrobert ggc_force_collect = false;
1979*404b540aSrobert
1980*404b540aSrobert /* Release free pages so that we will not count the bytes allocated
1981*404b540aSrobert there as part of the total allocated memory. */
1982*404b540aSrobert for (zone = G.zones; zone; zone = zone->next_zone)
1983*404b540aSrobert release_pages (zone);
1984*404b540aSrobert
1985*404b540aSrobert /* Collect some information about the various sizes of
1986*404b540aSrobert allocation. */
1987*404b540aSrobert fprintf (stderr,
1988*404b540aSrobert "Memory still allocated at the end of the compilation process\n");
1989*404b540aSrobert
1990*404b540aSrobert fprintf (stderr, "%20s %10s %10s %10s\n",
1991*404b540aSrobert "Zone", "Allocated", "Used", "Overhead");
1992*404b540aSrobert for (zone = G.zones; zone; zone = zone->next_zone)
1993*404b540aSrobert {
1994*404b540aSrobert struct large_page_entry *large_page;
1995*404b540aSrobert size_t overhead, allocated, in_use;
1996*404b540aSrobert
1997*404b540aSrobert /* Skip empty zones. */
1998*404b540aSrobert if (!zone->pages && !zone->large_pages)
1999*404b540aSrobert continue;
2000*404b540aSrobert
2001*404b540aSrobert allocated = in_use = 0;
2002*404b540aSrobert
2003*404b540aSrobert overhead = sizeof (struct alloc_zone);
2004*404b540aSrobert
2005*404b540aSrobert for (large_page = zone->large_pages; large_page != NULL;
2006*404b540aSrobert large_page = large_page->next)
2007*404b540aSrobert {
2008*404b540aSrobert allocated += large_page->bytes;
2009*404b540aSrobert in_use += large_page->bytes;
2010*404b540aSrobert overhead += sizeof (struct large_page_entry);
2011*404b540aSrobert }
2012*404b540aSrobert
2013*404b540aSrobert /* There's no easy way to walk through the small pages finding
2014*404b540aSrobert used and unused objects. Instead, add all the pages, and
2015*404b540aSrobert subtract out the free list. */
2016*404b540aSrobert
2017*404b540aSrobert allocated += GGC_PAGE_SIZE * zone->n_small_pages;
2018*404b540aSrobert in_use += GGC_PAGE_SIZE * zone->n_small_pages;
2019*404b540aSrobert overhead += G.small_page_overhead * zone->n_small_pages;
2020*404b540aSrobert
2021*404b540aSrobert for (i = 0; i <= NUM_FREE_BINS; i++)
2022*404b540aSrobert {
2023*404b540aSrobert struct alloc_chunk *chunk = zone->free_chunks[i];
2024*404b540aSrobert while (chunk)
2025*404b540aSrobert {
2026*404b540aSrobert in_use -= ggc_get_size (chunk);
2027*404b540aSrobert chunk = chunk->next_free;
2028*404b540aSrobert }
2029*404b540aSrobert }
2030*404b540aSrobert
2031*404b540aSrobert fprintf (stderr, "%20s %10lu%c %10lu%c %10lu%c\n",
2032*404b540aSrobert zone->name,
2033*404b540aSrobert SCALE (allocated), LABEL (allocated),
2034*404b540aSrobert SCALE (in_use), LABEL (in_use),
2035*404b540aSrobert SCALE (overhead), LABEL (overhead));
2036*404b540aSrobert
2037*404b540aSrobert gcc_assert (in_use == zone->allocated);
2038*404b540aSrobert
2039*404b540aSrobert total_overhead += overhead;
2040*404b540aSrobert total_allocated += zone->allocated;
2041*404b540aSrobert total_bytes_mapped += zone->bytes_mapped;
2042*404b540aSrobert }
2043*404b540aSrobert
2044*404b540aSrobert /* Count the size of the page table as best we can. */
2045*404b540aSrobert #if HOST_BITS_PER_PTR <= 32
2046*404b540aSrobert pte_overhead = sizeof (G.lookup);
2047*404b540aSrobert for (i = 0; i < PAGE_L1_SIZE; i++)
2048*404b540aSrobert if (G.lookup[i])
2049*404b540aSrobert pte_overhead += PAGE_L2_SIZE * sizeof (struct page_entry *);
2050*404b540aSrobert #else
2051*404b540aSrobert {
2052*404b540aSrobert page_table table = G.lookup;
2053*404b540aSrobert pte_overhead = 0;
2054*404b540aSrobert while (table)
2055*404b540aSrobert {
2056*404b540aSrobert pte_overhead += sizeof (*table);
2057*404b540aSrobert for (i = 0; i < PAGE_L1_SIZE; i++)
2058*404b540aSrobert if (table->table[i])
2059*404b540aSrobert pte_overhead += PAGE_L2_SIZE * sizeof (struct page_entry *);
2060*404b540aSrobert table = table->next;
2061*404b540aSrobert }
2062*404b540aSrobert }
2063*404b540aSrobert #endif
2064*404b540aSrobert fprintf (stderr, "%20s %11s %11s %10lu%c\n", "Page Table",
2065*404b540aSrobert "", "", SCALE (pte_overhead), LABEL (pte_overhead));
2066*404b540aSrobert total_overhead += pte_overhead;
2067*404b540aSrobert
2068*404b540aSrobert fprintf (stderr, "%20s %10lu%c %10lu%c %10lu%c\n", "Total",
2069*404b540aSrobert SCALE (total_bytes_mapped), LABEL (total_bytes_mapped),
2070*404b540aSrobert SCALE (total_allocated), LABEL(total_allocated),
2071*404b540aSrobert SCALE (total_overhead), LABEL (total_overhead));
2072*404b540aSrobert
2073*404b540aSrobert #ifdef GATHER_STATISTICS
2074*404b540aSrobert {
2075*404b540aSrobert unsigned long long all_overhead = 0, all_allocated = 0;
2076*404b540aSrobert unsigned long long all_overhead_under32 = 0, all_allocated_under32 = 0;
2077*404b540aSrobert unsigned long long all_overhead_under64 = 0, all_allocated_under64 = 0;
2078*404b540aSrobert unsigned long long all_overhead_under128 = 0, all_allocated_under128 = 0;
2079*404b540aSrobert
2080*404b540aSrobert fprintf (stderr, "\nTotal allocations and overheads during the compilation process\n");
2081*404b540aSrobert
2082*404b540aSrobert for (zone = G.zones; zone; zone = zone->next_zone)
2083*404b540aSrobert {
2084*404b540aSrobert all_overhead += zone->stats.total_overhead;
2085*404b540aSrobert all_allocated += zone->stats.total_allocated;
2086*404b540aSrobert
2087*404b540aSrobert all_allocated_under32 += zone->stats.total_allocated_under32;
2088*404b540aSrobert all_overhead_under32 += zone->stats.total_overhead_under32;
2089*404b540aSrobert
2090*404b540aSrobert all_allocated_under64 += zone->stats.total_allocated_under64;
2091*404b540aSrobert all_overhead_under64 += zone->stats.total_overhead_under64;
2092*404b540aSrobert
2093*404b540aSrobert all_allocated_under128 += zone->stats.total_allocated_under128;
2094*404b540aSrobert all_overhead_under128 += zone->stats.total_overhead_under128;
2095*404b540aSrobert
2096*404b540aSrobert fprintf (stderr, "%20s: %10lld\n",
2097*404b540aSrobert zone->name, zone->stats.total_allocated);
2098*404b540aSrobert }
2099*404b540aSrobert
2100*404b540aSrobert fprintf (stderr, "\n");
2101*404b540aSrobert
2102*404b540aSrobert fprintf (stderr, "Total Overhead: %10lld\n",
2103*404b540aSrobert all_overhead);
2104*404b540aSrobert fprintf (stderr, "Total Allocated: %10lld\n",
2105*404b540aSrobert all_allocated);
2106*404b540aSrobert
2107*404b540aSrobert fprintf (stderr, "Total Overhead under 32B: %10lld\n",
2108*404b540aSrobert all_overhead_under32);
2109*404b540aSrobert fprintf (stderr, "Total Allocated under 32B: %10lld\n",
2110*404b540aSrobert all_allocated_under32);
2111*404b540aSrobert fprintf (stderr, "Total Overhead under 64B: %10lld\n",
2112*404b540aSrobert all_overhead_under64);
2113*404b540aSrobert fprintf (stderr, "Total Allocated under 64B: %10lld\n",
2114*404b540aSrobert all_allocated_under64);
2115*404b540aSrobert fprintf (stderr, "Total Overhead under 128B: %10lld\n",
2116*404b540aSrobert all_overhead_under128);
2117*404b540aSrobert fprintf (stderr, "Total Allocated under 128B: %10lld\n",
2118*404b540aSrobert all_allocated_under128);
2119*404b540aSrobert }
2120*404b540aSrobert #endif
2121*404b540aSrobert }
2122*404b540aSrobert
2123*404b540aSrobert /* Precompiled header support. */
2124*404b540aSrobert
2125*404b540aSrobert /* For precompiled headers, we sort objects based on their type. We
2126*404b540aSrobert also sort various objects into their own buckets; currently this
2127*404b540aSrobert covers strings and IDENTIFIER_NODE trees. The choices of how
2128*404b540aSrobert to sort buckets have not yet been tuned. */
2129*404b540aSrobert
2130*404b540aSrobert #define NUM_PCH_BUCKETS (gt_types_enum_last + 3)
2131*404b540aSrobert
2132*404b540aSrobert #define OTHER_BUCKET (gt_types_enum_last + 0)
2133*404b540aSrobert #define IDENTIFIER_BUCKET (gt_types_enum_last + 1)
2134*404b540aSrobert #define STRING_BUCKET (gt_types_enum_last + 2)
2135*404b540aSrobert
2136*404b540aSrobert struct ggc_pch_ondisk
2137*404b540aSrobert {
2138*404b540aSrobert size_t total;
2139*404b540aSrobert size_t type_totals[NUM_PCH_BUCKETS];
2140*404b540aSrobert };
2141*404b540aSrobert
2142*404b540aSrobert struct ggc_pch_data
2143*404b540aSrobert {
2144*404b540aSrobert struct ggc_pch_ondisk d;
2145*404b540aSrobert size_t base;
2146*404b540aSrobert size_t orig_base;
2147*404b540aSrobert size_t alloc_size;
2148*404b540aSrobert alloc_type *alloc_bits;
2149*404b540aSrobert size_t type_bases[NUM_PCH_BUCKETS];
2150*404b540aSrobert size_t start_offset;
2151*404b540aSrobert };
2152*404b540aSrobert
2153*404b540aSrobert /* Initialize the PCH data structure. */
2154*404b540aSrobert
2155*404b540aSrobert struct ggc_pch_data *
init_ggc_pch(void)2156*404b540aSrobert init_ggc_pch (void)
2157*404b540aSrobert {
2158*404b540aSrobert return xcalloc (sizeof (struct ggc_pch_data), 1);
2159*404b540aSrobert }
2160*404b540aSrobert
2161*404b540aSrobert /* Return which of the page-aligned buckets the object at X, with type
2162*404b540aSrobert TYPE, should be sorted into in the PCH. Strings will have
2163*404b540aSrobert IS_STRING set and TYPE will be gt_types_enum_last. Other objects
2164*404b540aSrobert of unknown type will also have TYPE equal to gt_types_enum_last. */
2165*404b540aSrobert
2166*404b540aSrobert static int
pch_bucket(void * x,enum gt_types_enum type,bool is_string)2167*404b540aSrobert pch_bucket (void *x, enum gt_types_enum type,
2168*404b540aSrobert bool is_string)
2169*404b540aSrobert {
2170*404b540aSrobert /* Sort identifiers into their own bucket, to improve locality
2171*404b540aSrobert when searching the identifier hash table. */
2172*404b540aSrobert if (type == gt_ggc_e_14lang_tree_node
2173*404b540aSrobert && TREE_CODE ((tree) x) == IDENTIFIER_NODE)
2174*404b540aSrobert return IDENTIFIER_BUCKET;
2175*404b540aSrobert else if (type == gt_types_enum_last)
2176*404b540aSrobert {
2177*404b540aSrobert if (is_string)
2178*404b540aSrobert return STRING_BUCKET;
2179*404b540aSrobert return OTHER_BUCKET;
2180*404b540aSrobert }
2181*404b540aSrobert return type;
2182*404b540aSrobert }
2183*404b540aSrobert
2184*404b540aSrobert /* Add the size of object X to the size of the PCH data. */
2185*404b540aSrobert
2186*404b540aSrobert void
ggc_pch_count_object(struct ggc_pch_data * d,void * x ATTRIBUTE_UNUSED,size_t size,bool is_string,enum gt_types_enum type)2187*404b540aSrobert ggc_pch_count_object (struct ggc_pch_data *d, void *x ATTRIBUTE_UNUSED,
2188*404b540aSrobert size_t size, bool is_string, enum gt_types_enum type)
2189*404b540aSrobert {
2190*404b540aSrobert /* NOTE: Right now we don't need to align up the size of any objects.
2191*404b540aSrobert Strings can be unaligned, and everything else is allocated to a
2192*404b540aSrobert MAX_ALIGNMENT boundary already. */
2193*404b540aSrobert
2194*404b540aSrobert d->d.type_totals[pch_bucket (x, type, is_string)] += size;
2195*404b540aSrobert }
2196*404b540aSrobert
2197*404b540aSrobert /* Return the total size of the PCH data. */
2198*404b540aSrobert
2199*404b540aSrobert size_t
ggc_pch_total_size(struct ggc_pch_data * d)2200*404b540aSrobert ggc_pch_total_size (struct ggc_pch_data *d)
2201*404b540aSrobert {
2202*404b540aSrobert enum gt_types_enum i;
2203*404b540aSrobert size_t alloc_size, total_size;
2204*404b540aSrobert
2205*404b540aSrobert total_size = 0;
2206*404b540aSrobert for (i = 0; i < NUM_PCH_BUCKETS; i++)
2207*404b540aSrobert {
2208*404b540aSrobert d->d.type_totals[i] = ROUND_UP (d->d.type_totals[i], GGC_PAGE_SIZE);
2209*404b540aSrobert total_size += d->d.type_totals[i];
2210*404b540aSrobert }
2211*404b540aSrobert d->d.total = total_size;
2212*404b540aSrobert
2213*404b540aSrobert /* Include the size of the allocation bitmap. */
2214*404b540aSrobert alloc_size = CEIL (d->d.total, BYTES_PER_ALLOC_BIT * 8);
2215*404b540aSrobert alloc_size = ROUND_UP (alloc_size, MAX_ALIGNMENT);
2216*404b540aSrobert d->alloc_size = alloc_size;
2217*404b540aSrobert
2218*404b540aSrobert return d->d.total + alloc_size;
2219*404b540aSrobert }
2220*404b540aSrobert
2221*404b540aSrobert /* Set the base address for the objects in the PCH file. */
2222*404b540aSrobert
2223*404b540aSrobert void
ggc_pch_this_base(struct ggc_pch_data * d,void * base_)2224*404b540aSrobert ggc_pch_this_base (struct ggc_pch_data *d, void *base_)
2225*404b540aSrobert {
2226*404b540aSrobert int i;
2227*404b540aSrobert size_t base = (size_t) base_;
2228*404b540aSrobert
2229*404b540aSrobert d->base = d->orig_base = base;
2230*404b540aSrobert for (i = 0; i < NUM_PCH_BUCKETS; i++)
2231*404b540aSrobert {
2232*404b540aSrobert d->type_bases[i] = base;
2233*404b540aSrobert base += d->d.type_totals[i];
2234*404b540aSrobert }
2235*404b540aSrobert
2236*404b540aSrobert if (d->alloc_bits == NULL)
2237*404b540aSrobert d->alloc_bits = xcalloc (1, d->alloc_size);
2238*404b540aSrobert }
2239*404b540aSrobert
2240*404b540aSrobert /* Allocate a place for object X of size SIZE in the PCH file. */
2241*404b540aSrobert
2242*404b540aSrobert char *
ggc_pch_alloc_object(struct ggc_pch_data * d,void * x,size_t size,bool is_string,enum gt_types_enum type)2243*404b540aSrobert ggc_pch_alloc_object (struct ggc_pch_data *d, void *x,
2244*404b540aSrobert size_t size, bool is_string,
2245*404b540aSrobert enum gt_types_enum type)
2246*404b540aSrobert {
2247*404b540aSrobert size_t alloc_word, alloc_bit;
2248*404b540aSrobert char *result;
2249*404b540aSrobert int bucket = pch_bucket (x, type, is_string);
2250*404b540aSrobert
2251*404b540aSrobert /* Record the start of the object in the allocation bitmap. We
2252*404b540aSrobert can't assert that the allocation bit is previously clear, because
2253*404b540aSrobert strings may violate the invariant that they are at least
2254*404b540aSrobert BYTES_PER_ALLOC_BIT long. This is harmless - ggc_get_size
2255*404b540aSrobert should not be called for strings. */
2256*404b540aSrobert alloc_word = ((d->type_bases[bucket] - d->orig_base)
2257*404b540aSrobert / (8 * sizeof (alloc_type) * BYTES_PER_ALLOC_BIT));
2258*404b540aSrobert alloc_bit = ((d->type_bases[bucket] - d->orig_base)
2259*404b540aSrobert / BYTES_PER_ALLOC_BIT) % (8 * sizeof (alloc_type));
2260*404b540aSrobert d->alloc_bits[alloc_word] |= 1L << alloc_bit;
2261*404b540aSrobert
2262*404b540aSrobert /* Place the object at the current pointer for this bucket. */
2263*404b540aSrobert result = (char *) d->type_bases[bucket];
2264*404b540aSrobert d->type_bases[bucket] += size;
2265*404b540aSrobert return result;
2266*404b540aSrobert }
2267*404b540aSrobert
2268*404b540aSrobert /* Prepare to write out the PCH data to file F. */
2269*404b540aSrobert
2270*404b540aSrobert void
ggc_pch_prepare_write(struct ggc_pch_data * d,FILE * f)2271*404b540aSrobert ggc_pch_prepare_write (struct ggc_pch_data *d,
2272*404b540aSrobert FILE *f)
2273*404b540aSrobert {
2274*404b540aSrobert /* We seek around a lot while writing. Record where the end
2275*404b540aSrobert of the padding in the PCH file is, so that we can
2276*404b540aSrobert locate each object's offset. */
2277*404b540aSrobert d->start_offset = ftell (f);
2278*404b540aSrobert }
2279*404b540aSrobert
2280*404b540aSrobert /* Write out object X of SIZE to file F. */
2281*404b540aSrobert
2282*404b540aSrobert void
ggc_pch_write_object(struct ggc_pch_data * d,FILE * f,void * x,void * newx,size_t size,bool is_string ATTRIBUTE_UNUSED)2283*404b540aSrobert ggc_pch_write_object (struct ggc_pch_data *d,
2284*404b540aSrobert FILE *f, void *x, void *newx,
2285*404b540aSrobert size_t size, bool is_string ATTRIBUTE_UNUSED)
2286*404b540aSrobert {
2287*404b540aSrobert if (fseek (f, (size_t) newx - d->orig_base + d->start_offset, SEEK_SET) != 0)
2288*404b540aSrobert fatal_error ("can't seek PCH file: %m");
2289*404b540aSrobert
2290*404b540aSrobert if (fwrite (x, size, 1, f) != 1)
2291*404b540aSrobert fatal_error ("can't write PCH file: %m");
2292*404b540aSrobert }
2293*404b540aSrobert
2294*404b540aSrobert void
ggc_pch_finish(struct ggc_pch_data * d,FILE * f)2295*404b540aSrobert ggc_pch_finish (struct ggc_pch_data *d, FILE *f)
2296*404b540aSrobert {
2297*404b540aSrobert /* Write out the allocation bitmap. */
2298*404b540aSrobert if (fseek (f, d->start_offset + d->d.total, SEEK_SET) != 0)
2299*404b540aSrobert fatal_error ("can't seek PCH file: %m");
2300*404b540aSrobert
2301*404b540aSrobert if (fwrite (d->alloc_bits, d->alloc_size, 1, f) != 1)
2302*404b540aSrobert fatal_error ("can't write PCH fle: %m");
2303*404b540aSrobert
2304*404b540aSrobert /* Done with the PCH, so write out our footer. */
2305*404b540aSrobert if (fwrite (&d->d, sizeof (d->d), 1, f) != 1)
2306*404b540aSrobert fatal_error ("can't write PCH file: %m");
2307*404b540aSrobert
2308*404b540aSrobert free (d->alloc_bits);
2309*404b540aSrobert free (d);
2310*404b540aSrobert }
2311*404b540aSrobert
2312*404b540aSrobert /* The PCH file from F has been mapped at ADDR. Read in any
2313*404b540aSrobert additional data from the file and set up the GC state. */
2314*404b540aSrobert
2315*404b540aSrobert void
ggc_pch_read(FILE * f,void * addr)2316*404b540aSrobert ggc_pch_read (FILE *f, void *addr)
2317*404b540aSrobert {
2318*404b540aSrobert struct ggc_pch_ondisk d;
2319*404b540aSrobert size_t alloc_size;
2320*404b540aSrobert struct alloc_zone *zone;
2321*404b540aSrobert struct page_entry *pch_page;
2322*404b540aSrobert char *p;
2323*404b540aSrobert
2324*404b540aSrobert if (fread (&d, sizeof (d), 1, f) != 1)
2325*404b540aSrobert fatal_error ("can't read PCH file: %m");
2326*404b540aSrobert
2327*404b540aSrobert alloc_size = CEIL (d.total, BYTES_PER_ALLOC_BIT * 8);
2328*404b540aSrobert alloc_size = ROUND_UP (alloc_size, MAX_ALIGNMENT);
2329*404b540aSrobert
2330*404b540aSrobert pch_zone.bytes = d.total;
2331*404b540aSrobert pch_zone.alloc_bits = (alloc_type *) ((char *) addr + pch_zone.bytes);
2332*404b540aSrobert pch_zone.page = (char *) addr;
2333*404b540aSrobert pch_zone.end = (char *) pch_zone.alloc_bits;
2334*404b540aSrobert
2335*404b540aSrobert /* We've just read in a PCH file. So, every object that used to be
2336*404b540aSrobert allocated is now free. */
2337*404b540aSrobert for (zone = G.zones; zone; zone = zone->next_zone)
2338*404b540aSrobert {
2339*404b540aSrobert struct small_page_entry *page, *next_page;
2340*404b540aSrobert struct large_page_entry *large_page, *next_large_page;
2341*404b540aSrobert
2342*404b540aSrobert zone->allocated = 0;
2343*404b540aSrobert
2344*404b540aSrobert /* Clear the zone's free chunk list. */
2345*404b540aSrobert memset (zone->free_chunks, 0, sizeof (zone->free_chunks));
2346*404b540aSrobert zone->high_free_bin = 0;
2347*404b540aSrobert zone->cached_free = NULL;
2348*404b540aSrobert zone->cached_free_size = 0;
2349*404b540aSrobert
2350*404b540aSrobert /* Move all the small pages onto the free list. */
2351*404b540aSrobert for (page = zone->pages; page != NULL; page = next_page)
2352*404b540aSrobert {
2353*404b540aSrobert next_page = page->next;
2354*404b540aSrobert memset (page->alloc_bits, 0,
2355*404b540aSrobert G.small_page_overhead - PAGE_OVERHEAD);
2356*404b540aSrobert free_small_page (page);
2357*404b540aSrobert }
2358*404b540aSrobert
2359*404b540aSrobert /* Discard all the large pages. */
2360*404b540aSrobert for (large_page = zone->large_pages; large_page != NULL;
2361*404b540aSrobert large_page = next_large_page)
2362*404b540aSrobert {
2363*404b540aSrobert next_large_page = large_page->next;
2364*404b540aSrobert free_large_page (large_page);
2365*404b540aSrobert }
2366*404b540aSrobert
2367*404b540aSrobert zone->pages = NULL;
2368*404b540aSrobert zone->large_pages = NULL;
2369*404b540aSrobert }
2370*404b540aSrobert
2371*404b540aSrobert /* Allocate the dummy page entry for the PCH, and set all pages
2372*404b540aSrobert mapped into the PCH to reference it. */
2373*404b540aSrobert pch_page = xcalloc (1, sizeof (struct page_entry));
2374*404b540aSrobert pch_page->page = pch_zone.page;
2375*404b540aSrobert pch_page->pch_p = true;
2376*404b540aSrobert
2377*404b540aSrobert for (p = pch_zone.page; p < pch_zone.end; p += GGC_PAGE_SIZE)
2378*404b540aSrobert set_page_table_entry (p, pch_page);
2379*404b540aSrobert }
2380