1// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package runtime
6
7import "unsafe"
8
9// Per-thread (in Go, per-P) cache for small objects.
10// No locking needed because it is per-thread (per-P).
11//
12// mcaches are allocated from non-GC'd memory, so any heap pointers
13// must be specially handled.
14//
15//go:notinheap
16type mcache struct {
17	// The following members are accessed on every malloc,
18	// so they are grouped here for better caching.
19	next_sample int32   // trigger heap sample after allocating this many bytes
20	local_scan  uintptr // bytes of scannable heap allocated
21
22	// Allocator cache for tiny objects w/o pointers.
23	// See "Tiny allocator" comment in malloc.go.
24
25	// tiny points to the beginning of the current tiny block, or
26	// nil if there is no current tiny block.
27	//
28	// tiny is a heap pointer. Since mcache is in non-GC'd memory,
29	// we handle it by clearing it in releaseAll during mark
30	// termination.
31	tiny             uintptr
32	tinyoffset       uintptr
33	local_tinyallocs uintptr // number of tiny allocs not counted in other stats
34
35	// The rest is not accessed on every malloc.
36
37	alloc [numSpanClasses]*mspan // spans to allocate from, indexed by spanClass
38
39	// Local allocator stats, flushed during GC.
40	local_nlookup    uintptr                  // number of pointer lookups
41	local_largefree  uintptr                  // bytes freed for large objects (>maxsmallsize)
42	local_nlargefree uintptr                  // number of frees for large objects (>maxsmallsize)
43	local_nsmallfree [_NumSizeClasses]uintptr // number of frees for small objects (<=maxsmallsize)
44}
45
46// A gclink is a node in a linked list of blocks, like mlink,
47// but it is opaque to the garbage collector.
48// The GC does not trace the pointers during collection,
49// and the compiler does not emit write barriers for assignments
50// of gclinkptr values. Code should store references to gclinks
51// as gclinkptr, not as *gclink.
52type gclink struct {
53	next gclinkptr
54}
55
56// A gclinkptr is a pointer to a gclink, but it is opaque
57// to the garbage collector.
58type gclinkptr uintptr
59
60// ptr returns the *gclink form of p.
61// The result should be used for accessing fields, not stored
62// in other data structures.
63func (p gclinkptr) ptr() *gclink {
64	return (*gclink)(unsafe.Pointer(p))
65}
66
67// dummy MSpan that contains no free objects.
68var emptymspan mspan
69
70func allocmcache() *mcache {
71	lock(&mheap_.lock)
72	c := (*mcache)(mheap_.cachealloc.alloc())
73	unlock(&mheap_.lock)
74	for i := range c.alloc {
75		c.alloc[i] = &emptymspan
76	}
77	c.next_sample = nextSample()
78	return c
79}
80
81func freemcache(c *mcache) {
82	systemstack(func() {
83		c.releaseAll()
84
85		// NOTE(rsc,rlh): If gcworkbuffree comes back, we need to coordinate
86		// with the stealing of gcworkbufs during garbage collection to avoid
87		// a race where the workbuf is double-freed.
88		// gcworkbuffree(c.gcworkbuf)
89
90		lock(&mheap_.lock)
91		purgecachedstats(c)
92		mheap_.cachealloc.free(unsafe.Pointer(c))
93		unlock(&mheap_.lock)
94	})
95}
96
97// Gets a span that has a free object in it and assigns it
98// to be the cached span for the given sizeclass. Returns this span.
99func (c *mcache) refill(spc spanClass) {
100	_g_ := getg()
101
102	_g_.m.locks++
103	// Return the current cached span to the central lists.
104	s := c.alloc[spc]
105
106	if uintptr(s.allocCount) != s.nelems {
107		throw("refill of span with free space remaining")
108	}
109
110	if s != &emptymspan {
111		s.incache = false
112	}
113
114	// Get a new cached span from the central lists.
115	s = mheap_.central[spc].mcentral.cacheSpan()
116	if s == nil {
117		throw("out of memory")
118	}
119
120	if uintptr(s.allocCount) == s.nelems {
121		throw("span has no free space")
122	}
123
124	c.alloc[spc] = s
125	_g_.m.locks--
126}
127
128func (c *mcache) releaseAll() {
129	for i := range c.alloc {
130		s := c.alloc[i]
131		if s != &emptymspan {
132			mheap_.central[i].mcentral.uncacheSpan(s)
133			c.alloc[i] = &emptymspan
134		}
135	}
136	// Clear tinyalloc pool.
137	c.tiny = 0
138	c.tinyoffset = 0
139}
140