1// Copyright 2009 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package runtime 6 7import "unsafe" 8 9// Per-thread (in Go, per-P) cache for small objects. 10// No locking needed because it is per-thread (per-P). 11// 12// mcaches are allocated from non-GC'd memory, so any heap pointers 13// must be specially handled. 14// 15//go:notinheap 16type mcache struct { 17 // The following members are accessed on every malloc, 18 // so they are grouped here for better caching. 19 next_sample int32 // trigger heap sample after allocating this many bytes 20 local_scan uintptr // bytes of scannable heap allocated 21 22 // Allocator cache for tiny objects w/o pointers. 23 // See "Tiny allocator" comment in malloc.go. 24 25 // tiny points to the beginning of the current tiny block, or 26 // nil if there is no current tiny block. 27 // 28 // tiny is a heap pointer. Since mcache is in non-GC'd memory, 29 // we handle it by clearing it in releaseAll during mark 30 // termination. 31 tiny uintptr 32 tinyoffset uintptr 33 local_tinyallocs uintptr // number of tiny allocs not counted in other stats 34 35 // The rest is not accessed on every malloc. 36 37 alloc [numSpanClasses]*mspan // spans to allocate from, indexed by spanClass 38 39 // Local allocator stats, flushed during GC. 40 local_nlookup uintptr // number of pointer lookups 41 local_largefree uintptr // bytes freed for large objects (>maxsmallsize) 42 local_nlargefree uintptr // number of frees for large objects (>maxsmallsize) 43 local_nsmallfree [_NumSizeClasses]uintptr // number of frees for small objects (<=maxsmallsize) 44} 45 46// A gclink is a node in a linked list of blocks, like mlink, 47// but it is opaque to the garbage collector. 48// The GC does not trace the pointers during collection, 49// and the compiler does not emit write barriers for assignments 50// of gclinkptr values. Code should store references to gclinks 51// as gclinkptr, not as *gclink. 52type gclink struct { 53 next gclinkptr 54} 55 56// A gclinkptr is a pointer to a gclink, but it is opaque 57// to the garbage collector. 58type gclinkptr uintptr 59 60// ptr returns the *gclink form of p. 61// The result should be used for accessing fields, not stored 62// in other data structures. 63func (p gclinkptr) ptr() *gclink { 64 return (*gclink)(unsafe.Pointer(p)) 65} 66 67// dummy MSpan that contains no free objects. 68var emptymspan mspan 69 70func allocmcache() *mcache { 71 lock(&mheap_.lock) 72 c := (*mcache)(mheap_.cachealloc.alloc()) 73 unlock(&mheap_.lock) 74 for i := range c.alloc { 75 c.alloc[i] = &emptymspan 76 } 77 c.next_sample = nextSample() 78 return c 79} 80 81func freemcache(c *mcache) { 82 systemstack(func() { 83 c.releaseAll() 84 85 // NOTE(rsc,rlh): If gcworkbuffree comes back, we need to coordinate 86 // with the stealing of gcworkbufs during garbage collection to avoid 87 // a race where the workbuf is double-freed. 88 // gcworkbuffree(c.gcworkbuf) 89 90 lock(&mheap_.lock) 91 purgecachedstats(c) 92 mheap_.cachealloc.free(unsafe.Pointer(c)) 93 unlock(&mheap_.lock) 94 }) 95} 96 97// Gets a span that has a free object in it and assigns it 98// to be the cached span for the given sizeclass. Returns this span. 99func (c *mcache) refill(spc spanClass) { 100 _g_ := getg() 101 102 _g_.m.locks++ 103 // Return the current cached span to the central lists. 104 s := c.alloc[spc] 105 106 if uintptr(s.allocCount) != s.nelems { 107 throw("refill of span with free space remaining") 108 } 109 110 if s != &emptymspan { 111 s.incache = false 112 } 113 114 // Get a new cached span from the central lists. 115 s = mheap_.central[spc].mcentral.cacheSpan() 116 if s == nil { 117 throw("out of memory") 118 } 119 120 if uintptr(s.allocCount) == s.nelems { 121 throw("span has no free space") 122 } 123 124 c.alloc[spc] = s 125 _g_.m.locks-- 126} 127 128func (c *mcache) releaseAll() { 129 for i := range c.alloc { 130 s := c.alloc[i] 131 if s != &emptymspan { 132 mheap_.central[i].mcentral.uncacheSpan(s) 133 c.alloc[i] = &emptymspan 134 } 135 } 136 // Clear tinyalloc pool. 137 c.tiny = 0 138 c.tinyoffset = 0 139} 140