1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 // Per-P malloc cache for small objects.
6 //
7 // See malloc.h for an overview.
8
9 #include "runtime.h"
10 #include "arch.h"
11 #include "malloc.h"
12
13 extern volatile intgo runtime_MemProfileRate
14 __asm__ (GOSYM_PREFIX "runtime.MemProfileRate");
15
16 // dummy MSpan that contains no free objects.
17 static MSpan emptymspan;
18
19 MCache*
runtime_allocmcache(void)20 runtime_allocmcache(void)
21 {
22 intgo rate;
23 MCache *c;
24 int32 i;
25
26 runtime_lock(&runtime_mheap);
27 c = runtime_FixAlloc_Alloc(&runtime_mheap.cachealloc);
28 runtime_unlock(&runtime_mheap);
29 runtime_memclr((byte*)c, sizeof(*c));
30 for(i = 0; i < NumSizeClasses; i++)
31 c->alloc[i] = &emptymspan;
32
33 // Set first allocation sample size.
34 rate = runtime_MemProfileRate;
35 if(rate > 0x3fffffff) // make 2*rate not overflow
36 rate = 0x3fffffff;
37 if(rate != 0)
38 c->next_sample = runtime_fastrand1() % (2*rate);
39
40 return c;
41 }
42
43 void
runtime_freemcache(MCache * c)44 runtime_freemcache(MCache *c)
45 {
46 runtime_MCache_ReleaseAll(c);
47 runtime_lock(&runtime_mheap);
48 runtime_purgecachedstats(c);
49 runtime_FixAlloc_Free(&runtime_mheap.cachealloc, c);
50 runtime_unlock(&runtime_mheap);
51 }
52
53 // Gets a span that has a free object in it and assigns it
54 // to be the cached span for the given sizeclass. Returns this span.
55 MSpan*
runtime_MCache_Refill(MCache * c,int32 sizeclass)56 runtime_MCache_Refill(MCache *c, int32 sizeclass)
57 {
58 MCacheList *l;
59 MSpan *s;
60
61 runtime_m()->locks++;
62 // Return the current cached span to the central lists.
63 s = c->alloc[sizeclass];
64 if(s->freelist != nil)
65 runtime_throw("refill on a nonempty span");
66 if(s != &emptymspan)
67 runtime_MCentral_UncacheSpan(&runtime_mheap.central[sizeclass], s);
68
69 // Push any explicitly freed objects to the central lists.
70 // Not required, but it seems like a good time to do it.
71 l = &c->free[sizeclass];
72 if(l->nlist > 0) {
73 runtime_MCentral_FreeList(&runtime_mheap.central[sizeclass], l->list);
74 l->list = nil;
75 l->nlist = 0;
76 }
77
78 // Get a new cached span from the central lists.
79 s = runtime_MCentral_CacheSpan(&runtime_mheap.central[sizeclass]);
80 if(s == nil)
81 runtime_throw("out of memory");
82 if(s->freelist == nil) {
83 runtime_printf("%d %d\n", s->ref, (int32)((s->npages << PageShift) / s->elemsize));
84 runtime_throw("empty span");
85 }
86 c->alloc[sizeclass] = s;
87 runtime_m()->locks--;
88 return s;
89 }
90
91 void
runtime_MCache_Free(MCache * c,MLink * p,int32 sizeclass,uintptr size)92 runtime_MCache_Free(MCache *c, MLink *p, int32 sizeclass, uintptr size)
93 {
94 MCacheList *l;
95
96 // Put on free list.
97 l = &c->free[sizeclass];
98 p->next = l->list;
99 l->list = p;
100 l->nlist++;
101
102 // We transfer a span at a time from MCentral to MCache,
103 // so we'll do the same in the other direction.
104 if(l->nlist >= (runtime_class_to_allocnpages[sizeclass]<<PageShift)/size) {
105 runtime_MCentral_FreeList(&runtime_mheap.central[sizeclass], l->list);
106 l->list = nil;
107 l->nlist = 0;
108 }
109 }
110
111 void
runtime_MCache_ReleaseAll(MCache * c)112 runtime_MCache_ReleaseAll(MCache *c)
113 {
114 int32 i;
115 MSpan *s;
116 MCacheList *l;
117
118 for(i=0; i<NumSizeClasses; i++) {
119 s = c->alloc[i];
120 if(s != &emptymspan) {
121 runtime_MCentral_UncacheSpan(&runtime_mheap.central[i], s);
122 c->alloc[i] = &emptymspan;
123 }
124 l = &c->free[i];
125 if(l->nlist > 0) {
126 runtime_MCentral_FreeList(&runtime_mheap.central[i], l->list);
127 l->list = nil;
128 l->nlist = 0;
129 }
130 }
131 }
132