1 #include <Core/Core.h>
2
3 #define LTIMING(x) // RTIMING(x)
4 // #define LSTAT
5
6 namespace Upp {
7
8 #ifdef UPP_HEAP
9
10 #include "HeapImp.h"
11
12 // this part reserves very large (HPAGE*4 KB, default 16MB)
13 // chunks form the system and then serves as 4KB rounded allocator
14 // used as manager of huge memory blocks. 4KB and 64KB blocks are allocated from here too
15 // also able to deal with bigger blocks, those are directly allocated / freed from system
16
17 BlkHeader_<4096> HugeHeapDetail::freelist[20][1]; // only single global Huge heap...
18 Heap::HugePage *Heap::huge_pages;
19
20 #ifdef LSTAT
21 static int hstat[65536];
22
23 EXITBLOCK {
24 int cnt = 0;
25 for(int i = 0; i < 65536; i++) {
26 cnt += hstat[i];
27 if(hstat[i])
28 RLOG(i * 4 << " KB: " << hstat[i] << " / " << cnt);
29 }
30 }
31 #endif
32
33 size_t sKBLimit = INT_MAX;
34
MemoryLimitKb(int kb)35 void MemoryLimitKb(int kb)
36 {
37 sKBLimit = kb;
38 }
39
40 static MemoryProfile *sPeak;
41
PeakMemoryProfile()42 MemoryProfile *PeakMemoryProfile()
43 {
44 if(sPeak)
45 return sPeak;
46 sPeak = (MemoryProfile *)MemoryAllocPermanent(sizeof(MemoryProfile));
47 memset((void *)sPeak, 0, sizeof(MemoryProfile));
48 return NULL;
49 }
50
51 Heap::HugePage *Heap::free_huge_pages;
52 int Heap::free_hpages;
53
HugeAlloc(size_t count)54 void *Heap::HugeAlloc(size_t count) // count in 4kb pages
55 {
56 LTIMING("HugeAlloc");
57 ASSERT(count);
58
59 #ifdef LSTAT
60 if(count < 65536)
61 hstat[count]++;
62 #endif
63
64 huge_4KB_count += count;
65
66 auto MaxMem = [&] {
67 if(huge_4KB_count > huge_4KB_count_max) {
68 huge_4KB_count_max = huge_4KB_count;
69 if(4 * (Heap::huge_4KB_count - Heap::free_4KB) > sKBLimit)
70 Panic("MemoryLimitKb breached!");
71 if(sPeak)
72 Make(*sPeak);
73 }
74 };
75
76 if(!D::freelist[0]->next) { // initialization
77 for(int i = 0; i < __countof(D::freelist); i++)
78 Dbl_Self(D::freelist[i]);
79 }
80
81 if(count > sys_block_limit) { // we are wasting 4KB to store just 4 bytes here, but this is n MB after all..
82 LTIMING("SysAlloc");
83 byte *sysblk = (byte *)SysAllocRaw((count + 1) * 4096, 0);
84 BlkHeader *h = (BlkHeader *)(sysblk + 4096);
85 h->size = 0;
86 *((size_t *)sysblk) = count;
87 sys_count++;
88 sys_size += 4096 * count;
89 MaxMem();
90 return h;
91 }
92
93 LTIMING("Huge Alloc");
94
95 word wcount = (word)count;
96
97 for(int pass = 0; pass < 2; pass++) {
98 for(int i = Cv(wcount); i < __countof(D::freelist); i++) {
99 BlkHeader *l = D::freelist[i];
100 BlkHeader *h = l->next;
101 while(h != l) {
102 word sz = h->GetSize();
103 if(sz >= count) {
104 if(h->IsFirst() && h->IsLast()) // this is whole free page
105 free_hpages--;
106 void *ptr = MakeAlloc(h, wcount);
107 MaxMem();
108 return ptr;
109 }
110 h = h->next;
111 }
112 }
113
114 if(!FreeSmallEmpty(wcount, INT_MAX)) { // try to coalesce 4KB small free blocks back to huge storage
115 void *ptr = SysAllocRaw(HPAGE * 4096, 0); // failed, add HPAGE from the system
116
117 HugePage *pg; // record in set of huge pages
118 if(free_huge_pages) {
119 pg = free_huge_pages;
120 free_huge_pages = free_huge_pages->next;
121 }
122 else
123 pg = (HugePage *)MemoryAllocPermanent(sizeof(HugePage));
124
125 pg->page = ptr;
126 pg->next = huge_pages;
127 huge_pages = pg;
128 huge_chunks++;
129 free_hpages++;
130 AddChunk((BlkHeader *)ptr, HPAGE);
131 }
132 }
133 Panic("Out of memory");
134 return NULL;
135 }
136
HugeFree(void * ptr)137 int Heap::HugeFree(void *ptr)
138 {
139 LTIMING("HugeFree");
140 BlkHeader *h = (BlkHeader *)ptr;
141 if(h->size == 0) {
142 LTIMING("Sys Free");
143 byte *sysblk = (byte *)h - 4096;
144 size_t count = *((size_t *)sysblk);
145 SysFreeRaw(sysblk, (count + 1) * 4096);
146 huge_4KB_count -= count;
147 sys_count--;
148 sys_size -= 4096 * count;
149 return 0;
150 }
151 LTIMING("Huge Free");
152 huge_4KB_count -= h->GetSize();
153 h = BlkHeap::Free(h);
154 int sz = h->GetSize();
155 if(h->IsFirst() && h->IsLast()) {
156 if(free_hpages >= max_free_hpages) { // we have enough pages in the reserve, return to the system
157 LTIMING("Free Huge Page");
158 h->UnlinkFree();
159 HugePage *p = NULL;
160 while(huge_pages) { // remove the page from the set of huge pages
161 HugePage *n = huge_pages->next;
162 if(huge_pages->page != h) {
163 huge_pages->next = p;
164 p = huge_pages;
165 }
166 huge_pages = n;
167 }
168 huge_pages = p;
169 huge_chunks--;
170 SysFreeRaw(h, sz * 4096);
171 }
172 else
173 free_hpages++;
174 }
175 return sz;
176 }
177
HugeTryRealloc(void * ptr,size_t count)178 bool Heap::HugeTryRealloc(void *ptr, size_t count)
179 {
180 return count <= HPAGE && BlkHeap::TryRealloc(ptr, count, huge_4KB_count);
181 }
182
183 #endif
184
185 }