1 /*
2 * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/z/zGlobals.hpp"
26 #include "gc/z/zList.inline.hpp"
27 #include "gc/z/zNUMA.hpp"
28 #include "gc/z/zPage.inline.hpp"
29 #include "gc/z/zPageCache.hpp"
30 #include "gc/z/zStat.hpp"
31 #include "gc/z/zValue.inline.hpp"
32 #include "memory/allocation.hpp"
33 #include "runtime/globals.hpp"
34 #include "runtime/os.hpp"
35
36 static const ZStatCounter ZCounterPageCacheHitL1("Memory", "Page Cache Hit L1", ZStatUnitOpsPerSecond);
37 static const ZStatCounter ZCounterPageCacheHitL2("Memory", "Page Cache Hit L2", ZStatUnitOpsPerSecond);
38 static const ZStatCounter ZCounterPageCacheHitL3("Memory", "Page Cache Hit L3", ZStatUnitOpsPerSecond);
39 static const ZStatCounter ZCounterPageCacheMiss("Memory", "Page Cache Miss", ZStatUnitOpsPerSecond);
40
41 class ZPageCacheFlushClosure : public StackObj {
42 friend class ZPageCache;
43
44 protected:
45 const size_t _requested;
46 size_t _flushed;
47
48 public:
49 ZPageCacheFlushClosure(size_t requested);
50 virtual bool do_page(const ZPage* page) = 0;
51 };
52
ZPageCacheFlushClosure(size_t requested)53 ZPageCacheFlushClosure::ZPageCacheFlushClosure(size_t requested) :
54 _requested(requested),
55 _flushed(0) {}
56
ZPageCache()57 ZPageCache::ZPageCache() :
58 _small(),
59 _medium(),
60 _large(),
61 _last_commit(0) {}
62
alloc_small_page()63 ZPage* ZPageCache::alloc_small_page() {
64 const uint32_t numa_id = ZNUMA::id();
65 const uint32_t numa_count = ZNUMA::count();
66
67 // Try NUMA local page cache
68 ZPage* const l1_page = _small.get(numa_id).remove_first();
69 if (l1_page != NULL) {
70 ZStatInc(ZCounterPageCacheHitL1);
71 return l1_page;
72 }
73
74 // Try NUMA remote page cache(s)
75 uint32_t remote_numa_id = numa_id + 1;
76 const uint32_t remote_numa_count = numa_count - 1;
77 for (uint32_t i = 0; i < remote_numa_count; i++) {
78 if (remote_numa_id == numa_count) {
79 remote_numa_id = 0;
80 }
81
82 ZPage* const l2_page = _small.get(remote_numa_id).remove_first();
83 if (l2_page != NULL) {
84 ZStatInc(ZCounterPageCacheHitL2);
85 return l2_page;
86 }
87
88 remote_numa_id++;
89 }
90
91 return NULL;
92 }
93
alloc_medium_page()94 ZPage* ZPageCache::alloc_medium_page() {
95 ZPage* const page = _medium.remove_first();
96 if (page != NULL) {
97 ZStatInc(ZCounterPageCacheHitL1);
98 return page;
99 }
100
101 return NULL;
102 }
103
alloc_large_page(size_t size)104 ZPage* ZPageCache::alloc_large_page(size_t size) {
105 // Find a page with the right size
106 ZListIterator<ZPage> iter(&_large);
107 for (ZPage* page; iter.next(&page);) {
108 if (size == page->size()) {
109 // Page found
110 _large.remove(page);
111 ZStatInc(ZCounterPageCacheHitL1);
112 return page;
113 }
114 }
115
116 return NULL;
117 }
118
alloc_oversized_medium_page(size_t size)119 ZPage* ZPageCache::alloc_oversized_medium_page(size_t size) {
120 if (size <= ZPageSizeMedium) {
121 return _medium.remove_first();
122 }
123
124 return NULL;
125 }
126
alloc_oversized_large_page(size_t size)127 ZPage* ZPageCache::alloc_oversized_large_page(size_t size) {
128 // Find a page that is large enough
129 ZListIterator<ZPage> iter(&_large);
130 for (ZPage* page; iter.next(&page);) {
131 if (size <= page->size()) {
132 // Page found
133 _large.remove(page);
134 return page;
135 }
136 }
137
138 return NULL;
139 }
140
alloc_oversized_page(size_t size)141 ZPage* ZPageCache::alloc_oversized_page(size_t size) {
142 ZPage* page = alloc_oversized_large_page(size);
143 if (page == NULL) {
144 page = alloc_oversized_medium_page(size);
145 }
146
147 if (page != NULL) {
148 ZStatInc(ZCounterPageCacheHitL3);
149 }
150
151 return page;
152 }
153
alloc_page(uint8_t type,size_t size)154 ZPage* ZPageCache::alloc_page(uint8_t type, size_t size) {
155 ZPage* page;
156
157 // Try allocate exact page
158 if (type == ZPageTypeSmall) {
159 page = alloc_small_page();
160 } else if (type == ZPageTypeMedium) {
161 page = alloc_medium_page();
162 } else {
163 page = alloc_large_page(size);
164 }
165
166 if (page == NULL) {
167 // Try allocate potentially oversized page
168 ZPage* const oversized = alloc_oversized_page(size);
169 if (oversized != NULL) {
170 if (size < oversized->size()) {
171 // Split oversized page
172 page = oversized->split(type, size);
173
174 // Cache remainder
175 free_page(oversized);
176 } else {
177 // Re-type correctly sized page
178 page = oversized->retype(type);
179 }
180 }
181 }
182
183 if (page == NULL) {
184 ZStatInc(ZCounterPageCacheMiss);
185 }
186
187 return page;
188 }
189
free_page(ZPage * page)190 void ZPageCache::free_page(ZPage* page) {
191 const uint8_t type = page->type();
192 if (type == ZPageTypeSmall) {
193 _small.get(page->numa_id()).insert_first(page);
194 } else if (type == ZPageTypeMedium) {
195 _medium.insert_first(page);
196 } else {
197 _large.insert_first(page);
198 }
199 }
200
flush_list_inner(ZPageCacheFlushClosure * cl,ZList<ZPage> * from,ZList<ZPage> * to)201 bool ZPageCache::flush_list_inner(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to) {
202 ZPage* const page = from->last();
203 if (page == NULL || !cl->do_page(page)) {
204 // Don't flush page
205 return false;
206 }
207
208 // Flush page
209 from->remove(page);
210 to->insert_last(page);
211 return true;
212 }
213
flush_list(ZPageCacheFlushClosure * cl,ZList<ZPage> * from,ZList<ZPage> * to)214 void ZPageCache::flush_list(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to) {
215 while (flush_list_inner(cl, from, to));
216 }
217
flush_per_numa_lists(ZPageCacheFlushClosure * cl,ZPerNUMA<ZList<ZPage>> * from,ZList<ZPage> * to)218 void ZPageCache::flush_per_numa_lists(ZPageCacheFlushClosure* cl, ZPerNUMA<ZList<ZPage> >* from, ZList<ZPage>* to) {
219 const uint32_t numa_count = ZNUMA::count();
220 uint32_t numa_done = 0;
221 uint32_t numa_next = 0;
222
223 // Flush lists round-robin
224 while (numa_done < numa_count) {
225 ZList<ZPage>* numa_list = from->addr(numa_next);
226 if (++numa_next == numa_count) {
227 numa_next = 0;
228 }
229
230 if (flush_list_inner(cl, numa_list, to)) {
231 // Not done
232 numa_done = 0;
233 } else {
234 // Done
235 numa_done++;
236 }
237 }
238 }
239
flush(ZPageCacheFlushClosure * cl,ZList<ZPage> * to)240 void ZPageCache::flush(ZPageCacheFlushClosure* cl, ZList<ZPage>* to) {
241 // Prefer flushing large, then medium and last small pages
242 flush_list(cl, &_large, to);
243 flush_list(cl, &_medium, to);
244 flush_per_numa_lists(cl, &_small, to);
245
246 if (cl->_flushed > cl->_requested) {
247 // Overflushed, re-insert part of last page into the cache
248 const size_t overflushed = cl->_flushed - cl->_requested;
249 ZPage* const reinsert = to->last()->split(overflushed);
250 free_page(reinsert);
251 cl->_flushed -= overflushed;
252 }
253 }
254
255 class ZPageCacheFlushForAllocationClosure : public ZPageCacheFlushClosure {
256 public:
ZPageCacheFlushForAllocationClosure(size_t requested)257 ZPageCacheFlushForAllocationClosure(size_t requested) :
258 ZPageCacheFlushClosure(requested) {}
259
do_page(const ZPage * page)260 virtual bool do_page(const ZPage* page) {
261 if (_flushed < _requested) {
262 // Flush page
263 _flushed += page->size();
264 return true;
265 }
266
267 // Don't flush page
268 return false;
269 }
270 };
271
flush_for_allocation(size_t requested,ZList<ZPage> * to)272 void ZPageCache::flush_for_allocation(size_t requested, ZList<ZPage>* to) {
273 ZPageCacheFlushForAllocationClosure cl(requested);
274 flush(&cl, to);
275 }
276
277 class ZPageCacheFlushForUncommitClosure : public ZPageCacheFlushClosure {
278 private:
279 const uint64_t _now;
280 uint64_t* _timeout;
281
282 public:
ZPageCacheFlushForUncommitClosure(size_t requested,uint64_t now,uint64_t * timeout)283 ZPageCacheFlushForUncommitClosure(size_t requested, uint64_t now, uint64_t* timeout) :
284 ZPageCacheFlushClosure(requested),
285 _now(now),
286 _timeout(timeout) {
287 // Set initial timeout
288 *_timeout = ZUncommitDelay;
289 }
290
do_page(const ZPage * page)291 virtual bool do_page(const ZPage* page) {
292 const uint64_t expires = page->last_used() + ZUncommitDelay;
293 if (expires > _now) {
294 // Don't flush page, record shortest non-expired timeout
295 *_timeout = MIN2(*_timeout, expires - _now);
296 return false;
297 }
298
299 if (_flushed >= _requested) {
300 // Don't flush page, requested amount flushed
301 return false;
302 }
303
304 // Flush page
305 _flushed += page->size();
306 return true;
307 }
308 };
309
flush_for_uncommit(size_t requested,ZList<ZPage> * to,uint64_t * timeout)310 size_t ZPageCache::flush_for_uncommit(size_t requested, ZList<ZPage>* to, uint64_t* timeout) {
311 const uint64_t now = os::elapsedTime();
312 const uint64_t expires = _last_commit + ZUncommitDelay;
313 if (expires > now) {
314 // Delay uncommit, set next timeout
315 *timeout = expires - now;
316 return 0;
317 }
318
319 if (requested == 0) {
320 // Nothing to flush, set next timeout
321 *timeout = ZUncommitDelay;
322 return 0;
323 }
324
325 ZPageCacheFlushForUncommitClosure cl(requested, now, timeout);
326 flush(&cl, to);
327
328 return cl._flushed;
329 }
330
set_last_commit()331 void ZPageCache::set_last_commit() {
332 _last_commit = ceil(os::elapsedTime());
333 }
334
pages_do(ZPageClosure * cl) const335 void ZPageCache::pages_do(ZPageClosure* cl) const {
336 // Small
337 ZPerNUMAConstIterator<ZList<ZPage> > iter_numa(&_small);
338 for (const ZList<ZPage>* list; iter_numa.next(&list);) {
339 ZListIterator<ZPage> iter_small(list);
340 for (ZPage* page; iter_small.next(&page);) {
341 cl->do_page(page);
342 }
343 }
344
345 // Medium
346 ZListIterator<ZPage> iter_medium(&_medium);
347 for (ZPage* page; iter_medium.next(&page);) {
348 cl->do_page(page);
349 }
350
351 // Large
352 ZListIterator<ZPage> iter_large(&_large);
353 for (ZPage* page; iter_large.next(&page);) {
354 cl->do_page(page);
355 }
356 }
357