1 /*
2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "gc/shared/satbMarkQueue.hpp"
28 #include "gc/shared/strongRootsScope.hpp"
29 #include "gc/shared/taskTerminator.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
31 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
32 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
33 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
34 #include "gc/shenandoah/shenandoahMark.inline.hpp"
35 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
36 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
37 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
39 #include "gc/shenandoah/shenandoahStringDedup.hpp"
40 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
41 #include "gc/shenandoah/shenandoahUtils.hpp"
42 #include "memory/iterator.inline.hpp"
43 #include "memory/resourceArea.hpp"
44
45 class ShenandoahUpdateRootsTask : public AbstractGangTask {
46 private:
47 ShenandoahRootUpdater* _root_updater;
48 bool _check_alive;
49 public:
ShenandoahUpdateRootsTask(ShenandoahRootUpdater * root_updater,bool check_alive)50 ShenandoahUpdateRootsTask(ShenandoahRootUpdater* root_updater, bool check_alive) :
51 AbstractGangTask("Shenandoah Update Roots"),
52 _root_updater(root_updater),
53 _check_alive(check_alive){
54 }
55
work(uint worker_id)56 void work(uint worker_id) {
57 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
58 ShenandoahParallelWorkerSession worker_session(worker_id);
59
60 ShenandoahHeap* heap = ShenandoahHeap::heap();
61 ShenandoahUpdateRefsClosure cl;
62 if (_check_alive) {
63 ShenandoahForwardedIsAliveClosure is_alive;
64 _root_updater->roots_do<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>(worker_id, &is_alive, &cl);
65 } else {
66 AlwaysTrueClosure always_true;;
67 _root_updater->roots_do<AlwaysTrueClosure, ShenandoahUpdateRefsClosure>(worker_id, &always_true, &cl);
68 }
69 }
70 };
71
72 class ShenandoahConcurrentMarkingTask : public AbstractGangTask {
73 private:
74 ShenandoahConcurrentMark* const _cm;
75 TaskTerminator* const _terminator;
76
77 public:
ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark * cm,TaskTerminator * terminator)78 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) :
79 AbstractGangTask("Shenandoah Concurrent Mark"), _cm(cm), _terminator(terminator) {
80 }
81
work(uint worker_id)82 void work(uint worker_id) {
83 ShenandoahHeap* heap = ShenandoahHeap::heap();
84 ShenandoahConcurrentWorkerSession worker_session(worker_id);
85 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
86 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
87 ShenandoahReferenceProcessor* rp = heap->ref_processor();
88 assert(rp != NULL, "need reference processor");
89 _cm->mark_loop(worker_id, _terminator, rp,
90 true /*cancellable*/,
91 ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP);
92 }
93 };
94
95 class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure {
96 private:
97 SATBMarkQueueSet& _satb_qset;
98 OopClosure* const _cl;
99 uintx _claim_token;
100
101 public:
ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet & satb_qset,OopClosure * cl)102 ShenandoahSATBAndRemarkThreadsClosure(SATBMarkQueueSet& satb_qset, OopClosure* cl) :
103 _satb_qset(satb_qset),
104 _cl(cl),
105 _claim_token(Threads::thread_claim_token()) {}
106
do_thread(Thread * thread)107 void do_thread(Thread* thread) {
108 if (thread->claim_threads_do(true, _claim_token)) {
109 // Transfer any partial buffer to the qset for completed buffer processing.
110 _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
111 if (thread->is_Java_thread()) {
112 if (_cl != NULL) {
113 ResourceMark rm;
114 thread->oops_do(_cl, NULL);
115 }
116 }
117 }
118 }
119 };
120
121 class ShenandoahFinalMarkingTask : public AbstractGangTask {
122 private:
123 ShenandoahConcurrentMark* _cm;
124 TaskTerminator* _terminator;
125 bool _dedup_string;
126
127 public:
ShenandoahFinalMarkingTask(ShenandoahConcurrentMark * cm,TaskTerminator * terminator,bool dedup_string)128 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) :
129 AbstractGangTask("Shenandoah Final Mark"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
130 }
131
work(uint worker_id)132 void work(uint worker_id) {
133 ShenandoahHeap* heap = ShenandoahHeap::heap();
134
135 ShenandoahParallelWorkerSession worker_session(worker_id);
136 ShenandoahReferenceProcessor* rp = heap->ref_processor();
137
138 // First drain remaining SATB buffers.
139 {
140 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
141
142 ShenandoahSATBBufferClosure cl(q);
143 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
144 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
145 assert(!heap->has_forwarded_objects(), "Not expected");
146
147 ShenandoahMarkRefsClosure<NO_DEDUP> mark_cl(q, rp);
148 ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set,
149 ShenandoahIUBarrier ? &mark_cl : NULL);
150 Threads::threads_do(&tc);
151 }
152 _cm->mark_loop(worker_id, _terminator, rp,
153 false /*not cancellable*/,
154 _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP);
155 assert(_cm->task_queues()->is_empty(), "Should be empty");
156 }
157 };
158
ShenandoahConcurrentMark()159 ShenandoahConcurrentMark::ShenandoahConcurrentMark() :
160 ShenandoahMark() {}
161
162 // Mark concurrent roots during concurrent phases
163 class ShenandoahMarkConcurrentRootsTask : public AbstractGangTask {
164 private:
165 SuspendibleThreadSetJoiner _sts_joiner;
166 ShenandoahConcurrentRootScanner _root_scanner;
167 ShenandoahObjToScanQueueSet* const _queue_set;
168 ShenandoahReferenceProcessor* const _rp;
169
170 public:
171 ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
172 ShenandoahReferenceProcessor* rp,
173 ShenandoahPhaseTimings::Phase phase,
174 uint nworkers);
175 void work(uint worker_id);
176 };
177
ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet * qs,ShenandoahReferenceProcessor * rp,ShenandoahPhaseTimings::Phase phase,uint nworkers)178 ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
179 ShenandoahReferenceProcessor* rp,
180 ShenandoahPhaseTimings::Phase phase,
181 uint nworkers) :
182 AbstractGangTask("Shenandoah Concurrent Mark Roots"),
183 _root_scanner(nworkers, phase),
184 _queue_set(qs),
185 _rp(rp) {
186 assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
187 }
188
work(uint worker_id)189 void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) {
190 ShenandoahConcurrentWorkerSession worker_session(worker_id);
191 ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
192 // Cannot enable string deduplication during root scanning. Otherwise,
193 // may result lock inversion between stack watermark and string dedup queue lock.
194 ShenandoahMarkRefsClosure<NO_DEDUP> cl(q, _rp);
195 _root_scanner.roots_do(&cl, worker_id);
196 }
197
mark_concurrent_roots()198 void ShenandoahConcurrentMark::mark_concurrent_roots() {
199 ShenandoahHeap* const heap = ShenandoahHeap::heap();
200 assert(!heap->has_forwarded_objects(), "Not expected");
201
202 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
203
204 WorkGang* workers = heap->workers();
205 ShenandoahReferenceProcessor* rp = heap->ref_processor();
206 task_queues()->reserve(workers->active_workers());
207 ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
208
209 workers->run_task(&task);
210 }
211
212 class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
213 private:
214 SATBMarkQueueSet& _qset;
215 public:
ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet & qset)216 ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) :
217 HandshakeClosure("Shenandoah Flush SATB Handshake"),
218 _qset(qset) {}
219
do_thread(Thread * thread)220 void do_thread(Thread* thread) {
221 _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
222 }
223 };
224
concurrent_mark()225 void ShenandoahConcurrentMark::concurrent_mark() {
226 ShenandoahHeap* const heap = ShenandoahHeap::heap();
227 WorkGang* workers = heap->workers();
228 uint nworkers = workers->active_workers();
229 task_queues()->reserve(nworkers);
230
231 ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
232 ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
233 for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
234 TaskTerminator terminator(nworkers, task_queues());
235 ShenandoahConcurrentMarkingTask task(this, &terminator);
236 workers->run_task(&task);
237
238 if (heap->cancelled_gc()) {
239 // GC is cancelled, break out.
240 break;
241 }
242
243 size_t before = qset.completed_buffers_num();
244 Handshake::execute(&flush_satb);
245 size_t after = qset.completed_buffers_num();
246
247 if (before == after) {
248 // No more retries needed, break out.
249 break;
250 }
251 }
252 assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled");
253 }
254
finish_mark()255 void ShenandoahConcurrentMark::finish_mark() {
256 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
257 assert(Thread::current()->is_VM_thread(), "Must by VM Thread");
258 finish_mark_work();
259 assert(task_queues()->is_empty(), "Should be empty");
260 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
261 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
262
263 ShenandoahHeap* const heap = ShenandoahHeap::heap();
264 heap->set_concurrent_mark_in_progress(false);
265 heap->mark_complete_marking_context();
266 }
267
finish_mark_work()268 void ShenandoahConcurrentMark::finish_mark_work() {
269 // Finally mark everything else we've got in our queues during the previous steps.
270 // It does two different things for concurrent vs. mark-compact GC:
271 // - For concurrent GC, it starts with empty task queues, drains the remaining
272 // SATB buffers, and then completes the marking closure.
273 // - For mark-compact GC, it starts out with the task queues seeded by initial
274 // root scan, and completes the closure, thus marking through all live objects
275 // The implementation is the same, so it's shared here.
276 ShenandoahHeap* const heap = ShenandoahHeap::heap();
277 ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark);
278 uint nworkers = heap->workers()->active_workers();
279 task_queues()->reserve(nworkers);
280
281 StrongRootsScope scope(nworkers);
282 TaskTerminator terminator(nworkers, task_queues());
283 ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
284 heap->workers()->run_task(&task);
285
286 assert(task_queues()->is_empty(), "Should be empty");
287 }
288
289
cancel()290 void ShenandoahConcurrentMark::cancel() {
291 clear();
292 ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor();
293 rp->abandon_partial_discovery();
294 }
295