1 /*
2 * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "classfile/classLoaderData.hpp"
26 #include "gc/shared/gc_globals.hpp"
27 #include "gc/z/zAddress.inline.hpp"
28 #include "gc/z/zHeap.inline.hpp"
29 #include "gc/z/zNMethod.hpp"
30 #include "gc/z/zOop.hpp"
31 #include "gc/z/zPageAllocator.hpp"
32 #include "gc/z/zResurrection.hpp"
33 #include "gc/z/zRootsIterator.hpp"
34 #include "gc/z/zStackWatermark.hpp"
35 #include "gc/z/zStat.hpp"
36 #include "gc/z/zVerify.hpp"
37 #include "memory/iterator.inline.hpp"
38 #include "memory/resourceArea.hpp"
39 #include "oops/oop.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/globals.hpp"
42 #include "runtime/handles.hpp"
43 #include "runtime/safepoint.hpp"
44 #include "runtime/stackFrameStream.inline.hpp"
45 #include "runtime/stackWatermark.inline.hpp"
46 #include "runtime/stackWatermarkSet.inline.hpp"
47 #include "runtime/thread.hpp"
48 #include "utilities/debug.hpp"
49 #include "utilities/globalDefinitions.hpp"
50 #include "utilities/preserveException.hpp"
51
52 #define BAD_OOP_ARG(o, p) "Bad oop " PTR_FORMAT " found at " PTR_FORMAT, p2i(o), p2i(p)
53
z_verify_oop(oop * p)54 static void z_verify_oop(oop* p) {
55 const oop o = RawAccess<>::oop_load(p);
56 if (o != NULL) {
57 const uintptr_t addr = ZOop::to_address(o);
58 guarantee(ZAddress::is_good(addr), BAD_OOP_ARG(o, p));
59 guarantee(oopDesc::is_oop(ZOop::from_address(addr)), BAD_OOP_ARG(o, p));
60 }
61 }
62
z_verify_possibly_weak_oop(oop * p)63 static void z_verify_possibly_weak_oop(oop* p) {
64 const oop o = RawAccess<>::oop_load(p);
65 if (o != NULL) {
66 const uintptr_t addr = ZOop::to_address(o);
67 guarantee(ZAddress::is_good(addr) || ZAddress::is_finalizable_good(addr), BAD_OOP_ARG(o, p));
68 guarantee(oopDesc::is_oop(ZOop::from_address(ZAddress::good(addr))), BAD_OOP_ARG(o, p));
69 }
70 }
71
72 class ZVerifyRootClosure : public OopClosure {
73 private:
74 const bool _verify_fixed;
75
76 public:
ZVerifyRootClosure(bool verify_fixed)77 ZVerifyRootClosure(bool verify_fixed) :
78 _verify_fixed(verify_fixed) {}
79
do_oop(oop * p)80 virtual void do_oop(oop* p) {
81 if (_verify_fixed) {
82 z_verify_oop(p);
83 } else {
84 // Don't know the state of the oop.
85 oop obj = *p;
86 obj = NativeAccess<AS_NO_KEEPALIVE>::oop_load(&obj);
87 z_verify_oop(&obj);
88 }
89 }
90
do_oop(narrowOop *)91 virtual void do_oop(narrowOop*) {
92 ShouldNotReachHere();
93 }
94
verify_fixed() const95 bool verify_fixed() const {
96 return _verify_fixed;
97 }
98 };
99
100 class ZVerifyCodeBlobClosure : public CodeBlobToOopClosure {
101 public:
ZVerifyCodeBlobClosure(ZVerifyRootClosure * _cl)102 ZVerifyCodeBlobClosure(ZVerifyRootClosure* _cl) :
103 CodeBlobToOopClosure(_cl, false /* fix_relocations */) {}
104
do_code_blob(CodeBlob * cb)105 virtual void do_code_blob(CodeBlob* cb) {
106 CodeBlobToOopClosure::do_code_blob(cb);
107 }
108 };
109
110 class ZVerifyStack : public OopClosure {
111 private:
112 ZVerifyRootClosure* const _cl;
113 JavaThread* const _jt;
114 uint64_t _last_good;
115 bool _verifying_bad_frames;
116
117 public:
ZVerifyStack(ZVerifyRootClosure * cl,JavaThread * jt)118 ZVerifyStack(ZVerifyRootClosure* cl, JavaThread* jt) :
119 _cl(cl),
120 _jt(jt),
121 _last_good(0),
122 _verifying_bad_frames(false) {
123 ZStackWatermark* const stack_watermark = StackWatermarkSet::get<ZStackWatermark>(jt, StackWatermarkKind::gc);
124
125 if (_cl->verify_fixed()) {
126 assert(stack_watermark->processing_started(), "Should already have been fixed");
127 assert(stack_watermark->processing_completed(), "Should already have been fixed");
128 } else {
129 // We don't really know the state of the stack, verify watermark.
130 if (!stack_watermark->processing_started()) {
131 _verifying_bad_frames = true;
132 } else {
133 // Not time yet to verify bad frames
134 _last_good = stack_watermark->last_processed();
135 }
136 }
137 }
138
do_oop(oop * p)139 void do_oop(oop* p) {
140 if (_verifying_bad_frames) {
141 const oop obj = *p;
142 guarantee(!ZAddress::is_good(ZOop::to_address(obj)), BAD_OOP_ARG(obj, p));
143 }
144 _cl->do_oop(p);
145 }
146
do_oop(narrowOop * p)147 void do_oop(narrowOop* p) {
148 ShouldNotReachHere();
149 }
150
prepare_next_frame(frame & frame)151 void prepare_next_frame(frame& frame) {
152 if (_cl->verify_fixed()) {
153 // All frames need to be good
154 return;
155 }
156
157 // The verification has two modes, depending on whether we have reached the
158 // last processed frame or not. Before it is reached, we expect everything to
159 // be good. After reaching it, we expect everything to be bad.
160 const uintptr_t sp = reinterpret_cast<uintptr_t>(frame.sp());
161
162 if (!_verifying_bad_frames && sp == _last_good) {
163 // Found the last good frame, now verify the bad ones
164 _verifying_bad_frames = true;
165 }
166 }
167
verify_frames()168 void verify_frames() {
169 ZVerifyCodeBlobClosure cb_cl(_cl);
170 for (StackFrameStream frames(_jt, true /* update */, false /* process_frames */);
171 !frames.is_done();
172 frames.next()) {
173 frame& frame = *frames.current();
174 frame.oops_do(this, &cb_cl, frames.register_map(), DerivedPointerIterationMode::_ignore);
175 prepare_next_frame(frame);
176 }
177 }
178 };
179
180 class ZVerifyOopClosure : public ClaimMetadataVisitingOopIterateClosure {
181 private:
182 const bool _verify_weaks;
183
184 public:
ZVerifyOopClosure(bool verify_weaks)185 ZVerifyOopClosure(bool verify_weaks) :
186 ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_other),
187 _verify_weaks(verify_weaks) {}
188
do_oop(oop * p)189 virtual void do_oop(oop* p) {
190 if (_verify_weaks) {
191 z_verify_possibly_weak_oop(p);
192 } else {
193 // We should never encounter finalizable oops through strong
194 // paths. This assumes we have only visited strong roots.
195 z_verify_oop(p);
196 }
197 }
198
do_oop(narrowOop * p)199 virtual void do_oop(narrowOop* p) {
200 ShouldNotReachHere();
201 }
202
reference_iteration_mode()203 virtual ReferenceIterationMode reference_iteration_mode() {
204 return _verify_weaks ? DO_FIELDS : DO_FIELDS_EXCEPT_REFERENT;
205 }
206 };
207
208 typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_none> ZVerifyCLDClosure;
209
210 class ZVerifyThreadClosure : public ThreadClosure {
211 private:
212 ZVerifyRootClosure* const _cl;
213
214 public:
ZVerifyThreadClosure(ZVerifyRootClosure * cl)215 ZVerifyThreadClosure(ZVerifyRootClosure* cl) :
216 _cl(cl) {}
217
do_thread(Thread * thread)218 virtual void do_thread(Thread* thread) {
219 thread->oops_do_no_frames(_cl, NULL);
220
221 JavaThread* const jt = thread->as_Java_thread();
222 if (!jt->has_last_Java_frame()) {
223 return;
224 }
225
226 ZVerifyStack verify_stack(_cl, jt);
227 verify_stack.verify_frames();
228 }
229 };
230
231 class ZVerifyNMethodClosure : public NMethodClosure {
232 private:
233 OopClosure* const _cl;
234 BarrierSetNMethod* const _bs_nm;
235 const bool _verify_fixed;
236
trust_nmethod_state() const237 bool trust_nmethod_state() const {
238 // The root iterator will visit non-processed
239 // nmethods class unloading is turned off.
240 return ClassUnloading || _verify_fixed;
241 }
242
243 public:
ZVerifyNMethodClosure(OopClosure * cl,bool verify_fixed)244 ZVerifyNMethodClosure(OopClosure* cl, bool verify_fixed) :
245 _cl(cl),
246 _bs_nm(BarrierSet::barrier_set()->barrier_set_nmethod()),
247 _verify_fixed(verify_fixed) {}
248
do_nmethod(nmethod * nm)249 virtual void do_nmethod(nmethod* nm) {
250 assert(!trust_nmethod_state() || !_bs_nm->is_armed(nm), "Should not encounter any armed nmethods");
251
252 ZNMethod::nmethod_oops_do(nm, _cl);
253 }
254 };
255
roots_strong(bool verify_fixed)256 void ZVerify::roots_strong(bool verify_fixed) {
257 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
258 assert(!ZResurrection::is_blocked(), "Invalid phase");
259
260 ZVerifyRootClosure cl(verify_fixed);
261 ZVerifyCLDClosure cld_cl(&cl);
262 ZVerifyThreadClosure thread_cl(&cl);
263 ZVerifyNMethodClosure nm_cl(&cl, verify_fixed);
264
265 ZRootsIterator iter(ClassLoaderData::_claim_none);
266 iter.apply(&cl,
267 &cld_cl,
268 &thread_cl,
269 &nm_cl);
270 }
271
roots_weak()272 void ZVerify::roots_weak() {
273 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
274 assert(!ZResurrection::is_blocked(), "Invalid phase");
275
276 ZVerifyRootClosure cl(true /* verify_fixed */);
277 ZWeakRootsIterator iter;
278 iter.apply(&cl);
279 }
280
objects(bool verify_weaks)281 void ZVerify::objects(bool verify_weaks) {
282 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
283 assert(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
284 assert(!ZResurrection::is_blocked(), "Invalid phase");
285
286 ZVerifyOopClosure cl(verify_weaks);
287 ObjectToOopClosure object_cl(&cl);
288 ZHeap::heap()->object_iterate(&object_cl, verify_weaks);
289 }
290
before_zoperation()291 void ZVerify::before_zoperation() {
292 // Verify strong roots
293 ZStatTimerDisable disable;
294 if (ZVerifyRoots) {
295 roots_strong(false /* verify_fixed */);
296 }
297 }
298
after_mark()299 void ZVerify::after_mark() {
300 // Verify all strong roots and strong references
301 ZStatTimerDisable disable;
302 if (ZVerifyRoots) {
303 roots_strong(true /* verify_fixed */);
304 }
305 if (ZVerifyObjects) {
306 objects(false /* verify_weaks */);
307 }
308 }
309
after_weak_processing()310 void ZVerify::after_weak_processing() {
311 // Verify all roots and all references
312 ZStatTimerDisable disable;
313 if (ZVerifyRoots) {
314 roots_strong(true /* verify_fixed */);
315 roots_weak();
316 }
317 if (ZVerifyObjects) {
318 objects(true /* verify_weaks */);
319 }
320 }
321
322 template <bool Map>
323 class ZPageDebugMapOrUnmapClosure : public ZPageClosure {
324 private:
325 const ZPageAllocator* const _allocator;
326
327 public:
ZPageDebugMapOrUnmapClosure(const ZPageAllocator * allocator)328 ZPageDebugMapOrUnmapClosure(const ZPageAllocator* allocator) :
329 _allocator(allocator) {}
330
do_page(const ZPage * page)331 void do_page(const ZPage* page) {
332 if (Map) {
333 _allocator->debug_map_page(page);
334 } else {
335 _allocator->debug_unmap_page(page);
336 }
337 }
338 };
339
ZVerifyViewsFlip(const ZPageAllocator * allocator)340 ZVerifyViewsFlip::ZVerifyViewsFlip(const ZPageAllocator* allocator) :
341 _allocator(allocator) {
342 if (ZVerifyViews) {
343 // Unmap all pages
344 ZPageDebugMapOrUnmapClosure<false /* Map */> cl(_allocator);
345 ZHeap::heap()->pages_do(&cl);
346 }
347 }
348
~ZVerifyViewsFlip()349 ZVerifyViewsFlip::~ZVerifyViewsFlip() {
350 if (ZVerifyViews) {
351 // Map all pages
352 ZPageDebugMapOrUnmapClosure<true /* Map */> cl(_allocator);
353 ZHeap::heap()->pages_do(&cl);
354 }
355 }
356
357 #ifdef ASSERT
358
359 class ZVerifyBadOopClosure : public OopClosure {
360 public:
do_oop(oop * p)361 virtual void do_oop(oop* p) {
362 const oop o = *p;
363 assert(!ZAddress::is_good(ZOop::to_address(o)), "Should not be good: " PTR_FORMAT, p2i(o));
364 }
365
do_oop(narrowOop * p)366 virtual void do_oop(narrowOop* p) {
367 ShouldNotReachHere();
368 }
369 };
370
371 // This class encapsulates various marks we need to deal with calling the
372 // frame iteration code from arbitrary points in the runtime. It is mostly
373 // due to problems that we might want to eventually clean up inside of the
374 // frame iteration code, such as creating random handles even though there
375 // is no safepoint to protect against, and fiddling around with exceptions.
376 class StackWatermarkProcessingMark {
377 ResetNoHandleMark _rnhm;
378 HandleMark _hm;
379 PreserveExceptionMark _pem;
380 ResourceMark _rm;
381
382 public:
StackWatermarkProcessingMark(Thread * thread)383 StackWatermarkProcessingMark(Thread* thread) :
384 _rnhm(),
385 _hm(thread),
386 _pem(thread),
387 _rm(thread) {}
388 };
389
verify_frame_bad(const frame & fr,RegisterMap & register_map)390 void ZVerify::verify_frame_bad(const frame& fr, RegisterMap& register_map) {
391 ZVerifyBadOopClosure verify_cl;
392 fr.oops_do(&verify_cl, NULL, ®ister_map, DerivedPointerIterationMode::_ignore);
393 }
394
verify_thread_head_bad(JavaThread * jt)395 void ZVerify::verify_thread_head_bad(JavaThread* jt) {
396 ZVerifyBadOopClosure verify_cl;
397 jt->oops_do_no_frames(&verify_cl, NULL);
398 }
399
verify_thread_frames_bad(JavaThread * jt)400 void ZVerify::verify_thread_frames_bad(JavaThread* jt) {
401 if (jt->has_last_Java_frame()) {
402 ZVerifyBadOopClosure verify_cl;
403 StackWatermarkProcessingMark swpm(Thread::current());
404 // Traverse the execution stack
405 for (StackFrameStream fst(jt, true /* update */, false /* process_frames */); !fst.is_done(); fst.next()) {
406 fst.current()->oops_do(&verify_cl, NULL /* code_cl */, fst.register_map(), DerivedPointerIterationMode::_ignore);
407 }
408 }
409 }
410
411 #endif // ASSERT
412