1 /*
2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "classfile/javaClasses.hpp"
26 #include "gc/z/c2/zBarrierSetC2.hpp"
27 #include "gc/z/zBarrierSet.hpp"
28 #include "gc/z/zBarrierSetAssembler.hpp"
29 #include "gc/z/zBarrierSetRuntime.hpp"
30 #include "opto/arraycopynode.hpp"
31 #include "opto/addnode.hpp"
32 #include "opto/block.hpp"
33 #include "opto/compile.hpp"
34 #include "opto/graphKit.hpp"
35 #include "opto/machnode.hpp"
36 #include "opto/macro.hpp"
37 #include "opto/memnode.hpp"
38 #include "opto/node.hpp"
39 #include "opto/output.hpp"
40 #include "opto/regalloc.hpp"
41 #include "opto/rootnode.hpp"
42 #include "opto/type.hpp"
43 #include "utilities/growableArray.hpp"
44 #include "utilities/macros.hpp"
45
46 class ZBarrierSetC2State : public ResourceObj {
47 private:
48 GrowableArray<ZLoadBarrierStubC2*>* _stubs;
49 Node_Array _live;
50
51 public:
ZBarrierSetC2State(Arena * arena)52 ZBarrierSetC2State(Arena* arena) :
53 _stubs(new (arena) GrowableArray<ZLoadBarrierStubC2*>(arena, 8, 0, NULL)),
54 _live(arena) {}
55
stubs()56 GrowableArray<ZLoadBarrierStubC2*>* stubs() {
57 return _stubs;
58 }
59
live(const Node * node)60 RegMask* live(const Node* node) {
61 if (!node->is_Mach()) {
62 // Don't need liveness for non-MachNodes
63 return NULL;
64 }
65
66 const MachNode* const mach = node->as_Mach();
67 if (mach->barrier_data() != ZLoadBarrierStrong &&
68 mach->barrier_data() != ZLoadBarrierWeak) {
69 // Don't need liveness data for nodes without barriers
70 return NULL;
71 }
72
73 RegMask* live = (RegMask*)_live[node->_idx];
74 if (live == NULL) {
75 live = new (Compile::current()->comp_arena()->Amalloc_D(sizeof(RegMask))) RegMask();
76 _live.map(node->_idx, (Node*)live);
77 }
78
79 return live;
80 }
81 };
82
barrier_set_state()83 static ZBarrierSetC2State* barrier_set_state() {
84 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
85 }
86
create(const MachNode * node,Address ref_addr,Register ref,Register tmp,bool weak)87 ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) {
88 ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref, tmp, weak);
89 if (!Compile::current()->output()->in_scratch_emit_size()) {
90 barrier_set_state()->stubs()->append(stub);
91 }
92
93 return stub;
94 }
95
ZLoadBarrierStubC2(const MachNode * node,Address ref_addr,Register ref,Register tmp,bool weak)96 ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) :
97 _node(node),
98 _ref_addr(ref_addr),
99 _ref(ref),
100 _tmp(tmp),
101 _weak(weak),
102 _entry(),
103 _continuation() {
104 assert_different_registers(ref, ref_addr.base());
105 assert_different_registers(ref, ref_addr.index());
106 }
107
ref_addr() const108 Address ZLoadBarrierStubC2::ref_addr() const {
109 return _ref_addr;
110 }
111
ref() const112 Register ZLoadBarrierStubC2::ref() const {
113 return _ref;
114 }
115
tmp() const116 Register ZLoadBarrierStubC2::tmp() const {
117 return _tmp;
118 }
119
slow_path() const120 address ZLoadBarrierStubC2::slow_path() const {
121 const DecoratorSet decorators = _weak ? ON_WEAK_OOP_REF : ON_STRONG_OOP_REF;
122 return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
123 }
124
live() const125 RegMask& ZLoadBarrierStubC2::live() const {
126 return *barrier_set_state()->live(_node);
127 }
128
entry()129 Label* ZLoadBarrierStubC2::entry() {
130 // The _entry will never be bound when in_scratch_emit_size() is true.
131 // However, we still need to return a label that is not bound now, but
132 // will eventually be bound. Any lable will do, as it will only act as
133 // a placeholder, so we return the _continuation label.
134 return Compile::current()->output()->in_scratch_emit_size() ? &_continuation : &_entry;
135 }
136
continuation()137 Label* ZLoadBarrierStubC2::continuation() {
138 return &_continuation;
139 }
140
create_barrier_state(Arena * comp_arena) const141 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
142 return new (comp_arena) ZBarrierSetC2State(comp_arena);
143 }
144
late_barrier_analysis() const145 void ZBarrierSetC2::late_barrier_analysis() const {
146 analyze_dominating_barriers();
147 compute_liveness_at_stubs();
148 }
149
emit_stubs(CodeBuffer & cb) const150 void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
151 MacroAssembler masm(&cb);
152 GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
153
154 for (int i = 0; i < stubs->length(); i++) {
155 // Make sure there is enough space in the code buffer
156 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == NULL) {
157 ciEnv::current()->record_failure("CodeCache is full");
158 return;
159 }
160
161 ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
162 }
163
164 masm.flush();
165 }
166
estimate_stub_size() const167 int ZBarrierSetC2::estimate_stub_size() const {
168 Compile* const C = Compile::current();
169 BufferBlob* const blob = C->output()->scratch_buffer_blob();
170 GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
171 int size = 0;
172
173 for (int i = 0; i < stubs->length(); i++) {
174 CodeBuffer cb(blob->content_begin(), (address)C->output()->scratch_locs_memory() - blob->content_begin());
175 MacroAssembler masm(&cb);
176 ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
177 size += cb.insts_size();
178 }
179
180 return size;
181 }
182
set_barrier_data(C2Access & access)183 static void set_barrier_data(C2Access& access) {
184 if (ZBarrierSet::barrier_needed(access.decorators(), access.type())) {
185 if (access.decorators() & ON_WEAK_OOP_REF) {
186 access.set_barrier_data(ZLoadBarrierWeak);
187 } else {
188 access.set_barrier_data(ZLoadBarrierStrong);
189 }
190 }
191 }
192
load_at_resolved(C2Access & access,const Type * val_type) const193 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
194 set_barrier_data(access);
195 return BarrierSetC2::load_at_resolved(access, val_type);
196 }
197
atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess & access,Node * expected_val,Node * new_val,const Type * val_type) const198 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
199 Node* new_val, const Type* val_type) const {
200 set_barrier_data(access);
201 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
202 }
203
atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess & access,Node * expected_val,Node * new_val,const Type * value_type) const204 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
205 Node* new_val, const Type* value_type) const {
206 set_barrier_data(access);
207 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
208 }
209
atomic_xchg_at_resolved(C2AtomicParseAccess & access,Node * new_val,const Type * val_type) const210 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
211 set_barrier_data(access);
212 return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
213 }
214
array_copy_requires_gc_barriers(bool tightly_coupled_alloc,BasicType type,bool is_clone,ArrayCopyPhase phase) const215 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
216 bool is_clone, ArrayCopyPhase phase) const {
217 return type == T_OBJECT || type == T_ARRAY;
218 }
219
220 // This TypeFunc assumes a 64bit system
clone_type()221 static const TypeFunc* clone_type() {
222 // Create input type (domain)
223 const Type** domain_fields = TypeTuple::fields(4);
224 domain_fields[TypeFunc::Parms + 0] = TypeInstPtr::NOTNULL; // src
225 domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL; // dst
226 domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG; // size lower
227 domain_fields[TypeFunc::Parms + 3] = Type::HALF; // size upper
228 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
229
230 // Create result type (range)
231 const Type** range_fields = TypeTuple::fields(0);
232 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
233
234 return TypeFunc::make(domain, range);
235 }
236
clone_at_expansion(PhaseMacroExpand * phase,ArrayCopyNode * ac) const237 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
238 Node* const src = ac->in(ArrayCopyNode::Src);
239 if (ac->is_clone_array()) {
240 // Clone primitive array
241 BarrierSetC2::clone_at_expansion(phase, ac);
242 return;
243 }
244
245 // Clone instance
246 Node* const ctrl = ac->in(TypeFunc::Control);
247 Node* const mem = ac->in(TypeFunc::Memory);
248 Node* const dst = ac->in(ArrayCopyNode::Dest);
249 Node* const size = ac->in(ArrayCopyNode::Length);
250
251 assert(ac->is_clone_inst(), "Sanity check");
252 assert(size->bottom_type()->is_long(), "Should be long");
253
254 // The native clone we are calling here expects the instance size in words
255 // Add header/offset size to payload size to get instance size.
256 Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(false) >> LogBytesPerLong);
257 Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
258
259 Node* const call = phase->make_leaf_call(ctrl,
260 mem,
261 clone_type(),
262 ZBarrierSetRuntime::clone_addr(),
263 "ZBarrierSetRuntime::clone",
264 TypeRawPtr::BOTTOM,
265 src,
266 dst,
267 full_size,
268 phase->top());
269 phase->transform_later(call);
270 phase->igvn().replace_node(ac, call);
271 }
272
273 // == Dominating barrier elision ==
274
block_has_safepoint(const Block * block,uint from,uint to)275 static bool block_has_safepoint(const Block* block, uint from, uint to) {
276 for (uint i = from; i < to; i++) {
277 if (block->get_node(i)->is_MachSafePoint()) {
278 // Safepoint found
279 return true;
280 }
281 }
282
283 // Safepoint not found
284 return false;
285 }
286
block_has_safepoint(const Block * block)287 static bool block_has_safepoint(const Block* block) {
288 return block_has_safepoint(block, 0, block->number_of_nodes());
289 }
290
block_index(const Block * block,const Node * node)291 static uint block_index(const Block* block, const Node* node) {
292 for (uint j = 0; j < block->number_of_nodes(); ++j) {
293 if (block->get_node(j) == node) {
294 return j;
295 }
296 }
297 ShouldNotReachHere();
298 return 0;
299 }
300
analyze_dominating_barriers() const301 void ZBarrierSetC2::analyze_dominating_barriers() const {
302 ResourceMark rm;
303 Compile* const C = Compile::current();
304 PhaseCFG* const cfg = C->cfg();
305 Block_List worklist;
306 Node_List mem_ops;
307 Node_List barrier_loads;
308
309 // Step 1 - Find accesses, and track them in lists
310 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
311 const Block* const block = cfg->get_block(i);
312 for (uint j = 0; j < block->number_of_nodes(); ++j) {
313 const Node* const node = block->get_node(j);
314 if (!node->is_Mach()) {
315 continue;
316 }
317
318 MachNode* const mach = node->as_Mach();
319 switch (mach->ideal_Opcode()) {
320 case Op_LoadP:
321 case Op_CompareAndExchangeP:
322 case Op_CompareAndSwapP:
323 case Op_GetAndSetP:
324 if (mach->barrier_data() == ZLoadBarrierStrong) {
325 barrier_loads.push(mach);
326 }
327 case Op_StoreP:
328 mem_ops.push(mach);
329 break;
330
331 default:
332 break;
333 }
334 }
335 }
336
337 // Step 2 - Find dominating accesses for each load
338 for (uint i = 0; i < barrier_loads.size(); i++) {
339 MachNode* const load = barrier_loads.at(i)->as_Mach();
340 const TypePtr* load_adr_type = NULL;
341 intptr_t load_offset = 0;
342 const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type);
343 Block* const load_block = cfg->get_block_for_node(load);
344 const uint load_index = block_index(load_block, load);
345
346 for (uint j = 0; j < mem_ops.size(); j++) {
347 MachNode* mem = mem_ops.at(j)->as_Mach();
348 const TypePtr* mem_adr_type = NULL;
349 intptr_t mem_offset = 0;
350 const Node* mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type);
351 Block* mem_block = cfg->get_block_for_node(mem);
352 uint mem_index = block_index(mem_block, mem);
353
354 if (load_obj == NodeSentinel || mem_obj == NodeSentinel ||
355 load_obj == NULL || mem_obj == NULL ||
356 load_offset < 0 || mem_offset < 0) {
357 continue;
358 }
359
360 if (mem_obj != load_obj || mem_offset != load_offset) {
361 // Not the same addresses, not a candidate
362 continue;
363 }
364
365 if (load_block == mem_block) {
366 // Earlier accesses in the same block
367 if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) {
368 load->set_barrier_data(ZLoadBarrierElided);
369 }
370 } else if (mem_block->dominates(load_block)) {
371 // Dominating block? Look around for safepoints
372 ResourceMark rm;
373 Block_List stack;
374 VectorSet visited(Thread::current()->resource_area());
375 stack.push(load_block);
376 bool safepoint_found = block_has_safepoint(load_block);
377 while (!safepoint_found && stack.size() > 0) {
378 Block* block = stack.pop();
379 if (visited.test_set(block->_pre_order)) {
380 continue;
381 }
382 if (block_has_safepoint(block)) {
383 safepoint_found = true;
384 break;
385 }
386 if (block == mem_block) {
387 continue;
388 }
389
390 // Push predecessor blocks
391 for (uint p = 1; p < block->num_preds(); ++p) {
392 Block* pred = cfg->get_block_for_node(block->pred(p));
393 stack.push(pred);
394 }
395 }
396
397 if (!safepoint_found) {
398 load->set_barrier_data(ZLoadBarrierElided);
399 }
400 }
401 }
402 }
403 }
404
405 // == Reduced spilling optimization ==
406
compute_liveness_at_stubs() const407 void ZBarrierSetC2::compute_liveness_at_stubs() const {
408 ResourceMark rm;
409 Compile* const C = Compile::current();
410 Arena* const A = Thread::current()->resource_area();
411 PhaseCFG* const cfg = C->cfg();
412 PhaseRegAlloc* const regalloc = C->regalloc();
413 RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
414 ZBarrierSetAssembler* const bs = ZBarrierSet::assembler();
415 Block_List worklist;
416
417 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
418 new ((void*)(live + i)) RegMask();
419 worklist.push(cfg->get_block(i));
420 }
421
422 while (worklist.size() > 0) {
423 const Block* const block = worklist.pop();
424 RegMask& old_live = live[block->_pre_order];
425 RegMask new_live;
426
427 // Initialize to union of successors
428 for (uint i = 0; i < block->_num_succs; i++) {
429 const uint succ_id = block->_succs[i]->_pre_order;
430 new_live.OR(live[succ_id]);
431 }
432
433 // Walk block backwards, computing liveness
434 for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
435 const Node* const node = block->get_node(i);
436
437 // Remove def bits
438 const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
439 const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
440 if (first != OptoReg::Bad) {
441 new_live.Remove(first);
442 }
443 if (second != OptoReg::Bad) {
444 new_live.Remove(second);
445 }
446
447 // Add use bits
448 for (uint j = 1; j < node->req(); ++j) {
449 const Node* const use = node->in(j);
450 const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
451 const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
452 if (first != OptoReg::Bad) {
453 new_live.Insert(first);
454 }
455 if (second != OptoReg::Bad) {
456 new_live.Insert(second);
457 }
458 }
459
460 // If this node tracks liveness, update it
461 RegMask* const regs = barrier_set_state()->live(node);
462 if (regs != NULL) {
463 regs->OR(new_live);
464 }
465 }
466
467 // Now at block top, see if we have any changes
468 new_live.SUBTRACT(old_live);
469 if (new_live.is_NotEmpty()) {
470 // Liveness has refined, update and propagate to prior blocks
471 old_live.OR(new_live);
472 for (uint i = 1; i < block->num_preds(); ++i) {
473 Block* const pred = cfg->get_block_for_node(block->pred(i));
474 worklist.push(pred);
475 }
476 }
477 }
478 }
479