1// 2// Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. 3// Copyright (c) 2021 SAP SE. All rights reserved. 4// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5// 6// This code is free software; you can redistribute it and/or modify it 7// under the terms of the GNU General Public License version 2 only, as 8// published by the Free Software Foundation. 9// 10// This code is distributed in the hope that it will be useful, but WITHOUT 11// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13// version 2 for more details (a copy is included in the LICENSE file that 14// accompanied this code). 15// 16// You should have received a copy of the GNU General Public License version 17// 2 along with this work; if not, write to the Free Software Foundation, 18// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19// 20// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21// or visit www.oracle.com if you need additional information or have any 22// questions. 23// 24 25source_hpp %{ 26 27#include "gc/shared/gc_globals.hpp" 28#include "gc/z/c2/zBarrierSetC2.hpp" 29#include "gc/z/zThreadLocalData.hpp" 30 31%} 32 33source %{ 34 35static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, 36 Register tmp, uint8_t barrier_data) { 37 if (barrier_data == ZLoadBarrierElided) { 38 return; 39 } 40 41 ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data); 42 __ ld(tmp, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread); 43 __ and_(tmp, tmp, ref); 44 __ bne_far(CCR0, *stub->entry(), MacroAssembler::bc_far_optimize_on_relocate); 45 __ bind(*stub->continuation()); 46} 47 48static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, 49 Register tmp) { 50 ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, ZLoadBarrierStrong); 51 __ b(*stub->entry()); 52 __ bind(*stub->continuation()); 53} 54 55static void z_compare_and_swap(MacroAssembler& _masm, const MachNode* node, 56 Register res, Register mem, Register oldval, Register newval, 57 Register tmp_xchg, Register tmp_mask, 58 bool weak, bool acquire) { 59 // z-specific load barrier requires strong CAS operations. 60 // Weak CAS operations are thus only emitted if the barrier is elided. 61 __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem, 62 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, NULL, true, 63 weak && node->barrier_data() == ZLoadBarrierElided); 64 65 if (node->barrier_data() != ZLoadBarrierElided) { 66 Label skip_barrier; 67 68 __ ld(tmp_mask, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread); 69 __ and_(tmp_mask, tmp_mask, tmp_xchg); 70 __ beq(CCR0, skip_barrier); 71 72 // CAS must have failed because pointer in memory is bad. 73 z_load_barrier_slow_path(_masm, node, Address(mem), tmp_xchg, res /* used as tmp */); 74 75 __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem, 76 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, NULL, true, weak); 77 78 __ bind(skip_barrier); 79 } 80 81 if (acquire) { 82 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 83 // Uses the isync instruction as an acquire barrier. 84 // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync). 85 __ isync(); 86 } else { 87 __ sync(); 88 } 89 } 90} 91 92static void z_compare_and_exchange(MacroAssembler& _masm, const MachNode* node, 93 Register res, Register mem, Register oldval, Register newval, Register tmp, 94 bool weak, bool acquire) { 95 // z-specific load barrier requires strong CAS operations. 96 // Weak CAS operations are thus only emitted if the barrier is elided. 97 __ cmpxchgd(CCR0, res, oldval, newval, mem, 98 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, NULL, true, 99 weak && node->barrier_data() == ZLoadBarrierElided); 100 101 if (node->barrier_data() != ZLoadBarrierElided) { 102 Label skip_barrier; 103 __ ld(tmp, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread); 104 __ and_(tmp, tmp, res); 105 __ beq(CCR0, skip_barrier); 106 107 z_load_barrier_slow_path(_masm, node, Address(mem), res, tmp); 108 109 __ cmpxchgd(CCR0, res, oldval, newval, mem, 110 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, NULL, true, weak); 111 112 __ bind(skip_barrier); 113 } 114 115 if (acquire) { 116 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 117 // Uses the isync instruction as an acquire barrier. 118 // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync). 119 __ isync(); 120 } else { 121 __ sync(); 122 } 123 } 124} 125 126%} 127 128instruct zLoadP(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0) 129%{ 130 match(Set dst (LoadP mem)); 131 effect(TEMP_DEF dst, TEMP tmp, KILL cr0); 132 ins_cost(MEMORY_REF_COST); 133 134 predicate((UseZGC && n->as_Load()->barrier_data() != 0) 135 && (n->as_Load()->is_unordered() || followed_by_acquire(n))); 136 137 format %{ "LD $dst, $mem" %} 138 ins_encode %{ 139 assert($mem$$index == 0, "sanity"); 140 __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register); 141 z_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data()); 142 %} 143 ins_pipe(pipe_class_default); 144%} 145 146// Load Pointer Volatile 147instruct zLoadP_acq(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0) 148%{ 149 match(Set dst (LoadP mem)); 150 effect(TEMP_DEF dst, TEMP tmp, KILL cr0); 151 ins_cost(3 * MEMORY_REF_COST); 152 153 // Predicate on instruction order is implicitly present due to the predicate of the cheaper zLoadP operation 154 predicate(UseZGC && n->as_Load()->barrier_data() != 0); 155 156 format %{ "LD acq $dst, $mem" %} 157 ins_encode %{ 158 __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register); 159 z_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data()); 160 161 // Uses the isync instruction as an acquire barrier. 162 // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync). 163 __ isync(); 164 %} 165 ins_pipe(pipe_class_default); 166%} 167 168instruct zCompareAndSwapP(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, 169 iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{ 170 match(Set res (CompareAndSwapP mem (Binary oldval newval))); 171 effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0); 172 173 predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong) 174 && (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst)); 175 176 format %{ "CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %} 177 ins_encode %{ 178 z_compare_and_swap(_masm, this, 179 $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, 180 $tmp_xchg$$Register, $tmp_mask$$Register, 181 false /* weak */, false /* acquire */); 182 %} 183 ins_pipe(pipe_class_default); 184%} 185 186instruct zCompareAndSwapP_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, 187 iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{ 188 match(Set res (CompareAndSwapP mem (Binary oldval newval))); 189 effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0); 190 191 predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong) 192 && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst)); 193 194 format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %} 195 ins_encode %{ 196 z_compare_and_swap(_masm, this, 197 $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, 198 $tmp_xchg$$Register, $tmp_mask$$Register, 199 false /* weak */, true /* acquire */); 200 %} 201 ins_pipe(pipe_class_default); 202%} 203 204instruct zCompareAndSwapPWeak(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, 205 iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{ 206 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); 207 effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0); 208 209 predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong) 210 && ((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst); 211 212 format %{ "weak CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %} 213 ins_encode %{ 214 z_compare_and_swap(_masm, this, 215 $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, 216 $tmp_xchg$$Register, $tmp_mask$$Register, 217 true /* weak */, false /* acquire */); 218 %} 219 ins_pipe(pipe_class_default); 220%} 221 222instruct zCompareAndSwapPWeak_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, 223 iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{ 224 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); 225 effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0); 226 227 predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong) 228 && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst)); 229 230 format %{ "weak CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %} 231 ins_encode %{ 232 z_compare_and_swap(_masm, this, 233 $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, 234 $tmp_xchg$$Register, $tmp_mask$$Register, 235 true /* weak */, true /* acquire */); 236 %} 237 ins_pipe(pipe_class_default); 238%} 239 240instruct zCompareAndExchangeP(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, 241 iRegPdst tmp, flagsRegCR0 cr0) %{ 242 match(Set res (CompareAndExchangeP mem (Binary oldval newval))); 243 effect(TEMP_DEF res, TEMP tmp, KILL cr0); 244 245 predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong) 246 && ( 247 ((CompareAndSwapNode*)n)->order() != MemNode::acquire 248 && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst 249 )); 250 251 format %{ "CMPXCHG $res, $mem, $oldval, $newval; as ptr; ptr" %} 252 ins_encode %{ 253 z_compare_and_exchange(_masm, this, 254 $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register, 255 false /* weak */, false /* acquire */); 256 %} 257 ins_pipe(pipe_class_default); 258%} 259 260instruct zCompareAndExchangeP_acq(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval, 261 iRegPdst tmp, flagsRegCR0 cr0) %{ 262 match(Set res (CompareAndExchangeP mem (Binary oldval newval))); 263 effect(TEMP_DEF res, TEMP tmp, KILL cr0); 264 265 predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong) 266 && ( 267 ((CompareAndSwapNode*)n)->order() == MemNode::acquire 268 || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst 269 )); 270 271 format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as ptr; ptr" %} 272 ins_encode %{ 273 z_compare_and_exchange(_masm, this, 274 $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register, 275 false /* weak */, true /* acquire */); 276 %} 277 ins_pipe(pipe_class_default); 278%} 279 280instruct zGetAndSetP(iRegPdst res, iRegPdst mem, iRegPsrc newval, iRegPdst tmp, flagsRegCR0 cr0) %{ 281 match(Set res (GetAndSetP mem newval)); 282 effect(TEMP_DEF res, TEMP tmp, KILL cr0); 283 284 predicate(UseZGC && n->as_LoadStore()->barrier_data() != 0); 285 286 format %{ "GetAndSetP $res, $mem, $newval" %} 287 ins_encode %{ 288 __ getandsetd($res$$Register, $newval$$Register, $mem$$Register, MacroAssembler::cmpxchgx_hint_atomic_update()); 289 z_load_barrier(_masm, this, Address(noreg, (intptr_t) 0), $res$$Register, $tmp$$Register, barrier_data()); 290 291 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 292 __ isync(); 293 } else { 294 __ sync(); 295 } 296 %} 297 ins_pipe(pipe_class_default); 298%} 299