1//
2// Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
3// Copyright (c) 2021 SAP SE. All rights reserved.
4// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5//
6// This code is free software; you can redistribute it and/or modify it
7// under the terms of the GNU General Public License version 2 only, as
8// published by the Free Software Foundation.
9//
10// This code is distributed in the hope that it will be useful, but WITHOUT
11// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
13// version 2 for more details (a copy is included in the LICENSE file that
14// accompanied this code).
15//
16// You should have received a copy of the GNU General Public License version
17// 2 along with this work; if not, write to the Free Software Foundation,
18// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19//
20// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21// or visit www.oracle.com if you need additional information or have any
22// questions.
23//
24
25source_hpp %{
26
27#include "gc/shared/gc_globals.hpp"
28#include "gc/z/c2/zBarrierSetC2.hpp"
29#include "gc/z/zThreadLocalData.hpp"
30
31%}
32
33source %{
34
35static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref,
36                           Register tmp, uint8_t barrier_data) {
37  if (barrier_data == ZLoadBarrierElided) {
38    return;
39  }
40
41  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
42  __ ld(tmp, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread);
43  __ and_(tmp, tmp, ref);
44  __ bne_far(CCR0, *stub->entry(), MacroAssembler::bc_far_optimize_on_relocate);
45  __ bind(*stub->continuation());
46}
47
48static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref,
49                                     Register tmp) {
50  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, ZLoadBarrierStrong);
51  __ b(*stub->entry());
52  __ bind(*stub->continuation());
53}
54
55static void z_compare_and_swap(MacroAssembler& _masm, const MachNode* node,
56                              Register res, Register mem, Register oldval, Register newval,
57                              Register tmp_xchg, Register tmp_mask,
58                              bool weak, bool acquire) {
59  // z-specific load barrier requires strong CAS operations.
60  // Weak CAS operations are thus only emitted if the barrier is elided.
61  __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem,
62              MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, NULL, true,
63              weak && node->barrier_data() == ZLoadBarrierElided);
64
65  if (node->barrier_data() != ZLoadBarrierElided) {
66    Label skip_barrier;
67
68    __ ld(tmp_mask, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread);
69    __ and_(tmp_mask, tmp_mask, tmp_xchg);
70    __ beq(CCR0, skip_barrier);
71
72    // CAS must have failed because pointer in memory is bad.
73    z_load_barrier_slow_path(_masm, node, Address(mem), tmp_xchg, res /* used as tmp */);
74
75    __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem,
76                MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, NULL, true, weak);
77
78    __ bind(skip_barrier);
79  }
80
81  if (acquire) {
82    if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
83      // Uses the isync instruction as an acquire barrier.
84      // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync).
85      __ isync();
86    } else {
87      __ sync();
88    }
89  }
90}
91
92static void z_compare_and_exchange(MacroAssembler& _masm, const MachNode* node,
93                                   Register res, Register mem, Register oldval, Register newval, Register tmp,
94                                   bool weak, bool acquire) {
95  // z-specific load barrier requires strong CAS operations.
96  // Weak CAS operations are thus only emitted if the barrier is elided.
97  __ cmpxchgd(CCR0, res, oldval, newval, mem,
98              MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, NULL, true,
99              weak && node->barrier_data() == ZLoadBarrierElided);
100
101  if (node->barrier_data() != ZLoadBarrierElided) {
102    Label skip_barrier;
103    __ ld(tmp, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread);
104    __ and_(tmp, tmp, res);
105    __ beq(CCR0, skip_barrier);
106
107    z_load_barrier_slow_path(_masm, node, Address(mem), res, tmp);
108
109    __ cmpxchgd(CCR0, res, oldval, newval, mem,
110                MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, NULL, true, weak);
111
112    __ bind(skip_barrier);
113  }
114
115  if (acquire) {
116    if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
117      // Uses the isync instruction as an acquire barrier.
118      // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync).
119      __ isync();
120    } else {
121      __ sync();
122    }
123  }
124}
125
126%}
127
128instruct zLoadP(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0)
129%{
130  match(Set dst (LoadP mem));
131  effect(TEMP_DEF dst, TEMP tmp, KILL cr0);
132  ins_cost(MEMORY_REF_COST);
133
134  predicate((UseZGC && n->as_Load()->barrier_data() != 0)
135            && (n->as_Load()->is_unordered() || followed_by_acquire(n)));
136
137  format %{ "LD $dst, $mem" %}
138  ins_encode %{
139    assert($mem$$index == 0, "sanity");
140    __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register);
141    z_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data());
142  %}
143  ins_pipe(pipe_class_default);
144%}
145
146// Load Pointer Volatile
147instruct zLoadP_acq(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0)
148%{
149  match(Set dst (LoadP mem));
150  effect(TEMP_DEF dst, TEMP tmp, KILL cr0);
151  ins_cost(3 * MEMORY_REF_COST);
152
153  // Predicate on instruction order is implicitly present due to the predicate of the cheaper zLoadP operation
154  predicate(UseZGC && n->as_Load()->barrier_data() != 0);
155
156  format %{ "LD acq $dst, $mem" %}
157  ins_encode %{
158    __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register);
159    z_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data());
160
161    // Uses the isync instruction as an acquire barrier.
162    // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync).
163    __ isync();
164  %}
165  ins_pipe(pipe_class_default);
166%}
167
168instruct zCompareAndSwapP(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
169                          iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
170  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
171  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
172
173  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
174            && (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst));
175
176  format %{ "CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
177  ins_encode %{
178    z_compare_and_swap(_masm, this,
179                       $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
180                       $tmp_xchg$$Register, $tmp_mask$$Register,
181                       false /* weak */, false /* acquire */);
182  %}
183  ins_pipe(pipe_class_default);
184%}
185
186instruct zCompareAndSwapP_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
187                              iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
188  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
189  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
190
191  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
192            && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst));
193
194  format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %}
195  ins_encode %{
196    z_compare_and_swap(_masm, this,
197                       $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
198                       $tmp_xchg$$Register, $tmp_mask$$Register,
199                       false /* weak */, true /* acquire */);
200  %}
201  ins_pipe(pipe_class_default);
202%}
203
204instruct zCompareAndSwapPWeak(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
205                              iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
206  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
207  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
208
209  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
210            && ((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst);
211
212  format %{ "weak CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
213  ins_encode %{
214    z_compare_and_swap(_masm, this,
215                       $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
216                       $tmp_xchg$$Register, $tmp_mask$$Register,
217                       true /* weak */, false /* acquire */);
218  %}
219  ins_pipe(pipe_class_default);
220%}
221
222instruct zCompareAndSwapPWeak_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
223                                  iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
224  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
225  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
226
227  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
228            && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst));
229
230  format %{ "weak CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %}
231  ins_encode %{
232    z_compare_and_swap(_masm, this,
233                       $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
234                       $tmp_xchg$$Register, $tmp_mask$$Register,
235                       true /* weak */, true /* acquire */);
236  %}
237  ins_pipe(pipe_class_default);
238%}
239
240instruct zCompareAndExchangeP(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
241                              iRegPdst tmp, flagsRegCR0 cr0) %{
242  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
243  effect(TEMP_DEF res, TEMP tmp, KILL cr0);
244
245  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
246            && (
247              ((CompareAndSwapNode*)n)->order() != MemNode::acquire
248              && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst
249            ));
250
251  format %{ "CMPXCHG $res, $mem, $oldval, $newval; as ptr; ptr" %}
252  ins_encode %{
253    z_compare_and_exchange(_masm, this,
254                           $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register,
255                           false /* weak */, false /* acquire */);
256  %}
257  ins_pipe(pipe_class_default);
258%}
259
260instruct zCompareAndExchangeP_acq(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
261                                  iRegPdst tmp, flagsRegCR0 cr0) %{
262  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
263  effect(TEMP_DEF res, TEMP tmp, KILL cr0);
264
265  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
266            && (
267              ((CompareAndSwapNode*)n)->order() == MemNode::acquire
268              || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst
269            ));
270
271  format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as ptr; ptr" %}
272  ins_encode %{
273    z_compare_and_exchange(_masm, this,
274                           $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register,
275                           false /* weak */, true /* acquire */);
276  %}
277  ins_pipe(pipe_class_default);
278%}
279
280instruct zGetAndSetP(iRegPdst res, iRegPdst mem, iRegPsrc newval, iRegPdst tmp, flagsRegCR0 cr0) %{
281  match(Set res (GetAndSetP mem newval));
282  effect(TEMP_DEF res, TEMP tmp, KILL cr0);
283
284  predicate(UseZGC && n->as_LoadStore()->barrier_data() != 0);
285
286  format %{ "GetAndSetP $res, $mem, $newval" %}
287  ins_encode %{
288    __ getandsetd($res$$Register, $newval$$Register, $mem$$Register, MacroAssembler::cmpxchgx_hint_atomic_update());
289    z_load_barrier(_masm, this, Address(noreg, (intptr_t) 0), $res$$Register, $tmp$$Register, barrier_data());
290
291    if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
292      __ isync();
293    } else {
294      __ sync();
295    }
296  %}
297  ins_pipe(pipe_class_default);
298%}
299