1//=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// AArch64 Atomic operand code-gen constructs.
10//
11//===----------------------------------------------------------------------===//
12
13//===----------------------------------
14// Atomic fences
15//===----------------------------------
16let AddedComplexity = 15, Size = 0 in
17def CompilerBarrier : Pseudo<(outs), (ins i32imm:$ordering),
18                             [(atomic_fence timm:$ordering, 0)]>, Sched<[]>;
19def : Pat<(atomic_fence (i64 4), (timm)), (DMB (i32 0x9))>;
20def : Pat<(atomic_fence (timm), (timm)), (DMB (i32 0xb))>;
21
22//===----------------------------------
23// Atomic loads
24//===----------------------------------
25
26// When they're actually atomic, only one addressing mode (GPR64sp) is
27// supported, but when they're relaxed and anything can be used, all the
28// standard modes would be valid and may give efficiency gains.
29
30// An atomic load operation that does not need either acquire or release
31// semantics.
32class relaxed_load<PatFrags base>
33  : PatFrag<(ops node:$ptr), (base node:$ptr)> {
34  let IsAtomic = 1;
35  let IsAtomicOrderingAcquireOrStronger = 0;
36}
37
38// A atomic load operation that actually needs acquire semantics.
39class acquiring_load<PatFrags base>
40  : PatFrag<(ops node:$ptr), (base node:$ptr)> {
41  let IsAtomic = 1;
42  let IsAtomicOrderingAcquire = 1;
43}
44
45// An atomic load operation that needs sequential consistency.
46class seq_cst_load<PatFrags base>
47  : PatFrag<(ops node:$ptr), (base node:$ptr)> {
48  let IsAtomic = 1;
49  let IsAtomicOrderingSequentiallyConsistent = 1;
50}
51
52// RCPC extension, currently opt-in under a separate feature.
53let Predicates = [HasLDAPR] in {
54  // v8.3 Release Consistent Processor Consistent support, optional in v8.2.
55  // 8-bit loads
56  def : Pat<(acquiring_load<atomic_load_8> GPR64sp:$ptr), (LDAPRB GPR64sp:$ptr)>;
57  // 16-bit loads
58  def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDAPRH GPR64sp:$ptr)>;
59  // 32-bit loads
60  def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDAPRW GPR64sp:$ptr)>;
61  // 64-bit loads
62  def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDAPRX GPR64sp:$ptr)>;
63}
64
65// 8-bit loads
66def : Pat<(seq_cst_load<atomic_load_az_8>  GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
67def : Pat<(acquiring_load<atomic_load_az_8>  GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
68def : Pat<(relaxed_load<atomic_load_az_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm,
69                                                     ro_Wextend8:$offset)),
70          (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>;
71def : Pat<(relaxed_load<atomic_load_az_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm,
72                                                        ro_Xextend8:$offset)),
73          (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>;
74def : Pat<(relaxed_load<atomic_load_az_8> (am_indexed8 GPR64sp:$Rn,
75                                           uimm12s1:$offset)),
76          (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
77def : Pat<(relaxed_load<atomic_load_az_8>
78               (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
79          (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
80
81// 16-bit loads
82def : Pat<(seq_cst_load<atomic_load_az_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
83def : Pat<(acquiring_load<atomic_load_az_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
84def : Pat<(relaxed_load<atomic_load_az_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
85                                                       ro_Wextend16:$extend)),
86          (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
87def : Pat<(relaxed_load<atomic_load_az_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
88                                                          ro_Xextend16:$extend)),
89          (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
90def : Pat<(relaxed_load<atomic_load_az_16> (am_indexed16 GPR64sp:$Rn,
91                                                         uimm12s2:$offset)),
92          (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>;
93def : Pat<(relaxed_load<atomic_load_az_16>
94               (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
95          (LDURHHi GPR64sp:$Rn, simm9:$offset)>;
96
97// 32-bit loads
98def : Pat<(seq_cst_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>;
99def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>;
100def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
101                                                       ro_Wextend32:$extend)),
102          (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
103def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
104                                                       ro_Xextend32:$extend)),
105          (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
106def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn,
107                                                      uimm12s4:$offset)),
108          (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>;
109def : Pat<(relaxed_load<atomic_load_32>
110               (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),
111          (LDURWi GPR64sp:$Rn, simm9:$offset)>;
112
113// 64-bit loads
114def : Pat<(seq_cst_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>;
115def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>;
116def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
117                                                       ro_Wextend64:$extend)),
118          (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
119def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
120                                                       ro_Xextend64:$extend)),
121          (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
122def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn,
123                                                      uimm12s8:$offset)),
124          (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>;
125def : Pat<(relaxed_load<atomic_load_64>
126               (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
127          (LDURXi GPR64sp:$Rn, simm9:$offset)>;
128
129// FP 32-bit loads
130def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
131                                                       ro_Wextend32:$extend))))),
132          (LDRSroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
133def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
134                                                       ro_Xextend32:$extend))))),
135          (LDRSroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
136def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn,
137                                                      uimm12s8:$offset))))),
138          (LDRSui GPR64sp:$Rn, uimm12s8:$offset)>;
139def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32>
140               (am_unscaled32 GPR64sp:$Rn, simm9:$offset))))),
141          (LDURSi GPR64sp:$Rn, simm9:$offset)>;
142
143// FP 64-bit loads
144def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
145                                                       ro_Wextend64:$extend))))),
146          (LDRDroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
147def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
148                                                       ro_Xextend64:$extend))))),
149          (LDRDroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
150def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn,
151                                                      uimm12s8:$offset))))),
152          (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
153def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64>
154               (am_unscaled64 GPR64sp:$Rn, simm9:$offset))))),
155          (LDURDi GPR64sp:$Rn, simm9:$offset)>;
156
157//===----------------------------------
158// Atomic stores
159//===----------------------------------
160
161// When they're actually atomic, only one addressing mode (GPR64sp) is
162// supported, but when they're relaxed and anything can be used, all the
163// standard modes would be valid and may give efficiency gains.
164
165// A store operation that actually needs release semantics.
166class releasing_store<PatFrag base>
167  : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
168  let IsAtomic = 1;
169  let IsAtomicOrderingReleaseOrStronger = 1;
170}
171
172// An atomic store operation that doesn't actually need to be atomic on AArch64.
173class relaxed_store<PatFrag base>
174  : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
175  let IsAtomic = 1;
176  let IsAtomicOrderingReleaseOrStronger = 0;
177}
178
179// 8-bit stores
180def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val),
181          (STLRB GPR32:$val, GPR64sp:$ptr)>;
182def : Pat<(relaxed_store<atomic_store_8>
183               (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend),
184               GPR32:$val),
185          (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>;
186def : Pat<(relaxed_store<atomic_store_8>
187               (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend),
188               GPR32:$val),
189          (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>;
190def : Pat<(relaxed_store<atomic_store_8>
191               (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val),
192          (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>;
193def : Pat<(relaxed_store<atomic_store_8>
194               (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
195          (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
196
197// 16-bit stores
198def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val),
199          (STLRH GPR32:$val, GPR64sp:$ptr)>;
200def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
201                                                         ro_Wextend16:$extend),
202                                          GPR32:$val),
203          (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
204def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
205                                                         ro_Xextend16:$extend),
206                                          GPR32:$val),
207          (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
208def : Pat<(relaxed_store<atomic_store_16>
209              (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val),
210          (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>;
211def : Pat<(relaxed_store<atomic_store_16>
212               (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
213          (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
214
215// 32-bit stores
216def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val),
217          (STLRW GPR32:$val, GPR64sp:$ptr)>;
218def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
219                                                         ro_Wextend32:$extend),
220                                          GPR32:$val),
221          (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
222def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
223                                                         ro_Xextend32:$extend),
224                                          GPR32:$val),
225          (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
226def : Pat<(relaxed_store<atomic_store_32>
227              (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val),
228          (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
229def : Pat<(relaxed_store<atomic_store_32>
230               (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
231          (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
232
233// 64-bit stores
234def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val),
235          (STLRX GPR64:$val, GPR64sp:$ptr)>;
236def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
237                                                         ro_Wextend16:$extend),
238                                          GPR64:$val),
239          (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
240def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
241                                                         ro_Xextend16:$extend),
242                                          GPR64:$val),
243          (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
244def : Pat<(relaxed_store<atomic_store_64>
245              (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val),
246          (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>;
247def : Pat<(relaxed_store<atomic_store_64>
248               (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),
249          (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
250
251// FP 32-bit stores
252def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
253                                                         ro_Wextend32:$extend),
254                                          (i32 (bitconvert (f32 FPR32Op:$val)))),
255          (STRSroW FPR32Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
256def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
257                                                         ro_Xextend32:$extend),
258                                          (i32 (bitconvert (f32 FPR32Op:$val)))),
259          (STRSroX FPR32Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
260def : Pat<(relaxed_store<atomic_store_32>
261              (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))),
262          (STRSui FPR32Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
263def : Pat<(relaxed_store<atomic_store_32>
264               (am_unscaled32 GPR64sp:$Rn, simm9:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))),
265          (STURSi FPR32Op:$val, GPR64sp:$Rn, simm9:$offset)>;
266
267// FP 64-bit stores
268def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
269                                                         ro_Wextend64:$extend),
270                                          (i64 (bitconvert (f64 FPR64Op:$val)))),
271          (STRDroW FPR64Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
272def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
273                                                         ro_Xextend64:$extend),
274                                          (i64 (bitconvert (f64 FPR64Op:$val)))),
275          (STRDroX FPR64Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
276def : Pat<(relaxed_store<atomic_store_64>
277              (am_indexed64 GPR64sp:$Rn, uimm12s4:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))),
278          (STRDui FPR64Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
279def : Pat<(relaxed_store<atomic_store_64>
280               (am_unscaled64 GPR64sp:$Rn, simm9:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))),
281          (STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>;
282
283//===----------------------------------
284// Low-level exclusive operations
285//===----------------------------------
286
287// Load-exclusives.
288
289def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
290  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
291}]> {
292  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];
293}
294
295def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
296  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
297}]> {
298  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];
299}
300
301def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
302  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
303}]> {
304  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];
305}
306
307def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
308  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
309}]> {
310  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
311}
312
313def : Pat<(ldxr_1 GPR64sp:$addr),
314          (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
315def : Pat<(ldxr_2 GPR64sp:$addr),
316          (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
317def : Pat<(ldxr_4 GPR64sp:$addr),
318          (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
319def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>;
320
321def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff),
322          (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
323def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff),
324          (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
325def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff),
326          (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
327
328// Load-exclusives.
329
330def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
331  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
332}]> {
333  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];
334}
335
336def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
337  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
338}]> {
339  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];
340}
341
342def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
343  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
344}]> {
345  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];
346}
347
348def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
349  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
350}]> {
351  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
352}
353
354def : Pat<(ldaxr_1 GPR64sp:$addr),
355          (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
356def : Pat<(ldaxr_2 GPR64sp:$addr),
357          (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
358def : Pat<(ldaxr_4 GPR64sp:$addr),
359          (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
360def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>;
361
362def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff),
363          (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
364def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff),
365          (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
366def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff),
367          (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
368
369// Store-exclusives.
370
371def stxr_1 : PatFrag<(ops node:$val, node:$ptr),
372                     (int_aarch64_stxr node:$val, node:$ptr), [{
373  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
374}]> {
375  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];
376}
377
378def stxr_2 : PatFrag<(ops node:$val, node:$ptr),
379                     (int_aarch64_stxr node:$val, node:$ptr), [{
380  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
381}]> {
382  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];
383}
384
385def stxr_4 : PatFrag<(ops node:$val, node:$ptr),
386                     (int_aarch64_stxr node:$val, node:$ptr), [{
387  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
388}]> {
389  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];
390}
391
392def stxr_8 : PatFrag<(ops node:$val, node:$ptr),
393                     (int_aarch64_stxr node:$val, node:$ptr), [{
394  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
395}]> {
396  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
397}
398
399
400def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr),
401          (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
402def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr),
403          (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
404def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr),
405          (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
406def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr),
407          (STXRX GPR64:$val, GPR64sp:$addr)>;
408
409def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
410          (STXRB GPR32:$val, GPR64sp:$addr)>;
411def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
412          (STXRH GPR32:$val, GPR64sp:$addr)>;
413def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr),
414          (STXRW GPR32:$val, GPR64sp:$addr)>;
415
416def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
417          (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
418def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
419          (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
420def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
421          (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
422
423// Store-release-exclusives.
424
425def stlxr_1 : PatFrag<(ops node:$val, node:$ptr),
426                     (int_aarch64_stlxr node:$val, node:$ptr), [{
427  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
428}]> {
429  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];
430}
431
432def stlxr_2 : PatFrag<(ops node:$val, node:$ptr),
433                     (int_aarch64_stlxr node:$val, node:$ptr), [{
434  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
435}]> {
436  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];
437}
438
439def stlxr_4 : PatFrag<(ops node:$val, node:$ptr),
440                     (int_aarch64_stlxr node:$val, node:$ptr), [{
441  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
442}]> {
443  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];
444}
445
446def stlxr_8 : PatFrag<(ops node:$val, node:$ptr),
447                     (int_aarch64_stlxr node:$val, node:$ptr), [{
448  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
449}]> {
450  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
451}
452
453
454def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr),
455          (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
456def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr),
457          (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
458def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr),
459          (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
460def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr),
461          (STLXRX GPR64:$val, GPR64sp:$addr)>;
462
463def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
464          (STLXRB GPR32:$val, GPR64sp:$addr)>;
465def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
466          (STLXRH GPR32:$val, GPR64sp:$addr)>;
467def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr),
468          (STLXRW GPR32:$val, GPR64sp:$addr)>;
469
470def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
471          (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
472def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
473          (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
474def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
475          (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
476
477
478// And clear exclusive.
479
480def : Pat<(int_aarch64_clrex), (CLREX 0xf)>;
481
482//===----------------------------------
483// Atomic cmpxchg for -O0
484//===----------------------------------
485
486// The fast register allocator used during -O0 inserts spills to cover any VRegs
487// live across basic block boundaries. When this happens between an LDXR and an
488// STXR it can clear the exclusive monitor, causing all cmpxchg attempts to
489// fail.
490
491// Unfortunately, this means we have to have an alternative (expanded
492// post-regalloc) path for -O0 compilations. Fortunately this path can be
493// significantly more naive than the standard expansion: we conservatively
494// assume seq_cst, strong cmpxchg and omit clrex on failure.
495
496let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch",
497    mayLoad = 1, mayStore = 1 in {
498def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
499                        (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
500                 Sched<[WriteAtomic]>;
501
502def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
503                         (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
504                  Sched<[WriteAtomic]>;
505
506def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
507                         (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
508                  Sched<[WriteAtomic]>;
509
510def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch),
511                         (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>,
512                  Sched<[WriteAtomic]>;
513}
514
515let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch",
516    mayLoad = 1, mayStore = 1 in {
517class cmp_swap_128 : Pseudo<(outs GPR64common:$RdLo, GPR64common:$RdHi,
518                                  GPR32common:$scratch),
519                           (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi,
520                                GPR64:$newLo, GPR64:$newHi), []>,
521                     Sched<[WriteAtomic]>;
522def CMP_SWAP_128 : cmp_swap_128;
523def CMP_SWAP_128_RELEASE : cmp_swap_128;
524def CMP_SWAP_128_ACQUIRE : cmp_swap_128;
525def CMP_SWAP_128_MONOTONIC : cmp_swap_128;
526}
527
528// v8.1 Atomic instructions:
529let Predicates = [HasLSE] in {
530  defm : LDOPregister_patterns<"LDADD", "atomic_load_add">;
531  defm : LDOPregister_patterns<"LDSET", "atomic_load_or">;
532  defm : LDOPregister_patterns<"LDEOR", "atomic_load_xor">;
533  defm : LDOPregister_patterns<"LDCLR", "atomic_load_clr">;
534  defm : LDOPregister_patterns<"LDSMAX", "atomic_load_max">;
535  defm : LDOPregister_patterns<"LDSMIN", "atomic_load_min">;
536  defm : LDOPregister_patterns<"LDUMAX", "atomic_load_umax">;
537  defm : LDOPregister_patterns<"LDUMIN", "atomic_load_umin">;
538  defm : LDOPregister_patterns<"SWP", "atomic_swap">;
539  defm : CASregister_patterns<"CAS", "atomic_cmp_swap">;
540
541  // These two patterns are only needed for global isel, selection dag isel
542  // converts atomic load-sub into a sub and atomic load-add, and likewise for
543  // and -> clr.
544  defm : LDOPregister_patterns_mod<"LDADD", "atomic_load_sub", "SUB">;
545  defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">;
546}
547