1; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
2; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-REG
3
4
5; Point of CHECK-REG is to make sure UNPREDICTABLE instructions aren't created
6; (i.e. reusing a register for status & data in store exclusive).
7; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], w[[NEW]], [x{{[0-9]+}}]
8; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], x[[NEW]], [x{{[0-9]+}}]
9
10@var8 = global i8 0
11@var16 = global i16 0
12@var32 = global i32 0
13@var64 = global i64 0
14
15define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
16; CHECK-LABEL: test_atomic_load_add_i8:
17   %old = atomicrmw add i8* @var8, i8 %offset seq_cst
18; CHECK-NOT: dmb
19; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
20; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
21
22; CHECK: .LBB{{[0-9]+}}_1:
23; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
24  ; w0 below is a reasonable guess but could change: it certainly comes into the
25  ;  function there.
26; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
27; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
28; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
29; CHECK-NOT: dmb
30
31; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
32   ret i8 %old
33}
34
35define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
36; CHECK-LABEL: test_atomic_load_add_i16:
37   %old = atomicrmw add i16* @var16, i16 %offset acquire
38; CHECK-NOT: dmb
39; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
40; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
41
42; CHECK: .LBB{{[0-9]+}}_1:
43; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
44  ; w0 below is a reasonable guess but could change: it certainly comes into the
45  ;  function there.
46; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
47; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
48; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
49; CHECK-NOT: dmb
50
51; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
52   ret i16 %old
53}
54
55define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
56; CHECK-LABEL: test_atomic_load_add_i32:
57   %old = atomicrmw add i32* @var32, i32 %offset release
58; CHECK-NOT: dmb
59; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
60; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
61
62; CHECK: .LBB{{[0-9]+}}_1:
63; ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
64  ; w0 below is a reasonable guess but could change: it certainly comes into the
65  ;  function there.
66; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
67; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
68; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
69; CHECK-NOT: dmb
70
71; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
72   ret i32 %old
73}
74
75define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
76; CHECK-LABEL: test_atomic_load_add_i64:
77   %old = atomicrmw add i64* @var64, i64 %offset monotonic
78; CHECK-NOT: dmb
79; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
80; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
81
82; CHECK: .LBB{{[0-9]+}}_1:
83; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
84  ; x0 below is a reasonable guess but could change: it certainly comes into the
85  ; function there.
86; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0
87; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
88; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
89; CHECK-NOT: dmb
90
91; CHECK: mov x0, x[[OLD]]
92   ret i64 %old
93}
94
95define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
96; CHECK-LABEL: test_atomic_load_sub_i8:
97   %old = atomicrmw sub i8* @var8, i8 %offset monotonic
98; CHECK-NOT: dmb
99; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
100; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
101
102; CHECK: .LBB{{[0-9]+}}_1:
103; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
104  ; w0 below is a reasonable guess but could change: it certainly comes into the
105  ;  function there.
106; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
107; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
108; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
109; CHECK-NOT: dmb
110
111; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
112   ret i8 %old
113}
114
115define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
116; CHECK-LABEL: test_atomic_load_sub_i16:
117   %old = atomicrmw sub i16* @var16, i16 %offset release
118; CHECK-NOT: dmb
119; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
120; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
121
122; CHECK: .LBB{{[0-9]+}}_1:
123; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
124  ; w0 below is a reasonable guess but could change: it certainly comes into the
125  ;  function there.
126; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
127; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
128; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
129; CHECK-NOT: dmb
130
131; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
132   ret i16 %old
133}
134
135define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
136; CHECK-LABEL: test_atomic_load_sub_i32:
137   %old = atomicrmw sub i32* @var32, i32 %offset acquire
138; CHECK-NOT: dmb
139; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
140; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
141
142; CHECK: .LBB{{[0-9]+}}_1:
143; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
144  ; w0 below is a reasonable guess but could change: it certainly comes into the
145  ;  function there.
146; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
147; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
148; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
149; CHECK-NOT: dmb
150
151; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
152   ret i32 %old
153}
154
155define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
156; CHECK-LABEL: test_atomic_load_sub_i64:
157   %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
158; CHECK-NOT: dmb
159; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
160; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
161
162; CHECK: .LBB{{[0-9]+}}_1:
163; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
164  ; x0 below is a reasonable guess but could change: it certainly comes into the
165  ; function there.
166; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0
167; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
168; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
169; CHECK-NOT: dmb
170
171; CHECK: mov x0, x[[OLD]]
172   ret i64 %old
173}
174
175define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
176; CHECK-LABEL: test_atomic_load_and_i8:
177   %old = atomicrmw and i8* @var8, i8 %offset release
178; CHECK-NOT: dmb
179; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
180; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
181
182; CHECK: .LBB{{[0-9]+}}_1:
183; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
184  ; w0 below is a reasonable guess but could change: it certainly comes into the
185  ;  function there.
186; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
187; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
188; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
189; CHECK-NOT: dmb
190
191; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
192   ret i8 %old
193}
194
195define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
196; CHECK-LABEL: test_atomic_load_and_i16:
197   %old = atomicrmw and i16* @var16, i16 %offset monotonic
198; CHECK-NOT: dmb
199; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
200; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
201
202; CHECK: .LBB{{[0-9]+}}_1:
203; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
204  ; w0 below is a reasonable guess but could change: it certainly comes into the
205  ;  function there.
206; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
207; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
208; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
209; CHECK-NOT: dmb
210
211; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
212   ret i16 %old
213}
214
215define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
216; CHECK-LABEL: test_atomic_load_and_i32:
217   %old = atomicrmw and i32* @var32, i32 %offset seq_cst
218; CHECK-NOT: dmb
219; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
220; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
221
222; CHECK: .LBB{{[0-9]+}}_1:
223; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
224  ; w0 below is a reasonable guess but could change: it certainly comes into the
225  ;  function there.
226; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
227; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
228; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
229; CHECK-NOT: dmb
230
231; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
232   ret i32 %old
233}
234
235define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
236; CHECK-LABEL: test_atomic_load_and_i64:
237   %old = atomicrmw and i64* @var64, i64 %offset acquire
238; CHECK-NOT: dmb
239; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
240; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
241
242; CHECK: .LBB{{[0-9]+}}_1:
243; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
244  ; x0 below is a reasonable guess but could change: it certainly comes into the
245  ; function there.
246; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0
247; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
248; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
249; CHECK-NOT: dmb
250
251; CHECK: mov x0, x[[OLD]]
252   ret i64 %old
253}
254
255define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
256; CHECK-LABEL: test_atomic_load_or_i8:
257   %old = atomicrmw or i8* @var8, i8 %offset seq_cst
258; CHECK-NOT: dmb
259; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
260; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
261
262; CHECK: .LBB{{[0-9]+}}_1:
263; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
264  ; w0 below is a reasonable guess but could change: it certainly comes into the
265  ;  function there.
266; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
267; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
268; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
269; CHECK-NOT: dmb
270
271; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
272   ret i8 %old
273}
274
275define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
276; CHECK-LABEL: test_atomic_load_or_i16:
277   %old = atomicrmw or i16* @var16, i16 %offset monotonic
278; CHECK-NOT: dmb
279; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
280; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
281
282; CHECK: .LBB{{[0-9]+}}_1:
283; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
284  ; w0 below is a reasonable guess but could change: it certainly comes into the
285  ;  function there.
286; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
287; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
288; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
289; CHECK-NOT: dmb
290
291; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
292   ret i16 %old
293}
294
295define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
296; CHECK-LABEL: test_atomic_load_or_i32:
297   %old = atomicrmw or i32* @var32, i32 %offset acquire
298; CHECK-NOT: dmb
299; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
300; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
301
302; CHECK: .LBB{{[0-9]+}}_1:
303; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
304  ; w0 below is a reasonable guess but could change: it certainly comes into the
305  ;  function there.
306; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
307; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
308; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
309; CHECK-NOT: dmb
310
311; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
312   ret i32 %old
313}
314
315define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
316; CHECK-LABEL: test_atomic_load_or_i64:
317   %old = atomicrmw or i64* @var64, i64 %offset release
318; CHECK-NOT: dmb
319; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
320; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
321
322; CHECK: .LBB{{[0-9]+}}_1:
323; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
324  ; x0 below is a reasonable guess but could change: it certainly comes into the
325  ; function there.
326; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0
327; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
328; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
329; CHECK-NOT: dmb
330
331; CHECK: mov x0, x[[OLD]]
332   ret i64 %old
333}
334
335define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
336; CHECK-LABEL: test_atomic_load_xor_i8:
337   %old = atomicrmw xor i8* @var8, i8 %offset acquire
338; CHECK-NOT: dmb
339; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
340; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
341
342; CHECK: .LBB{{[0-9]+}}_1:
343; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
344  ; w0 below is a reasonable guess but could change: it certainly comes into the
345  ;  function there.
346; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
347; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
348; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
349; CHECK-NOT: dmb
350
351; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
352   ret i8 %old
353}
354
355define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
356; CHECK-LABEL: test_atomic_load_xor_i16:
357   %old = atomicrmw xor i16* @var16, i16 %offset release
358; CHECK-NOT: dmb
359; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
360; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
361
362; CHECK: .LBB{{[0-9]+}}_1:
363; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
364  ; w0 below is a reasonable guess but could change: it certainly comes into the
365  ;  function there.
366; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
367; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
368; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
369; CHECK-NOT: dmb
370
371; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
372   ret i16 %old
373}
374
375define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
376; CHECK-LABEL: test_atomic_load_xor_i32:
377   %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
378; CHECK-NOT: dmb
379; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
380; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
381
382; CHECK: .LBB{{[0-9]+}}_1:
383; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
384  ; w0 below is a reasonable guess but could change: it certainly comes into the
385  ;  function there.
386; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
387; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
388; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
389; CHECK-NOT: dmb
390
391; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
392   ret i32 %old
393}
394
395define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
396; CHECK-LABEL: test_atomic_load_xor_i64:
397   %old = atomicrmw xor i64* @var64, i64 %offset monotonic
398; CHECK-NOT: dmb
399; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
400; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
401
402; CHECK: .LBB{{[0-9]+}}_1:
403; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
404  ; x0 below is a reasonable guess but could change: it certainly comes into the
405  ; function there.
406; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0
407; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
408; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
409; CHECK-NOT: dmb
410
411; CHECK: mov x0, x[[OLD]]
412   ret i64 %old
413}
414
415define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
416; CHECK-LABEL: test_atomic_load_xchg_i8:
417   %old = atomicrmw xchg i8* @var8, i8 %offset monotonic
418; CHECK-NOT: dmb
419; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
420; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
421
422; CHECK: .LBB{{[0-9]+}}_1:
423; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
424  ; w0 below is a reasonable guess but could change: it certainly comes into the
425  ; function there.
426; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
427; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
428; CHECK-NOT: dmb
429
430; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
431   ret i8 %old
432}
433
434define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
435; CHECK-LABEL: test_atomic_load_xchg_i16:
436   %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
437; CHECK-NOT: dmb
438; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
439; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
440
441; CHECK: .LBB{{[0-9]+}}_1:
442; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
443  ; w0 below is a reasonable guess but could change: it certainly comes into the
444  ; function there.
445; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
446; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
447; CHECK-NOT: dmb
448
449; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
450   ret i16 %old
451}
452
453define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
454; CHECK-LABEL: test_atomic_load_xchg_i32:
455   %old = atomicrmw xchg i32* @var32, i32 %offset release
456; CHECK-NOT: dmb
457; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
458; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
459
460; CHECK: .LBB{{[0-9]+}}_1:
461; ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
462  ; w0 below is a reasonable guess but could change: it certainly comes into the
463  ;  function there.
464; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
465; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
466; CHECK-NOT: dmb
467
468; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
469   ret i32 %old
470}
471
472define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
473; CHECK-LABEL: test_atomic_load_xchg_i64:
474   %old = atomicrmw xchg i64* @var64, i64 %offset acquire
475; CHECK-NOT: dmb
476; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
477; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
478
479; CHECK: .LBB{{[0-9]+}}_1:
480; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
481  ; x0 below is a reasonable guess but could change: it certainly comes into the
482  ; function there.
483; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]]
484; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
485; CHECK-NOT: dmb
486
487; CHECK: mov x0, x[[OLD]]
488   ret i64 %old
489}
490
491
492define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
493; CHECK-LABEL: test_atomic_load_min_i8:
494   %old = atomicrmw min i8* @var8, i8 %offset acquire
495; CHECK-NOT: dmb
496; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
497; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
498
499; CHECK: .LBB{{[0-9]+}}_1:
500; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
501  ; w0 below is a reasonable guess but could change: it certainly comes into the
502  ;  function there.
503
504; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]]
505; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb
506; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
507
508; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
509; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
510; CHECK-NOT: dmb
511
512; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
513   ret i8 %old
514}
515
516define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
517; CHECK-LABEL: test_atomic_load_min_i16:
518   %old = atomicrmw min i16* @var16, i16 %offset release
519; CHECK-NOT: dmb
520; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
521; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
522
523; CHECK: .LBB{{[0-9]+}}_1:
524; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
525  ; w0 below is a reasonable guess but could change: it certainly comes into the
526  ;  function there.
527
528; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]]
529; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth
530; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
531
532
533; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
534; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
535; CHECK-NOT: dmb
536
537; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
538   ret i16 %old
539}
540
541define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
542; CHECK-LABEL: test_atomic_load_min_i32:
543   %old = atomicrmw min i32* @var32, i32 %offset monotonic
544; CHECK-NOT: dmb
545; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
546; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
547
548; CHECK: .LBB{{[0-9]+}}_1:
549; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
550  ; w0 below is a reasonable guess but could change: it certainly comes into the
551  ;  function there.
552
553; CHECK-NEXT: cmp w[[OLD]], w0
554; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
555
556
557; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
558; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
559; CHECK-NOT: dmb
560
561; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
562   ret i32 %old
563}
564
565define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
566; CHECK-LABEL: test_atomic_load_min_i64:
567   %old = atomicrmw min i64* @var64, i64 %offset seq_cst
568; CHECK-NOT: dmb
569; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
570; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
571
572; CHECK: .LBB{{[0-9]+}}_1:
573; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
574  ; x0 below is a reasonable guess but could change: it certainly comes into the
575  ; function there.
576
577; CHECK-NEXT: cmp x[[OLD]], x0
578; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, le
579
580
581; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
582; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
583; CHECK-NOT: dmb
584
585; CHECK: mov x0, x[[OLD]]
586   ret i64 %old
587}
588
589define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
590; CHECK-LABEL: test_atomic_load_max_i8:
591   %old = atomicrmw max i8* @var8, i8 %offset seq_cst
592; CHECK-NOT: dmb
593; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
594; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
595
596; CHECK: .LBB{{[0-9]+}}_1:
597; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
598  ; w0 below is a reasonable guess but could change: it certainly comes into the
599  ;  function there.
600
601; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]]
602; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb
603; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
604
605
606; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
607; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
608; CHECK-NOT: dmb
609
610; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
611   ret i8 %old
612}
613
614define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
615; CHECK-LABEL: test_atomic_load_max_i16:
616   %old = atomicrmw max i16* @var16, i16 %offset acquire
617; CHECK-NOT: dmb
618; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
619; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
620
621; CHECK: .LBB{{[0-9]+}}_1:
622; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
623  ; w0 below is a reasonable guess but could change: it certainly comes into the
624  ;  function there.
625
626; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]]
627; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth
628; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
629
630
631; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
632; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
633; CHECK-NOT: dmb
634
635; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
636   ret i16 %old
637}
638
639define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
640; CHECK-LABEL: test_atomic_load_max_i32:
641   %old = atomicrmw max i32* @var32, i32 %offset release
642; CHECK-NOT: dmb
643; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
644; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
645
646; CHECK: .LBB{{[0-9]+}}_1:
647; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
648  ; w0 below is a reasonable guess but could change: it certainly comes into the
649  ;  function there.
650
651; CHECK-NEXT: cmp w[[OLD]], w0
652; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
653
654
655; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
656; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
657; CHECK-NOT: dmb
658
659; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
660   ret i32 %old
661}
662
663define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
664; CHECK-LABEL: test_atomic_load_max_i64:
665   %old = atomicrmw max i64* @var64, i64 %offset monotonic
666; CHECK-NOT: dmb
667; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
668; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
669
670; CHECK: .LBB{{[0-9]+}}_1:
671; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
672  ; x0 below is a reasonable guess but could change: it certainly comes into the
673  ; function there.
674
675; CHECK-NEXT: cmp x[[OLD]], x0
676; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt
677
678
679; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
680; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
681; CHECK-NOT: dmb
682
683; CHECK: mov x0, x[[OLD]]
684   ret i64 %old
685}
686
687define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
688; CHECK-LABEL: test_atomic_load_umin_i8:
689   %old = atomicrmw umin i8* @var8, i8 %offset monotonic
690; CHECK-NOT: dmb
691; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
692; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
693
694; CHECK: .LBB{{[0-9]+}}_1:
695; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
696  ; w0 below is a reasonable guess but could change: it certainly comes into the
697  ;  function there.
698
699; CHECK-NEXT: cmp w[[OLD]], w0, uxtb
700; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
701
702
703; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
704; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
705; CHECK-NOT: dmb
706
707; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
708   ret i8 %old
709}
710
711define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
712; CHECK-LABEL: test_atomic_load_umin_i16:
713   %old = atomicrmw umin i16* @var16, i16 %offset acquire
714; CHECK-NOT: dmb
715; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
716; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
717
718; CHECK: .LBB{{[0-9]+}}_1:
719; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
720  ; w0 below is a reasonable guess but could change: it certainly comes into the
721  ;  function there.
722
723; CHECK-NEXT: cmp w[[OLD]], w0, uxth
724; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
725
726
727; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
728; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
729; CHECK-NOT: dmb
730
731; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
732   ret i16 %old
733}
734
735define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
736; CHECK-LABEL: test_atomic_load_umin_i32:
737   %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
738; CHECK-NOT: dmb
739; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
740; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
741
742; CHECK: .LBB{{[0-9]+}}_1:
743; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
744  ; w0 below is a reasonable guess but could change: it certainly comes into the
745  ;  function there.
746
747; CHECK-NEXT: cmp w[[OLD]], w0
748; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
749
750
751; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
752; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
753; CHECK-NOT: dmb
754
755; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
756   ret i32 %old
757}
758
759define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
760; CHECK-LABEL: test_atomic_load_umin_i64:
761   %old = atomicrmw umin i64* @var64, i64 %offset acq_rel
762; CHECK-NOT: dmb
763; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
764; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
765
766; CHECK: .LBB{{[0-9]+}}_1:
767; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
768  ; x0 below is a reasonable guess but could change: it certainly comes into the
769  ; function there.
770
771; CHECK-NEXT: cmp x[[OLD]], x0
772; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, ls
773
774
775; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
776; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
777; CHECK-NOT: dmb
778
779; CHECK: mov x0, x[[OLD]]
780   ret i64 %old
781}
782
783define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
784; CHECK-LABEL: test_atomic_load_umax_i8:
785   %old = atomicrmw umax i8* @var8, i8 %offset acq_rel
786; CHECK-NOT: dmb
787; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
788; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
789
790; CHECK: .LBB{{[0-9]+}}_1:
791; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
792  ; w0 below is a reasonable guess but could change: it certainly comes into the
793  ;  function there.
794
795; CHECK-NEXT: cmp w[[OLD]], w0, uxtb
796; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
797
798
799; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
800; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
801; CHECK-NOT: dmb
802
803; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
804   ret i8 %old
805}
806
807define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
808; CHECK-LABEL: test_atomic_load_umax_i16:
809   %old = atomicrmw umax i16* @var16, i16 %offset monotonic
810; CHECK-NOT: dmb
811; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
812; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
813
814; CHECK: .LBB{{[0-9]+}}_1:
815; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
816  ; w0 below is a reasonable guess but could change: it certainly comes into the
817  ;  function there.
818
819; CHECK-NEXT: cmp w[[OLD]], w0, uxth
820; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
821
822
823; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
824; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
825; CHECK-NOT: dmb
826
827; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
828   ret i16 %old
829}
830
831define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
832; CHECK-LABEL: test_atomic_load_umax_i32:
833   %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
834; CHECK-NOT: dmb
835; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
836; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
837
838; CHECK: .LBB{{[0-9]+}}_1:
839; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
840  ; w0 below is a reasonable guess but could change: it certainly comes into the
841  ;  function there.
842
843; CHECK-NEXT: cmp w[[OLD]], w0
844; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
845
846
847; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
848; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
849; CHECK-NOT: dmb
850
851; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
852   ret i32 %old
853}
854
855define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
856; CHECK-LABEL: test_atomic_load_umax_i64:
857   %old = atomicrmw umax i64* @var64, i64 %offset release
858; CHECK-NOT: dmb
859; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
860; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
861
862; CHECK: .LBB{{[0-9]+}}_1:
863; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
864  ; x0 below is a reasonable guess but could change: it certainly comes into the
865  ; function there.
866
867; CHECK-NEXT: cmp x[[OLD]], x0
868; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi
869
870
871; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
872; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
873; CHECK-NOT: dmb
874
875; CHECK: mov x0, x[[OLD]]
876   ret i64 %old
877}
878
879define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
880; CHECK-LABEL: test_atomic_cmpxchg_i8:
881   %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
882   %old = extractvalue { i8, i1 } %pair, 0
883
884; CHECK-NOT: dmb
885; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
886; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
887
888; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
889; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
890  ; w0 below is a reasonable guess but could change: it certainly comes into the
891  ;  function there.
892; CHECK-NEXT: cmp w[[OLD]], w0
893; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
894; CHECK: stxrb [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
895; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
896; CHECK-NOT: dmb
897
898; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
899   ret i8 %old
900}
901
902define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
903; CHECK-LABEL: test_atomic_cmpxchg_i16:
904   %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
905   %old = extractvalue { i16, i1 } %pair, 0
906
907; CHECK-NOT: dmb
908; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
909; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
910
911; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
912; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
913  ; w0 below is a reasonable guess but could change: it certainly comes into the
914  ;  function there.
915; CHECK-NEXT: cmp w[[OLD]], w0
916; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
917; CHECK: stlxrh [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
918; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
919; CHECK-NOT: dmb
920
921; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
922   ret i16 %old
923}
924
925define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
926; CHECK-LABEL: test_atomic_cmpxchg_i32:
927   %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
928   %old = extractvalue { i32, i1 } %pair, 0
929
930; CHECK-NOT: dmb
931; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
932; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
933
934; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
935; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
936  ; w0 below is a reasonable guess but could change: it certainly comes into the
937  ;  function there.
938; CHECK-NEXT: cmp w[[OLD]], w0
939; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
940; CHECK: stlxr [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
941; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
942; CHECK-NOT: dmb
943
944; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
945   ret i32 %old
946}
947
948define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
949; CHECK-LABEL: test_atomic_cmpxchg_i64:
950   %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
951   %old = extractvalue { i64, i1 } %pair, 0
952
953; CHECK-NOT: dmb
954; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
955; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
956
957; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
958; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
959  ; w0 below is a reasonable guess but could change: it certainly comes into the
960  ;  function there.
961; CHECK-NEXT: cmp x[[OLD]], x0
962; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
963  ; As above, w1 is a reasonable guess.
964; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]]
965; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
966; CHECK-NOT: dmb
967
968; CHECK: str x[[OLD]],
969   store i64 %old, i64* @var64
970   ret void
971}
972
973define i8 @test_atomic_load_monotonic_i8() nounwind {
974; CHECK-LABEL: test_atomic_load_monotonic_i8:
975  %val = load atomic i8* @var8 monotonic, align 1
976; CHECK-NOT: dmb
977; CHECK: adrp x[[HIADDR:[0-9]+]], var8
978; CHECK: ldrb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
979; CHECK-NOT: dmb
980
981  ret i8 %val
982}
983
984define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
985; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:
986  %addr_int = add i64 %base, %off
987  %addr = inttoptr i64 %addr_int to i8*
988
989  %val = load atomic i8* %addr monotonic, align 1
990; CHECK-NOT: dmb
991; CHECK: ldrb w0, [x0, x1]
992; CHECK-NOT: dmb
993
994  ret i8 %val
995}
996
997define i8 @test_atomic_load_acquire_i8() nounwind {
998; CHECK-LABEL: test_atomic_load_acquire_i8:
999  %val = load atomic i8* @var8 acquire, align 1
1000; CHECK-NOT: dmb
1001; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1002; CHECK-NOT: dmb
1003; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1004; CHECK-NOT: dmb
1005; CHECK: ldarb w0, [x[[ADDR]]]
1006; CHECK-NOT: dmb
1007  ret i8 %val
1008}
1009
1010define i8 @test_atomic_load_seq_cst_i8() nounwind {
1011; CHECK-LABEL: test_atomic_load_seq_cst_i8:
1012  %val = load atomic i8* @var8 seq_cst, align 1
1013; CHECK-NOT: dmb
1014; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1015; CHECK-NOT: dmb
1016; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
1017; CHECK-NOT: dmb
1018; CHECK: ldarb w0, [x[[ADDR]]]
1019; CHECK-NOT: dmb
1020  ret i8 %val
1021}
1022
1023define i16 @test_atomic_load_monotonic_i16() nounwind {
1024; CHECK-LABEL: test_atomic_load_monotonic_i16:
1025  %val = load atomic i16* @var16 monotonic, align 2
1026; CHECK-NOT: dmb
1027; CHECK: adrp x[[HIADDR:[0-9]+]], var16
1028; CHECK-NOT: dmb
1029; CHECK: ldrh w0, [x[[HIADDR]], {{#?}}:lo12:var16]
1030; CHECK-NOT: dmb
1031
1032  ret i16 %val
1033}
1034
1035define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
1036; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:
1037  %addr_int = add i64 %base, %off
1038  %addr = inttoptr i64 %addr_int to i32*
1039
1040  %val = load atomic i32* %addr monotonic, align 4
1041; CHECK-NOT: dmb
1042; CHECK: ldr w0, [x0, x1]
1043; CHECK-NOT: dmb
1044
1045  ret i32 %val
1046}
1047
1048define i64 @test_atomic_load_seq_cst_i64() nounwind {
1049; CHECK-LABEL: test_atomic_load_seq_cst_i64:
1050  %val = load atomic i64* @var64 seq_cst, align 8
1051; CHECK-NOT: dmb
1052; CHECK: adrp [[HIADDR:x[0-9]+]], var64
1053; CHECK-NOT: dmb
1054; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64
1055; CHECK-NOT: dmb
1056; CHECK: ldar x0, [x[[ADDR]]]
1057; CHECK-NOT: dmb
1058  ret i64 %val
1059}
1060
1061define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
1062; CHECK-LABEL: test_atomic_store_monotonic_i8:
1063  store atomic i8 %val, i8* @var8 monotonic, align 1
1064; CHECK: adrp x[[HIADDR:[0-9]+]], var8
1065; CHECK: strb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
1066
1067  ret void
1068}
1069
1070define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
1071; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:
1072
1073  %addr_int = add i64 %base, %off
1074  %addr = inttoptr i64 %addr_int to i8*
1075
1076  store atomic i8 %val, i8* %addr monotonic, align 1
1077; CHECK: strb w2, [x0, x1]
1078
1079  ret void
1080}
1081define void @test_atomic_store_release_i8(i8 %val) nounwind {
1082; CHECK-LABEL: test_atomic_store_release_i8:
1083  store atomic i8 %val, i8* @var8 release, align 1
1084; CHECK-NOT: dmb
1085; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1086; CHECK-NOT: dmb
1087; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
1088; CHECK-NOT: dmb
1089; CHECK: stlrb w0, [x[[ADDR]]]
1090; CHECK-NOT: dmb
1091  ret void
1092}
1093
1094define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
1095; CHECK-LABEL: test_atomic_store_seq_cst_i8:
1096  store atomic i8 %val, i8* @var8 seq_cst, align 1
1097; CHECK-NOT: dmb
1098; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1099; CHECK-NOT: dmb
1100; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
1101; CHECK-NOT: dmb
1102; CHECK: stlrb w0, [x[[ADDR]]]
1103; CHECK-NOT: dmb
1104
1105  ret void
1106}
1107
1108define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
1109; CHECK-LABEL: test_atomic_store_monotonic_i16:
1110  store atomic i16 %val, i16* @var16 monotonic, align 2
1111; CHECK-NOT: dmb
1112; CHECK: adrp x[[HIADDR:[0-9]+]], var16
1113; CHECK-NOT: dmb
1114; CHECK: strh w0, [x[[HIADDR]], {{#?}}:lo12:var16]
1115; CHECK-NOT: dmb
1116  ret void
1117}
1118
1119define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
1120; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:
1121
1122  %addr_int = add i64 %base, %off
1123  %addr = inttoptr i64 %addr_int to i32*
1124
1125  store atomic i32 %val, i32* %addr monotonic, align 4
1126; CHECK-NOT: dmb
1127; CHECK: str w2, [x0, x1]
1128; CHECK-NOT: dmb
1129
1130  ret void
1131}
1132
1133define void @test_atomic_store_release_i64(i64 %val) nounwind {
1134; CHECK-LABEL: test_atomic_store_release_i64:
1135  store atomic i64 %val, i64* @var64 release, align 8
1136; CHECK-NOT: dmb
1137; CHECK: adrp [[HIADDR:x[0-9]+]], var64
1138; CHECK-NOT: dmb
1139; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64
1140; CHECK-NOT: dmb
1141; CHECK: stlr x0, [x[[ADDR]]]
1142; CHECK-NOT: dmb
1143  ret void
1144}
1145