1; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=verde -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
2; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
3; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mtriple=r600---amdgiz -mcpu=redwood -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=EG -check-prefix=FUNC %s
4
5declare i32 @llvm.r600.read.tidig.x() #0
6
7; FUNC-LABEL: {{^}}ashr_v2i32:
8; SI: v_ashr_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
9; SI: v_ashr_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
10
11; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
12; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
13
14; EG: ASHR {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
15; EG: ASHR {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
16define amdgpu_kernel void @ashr_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) {
17  %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1
18  %a = load <2 x i32>, <2 x i32> addrspace(1)* %in
19  %b = load <2 x i32>, <2 x i32> addrspace(1)* %b_ptr
20  %result = ashr <2 x i32> %a, %b
21  store <2 x i32> %result, <2 x i32> addrspace(1)* %out
22  ret void
23}
24
25; FUNC-LABEL: {{^}}ashr_v4i32:
26; SI: v_ashr_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
27; SI: v_ashr_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
28; SI: v_ashr_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
29; SI: v_ashr_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
30
31; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
32; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
33; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
34; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
35
36; EG: ASHR {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
37; EG: ASHR {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
38; EG: ASHR {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
39; EG: ASHR {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
40define amdgpu_kernel void @ashr_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
41  %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1
42  %a = load <4 x i32>, <4 x i32> addrspace(1)* %in
43  %b = load <4 x i32>, <4 x i32> addrspace(1)* %b_ptr
44  %result = ashr <4 x i32> %a, %b
45  store <4 x i32> %result, <4 x i32> addrspace(1)* %out
46  ret void
47}
48
49; FUNC-LABEL: {{^}}ashr_v2i16:
50; FIXME: The ashr operation is uniform, but because its operands come from a
51; global load we end up with the vector instructions rather than scalar.
52; VI: v_ashrrev_i32_sdwa v{{[0-9]+}}, sext(v{{[0-9]+}}), sext(v{{[0-9]+}}) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
53; VI: v_ashrrev_i32_sdwa v{{[0-9]+}}, sext(v{{[0-9]+}}), sext(v{{[0-9]+}}) dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
54define amdgpu_kernel void @ashr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) {
55  %b_ptr = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in, i16 1
56  %a = load <2 x i16>, <2 x i16> addrspace(1)* %in
57  %b = load <2 x i16>, <2 x i16> addrspace(1)* %b_ptr
58  %result = ashr <2 x i16> %a, %b
59  store <2 x i16> %result, <2 x i16> addrspace(1)* %out
60  ret void
61}
62
63; FUNC-LABEL: {{^}}ashr_v4i16:
64; FIXME: The ashr operation is uniform, but because its operands come from a
65; global load we end up with the vector instructions rather than scalar.
66; VI: v_ashrrev_i32_sdwa v{{[0-9]+}}, sext(v{{[0-9]+}}), sext(v{{[0-9]+}}) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
67; VI: v_ashrrev_i32_sdwa v{{[0-9]+}}, sext(v{{[0-9]+}}), sext(v{{[0-9]+}}) dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
68; VI: v_ashrrev_i32_sdwa v{{[0-9]+}}, sext(v{{[0-9]+}}), sext(v{{[0-9]+}}) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
69; VI: v_ashrrev_i32_sdwa v{{[0-9]+}}, sext(v{{[0-9]+}}), sext(v{{[0-9]+}}) dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
70define amdgpu_kernel void @ashr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) {
71  %b_ptr = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %in, i16 1
72  %a = load <4 x i16>, <4 x i16> addrspace(1)* %in
73  %b = load <4 x i16>, <4 x i16> addrspace(1)* %b_ptr
74  %result = ashr <4 x i16> %a, %b
75  store <4 x i16> %result, <4 x i16> addrspace(1)* %out
76  ret void
77}
78
79; FUNC-LABEL: {{^}}s_ashr_i64:
80; GCN: s_ashr_i64 s[{{[0-9]}}:{{[0-9]}}], s[{{[0-9]}}:{{[0-9]}}], 8
81
82; EG: ASHR
83define amdgpu_kernel void @s_ashr_i64(i64 addrspace(1)* %out, i32 %in) {
84entry:
85  %in.ext = sext i32 %in to i64
86  %ashr = ashr i64 %in.ext, 8
87  store i64 %ashr, i64 addrspace(1)* %out
88  ret void
89}
90
91; FUNC-LABEL: {{^}}ashr_i64_2:
92; SI: v_ashr_i64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
93
94; VI: v_ashrrev_i64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
95
96; EG: SUB_INT {{\*? *}}[[COMPSH:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHIFT:T[0-9]+\.[XYZW]]]
97; EG: LSHL {{\* *}}[[TEMP:T[0-9]+\.[XYZW]]], [[OPHI:T[0-9]+\.[XYZW]]], {{[[COMPSH]]|PV.[XYZW]}}
98; EG-DAG: ADD_INT {{\*? *}}[[BIGSH:T[0-9]+\.[XYZW]]], [[SHIFT]], literal
99; EG-DAG: LSHL {{\*? *}}[[OVERF:T[0-9]+\.[XYZW]]], {{[[TEMP]]|PV.[XYZW]}}, 1
100; EG-DAG: LSHR {{\*? *}}[[LOSMTMP:T[0-9]+\.[XYZW]]], [[OPLO:T[0-9]+\.[XYZW]]], [[SHIFT]]
101; EG-DAG: OR_INT {{\*? *}}[[LOSM:T[0-9]+\.[XYZW]]], {{[[LOSMTMP]]|PV.[XYZW]|PS}}, {{[[OVERF]]|PV.[XYZW]}}
102; EG-DAG: ASHR {{\*? *}}[[HISM:T[0-9]+\.[XYZW]]], [[OPHI]], {{PS|PV.[XYZW]|[[SHIFT]]}}
103; EG-DAG: ASHR {{\*? *}}[[LOBIG:T[0-9]+\.[XYZW]]], [[OPHI]], literal
104; EG-DAG: ASHR {{\*? *}}[[HIBIG:T[0-9]+\.[XYZW]]], [[OPHI]], literal
105; EG-DAG: SETGT_UINT {{\*? *}}[[RESC:T[0-9]+\.[XYZW]]], [[SHIFT]], literal
106; EG-DAG: CNDE_INT {{\*? *}}[[RESLO:T[0-9]+\.[XYZW]]], {{T[0-9]+\.[XYZW]}}
107; EG-DAG: CNDE_INT {{\*? *}}[[RESHI:T[0-9]+\.[XYZW]]], {{T[0-9]+\.[XYZW]}}
108define amdgpu_kernel void @ashr_i64_2(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
109entry:
110  %b_ptr = getelementptr i64, i64 addrspace(1)* %in, i64 1
111  %a = load i64, i64 addrspace(1)* %in
112  %b = load i64, i64 addrspace(1)* %b_ptr
113  %result = ashr i64 %a, %b
114  store i64 %result, i64 addrspace(1)* %out
115  ret void
116}
117
118; FUNC-LABEL: {{^}}ashr_v2i64:
119; SI: v_ashr_i64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
120; VI: v_ashrrev_i64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
121
122; EG-DAG: SUB_INT {{\*? *}}[[COMPSHA:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHA:T[0-9]+\.[XYZW]]]
123; EG-DAG: SUB_INT {{\*? *}}[[COMPSHB:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHB:T[0-9]+\.[XYZW]]]
124; EG-DAG: LSHL {{\*? *}}[[COMPSHA]]
125; EG-DAG: LSHL {{\*? *}}[[COMPSHB]]
126; EG-DAG: LSHL {{.*}}, 1
127; EG-DAG: LSHL {{.*}}, 1
128; EG-DAG: ASHR {{.*}}, [[SHA]]
129; EG-DAG: ASHR {{.*}}, [[SHB]]
130; EG-DAG: LSHR {{.*}}, [[SHA]]
131; EG-DAG: LSHR {{.*}}, [[SHB]]
132; EG-DAG: OR_INT
133; EG-DAG: OR_INT
134; EG-DAG: ADD_INT  {{\*? *}}[[BIGSHA:T[0-9]+\.[XYZW]]]{{.*}}, literal
135; EG-DAG: ADD_INT  {{\*? *}}[[BIGSHB:T[0-9]+\.[XYZW]]]{{.*}}, literal
136; EG-DAG: ASHR
137; EG-DAG: ASHR
138; EG-DAG: ASHR {{.*}}, literal
139; EG-DAG: ASHR {{.*}}, literal
140; EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHA]], literal
141; EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHB]], literal
142; EG-DAG: CNDE_INT
143; EG-DAG: CNDE_INT
144; EG-DAG: CNDE_INT
145; EG-DAG: CNDE_INT
146define amdgpu_kernel void @ashr_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %in) {
147  %b_ptr = getelementptr <2 x i64>, <2 x i64> addrspace(1)* %in, i64 1
148  %a = load <2 x i64>, <2 x i64> addrspace(1)* %in
149  %b = load <2 x i64>, <2 x i64> addrspace(1)* %b_ptr
150  %result = ashr <2 x i64> %a, %b
151  store <2 x i64> %result, <2 x i64> addrspace(1)* %out
152  ret void
153}
154
155; FIXME: Broken on r600
156; XFUNC-LABEL: {{^}}s_ashr_v2i64:
157; XGCN: s_ashr_i64 {{s\[[0-9]+:[0-9]+\], s\[[0-9]+:[0-9]+\], s[0-9]+}}
158; XGCN: s_ashr_i64 {{s\[[0-9]+:[0-9]+\], s\[[0-9]+:[0-9]+\], s[0-9]+}}
159; define amdgpu_kernel void @s_ashr_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %in, <2 x i64> %a, <2 x i64> %b) {
160;   %result = ashr <2 x i64> %a, %b
161;   store <2 x i64> %result, <2 x i64> addrspace(1)* %out
162;   ret void
163; }
164
165; FUNC-LABEL: {{^}}ashr_v4i64:
166; SI: v_ashr_i64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
167; SI: v_ashr_i64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
168; SI: v_ashr_i64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
169; SI: v_ashr_i64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
170
171; VI: v_ashrrev_i64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
172; VI: v_ashrrev_i64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
173; VI: v_ashrrev_i64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
174; VI: v_ashrrev_i64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
175
176; EG-DAG: SUB_INT {{\*? *}}[[COMPSHA:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHA:T[0-9]+\.[XYZW]]]
177; EG-DAG: SUB_INT {{\*? *}}[[COMPSHB:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHB:T[0-9]+\.[XYZW]]]
178; EG-DAG: SUB_INT {{\*? *}}[[COMPSHC:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHC:T[0-9]+\.[XYZW]]]
179; EG-DAG: SUB_INT {{\*? *}}[[COMPSHD:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHD:T[0-9]+\.[XYZW]]]
180; EG-DAG: LSHL {{\*? *}}[[COMPSHA]]
181; EG-DAG: LSHL {{\*? *}}[[COMPSHB]]
182; EG-DAG: LSHL {{\*? *}}[[COMPSHC]]
183; EG-DAG: LSHL {{\*? *}}[[COMPSHD]]
184; EG-DAG: LSHL {{.*}}, 1
185; EG-DAG: LSHL {{.*}}, 1
186; EG-DAG: LSHL {{.*}}, 1
187; EG-DAG: LSHL {{.*}}, 1
188; EG-DAG: ASHR {{.*}}, [[SHA]]
189; EG-DAG: ASHR {{.*}}, [[SHB]]
190; EG-DAG: ASHR {{.*}}, [[SHC]]
191; EG-DAG: ASHR {{.*}}, [[SHD]]
192; EG-DAG: LSHR {{.*}}, [[SHA]]
193; EG-DAG: LSHR {{.*}}, [[SHB]]
194; EG-DAG: LSHR {{.*}}, [[SHA]]
195; EG-DAG: LSHR {{.*}}, [[SHB]]
196; EG-DAG: OR_INT
197; EG-DAG: OR_INT
198; EG-DAG: OR_INT
199; EG-DAG: OR_INT
200; EG-DAG: ADD_INT  {{\*? *}}[[BIGSHA:T[0-9]+\.[XYZW]]]{{.*}}, literal
201; EG-DAG: ADD_INT  {{\*? *}}[[BIGSHB:T[0-9]+\.[XYZW]]]{{.*}}, literal
202; EG-DAG: ADD_INT  {{\*? *}}[[BIGSHC:T[0-9]+\.[XYZW]]]{{.*}}, literal
203; EG-DAG: ADD_INT  {{\*? *}}[[BIGSHD:T[0-9]+\.[XYZW]]]{{.*}}, literal
204; EG-DAG: ASHR
205; EG-DAG: ASHR
206; EG-DAG: ASHR
207; EG-DAG: ASHR
208; EG-DAG: ASHR {{.*}}, literal
209; EG-DAG: ASHR {{.*}}, literal
210; EG-DAG: ASHR {{.*}}, literal
211; EG-DAG: ASHR {{.*}}, literal
212; EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHA]], literal
213; EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHB]], literal
214; EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHC]], literal
215; EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHD]], literal
216; EG-DAG: CNDE_INT
217; EG-DAG: CNDE_INT
218; EG-DAG: CNDE_INT
219; EG-DAG: CNDE_INT
220; EG-DAG: CNDE_INT
221; EG-DAG: CNDE_INT
222; EG-DAG: CNDE_INT
223; EG-DAG: CNDE_INT
224define amdgpu_kernel void @ashr_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) {
225  %b_ptr = getelementptr <4 x i64>, <4 x i64> addrspace(1)* %in, i64 1
226  %a = load <4 x i64>, <4 x i64> addrspace(1)* %in
227  %b = load <4 x i64>, <4 x i64> addrspace(1)* %b_ptr
228  %result = ashr <4 x i64> %a, %b
229  store <4 x i64> %result, <4 x i64> addrspace(1)* %out
230  ret void
231}
232
233; GCN-LABEL: {{^}}s_ashr_32_i64:
234; GCN: s_load_dword s[[HI:[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, {{0x14|0x50}}
235; GCN: s_ashr_i32 s[[SHIFT:[0-9]+]], s[[HI]], 31
236; GCN: s_add_u32 s{{[0-9]+}}, s[[HI]], s{{[0-9]+}}
237; GCN: s_addc_u32 s{{[0-9]+}}, s[[SHIFT]], s{{[0-9]+}}
238define amdgpu_kernel void @s_ashr_32_i64(i64 addrspace(1)* %out, [8 x i32], i64 %a, [8 x i32], i64 %b) {
239  %result = ashr i64 %a, 32
240  %add = add i64 %result, %b
241  store i64 %add, i64 addrspace(1)* %out
242  ret void
243}
244
245; GCN-LABEL: {{^}}v_ashr_32_i64:
246; SI: buffer_load_dword v[[HI:[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
247; VI: flat_load_dword v[[HI:[0-9]+]]
248; GCN: v_ashrrev_i32_e32 v[[SHIFT:[0-9]+]], 31, v[[HI]]
249; GCN: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[HI]]:[[SHIFT]]{{\]}}
250define amdgpu_kernel void @v_ashr_32_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
251  %tid = call i32 @llvm.r600.read.tidig.x() #0
252  %gep.in = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
253  %gep.out = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
254  %a = load i64, i64 addrspace(1)* %gep.in
255  %result = ashr i64 %a, 32
256  store i64 %result, i64 addrspace(1)* %gep.out
257  ret void
258}
259
260; GCN-LABEL: {{^}}s_ashr_63_i64:
261; GCN: s_load_dword s[[HI:[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, {{0x14|0x50}}
262; GCN: s_ashr_i32 s[[SHIFT:[0-9]+]], s[[HI]], 31
263; GCN: s_add_u32 {{s[0-9]+}}, s[[SHIFT]], {{s[0-9]+}}
264; GCN: s_addc_u32 {{s[0-9]+}}, s[[SHIFT]], {{s[0-9]+}}
265define amdgpu_kernel void @s_ashr_63_i64(i64 addrspace(1)* %out, [8 x i32], i64 %a, [8 x i32], i64 %b) {
266  %result = ashr i64 %a, 63
267  %add = add i64 %result, %b
268  store i64 %add, i64 addrspace(1)* %out
269  ret void
270}
271
272; GCN-LABEL: {{^}}v_ashr_63_i64:
273; SI: buffer_load_dword v[[HI:[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
274; VI: flat_load_dword v[[HI:[0-9]+]]
275; GCN: v_ashrrev_i32_e32 v[[SHIFT:[0-9]+]], 31, v[[HI]]
276; GCN: v_mov_b32_e32 v[[COPY:[0-9]+]], v[[SHIFT]]
277; GCN: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[SHIFT]]:[[COPY]]{{\]}}
278define amdgpu_kernel void @v_ashr_63_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
279  %tid = call i32 @llvm.r600.read.tidig.x() #0
280  %gep.in = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
281  %gep.out = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
282  %a = load i64, i64 addrspace(1)* %gep.in
283  %result = ashr i64 %a, 63
284  store i64 %result, i64 addrspace(1)* %gep.out
285  ret void
286}
287
288attributes #0 = { nounwind readnone }
289