1; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,SI,FUNC %s 2; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,VI,FUNC %s 3; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=FUNC,GFX9_10 %s 4; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1010 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=FUNC,GFX9_10 %s 5; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=redwood < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=EG,FUNC %s 6 7; mul24 and mad24 are affected 8 9; FUNC-LABEL: {{^}}test_mul_v2i32: 10; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} 11; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} 12 13; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}} 14; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}} 15 16define amdgpu_kernel void @test_mul_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) { 17 %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1 18 %a = load <2 x i32>, <2 x i32> addrspace(1) * %in 19 %b = load <2 x i32>, <2 x i32> addrspace(1) * %b_ptr 20 %result = mul <2 x i32> %a, %b 21 store <2 x i32> %result, <2 x i32> addrspace(1)* %out 22 ret void 23} 24 25; FUNC-LABEL: {{^}}v_mul_v4i32: 26; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} 27; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} 28; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} 29; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} 30 31; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}} 32; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}} 33; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}} 34; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}} 35 36define amdgpu_kernel void @v_mul_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { 37 %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1 38 %a = load <4 x i32>, <4 x i32> addrspace(1) * %in 39 %b = load <4 x i32>, <4 x i32> addrspace(1) * %b_ptr 40 %result = mul <4 x i32> %a, %b 41 store <4 x i32> %result, <4 x i32> addrspace(1)* %out 42 ret void 43} 44 45; FUNC-LABEL: {{^}}s_trunc_i64_mul_to_i32: 46; GCN: s_load_dword 47; GCN: s_load_dword 48; GCN: s_mul_i32 49; GCN: buffer_store_dword 50define amdgpu_kernel void @s_trunc_i64_mul_to_i32(i32 addrspace(1)* %out, i64 %a, i64 %b) { 51 %mul = mul i64 %b, %a 52 %trunc = trunc i64 %mul to i32 53 store i32 %trunc, i32 addrspace(1)* %out, align 8 54 ret void 55} 56 57; FUNC-LABEL: {{^}}v_trunc_i64_mul_to_i32: 58; GCN: s_load_dword 59; GCN: s_load_dword 60; GCN: v_mul_lo_u32 61; GCN: buffer_store_dword 62define amdgpu_kernel void @v_trunc_i64_mul_to_i32(i32 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind { 63 %a = load i64, i64 addrspace(1)* %aptr, align 8 64 %b = load i64, i64 addrspace(1)* %bptr, align 8 65 %mul = mul i64 %b, %a 66 %trunc = trunc i64 %mul to i32 67 store i32 %trunc, i32 addrspace(1)* %out, align 8 68 ret void 69} 70 71; This 64-bit multiply should just use MUL_HI and MUL_LO, since the top 72; 32-bits of both arguments are sign bits. 73; FUNC-LABEL: {{^}}mul64_sext_c: 74; EG-DAG: MULLO_INT 75; EG-DAG: MULHI_INT 76; GCN-DAG: s_mul_i32 77; GCN-DAG: v_mul_hi_i32 78define amdgpu_kernel void @mul64_sext_c(i64 addrspace(1)* %out, i32 %in) { 79entry: 80 %0 = sext i32 %in to i64 81 %1 = mul i64 %0, 80 82 store i64 %1, i64 addrspace(1)* %out 83 ret void 84} 85 86; FUNC-LABEL: {{^}}v_mul64_sext_c: 87; EG-DAG: MULLO_INT 88; EG-DAG: MULHI_INT 89; GCN-DAG: v_mul_lo_u32 90; GCN-DAG: v_mul_hi_i32 91; GCN: s_endpgm 92define amdgpu_kernel void @v_mul64_sext_c(i64 addrspace(1)* %out, i32 addrspace(1)* %in) { 93 %val = load i32, i32 addrspace(1)* %in, align 4 94 %ext = sext i32 %val to i64 95 %mul = mul i64 %ext, 80 96 store i64 %mul, i64 addrspace(1)* %out, align 8 97 ret void 98} 99 100; FUNC-LABEL: {{^}}v_mul64_sext_inline_imm: 101; GCN-DAG: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, 9 102; GCN-DAG: v_mul_hi_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9 103; GCN: s_endpgm 104define amdgpu_kernel void @v_mul64_sext_inline_imm(i64 addrspace(1)* %out, i32 addrspace(1)* %in) { 105 %val = load i32, i32 addrspace(1)* %in, align 4 106 %ext = sext i32 %val to i64 107 %mul = mul i64 %ext, 9 108 store i64 %mul, i64 addrspace(1)* %out, align 8 109 ret void 110} 111 112; FUNC-LABEL: {{^}}s_mul_i32: 113; GCN: s_load_dword [[SRC0:s[0-9]+]], 114; GCN: s_load_dword [[SRC1:s[0-9]+]], 115; GCN: s_mul_i32 [[SRESULT:s[0-9]+]], [[SRC0]], [[SRC1]] 116; GCN: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]] 117; GCN: buffer_store_dword [[VRESULT]], 118; GCN: s_endpgm 119define amdgpu_kernel void @s_mul_i32(i32 addrspace(1)* %out, [8 x i32], i32 %a, [8 x i32], i32 %b) nounwind { 120 %mul = mul i32 %a, %b 121 store i32 %mul, i32 addrspace(1)* %out, align 4 122 ret void 123} 124 125; FUNC-LABEL: {{^}}v_mul_i32: 126; GCN: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} 127define amdgpu_kernel void @v_mul_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) { 128 %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1 129 %a = load i32, i32 addrspace(1)* %in 130 %b = load i32, i32 addrspace(1)* %b_ptr 131 %result = mul i32 %a, %b 132 store i32 %result, i32 addrspace(1)* %out 133 ret void 134} 135 136; A standard 64-bit multiply. The expansion should be around 6 instructions. 137; It would be difficult to match the expansion correctly without writing 138; a really complicated list of FileCheck expressions. I don't want 139; to confuse people who may 'break' this test with a correct optimization, 140; so this test just uses FUNC-LABEL to make sure the compiler does not 141; crash with a 'failed to select' error. 142 143; FUNC-LABEL: {{^}}s_mul_i64: 144; GFX9_10-DAG: s_mul_i32 145; GFX9_10-DAG: s_mul_hi_u32 146; GFX9_10-DAG: s_mul_i32 147; GFX9_10-DAG: s_mul_i32 148; GFX9_10: s_endpgm 149define amdgpu_kernel void @s_mul_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind { 150 %mul = mul i64 %a, %b 151 store i64 %mul, i64 addrspace(1)* %out, align 8 152 ret void 153} 154 155; FUNC-LABEL: {{^}}v_mul_i64: 156; GCN: v_mul_lo_u32 157define amdgpu_kernel void @v_mul_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) { 158 %a = load i64, i64 addrspace(1)* %aptr, align 8 159 %b = load i64, i64 addrspace(1)* %bptr, align 8 160 %mul = mul i64 %a, %b 161 store i64 %mul, i64 addrspace(1)* %out, align 8 162 ret void 163} 164 165; FUNC-LABEL: {{^}}mul32_in_branch: 166; GCN: s_mul_i32 167define amdgpu_kernel void @mul32_in_branch(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %a, i32 %b, i32 %c) { 168entry: 169 %0 = icmp eq i32 %a, 0 170 br i1 %0, label %if, label %else 171 172if: 173 %1 = load i32, i32 addrspace(1)* %in 174 br label %endif 175 176else: 177 %2 = mul i32 %a, %b 178 br label %endif 179 180endif: 181 %3 = phi i32 [%1, %if], [%2, %else] 182 store i32 %3, i32 addrspace(1)* %out 183 ret void 184} 185 186; FUNC-LABEL: {{^}}mul64_in_branch: 187; GCN-DAG: s_mul_i32 188; GCN-DAG: v_mul_hi_u32 189; GCN: s_endpgm 190define amdgpu_kernel void @mul64_in_branch(i64 addrspace(1)* %out, i64 addrspace(1)* %in, i64 %a, i64 %b, i64 %c) { 191entry: 192 %0 = icmp eq i64 %a, 0 193 br i1 %0, label %if, label %else 194 195if: 196 %1 = load i64, i64 addrspace(1)* %in 197 br label %endif 198 199else: 200 %2 = mul i64 %a, %b 201 br label %endif 202 203endif: 204 %3 = phi i64 [%1, %if], [%2, %else] 205 store i64 %3, i64 addrspace(1)* %out 206 ret void 207} 208 209; FIXME: Load dwordx4 210; FUNC-LABEL: {{^}}s_mul_i128: 211; GCN: s_load_dwordx4 212; GCN: s_load_dwordx4 213 214; SI: v_mul_hi_u32 215; SI: v_mul_hi_u32 216; SI: s_mul_i32 217; SI: v_mul_hi_u32 218; SI: s_mul_i32 219; SI: s_mul_i32 220 221; SI-DAG: s_mul_i32 222; SI-DAG: v_mul_hi_u32 223; SI-DAG: v_mul_hi_u32 224; SI-DAG: s_mul_i32 225; SI-DAG: s_mul_i32 226; SI-DAG: v_mul_hi_u32 227 228; VI: v_mul_hi_u32 229; VI: s_mul_i32 230; VI: s_mul_i32 231; VI: v_mul_hi_u32 232; VI: v_mul_hi_u32 233; VI: s_mul_i32 234; VI: v_mad_u64_u32 235; VI: s_mul_i32 236; VI: v_mad_u64_u32 237; VI: s_mul_i32 238; VI: s_mul_i32 239; VI: v_mad_u64_u32 240; VI: s_mul_i32 241 242 243; GCN: buffer_store_dwordx4 244define amdgpu_kernel void @s_mul_i128(i128 addrspace(1)* %out, [8 x i32], i128 %a, [8 x i32], i128 %b) nounwind #0 { 245 %mul = mul i128 %a, %b 246 store i128 %mul, i128 addrspace(1)* %out 247 ret void 248} 249 250; FUNC-LABEL: {{^}}v_mul_i128: 251; GCN: {{buffer|flat}}_load_dwordx4 252; GCN: {{buffer|flat}}_load_dwordx4 253 254; SI-DAG: v_mul_lo_u32 255; SI-DAG: v_mul_hi_u32 256; SI-DAG: v_mul_hi_u32 257; SI-DAG: v_mul_lo_u32 258; SI-DAG: v_mul_hi_u32 259; SI-DAG: v_mul_hi_u32 260; SI-DAG: v_mul_lo_u32 261; SI-DAG: v_mul_lo_u32 262; SI-DAG: v_add_i32_e32 263 264; SI-DAG: v_mul_hi_u32 265; SI-DAG: v_mul_lo_u32 266; SI-DAG: v_mul_hi_u32 267; SI-DAG: v_mul_lo_u32 268; SI-DAG: v_mul_lo_u32 269; SI-DAG: v_mul_lo_u32 270; SI-DAG: v_mul_lo_u32 271; SI-DAG: v_mul_lo_u32 272 273; VI-DAG: v_mul_lo_u32 274; VI-DAG: v_mul_hi_u32 275; VI: v_mad_u64_u32 276; VI: v_mad_u64_u32 277; VI: v_mad_u64_u32 278 279; GCN: {{buffer|flat}}_store_dwordx4 280define amdgpu_kernel void @v_mul_i128(i128 addrspace(1)* %out, i128 addrspace(1)* %aptr, i128 addrspace(1)* %bptr) #0 { 281 %tid = call i32 @llvm.amdgcn.workitem.id.x() 282 %gep.a = getelementptr inbounds i128, i128 addrspace(1)* %aptr, i32 %tid 283 %gep.b = getelementptr inbounds i128, i128 addrspace(1)* %bptr, i32 %tid 284 %gep.out = getelementptr inbounds i128, i128 addrspace(1)* %bptr, i32 %tid 285 %a = load i128, i128 addrspace(1)* %gep.a 286 %b = load i128, i128 addrspace(1)* %gep.b 287 %mul = mul i128 %a, %b 288 store i128 %mul, i128 addrspace(1)* %gep.out 289 ret void 290} 291 292declare i32 @llvm.amdgcn.workitem.id.x() #1 293 294attributes #0 = { nounwind } 295attributes #1 = { nounwind readnone} 296