1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -march=amdgcn -mcpu=gfx902 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s 3; RUN: llc -march=amdgcn -mcpu=gfx1030 -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefix=GCN-SCRATCH %s 4 5define amdgpu_kernel void @vector_clause(<4 x i32> addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture %arg1) { 6; GCN-LABEL: vector_clause: 7; GCN: ; %bb.0: ; %bb 8; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 9; GCN-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c 10; GCN-NEXT: v_lshlrev_b32_e32 v16, 4, v0 11; GCN-NEXT: s_waitcnt lgkmcnt(0) 12; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[2:3] 13; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[2:3] offset:16 14; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[2:3] offset:32 15; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[2:3] offset:48 16; GCN-NEXT: s_waitcnt vmcnt(3) 17; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[4:5] 18; GCN-NEXT: s_waitcnt vmcnt(3) 19; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[4:5] offset:16 20; GCN-NEXT: s_waitcnt vmcnt(3) 21; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[4:5] offset:32 22; GCN-NEXT: s_waitcnt vmcnt(3) 23; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[4:5] offset:48 24; GCN-NEXT: s_endpgm 25; 26; GCN-SCRATCH-LABEL: vector_clause: 27; GCN-SCRATCH: ; %bb.0: ; %bb 28; GCN-SCRATCH-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 29; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v16, 4, v0 30; GCN-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c 31; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) 32; GCN-SCRATCH-NEXT: s_clause 0x3 33; GCN-SCRATCH-NEXT: global_load_dwordx4 v[0:3], v16, s[2:3] 34; GCN-SCRATCH-NEXT: global_load_dwordx4 v[4:7], v16, s[2:3] offset:16 35; GCN-SCRATCH-NEXT: global_load_dwordx4 v[8:11], v16, s[2:3] offset:32 36; GCN-SCRATCH-NEXT: global_load_dwordx4 v[12:15], v16, s[2:3] offset:48 37; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(3) 38; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] 39; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(2) 40; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 41; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1) 42; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 43; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) 44; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 45; GCN-SCRATCH-NEXT: s_endpgm 46bb: 47 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() 48 %tmp2 = zext i32 %tmp to i64 49 %tmp3 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp2 50 %tmp4 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp3, align 16 51 %tmp5 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp2 52 %tmp6 = add nuw nsw i64 %tmp2, 1 53 %tmp7 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp6 54 %tmp8 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp7, align 16 55 %tmp9 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp6 56 %tmp10 = add nuw nsw i64 %tmp2, 2 57 %tmp11 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp10 58 %tmp12 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp11, align 16 59 %tmp13 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp10 60 %tmp14 = add nuw nsw i64 %tmp2, 3 61 %tmp15 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp14 62 %tmp16 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp15, align 16 63 %tmp17 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp14 64 store <4 x i32> %tmp4, <4 x i32> addrspace(1)* %tmp5, align 16 65 store <4 x i32> %tmp8, <4 x i32> addrspace(1)* %tmp9, align 16 66 store <4 x i32> %tmp12, <4 x i32> addrspace(1)* %tmp13, align 16 67 store <4 x i32> %tmp16, <4 x i32> addrspace(1)* %tmp17, align 16 68 ret void 69} 70 71define amdgpu_kernel void @scalar_clause(<4 x i32> addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture %arg1) { 72; GCN-LABEL: scalar_clause: 73; GCN: ; %bb.0: ; %bb 74; GCN-NEXT: s_load_dwordx2 s[16:17], s[0:1], 0x24 75; GCN-NEXT: s_load_dwordx2 s[18:19], s[0:1], 0x2c 76; GCN-NEXT: v_mov_b32_e32 v12, 0 77; GCN-NEXT: s_waitcnt lgkmcnt(0) 78; GCN-NEXT: s_load_dwordx4 s[0:3], s[16:17], 0x0 79; GCN-NEXT: s_load_dwordx4 s[4:7], s[16:17], 0x10 80; GCN-NEXT: s_load_dwordx4 s[8:11], s[16:17], 0x20 81; GCN-NEXT: s_load_dwordx4 s[12:15], s[16:17], 0x30 82; GCN-NEXT: s_waitcnt lgkmcnt(0) 83; GCN-NEXT: v_mov_b32_e32 v0, s0 84; GCN-NEXT: v_mov_b32_e32 v4, s4 85; GCN-NEXT: v_mov_b32_e32 v8, s8 86; GCN-NEXT: v_mov_b32_e32 v1, s1 87; GCN-NEXT: v_mov_b32_e32 v2, s2 88; GCN-NEXT: v_mov_b32_e32 v3, s3 89; GCN-NEXT: v_mov_b32_e32 v5, s5 90; GCN-NEXT: v_mov_b32_e32 v6, s6 91; GCN-NEXT: v_mov_b32_e32 v7, s7 92; GCN-NEXT: v_mov_b32_e32 v9, s9 93; GCN-NEXT: v_mov_b32_e32 v10, s10 94; GCN-NEXT: v_mov_b32_e32 v11, s11 95; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[18:19] 96; GCN-NEXT: global_store_dwordx4 v12, v[4:7], s[18:19] offset:16 97; GCN-NEXT: global_store_dwordx4 v12, v[8:11], s[18:19] offset:32 98; GCN-NEXT: v_mov_b32_e32 v0, s12 99; GCN-NEXT: v_mov_b32_e32 v1, s13 100; GCN-NEXT: v_mov_b32_e32 v2, s14 101; GCN-NEXT: v_mov_b32_e32 v3, s15 102; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[18:19] offset:48 103; GCN-NEXT: s_endpgm 104; 105; GCN-SCRATCH-LABEL: scalar_clause: 106; GCN-SCRATCH: ; %bb.0: ; %bb 107; GCN-SCRATCH-NEXT: s_clause 0x1 108; GCN-SCRATCH-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x24 109; GCN-SCRATCH-NEXT: s_load_dwordx2 s[16:17], s[0:1], 0x2c 110; GCN-SCRATCH-NEXT: v_mov_b32_e32 v16, 0 111; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) 112; GCN-SCRATCH-NEXT: s_clause 0x3 113; GCN-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[12:13], 0x0 114; GCN-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[12:13], 0x10 115; GCN-SCRATCH-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x20 116; GCN-SCRATCH-NEXT: s_load_dwordx4 s[12:15], s[12:13], 0x30 117; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) 118; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, s0 119; GCN-SCRATCH-NEXT: v_mov_b32_e32 v4, s4 120; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, s1 121; GCN-SCRATCH-NEXT: v_mov_b32_e32 v2, s2 122; GCN-SCRATCH-NEXT: v_mov_b32_e32 v3, s3 123; GCN-SCRATCH-NEXT: v_mov_b32_e32 v8, s8 124; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, s5 125; GCN-SCRATCH-NEXT: v_mov_b32_e32 v6, s6 126; GCN-SCRATCH-NEXT: v_mov_b32_e32 v7, s7 127; GCN-SCRATCH-NEXT: v_mov_b32_e32 v12, s12 128; GCN-SCRATCH-NEXT: v_mov_b32_e32 v9, s9 129; GCN-SCRATCH-NEXT: v_mov_b32_e32 v10, s10 130; GCN-SCRATCH-NEXT: v_mov_b32_e32 v11, s11 131; GCN-SCRATCH-NEXT: v_mov_b32_e32 v13, s13 132; GCN-SCRATCH-NEXT: v_mov_b32_e32 v14, s14 133; GCN-SCRATCH-NEXT: v_mov_b32_e32 v15, s15 134; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] 135; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 136; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 137; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 138; GCN-SCRATCH-NEXT: s_endpgm 139bb: 140 %tmp = load <4 x i32>, <4 x i32> addrspace(1)* %arg, align 16 141 %tmp2 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 1 142 %tmp3 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp2, align 16 143 %tmp4 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 1 144 %tmp5 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 2 145 %tmp6 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp5, align 16 146 %tmp7 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 2 147 %tmp8 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 3 148 %tmp9 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp8, align 16 149 %tmp10 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 3 150 store <4 x i32> %tmp, <4 x i32> addrspace(1)* %arg1, align 16 151 store <4 x i32> %tmp3, <4 x i32> addrspace(1)* %tmp4, align 16 152 store <4 x i32> %tmp6, <4 x i32> addrspace(1)* %tmp7, align 16 153 store <4 x i32> %tmp9, <4 x i32> addrspace(1)* %tmp10, align 16 154 ret void 155} 156 157define void @mubuf_clause(<4 x i32> addrspace(5)* noalias nocapture readonly %arg, <4 x i32> addrspace(5)* noalias nocapture %arg1) { 158; GCN-LABEL: mubuf_clause: 159; GCN: ; %bb.0: ; %bb 160; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 161; GCN-NEXT: v_and_b32_e32 v2, 0x3ff, v2 162; GCN-NEXT: v_lshlrev_b32_e32 v2, 4, v2 163; GCN-NEXT: v_add_u32_e32 v0, v0, v2 164; GCN-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen 165; GCN-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 166; GCN-NEXT: buffer_load_dword v5, v0, s[0:3], 0 offen offset:8 167; GCN-NEXT: buffer_load_dword v6, v0, s[0:3], 0 offen offset:12 168; GCN-NEXT: buffer_load_dword v7, v0, s[0:3], 0 offen offset:16 169; GCN-NEXT: buffer_load_dword v8, v0, s[0:3], 0 offen offset:20 170; GCN-NEXT: buffer_load_dword v9, v0, s[0:3], 0 offen offset:24 171; GCN-NEXT: buffer_load_dword v10, v0, s[0:3], 0 offen offset:28 172; GCN-NEXT: buffer_load_dword v11, v0, s[0:3], 0 offen offset:32 173; GCN-NEXT: buffer_load_dword v12, v0, s[0:3], 0 offen offset:36 174; GCN-NEXT: buffer_load_dword v13, v0, s[0:3], 0 offen offset:40 175; GCN-NEXT: buffer_load_dword v14, v0, s[0:3], 0 offen offset:44 176; GCN-NEXT: buffer_load_dword v15, v0, s[0:3], 0 offen offset:48 177; GCN-NEXT: buffer_load_dword v16, v0, s[0:3], 0 offen offset:52 178; GCN-NEXT: buffer_load_dword v17, v0, s[0:3], 0 offen offset:56 179; GCN-NEXT: s_nop 0 180; GCN-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:60 181; GCN-NEXT: v_add_u32_e32 v1, v1, v2 182; GCN-NEXT: s_waitcnt vmcnt(12) 183; GCN-NEXT: buffer_store_dword v6, v1, s[0:3], 0 offen offset:12 184; GCN-NEXT: buffer_store_dword v5, v1, s[0:3], 0 offen offset:8 185; GCN-NEXT: buffer_store_dword v4, v1, s[0:3], 0 offen offset:4 186; GCN-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen 187; GCN-NEXT: s_waitcnt vmcnt(12) 188; GCN-NEXT: buffer_store_dword v10, v1, s[0:3], 0 offen offset:28 189; GCN-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen offset:24 190; GCN-NEXT: buffer_store_dword v8, v1, s[0:3], 0 offen offset:20 191; GCN-NEXT: buffer_store_dword v7, v1, s[0:3], 0 offen offset:16 192; GCN-NEXT: s_waitcnt vmcnt(12) 193; GCN-NEXT: buffer_store_dword v14, v1, s[0:3], 0 offen offset:44 194; GCN-NEXT: buffer_store_dword v13, v1, s[0:3], 0 offen offset:40 195; GCN-NEXT: buffer_store_dword v12, v1, s[0:3], 0 offen offset:36 196; GCN-NEXT: buffer_store_dword v11, v1, s[0:3], 0 offen offset:32 197; GCN-NEXT: s_waitcnt vmcnt(12) 198; GCN-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:60 199; GCN-NEXT: buffer_store_dword v17, v1, s[0:3], 0 offen offset:56 200; GCN-NEXT: buffer_store_dword v16, v1, s[0:3], 0 offen offset:52 201; GCN-NEXT: buffer_store_dword v15, v1, s[0:3], 0 offen offset:48 202; GCN-NEXT: s_waitcnt vmcnt(0) 203; GCN-NEXT: s_setpc_b64 s[30:31] 204; 205; GCN-SCRATCH-LABEL: mubuf_clause: 206; GCN-SCRATCH: ; %bb.0: ; %bb 207; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 208; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0 209; GCN-SCRATCH-NEXT: v_and_b32_e32 v2, 0x3ff, v2 210; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v18, 4, v2 211; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v0, v0, v18 212; GCN-SCRATCH-NEXT: s_clause 0x3 213; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[2:5], v0, off 214; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[6:9], v0, off offset:16 215; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[10:13], v0, off offset:32 216; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[14:17], v0, off offset:48 217; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v0, v1, v18 218; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(3) 219; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[2:5], off 220; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(2) 221; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:16 222; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1) 223; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:32 224; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) 225; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:48 226; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0 227; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31] 228bb: 229 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() 230 %tmp2 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp 231 %tmp3 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp2, align 16 232 %tmp4 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp 233 %tmp5 = add nuw nsw i32 %tmp, 1 234 %tmp6 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp5 235 %tmp7 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp6, align 16 236 %tmp8 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp5 237 %tmp9 = add nuw nsw i32 %tmp, 2 238 %tmp10 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp9 239 %tmp11 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp10, align 16 240 %tmp12 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp9 241 %tmp13 = add nuw nsw i32 %tmp, 3 242 %tmp14 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp13 243 %tmp15 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp14, align 16 244 %tmp16 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp13 245 store <4 x i32> %tmp3, <4 x i32> addrspace(5)* %tmp4, align 16 246 store <4 x i32> %tmp7, <4 x i32> addrspace(5)* %tmp8, align 16 247 store <4 x i32> %tmp11, <4 x i32> addrspace(5)* %tmp12, align 16 248 store <4 x i32> %tmp15, <4 x i32> addrspace(5)* %tmp16, align 16 249 ret void 250} 251 252define amdgpu_kernel void @vector_clause_indirect(i64 addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture readnone %arg1, <4 x i32> addrspace(1)* noalias nocapture %arg2) { 253; GCN-LABEL: vector_clause_indirect: 254; GCN: ; %bb.0: ; %bb 255; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 256; GCN-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 257; GCN-NEXT: v_lshlrev_b32_e32 v0, 3, v0 258; GCN-NEXT: s_waitcnt lgkmcnt(0) 259; GCN-NEXT: global_load_dwordx2 v[8:9], v0, s[2:3] 260; GCN-NEXT: s_waitcnt vmcnt(0) 261; GCN-NEXT: global_load_dwordx4 v[0:3], v[8:9], off 262; GCN-NEXT: global_load_dwordx4 v[4:7], v[8:9], off offset:16 263; GCN-NEXT: v_mov_b32_e32 v8, 0 264; GCN-NEXT: s_waitcnt vmcnt(1) 265; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5] 266; GCN-NEXT: s_waitcnt vmcnt(1) 267; GCN-NEXT: global_store_dwordx4 v8, v[4:7], s[4:5] offset:16 268; GCN-NEXT: s_endpgm 269; 270; GCN-SCRATCH-LABEL: vector_clause_indirect: 271; GCN-SCRATCH: ; %bb.0: ; %bb 272; GCN-SCRATCH-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 273; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v0, 3, v0 274; GCN-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 275; GCN-SCRATCH-NEXT: v_mov_b32_e32 v8, 0 276; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) 277; GCN-SCRATCH-NEXT: global_load_dwordx2 v[4:5], v0, s[2:3] 278; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) 279; GCN-SCRATCH-NEXT: s_clause 0x1 280; GCN-SCRATCH-NEXT: global_load_dwordx4 v[0:3], v[4:5], off 281; GCN-SCRATCH-NEXT: global_load_dwordx4 v[4:7], v[4:5], off offset:16 282; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1) 283; GCN-SCRATCH-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] 284; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) 285; GCN-SCRATCH-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 286; GCN-SCRATCH-NEXT: s_endpgm 287bb: 288 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() 289 %tmp3 = zext i32 %tmp to i64 290 %tmp4 = getelementptr inbounds i64, i64 addrspace(1)* %arg, i64 %tmp3 291 %tmp5 = bitcast i64 addrspace(1)* %tmp4 to <4 x i32> addrspace(1)* addrspace(1)* 292 %tmp6 = load <4 x i32> addrspace(1)*, <4 x i32> addrspace(1)* addrspace(1)* %tmp5, align 8 293 %tmp7 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp6, align 16 294 %tmp8 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %tmp6, i64 1 295 %tmp9 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp8, align 16 296 store <4 x i32> %tmp7, <4 x i32> addrspace(1)* %arg2, align 16 297 %tmp10 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg2, i64 1 298 store <4 x i32> %tmp9, <4 x i32> addrspace(1)* %tmp10, align 16 299 ret void 300} 301 302define void @load_global_d16_hi(i16 addrspace(1)* %in, i16 %reg, <2 x i16> addrspace(1)* %out) { 303; GCN-LABEL: load_global_d16_hi: 304; GCN: ; %bb.0: ; %entry 305; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 306; GCN-NEXT: v_mov_b32_e32 v5, v2 307; GCN-NEXT: global_load_short_d16_hi v5, v[0:1], off 308; GCN-NEXT: s_nop 0 309; GCN-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:64 310; GCN-NEXT: s_waitcnt vmcnt(1) 311; GCN-NEXT: global_store_dword v[3:4], v5, off 312; GCN-NEXT: s_waitcnt vmcnt(1) 313; GCN-NEXT: global_store_dword v[3:4], v2, off offset:128 314; GCN-NEXT: s_waitcnt vmcnt(0) 315; GCN-NEXT: s_setpc_b64 s[30:31] 316; 317; GCN-SCRATCH-LABEL: load_global_d16_hi: 318; GCN-SCRATCH: ; %bb.0: ; %entry 319; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 320; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0 321; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, v2 322; GCN-SCRATCH-NEXT: s_clause 0x1 323; GCN-SCRATCH-NEXT: global_load_short_d16_hi v5, v[0:1], off 324; GCN-SCRATCH-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:64 325; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1) 326; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v5, off 327; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) 328; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v2, off offset:128 329; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0 330; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31] 331entry: 332 %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 32 333 %load1 = load i16, i16 addrspace(1)* %in 334 %load2 = load i16, i16 addrspace(1)* %gep 335 %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0 336 %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1 337 store <2 x i16> %build1, <2 x i16> addrspace(1)* %out 338 %build2 = insertelement <2 x i16> undef, i16 %reg, i32 0 339 %build3 = insertelement <2 x i16> %build2, i16 %load2, i32 1 340 %gep2 = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 32 341 store <2 x i16> %build3, <2 x i16> addrspace(1)* %gep2 342 ret void 343} 344 345define void @load_global_d16_lo(i16 addrspace(1)* %in, i32 %reg, <2 x i16> addrspace(1)* %out) { 346; GCN-LABEL: load_global_d16_lo: 347; GCN: ; %bb.0: ; %entry 348; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 349; GCN-NEXT: v_mov_b32_e32 v5, v2 350; GCN-NEXT: global_load_short_d16 v5, v[0:1], off 351; GCN-NEXT: s_nop 0 352; GCN-NEXT: global_load_short_d16 v2, v[0:1], off offset:64 353; GCN-NEXT: s_waitcnt vmcnt(1) 354; GCN-NEXT: global_store_dword v[3:4], v5, off 355; GCN-NEXT: s_waitcnt vmcnt(1) 356; GCN-NEXT: global_store_dword v[3:4], v2, off offset:128 357; GCN-NEXT: s_waitcnt vmcnt(0) 358; GCN-NEXT: s_setpc_b64 s[30:31] 359; 360; GCN-SCRATCH-LABEL: load_global_d16_lo: 361; GCN-SCRATCH: ; %bb.0: ; %entry 362; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 363; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0 364; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, v2 365; GCN-SCRATCH-NEXT: s_clause 0x1 366; GCN-SCRATCH-NEXT: global_load_short_d16 v5, v[0:1], off 367; GCN-SCRATCH-NEXT: global_load_short_d16 v2, v[0:1], off offset:64 368; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1) 369; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v5, off 370; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) 371; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v2, off offset:128 372; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0 373; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31] 374entry: 375 %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 32 376 %reg.bc1 = bitcast i32 %reg to <2 x i16> 377 %reg.bc2 = bitcast i32 %reg to <2 x i16> 378 %load1 = load i16, i16 addrspace(1)* %in 379 %load2 = load i16, i16 addrspace(1)* %gep 380 %build1 = insertelement <2 x i16> %reg.bc1, i16 %load1, i32 0 381 %build2 = insertelement <2 x i16> %reg.bc2, i16 %load2, i32 0 382 %gep2 = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 32 383 store <2 x i16> %build1, <2 x i16> addrspace(1)* %out 384 store <2 x i16> %build2, <2 x i16> addrspace(1)* %gep2 385 ret void 386} 387 388define amdgpu_kernel void @flat_scratch_load(float %a, float %b, <8 x i32> %desc) { 389; GCN-LABEL: flat_scratch_load: 390; GCN: ; %bb.0: ; %.entry 391; GCN-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 392; GCN-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 393; GCN-NEXT: s_mov_b32 s18, -1 394; GCN-NEXT: s_mov_b32 s19, 0xe00000 395; GCN-NEXT: s_add_u32 s16, s16, s3 396; GCN-NEXT: s_addc_u32 s17, s17, 0 397; GCN-NEXT: s_mov_b64 s[12:13], exec 398; GCN-NEXT: s_wqm_b64 exec, exec 399; GCN-NEXT: v_mov_b32_e32 v0, 0x40b00000 400; GCN-NEXT: s_load_dwordx2 s[14:15], s[0:1], 0x24 401; GCN-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x44 402; GCN-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:4 403; GCN-NEXT: s_waitcnt vmcnt(0) 404; GCN-NEXT: ;;#ASMSTART 405; GCN-NEXT: ;;#ASMEND 406; GCN-NEXT: buffer_load_dword v2, off, s[16:19], 0 offset:4 407; GCN-NEXT: s_brev_b32 s0, 1 408; GCN-NEXT: s_waitcnt lgkmcnt(0) 409; GCN-NEXT: v_mov_b32_e32 v0, s14 410; GCN-NEXT: s_mov_b32 s3, 0 411; GCN-NEXT: s_mov_b32 s1, s0 412; GCN-NEXT: s_mov_b32 s2, s0 413; GCN-NEXT: v_mov_b32_e32 v1, s15 414; GCN-NEXT: s_and_b64 exec, exec, s[12:13] 415; GCN-NEXT: image_sample v0, v[0:1], s[4:11], s[0:3] dmask:0x1 416; GCN-NEXT: s_waitcnt vmcnt(0) 417; GCN-NEXT: v_add_f32_e32 v0, v2, v0 418; GCN-NEXT: exp mrt0 v0, off, off, off done vm 419; GCN-NEXT: s_endpgm 420; 421; GCN-SCRATCH-LABEL: flat_scratch_load: 422; GCN-SCRATCH: ; %bb.0: ; %.entry 423; GCN-SCRATCH-NEXT: s_add_u32 s2, s2, s5 424; GCN-SCRATCH-NEXT: s_addc_u32 s3, s3, 0 425; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2 426; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3 427; GCN-SCRATCH-NEXT: s_mov_b32 s9, exec_lo 428; GCN-SCRATCH-NEXT: s_wqm_b32 exec_lo, exec_lo 429; GCN-SCRATCH-NEXT: s_clause 0x1 430; GCN-SCRATCH-NEXT: s_load_dwordx2 s[10:11], s[0:1], 0x24 431; GCN-SCRATCH-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x44 432; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, 0x40b00000 433; GCN-SCRATCH-NEXT: s_brev_b32 s8, 1 434; GCN-SCRATCH-NEXT: scratch_store_dword off, v0, off offset:4 435; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0 436; GCN-SCRATCH-NEXT: ;;#ASMSTART 437; GCN-SCRATCH-NEXT: ;;#ASMEND 438; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) 439; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, s10 440; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, s11 441; GCN-SCRATCH-NEXT: s_and_b32 exec_lo, exec_lo, s9 442; GCN-SCRATCH-NEXT: s_mov_b32 s11, 0 443; GCN-SCRATCH-NEXT: s_mov_b32 s9, s8 444; GCN-SCRATCH-NEXT: s_mov_b32 s10, s8 445; GCN-SCRATCH-NEXT: scratch_load_dword v2, off, off offset:4 446; GCN-SCRATCH-NEXT: image_sample v0, v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D 447; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) 448; GCN-SCRATCH-NEXT: v_add_f32_e32 v0, v2, v0 449; GCN-SCRATCH-NEXT: exp mrt0 v0, off, off, off done vm 450; GCN-SCRATCH-NEXT: s_endpgm 451.entry: 452 %alloca = alloca float, align 4, addrspace(5) 453 store volatile float 5.5, float addrspace(5)* %alloca 454 call void asm sideeffect "", ""() 455 ; There was a bug with flat scratch instructions that do not not use any address registers (ST mode). 456 ; To trigger, the scratch_load has to be immediately before the image_sample in MIR. 457 %load = load float, float addrspace(5)* %alloca 458 %val = call <2 x float> @llvm.amdgcn.image.sample.2d.v2f32.f32(i32 9, float %a, float %b, <8 x i32> %desc, <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 0>, i1 false, i32 0, i32 0) 459 %val0 = extractelement <2 x float> %val, i32 0 460 %valadd = fadd float %load, %val0 461 call void @llvm.amdgcn.exp.f32(i32 immarg 0, i32 immarg 1, float %valadd, float undef, float undef, float undef, i1 immarg true, i1 immarg true) 462 ret void 463} 464 465define amdgpu_kernel void @flat_scratch_load_clause(float %a, float %b, <8 x i32> %desc) { 466; GCN-LABEL: flat_scratch_load_clause: 467; GCN: ; %bb.0: ; %.entry 468; GCN-NEXT: s_mov_b32 s4, SCRATCH_RSRC_DWORD0 469; GCN-NEXT: s_mov_b32 s5, SCRATCH_RSRC_DWORD1 470; GCN-NEXT: s_mov_b32 s6, -1 471; GCN-NEXT: s_mov_b32 s7, 0xe00000 472; GCN-NEXT: s_add_u32 s4, s4, s3 473; GCN-NEXT: s_addc_u32 s5, s5, 0 474; GCN-NEXT: v_mov_b32_e32 v0, 0x40b00000 475; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:4 476; GCN-NEXT: s_waitcnt vmcnt(0) 477; GCN-NEXT: v_mov_b32_e32 v0, 0x40d00000 478; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:8 479; GCN-NEXT: s_waitcnt vmcnt(0) 480; GCN-NEXT: ;;#ASMSTART 481; GCN-NEXT: ;;#ASMEND 482; GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0 offset:4 483; GCN-NEXT: buffer_load_dword v1, off, s[4:7], 0 offset:8 484; GCN-NEXT: s_waitcnt vmcnt(0) 485; GCN-NEXT: v_add_f32_e32 v0, v0, v1 486; GCN-NEXT: exp mrt0 v0, off, off, off done vm 487; GCN-NEXT: s_endpgm 488; 489; GCN-SCRATCH-LABEL: flat_scratch_load_clause: 490; GCN-SCRATCH: ; %bb.0: ; %.entry 491; GCN-SCRATCH-NEXT: s_add_u32 s2, s2, s5 492; GCN-SCRATCH-NEXT: s_addc_u32 s3, s3, 0 493; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2 494; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3 495; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, 0x40b00000 496; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, 0x40d00000 497; GCN-SCRATCH-NEXT: scratch_store_dword off, v0, off offset:4 498; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0 499; GCN-SCRATCH-NEXT: scratch_store_dword off, v1, off offset:8 500; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0 501; GCN-SCRATCH-NEXT: ;;#ASMSTART 502; GCN-SCRATCH-NEXT: ;;#ASMEND 503; GCN-SCRATCH-NEXT: s_clause 0x1 504; GCN-SCRATCH-NEXT: scratch_load_dword v0, off, off offset:4 505; GCN-SCRATCH-NEXT: scratch_load_dword v1, off, off offset:8 506; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) 507; GCN-SCRATCH-NEXT: v_add_f32_e32 v0, v0, v1 508; GCN-SCRATCH-NEXT: exp mrt0 v0, off, off, off done vm 509; GCN-SCRATCH-NEXT: s_endpgm 510.entry: 511 %alloca = alloca float, align 4, addrspace(5) 512 %alloca2 = alloca float, align 4, addrspace(5) 513 store volatile float 5.5, float addrspace(5)* %alloca 514 store volatile float 6.5, float addrspace(5)* %alloca2 515 call void asm sideeffect "", ""() 516 %load0 = load float, float addrspace(5)* %alloca 517 %load1 = load float, float addrspace(5)* %alloca2 518 %valadd = fadd float %load0, %load1 519 call void @llvm.amdgcn.exp.f32(i32 immarg 0, i32 immarg 1, float %valadd, float undef, float undef, float undef, i1 immarg true, i1 immarg true) 520 ret void 521} 522 523declare i32 @llvm.amdgcn.workitem.id.x() 524declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg) 525declare <2 x float> @llvm.amdgcn.image.sample.2d.v2f32.f32(i32 immarg, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) 526