1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def smrd_offset_8 : NamedOperandU32<"SMRDOffset8", 10 NamedMatchClass<"SMRDOffset8">> { 11 let OperandType = "OPERAND_IMMEDIATE"; 12} 13 14class SMEMOffset : NamedOperandU32<"SMEMOffset", 15 NamedMatchClass<"SMEMOffset">> { 16 let OperandType = "OPERAND_IMMEDIATE"; 17 let EncoderMethod = "getSMEMOffsetEncoding"; 18 let DecoderMethod = "decodeSMEMOffset"; 19} 20 21def smem_offset : SMEMOffset; 22 23def smem_offset_mod : SMEMOffset { 24 let PrintMethod = "printSMEMOffsetMod"; 25} 26 27//===----------------------------------------------------------------------===// 28// Scalar Memory classes 29//===----------------------------------------------------------------------===// 30 31class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> : 32 InstSI <outs, ins, "", pattern>, 33 SIMCInstr<opName, SIEncodingFamily.NONE> { 34 let isPseudo = 1; 35 let isCodeGenOnly = 1; 36 37 let LGKM_CNT = 1; 38 let SMRD = 1; 39 let mayStore = 0; 40 let mayLoad = 1; 41 let hasSideEffects = 0; 42 let UseNamedOperandTable = 1; 43 let SchedRW = [WriteSMEM]; 44 45 string Mnemonic = opName; 46 string AsmOperands = asmOps; 47 48 bits<1> has_sbase = 1; 49 bits<1> has_sdst = 1; 50 bit has_glc = 0; 51 bit has_dlc = 0; 52 bit has_offset = 0; 53 bit has_soffset = 0; 54 bit is_buffer = 0; 55} 56 57class SM_Real <SM_Pseudo ps, string opName = ps.Mnemonic> 58 : InstSI<ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands> { 59 60 let isPseudo = 0; 61 let isCodeGenOnly = 0; 62 63 Instruction Opcode = !cast<Instruction>(NAME); 64 65 // copy relevant pseudo op flags 66 let LGKM_CNT = ps.LGKM_CNT; 67 let SMRD = ps.SMRD; 68 let mayStore = ps.mayStore; 69 let mayLoad = ps.mayLoad; 70 let hasSideEffects = ps.hasSideEffects; 71 let UseNamedOperandTable = ps.UseNamedOperandTable; 72 let SchedRW = ps.SchedRW; 73 let SubtargetPredicate = ps.SubtargetPredicate; 74 let AsmMatchConverter = ps.AsmMatchConverter; 75 let IsAtomicRet = ps.IsAtomicRet; 76 let IsAtomicNoRet = ps.IsAtomicNoRet; 77 78 let TSFlags = ps.TSFlags; 79 80 bit is_buffer = ps.is_buffer; 81 82 // encoding 83 bits<7> sbase; 84 bits<7> sdst; 85 bits<32> offset; 86 bits<8> soffset; 87 bits<5> cpol; 88} 89 90class OffsetMode<bit hasOffset, bit hasSOffset, string variant, 91 dag ins, string asm> { 92 bit HasOffset = hasOffset; 93 bit HasSOffset = hasSOffset; 94 string Variant = variant; 95 dag Ins = ins; 96 string Asm = asm; 97} 98 99def IMM_Offset : OffsetMode<1, 0, "_IMM", (ins smem_offset:$offset), "$offset">; 100def SGPR_Offset : OffsetMode<0, 1, "_SGPR", (ins SReg_32:$soffset), "$soffset">; 101def SGPR_IMM_Offset : OffsetMode<1, 1, "_SGPR_IMM", 102 (ins SReg_32:$soffset, smem_offset_mod:$offset), 103 "$soffset$offset">; 104 105class SM_Probe_Pseudo <string opName, string variant, RegisterClass baseClass, 106 dag offsets, string asmOffsets, 107 bit hasOffset, bit hasSOffset> 108 : SM_Pseudo<opName, (outs), 109 !con((ins i8imm:$sdata, baseClass:$sbase), offsets), 110 " $sdata, $sbase, " # asmOffsets> { 111 let mayLoad = 0; 112 let mayStore = 0; 113 let has_glc = 0; 114 let LGKM_CNT = 0; 115 let ScalarStore = 0; 116 let hasSideEffects = 1; 117 let has_offset = hasOffset; 118 let has_soffset = hasSOffset; 119 let PseudoInstr = opName # variant; 120} 121 122class SM_Load_Pseudo <string opName, RegisterClass baseClass, 123 RegisterClass dstClass, OffsetMode offsets> 124 : SM_Pseudo<opName, (outs dstClass:$sdst), 125 !con((ins baseClass:$sbase), offsets.Ins, (ins CPol:$cpol)), 126 " $sdst, $sbase, " # offsets.Asm # "$cpol", []> { 127 RegisterClass BaseClass = baseClass; 128 let mayLoad = 1; 129 let mayStore = 0; 130 let has_glc = 1; 131 let has_dlc = 1; 132 let has_offset = offsets.HasOffset; 133 let has_soffset = offsets.HasSOffset; 134 let PseudoInstr = opName # offsets.Variant; 135} 136 137class SM_Store_Pseudo <string opName, RegisterClass baseClass, 138 RegisterClass srcClass, dag ins, string asmOps> 139 : SM_Pseudo<opName, (outs), ins, asmOps, []> { 140 RegisterClass BaseClass = baseClass; 141 RegisterClass SrcClass = srcClass; 142 let mayLoad = 0; 143 let mayStore = 1; 144 let has_glc = 1; 145 let has_dlc = 1; 146 let ScalarStore = 1; 147} 148 149class SM_Discard_Pseudo <string opName, string variant, dag offsets, 150 string asmOffsets, bit hasOffset, bit hasSOffset> 151 : SM_Pseudo<opName, (outs), !con((ins SReg_64:$sbase), offsets), 152 " $sbase, " # asmOffsets> { 153 let mayLoad = 0; 154 let mayStore = 0; 155 let has_glc = 0; 156 let has_sdst = 0; 157 let ScalarStore = 0; 158 let hasSideEffects = 1; 159 let has_offset = hasOffset; 160 let has_soffset = hasSOffset; 161 let PseudoInstr = opName # variant; 162} 163 164multiclass SM_Pseudo_Loads<string opName, 165 RegisterClass baseClass, 166 RegisterClass dstClass> { 167 def _IMM : SM_Load_Pseudo <opName, baseClass, dstClass, IMM_Offset>; 168 def _SGPR : SM_Load_Pseudo <opName, baseClass, dstClass, SGPR_Offset>; 169 def _SGPR_IMM : SM_Load_Pseudo <opName, baseClass, dstClass, SGPR_IMM_Offset>; 170} 171 172multiclass SM_Pseudo_Stores<string opName, 173 RegisterClass baseClass, 174 RegisterClass srcClass> { 175 def _IMM : SM_Store_Pseudo <opName, baseClass, srcClass, 176 (ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, CPol:$cpol), 177 " $sdata, $sbase, $offset$cpol"> { 178 let has_offset = 1; 179 let PseudoInstr = opName # "_IMM"; 180 } 181 182 def _SGPR : SM_Store_Pseudo <opName, baseClass, srcClass, 183 (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soffset, CPol:$cpol), 184 " $sdata, $sbase, $soffset$cpol"> { 185 let has_soffset = 1; 186 let PseudoInstr = opName # "_SGPR"; 187 } 188 189 def _SGPR_IMM : SM_Store_Pseudo <opName, baseClass, srcClass, 190 (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soffset, i32imm:$offset, 191 CPol:$cpol), 192 " $sdata, $sbase, $soffset$offset$cpol"> { 193 let has_offset = 1; 194 let has_soffset = 1; 195 let PseudoInstr = opName # "_SGPR_IMM"; 196 } 197} 198 199multiclass SM_Pseudo_Discards<string opName> { 200 def _IMM : SM_Discard_Pseudo <opName, "_IMM", 201 (ins smem_offset:$offset), "$offset", 1, 0>; 202 def _SGPR : SM_Discard_Pseudo <opName, "_SGPR", 203 (ins SReg_32:$soffset), "$soffset", 0, 1>; 204 def _SGPR_IMM : SM_Discard_Pseudo <opName, "_SGPR_IMM", 205 (ins SReg_32:$soffset, smem_offset_mod:$offset), "$soffset$offset", 1, 1>; 206} 207 208class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 209 opName, (outs SReg_64_XEXEC:$sdst), (ins), 210 " $sdst", [(set i64:$sdst, (node))]> { 211 let hasSideEffects = 1; 212 213 let mayStore = 0; 214 let mayLoad = 0; 215 let has_sbase = 0; 216} 217 218class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 219 opName, (outs), (ins), "", [(node)]> { 220 let hasSideEffects = 1; 221 let mayLoad = 0; 222 let mayStore = 0; 223 let has_sdst = 0; 224 let has_sbase = 0; 225} 226 227multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> { 228 def _IMM : SM_Probe_Pseudo <opName, "_IMM", baseClass, 229 (ins smem_offset:$offset), "$offset", 1, 0>; 230 def _SGPR : SM_Probe_Pseudo <opName, "_SGPR", baseClass, 231 (ins SReg_32:$soffset), "$soffset", 0, 1>; 232 def _SGPR_IMM : SM_Probe_Pseudo <opName, "_SGPR_IMM", baseClass, 233 (ins SReg_32:$soffset, smem_offset_mod:$offset), "$soffset$offset", 1, 1>; 234} 235 236class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo< 237 opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins), 238 " $sdst", [(set i32:$sdst, (node))]> { 239 let hasSideEffects = 1; 240 let mayStore = 0; 241 let mayLoad = 0; 242 let has_sbase = 0; 243} 244 245//===----------------------------------------------------------------------===// 246// Scalar Atomic Memory Classes 247//===----------------------------------------------------------------------===// 248 249class SM_Atomic_Pseudo <string opName, 250 dag outs, dag ins, string asmOps, bit isRet> 251 : SM_Pseudo<opName, outs, ins, asmOps, []> { 252 253 bit glc = isRet; 254 255 let mayLoad = 1; 256 let mayStore = 1; 257 let has_glc = 1; 258 let has_dlc = 1; 259 let has_soffset = 1; 260 261 // Should these be set? 262 let ScalarStore = 1; 263 let hasSideEffects = 1; 264 let maybeAtomic = 1; 265 266 let IsAtomicNoRet = !not(isRet); 267 let IsAtomicRet = isRet; 268 269 let AsmMatchConverter = "cvtSMEMAtomic"; 270} 271 272class SM_Pseudo_Atomic<string opName, 273 RegisterClass baseClass, 274 RegisterClass dataClass, 275 OffsetMode offsets, 276 bit isRet, 277 string opNameWithSuffix = 278 opName # offsets.Variant # !if(isRet, "_RTN", ""), 279 Operand CPolTy = !if(isRet, CPol_GLC1, CPol)> : 280 SM_Atomic_Pseudo<opName, 281 !if(isRet, (outs dataClass:$sdst), (outs)), 282 !con((ins dataClass:$sdata, baseClass:$sbase), offsets.Ins, 283 (ins CPolTy:$cpol)), 284 !if(isRet, " $sdst", " $sdata") # 285 ", $sbase, " # offsets.Asm # "$cpol", 286 isRet>, 287 AtomicNoRet <opNameWithSuffix, isRet> { 288 let has_offset = offsets.HasOffset; 289 let has_soffset = offsets.HasSOffset; 290 let PseudoInstr = opNameWithSuffix; 291 292 let Constraints = !if(isRet, "$sdst = $sdata", ""); 293 let DisableEncoding = !if(isRet, "$sdata", ""); 294} 295 296multiclass SM_Pseudo_Atomics<string opName, 297 RegisterClass baseClass, 298 RegisterClass dataClass> { 299 def _IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 0>; 300 def _SGPR : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 0>; 301 def _SGPR_IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 0>; 302 def _IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 1>; 303 def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 1>; 304 def _SGPR_IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 1>; 305} 306 307//===----------------------------------------------------------------------===// 308// Scalar Memory Instructions 309//===----------------------------------------------------------------------===// 310 311// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit 312// SMRD instructions, because the SReg_32_XM0 register class does not include M0 313// and writing to M0 from an SMRD instruction will hang the GPU. 314 315// XXX - SMEM instructions do not allow exec for data operand, but 316// does sdst for SMRD on SI/CI? 317defm S_LOAD_DWORD : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 318defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64_XEXEC>; 319defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>; 320defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>; 321defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>; 322 323let is_buffer = 1 in { 324defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads < 325 "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC 326>; 327 328// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on 329// SI/CI, bit disallowed for SMEM on VI. 330defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads < 331 "s_buffer_load_dwordx2", SReg_128, SReg_64_XEXEC 332>; 333 334defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads < 335 "s_buffer_load_dwordx4", SReg_128, SReg_128 336>; 337 338defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads < 339 "s_buffer_load_dwordx8", SReg_128, SReg_256 340>; 341 342defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads < 343 "s_buffer_load_dwordx16", SReg_128, SReg_512 344>; 345} 346 347let SubtargetPredicate = HasScalarStores in { 348defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 349defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>; 350defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>; 351 352let is_buffer = 1 in { 353defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores < 354 "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC 355>; 356 357defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores < 358 "s_buffer_store_dwordx2", SReg_128, SReg_64_XEXEC 359>; 360 361defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores < 362 "s_buffer_store_dwordx4", SReg_128, SReg_128 363>; 364} 365} // End SubtargetPredicate = HasScalarStores 366 367let SubtargetPredicate = HasSMemTimeInst in 368def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>; 369def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>; 370 371let SubtargetPredicate = isGFX7GFX8GFX9 in { 372def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>; 373} // let SubtargetPredicate = isGFX7GFX8GFX9 374 375let SubtargetPredicate = isGFX8Plus in { 376let OtherPredicates = [HasScalarStores] in { 377def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>; 378def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>; 379} // End OtherPredicates = [HasScalarStores] 380 381defm S_ATC_PROBE : SM_Pseudo_Probe <"s_atc_probe", SReg_64>; 382let is_buffer = 1 in { 383defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>; 384} 385} // SubtargetPredicate = isGFX8Plus 386 387let SubtargetPredicate = HasSMemRealTime in 388def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>; 389 390let SubtargetPredicate = isGFX10Plus in 391def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">; 392let SubtargetPredicate = HasGetWaveIdInst in 393def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>; 394 395 396let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in { 397defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <"s_scratch_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 398defm S_SCRATCH_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>; 399defm S_SCRATCH_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>; 400 401defm S_SCRATCH_STORE_DWORD : SM_Pseudo_Stores <"s_scratch_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 402defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>; 403defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>; 404} // SubtargetPredicate = HasScalarFlatScratchInsts 405 406let SubtargetPredicate = HasScalarAtomics in { 407 408let is_buffer = 1 in { 409defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>; 410defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>; 411defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>; 412defm S_BUFFER_ATOMIC_SUB : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>; 413defm S_BUFFER_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>; 414defm S_BUFFER_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>; 415defm S_BUFFER_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>; 416defm S_BUFFER_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>; 417defm S_BUFFER_ATOMIC_AND : SM_Pseudo_Atomics <"s_buffer_atomic_and", SReg_128, SReg_32_XM0_XEXEC>; 418defm S_BUFFER_ATOMIC_OR : SM_Pseudo_Atomics <"s_buffer_atomic_or", SReg_128, SReg_32_XM0_XEXEC>; 419defm S_BUFFER_ATOMIC_XOR : SM_Pseudo_Atomics <"s_buffer_atomic_xor", SReg_128, SReg_32_XM0_XEXEC>; 420defm S_BUFFER_ATOMIC_INC : SM_Pseudo_Atomics <"s_buffer_atomic_inc", SReg_128, SReg_32_XM0_XEXEC>; 421defm S_BUFFER_ATOMIC_DEC : SM_Pseudo_Atomics <"s_buffer_atomic_dec", SReg_128, SReg_32_XM0_XEXEC>; 422 423defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_swap_x2", SReg_128, SReg_64_XEXEC>; 424defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap_x2", SReg_128, SReg_128>; 425defm S_BUFFER_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_add_x2", SReg_128, SReg_64_XEXEC>; 426defm S_BUFFER_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_sub_x2", SReg_128, SReg_64_XEXEC>; 427defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smin_x2", SReg_128, SReg_64_XEXEC>; 428defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>; 429defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>; 430defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>; 431defm S_BUFFER_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>; 432defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>; 433defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>; 434defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>; 435defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>; 436} 437 438defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>; 439defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>; 440defm S_ATOMIC_ADD : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>; 441defm S_ATOMIC_SUB : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>; 442defm S_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>; 443defm S_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>; 444defm S_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>; 445defm S_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_atomic_umax", SReg_64, SReg_32_XM0_XEXEC>; 446defm S_ATOMIC_AND : SM_Pseudo_Atomics <"s_atomic_and", SReg_64, SReg_32_XM0_XEXEC>; 447defm S_ATOMIC_OR : SM_Pseudo_Atomics <"s_atomic_or", SReg_64, SReg_32_XM0_XEXEC>; 448defm S_ATOMIC_XOR : SM_Pseudo_Atomics <"s_atomic_xor", SReg_64, SReg_32_XM0_XEXEC>; 449defm S_ATOMIC_INC : SM_Pseudo_Atomics <"s_atomic_inc", SReg_64, SReg_32_XM0_XEXEC>; 450defm S_ATOMIC_DEC : SM_Pseudo_Atomics <"s_atomic_dec", SReg_64, SReg_32_XM0_XEXEC>; 451 452defm S_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_atomic_swap_x2", SReg_64, SReg_64_XEXEC>; 453defm S_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_atomic_cmpswap_x2", SReg_64, SReg_128>; 454defm S_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_atomic_add_x2", SReg_64, SReg_64_XEXEC>; 455defm S_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_atomic_sub_x2", SReg_64, SReg_64_XEXEC>; 456defm S_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_atomic_smin_x2", SReg_64, SReg_64_XEXEC>; 457defm S_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_atomic_umin_x2", SReg_64, SReg_64_XEXEC>; 458defm S_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_atomic_smax_x2", SReg_64, SReg_64_XEXEC>; 459defm S_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_atomic_umax_x2", SReg_64, SReg_64_XEXEC>; 460defm S_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_atomic_and_x2", SReg_64, SReg_64_XEXEC>; 461defm S_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_atomic_or_x2", SReg_64, SReg_64_XEXEC>; 462defm S_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_atomic_xor_x2", SReg_64, SReg_64_XEXEC>; 463defm S_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_atomic_inc_x2", SReg_64, SReg_64_XEXEC>; 464defm S_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_atomic_dec_x2", SReg_64, SReg_64_XEXEC>; 465 466} // let SubtargetPredicate = HasScalarAtomics 467 468let SubtargetPredicate = HasScalarAtomics in { 469defm S_DCACHE_DISCARD : SM_Pseudo_Discards <"s_dcache_discard">; 470defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards <"s_dcache_discard_x2">; 471} 472 473//===----------------------------------------------------------------------===// 474// Targets 475//===----------------------------------------------------------------------===// 476 477//===----------------------------------------------------------------------===// 478// SI 479//===----------------------------------------------------------------------===// 480 481class SMRD_Real_si <bits<5> op, SM_Pseudo ps> 482 : SM_Real<ps> 483 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 484 , Enc32 { 485 486 let AssemblerPredicate = isGFX6GFX7; 487 let DecoderNamespace = "GFX6GFX7"; 488 489 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?)); 490 let Inst{8} = ps.has_offset; 491 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 492 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 493 let Inst{26-22} = op; 494 let Inst{31-27} = 0x18; //encoding 495} 496 497multiclass SM_Real_Loads_si<bits<5> op, string ps, 498 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 499 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 500 501 def _IMM_si : SMRD_Real_si <op, immPs> { 502 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, CPol:$cpol); 503 } 504 505 def _SGPR_si : SMRD_Real_si <op, sgprPs> { 506 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol); 507 } 508 509} 510 511defm S_LOAD_DWORD : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">; 512defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">; 513defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">; 514defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">; 515defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">; 516defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">; 517defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09, "S_BUFFER_LOAD_DWORDX2">; 518defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a, "S_BUFFER_LOAD_DWORDX4">; 519defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b, "S_BUFFER_LOAD_DWORDX8">; 520defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c, "S_BUFFER_LOAD_DWORDX16">; 521 522def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>; 523def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>; 524 525 526//===----------------------------------------------------------------------===// 527// VI and GFX9. 528//===----------------------------------------------------------------------===// 529 530class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> 531 : SM_Real<ps> 532 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> 533 , Enc64 { 534 field bit IsGFX9SpecificEncoding = false; 535 let AssemblerPredicate = !if(IsGFX9SpecificEncoding, isGFX9Only, isGFX8GFX9); 536 let DecoderNamespace = "GFX8"; 537 538 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 539 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 540 541 // Note that for GFX9 instructions with immediate offsets, soffset_en 542 // must be defined, whereas in GFX8 it's undefined in all cases, 543 // meaning GFX9 is not perfectly backward-compatible with GFX8, despite 544 // documentation suggesting otherwise. 545 field bit SOffsetEn = !if(IsGFX9SpecificEncoding, 546 !if(ps.has_offset, ps.has_soffset, !if(ps.has_soffset, 0, ?)), 547 ?); 548 let Inst{14} = SOffsetEn; 549 550 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 551 552 // imm 553 // TODO: Shall not be defined if the instruction has no offset nor 554 // soffset. 555 let Inst{17} = ps.has_offset; 556 557 let Inst{25-18} = op; 558 let Inst{31-26} = 0x30; //encoding 559 560 // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed. 561 // Offset value is corrected accordingly when offset is encoded/decoded. 562 // TODO: Forbid non-M0 register offsets for GFX8 stores and atomics. 563 field bits<21> Offset; 564 let Offset{6-0} = !if(ps.has_offset, offset{6-0}, 565 !if(ps.has_soffset, soffset{6-0}, ?)); 566 let Offset{20-7} = !if(ps.has_offset, offset{20-7}, ?); 567 let Inst{52-32} = Offset; 568 569 // soffset 570 let Inst{63-57} = !if(!and(IsGFX9SpecificEncoding, ps.has_soffset), 571 soffset{6-0}, ?); 572} 573 574class SMEM_Real_Load_vi<bits<8> op, string ps, OffsetMode offsets> 575 : SMEM_Real_vi<op, !cast<SM_Pseudo>(ps # offsets.Variant)> { 576 RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps # offsets.Variant).BaseClass; 577 let InOperandList = !con((ins BaseClass:$sbase), offsets.Ins, (ins CPol:$cpol)); 578} 579 580// The alternative GFX9 SGPR encoding using soffset to encode the 581// offset register. Not available in assembler and goes to the GFX9 582// encoding family to avoid conflicts with the primary SGPR variant. 583class SMEM_Real_SGPR_alt_gfx9 { 584 bit IsGFX9SpecificEncoding = true; 585 bit SOffsetEn = 1; 586 bit Offset = ?; 587 int Subtarget = SIEncodingFamily.GFX9; 588 string AsmVariantName = "NonParsable"; 589} 590 591multiclass SM_Real_Loads_vi<bits<8> op, string ps> { 592 def _IMM_vi : SMEM_Real_Load_vi <op, ps, IMM_Offset>; 593 def _SGPR_vi : SMEM_Real_Load_vi <op, ps, SGPR_Offset>; 594 def _SGPR_alt_gfx9 : SMEM_Real_Load_vi <op, ps, SGPR_Offset>, 595 SMEM_Real_SGPR_alt_gfx9; 596 let IsGFX9SpecificEncoding = true in 597 def _SGPR_IMM_gfx9 : SMEM_Real_Load_vi <op, ps, SGPR_IMM_Offset>; 598} 599 600class SMEM_Real_Store_Base_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> { 601 // encoding 602 bits<7> sdata; 603 604 let sdst = ?; 605 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 606} 607 608class SMEM_Real_Store_vi <bits<8> op, string ps, dag offsets> 609 : SMEM_Real_Store_Base_vi <op, !cast<SM_Pseudo>(ps)> { 610 RegisterClass SrcClass = !cast<SM_Store_Pseudo>(ps).SrcClass; 611 RegisterClass BaseClass = !cast<SM_Store_Pseudo>(ps).BaseClass; 612 let InOperandList = !con((ins SrcClass:$sdata, BaseClass:$sbase), 613 offsets, (ins CPol:$cpol)); 614} 615 616multiclass SM_Real_Stores_vi<bits<8> op, string ps> { 617 def _IMM_vi : SMEM_Real_Store_vi <op, ps#_IMM, (ins smem_offset:$offset)>; 618 def _SGPR_vi : SMEM_Real_Store_vi <op, ps#_SGPR, (ins SReg_32:$soffset)>; 619 def _SGPR_alt_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR", 620 (ins SReg_32:$soffset)>, 621 SMEM_Real_SGPR_alt_gfx9; 622 let IsGFX9SpecificEncoding = true in 623 def _SGPR_IMM_gfx9 : SMEM_Real_Store_vi < 624 op, ps#"_SGPR_IMM", (ins SReg_32:$soffset, smem_offset_mod:$offset)>; 625} 626 627multiclass SM_Real_Probe_vi<bits<8> op, string ps> { 628 def _IMM_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 629 def _SGPR_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 630 def _SGPR_alt_gfx9 631 : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>, 632 SMEM_Real_SGPR_alt_gfx9; 633 let IsGFX9SpecificEncoding = true in 634 def _SGPR_IMM_gfx9 635 : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>; 636} 637 638defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">; 639defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">; 640defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">; 641defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">; 642defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">; 643defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">; 644defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">; 645defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">; 646defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">; 647defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">; 648 649defm S_STORE_DWORD : SM_Real_Stores_vi <0x10, "S_STORE_DWORD">; 650defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11, "S_STORE_DWORDX2">; 651defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12, "S_STORE_DWORDX4">; 652 653defm S_BUFFER_STORE_DWORD : SM_Real_Stores_vi <0x18, "S_BUFFER_STORE_DWORD">; 654defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_vi <0x19, "S_BUFFER_STORE_DWORDX2">; 655defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_vi <0x1a, "S_BUFFER_STORE_DWORDX4">; 656 657// These instructions use same encoding 658def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>; 659def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>; 660def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>; 661def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>; 662def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>; 663def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>; 664 665defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_vi <0x05, "S_SCRATCH_LOAD_DWORD">; 666defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_vi <0x06, "S_SCRATCH_LOAD_DWORDX2">; 667defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_vi <0x07, "S_SCRATCH_LOAD_DWORDX4">; 668 669defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_vi <0x15, "S_SCRATCH_STORE_DWORD">; 670defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16, "S_SCRATCH_STORE_DWORDX2">; 671defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17, "S_SCRATCH_STORE_DWORDX4">; 672 673defm S_ATC_PROBE : SM_Real_Probe_vi <0x26, "S_ATC_PROBE">; 674defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27, "S_ATC_PROBE_BUFFER">; 675 676//===----------------------------------------------------------------------===// 677// GFX9 678//===----------------------------------------------------------------------===// 679 680class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps> 681 : SMEM_Real_vi <op, ps>, 682 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 683 684 bits<7> sdata; 685 686 let Constraints = ps.Constraints; 687 let DisableEncoding = ps.DisableEncoding; 688 689 let cpol{CPolBit.GLC} = ps.glc; 690 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 691} 692 693multiclass SM_Real_Atomics_vi<bits<8> op, string ps> { 694 def _IMM_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 695 def _SGPR_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 696 def _SGPR_alt_gfx9 697 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>, 698 SMEM_Real_SGPR_alt_gfx9; 699 let IsGFX9SpecificEncoding = true in 700 def _SGPR_IMM_gfx9 701 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>; 702 def _IMM_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 703 def _SGPR_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 704 def _SGPR_RTN_alt_gfx9 705 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>, 706 SMEM_Real_SGPR_alt_gfx9; 707 let IsGFX9SpecificEncoding = true in 708 def _SGPR_IMM_RTN_gfx9 709 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>; 710} 711 712defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_vi <0x40, "S_BUFFER_ATOMIC_SWAP">; 713defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 714defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_vi <0x42, "S_BUFFER_ATOMIC_ADD">; 715defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_vi <0x43, "S_BUFFER_ATOMIC_SUB">; 716defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_vi <0x44, "S_BUFFER_ATOMIC_SMIN">; 717defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_vi <0x45, "S_BUFFER_ATOMIC_UMIN">; 718defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_vi <0x46, "S_BUFFER_ATOMIC_SMAX">; 719defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_vi <0x47, "S_BUFFER_ATOMIC_UMAX">; 720defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_vi <0x48, "S_BUFFER_ATOMIC_AND">; 721defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_vi <0x49, "S_BUFFER_ATOMIC_OR">; 722defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_vi <0x4a, "S_BUFFER_ATOMIC_XOR">; 723defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_vi <0x4b, "S_BUFFER_ATOMIC_INC">; 724defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_vi <0x4c, "S_BUFFER_ATOMIC_DEC">; 725 726defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 727defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 728defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 729defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 730defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 731defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 732defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 733defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 734defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0x68, "S_BUFFER_ATOMIC_AND_X2">; 735defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0x69, "S_BUFFER_ATOMIC_OR_X2">; 736defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 737defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 738defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 739 740defm S_ATOMIC_SWAP : SM_Real_Atomics_vi <0x80, "S_ATOMIC_SWAP">; 741defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x81, "S_ATOMIC_CMPSWAP">; 742defm S_ATOMIC_ADD : SM_Real_Atomics_vi <0x82, "S_ATOMIC_ADD">; 743defm S_ATOMIC_SUB : SM_Real_Atomics_vi <0x83, "S_ATOMIC_SUB">; 744defm S_ATOMIC_SMIN : SM_Real_Atomics_vi <0x84, "S_ATOMIC_SMIN">; 745defm S_ATOMIC_UMIN : SM_Real_Atomics_vi <0x85, "S_ATOMIC_UMIN">; 746defm S_ATOMIC_SMAX : SM_Real_Atomics_vi <0x86, "S_ATOMIC_SMAX">; 747defm S_ATOMIC_UMAX : SM_Real_Atomics_vi <0x87, "S_ATOMIC_UMAX">; 748defm S_ATOMIC_AND : SM_Real_Atomics_vi <0x88, "S_ATOMIC_AND">; 749defm S_ATOMIC_OR : SM_Real_Atomics_vi <0x89, "S_ATOMIC_OR">; 750defm S_ATOMIC_XOR : SM_Real_Atomics_vi <0x8a, "S_ATOMIC_XOR">; 751defm S_ATOMIC_INC : SM_Real_Atomics_vi <0x8b, "S_ATOMIC_INC">; 752defm S_ATOMIC_DEC : SM_Real_Atomics_vi <0x8c, "S_ATOMIC_DEC">; 753 754defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0xa0, "S_ATOMIC_SWAP_X2">; 755defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0xa1, "S_ATOMIC_CMPSWAP_X2">; 756defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0xa2, "S_ATOMIC_ADD_X2">; 757defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0xa3, "S_ATOMIC_SUB_X2">; 758defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0xa4, "S_ATOMIC_SMIN_X2">; 759defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0xa5, "S_ATOMIC_UMIN_X2">; 760defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0xa6, "S_ATOMIC_SMAX_X2">; 761defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0xa7, "S_ATOMIC_UMAX_X2">; 762defm S_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0xa8, "S_ATOMIC_AND_X2">; 763defm S_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0xa9, "S_ATOMIC_OR_X2">; 764defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0xaa, "S_ATOMIC_XOR_X2">; 765defm S_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0xab, "S_ATOMIC_INC_X2">; 766defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0xac, "S_ATOMIC_DEC_X2">; 767 768multiclass SM_Real_Discard_vi<bits<8> op, string ps> { 769 def _IMM_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>; 770 def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>; 771 def _SGPR_alt_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>, 772 SMEM_Real_SGPR_alt_gfx9; 773 let IsGFX9SpecificEncoding = true in 774 def _SGPR_IMM_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR_IMM)>; 775} 776 777defm S_DCACHE_DISCARD : SM_Real_Discard_vi <0x28, "S_DCACHE_DISCARD">; 778defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29, "S_DCACHE_DISCARD_X2">; 779 780//===----------------------------------------------------------------------===// 781// CI 782//===----------------------------------------------------------------------===// 783 784def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset", 785 NamedMatchClass<"SMRDLiteralOffset">> { 786 let OperandType = "OPERAND_IMMEDIATE"; 787} 788 789class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : 790 SM_Real<ps>, 791 Enc64 { 792 793 let AssemblerPredicate = isGFX7Only; 794 let DecoderNamespace = "GFX7"; 795 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, CPol:$cpol); 796 797 let Inst{7-0} = 0xff; 798 let Inst{8} = 0; 799 let Inst{14-9} = sbase{6-1}; 800 let Inst{21-15} = sdst{6-0}; 801 let Inst{26-22} = op; 802 let Inst{31-27} = 0x18; //encoding 803 let Inst{63-32} = offset{31-0}; 804} 805 806def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>; 807def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>; 808def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>; 809def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>; 810def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>; 811def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>; 812def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>; 813def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>; 814def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>; 815def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>; 816 817class SMRD_Real_ci <bits<5> op, SM_Pseudo ps> 818 : SM_Real<ps> 819 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 820 , Enc32 { 821 822 let AssemblerPredicate = isGFX7Only; 823 let DecoderNamespace = "GFX7"; 824 825 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?)); 826 let Inst{8} = ps.has_offset; 827 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 828 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 829 let Inst{26-22} = op; 830 let Inst{31-27} = 0x18; //encoding 831} 832 833def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>; 834 835//===----------------------------------------------------------------------===// 836// Scalar Memory Patterns 837//===----------------------------------------------------------------------===// 838 839def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]> { 840 let GISelPredicateCode = [{ 841 if (!MI.hasOneMemOperand()) 842 return false; 843 if (!isInstrUniform(MI)) 844 return false; 845 846 // FIXME: We should probably be caching this. 847 SmallVector<GEPInfo, 4> AddrInfo; 848 getAddrModeInfo(MI, MRI, AddrInfo); 849 850 if (hasVgprParts(AddrInfo)) 851 return false; 852 return true; 853 }]; 854} 855 856def SMRDImm : ComplexPattern<iPTR, 2, "SelectSMRDImm">; 857def SMRDImm32 : ComplexPattern<iPTR, 2, "SelectSMRDImm32">; 858def SMRDSgpr : ComplexPattern<iPTR, 2, "SelectSMRDSgpr">; 859def SMRDSgprImm : ComplexPattern<iPTR, 3, "SelectSMRDSgprImm">; 860def SMRDBufferImm : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">; 861def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">; 862 863multiclass SMRD_Pattern <string Instr, ValueType vt> { 864 865 // 1. IMM offset 866 def : GCNPat < 867 (smrd_load (SMRDImm i64:$sbase, i32:$offset)), 868 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0)) 869 >; 870 871 // 2. 32-bit IMM offset on CI 872 def : GCNPat < 873 (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)), 874 (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0))> { 875 let OtherPredicates = [isGFX7Only]; 876 } 877 878 // 3. SGPR offset 879 def : GCNPat < 880 (smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)), 881 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $soffset, 0)) 882 >; 883 884 // 4. SGPR+IMM offset 885 def : GCNPat < 886 (smrd_load (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)), 887 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, $offset, 0))> { 888 let OtherPredicates = [isGFX9Plus]; 889 } 890 891 // 5. No offset 892 def : GCNPat < 893 (vt (smrd_load (i64 SReg_64:$sbase))), 894 (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0)) 895 >; 896} 897 898multiclass SMLoad_Pattern <string Instr, ValueType vt> { 899 // 1. Offset as an immediate 900 def : GCNPat < 901 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy), 902 (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> { 903 let AddedComplexity = 2; 904 } 905 906 // 2. 32-bit IMM offset on CI 907 def : GCNPat < 908 (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)), 909 (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset, 910 (extract_cpol $cachepolicy))> { 911 let OtherPredicates = [isGFX7Only]; 912 let AddedComplexity = 1; 913 } 914 915 // 3. Offset loaded in an 32bit SGPR 916 def : GCNPat < 917 (SIsbuffer_load v4i32:$sbase, i32:$offset, timm:$cachepolicy), 918 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$offset, (extract_cpol $cachepolicy))) 919 >; 920} 921 922// Global and constant loads can be selected to either MUBUF or SMRD 923// instructions, but SMRD instructions are faster so we want the instruction 924// selector to prefer those. 925let AddedComplexity = 100 in { 926 927foreach vt = Reg32Types.types in { 928defm : SMRD_Pattern <"S_LOAD_DWORD", vt>; 929} 930 931foreach vt = SReg_64.RegTypes in { 932defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>; 933} 934 935foreach vt = SReg_128.RegTypes in { 936defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>; 937} 938 939foreach vt = SReg_256.RegTypes in { 940defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>; 941} 942 943foreach vt = SReg_512.RegTypes in { 944defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>; 945} 946 947defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>; 948defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>; 949defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>; 950defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>; 951defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16i32>; 952 953defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", f32>; 954defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2f32>; 955defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4f32>; 956defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8f32>; 957defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16f32>; 958} // End let AddedComplexity = 100 959 960let OtherPredicates = [HasSMemTimeInst] in { 961def : GCNPat < 962 (i64 (readcyclecounter)), 963 (S_MEMTIME) 964>; 965} // let OtherPredicates = [HasSMemTimeInst] 966 967let OtherPredicates = [HasShaderCyclesRegister] in { 968def : GCNPat < 969 (i64 (readcyclecounter)), 970 (REG_SEQUENCE SReg_64, 971 (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0, 972 (S_MOV_B32 (i32 0)), sub1)> { 973 // Prefer this to s_memtime because it has lower and more predictable latency. 974 let AddedComplexity = 1; 975} 976} // let OtherPredicates = [HasShaderCyclesRegister] 977 978//===----------------------------------------------------------------------===// 979// GFX10. 980//===----------------------------------------------------------------------===// 981 982class SMEM_Real_10Plus_common<bits<8> op, SM_Pseudo ps, string opName, 983 int subtarget, RegisterWithSubRegs sgpr_null> : 984 SM_Real<ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>, Enc64 { 985 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 986 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 987 let Inst{25-18} = op; 988 let Inst{31-26} = 0x3d; 989 // There are SMEM instructions that do not employ any of the offset 990 // fields, in which case we need them to remain undefined. 991 let Inst{52-32} = !if(ps.has_offset, offset{20-0}, !if(ps.has_soffset, 0, ?)); 992 let Inst{63-57} = !if(ps.has_soffset, soffset{6-0}, 993 !if(ps.has_offset, sgpr_null.HWEncoding{6-0}, ?)); 994} 995 996class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> 997 : SMEM_Real_10Plus_common<op, ps, ps.Mnemonic, SIEncodingFamily.GFX10, 998 SGPR_NULL_gfxpre11> { 999 let AssemblerPredicate = isGFX10Only; 1000 let DecoderNamespace = "GFX10"; 1001 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ?); 1002 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 1003} 1004 1005class SMEM_Real_Load_gfx10<bits<8> op, string ps, OffsetMode offsets> 1006 : SMEM_Real_gfx10<op, !cast<SM_Pseudo>(ps # offsets.Variant)> { 1007 RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps # offsets.Variant).BaseClass; 1008 let InOperandList = !con((ins BaseClass:$sbase), offsets.Ins, (ins CPol:$cpol)); 1009} 1010 1011multiclass SM_Real_Loads_gfx10<bits<8> op, string ps> { 1012 def _IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps, IMM_Offset>; 1013 def _SGPR_gfx10 : SMEM_Real_Load_gfx10<op, ps, SGPR_Offset>; 1014 def _SGPR_IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps, SGPR_IMM_Offset>; 1015} 1016 1017class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> { 1018 bits<7> sdata; 1019 1020 let sdst = ?; 1021 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 1022} 1023 1024multiclass SM_Real_Stores_gfx10<bits<8> op, string ps, 1025 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM), 1026 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> { 1027 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs> { 1028 let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol); 1029 } 1030 1031 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs> { 1032 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol); 1033 } 1034 1035 def _SGPR_IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Store_Pseudo>(ps#_SGPR_IMM)> { 1036 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, 1037 SReg_32:$soffset, smem_offset_mod:$offset, CPol:$cpol); 1038 } 1039} 1040 1041defm S_LOAD_DWORD : SM_Real_Loads_gfx10<0x000, "S_LOAD_DWORD">; 1042defm S_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x001, "S_LOAD_DWORDX2">; 1043defm S_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x002, "S_LOAD_DWORDX4">; 1044defm S_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x003, "S_LOAD_DWORDX8">; 1045defm S_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x004, "S_LOAD_DWORDX16">; 1046 1047let SubtargetPredicate = HasScalarFlatScratchInsts in { 1048defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_gfx10<0x005, "S_SCRATCH_LOAD_DWORD">; 1049defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x006, "S_SCRATCH_LOAD_DWORDX2">; 1050defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x007, "S_SCRATCH_LOAD_DWORDX4">; 1051} // End SubtargetPredicate = HasScalarFlatScratchInsts 1052 1053defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_gfx10<0x008, "S_BUFFER_LOAD_DWORD">; 1054defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x009, "S_BUFFER_LOAD_DWORDX2">; 1055defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x00a, "S_BUFFER_LOAD_DWORDX4">; 1056defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x00b, "S_BUFFER_LOAD_DWORDX8">; 1057defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x00c, "S_BUFFER_LOAD_DWORDX16">; 1058 1059let SubtargetPredicate = HasScalarStores in { 1060defm S_STORE_DWORD : SM_Real_Stores_gfx10<0x010, "S_STORE_DWORD">; 1061defm S_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x011, "S_STORE_DWORDX2">; 1062defm S_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x012, "S_STORE_DWORDX4">; 1063let OtherPredicates = [HasScalarFlatScratchInsts] in { 1064defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_gfx10<0x015, "S_SCRATCH_STORE_DWORD">; 1065defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016, "S_SCRATCH_STORE_DWORDX2">; 1066defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017, "S_SCRATCH_STORE_DWORDX4">; 1067} // End OtherPredicates = [HasScalarFlatScratchInsts] 1068defm S_BUFFER_STORE_DWORD : SM_Real_Stores_gfx10<0x018, "S_BUFFER_STORE_DWORD">; 1069defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x019, "S_BUFFER_STORE_DWORDX2">; 1070defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x01a, "S_BUFFER_STORE_DWORDX4">; 1071} // End SubtargetPredicate = HasScalarStores 1072 1073def S_MEMREALTIME_gfx10 : SMEM_Real_gfx10<0x025, S_MEMREALTIME>; 1074def S_MEMTIME_gfx10 : SMEM_Real_gfx10<0x024, S_MEMTIME>; 1075def S_GL1_INV_gfx10 : SMEM_Real_gfx10<0x01f, S_GL1_INV>; 1076def S_GET_WAVEID_IN_WORKGROUP_gfx10 : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>; 1077def S_DCACHE_INV_gfx10 : SMEM_Real_gfx10<0x020, S_DCACHE_INV>; 1078 1079let SubtargetPredicate = HasScalarStores in { 1080def S_DCACHE_WB_gfx10 : SMEM_Real_gfx10<0x021, S_DCACHE_WB>; 1081} // End SubtargetPredicate = HasScalarStores 1082 1083multiclass SM_Real_Probe_gfx10<bits<8> op, string ps> { 1084 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1085 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1086 def _SGPR_IMM_gfx10 1087 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>; 1088} 1089 1090defm S_ATC_PROBE : SM_Real_Probe_gfx10 <0x26, "S_ATC_PROBE">; 1091defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27, "S_ATC_PROBE_BUFFER">; 1092 1093class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps> 1094 : SMEM_Real_gfx10 <op, ps>, 1095 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 1096 1097 bits<7> sdata; 1098 1099 let Constraints = ps.Constraints; 1100 let DisableEncoding = ps.DisableEncoding; 1101 1102 let cpol{CPolBit.GLC} = ps.glc; 1103 1104 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 1105 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 1106} 1107 1108multiclass SM_Real_Atomics_gfx10<bits<8> op, string ps> { 1109 def _IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 1110 def _SGPR_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 1111 def _SGPR_IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>; 1112 def _IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 1113 def _SGPR_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 1114 def _SGPR_IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>; 1115} 1116 1117let SubtargetPredicate = HasScalarAtomics in { 1118 1119defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x40, "S_BUFFER_ATOMIC_SWAP">; 1120defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 1121defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x42, "S_BUFFER_ATOMIC_ADD">; 1122defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x43, "S_BUFFER_ATOMIC_SUB">; 1123defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x44, "S_BUFFER_ATOMIC_SMIN">; 1124defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x45, "S_BUFFER_ATOMIC_UMIN">; 1125defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x46, "S_BUFFER_ATOMIC_SMAX">; 1126defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x47, "S_BUFFER_ATOMIC_UMAX">; 1127defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x48, "S_BUFFER_ATOMIC_AND">; 1128defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x49, "S_BUFFER_ATOMIC_OR">; 1129defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x4a, "S_BUFFER_ATOMIC_XOR">; 1130defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x4b, "S_BUFFER_ATOMIC_INC">; 1131defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x4c, "S_BUFFER_ATOMIC_DEC">; 1132 1133defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 1134defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 1135defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 1136defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 1137defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 1138defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 1139defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 1140defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 1141defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0x68, "S_BUFFER_ATOMIC_AND_X2">; 1142defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0x69, "S_BUFFER_ATOMIC_OR_X2">; 1143defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 1144defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 1145defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 1146 1147defm S_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x80, "S_ATOMIC_SWAP">; 1148defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x81, "S_ATOMIC_CMPSWAP">; 1149defm S_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x82, "S_ATOMIC_ADD">; 1150defm S_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x83, "S_ATOMIC_SUB">; 1151defm S_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x84, "S_ATOMIC_SMIN">; 1152defm S_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x85, "S_ATOMIC_UMIN">; 1153defm S_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x86, "S_ATOMIC_SMAX">; 1154defm S_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x87, "S_ATOMIC_UMAX">; 1155defm S_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x88, "S_ATOMIC_AND">; 1156defm S_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x89, "S_ATOMIC_OR">; 1157defm S_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x8a, "S_ATOMIC_XOR">; 1158defm S_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x8b, "S_ATOMIC_INC">; 1159defm S_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x8c, "S_ATOMIC_DEC">; 1160 1161defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0xa0, "S_ATOMIC_SWAP_X2">; 1162defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0xa1, "S_ATOMIC_CMPSWAP_X2">; 1163defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0xa2, "S_ATOMIC_ADD_X2">; 1164defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0xa3, "S_ATOMIC_SUB_X2">; 1165defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0xa4, "S_ATOMIC_SMIN_X2">; 1166defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0xa5, "S_ATOMIC_UMIN_X2">; 1167defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0xa6, "S_ATOMIC_SMAX_X2">; 1168defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0xa7, "S_ATOMIC_UMAX_X2">; 1169defm S_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0xa8, "S_ATOMIC_AND_X2">; 1170defm S_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0xa9, "S_ATOMIC_OR_X2">; 1171defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0xaa, "S_ATOMIC_XOR_X2">; 1172defm S_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0xab, "S_ATOMIC_INC_X2">; 1173defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0xac, "S_ATOMIC_DEC_X2">; 1174 1175multiclass SM_Real_Discard_gfx10<bits<8> op, string ps> { 1176 def _IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1177 def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1178 def _SGPR_IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>; 1179} 1180 1181defm S_DCACHE_DISCARD : SM_Real_Discard_gfx10 <0x28, "S_DCACHE_DISCARD">; 1182defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29, "S_DCACHE_DISCARD_X2">; 1183 1184} // End SubtargetPredicate = HasScalarAtomics 1185 1186def SMInfoTable : GenericTable { 1187 let FilterClass = "SM_Real"; 1188 let CppTypeName = "SMInfo"; 1189 let Fields = ["Opcode", "is_buffer"]; 1190 1191 let PrimaryKey = ["Opcode"]; 1192 let PrimaryKeyName = "getSMEMOpcodeHelper"; 1193} 1194 1195//===----------------------------------------------------------------------===// 1196// GFX11. 1197//===----------------------------------------------------------------------===// 1198 1199class SMEM_Real_gfx11<bits<8> op, SM_Pseudo ps, string opName = ps.Mnemonic> : 1200 SMEM_Real_10Plus_common<op, ps, opName, SIEncodingFamily.GFX11, 1201 SGPR_NULL_gfx11plus> { 1202 let AssemblerPredicate = isGFX11Plus; 1203 let DecoderNamespace = "GFX11"; 1204 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 1205 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, 0); 1206} 1207 1208class SMEM_Real_Load_gfx11<bits<8> op, string ps, string opName, OffsetMode offsets> : 1209 SMEM_Real_gfx11<op, !cast<SM_Pseudo>(ps # offsets.Variant), opName> { 1210 RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps # offsets.Variant).BaseClass; 1211 let InOperandList = !con((ins BaseClass:$sbase), offsets.Ins, (ins CPol:$cpol)); 1212} 1213 1214multiclass SM_Real_Loads_gfx11<bits<8> op, string ps, string opName> { 1215 def _IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps, opName, IMM_Offset>; 1216 def _SGPR_gfx11 : SMEM_Real_Load_gfx11<op, ps, opName, SGPR_Offset>; 1217 def _SGPR_IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps, opName, SGPR_IMM_Offset>; 1218 def : MnemonicAlias<!cast<SM_Pseudo>(ps#"_IMM").Mnemonic, opName>, 1219 Requires<[isGFX11Plus]>; 1220} 1221 1222defm S_LOAD_B32 : SM_Real_Loads_gfx11<0x000, "S_LOAD_DWORD", "s_load_b32">; 1223defm S_LOAD_B64 : SM_Real_Loads_gfx11<0x001, "S_LOAD_DWORDX2", "s_load_b64">; 1224defm S_LOAD_B128 : SM_Real_Loads_gfx11<0x002, "S_LOAD_DWORDX4", "s_load_b128">; 1225defm S_LOAD_B256 : SM_Real_Loads_gfx11<0x003, "S_LOAD_DWORDX8", "s_load_b256">; 1226defm S_LOAD_B512 : SM_Real_Loads_gfx11<0x004, "S_LOAD_DWORDX16", "s_load_b512">; 1227 1228defm S_BUFFER_LOAD_B32 : SM_Real_Loads_gfx11<0x008, "S_BUFFER_LOAD_DWORD", "s_buffer_load_b32">; 1229defm S_BUFFER_LOAD_B64 : SM_Real_Loads_gfx11<0x009, "S_BUFFER_LOAD_DWORDX2", "s_buffer_load_b64">; 1230defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx11<0x00a, "S_BUFFER_LOAD_DWORDX4", "s_buffer_load_b128">; 1231defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx11<0x00b, "S_BUFFER_LOAD_DWORDX8", "s_buffer_load_b256">; 1232defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx11<0x00c, "S_BUFFER_LOAD_DWORDX16", "s_buffer_load_b512">; 1233 1234def S_GL1_INV_gfx11 : SMEM_Real_gfx11<0x020, S_GL1_INV>; 1235def S_DCACHE_INV_gfx11 : SMEM_Real_gfx11<0x021, S_DCACHE_INV>; 1236 1237class SMEM_Real_Store_gfx11 <bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx11<op, ps> { 1238 // encoding 1239 bits<7> sdata; 1240 1241 let sdst = ?; 1242 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 1243} 1244 1245multiclass SM_Real_Probe_gfx11<bits<8> op, string ps> { 1246 def _IMM_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 1247 def _SGPR_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 1248 def _SGPR_IMM_gfx11 1249 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>; 1250} 1251 1252defm S_ATC_PROBE : SM_Real_Probe_gfx11 <0x22, "S_ATC_PROBE">; 1253defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx11 <0x23, "S_ATC_PROBE_BUFFER">; 1254