1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def smrd_offset_8 : NamedOperandU32<"SMRDOffset8", 10 NamedMatchClass<"SMRDOffset8">> { 11 let OperandType = "OPERAND_IMMEDIATE"; 12} 13 14def smem_offset : NamedOperandU32<"SMEMOffset", 15 NamedMatchClass<"SMEMOffset">> { 16 let OperandType = "OPERAND_IMMEDIATE"; 17 let EncoderMethod = "getSMEMOffsetEncoding"; 18 let DecoderMethod = "decodeSMEMOffset"; 19} 20 21//===----------------------------------------------------------------------===// 22// Scalar Memory classes 23//===----------------------------------------------------------------------===// 24 25class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> : 26 InstSI <outs, ins, "", pattern>, 27 SIMCInstr<opName, SIEncodingFamily.NONE> { 28 let isPseudo = 1; 29 let isCodeGenOnly = 1; 30 31 let LGKM_CNT = 1; 32 let SMRD = 1; 33 let mayStore = 0; 34 let mayLoad = 1; 35 let hasSideEffects = 0; 36 let UseNamedOperandTable = 1; 37 let SchedRW = [WriteSMEM]; 38 39 string Mnemonic = opName; 40 string AsmOperands = asmOps; 41 42 bits<1> has_sbase = 1; 43 bits<1> has_sdst = 1; 44 bit has_glc = 0; 45 bit has_dlc = 0; 46 bits<1> has_offset = 1; 47 bits<1> offset_is_imm = 0; 48 bit is_buffer = 0; 49} 50 51class SM_Real <SM_Pseudo ps> 52 : InstSI<ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 53 54 let isPseudo = 0; 55 let isCodeGenOnly = 0; 56 57 Instruction Opcode = !cast<Instruction>(NAME); 58 59 // copy relevant pseudo op flags 60 let LGKM_CNT = ps.LGKM_CNT; 61 let SMRD = ps.SMRD; 62 let mayStore = ps.mayStore; 63 let mayLoad = ps.mayLoad; 64 let hasSideEffects = ps.hasSideEffects; 65 let UseNamedOperandTable = ps.UseNamedOperandTable; 66 let SchedRW = ps.SchedRW; 67 let SubtargetPredicate = ps.SubtargetPredicate; 68 let AsmMatchConverter = ps.AsmMatchConverter; 69 let IsAtomicRet = ps.IsAtomicRet; 70 let IsAtomicNoRet = ps.IsAtomicNoRet; 71 72 let TSFlags = ps.TSFlags; 73 74 bit is_buffer = ps.is_buffer; 75 76 // encoding 77 bits<7> sbase; 78 bits<7> sdst; 79 bits<32> offset; 80 bits<1> imm = !if(ps.has_offset, ps.offset_is_imm, 0); 81 bits<5> cpol; 82} 83 84class SM_Probe_Pseudo <string opName, dag ins, bit isImm> 85 : SM_Pseudo<opName, (outs), ins, " $sdata, $sbase, $offset"> { 86 let mayLoad = 0; 87 let mayStore = 0; 88 let has_glc = 0; 89 let LGKM_CNT = 0; 90 let ScalarStore = 0; 91 let hasSideEffects = 1; 92 let offset_is_imm = isImm; 93 let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR"); 94} 95 96class SM_Load_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> 97 : SM_Pseudo<opName, outs, ins, asmOps, pattern> { 98 RegisterClass BaseClass; 99 let mayLoad = 1; 100 let mayStore = 0; 101 let has_glc = 1; 102 let has_dlc = 1; 103} 104 105class SM_Store_Pseudo <string opName, dag ins, string asmOps, list<dag> pattern = []> 106 : SM_Pseudo<opName, (outs), ins, asmOps, pattern> { 107 RegisterClass BaseClass; 108 RegisterClass SrcClass; 109 let mayLoad = 0; 110 let mayStore = 1; 111 let has_glc = 1; 112 let has_dlc = 1; 113 let ScalarStore = 1; 114} 115 116class SM_Discard_Pseudo <string opName, dag ins, bit isImm> 117 : SM_Pseudo<opName, (outs), ins, " $sbase, $offset"> { 118 let mayLoad = 0; 119 let mayStore = 0; 120 let has_glc = 0; 121 let has_sdst = 0; 122 let ScalarStore = 0; 123 let hasSideEffects = 1; 124 let offset_is_imm = isImm; 125 let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR"); 126} 127 128multiclass SM_Pseudo_Loads<string opName, 129 RegisterClass baseClass, 130 RegisterClass dstClass> { 131 def _IMM : SM_Load_Pseudo <opName, 132 (outs dstClass:$sdst), 133 (ins baseClass:$sbase, i32imm:$offset, CPol:$cpol), 134 " $sdst, $sbase, $offset$cpol", []> { 135 let offset_is_imm = 1; 136 let BaseClass = baseClass; 137 let PseudoInstr = opName # "_IMM"; 138 let has_glc = 1; 139 let has_dlc = 1; 140 } 141 142 def _SGPR : SM_Load_Pseudo <opName, 143 (outs dstClass:$sdst), 144 (ins baseClass:$sbase, SReg_32:$soff, CPol:$cpol), 145 " $sdst, $sbase, $offset$cpol", []> { 146 let BaseClass = baseClass; 147 let PseudoInstr = opName # "_SGPR"; 148 let has_glc = 1; 149 let has_dlc = 1; 150 } 151} 152 153multiclass SM_Pseudo_Stores<string opName, 154 RegisterClass baseClass, 155 RegisterClass srcClass> { 156 def _IMM : SM_Store_Pseudo <opName, 157 (ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, CPol:$cpol), 158 " $sdata, $sbase, $offset$cpol", []> { 159 let offset_is_imm = 1; 160 let BaseClass = baseClass; 161 let SrcClass = srcClass; 162 let PseudoInstr = opName # "_IMM"; 163 } 164 165 def _SGPR : SM_Store_Pseudo <opName, 166 (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soff, CPol:$cpol), 167 " $sdata, $sbase, $offset$cpol", []> { 168 let BaseClass = baseClass; 169 let SrcClass = srcClass; 170 let PseudoInstr = opName # "_SGPR"; 171 } 172} 173 174multiclass SM_Pseudo_Discards<string opName> { 175 def _IMM : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smem_offset:$offset), 1>; 176 def _SGPR : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, SReg_32:$offset), 0>; 177} 178 179class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 180 opName, (outs SReg_64_XEXEC:$sdst), (ins), 181 " $sdst", [(set i64:$sdst, (node))]> { 182 let hasSideEffects = 1; 183 184 let mayStore = 0; 185 let mayLoad = 0; 186 let has_sbase = 0; 187 let has_offset = 0; 188} 189 190class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 191 opName, (outs), (ins), "", [(node)]> { 192 let hasSideEffects = 1; 193 let mayStore = 0; 194 let has_sdst = 0; 195 let has_sbase = 0; 196 let has_offset = 0; 197} 198 199multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> { 200 def _IMM : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smem_offset:$offset), 1>; 201 def _SGPR : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, SReg_32:$offset), 0>; 202} 203 204class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo< 205 opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins), 206 " $sdst", [(set i32:$sdst, (node))]> { 207 let hasSideEffects = 1; 208 let mayStore = 0; 209 let mayLoad = 1; 210 let has_sbase = 0; 211 let has_offset = 0; 212} 213 214//===----------------------------------------------------------------------===// 215// Scalar Atomic Memory Classes 216//===----------------------------------------------------------------------===// 217 218class SM_Atomic_Pseudo <string opName, 219 dag outs, dag ins, string asmOps, bit isRet> 220 : SM_Pseudo<opName, outs, ins, asmOps, []> { 221 222 bit glc = isRet; 223 224 let mayLoad = 1; 225 let mayStore = 1; 226 let has_glc = 1; 227 let has_dlc = 1; 228 229 // Should these be set? 230 let ScalarStore = 1; 231 let hasSideEffects = 1; 232 let maybeAtomic = 1; 233 234 let IsAtomicNoRet = !not(isRet); 235 let IsAtomicRet = isRet; 236 237 let AsmMatchConverter = "cvtSMEMAtomic"; 238} 239 240class SM_Pseudo_Atomic<string opName, 241 RegisterClass baseClass, 242 RegisterClass dataClass, 243 bit isImm, 244 bit isRet, 245 string opNameWithSuffix = opName # !if(isImm, 246 !if(isRet, "_IMM_RTN", "_IMM"), 247 !if(isRet, "_SGPR_RTN", "_SGPR")), 248 Operand CPolTy = !if(isRet, CPol_GLC1, CPol)> : 249 SM_Atomic_Pseudo<opName, 250 !if(isRet, (outs dataClass:$sdst), (outs)), 251 !if(isImm, 252 (ins dataClass:$sdata, baseClass:$sbase, smem_offset:$offset, CPolTy:$cpol), 253 (ins dataClass:$sdata, baseClass:$sbase, SReg_32:$offset, CPolTy:$cpol)), 254 !if(isRet, " $sdst", " $sdata") # ", $sbase, $offset$cpol", 255 isRet>, 256 AtomicNoRet <opNameWithSuffix, isRet> { 257 let offset_is_imm = isImm; 258 let PseudoInstr = opNameWithSuffix; 259 260 let Constraints = !if(isRet, "$sdst = $sdata", ""); 261 let DisableEncoding = !if(isRet, "$sdata", ""); 262} 263 264multiclass SM_Pseudo_Atomics<string opName, 265 RegisterClass baseClass, 266 RegisterClass dataClass> { 267 def _IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 0>; 268 def _SGPR : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 0>; 269 def _IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 1>; 270 def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 1>; 271} 272 273//===----------------------------------------------------------------------===// 274// Scalar Memory Instructions 275//===----------------------------------------------------------------------===// 276 277// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit 278// SMRD instructions, because the SReg_32_XM0 register class does not include M0 279// and writing to M0 from an SMRD instruction will hang the GPU. 280 281// XXX - SMEM instructions do not allow exec for data operand, but 282// does sdst for SMRD on SI/CI? 283defm S_LOAD_DWORD : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 284defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64_XEXEC>; 285defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>; 286defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>; 287defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>; 288 289let is_buffer = 1 in { 290defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads < 291 "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC 292>; 293 294// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on 295// SI/CI, bit disallowed for SMEM on VI. 296defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads < 297 "s_buffer_load_dwordx2", SReg_128, SReg_64_XEXEC 298>; 299 300defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads < 301 "s_buffer_load_dwordx4", SReg_128, SReg_128 302>; 303 304defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads < 305 "s_buffer_load_dwordx8", SReg_128, SReg_256 306>; 307 308defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads < 309 "s_buffer_load_dwordx16", SReg_128, SReg_512 310>; 311} 312 313let SubtargetPredicate = HasScalarStores in { 314defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 315defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>; 316defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>; 317 318let is_buffer = 1 in { 319defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores < 320 "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC 321>; 322 323defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores < 324 "s_buffer_store_dwordx2", SReg_128, SReg_64_XEXEC 325>; 326 327defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores < 328 "s_buffer_store_dwordx4", SReg_128, SReg_128 329>; 330} 331} // End SubtargetPredicate = HasScalarStores 332 333let SubtargetPredicate = HasSMemTimeInst in 334def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>; 335def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>; 336 337let SubtargetPredicate = isGFX7GFX8GFX9 in { 338def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>; 339} // let SubtargetPredicate = isGFX7GFX8GFX9 340 341let SubtargetPredicate = isGFX8Plus in { 342let OtherPredicates = [HasScalarStores] in { 343def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>; 344def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>; 345} // End OtherPredicates = [HasScalarStores] 346 347defm S_ATC_PROBE : SM_Pseudo_Probe <"s_atc_probe", SReg_64>; 348let is_buffer = 1 in { 349defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>; 350} 351} // SubtargetPredicate = isGFX8Plus 352 353let SubtargetPredicate = HasSMemRealTime in 354def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>; 355 356let SubtargetPredicate = isGFX10Plus in 357def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">; 358let SubtargetPredicate = HasGetWaveIdInst in 359def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>; 360 361 362let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in { 363defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <"s_scratch_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 364defm S_SCRATCH_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>; 365defm S_SCRATCH_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>; 366 367defm S_SCRATCH_STORE_DWORD : SM_Pseudo_Stores <"s_scratch_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 368defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>; 369defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>; 370} // SubtargetPredicate = HasScalarFlatScratchInsts 371 372let SubtargetPredicate = HasScalarAtomics in { 373 374let is_buffer = 1 in { 375defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>; 376defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>; 377defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>; 378defm S_BUFFER_ATOMIC_SUB : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>; 379defm S_BUFFER_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>; 380defm S_BUFFER_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>; 381defm S_BUFFER_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>; 382defm S_BUFFER_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>; 383defm S_BUFFER_ATOMIC_AND : SM_Pseudo_Atomics <"s_buffer_atomic_and", SReg_128, SReg_32_XM0_XEXEC>; 384defm S_BUFFER_ATOMIC_OR : SM_Pseudo_Atomics <"s_buffer_atomic_or", SReg_128, SReg_32_XM0_XEXEC>; 385defm S_BUFFER_ATOMIC_XOR : SM_Pseudo_Atomics <"s_buffer_atomic_xor", SReg_128, SReg_32_XM0_XEXEC>; 386defm S_BUFFER_ATOMIC_INC : SM_Pseudo_Atomics <"s_buffer_atomic_inc", SReg_128, SReg_32_XM0_XEXEC>; 387defm S_BUFFER_ATOMIC_DEC : SM_Pseudo_Atomics <"s_buffer_atomic_dec", SReg_128, SReg_32_XM0_XEXEC>; 388 389defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_swap_x2", SReg_128, SReg_64_XEXEC>; 390defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap_x2", SReg_128, SReg_128>; 391defm S_BUFFER_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_add_x2", SReg_128, SReg_64_XEXEC>; 392defm S_BUFFER_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_sub_x2", SReg_128, SReg_64_XEXEC>; 393defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smin_x2", SReg_128, SReg_64_XEXEC>; 394defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>; 395defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>; 396defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>; 397defm S_BUFFER_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>; 398defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>; 399defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>; 400defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>; 401defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>; 402} 403 404defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>; 405defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>; 406defm S_ATOMIC_ADD : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>; 407defm S_ATOMIC_SUB : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>; 408defm S_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>; 409defm S_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>; 410defm S_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>; 411defm S_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_atomic_umax", SReg_64, SReg_32_XM0_XEXEC>; 412defm S_ATOMIC_AND : SM_Pseudo_Atomics <"s_atomic_and", SReg_64, SReg_32_XM0_XEXEC>; 413defm S_ATOMIC_OR : SM_Pseudo_Atomics <"s_atomic_or", SReg_64, SReg_32_XM0_XEXEC>; 414defm S_ATOMIC_XOR : SM_Pseudo_Atomics <"s_atomic_xor", SReg_64, SReg_32_XM0_XEXEC>; 415defm S_ATOMIC_INC : SM_Pseudo_Atomics <"s_atomic_inc", SReg_64, SReg_32_XM0_XEXEC>; 416defm S_ATOMIC_DEC : SM_Pseudo_Atomics <"s_atomic_dec", SReg_64, SReg_32_XM0_XEXEC>; 417 418defm S_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_atomic_swap_x2", SReg_64, SReg_64_XEXEC>; 419defm S_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_atomic_cmpswap_x2", SReg_64, SReg_128>; 420defm S_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_atomic_add_x2", SReg_64, SReg_64_XEXEC>; 421defm S_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_atomic_sub_x2", SReg_64, SReg_64_XEXEC>; 422defm S_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_atomic_smin_x2", SReg_64, SReg_64_XEXEC>; 423defm S_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_atomic_umin_x2", SReg_64, SReg_64_XEXEC>; 424defm S_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_atomic_smax_x2", SReg_64, SReg_64_XEXEC>; 425defm S_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_atomic_umax_x2", SReg_64, SReg_64_XEXEC>; 426defm S_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_atomic_and_x2", SReg_64, SReg_64_XEXEC>; 427defm S_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_atomic_or_x2", SReg_64, SReg_64_XEXEC>; 428defm S_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_atomic_xor_x2", SReg_64, SReg_64_XEXEC>; 429defm S_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_atomic_inc_x2", SReg_64, SReg_64_XEXEC>; 430defm S_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_atomic_dec_x2", SReg_64, SReg_64_XEXEC>; 431 432} // let SubtargetPredicate = HasScalarAtomics 433 434let SubtargetPredicate = HasScalarAtomics in { 435defm S_DCACHE_DISCARD : SM_Pseudo_Discards <"s_dcache_discard">; 436defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards <"s_dcache_discard_x2">; 437} 438 439//===----------------------------------------------------------------------===// 440// Targets 441//===----------------------------------------------------------------------===// 442 443//===----------------------------------------------------------------------===// 444// SI 445//===----------------------------------------------------------------------===// 446 447class SMRD_Real_si <bits<5> op, SM_Pseudo ps> 448 : SM_Real<ps> 449 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 450 , Enc32 { 451 452 let AssemblerPredicate = isGFX6GFX7; 453 let DecoderNamespace = "GFX6GFX7"; 454 455 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?); 456 let Inst{8} = imm; 457 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 458 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 459 let Inst{26-22} = op; 460 let Inst{31-27} = 0x18; //encoding 461} 462 463// FIXME: Assembler should reject trying to use glc on SMRD 464// instructions on SI. 465multiclass SM_Real_Loads_si<bits<5> op, string ps, 466 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 467 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 468 469 def _IMM_si : SMRD_Real_si <op, immPs> { 470 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, CPol:$cpol); 471 } 472 473 // FIXME: The operand name $offset is inconsistent with $soff used 474 // in the pseudo 475 def _SGPR_si : SMRD_Real_si <op, sgprPs> { 476 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, CPol:$cpol); 477 } 478 479} 480 481defm S_LOAD_DWORD : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">; 482defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">; 483defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">; 484defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">; 485defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">; 486defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">; 487defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09, "S_BUFFER_LOAD_DWORDX2">; 488defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a, "S_BUFFER_LOAD_DWORDX4">; 489defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b, "S_BUFFER_LOAD_DWORDX8">; 490defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c, "S_BUFFER_LOAD_DWORDX16">; 491 492def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>; 493def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>; 494 495 496//===----------------------------------------------------------------------===// 497// VI 498//===----------------------------------------------------------------------===// 499 500class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> 501 : SM_Real<ps> 502 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> 503 , Enc64 { 504 let AssemblerPredicate = isGFX8GFX9; 505 let DecoderNamespace = "GFX8"; 506 507 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 508 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 509 510 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 511 let Inst{17} = imm; 512 let Inst{25-18} = op; 513 let Inst{31-26} = 0x30; //encoding 514 515 // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed. 516 // Offset value is corrected accordingly when offset is encoded/decoded. 517 let Inst{38-32} = !if(ps.has_offset, offset{6-0}, ?); 518 let Inst{52-39} = !if(ps.has_offset, !if(imm, offset{20-7}, ?), ?); 519} 520 521multiclass SM_Real_Loads_vi<bits<8> op, string ps, 522 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 523 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 524 def _IMM_vi : SMEM_Real_vi <op, immPs> { 525 let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol); 526 } 527 def _SGPR_vi : SMEM_Real_vi <op, sgprPs> { 528 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, CPol:$cpol); 529 } 530} 531 532class SMEM_Real_Store_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> { 533 // encoding 534 bits<7> sdata; 535 536 let sdst = ?; 537 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 538} 539 540multiclass SM_Real_Stores_vi<bits<8> op, string ps, 541 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM), 542 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> { 543 // FIXME: The operand name $offset is inconsistent with $soff used 544 // in the pseudo 545 def _IMM_vi : SMEM_Real_Store_vi <op, immPs> { 546 let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol); 547 } 548 549 def _SGPR_vi : SMEM_Real_Store_vi <op, sgprPs> { 550 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, CPol:$cpol); 551 } 552} 553 554multiclass SM_Real_Probe_vi<bits<8> op, string ps> { 555 def _IMM_vi : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 556 def _SGPR_vi : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 557} 558 559defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">; 560defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">; 561defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">; 562defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">; 563defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">; 564defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">; 565defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">; 566defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">; 567defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">; 568defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">; 569 570defm S_STORE_DWORD : SM_Real_Stores_vi <0x10, "S_STORE_DWORD">; 571defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11, "S_STORE_DWORDX2">; 572defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12, "S_STORE_DWORDX4">; 573 574defm S_BUFFER_STORE_DWORD : SM_Real_Stores_vi <0x18, "S_BUFFER_STORE_DWORD">; 575defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_vi <0x19, "S_BUFFER_STORE_DWORDX2">; 576defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_vi <0x1a, "S_BUFFER_STORE_DWORDX4">; 577 578// These instructions use same encoding 579def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>; 580def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>; 581def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>; 582def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>; 583def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>; 584def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>; 585 586defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_vi <0x05, "S_SCRATCH_LOAD_DWORD">; 587defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_vi <0x06, "S_SCRATCH_LOAD_DWORDX2">; 588defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_vi <0x07, "S_SCRATCH_LOAD_DWORDX4">; 589 590defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_vi <0x15, "S_SCRATCH_STORE_DWORD">; 591defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16, "S_SCRATCH_STORE_DWORDX2">; 592defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17, "S_SCRATCH_STORE_DWORDX4">; 593 594defm S_ATC_PROBE : SM_Real_Probe_vi <0x26, "S_ATC_PROBE">; 595defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27, "S_ATC_PROBE_BUFFER">; 596 597//===----------------------------------------------------------------------===// 598// GFX9 599//===----------------------------------------------------------------------===// 600 601class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps> 602 : SMEM_Real_vi <op, ps>, 603 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 604 605 bits<7> sdata; 606 607 let Constraints = ps.Constraints; 608 let DisableEncoding = ps.DisableEncoding; 609 610 let cpol{CPolBit.GLC} = ps.glc; 611 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 612} 613 614multiclass SM_Real_Atomics_vi<bits<8> op, string ps> { 615 def _IMM_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 616 def _SGPR_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 617 def _IMM_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 618 def _SGPR_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 619} 620 621defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_vi <0x40, "S_BUFFER_ATOMIC_SWAP">; 622defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 623defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_vi <0x42, "S_BUFFER_ATOMIC_ADD">; 624defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_vi <0x43, "S_BUFFER_ATOMIC_SUB">; 625defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_vi <0x44, "S_BUFFER_ATOMIC_SMIN">; 626defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_vi <0x45, "S_BUFFER_ATOMIC_UMIN">; 627defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_vi <0x46, "S_BUFFER_ATOMIC_SMAX">; 628defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_vi <0x47, "S_BUFFER_ATOMIC_UMAX">; 629defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_vi <0x48, "S_BUFFER_ATOMIC_AND">; 630defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_vi <0x49, "S_BUFFER_ATOMIC_OR">; 631defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_vi <0x4a, "S_BUFFER_ATOMIC_XOR">; 632defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_vi <0x4b, "S_BUFFER_ATOMIC_INC">; 633defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_vi <0x4c, "S_BUFFER_ATOMIC_DEC">; 634 635defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 636defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 637defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 638defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 639defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 640defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 641defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 642defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 643defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0x68, "S_BUFFER_ATOMIC_AND_X2">; 644defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0x69, "S_BUFFER_ATOMIC_OR_X2">; 645defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 646defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 647defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 648 649defm S_ATOMIC_SWAP : SM_Real_Atomics_vi <0x80, "S_ATOMIC_SWAP">; 650defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x81, "S_ATOMIC_CMPSWAP">; 651defm S_ATOMIC_ADD : SM_Real_Atomics_vi <0x82, "S_ATOMIC_ADD">; 652defm S_ATOMIC_SUB : SM_Real_Atomics_vi <0x83, "S_ATOMIC_SUB">; 653defm S_ATOMIC_SMIN : SM_Real_Atomics_vi <0x84, "S_ATOMIC_SMIN">; 654defm S_ATOMIC_UMIN : SM_Real_Atomics_vi <0x85, "S_ATOMIC_UMIN">; 655defm S_ATOMIC_SMAX : SM_Real_Atomics_vi <0x86, "S_ATOMIC_SMAX">; 656defm S_ATOMIC_UMAX : SM_Real_Atomics_vi <0x87, "S_ATOMIC_UMAX">; 657defm S_ATOMIC_AND : SM_Real_Atomics_vi <0x88, "S_ATOMIC_AND">; 658defm S_ATOMIC_OR : SM_Real_Atomics_vi <0x89, "S_ATOMIC_OR">; 659defm S_ATOMIC_XOR : SM_Real_Atomics_vi <0x8a, "S_ATOMIC_XOR">; 660defm S_ATOMIC_INC : SM_Real_Atomics_vi <0x8b, "S_ATOMIC_INC">; 661defm S_ATOMIC_DEC : SM_Real_Atomics_vi <0x8c, "S_ATOMIC_DEC">; 662 663defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0xa0, "S_ATOMIC_SWAP_X2">; 664defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0xa1, "S_ATOMIC_CMPSWAP_X2">; 665defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0xa2, "S_ATOMIC_ADD_X2">; 666defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0xa3, "S_ATOMIC_SUB_X2">; 667defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0xa4, "S_ATOMIC_SMIN_X2">; 668defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0xa5, "S_ATOMIC_UMIN_X2">; 669defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0xa6, "S_ATOMIC_SMAX_X2">; 670defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0xa7, "S_ATOMIC_UMAX_X2">; 671defm S_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0xa8, "S_ATOMIC_AND_X2">; 672defm S_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0xa9, "S_ATOMIC_OR_X2">; 673defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0xaa, "S_ATOMIC_XOR_X2">; 674defm S_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0xab, "S_ATOMIC_INC_X2">; 675defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0xac, "S_ATOMIC_DEC_X2">; 676 677multiclass SM_Real_Discard_vi<bits<8> op, string ps> { 678 def _IMM_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>; 679 def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>; 680} 681 682defm S_DCACHE_DISCARD : SM_Real_Discard_vi <0x28, "S_DCACHE_DISCARD">; 683defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29, "S_DCACHE_DISCARD_X2">; 684 685//===----------------------------------------------------------------------===// 686// CI 687//===----------------------------------------------------------------------===// 688 689def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset", 690 NamedMatchClass<"SMRDLiteralOffset">> { 691 let OperandType = "OPERAND_IMMEDIATE"; 692} 693 694class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : 695 SM_Real<ps>, 696 Enc64 { 697 698 let AssemblerPredicate = isGFX7Only; 699 let DecoderNamespace = "GFX7"; 700 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, CPol:$cpol); 701 702 let Inst{7-0} = 0xff; 703 let Inst{8} = 0; 704 let Inst{14-9} = sbase{6-1}; 705 let Inst{21-15} = sdst{6-0}; 706 let Inst{26-22} = op; 707 let Inst{31-27} = 0x18; //encoding 708 let Inst{63-32} = offset{31-0}; 709} 710 711def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>; 712def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>; 713def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>; 714def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>; 715def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>; 716def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>; 717def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>; 718def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>; 719def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>; 720def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>; 721 722class SMRD_Real_ci <bits<5> op, SM_Pseudo ps> 723 : SM_Real<ps> 724 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 725 , Enc32 { 726 727 let AssemblerPredicate = isGFX7Only; 728 let DecoderNamespace = "GFX7"; 729 730 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?); 731 let Inst{8} = imm; 732 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 733 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 734 let Inst{26-22} = op; 735 let Inst{31-27} = 0x18; //encoding 736} 737 738def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>; 739 740//===----------------------------------------------------------------------===// 741// Scalar Memory Patterns 742//===----------------------------------------------------------------------===// 743 744def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]> { 745 let GISelPredicateCode = [{ 746 if (!MI.hasOneMemOperand()) 747 return false; 748 if (!isInstrUniform(MI)) 749 return false; 750 751 // FIXME: We should probably be caching this. 752 SmallVector<GEPInfo, 4> AddrInfo; 753 getAddrModeInfo(MI, MRI, AddrInfo); 754 755 if (hasVgprParts(AddrInfo)) 756 return false; 757 return true; 758 }]; 759} 760 761def SMRDImm : ComplexPattern<iPTR, 2, "SelectSMRDImm">; 762def SMRDImm32 : ComplexPattern<iPTR, 2, "SelectSMRDImm32">; 763def SMRDSgpr : ComplexPattern<iPTR, 2, "SelectSMRDSgpr">; 764def SMRDBufferImm : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">; 765def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">; 766 767multiclass SMRD_Pattern <string Instr, ValueType vt> { 768 769 // 1. IMM offset 770 def : GCNPat < 771 (smrd_load (SMRDImm i64:$sbase, i32:$offset)), 772 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0)) 773 >; 774 775 // 2. 32-bit IMM offset on CI 776 def : GCNPat < 777 (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)), 778 (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0))> { 779 let OtherPredicates = [isGFX7Only]; 780 } 781 782 // 3. SGPR offset 783 def : GCNPat < 784 (smrd_load (SMRDSgpr i64:$sbase, i32:$offset)), 785 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, 0)) 786 >; 787 788 // 4. No offset 789 def : GCNPat < 790 (vt (smrd_load (i64 SReg_64:$sbase))), 791 (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0)) 792 >; 793} 794 795multiclass SMLoad_Pattern <string Instr, ValueType vt> { 796 // 1. Offset as an immediate 797 def : GCNPat < 798 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy), 799 (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> { 800 let AddedComplexity = 2; 801 } 802 803 // 2. 32-bit IMM offset on CI 804 def : GCNPat < 805 (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)), 806 (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset, 807 (extract_cpol $cachepolicy))> { 808 let OtherPredicates = [isGFX7Only]; 809 let AddedComplexity = 1; 810 } 811 812 // 3. Offset loaded in an 32bit SGPR 813 def : GCNPat < 814 (SIsbuffer_load v4i32:$sbase, i32:$offset, timm:$cachepolicy), 815 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$offset, (extract_cpol $cachepolicy))) 816 >; 817} 818 819// Global and constant loads can be selected to either MUBUF or SMRD 820// instructions, but SMRD instructions are faster so we want the instruction 821// selector to prefer those. 822let AddedComplexity = 100 in { 823 824foreach vt = Reg32Types.types in { 825defm : SMRD_Pattern <"S_LOAD_DWORD", vt>; 826} 827 828foreach vt = SReg_64.RegTypes in { 829defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>; 830} 831 832foreach vt = SReg_128.RegTypes in { 833defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>; 834} 835 836foreach vt = SReg_256.RegTypes in { 837defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>; 838} 839 840foreach vt = SReg_512.RegTypes in { 841defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>; 842} 843 844defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>; 845defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>; 846defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>; 847defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>; 848defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16i32>; 849 850defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", f32>; 851defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2f32>; 852defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4f32>; 853defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8f32>; 854defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16f32>; 855} // End let AddedComplexity = 100 856 857let OtherPredicates = [HasSMemTimeInst] in { 858def : GCNPat < 859 (i64 (readcyclecounter)), 860 (S_MEMTIME) 861>; 862} // let OtherPredicates = [HasSMemTimeInst] 863 864let OtherPredicates = [HasShaderCyclesRegister] in { 865def : GCNPat < 866 (i64 (readcyclecounter)), 867 (REG_SEQUENCE SReg_64, 868 (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0, 869 (S_MOV_B32 (i32 0)), sub1)> { 870 // Prefer this to s_memtime because it has lower and more predictable latency. 871 let AddedComplexity = 1; 872} 873} // let OtherPredicates = [HasShaderCyclesRegister] 874 875//===----------------------------------------------------------------------===// 876// GFX10. 877//===----------------------------------------------------------------------===// 878 879class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> : 880 SM_Real<ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10>, Enc64 { 881 let AssemblerPredicate = isGFX10Plus; 882 let DecoderNamespace = "GFX10"; 883 884 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 885 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 886 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ?); 887 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 888 let Inst{25-18} = op; 889 let Inst{31-26} = 0x3d; 890 let Inst{52-32} = !if(ps.offset_is_imm, !if(ps.has_offset, offset{20-0}, ?), ?); 891 let Inst{63-57} = !if(ps.offset_is_imm, !cast<int>(SGPR_NULL.HWEncoding), 892 !if(ps.has_offset, offset{6-0}, ?)); 893} 894 895multiclass SM_Real_Loads_gfx10<bits<8> op, string ps, 896 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 897 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 898 def _IMM_gfx10 : SMEM_Real_gfx10<op, immPs> { 899 let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol); 900 } 901 def _SGPR_gfx10 : SMEM_Real_gfx10<op, sgprPs> { 902 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, CPol:$cpol); 903 } 904} 905 906class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> { 907 bits<7> sdata; 908 909 let sdst = ?; 910 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 911} 912 913multiclass SM_Real_Stores_gfx10<bits<8> op, string ps, 914 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM), 915 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> { 916 // FIXME: The operand name $offset is inconsistent with $soff used 917 // in the pseudo 918 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs> { 919 let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol); 920 } 921 922 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs> { 923 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, CPol:$cpol); 924 } 925} 926 927defm S_LOAD_DWORD : SM_Real_Loads_gfx10<0x000, "S_LOAD_DWORD">; 928defm S_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x001, "S_LOAD_DWORDX2">; 929defm S_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x002, "S_LOAD_DWORDX4">; 930defm S_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x003, "S_LOAD_DWORDX8">; 931defm S_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x004, "S_LOAD_DWORDX16">; 932 933let SubtargetPredicate = HasScalarFlatScratchInsts in { 934defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_gfx10<0x005, "S_SCRATCH_LOAD_DWORD">; 935defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x006, "S_SCRATCH_LOAD_DWORDX2">; 936defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x007, "S_SCRATCH_LOAD_DWORDX4">; 937} // End SubtargetPredicate = HasScalarFlatScratchInsts 938 939defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_gfx10<0x008, "S_BUFFER_LOAD_DWORD">; 940defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x009, "S_BUFFER_LOAD_DWORDX2">; 941defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x00a, "S_BUFFER_LOAD_DWORDX4">; 942defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x00b, "S_BUFFER_LOAD_DWORDX8">; 943defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x00c, "S_BUFFER_LOAD_DWORDX16">; 944 945let SubtargetPredicate = HasScalarStores in { 946defm S_STORE_DWORD : SM_Real_Stores_gfx10<0x010, "S_STORE_DWORD">; 947defm S_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x011, "S_STORE_DWORDX2">; 948defm S_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x012, "S_STORE_DWORDX4">; 949let OtherPredicates = [HasScalarFlatScratchInsts] in { 950defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_gfx10<0x015, "S_SCRATCH_STORE_DWORD">; 951defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016, "S_SCRATCH_STORE_DWORDX2">; 952defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017, "S_SCRATCH_STORE_DWORDX4">; 953} // End OtherPredicates = [HasScalarFlatScratchInsts] 954defm S_BUFFER_STORE_DWORD : SM_Real_Stores_gfx10<0x018, "S_BUFFER_STORE_DWORD">; 955defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x019, "S_BUFFER_STORE_DWORDX2">; 956defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x01a, "S_BUFFER_STORE_DWORDX4">; 957} // End SubtargetPredicate = HasScalarStores 958 959def S_MEMREALTIME_gfx10 : SMEM_Real_gfx10<0x025, S_MEMREALTIME>; 960def S_MEMTIME_gfx10 : SMEM_Real_gfx10<0x024, S_MEMTIME>; 961def S_GL1_INV_gfx10 : SMEM_Real_gfx10<0x01f, S_GL1_INV>; 962def S_GET_WAVEID_IN_WORKGROUP_gfx10 : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>; 963def S_DCACHE_INV_gfx10 : SMEM_Real_gfx10<0x020, S_DCACHE_INV>; 964 965let SubtargetPredicate = HasScalarStores in { 966def S_DCACHE_WB_gfx10 : SMEM_Real_gfx10<0x021, S_DCACHE_WB>; 967} // End SubtargetPredicate = HasScalarStores 968 969multiclass SM_Real_Probe_gfx10<bits<8> op, string ps> { 970 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 971 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 972} 973 974defm S_ATC_PROBE : SM_Real_Probe_gfx10 <0x26, "S_ATC_PROBE">; 975defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27, "S_ATC_PROBE_BUFFER">; 976 977class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps> 978 : SMEM_Real_gfx10 <op, ps>, 979 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 980 981 bits<7> sdata; 982 983 let Constraints = ps.Constraints; 984 let DisableEncoding = ps.DisableEncoding; 985 986 let cpol{CPolBit.GLC} = ps.glc; 987 988 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 989 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 990} 991 992multiclass SM_Real_Atomics_gfx10<bits<8> op, string ps> { 993 def _IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 994 def _SGPR_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 995 def _IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 996 def _SGPR_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 997} 998 999let SubtargetPredicate = HasScalarAtomics in { 1000 1001defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x40, "S_BUFFER_ATOMIC_SWAP">; 1002defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 1003defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x42, "S_BUFFER_ATOMIC_ADD">; 1004defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x43, "S_BUFFER_ATOMIC_SUB">; 1005defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x44, "S_BUFFER_ATOMIC_SMIN">; 1006defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x45, "S_BUFFER_ATOMIC_UMIN">; 1007defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x46, "S_BUFFER_ATOMIC_SMAX">; 1008defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x47, "S_BUFFER_ATOMIC_UMAX">; 1009defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x48, "S_BUFFER_ATOMIC_AND">; 1010defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x49, "S_BUFFER_ATOMIC_OR">; 1011defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x4a, "S_BUFFER_ATOMIC_XOR">; 1012defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x4b, "S_BUFFER_ATOMIC_INC">; 1013defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x4c, "S_BUFFER_ATOMIC_DEC">; 1014 1015defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 1016defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 1017defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 1018defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 1019defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 1020defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 1021defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 1022defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 1023defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0x68, "S_BUFFER_ATOMIC_AND_X2">; 1024defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0x69, "S_BUFFER_ATOMIC_OR_X2">; 1025defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 1026defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 1027defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 1028 1029defm S_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x80, "S_ATOMIC_SWAP">; 1030defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x81, "S_ATOMIC_CMPSWAP">; 1031defm S_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x82, "S_ATOMIC_ADD">; 1032defm S_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x83, "S_ATOMIC_SUB">; 1033defm S_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x84, "S_ATOMIC_SMIN">; 1034defm S_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x85, "S_ATOMIC_UMIN">; 1035defm S_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x86, "S_ATOMIC_SMAX">; 1036defm S_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x87, "S_ATOMIC_UMAX">; 1037defm S_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x88, "S_ATOMIC_AND">; 1038defm S_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x89, "S_ATOMIC_OR">; 1039defm S_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x8a, "S_ATOMIC_XOR">; 1040defm S_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x8b, "S_ATOMIC_INC">; 1041defm S_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x8c, "S_ATOMIC_DEC">; 1042 1043defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0xa0, "S_ATOMIC_SWAP_X2">; 1044defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0xa1, "S_ATOMIC_CMPSWAP_X2">; 1045defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0xa2, "S_ATOMIC_ADD_X2">; 1046defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0xa3, "S_ATOMIC_SUB_X2">; 1047defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0xa4, "S_ATOMIC_SMIN_X2">; 1048defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0xa5, "S_ATOMIC_UMIN_X2">; 1049defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0xa6, "S_ATOMIC_SMAX_X2">; 1050defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0xa7, "S_ATOMIC_UMAX_X2">; 1051defm S_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0xa8, "S_ATOMIC_AND_X2">; 1052defm S_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0xa9, "S_ATOMIC_OR_X2">; 1053defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0xaa, "S_ATOMIC_XOR_X2">; 1054defm S_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0xab, "S_ATOMIC_INC_X2">; 1055defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0xac, "S_ATOMIC_DEC_X2">; 1056 1057multiclass SM_Real_Discard_gfx10<bits<8> op, string ps> { 1058 def _IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1059 def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1060} 1061 1062defm S_DCACHE_DISCARD : SM_Real_Discard_gfx10 <0x28, "S_DCACHE_DISCARD">; 1063defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29, "S_DCACHE_DISCARD_X2">; 1064 1065} // End SubtargetPredicate = HasScalarAtomics 1066 1067def SMInfoTable : GenericTable { 1068 let FilterClass = "SM_Real"; 1069 let CppTypeName = "SMInfo"; 1070 let Fields = ["Opcode", "is_buffer"]; 1071 1072 let PrimaryKey = ["Opcode"]; 1073 let PrimaryKeyName = "getSMEMOpcodeHelper"; 1074} 1075