1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def smrd_offset_8 : NamedOperandU32<"SMRDOffset8", 10 NamedMatchClass<"SMRDOffset8">> { 11 let OperandType = "OPERAND_IMMEDIATE"; 12} 13 14def smem_offset : NamedOperandU32<"SMEMOffset", 15 NamedMatchClass<"SMEMOffset">> { 16 let OperandType = "OPERAND_IMMEDIATE"; 17 let EncoderMethod = "getSMEMOffsetEncoding"; 18 let DecoderMethod = "decodeSMEMOffset"; 19} 20 21//===----------------------------------------------------------------------===// 22// Scalar Memory classes 23//===----------------------------------------------------------------------===// 24 25class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> : 26 InstSI <outs, ins, "", pattern>, 27 SIMCInstr<opName, SIEncodingFamily.NONE> { 28 let isPseudo = 1; 29 let isCodeGenOnly = 1; 30 31 let LGKM_CNT = 1; 32 let SMRD = 1; 33 let mayStore = 0; 34 let mayLoad = 1; 35 let hasSideEffects = 0; 36 let UseNamedOperandTable = 1; 37 let SchedRW = [WriteSMEM]; 38 39 string Mnemonic = opName; 40 string AsmOperands = asmOps; 41 42 bits<1> has_sbase = 1; 43 bits<1> has_sdst = 1; 44 bit has_glc = 0; 45 bit has_dlc = 0; 46 bits<1> has_offset = 1; 47 bits<1> offset_is_imm = 0; 48 bit is_buffer = 0; 49} 50 51class SM_Real <SM_Pseudo ps> 52 : InstSI<ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 53 54 let isPseudo = 0; 55 let isCodeGenOnly = 0; 56 57 Instruction Opcode = !cast<Instruction>(NAME); 58 59 // copy relevant pseudo op flags 60 let SubtargetPredicate = ps.SubtargetPredicate; 61 let AsmMatchConverter = ps.AsmMatchConverter; 62 let UseNamedOperandTable = ps.UseNamedOperandTable; 63 let SMRD = ps.SMRD; 64 65 bit is_buffer = ps.is_buffer; 66 67 // encoding 68 bits<7> sbase; 69 bits<7> sdst; 70 bits<32> offset; 71 bits<1> imm = !if(ps.has_offset, ps.offset_is_imm, 0); 72} 73 74class SM_Probe_Pseudo <string opName, dag ins, bit isImm> 75 : SM_Pseudo<opName, (outs), ins, " $sdata, $sbase, $offset"> { 76 let mayLoad = 0; 77 let mayStore = 0; 78 let has_glc = 0; 79 let LGKM_CNT = 0; 80 let ScalarStore = 0; 81 let hasSideEffects = 1; 82 let offset_is_imm = isImm; 83 let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR"); 84} 85 86class SM_Load_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> 87 : SM_Pseudo<opName, outs, ins, asmOps, pattern> { 88 RegisterClass BaseClass; 89 let mayLoad = 1; 90 let mayStore = 0; 91 let has_glc = 1; 92 let has_dlc = 1; 93} 94 95class SM_Store_Pseudo <string opName, dag ins, string asmOps, list<dag> pattern = []> 96 : SM_Pseudo<opName, (outs), ins, asmOps, pattern> { 97 RegisterClass BaseClass; 98 RegisterClass SrcClass; 99 let mayLoad = 0; 100 let mayStore = 1; 101 let has_glc = 1; 102 let has_dlc = 1; 103 let ScalarStore = 1; 104} 105 106class SM_Discard_Pseudo <string opName, dag ins, bit isImm> 107 : SM_Pseudo<opName, (outs), ins, " $sbase, $offset"> { 108 let mayLoad = 0; 109 let mayStore = 0; 110 let has_glc = 0; 111 let has_sdst = 0; 112 let ScalarStore = 0; 113 let hasSideEffects = 1; 114 let offset_is_imm = isImm; 115 let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR"); 116} 117 118multiclass SM_Pseudo_Loads<string opName, 119 RegisterClass baseClass, 120 RegisterClass dstClass> { 121 def _IMM : SM_Load_Pseudo <opName, 122 (outs dstClass:$sdst), 123 (ins baseClass:$sbase, i32imm:$offset, i1imm:$glc, i1imm:$dlc), 124 " $sdst, $sbase, $offset$glc$dlc", []> { 125 let offset_is_imm = 1; 126 let BaseClass = baseClass; 127 let PseudoInstr = opName # "_IMM"; 128 let has_glc = 1; 129 let has_dlc = 1; 130 } 131 132 def _SGPR : SM_Load_Pseudo <opName, 133 (outs dstClass:$sdst), 134 (ins baseClass:$sbase, SReg_32:$soff, i1imm:$glc, i1imm:$dlc), 135 " $sdst, $sbase, $offset$glc$dlc", []> { 136 let BaseClass = baseClass; 137 let PseudoInstr = opName # "_SGPR"; 138 let has_glc = 1; 139 let has_dlc = 1; 140 } 141} 142 143multiclass SM_Pseudo_Stores<string opName, 144 RegisterClass baseClass, 145 RegisterClass srcClass> { 146 def _IMM : SM_Store_Pseudo <opName, 147 (ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, i1imm:$glc, i1imm:$dlc), 148 " $sdata, $sbase, $offset$glc$dlc", []> { 149 let offset_is_imm = 1; 150 let BaseClass = baseClass; 151 let SrcClass = srcClass; 152 let PseudoInstr = opName # "_IMM"; 153 } 154 155 def _SGPR : SM_Store_Pseudo <opName, 156 (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soff, i1imm:$glc, i1imm:$dlc), 157 " $sdata, $sbase, $offset$glc$dlc", []> { 158 let BaseClass = baseClass; 159 let SrcClass = srcClass; 160 let PseudoInstr = opName # "_SGPR"; 161 } 162} 163 164multiclass SM_Pseudo_Discards<string opName> { 165 def _IMM : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smem_offset:$offset), 1>; 166 def _SGPR : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, SReg_32:$offset), 0>; 167} 168 169class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 170 opName, (outs SReg_64_XEXEC:$sdst), (ins), 171 " $sdst", [(set i64:$sdst, (node))]> { 172 let hasSideEffects = 1; 173 174 // FIXME: This should be definitively mayStore = 0. TableGen 175 // brokenly tries to infer these based on the intrinsic properties 176 // corresponding to the IR attributes. The target intrinsics are 177 // considered as writing to memory for IR dependency purposes, but 178 // those can be modeled with hasSideEffects here. These also end up 179 // inferring differently for llvm.readcyclecounter and the amdgcn 180 // intrinsics. 181 let mayStore = ?; 182 let mayLoad = 1; 183 let has_sbase = 0; 184 let has_offset = 0; 185} 186 187class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 188 opName, (outs), (ins), "", [(node)]> { 189 let hasSideEffects = 1; 190 let mayStore = 0; 191 let has_sdst = 0; 192 let has_sbase = 0; 193 let has_offset = 0; 194} 195 196multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> { 197 def _IMM : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smem_offset:$offset), 1>; 198 def _SGPR : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, SReg_32:$offset), 0>; 199} 200 201class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo< 202 opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins), 203 " $sdst", [(set i32:$sdst, (node))]> { 204 let hasSideEffects = 1; 205 let mayStore = 0; 206 let mayLoad = 1; 207 let has_sbase = 0; 208 let has_offset = 0; 209} 210 211//===----------------------------------------------------------------------===// 212// Scalar Atomic Memory Classes 213//===----------------------------------------------------------------------===// 214 215class SM_Atomic_Pseudo <string opName, 216 dag outs, dag ins, string asmOps, bit isRet> 217 : SM_Pseudo<opName, outs, ins, asmOps, []> { 218 219 bit glc = isRet; 220 221 let mayLoad = 1; 222 let mayStore = 1; 223 let has_glc = 1; 224 let has_dlc = 1; 225 226 // Should these be set? 227 let ScalarStore = 1; 228 let hasSideEffects = 1; 229 let maybeAtomic = 1; 230} 231 232class SM_Pseudo_Atomic<string opName, 233 RegisterClass baseClass, 234 RegisterClass dataClass, 235 bit isImm, 236 bit isRet> : 237 SM_Atomic_Pseudo<opName, 238 !if(isRet, (outs dataClass:$sdst), (outs)), 239 !if(isImm, 240 (ins dataClass:$sdata, baseClass:$sbase, smem_offset:$offset, DLC:$dlc), 241 (ins dataClass:$sdata, baseClass:$sbase, SReg_32:$offset, DLC:$dlc)), 242 !if(isRet, " $sdst", " $sdata") # ", $sbase, $offset" # !if(isRet, " glc", "") # "$dlc", 243 isRet> { 244 let offset_is_imm = isImm; 245 let PseudoInstr = opName # !if(isImm, 246 !if(isRet, "_IMM_RTN", "_IMM"), 247 !if(isRet, "_SGPR_RTN", "_SGPR")); 248 249 let Constraints = !if(isRet, "$sdst = $sdata", ""); 250 let DisableEncoding = !if(isRet, "$sdata", ""); 251} 252 253multiclass SM_Pseudo_Atomics<string opName, 254 RegisterClass baseClass, 255 RegisterClass dataClass> { 256 def _IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 0>; 257 def _SGPR : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 0>; 258 def _IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 1>; 259 def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 1>; 260} 261 262//===----------------------------------------------------------------------===// 263// Scalar Memory Instructions 264//===----------------------------------------------------------------------===// 265 266// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit 267// SMRD instructions, because the SReg_32_XM0 register class does not include M0 268// and writing to M0 from an SMRD instruction will hang the GPU. 269 270// XXX - SMEM instructions do not allow exec for data operand, but 271// does sdst for SMRD on SI/CI? 272defm S_LOAD_DWORD : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 273defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64_XEXEC>; 274defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>; 275defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>; 276defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>; 277 278let is_buffer = 1 in { 279defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads < 280 "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC 281>; 282 283// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on 284// SI/CI, bit disallowed for SMEM on VI. 285defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads < 286 "s_buffer_load_dwordx2", SReg_128, SReg_64_XEXEC 287>; 288 289defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads < 290 "s_buffer_load_dwordx4", SReg_128, SReg_128 291>; 292 293defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads < 294 "s_buffer_load_dwordx8", SReg_128, SReg_256 295>; 296 297defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads < 298 "s_buffer_load_dwordx16", SReg_128, SReg_512 299>; 300} 301 302let SubtargetPredicate = HasScalarStores in { 303defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 304defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>; 305defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>; 306 307let is_buffer = 1 in { 308defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores < 309 "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC 310>; 311 312defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores < 313 "s_buffer_store_dwordx2", SReg_128, SReg_64_XEXEC 314>; 315 316defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores < 317 "s_buffer_store_dwordx4", SReg_128, SReg_128 318>; 319} 320} // End SubtargetPredicate = HasScalarStores 321 322let SubtargetPredicate = HasSMemTimeInst in 323def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>; 324def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>; 325 326let SubtargetPredicate = isGFX7GFX8GFX9 in { 327def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>; 328} // let SubtargetPredicate = isGFX7GFX8GFX9 329 330let SubtargetPredicate = isGFX8Plus in { 331let OtherPredicates = [HasScalarStores] in { 332def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>; 333def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>; 334} // End OtherPredicates = [HasScalarStores] 335 336defm S_ATC_PROBE : SM_Pseudo_Probe <"s_atc_probe", SReg_64>; 337let is_buffer = 1 in { 338defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>; 339} 340} // SubtargetPredicate = isGFX8Plus 341 342let SubtargetPredicate = HasSMemRealTime in 343def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>; 344 345let SubtargetPredicate = isGFX10Plus in 346def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">; 347let SubtargetPredicate = HasGetWaveIdInst in 348def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>; 349 350 351let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in { 352defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <"s_scratch_load_dword", SReg_64, SReg_32_XM0_XEXEC>; 353defm S_SCRATCH_LOAD_DWORDX2 : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>; 354defm S_SCRATCH_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>; 355 356defm S_SCRATCH_STORE_DWORD : SM_Pseudo_Stores <"s_scratch_store_dword", SReg_64, SReg_32_XM0_XEXEC>; 357defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>; 358defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>; 359} // SubtargetPredicate = HasScalarFlatScratchInsts 360 361let SubtargetPredicate = HasScalarAtomics in { 362 363let is_buffer = 1 in { 364defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>; 365defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>; 366defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>; 367defm S_BUFFER_ATOMIC_SUB : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>; 368defm S_BUFFER_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>; 369defm S_BUFFER_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>; 370defm S_BUFFER_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>; 371defm S_BUFFER_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>; 372defm S_BUFFER_ATOMIC_AND : SM_Pseudo_Atomics <"s_buffer_atomic_and", SReg_128, SReg_32_XM0_XEXEC>; 373defm S_BUFFER_ATOMIC_OR : SM_Pseudo_Atomics <"s_buffer_atomic_or", SReg_128, SReg_32_XM0_XEXEC>; 374defm S_BUFFER_ATOMIC_XOR : SM_Pseudo_Atomics <"s_buffer_atomic_xor", SReg_128, SReg_32_XM0_XEXEC>; 375defm S_BUFFER_ATOMIC_INC : SM_Pseudo_Atomics <"s_buffer_atomic_inc", SReg_128, SReg_32_XM0_XEXEC>; 376defm S_BUFFER_ATOMIC_DEC : SM_Pseudo_Atomics <"s_buffer_atomic_dec", SReg_128, SReg_32_XM0_XEXEC>; 377 378defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_swap_x2", SReg_128, SReg_64_XEXEC>; 379defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap_x2", SReg_128, SReg_128>; 380defm S_BUFFER_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_add_x2", SReg_128, SReg_64_XEXEC>; 381defm S_BUFFER_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_sub_x2", SReg_128, SReg_64_XEXEC>; 382defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smin_x2", SReg_128, SReg_64_XEXEC>; 383defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>; 384defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>; 385defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>; 386defm S_BUFFER_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>; 387defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>; 388defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>; 389defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>; 390defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>; 391} 392 393defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>; 394defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>; 395defm S_ATOMIC_ADD : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>; 396defm S_ATOMIC_SUB : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>; 397defm S_ATOMIC_SMIN : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>; 398defm S_ATOMIC_UMIN : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>; 399defm S_ATOMIC_SMAX : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>; 400defm S_ATOMIC_UMAX : SM_Pseudo_Atomics <"s_atomic_umax", SReg_64, SReg_32_XM0_XEXEC>; 401defm S_ATOMIC_AND : SM_Pseudo_Atomics <"s_atomic_and", SReg_64, SReg_32_XM0_XEXEC>; 402defm S_ATOMIC_OR : SM_Pseudo_Atomics <"s_atomic_or", SReg_64, SReg_32_XM0_XEXEC>; 403defm S_ATOMIC_XOR : SM_Pseudo_Atomics <"s_atomic_xor", SReg_64, SReg_32_XM0_XEXEC>; 404defm S_ATOMIC_INC : SM_Pseudo_Atomics <"s_atomic_inc", SReg_64, SReg_32_XM0_XEXEC>; 405defm S_ATOMIC_DEC : SM_Pseudo_Atomics <"s_atomic_dec", SReg_64, SReg_32_XM0_XEXEC>; 406 407defm S_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <"s_atomic_swap_x2", SReg_64, SReg_64_XEXEC>; 408defm S_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <"s_atomic_cmpswap_x2", SReg_64, SReg_128>; 409defm S_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <"s_atomic_add_x2", SReg_64, SReg_64_XEXEC>; 410defm S_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <"s_atomic_sub_x2", SReg_64, SReg_64_XEXEC>; 411defm S_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <"s_atomic_smin_x2", SReg_64, SReg_64_XEXEC>; 412defm S_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <"s_atomic_umin_x2", SReg_64, SReg_64_XEXEC>; 413defm S_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <"s_atomic_smax_x2", SReg_64, SReg_64_XEXEC>; 414defm S_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <"s_atomic_umax_x2", SReg_64, SReg_64_XEXEC>; 415defm S_ATOMIC_AND_X2 : SM_Pseudo_Atomics <"s_atomic_and_x2", SReg_64, SReg_64_XEXEC>; 416defm S_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_atomic_or_x2", SReg_64, SReg_64_XEXEC>; 417defm S_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_atomic_xor_x2", SReg_64, SReg_64_XEXEC>; 418defm S_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_atomic_inc_x2", SReg_64, SReg_64_XEXEC>; 419defm S_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_atomic_dec_x2", SReg_64, SReg_64_XEXEC>; 420 421} // let SubtargetPredicate = HasScalarAtomics 422 423let SubtargetPredicate = HasScalarAtomics in { 424defm S_DCACHE_DISCARD : SM_Pseudo_Discards <"s_dcache_discard">; 425defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards <"s_dcache_discard_x2">; 426} 427 428//===----------------------------------------------------------------------===// 429// Targets 430//===----------------------------------------------------------------------===// 431 432//===----------------------------------------------------------------------===// 433// SI 434//===----------------------------------------------------------------------===// 435 436class SMRD_Real_si <bits<5> op, SM_Pseudo ps> 437 : SM_Real<ps> 438 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 439 , Enc32 { 440 441 let AssemblerPredicate = isGFX6GFX7; 442 let DecoderNamespace = "GFX6GFX7"; 443 444 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?); 445 let Inst{8} = imm; 446 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 447 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 448 let Inst{26-22} = op; 449 let Inst{31-27} = 0x18; //encoding 450} 451 452// FIXME: Assembler should reject trying to use glc on SMRD 453// instructions on SI. 454multiclass SM_Real_Loads_si<bits<5> op, string ps, 455 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 456 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 457 458 def _IMM_si : SMRD_Real_si <op, immPs> { 459 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, GLC:$glc, DLC:$dlc); 460 } 461 462 // FIXME: The operand name $offset is inconsistent with $soff used 463 // in the pseudo 464 def _SGPR_si : SMRD_Real_si <op, sgprPs> { 465 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); 466 } 467 468} 469 470defm S_LOAD_DWORD : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">; 471defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">; 472defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">; 473defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">; 474defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">; 475defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">; 476defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09, "S_BUFFER_LOAD_DWORDX2">; 477defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a, "S_BUFFER_LOAD_DWORDX4">; 478defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b, "S_BUFFER_LOAD_DWORDX8">; 479defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c, "S_BUFFER_LOAD_DWORDX16">; 480 481def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>; 482def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>; 483 484 485//===----------------------------------------------------------------------===// 486// VI 487//===----------------------------------------------------------------------===// 488 489class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> 490 : SM_Real<ps> 491 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> 492 , Enc64 { 493 bit glc; 494 495 let AssemblerPredicate = isGFX8GFX9; 496 let DecoderNamespace = "GFX8"; 497 498 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 499 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 500 501 let Inst{16} = !if(ps.has_glc, glc, ?); 502 let Inst{17} = imm; 503 let Inst{25-18} = op; 504 let Inst{31-26} = 0x30; //encoding 505 506 // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed. 507 // Offset value is corrected accordingly when offset is encoded/decoded. 508 let Inst{52-32} = !if(ps.has_offset, offset{20-0}, ?); 509} 510 511multiclass SM_Real_Loads_vi<bits<8> op, string ps, 512 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 513 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 514 def _IMM_vi : SMEM_Real_vi <op, immPs> { 515 let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); 516 } 517 def _SGPR_vi : SMEM_Real_vi <op, sgprPs> { 518 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); 519 } 520} 521 522class SMEM_Real_Store_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> { 523 // encoding 524 bits<7> sdata; 525 526 let sdst = ?; 527 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 528} 529 530multiclass SM_Real_Stores_vi<bits<8> op, string ps, 531 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM), 532 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> { 533 // FIXME: The operand name $offset is inconsistent with $soff used 534 // in the pseudo 535 def _IMM_vi : SMEM_Real_Store_vi <op, immPs> { 536 let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); 537 } 538 539 def _SGPR_vi : SMEM_Real_Store_vi <op, sgprPs> { 540 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); 541 } 542} 543 544multiclass SM_Real_Probe_vi<bits<8> op, string ps> { 545 def _IMM_vi : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 546 def _SGPR_vi : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 547} 548 549defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">; 550defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">; 551defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">; 552defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">; 553defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">; 554defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">; 555defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">; 556defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">; 557defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">; 558defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">; 559 560defm S_STORE_DWORD : SM_Real_Stores_vi <0x10, "S_STORE_DWORD">; 561defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11, "S_STORE_DWORDX2">; 562defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12, "S_STORE_DWORDX4">; 563 564defm S_BUFFER_STORE_DWORD : SM_Real_Stores_vi <0x18, "S_BUFFER_STORE_DWORD">; 565defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_vi <0x19, "S_BUFFER_STORE_DWORDX2">; 566defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_vi <0x1a, "S_BUFFER_STORE_DWORDX4">; 567 568// These instructions use same encoding 569def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>; 570def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>; 571def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>; 572def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>; 573def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>; 574def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>; 575 576defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_vi <0x05, "S_SCRATCH_LOAD_DWORD">; 577defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_vi <0x06, "S_SCRATCH_LOAD_DWORDX2">; 578defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_vi <0x07, "S_SCRATCH_LOAD_DWORDX4">; 579 580defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_vi <0x15, "S_SCRATCH_STORE_DWORD">; 581defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16, "S_SCRATCH_STORE_DWORDX2">; 582defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17, "S_SCRATCH_STORE_DWORDX4">; 583 584defm S_ATC_PROBE : SM_Real_Probe_vi <0x26, "S_ATC_PROBE">; 585defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27, "S_ATC_PROBE_BUFFER">; 586 587//===----------------------------------------------------------------------===// 588// GFX9 589//===----------------------------------------------------------------------===// 590 591class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps> 592 : SMEM_Real_vi <op, ps> { 593 594 bits<7> sdata; 595 596 let Constraints = ps.Constraints; 597 let DisableEncoding = ps.DisableEncoding; 598 599 let glc = ps.glc; 600 let Inst{12-6} = !if(glc, sdst{6-0}, sdata{6-0}); 601} 602 603multiclass SM_Real_Atomics_vi<bits<8> op, string ps> { 604 def _IMM_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 605 def _SGPR_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 606 def _IMM_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 607 def _SGPR_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 608} 609 610defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_vi <0x40, "S_BUFFER_ATOMIC_SWAP">; 611defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 612defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_vi <0x42, "S_BUFFER_ATOMIC_ADD">; 613defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_vi <0x43, "S_BUFFER_ATOMIC_SUB">; 614defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_vi <0x44, "S_BUFFER_ATOMIC_SMIN">; 615defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_vi <0x45, "S_BUFFER_ATOMIC_UMIN">; 616defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_vi <0x46, "S_BUFFER_ATOMIC_SMAX">; 617defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_vi <0x47, "S_BUFFER_ATOMIC_UMAX">; 618defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_vi <0x48, "S_BUFFER_ATOMIC_AND">; 619defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_vi <0x49, "S_BUFFER_ATOMIC_OR">; 620defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_vi <0x4a, "S_BUFFER_ATOMIC_XOR">; 621defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_vi <0x4b, "S_BUFFER_ATOMIC_INC">; 622defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_vi <0x4c, "S_BUFFER_ATOMIC_DEC">; 623 624defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 625defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 626defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 627defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 628defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 629defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 630defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 631defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 632defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0x68, "S_BUFFER_ATOMIC_AND_X2">; 633defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0x69, "S_BUFFER_ATOMIC_OR_X2">; 634defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 635defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 636defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 637 638defm S_ATOMIC_SWAP : SM_Real_Atomics_vi <0x80, "S_ATOMIC_SWAP">; 639defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x81, "S_ATOMIC_CMPSWAP">; 640defm S_ATOMIC_ADD : SM_Real_Atomics_vi <0x82, "S_ATOMIC_ADD">; 641defm S_ATOMIC_SUB : SM_Real_Atomics_vi <0x83, "S_ATOMIC_SUB">; 642defm S_ATOMIC_SMIN : SM_Real_Atomics_vi <0x84, "S_ATOMIC_SMIN">; 643defm S_ATOMIC_UMIN : SM_Real_Atomics_vi <0x85, "S_ATOMIC_UMIN">; 644defm S_ATOMIC_SMAX : SM_Real_Atomics_vi <0x86, "S_ATOMIC_SMAX">; 645defm S_ATOMIC_UMAX : SM_Real_Atomics_vi <0x87, "S_ATOMIC_UMAX">; 646defm S_ATOMIC_AND : SM_Real_Atomics_vi <0x88, "S_ATOMIC_AND">; 647defm S_ATOMIC_OR : SM_Real_Atomics_vi <0x89, "S_ATOMIC_OR">; 648defm S_ATOMIC_XOR : SM_Real_Atomics_vi <0x8a, "S_ATOMIC_XOR">; 649defm S_ATOMIC_INC : SM_Real_Atomics_vi <0x8b, "S_ATOMIC_INC">; 650defm S_ATOMIC_DEC : SM_Real_Atomics_vi <0x8c, "S_ATOMIC_DEC">; 651 652defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0xa0, "S_ATOMIC_SWAP_X2">; 653defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0xa1, "S_ATOMIC_CMPSWAP_X2">; 654defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0xa2, "S_ATOMIC_ADD_X2">; 655defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0xa3, "S_ATOMIC_SUB_X2">; 656defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0xa4, "S_ATOMIC_SMIN_X2">; 657defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0xa5, "S_ATOMIC_UMIN_X2">; 658defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0xa6, "S_ATOMIC_SMAX_X2">; 659defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0xa7, "S_ATOMIC_UMAX_X2">; 660defm S_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0xa8, "S_ATOMIC_AND_X2">; 661defm S_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0xa9, "S_ATOMIC_OR_X2">; 662defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0xaa, "S_ATOMIC_XOR_X2">; 663defm S_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0xab, "S_ATOMIC_INC_X2">; 664defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0xac, "S_ATOMIC_DEC_X2">; 665 666multiclass SM_Real_Discard_vi<bits<8> op, string ps> { 667 def _IMM_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>; 668 def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>; 669} 670 671defm S_DCACHE_DISCARD : SM_Real_Discard_vi <0x28, "S_DCACHE_DISCARD">; 672defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29, "S_DCACHE_DISCARD_X2">; 673 674//===----------------------------------------------------------------------===// 675// CI 676//===----------------------------------------------------------------------===// 677 678def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset", 679 NamedMatchClass<"SMRDLiteralOffset">> { 680 let OperandType = "OPERAND_IMMEDIATE"; 681} 682 683class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : 684 SM_Real<ps>, 685 Enc64 { 686 687 let AssemblerPredicate = isGFX7Only; 688 let DecoderNamespace = "GFX7"; 689 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, GLC:$glc, DLC:$dlc); 690 691 let LGKM_CNT = ps.LGKM_CNT; 692 let mayLoad = ps.mayLoad; 693 let mayStore = ps.mayStore; 694 let hasSideEffects = ps.hasSideEffects; 695 let SchedRW = ps.SchedRW; 696 697 let Inst{7-0} = 0xff; 698 let Inst{8} = 0; 699 let Inst{14-9} = sbase{6-1}; 700 let Inst{21-15} = sdst{6-0}; 701 let Inst{26-22} = op; 702 let Inst{31-27} = 0x18; //encoding 703 let Inst{63-32} = offset{31-0}; 704} 705 706def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>; 707def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>; 708def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>; 709def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>; 710def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>; 711def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>; 712def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>; 713def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>; 714def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>; 715def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>; 716 717class SMRD_Real_ci <bits<5> op, SM_Pseudo ps> 718 : SM_Real<ps> 719 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 720 , Enc32 { 721 722 let AssemblerPredicate = isGFX7Only; 723 let DecoderNamespace = "GFX7"; 724 725 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?); 726 let Inst{8} = imm; 727 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 728 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 729 let Inst{26-22} = op; 730 let Inst{31-27} = 0x18; //encoding 731} 732 733def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>; 734 735//===----------------------------------------------------------------------===// 736// Scalar Memory Patterns 737//===----------------------------------------------------------------------===// 738 739def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]> { 740 let GISelPredicateCode = [{ 741 if (!MI.hasOneMemOperand()) 742 return false; 743 if (!isInstrUniform(MI)) 744 return false; 745 746 // FIXME: We should probably be caching this. 747 SmallVector<GEPInfo, 4> AddrInfo; 748 getAddrModeInfo(MI, MRI, AddrInfo); 749 750 if (hasVgprParts(AddrInfo)) 751 return false; 752 return true; 753 }]; 754} 755 756def SMRDImm : ComplexPattern<i64, 2, "SelectSMRDImm">; 757def SMRDImm32 : ComplexPattern<i64, 2, "SelectSMRDImm32">; 758def SMRDSgpr : ComplexPattern<i64, 2, "SelectSMRDSgpr">; 759def SMRDBufferImm : ComplexPattern<i32, 1, "SelectSMRDBufferImm">; 760def SMRDBufferImm32 : ComplexPattern<i32, 1, "SelectSMRDBufferImm32">; 761 762multiclass SMRD_Pattern <string Instr, ValueType vt> { 763 764 // 1. IMM offset 765 def : GCNPat < 766 (smrd_load (SMRDImm i64:$sbase, i32:$offset)), 767 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0, 0)) 768 >; 769 770 // 2. 32-bit IMM offset on CI 771 def : GCNPat < 772 (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)), 773 (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0, 0))> { 774 let OtherPredicates = [isGFX7Only]; 775 } 776 777 // 3. SGPR offset 778 def : GCNPat < 779 (smrd_load (SMRDSgpr i64:$sbase, i32:$offset)), 780 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, 0, 0)) 781 >; 782 783 // 4. No offset 784 def : GCNPat < 785 (vt (smrd_load (i64 SReg_64:$sbase))), 786 (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0, 0)) 787 >; 788} 789 790multiclass SMLoad_Pattern <string Instr, ValueType vt> { 791 // 1. Offset as an immediate 792 def : GCNPat < 793 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy), 794 (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_glc $cachepolicy), 795 (extract_dlc $cachepolicy)))> { 796 let AddedComplexity = 2; 797 } 798 799 // 2. 32-bit IMM offset on CI 800 def : GCNPat < 801 (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)), 802 (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset, 803 (extract_glc $cachepolicy), (extract_dlc $cachepolicy))> { 804 let OtherPredicates = [isGFX7Only]; 805 let AddedComplexity = 1; 806 } 807 808 // 3. Offset loaded in an 32bit SGPR 809 def : GCNPat < 810 (SIsbuffer_load v4i32:$sbase, i32:$offset, timm:$cachepolicy), 811 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$offset, (extract_glc $cachepolicy), 812 (extract_dlc $cachepolicy))) 813 >; 814} 815 816// Global and constant loads can be selected to either MUBUF or SMRD 817// instructions, but SMRD instructions are faster so we want the instruction 818// selector to prefer those. 819let AddedComplexity = 100 in { 820 821foreach vt = Reg32Types.types in { 822defm : SMRD_Pattern <"S_LOAD_DWORD", vt>; 823} 824 825foreach vt = SReg_64.RegTypes in { 826defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>; 827} 828 829foreach vt = SReg_128.RegTypes in { 830defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>; 831} 832 833foreach vt = SReg_256.RegTypes in { 834defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>; 835} 836 837foreach vt = SReg_512.RegTypes in { 838defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>; 839} 840 841defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>; 842defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>; 843defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>; 844defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>; 845defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16i32>; 846 847defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", f32>; 848defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2f32>; 849defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4f32>; 850defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8f32>; 851defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16f32>; 852} // End let AddedComplexity = 100 853 854let OtherPredicates = [HasSMemTimeInst] in { 855def : GCNPat < 856 (i64 (readcyclecounter)), 857 (S_MEMTIME) 858>; 859} // let OtherPredicates = [HasSMemTimeInst] 860 861let OtherPredicates = [HasNoSMemTimeInst] in { 862def : GCNPat < 863 (i64 (readcyclecounter)), 864 (REG_SEQUENCE SReg_64, 865 (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0, 866 (S_MOV_B32 (i32 0)), sub1) 867>; 868} // let OtherPredicates = [HasNoSMemTimeInst] 869 870//===----------------------------------------------------------------------===// 871// GFX10. 872//===----------------------------------------------------------------------===// 873 874class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> : 875 SM_Real<ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10>, Enc64 { 876 bit glc; 877 bit dlc; 878 879 let AssemblerPredicate = isGFX10Plus; 880 let DecoderNamespace = "GFX10"; 881 882 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 883 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 884 let Inst{14} = !if(ps.has_dlc, dlc, ?); 885 let Inst{16} = !if(ps.has_glc, glc, ?); 886 let Inst{25-18} = op; 887 let Inst{31-26} = 0x3d; 888 let Inst{52-32} = !if(ps.offset_is_imm, !if(ps.has_offset, offset{20-0}, ?), ?); 889 let Inst{63-57} = !if(ps.offset_is_imm, !cast<int>(SGPR_NULL.HWEncoding), 890 !if(ps.has_offset, offset{6-0}, ?)); 891} 892 893multiclass SM_Real_Loads_gfx10<bits<8> op, string ps, 894 SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), 895 SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { 896 def _IMM_gfx10 : SMEM_Real_gfx10<op, immPs> { 897 let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); 898 } 899 def _SGPR_gfx10 : SMEM_Real_gfx10<op, sgprPs> { 900 let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); 901 } 902} 903 904class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> { 905 bits<7> sdata; 906 907 let sdst = ?; 908 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 909} 910 911multiclass SM_Real_Stores_gfx10<bits<8> op, string ps, 912 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM), 913 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> { 914 // FIXME: The operand name $offset is inconsistent with $soff used 915 // in the pseudo 916 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs> { 917 let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); 918 } 919 920 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs> { 921 let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); 922 } 923} 924 925defm S_LOAD_DWORD : SM_Real_Loads_gfx10<0x000, "S_LOAD_DWORD">; 926defm S_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x001, "S_LOAD_DWORDX2">; 927defm S_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x002, "S_LOAD_DWORDX4">; 928defm S_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x003, "S_LOAD_DWORDX8">; 929defm S_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x004, "S_LOAD_DWORDX16">; 930 931let SubtargetPredicate = HasScalarFlatScratchInsts in { 932defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_gfx10<0x005, "S_SCRATCH_LOAD_DWORD">; 933defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x006, "S_SCRATCH_LOAD_DWORDX2">; 934defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x007, "S_SCRATCH_LOAD_DWORDX4">; 935} // End SubtargetPredicate = HasScalarFlatScratchInsts 936 937defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_gfx10<0x008, "S_BUFFER_LOAD_DWORD">; 938defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x009, "S_BUFFER_LOAD_DWORDX2">; 939defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x00a, "S_BUFFER_LOAD_DWORDX4">; 940defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x00b, "S_BUFFER_LOAD_DWORDX8">; 941defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x00c, "S_BUFFER_LOAD_DWORDX16">; 942 943let SubtargetPredicate = HasScalarStores in { 944defm S_STORE_DWORD : SM_Real_Stores_gfx10<0x010, "S_STORE_DWORD">; 945defm S_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x011, "S_STORE_DWORDX2">; 946defm S_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x012, "S_STORE_DWORDX4">; 947let OtherPredicates = [HasScalarFlatScratchInsts] in { 948defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_gfx10<0x015, "S_SCRATCH_STORE_DWORD">; 949defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016, "S_SCRATCH_STORE_DWORDX2">; 950defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017, "S_SCRATCH_STORE_DWORDX4">; 951} // End OtherPredicates = [HasScalarFlatScratchInsts] 952defm S_BUFFER_STORE_DWORD : SM_Real_Stores_gfx10<0x018, "S_BUFFER_STORE_DWORD">; 953defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x019, "S_BUFFER_STORE_DWORDX2">; 954defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x01a, "S_BUFFER_STORE_DWORDX4">; 955} // End SubtargetPredicate = HasScalarStores 956 957def S_MEMREALTIME_gfx10 : SMEM_Real_gfx10<0x025, S_MEMREALTIME>; 958def S_MEMTIME_gfx10 : SMEM_Real_gfx10<0x024, S_MEMTIME>; 959def S_GL1_INV_gfx10 : SMEM_Real_gfx10<0x01f, S_GL1_INV>; 960def S_GET_WAVEID_IN_WORKGROUP_gfx10 : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>; 961def S_DCACHE_INV_gfx10 : SMEM_Real_gfx10<0x020, S_DCACHE_INV>; 962 963let SubtargetPredicate = HasScalarStores in { 964def S_DCACHE_WB_gfx10 : SMEM_Real_gfx10<0x021, S_DCACHE_WB>; 965} // End SubtargetPredicate = HasScalarStores 966 967multiclass SM_Real_Probe_gfx10<bits<8> op, string ps> { 968 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 969 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 970} 971 972defm S_ATC_PROBE : SM_Real_Probe_gfx10 <0x26, "S_ATC_PROBE">; 973defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27, "S_ATC_PROBE_BUFFER">; 974 975class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps> 976 : SMEM_Real_gfx10 <op, ps> { 977 978 bits<7> sdata; 979 bit dlc; 980 981 let Constraints = ps.Constraints; 982 let DisableEncoding = ps.DisableEncoding; 983 984 let glc = ps.glc; 985 986 let Inst{14} = !if(ps.has_dlc, dlc, 0); 987 let Inst{12-6} = !if(glc, sdst{6-0}, sdata{6-0}); 988} 989 990multiclass SM_Real_Atomics_gfx10<bits<8> op, string ps> { 991 def _IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 992 def _SGPR_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 993 def _IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 994 def _SGPR_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 995} 996 997let SubtargetPredicate = HasScalarAtomics in { 998 999defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x40, "S_BUFFER_ATOMIC_SWAP">; 1000defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x41, "S_BUFFER_ATOMIC_CMPSWAP">; 1001defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x42, "S_BUFFER_ATOMIC_ADD">; 1002defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x43, "S_BUFFER_ATOMIC_SUB">; 1003defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x44, "S_BUFFER_ATOMIC_SMIN">; 1004defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x45, "S_BUFFER_ATOMIC_UMIN">; 1005defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x46, "S_BUFFER_ATOMIC_SMAX">; 1006defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x47, "S_BUFFER_ATOMIC_UMAX">; 1007defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x48, "S_BUFFER_ATOMIC_AND">; 1008defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x49, "S_BUFFER_ATOMIC_OR">; 1009defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x4a, "S_BUFFER_ATOMIC_XOR">; 1010defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x4b, "S_BUFFER_ATOMIC_INC">; 1011defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x4c, "S_BUFFER_ATOMIC_DEC">; 1012 1013defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0x60, "S_BUFFER_ATOMIC_SWAP_X2">; 1014defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">; 1015defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0x62, "S_BUFFER_ATOMIC_ADD_X2">; 1016defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0x63, "S_BUFFER_ATOMIC_SUB_X2">; 1017defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0x64, "S_BUFFER_ATOMIC_SMIN_X2">; 1018defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0x65, "S_BUFFER_ATOMIC_UMIN_X2">; 1019defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0x66, "S_BUFFER_ATOMIC_SMAX_X2">; 1020defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0x67, "S_BUFFER_ATOMIC_UMAX_X2">; 1021defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0x68, "S_BUFFER_ATOMIC_AND_X2">; 1022defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0x69, "S_BUFFER_ATOMIC_OR_X2">; 1023defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0x6a, "S_BUFFER_ATOMIC_XOR_X2">; 1024defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0x6b, "S_BUFFER_ATOMIC_INC_X2">; 1025defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0x6c, "S_BUFFER_ATOMIC_DEC_X2">; 1026 1027defm S_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x80, "S_ATOMIC_SWAP">; 1028defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x81, "S_ATOMIC_CMPSWAP">; 1029defm S_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x82, "S_ATOMIC_ADD">; 1030defm S_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x83, "S_ATOMIC_SUB">; 1031defm S_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x84, "S_ATOMIC_SMIN">; 1032defm S_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x85, "S_ATOMIC_UMIN">; 1033defm S_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x86, "S_ATOMIC_SMAX">; 1034defm S_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x87, "S_ATOMIC_UMAX">; 1035defm S_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x88, "S_ATOMIC_AND">; 1036defm S_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x89, "S_ATOMIC_OR">; 1037defm S_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x8a, "S_ATOMIC_XOR">; 1038defm S_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x8b, "S_ATOMIC_INC">; 1039defm S_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x8c, "S_ATOMIC_DEC">; 1040 1041defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0xa0, "S_ATOMIC_SWAP_X2">; 1042defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0xa1, "S_ATOMIC_CMPSWAP_X2">; 1043defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0xa2, "S_ATOMIC_ADD_X2">; 1044defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0xa3, "S_ATOMIC_SUB_X2">; 1045defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0xa4, "S_ATOMIC_SMIN_X2">; 1046defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0xa5, "S_ATOMIC_UMIN_X2">; 1047defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0xa6, "S_ATOMIC_SMAX_X2">; 1048defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0xa7, "S_ATOMIC_UMAX_X2">; 1049defm S_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0xa8, "S_ATOMIC_AND_X2">; 1050defm S_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0xa9, "S_ATOMIC_OR_X2">; 1051defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0xaa, "S_ATOMIC_XOR_X2">; 1052defm S_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0xab, "S_ATOMIC_INC_X2">; 1053defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0xac, "S_ATOMIC_DEC_X2">; 1054 1055multiclass SM_Real_Discard_gfx10<bits<8> op, string ps> { 1056 def _IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1057 def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1058} 1059 1060defm S_DCACHE_DISCARD : SM_Real_Discard_gfx10 <0x28, "S_DCACHE_DISCARD">; 1061defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29, "S_DCACHE_DISCARD_X2">; 1062 1063} // End SubtargetPredicate = HasScalarAtomics 1064 1065def SMInfoTable : GenericTable { 1066 let FilterClass = "SM_Real"; 1067 let CppTypeName = "SMInfo"; 1068 let Fields = ["Opcode", "is_buffer"]; 1069 1070 let PrimaryKey = ["Opcode"]; 1071 let PrimaryKeyName = "getSMEMOpcodeHelper"; 1072} 1073