1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 10def MUBUFAddr64 : ComplexPattern<i64, 9, "SelectMUBUFAddr64">; 11def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 12 13def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 14def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 15 16def MUBUFOffset : ComplexPattern<i64, 8, "SelectMUBUFOffset">; 17def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 18def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 19 20def BUFAddrKind { 21 int Offset = 0; 22 int OffEn = 1; 23 int IdxEn = 2; 24 int BothEn = 3; 25 int Addr64 = 4; 26} 27 28class getAddrName<int addrKind> { 29 string ret = 30 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 31 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 32 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 33 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 34 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 35 ""))))); 36} 37 38class MUBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43class MUBUFLdsTable <bit is_lds, string Name> { 44 bit IsLds = is_lds; 45 string OpName = Name; 46} 47 48class MTBUFAddr64Table <bit is_addr64, string Name> { 49 bit IsAddr64 = is_addr64; 50 string OpName = Name; 51} 52 53//===----------------------------------------------------------------------===// 54// MTBUF classes 55//===----------------------------------------------------------------------===// 56 57class MTBUFGetBaseOpcode<string Op> { 58 string ret = !subst("FORMAT_XY", "FORMAT_X", 59 !subst("FORMAT_XYZ", "FORMAT_X", 60 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 61} 62 63class getMTBUFElements<string Op> { 64 int ret = 1; 65} 66 67 68class MTBUF_Pseudo <string opName, dag outs, dag ins, 69 string asmOps, list<dag> pattern=[]> : 70 InstSI<outs, ins, "", pattern>, 71 SIMCInstr<opName, SIEncodingFamily.NONE> { 72 73 let isPseudo = 1; 74 let isCodeGenOnly = 1; 75 let Size = 8; 76 let UseNamedOperandTable = 1; 77 78 string Mnemonic = opName; 79 string AsmOperands = asmOps; 80 81 Instruction Opcode = !cast<Instruction>(NAME); 82 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 83 84 let VM_CNT = 1; 85 let EXP_CNT = 1; 86 let MTBUF = 1; 87 let Uses = [EXEC]; 88 let hasSideEffects = 0; 89 let SchedRW = [WriteVMEM]; 90 91 let AsmMatchConverter = "cvtMtbuf"; 92 93 bits<1> offen = 0; 94 bits<1> idxen = 0; 95 bits<1> addr64 = 0; 96 bits<1> has_vdata = 1; 97 bits<1> has_vaddr = 1; 98 bits<1> has_glc = 1; 99 bits<1> has_dlc = 1; 100 bits<1> glc_value = 0; // the value for glc if no such operand 101 bits<1> dlc_value = 0; // the value for dlc if no such operand 102 bits<1> has_srsrc = 1; 103 bits<1> has_soffset = 1; 104 bits<1> has_offset = 1; 105 bits<1> has_slc = 1; 106 bits<1> has_tfe = 1; 107 bits<4> elements = 0; 108} 109 110class MTBUF_Real <MTBUF_Pseudo ps> : 111 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 112 113 let isPseudo = 0; 114 let isCodeGenOnly = 0; 115 116 // copy relevant pseudo op flags 117 let SubtargetPredicate = ps.SubtargetPredicate; 118 let AsmMatchConverter = ps.AsmMatchConverter; 119 let Constraints = ps.Constraints; 120 let DisableEncoding = ps.DisableEncoding; 121 let TSFlags = ps.TSFlags; 122 123 bits<12> offset; 124 bits<1> glc; 125 bits<1> dlc; 126 bits<7> format; 127 bits<8> vaddr; 128 bits<8> vdata; 129 bits<7> srsrc; 130 bits<1> slc; 131 bits<1> tfe; 132 bits<8> soffset; 133 134 bits<4> dfmt = format{3-0}; 135 bits<3> nfmt = format{6-4}; 136} 137 138class getMTBUFInsDA<list<RegisterClass> vdataList, 139 list<RegisterClass> vaddrList=[]> { 140 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 141 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 142 dag InsNoData = !if(!empty(vaddrList), 143 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 144 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz), 145 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 146 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz) 147 ); 148 dag InsData = !if(!empty(vaddrList), 149 (ins vdataClass:$vdata, SReg_128:$srsrc, 150 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 151 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz), 152 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 153 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 154 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz) 155 ); 156 dag ret = !if(!empty(vdataList), InsNoData, InsData); 157} 158 159class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 160 dag ret = 161 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 162 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 163 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 164 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 165 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 166 (ins)))))); 167} 168 169class getMTBUFAsmOps<int addrKind> { 170 string Pfx = 171 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset", 172 !if(!eq(addrKind, BUFAddrKind.OffEn), 173 "$vaddr, $srsrc, $format, $soffset offen", 174 !if(!eq(addrKind, BUFAddrKind.IdxEn), 175 "$vaddr, $srsrc, $format, $soffset idxen", 176 !if(!eq(addrKind, BUFAddrKind.BothEn), 177 "$vaddr, $srsrc, $format, $soffset idxen offen", 178 !if(!eq(addrKind, BUFAddrKind.Addr64), 179 "$vaddr, $srsrc, $format, $soffset addr64", 180 ""))))); 181 string ret = Pfx # "$offset"; 182} 183 184class MTBUF_SetupAddr<int addrKind> { 185 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 186 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 187 188 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 189 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 190 191 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 192 193 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 194} 195 196class MTBUF_Load_Pseudo <string opName, 197 int addrKind, 198 RegisterClass vdataClass, 199 int elems, 200 list<dag> pattern=[], 201 // Workaround bug bz30254 202 int addrKindCopy = addrKind> 203 : MTBUF_Pseudo<opName, 204 (outs vdataClass:$vdata), 205 getMTBUFIns<addrKindCopy>.ret, 206 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 207 pattern>, 208 MTBUF_SetupAddr<addrKindCopy> { 209 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 210 let mayLoad = 1; 211 let mayStore = 0; 212 let elements = elems; 213} 214 215multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 216 int elems, ValueType load_vt = i32, 217 SDPatternOperator ld = null_frag> { 218 219 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 220 [(set load_vt:$vdata, 221 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format, 222 i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 223 MTBUFAddr64Table<0, NAME>; 224 225 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 226 [(set load_vt:$vdata, 227 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 228 i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 229 MTBUFAddr64Table<1, NAME>; 230 231 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 232 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 233 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 234 235 let DisableWQM = 1 in { 236 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 237 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 238 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 239 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 240 } 241} 242 243class MTBUF_Store_Pseudo <string opName, 244 int addrKind, 245 RegisterClass vdataClass, 246 int elems, 247 list<dag> pattern=[], 248 // Workaround bug bz30254 249 int addrKindCopy = addrKind, 250 RegisterClass vdataClassCopy = vdataClass> 251 : MTBUF_Pseudo<opName, 252 (outs), 253 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 254 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 255 pattern>, 256 MTBUF_SetupAddr<addrKindCopy> { 257 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 258 let mayLoad = 0; 259 let mayStore = 1; 260 let elements = elems; 261} 262 263multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 264 int elems, ValueType store_vt = i32, 265 SDPatternOperator st = null_frag> { 266 267 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 268 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 269 i16:$offset, i8:$format, i1:$glc, 270 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 271 MTBUFAddr64Table<0, NAME>; 272 273 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 274 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 275 i16:$offset, i8:$format, i1:$glc, 276 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 277 MTBUFAddr64Table<1, NAME>; 278 279 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 280 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 281 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 282 283 let DisableWQM = 1 in { 284 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 285 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 286 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 287 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 288 } 289} 290 291 292//===----------------------------------------------------------------------===// 293// MUBUF classes 294//===----------------------------------------------------------------------===// 295 296class MUBUFGetBaseOpcode<string Op> { 297 string ret = !subst("DWORDX2", "DWORD", 298 !subst("DWORDX3", "DWORD", 299 !subst("DWORDX4", "DWORD", Op))); 300} 301 302class MUBUF_Pseudo <string opName, dag outs, dag ins, 303 string asmOps, list<dag> pattern=[]> : 304 InstSI<outs, ins, "", pattern>, 305 SIMCInstr<opName, SIEncodingFamily.NONE> { 306 307 let isPseudo = 1; 308 let isCodeGenOnly = 1; 309 let Size = 8; 310 let UseNamedOperandTable = 1; 311 312 string Mnemonic = opName; 313 string AsmOperands = asmOps; 314 315 Instruction Opcode = !cast<Instruction>(NAME); 316 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 317 318 let VM_CNT = 1; 319 let EXP_CNT = 1; 320 let MUBUF = 1; 321 let Uses = [EXEC]; 322 let hasSideEffects = 0; 323 let SchedRW = [WriteVMEM]; 324 325 let AsmMatchConverter = "cvtMubuf"; 326 327 bits<1> offen = 0; 328 bits<1> idxen = 0; 329 bits<1> addr64 = 0; 330 bits<1> lds = 0; 331 bits<1> has_vdata = 1; 332 bits<1> has_vaddr = 1; 333 bits<1> has_glc = 1; 334 bits<1> has_dlc = 1; 335 bits<1> glc_value = 0; // the value for glc if no such operand 336 bits<1> dlc_value = 0; // the value for dlc if no such operand 337 bits<1> has_srsrc = 1; 338 bits<1> has_soffset = 1; 339 bits<1> has_offset = 1; 340 bits<1> has_slc = 1; 341 bits<1> has_tfe = 1; 342 bits<4> elements = 0; 343} 344 345class MUBUF_Real <MUBUF_Pseudo ps> : 346 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 347 348 let isPseudo = 0; 349 let isCodeGenOnly = 0; 350 351 // copy relevant pseudo op flags 352 let SubtargetPredicate = ps.SubtargetPredicate; 353 let AsmMatchConverter = ps.AsmMatchConverter; 354 let Constraints = ps.Constraints; 355 let DisableEncoding = ps.DisableEncoding; 356 let TSFlags = ps.TSFlags; 357 358 bits<12> offset; 359 bits<1> glc; 360 bits<1> dlc; 361 bits<8> vaddr; 362 bits<8> vdata; 363 bits<7> srsrc; 364 bits<1> slc; 365 bits<1> tfe; 366 bits<8> soffset; 367} 368 369 370// For cache invalidation instructions. 371class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 372 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 373 374 let AsmMatchConverter = ""; 375 376 let hasSideEffects = 1; 377 let mayLoad = 0; 378 let mayStore = 0; 379 380 // Set everything to 0. 381 let offen = 0; 382 let idxen = 0; 383 let addr64 = 0; 384 let has_vdata = 0; 385 let has_vaddr = 0; 386 let has_glc = 0; 387 let has_dlc = 0; 388 let glc_value = 0; 389 let dlc_value = 0; 390 let has_srsrc = 0; 391 let has_soffset = 0; 392 let has_offset = 0; 393 let has_slc = 0; 394 let has_tfe = 0; 395} 396 397class getMUBUFInsDA<list<RegisterClass> vdataList, 398 list<RegisterClass> vaddrList=[], 399 bit isLds = 0> { 400 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 401 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 402 dag InsNoData = !if(!empty(vaddrList), 403 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 404 offset:$offset, GLC:$glc, SLC:$slc), 405 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 406 offset:$offset, GLC:$glc, SLC:$slc) 407 ); 408 dag InsData = !if(!empty(vaddrList), 409 (ins vdataClass:$vdata, SReg_128:$srsrc, 410 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 411 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 412 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 413 ); 414 dag ret = !con( 415 !if(!empty(vdataList), InsNoData, InsData), 416 !if(isLds, (ins DLC:$dlc, SWZ:$swz), (ins TFE:$tfe, DLC:$dlc,SWZ:$swz)) 417 ); 418} 419 420class getMUBUFElements<ValueType vt> { 421 // eq does not support ValueType for some reason. 422 string vtAsStr = !cast<string>(vt); 423 424 int ret = 425 !if(!eq(vtAsStr, "f16"), 1, 426 !if(!eq(vtAsStr, "v2f16"), 2, 427 !if(!eq(vtAsStr, "v3f16"), 3, 428 !if(!eq(vtAsStr, "v4f16"), 4, 429 !if(!eq(vt.Size, 32), 1, 430 !if(!eq(vt.Size, 64), 2, 431 !if(!eq(vt.Size, 96), 3, 432 !if(!eq(vt.Size, 128), 4, 0) 433 ) 434 ) 435 ) 436 ) 437 ) 438 ) 439 ); 440} 441 442class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 443 dag ret = 444 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 445 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 446 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 447 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 448 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 449 (ins)))))); 450} 451 452class getMUBUFAsmOps<int addrKind> { 453 string Pfx = 454 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 455 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 456 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 457 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 458 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 459 ""))))); 460 string ret = Pfx # "$offset"; 461} 462 463class MUBUF_SetupAddr<int addrKind> { 464 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 465 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 466 467 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 468 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 469 470 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 471 472 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 473} 474 475class MUBUF_Load_Pseudo <string opName, 476 int addrKind, 477 ValueType vdata_vt, 478 bit HasTiedDest = 0, 479 bit isLds = 0, 480 list<dag> pattern=[], 481 // Workaround bug bz30254 482 int addrKindCopy = addrKind> 483 : MUBUF_Pseudo<opName, 484 (outs getVregSrcForVT<vdata_vt>.ret:$vdata), 485 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 486 !if(HasTiedDest, (ins getVregSrcForVT<vdata_vt>.ret:$vdata_in), (ins))), 487 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 488 !if(isLds, " lds", "$tfe") # "$dlc" # "$swz", 489 pattern>, 490 MUBUF_SetupAddr<addrKindCopy> { 491 let PseudoInstr = opName # !if(isLds, "_lds", "") # 492 "_" # getAddrName<addrKindCopy>.ret; 493 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 494 495 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 496 let mayLoad = 1; 497 let mayStore = 0; 498 let maybeAtomic = 1; 499 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 500 let has_tfe = !if(isLds, 0, 1); 501 let lds = isLds; 502 let elements = getMUBUFElements<vdata_vt>.ret; 503} 504 505class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 506 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 507 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 508>; 509 510class MUBUF_Addr64_Load_Pat <Instruction inst, 511 ValueType load_vt = i32, 512 SDPatternOperator ld = null_frag> : Pat < 513 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 514 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 515>; 516 517multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 518 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 519 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 520} 521 522 523// FIXME: tfe can't be an operand because it requires a separate 524// opcode because it needs an N+1 register class dest register. 525multiclass MUBUF_Pseudo_Loads<string opName, 526 ValueType load_vt = i32, 527 SDPatternOperator ld = null_frag, 528 bit TiedDest = 0, 529 bit isLds = 0> { 530 531 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>, 532 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 533 534 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, load_vt, TiedDest, isLds>, 535 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 536 537 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>; 538 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>; 539 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>; 540 541 let DisableWQM = 1 in { 542 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>; 543 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>; 544 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>; 545 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>; 546 } 547} 548 549multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32, 550 SDPatternOperator ld_nolds = null_frag, 551 SDPatternOperator ld_lds = null_frag> { 552 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>; 553 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>; 554} 555 556class MUBUF_Store_Pseudo <string opName, 557 int addrKind, 558 ValueType store_vt, 559 list<dag> pattern=[], 560 // Workaround bug bz30254 561 int addrKindCopy = addrKind> 562 : MUBUF_Pseudo<opName, 563 (outs), 564 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 565 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 566 pattern>, 567 MUBUF_SetupAddr<addrKindCopy> { 568 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 569 let mayLoad = 0; 570 let mayStore = 1; 571 let maybeAtomic = 1; 572 let elements = getMUBUFElements<store_vt>.ret; 573} 574 575multiclass MUBUF_Pseudo_Stores<string opName, 576 ValueType store_vt = i32, 577 SDPatternOperator st = null_frag> { 578 579 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt, 580 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 581 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 582 MUBUFAddr64Table<0, NAME>; 583 584 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt, 585 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 586 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 587 MUBUFAddr64Table<1, NAME>; 588 589 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>; 590 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>; 591 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>; 592 593 let DisableWQM = 1 in { 594 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt>; 595 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>; 596 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>; 597 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>; 598 } 599} 600 601class MUBUF_Pseudo_Store_Lds<string opName> 602 : MUBUF_Pseudo<opName, 603 (outs), 604 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc, SWZ:$swz), 605 " $srsrc, $soffset$offset lds$glc$slc$swz"> { 606 let mayLoad = 0; 607 let mayStore = 1; 608 let maybeAtomic = 1; 609 610 let has_vdata = 0; 611 let has_vaddr = 0; 612 let has_tfe = 0; 613 let lds = 1; 614 615 let Uses = [EXEC, M0]; 616 let AsmMatchConverter = "cvtMubufLds"; 617} 618 619class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 620 list<RegisterClass> vaddrList=[]> { 621 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 622 dag ret = !if(vdata_in, 623 !if(!empty(vaddrList), 624 (ins vdataClass:$vdata_in, 625 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 626 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 627 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 628 ), 629 !if(!empty(vaddrList), 630 (ins vdataClass:$vdata, 631 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 632 (ins vdataClass:$vdata, vaddrClass:$vaddr, 633 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 634 )); 635} 636 637class getMUBUFAtomicIns<int addrKind, 638 RegisterClass vdataClass, 639 bit vdata_in, 640 // Workaround bug bz30254 641 RegisterClass vdataClassCopy=vdataClass> { 642 dag ret = 643 !if(!eq(addrKind, BUFAddrKind.Offset), 644 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 645 !if(!eq(addrKind, BUFAddrKind.OffEn), 646 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 647 !if(!eq(addrKind, BUFAddrKind.IdxEn), 648 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 649 !if(!eq(addrKind, BUFAddrKind.BothEn), 650 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 651 !if(!eq(addrKind, BUFAddrKind.Addr64), 652 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 653 (ins)))))); 654} 655 656class MUBUF_Atomic_Pseudo<string opName, 657 int addrKind, 658 dag outs, 659 dag ins, 660 string asmOps, 661 list<dag> pattern=[], 662 // Workaround bug bz30254 663 int addrKindCopy = addrKind> 664 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 665 MUBUF_SetupAddr<addrKindCopy> { 666 let mayStore = 1; 667 let mayLoad = 1; 668 let hasPostISelHook = 1; 669 let hasSideEffects = 1; 670 let DisableWQM = 1; 671 let has_glc = 0; 672 let has_dlc = 0; 673 let has_tfe = 0; 674 let maybeAtomic = 1; 675} 676 677class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 678 RegisterClass vdataClass, 679 list<dag> pattern=[], 680 // Workaround bug bz30254 681 int addrKindCopy = addrKind, 682 RegisterClass vdataClassCopy = vdataClass> 683 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 684 (outs), 685 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 686 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 687 pattern>, 688 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 689 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 690 let glc_value = 0; 691 let dlc_value = 0; 692 let AsmMatchConverter = "cvtMubufAtomic"; 693} 694 695class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 696 RegisterClass vdataClass, 697 list<dag> pattern=[], 698 // Workaround bug bz30254 699 int addrKindCopy = addrKind, 700 RegisterClass vdataClassCopy = vdataClass> 701 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 702 (outs vdataClassCopy:$vdata), 703 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 704 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 705 pattern>, 706 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 707 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 708 let glc_value = 1; 709 let dlc_value = 0; 710 let Constraints = "$vdata = $vdata_in"; 711 let DisableEncoding = "$vdata_in"; 712 let AsmMatchConverter = "cvtMubufAtomicReturn"; 713} 714 715multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 716 RegisterClass vdataClass, 717 ValueType vdataType, 718 SDPatternOperator atomic, 719 bit isFP = isFloatType<vdataType>.ret> { 720 let FPAtomic = isFP in 721 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 722 MUBUFAddr64Table <0, NAME>; 723 724 let FPAtomic = isFP in 725 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 726 MUBUFAddr64Table <1, NAME>; 727 728 let FPAtomic = isFP in 729 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 730 731 let FPAtomic = isFP in 732 733 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 734 735 let FPAtomic = isFP in 736 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 737} 738 739multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 740 RegisterClass vdataClass, 741 ValueType vdataType, 742 SDPatternOperator atomic, 743 bit isFP = isFloatType<vdataType>.ret> { 744 let FPAtomic = isFP in 745 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 746 [(set vdataType:$vdata, 747 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 748 vdataType:$vdata_in))]>, 749 MUBUFAddr64Table <0, NAME # "_RTN">; 750 751 let FPAtomic = isFP in 752 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 753 [(set vdataType:$vdata, 754 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 755 vdataType:$vdata_in))]>, 756 MUBUFAddr64Table <1, NAME # "_RTN">; 757 758 let FPAtomic = isFP in 759 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 760 761 let FPAtomic = isFP in 762 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 763 764 let FPAtomic = isFP in 765 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 766} 767 768multiclass MUBUF_Pseudo_Atomics <string opName, 769 RegisterClass vdataClass, 770 ValueType vdataType, 771 SDPatternOperator atomic> : 772 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 773 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 774 775 776//===----------------------------------------------------------------------===// 777// MUBUF Instructions 778//===----------------------------------------------------------------------===// 779 780defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 781 "buffer_load_format_x", f32 782>; 783defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 784 "buffer_load_format_xy", v2f32 785>; 786defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 787 "buffer_load_format_xyz", v3f32 788>; 789defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 790 "buffer_load_format_xyzw", v4f32 791>; 792defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 793 "buffer_store_format_x", f32 794>; 795defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 796 "buffer_store_format_xy", v2f32 797>; 798defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 799 "buffer_store_format_xyz", v3f32 800>; 801defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 802 "buffer_store_format_xyzw", v4f32 803>; 804 805let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 806 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 807 "buffer_load_format_d16_x", i32 808 >; 809 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 810 "buffer_load_format_d16_xy", v2i32 811 >; 812 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 813 "buffer_load_format_d16_xyz", v3i32 814 >; 815 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 816 "buffer_load_format_d16_xyzw", v4i32 817 >; 818 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 819 "buffer_store_format_d16_x", i32 820 >; 821 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 822 "buffer_store_format_d16_xy", v2i32 823 >; 824 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 825 "buffer_store_format_d16_xyz", v3i32 826 >; 827 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 828 "buffer_store_format_d16_xyzw", v4i32 829 >; 830} // End HasUnpackedD16VMem. 831 832let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 833 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 834 "buffer_load_format_d16_x", f16 835 >; 836 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 837 "buffer_load_format_d16_xy", v2f16 838 >; 839 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 840 "buffer_load_format_d16_xyz", v3f16 841 >; 842 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 843 "buffer_load_format_d16_xyzw", v4f16 844 >; 845 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 846 "buffer_store_format_d16_x", f16 847 >; 848 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 849 "buffer_store_format_d16_xy", v2f16 850 >; 851 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 852 "buffer_store_format_d16_xyz", v3f16 853 >; 854 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 855 "buffer_store_format_d16_xyzw", v4f16 856 >; 857} // End HasPackedD16VMem. 858 859defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 860 "buffer_load_ubyte", i32 861>; 862defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 863 "buffer_load_sbyte", i32 864>; 865defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 866 "buffer_load_ushort", i32 867>; 868defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 869 "buffer_load_sshort", i32 870>; 871defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 872 "buffer_load_dword", i32 873>; 874defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 875 "buffer_load_dwordx2", v2i32 876>; 877defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 878 "buffer_load_dwordx3", v3i32 879>; 880defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 881 "buffer_load_dwordx4", v4i32 882>; 883 884defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 885defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 886defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 887defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 888defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 889defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 890defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 891defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 892defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 893defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 894 895// This is not described in AMD documentation, 896// but 'lds' versions of these opcodes are available 897// in at least GFX8+ chips. See Bug 37653. 898let SubtargetPredicate = isGFX8GFX9 in { 899defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 900 "buffer_load_dwordx2", v2i32, null_frag, 0, 1 901>; 902defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 903 "buffer_load_dwordx3", v3i32, null_frag, 0, 1 904>; 905defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 906 "buffer_load_dwordx4", v4i32, null_frag, 0, 1 907>; 908} 909 910defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 911 "buffer_store_byte", i32, truncstorei8_global 912>; 913defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 914 "buffer_store_short", i32, truncstorei16_global 915>; 916defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 917 "buffer_store_dword", i32, store_global 918>; 919defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 920 "buffer_store_dwordx2", v2i32, store_global 921>; 922defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 923 "buffer_store_dwordx3", v3i32, store_global 924>; 925defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 926 "buffer_store_dwordx4", v4i32, store_global 927>; 928defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 929 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32 930>; 931defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 932 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 933>; 934defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 935 "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32 936>; 937defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 938 "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32 939>; 940defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 941 "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32 942>; 943defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 944 "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32 945>; 946defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 947 "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32 948>; 949defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 950 "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32 951>; 952defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 953 "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32 954>; 955defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 956 "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32 957>; 958defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 959 "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32 960>; 961defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 962 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32 963>; 964defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 965 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32 966>; 967defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 968 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64 969>; 970defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 971 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 972>; 973defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 974 "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64 975>; 976defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 977 "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64 978>; 979defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 980 "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64 981>; 982defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 983 "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64 984>; 985defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 986 "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64 987>; 988defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 989 "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64 990>; 991defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 992 "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64 993>; 994defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 995 "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64 996>; 997defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 998 "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64 999>; 1000defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1001 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64 1002>; 1003defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1004 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64 1005>; 1006 1007let SubtargetPredicate = HasGFX10_BEncoding in 1008defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1009 "buffer_atomic_csub", VGPR_32, i32, atomic_csub_global_32 1010>; 1011 1012let SubtargetPredicate = isGFX8GFX9 in { 1013def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1014} 1015 1016let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1017/* 1018defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1019defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1020*/ 1021 1022def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1023 int_amdgcn_buffer_wbinvl1_sc>; 1024} 1025 1026let SubtargetPredicate = isGFX6GFX7GFX10 in { 1027 1028defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1029 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1030>; 1031defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1032 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1033>; 1034defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1035 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1036>; 1037defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1038 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1039>; 1040defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1041 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1042>; 1043defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1044 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1045>; 1046 1047} 1048 1049let SubtargetPredicate = HasD16LoadStore in { 1050 1051defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1052 "buffer_load_ubyte_d16", i32, null_frag, 1 1053>; 1054 1055defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1056 "buffer_load_ubyte_d16_hi", i32, null_frag, 1 1057>; 1058 1059defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1060 "buffer_load_sbyte_d16", i32, null_frag, 1 1061>; 1062 1063defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1064 "buffer_load_sbyte_d16_hi", i32, null_frag, 1 1065>; 1066 1067defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1068 "buffer_load_short_d16", i32, null_frag, 1 1069>; 1070 1071defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1072 "buffer_load_short_d16_hi", i32, null_frag, 1 1073>; 1074 1075defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1076 "buffer_store_byte_d16_hi", i32 1077>; 1078 1079defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1080 "buffer_store_short_d16_hi", i32 1081>; 1082 1083defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1084 "buffer_load_format_d16_hi_x", i32 1085>; 1086defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1087 "buffer_store_format_d16_hi_x", i32 1088>; 1089 1090} // End HasD16LoadStore 1091 1092def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1093 int_amdgcn_buffer_wbinvl1>; 1094 1095let SubtargetPredicate = HasAtomicFaddInsts in { 1096 1097defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1098 "buffer_atomic_add_f32", VGPR_32, f32, atomic_fadd_global_noret 1099>; 1100defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1101 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_pk_fadd_global_noret 1102>; 1103 1104} // End SubtargetPredicate = HasAtomicFaddInsts 1105 1106//===----------------------------------------------------------------------===// 1107// MTBUF Instructions 1108//===----------------------------------------------------------------------===// 1109 1110defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1111defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1112defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1113defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1114defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1115defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1116defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1117defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1118 1119let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1120 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1121 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1122 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1123 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1124 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1125 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1126 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1127 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1128} // End HasUnpackedD16VMem. 1129 1130let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1131 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1132 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1133 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1134 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1135 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1136 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1137 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1138 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1139} // End HasPackedD16VMem. 1140 1141let SubtargetPredicate = isGFX7Plus in { 1142 1143//===----------------------------------------------------------------------===// 1144// Instruction definitions for CI and newer. 1145//===----------------------------------------------------------------------===// 1146 1147def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1148 int_amdgcn_buffer_wbinvl1_vol>; 1149 1150} // End let SubtargetPredicate = isGFX7Plus 1151 1152let SubtargetPredicate = isGFX10Plus in { 1153 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1154 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1155} // End SubtargetPredicate = isGFX10Plus 1156 1157//===----------------------------------------------------------------------===// 1158// MUBUF Patterns 1159//===----------------------------------------------------------------------===// 1160 1161//===----------------------------------------------------------------------===// 1162// buffer_load/store_format patterns 1163//===----------------------------------------------------------------------===// 1164 1165multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1166 string opcode> { 1167 def : GCNPat< 1168 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1169 timm:$auxiliary, 0)), 1170 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1171 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1172 (extract_swz $auxiliary)) 1173 >; 1174 1175 def : GCNPat< 1176 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1177 timm:$auxiliary, 0)), 1178 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1179 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1180 (extract_swz $auxiliary)) 1181 >; 1182 1183 def : GCNPat< 1184 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1185 timm:$auxiliary, timm)), 1186 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1187 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1188 (extract_swz $auxiliary)) 1189 >; 1190 1191 def : GCNPat< 1192 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1193 timm:$auxiliary, timm)), 1194 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1195 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1196 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1197 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1198 (extract_swz $auxiliary)) 1199 >; 1200} 1201 1202defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1203defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1204defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1205defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1206defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1207defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1208defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1209defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1210 1211let SubtargetPredicate = HasUnpackedD16VMem in { 1212 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1213 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1214 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1215 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1216 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1217} // End HasUnpackedD16VMem. 1218 1219let SubtargetPredicate = HasPackedD16VMem in { 1220 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1221 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1222 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1223 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1224 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1225 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1226 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1227} // End HasPackedD16VMem. 1228 1229defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1230defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1231defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1232defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1233defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1234defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1235defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1236defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1237defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1238defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1239defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1240defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1241defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1242defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1243defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1244defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1245 1246multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1247 string opcode> { 1248 def : GCNPat< 1249 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1250 timm:$auxiliary, 0), 1251 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1252 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1253 (extract_swz $auxiliary)) 1254 >; 1255 1256 def : GCNPat< 1257 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1258 timm:$auxiliary, 0), 1259 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1260 (as_i16timm $offset), (extract_glc $auxiliary), 1261 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1262 (extract_swz $auxiliary)) 1263 >; 1264 1265 def : GCNPat< 1266 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1267 timm:$auxiliary, timm), 1268 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1269 (as_i16timm $offset), (extract_glc $auxiliary), 1270 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1271 (extract_swz $auxiliary)) 1272 >; 1273 1274 def : GCNPat< 1275 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1276 timm:$auxiliary, timm), 1277 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1278 getVregSrcForVT<vt>.ret:$vdata, 1279 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1280 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_glc $auxiliary), 1281 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1282 (extract_swz $auxiliary)) 1283 >; 1284} 1285 1286defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1287defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1288defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1289defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1290defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1291defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1292defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1293defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1294 1295let SubtargetPredicate = HasUnpackedD16VMem in { 1296 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1297 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1298 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1299 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1300 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1301} // End HasUnpackedD16VMem. 1302 1303let SubtargetPredicate = HasPackedD16VMem in { 1304 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1305 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1306 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1307 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1308 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1309 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1310 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1311} // End HasPackedD16VMem. 1312 1313defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1314defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1315defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1316defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1317defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1318defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1319defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1320defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1321defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1322defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1323defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1324defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1325defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1326defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1327 1328//===----------------------------------------------------------------------===// 1329// buffer_atomic patterns 1330//===----------------------------------------------------------------------===// 1331 1332multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt, 1333 string opcode> { 1334 def : GCNPat< 1335 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1336 timm:$offset, timm:$cachepolicy, 0)), 1337 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) 1338 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1339 (as_i16timm $offset), (extract_slc $cachepolicy)) 1340 >; 1341 1342 def : GCNPat< 1343 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1344 timm:$offset, timm:$cachepolicy, timm)), 1345 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1346 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1347 (as_i16timm $offset), (extract_slc $cachepolicy)) 1348 >; 1349 1350 def : GCNPat< 1351 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1352 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1353 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1354 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1355 (as_i16timm $offset), (extract_slc $cachepolicy)) 1356 >; 1357 1358 def : GCNPat< 1359 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1360 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1361 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1362 getVregSrcForVT<vt>.ret:$vdata_in, 1363 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1364 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1365 (extract_slc $cachepolicy)) 1366 >; 1367} 1368 1369defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">; 1370defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">; 1371defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">; 1372defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">; 1373defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">; 1374defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">; 1375defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">; 1376defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">; 1377defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">; 1378defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">; 1379defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">; 1380defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">; 1381defm : BufferAtomicPatterns<SIbuffer_atomic_csub, i32, "BUFFER_ATOMIC_CSUB">; 1382defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">; 1383defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64, "BUFFER_ATOMIC_ADD_X2">; 1384defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">; 1385defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">; 1386defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">; 1387defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">; 1388defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">; 1389defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">; 1390defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">; 1391defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">; 1392defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">; 1393defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">; 1394 1395multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1396 string opcode> { 1397 def : GCNPat< 1398 (name vt:$vdata_in, v4i32:$rsrc, 0, 1399 0, i32:$soffset, timm:$offset, 1400 timm:$cachepolicy, 0), 1401 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset, 1402 (as_i16imm $offset), (extract_slc $cachepolicy)) 1403 >; 1404 1405 def : GCNPat< 1406 (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1407 0, i32:$soffset, timm:$offset, 1408 timm:$cachepolicy, timm), 1409 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset, 1410 (as_i16imm $offset), (extract_slc $cachepolicy)) 1411 >; 1412 1413 def : GCNPat< 1414 (name vt:$vdata_in, v4i32:$rsrc, 0, 1415 i32:$voffset, i32:$soffset, timm:$offset, 1416 timm:$cachepolicy, 0), 1417 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset, 1418 (as_i16imm $offset), (extract_slc $cachepolicy)) 1419 >; 1420 1421 def : GCNPat< 1422 (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1423 i32:$voffset, i32:$soffset, timm:$offset, 1424 timm:$cachepolicy, timm), 1425 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1426 $vdata_in, 1427 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1428 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)) 1429 >; 1430} 1431 1432let SubtargetPredicate = HasAtomicFaddInsts in { 1433defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1434defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1435} 1436 1437def : GCNPat< 1438 (SIbuffer_atomic_cmpswap 1439 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1440 timm:$offset, timm:$cachepolicy, 0), 1441 (EXTRACT_SUBREG 1442 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1443 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1444 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1445 (extract_slc $cachepolicy)), sub0) 1446>; 1447 1448def : GCNPat< 1449 (SIbuffer_atomic_cmpswap 1450 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1451 0, i32:$soffset, timm:$offset, 1452 timm:$cachepolicy, timm), 1453 (EXTRACT_SUBREG 1454 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1455 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1456 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1457 sub0) 1458>; 1459 1460def : GCNPat< 1461 (SIbuffer_atomic_cmpswap 1462 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1463 i32:$voffset, i32:$soffset, timm:$offset, 1464 timm:$cachepolicy, 0), 1465 (EXTRACT_SUBREG 1466 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1467 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1468 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1469 sub0) 1470>; 1471 1472def : GCNPat< 1473 (SIbuffer_atomic_cmpswap 1474 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1475 i32:$voffset, i32:$soffset, timm:$offset, 1476 timm:$cachepolicy, timm), 1477 (EXTRACT_SUBREG 1478 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1479 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1480 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1481 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1482 sub0) 1483>; 1484 1485class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1486 PatFrag constant_ld> : GCNPat < 1487 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1488 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1489 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1490 >; 1491 1492multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1493 ValueType vt, PatFrag atomic_ld> { 1494 def : GCNPat < 1495 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1496 i16:$offset, i1:$slc))), 1497 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0) 1498 >; 1499 1500 def : GCNPat < 1501 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1502 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0) 1503 >; 1504} 1505 1506let SubtargetPredicate = isGFX6GFX7 in { 1507def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1508def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1509def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1510def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1511def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1512def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1513 1514defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1515defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1516} // End SubtargetPredicate = isGFX6GFX7 1517 1518multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1519 PatFrag ld> { 1520 1521 def : GCNPat < 1522 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1523 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1524 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1525 >; 1526} 1527 1528let OtherPredicates = [Has16BitInsts] in { 1529 1530defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1531defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1532defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1533defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1534defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1535defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1536 1537defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1538 1539} // End OtherPredicates = [Has16BitInsts] 1540 1541multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1542 MUBUF_Pseudo InstrOffset, 1543 ValueType vt, PatFrag ld> { 1544 def : GCNPat < 1545 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1546 i32:$soffset, u16imm:$offset))), 1547 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1548 >; 1549 1550 def : GCNPat < 1551 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1552 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1553 >; 1554} 1555 1556// XXX - Is it possible to have a complex pattern in a PatFrag? 1557multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1558 MUBUF_Pseudo InstrOffset, 1559 ValueType vt, PatFrag ld_frag> { 1560 def : GCNPat < 1561 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1562 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1563 >; 1564 1565 def : GCNPat < 1566 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1567 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1568 >; 1569} 1570 1571defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1572defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1573defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1574defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1575defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1576defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1577defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1578defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1579defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1580defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1581 1582foreach vt = Reg32Types.types in { 1583defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1584} 1585defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1586defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1587defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1588 1589let OtherPredicates = [D16PreservesUnusedBits] in { 1590defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1591defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1592defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1593defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1594defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1595defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1596 1597defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1598defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1599defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1600defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1601defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1602defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1603} 1604 1605multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1606 ValueType vt, PatFrag atomic_st> { 1607 // Store follows atomic op convention so address is first 1608 def : GCNPat < 1609 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1610 i16:$offset, i1:$slc), vt:$val), 1611 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0) 1612 >; 1613 1614 def : GCNPat < 1615 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1616 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0) 1617 >; 1618} 1619let SubtargetPredicate = isGFX6GFX7 in { 1620defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>; 1621defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>; 1622} // End Predicates = isGFX6GFX7 1623 1624 1625multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1626 PatFrag st> { 1627 1628 def : GCNPat < 1629 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1630 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)), 1631 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1632 >; 1633} 1634 1635defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1636defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1637 1638multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1639 MUBUF_Pseudo InstrOffset, 1640 ValueType vt, PatFrag st, 1641 RegisterClass rc = VGPR_32> { 1642 def : GCNPat < 1643 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1644 i32:$soffset, u16imm:$offset)), 1645 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1646 >; 1647 1648 def : GCNPat < 1649 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1650 u16imm:$offset)), 1651 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1652 >; 1653} 1654 1655defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1656defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1657defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1658defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1659 1660foreach vt = Reg32Types.types in { 1661defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1662} 1663 1664defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1665defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1666defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1667 1668 1669let OtherPredicates = [D16PreservesUnusedBits] in { 1670 // Hiding the extract high pattern in the PatFrag seems to not 1671 // automatically increase the complexity. 1672let AddedComplexity = 1 in { 1673defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1674defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1675} 1676} 1677 1678//===----------------------------------------------------------------------===// 1679// MTBUF Patterns 1680//===----------------------------------------------------------------------===// 1681 1682//===----------------------------------------------------------------------===// 1683// tbuffer_load/store_format patterns 1684//===----------------------------------------------------------------------===// 1685 1686multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1687 string opcode> { 1688 def : GCNPat< 1689 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1690 timm:$format, timm:$auxiliary, 0)), 1691 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1692 (as_i8timm $format), 1693 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1694 (extract_swz $auxiliary)) 1695 >; 1696 1697 def : GCNPat< 1698 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1699 timm:$format, timm:$auxiliary, timm)), 1700 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1701 (as_i8timm $format), 1702 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1703 (extract_swz $auxiliary)) 1704 >; 1705 1706 def : GCNPat< 1707 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1708 timm:$format, timm:$auxiliary, 0)), 1709 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1710 (as_i8timm $format), 1711 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1712 (extract_swz $auxiliary)) 1713 >; 1714 1715 def : GCNPat< 1716 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1717 timm:$format, timm:$auxiliary, timm)), 1718 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1719 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1720 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1721 (as_i8timm $format), 1722 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1723 (extract_swz $auxiliary)) 1724 >; 1725} 1726 1727defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1728defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1729defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1730defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1731defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1732defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1733defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1734defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1735 1736let SubtargetPredicate = HasUnpackedD16VMem in { 1737 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1738 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1739 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1740 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1741} // End HasUnpackedD16VMem. 1742 1743let SubtargetPredicate = HasPackedD16VMem in { 1744 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1745 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1746 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1747 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1748} // End HasPackedD16VMem. 1749 1750multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1751 string opcode> { 1752 def : GCNPat< 1753 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1754 timm:$format, timm:$auxiliary, 0), 1755 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1756 (as_i16timm $offset), (as_i8timm $format), 1757 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1758 (extract_swz $auxiliary)) 1759 >; 1760 1761 def : GCNPat< 1762 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1763 timm:$format, timm:$auxiliary, timm), 1764 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1765 (as_i16timm $offset), (as_i8timm $format), 1766 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1767 (extract_swz $auxiliary)) 1768 >; 1769 1770 def : GCNPat< 1771 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1772 timm:$format, timm:$auxiliary, 0), 1773 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1774 (as_i16timm $offset), (as_i8timm $format), 1775 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1776 (extract_swz $auxiliary)) 1777 >; 1778 1779 def : GCNPat< 1780 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1781 timm:$offset, timm:$format, timm:$auxiliary, timm), 1782 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1783 getVregSrcForVT<vt>.ret:$vdata, 1784 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1785 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 1786 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1787 (extract_swz $auxiliary)) 1788 >; 1789} 1790 1791defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1792defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1793defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1794defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1795defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1796defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1797defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1798defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1799 1800let SubtargetPredicate = HasUnpackedD16VMem in { 1801 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1802 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1803 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1804 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1805} // End HasUnpackedD16VMem. 1806 1807let SubtargetPredicate = HasPackedD16VMem in { 1808 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1809 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 1810 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1811 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1812} // End HasPackedD16VMem. 1813 1814//===----------------------------------------------------------------------===// 1815// Target-specific instruction encodings. 1816//===----------------------------------------------------------------------===// 1817 1818//===----------------------------------------------------------------------===// 1819// Base ENC_MUBUF for GFX6, GFX7, GFX10. 1820//===----------------------------------------------------------------------===// 1821 1822class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 1823 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 1824 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1825 let Inst{12} = ps.offen; 1826 let Inst{13} = ps.idxen; 1827 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1828 let Inst{16} = !if(ps.lds, 1, 0); 1829 let Inst{24-18} = op; 1830 let Inst{31-26} = 0x38; 1831 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1832 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1833 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1834 let Inst{54} = !if(ps.has_slc, slc, ?); 1835 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1836 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1837} 1838 1839class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 1840 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 1841 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 1842 let Inst{25} = op{7}; 1843} 1844 1845class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 1846 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 1847 let Inst{15} = ps.addr64; 1848} 1849 1850//===----------------------------------------------------------------------===// 1851// MUBUF - GFX10. 1852//===----------------------------------------------------------------------===// 1853 1854let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 1855 multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName, 1856 string asmName> { 1857 def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> { 1858 MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName); 1859 let AsmString = asmName # ps.AsmOperands; 1860 } 1861 } 1862 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 1863 def _BOTHEN_gfx10 : 1864 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1865 def _IDXEN_gfx10 : 1866 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1867 def _OFFEN_gfx10 : 1868 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1869 def _OFFSET_gfx10 : 1870 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1871 } 1872 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 1873 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1874 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 1875 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1876 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 1877 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1878 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 1879 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1880 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 1881 1882 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1883 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 1884 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1885 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 1886 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1887 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 1888 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1889 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 1890 } 1891 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 1892 def _BOTHEN_RTN_gfx10 : 1893 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1894 def _IDXEN_RTN_gfx10 : 1895 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1896 def _OFFEN_RTN_gfx10 : 1897 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1898 def _OFFSET_RTN_gfx10 : 1899 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1900 } 1901 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 1902 MUBUF_Real_AllAddr_gfx10<op>, MUBUF_Real_Atomics_RTN_gfx10<op>; 1903} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 1904 1905defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 1906defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 1907defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 1908defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 1909defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 1910defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 1911defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 1912defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 1913// FIXME-GFX10: Add following instructions: 1914//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 1915//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 1916defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 1917defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 1918defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 1919defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 1920defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 1921defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 1922defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 1923defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 1924 1925def BUFFER_GL0_INV_gfx10 : 1926 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 1927def BUFFER_GL1_INV_gfx10 : 1928 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 1929 1930//===----------------------------------------------------------------------===// 1931// MUBUF - GFX6, GFX7, GFX10. 1932//===----------------------------------------------------------------------===// 1933 1934let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 1935 multiclass MUBUF_Real_gfx6<bits<8> op> { 1936 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1937 } 1938} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 1939 1940let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 1941 multiclass MUBUF_Real_gfx7<bits<8> op> { 1942 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1943 } 1944} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 1945 1946let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 1947 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 1948 def _ADDR64_gfx6_gfx7 : 1949 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1950 def _BOTHEN_gfx6_gfx7 : 1951 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1952 def _IDXEN_gfx6_gfx7 : 1953 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1954 def _OFFEN_gfx6_gfx7 : 1955 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1956 def _OFFSET_gfx6_gfx7 : 1957 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1958 } 1959 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 1960 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1961 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 1962 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 1963 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 1964 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1965 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 1966 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1967 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 1968 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1969 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 1970 1971 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1972 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 1973 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 1974 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 1975 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1976 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 1977 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1978 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 1979 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1980 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 1981 } 1982 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> : 1983 MUBUF_Real_AllAddr_gfx6_gfx7<op> { 1984 def _ADDR64_RTN_gfx6_gfx7 : 1985 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1986 def _BOTHEN_RTN_gfx6_gfx7 : 1987 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1988 def _IDXEN_RTN_gfx6_gfx7 : 1989 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1990 def _OFFEN_RTN_gfx6_gfx7 : 1991 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1992 def _OFFSET_RTN_gfx6_gfx7 : 1993 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1994 } 1995} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 1996 1997multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 1998 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 1999 2000multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2001 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2002 2003multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2004 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2005 2006// FIXME-GFX6: Following instructions are available only on GFX6. 2007//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2008//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2009 2010defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2011defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2012defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2013defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2014defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2015defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2016defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2017defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2018defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2019defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2020defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2021defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2022defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2023defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2024defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2025defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2026defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2027defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2028defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2029defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2030defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2031defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2032 2033defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2034defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2035defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2036defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2037defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2038defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2039defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2040defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2041defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2042defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2043defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2044defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2045defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2046defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2047defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2048defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2049defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2050defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2051defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2052defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2053defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2054defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2055defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2056defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2057defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2058defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2059defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2060defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2061defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2062// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2063defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2064defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2065defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2066 2067defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2068 2069defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2070defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2071def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2072 2073//===----------------------------------------------------------------------===// 2074// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2075//===----------------------------------------------------------------------===// 2076 2077class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2078 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2079 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2080 let Inst{12} = ps.offen; 2081 let Inst{13} = ps.idxen; 2082 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2083 let Inst{18-16} = op; 2084 let Inst{31-26} = 0x3a; //encoding 2085 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2086 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2087 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2088 let Inst{54} = !if(ps.has_slc, slc, ?); 2089 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2090 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2091} 2092 2093//===----------------------------------------------------------------------===// 2094// MTBUF - GFX10. 2095//===----------------------------------------------------------------------===// 2096 2097class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2098 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2099 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 2100 let Inst{25-19} = format; 2101 let Inst{53} = op{3}; 2102} 2103 2104let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2105 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2106 def _BOTHEN_gfx10 : 2107 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2108 def _IDXEN_gfx10 : 2109 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2110 def _OFFEN_gfx10 : 2111 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2112 def _OFFSET_gfx10 : 2113 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2114 } 2115} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2116 2117defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2118defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2119defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2120defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2121defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2122defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2123defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2124defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2125 2126//===----------------------------------------------------------------------===// 2127// MTBUF - GFX6, GFX7, GFX10. 2128//===----------------------------------------------------------------------===// 2129 2130class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2131 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2132 let Inst{15} = ps.addr64; 2133 let Inst{22-19} = dfmt; 2134 let Inst{25-23} = nfmt; 2135} 2136 2137let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2138 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2139 def _ADDR64_gfx6_gfx7 : 2140 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2141 def _BOTHEN_gfx6_gfx7 : 2142 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2143 def _IDXEN_gfx6_gfx7 : 2144 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2145 def _OFFEN_gfx6_gfx7 : 2146 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2147 def _OFFSET_gfx6_gfx7 : 2148 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2149 } 2150} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2151 2152multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2153 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2154 2155defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2156defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2157defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2158defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2159defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2160defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2161defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2162defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2163 2164//===----------------------------------------------------------------------===// 2165// GFX8, GFX9 (VI). 2166//===----------------------------------------------------------------------===// 2167 2168class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 2169 MUBUF_Real<ps>, 2170 Enc64, 2171 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2172 let AssemblerPredicate = isGFX8GFX9; 2173 let DecoderNamespace = "GFX8"; 2174 2175 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2176 let Inst{12} = ps.offen; 2177 let Inst{13} = ps.idxen; 2178 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2179 let Inst{16} = !if(ps.lds, 1, 0); 2180 let Inst{17} = !if(ps.has_slc, slc, ?); 2181 let Inst{24-18} = op; 2182 let Inst{31-26} = 0x38; //encoding 2183 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2184 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2185 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2186 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2187 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2188} 2189 2190multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2191 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2192 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2193 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2194 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2195} 2196 2197multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2198 2199 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2200 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2201 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2202 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2203 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2204 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2205 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2206 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2207 2208 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2209 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2210 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2211 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2212 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2213 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2214 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2215 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2216} 2217 2218class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2219 MUBUF_Real<ps>, 2220 Enc64, 2221 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2222 let AssemblerPredicate=HasUnpackedD16VMem; 2223 let DecoderNamespace="GFX80_UNPACKED"; 2224 2225 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2226 let Inst{12} = ps.offen; 2227 let Inst{13} = ps.idxen; 2228 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2229 let Inst{16} = !if(ps.lds, 1, 0); 2230 let Inst{17} = !if(ps.has_slc, slc, ?); 2231 let Inst{24-18} = op; 2232 let Inst{31-26} = 0x38; //encoding 2233 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2234 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2235 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2236 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2237 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2238} 2239 2240multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2241 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2242 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2243 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2244 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2245} 2246 2247multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2248 MUBUF_Real_AllAddr_vi<op> { 2249 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2250 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2251 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2252 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2253} 2254 2255defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2256defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2257defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2258defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2259defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2260defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2261defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2262defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2263let SubtargetPredicate = HasUnpackedD16VMem in { 2264 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2265 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2266 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2267 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2268 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2269 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2270 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2271 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2272} // End HasUnpackedD16VMem. 2273let SubtargetPredicate = HasPackedD16VMem in { 2274 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2275 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2276 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2277 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2278 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2279 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2280 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2281 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2282} // End HasPackedD16VMem. 2283defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2284defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2285defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2286defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2287defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2288defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2289defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2290defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2291defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2292defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2293defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2294defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2295defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2296defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2297defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2298defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2299 2300defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2301defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2302defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2303defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2304defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2305defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2306 2307defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2308defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2309 2310defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2311defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2312defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2313defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2314defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2315defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2316defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2317defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2318defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2319defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2320defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2321defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2322defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2323 2324defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2325defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2326defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2327defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2328defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2329defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2330defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2331defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2332defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2333defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2334defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2335defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2336defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2337 2338def BUFFER_STORE_LDS_DWORD_vi : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>; 2339 2340def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2341def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2342 2343let SubtargetPredicate = HasAtomicFaddInsts in { 2344 2345defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_AllAddr_vi <0x4d>; 2346defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>; 2347 2348} // End SubtargetPredicate = HasAtomicFaddInsts 2349 2350class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2351 MTBUF_Real<ps>, 2352 Enc64, 2353 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2354 let AssemblerPredicate = isGFX8GFX9; 2355 let DecoderNamespace = "GFX8"; 2356 2357 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2358 let Inst{12} = ps.offen; 2359 let Inst{13} = ps.idxen; 2360 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2361 let Inst{18-15} = op; 2362 let Inst{22-19} = dfmt; 2363 let Inst{25-23} = nfmt; 2364 let Inst{31-26} = 0x3a; //encoding 2365 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2366 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2367 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2368 let Inst{54} = !if(ps.has_slc, slc, ?); 2369 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2370 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2371} 2372 2373multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2374 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2375 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2376 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2377 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2378} 2379 2380class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2381 MTBUF_Real<ps>, 2382 Enc64, 2383 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2384 let AssemblerPredicate=HasUnpackedD16VMem; 2385 let DecoderNamespace="GFX80_UNPACKED"; 2386 2387 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2388 let Inst{12} = ps.offen; 2389 let Inst{13} = ps.idxen; 2390 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2391 let Inst{18-15} = op; 2392 let Inst{22-19} = dfmt; 2393 let Inst{25-23} = nfmt; 2394 let Inst{31-26} = 0x3a; //encoding 2395 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2396 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2397 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2398 let Inst{54} = !if(ps.has_slc, slc, ?); 2399 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2400 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2401} 2402 2403multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2404 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2405 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2406 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2407 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2408} 2409 2410defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2411defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2412defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2413defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2414defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2415defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2416defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2417defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2418let SubtargetPredicate = HasUnpackedD16VMem in { 2419 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2420 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2421 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2422 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2423 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2424 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2425 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2426 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2427} // End HasUnpackedD16VMem. 2428let SubtargetPredicate = HasPackedD16VMem in { 2429 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2430 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2431 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2432 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2433 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2434 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2435 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2436 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2437} // End HasUnpackedD16VMem. 2438 2439def MUBUFInfoTable : GenericTable { 2440 let FilterClass = "MUBUF_Pseudo"; 2441 let CppTypeName = "MUBUFInfo"; 2442 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2443 2444 let PrimaryKey = ["Opcode"]; 2445 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2446} 2447 2448def getMUBUFInfoFromOpcode : SearchIndex { 2449 let Table = MUBUFInfoTable; 2450 let Key = ["Opcode"]; 2451} 2452 2453def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2454 let Table = MUBUFInfoTable; 2455 let Key = ["BaseOpcode", "elements"]; 2456} 2457 2458def MTBUFInfoTable : GenericTable { 2459 let FilterClass = "MTBUF_Pseudo"; 2460 let CppTypeName = "MTBUFInfo"; 2461 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2462 2463 let PrimaryKey = ["Opcode"]; 2464 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2465} 2466 2467def getMTBUFInfoFromOpcode : SearchIndex { 2468 let Table = MTBUFInfoTable; 2469 let Key = ["Opcode"]; 2470} 2471 2472def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2473 let Table = MTBUFInfoTable; 2474 let Key = ["BaseOpcode", "elements"]; 2475} 2476