1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<i64, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFAddrKind { 16 int Offset = 0; 17 int OffEn = 1; 18 int IdxEn = 2; 19 int BothEn = 3; 20 int Addr64 = 4; 21} 22 23class getAddrName<int addrKind> { 24 string ret = 25 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 26 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 27 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 28 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 29 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 30 ""))))); 31} 32 33class MUBUFAddr64Table <bit is_addr64, string Name> { 34 bit IsAddr64 = is_addr64; 35 string OpName = Name; 36} 37 38class MUBUFLdsTable <bit is_lds, string Name> { 39 bit IsLds = is_lds; 40 string OpName = Name; 41} 42 43class MTBUFAddr64Table <bit is_addr64, string Name> { 44 bit IsAddr64 = is_addr64; 45 string OpName = Name; 46} 47 48//===----------------------------------------------------------------------===// 49// MTBUF classes 50//===----------------------------------------------------------------------===// 51 52class MTBUFGetBaseOpcode<string Op> { 53 string ret = !subst("FORMAT_XY", "FORMAT_X", 54 !subst("FORMAT_XYZ", "FORMAT_X", 55 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 56} 57 58class getMTBUFElements<string Op> { 59 int ret = 1; 60} 61 62 63class MTBUF_Pseudo <string opName, dag outs, dag ins, 64 string asmOps, list<dag> pattern=[]> : 65 InstSI<outs, ins, "", pattern>, 66 SIMCInstr<opName, SIEncodingFamily.NONE> { 67 68 let isPseudo = 1; 69 let isCodeGenOnly = 1; 70 let Size = 8; 71 let UseNamedOperandTable = 1; 72 73 string Mnemonic = opName; 74 string AsmOperands = asmOps; 75 76 Instruction Opcode = !cast<Instruction>(NAME); 77 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 78 79 let VM_CNT = 1; 80 let EXP_CNT = 1; 81 let MTBUF = 1; 82 let Uses = [EXEC]; 83 let hasSideEffects = 0; 84 let SchedRW = [WriteVMEM]; 85 86 let AsmMatchConverter = "cvtMtbuf"; 87 88 bits<1> offen = 0; 89 bits<1> idxen = 0; 90 bits<1> addr64 = 0; 91 bits<1> has_vdata = 1; 92 bits<1> has_vaddr = 1; 93 bits<1> has_glc = 1; 94 bits<1> has_dlc = 1; 95 bits<1> glc_value = 0; // the value for glc if no such operand 96 bits<1> dlc_value = 0; // the value for dlc if no such operand 97 bits<1> has_srsrc = 1; 98 bits<1> has_soffset = 1; 99 bits<1> has_offset = 1; 100 bits<1> has_slc = 1; 101 bits<1> has_tfe = 1; 102 bits<4> elements = 0; 103 bits<1> has_sccb = 1; 104 bits<1> sccb_value = 0; 105} 106 107class MTBUF_Real <MTBUF_Pseudo ps> : 108 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 109 110 let isPseudo = 0; 111 let isCodeGenOnly = 0; 112 113 let VM_CNT = 1; 114 let EXP_CNT = 1; 115 let MTBUF = 1; 116 117 // copy relevant pseudo op flags 118 let UseNamedOperandTable = ps.UseNamedOperandTable; 119 let SubtargetPredicate = ps.SubtargetPredicate; 120 let AsmMatchConverter = ps.AsmMatchConverter; 121 let Constraints = ps.Constraints; 122 let DisableEncoding = ps.DisableEncoding; 123 let TSFlags = ps.TSFlags; 124 let SchedRW = ps.SchedRW; 125 let mayLoad = ps.mayLoad; 126 let mayStore = ps.mayStore; 127 let IsAtomicRet = ps.IsAtomicRet; 128 let IsAtomicNoRet = ps.IsAtomicNoRet; 129 130 bits<12> offset; 131 bits<5> cpol; 132 bits<7> format; 133 bits<8> vaddr; 134 bits<10> vdata; 135 bits<7> srsrc; 136 bits<1> tfe; 137 bits<8> soffset; 138 139 bits<4> dfmt = format{3-0}; 140 bits<3> nfmt = format{6-4}; 141 142 // GFX90A+ only: instruction uses AccVGPR for data 143 // Bit superceedes tfe. 144 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 145} 146 147class getMTBUFInsDA<list<RegisterClass> vdataList, 148 list<RegisterClass> vaddrList=[]> { 149 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 150 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 151 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 152 dag InsNoData = !if(!empty(vaddrList), 153 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 154 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz), 155 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 156 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz) 157 ); 158 dag InsData = !if(!empty(vaddrList), 159 (ins vdata_op:$vdata, SReg_128:$srsrc, 160 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 161 TFE:$tfe, SWZ:$swz), 162 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 163 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 164 TFE:$tfe, SWZ:$swz) 165 ); 166 dag ret = !if(!empty(vdataList), InsNoData, InsData); 167} 168 169class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 170 dag ret = 171 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 172 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 173 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 174 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 175 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 176 (ins)))))); 177} 178 179class getMTBUFAsmOps<int addrKind> { 180 string Pfx = 181 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 182 !if(!eq(addrKind, BUFAddrKind.OffEn), 183 "$vaddr, $srsrc,$format $soffset offen", 184 !if(!eq(addrKind, BUFAddrKind.IdxEn), 185 "$vaddr, $srsrc,$format $soffset idxen", 186 !if(!eq(addrKind, BUFAddrKind.BothEn), 187 "$vaddr, $srsrc,$format $soffset idxen offen", 188 !if(!eq(addrKind, BUFAddrKind.Addr64), 189 "$vaddr, $srsrc,$format $soffset addr64", 190 ""))))); 191 string ret = Pfx # "$offset"; 192} 193 194class MTBUF_SetupAddr<int addrKind> { 195 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 196 !eq(addrKind, BUFAddrKind.BothEn)); 197 198 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 199 !eq(addrKind, BUFAddrKind.BothEn)); 200 201 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 202 203 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 204} 205 206class MTBUF_Load_Pseudo <string opName, 207 int addrKind, 208 RegisterClass vdataClass, 209 int elems, 210 list<dag> pattern=[], 211 // Workaround bug bz30254 212 int addrKindCopy = addrKind> 213 : MTBUF_Pseudo<opName, 214 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 215 getMTBUFIns<addrKindCopy>.ret, 216 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 217 pattern>, 218 MTBUF_SetupAddr<addrKindCopy> { 219 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 220 let mayLoad = 1; 221 let mayStore = 0; 222 let elements = elems; 223} 224 225multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 226 int elems, ValueType load_vt = i32, 227 SDPatternOperator ld = null_frag> { 228 229 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 230 MTBUFAddr64Table<0, NAME>; 231 232 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 233 MTBUFAddr64Table<1, NAME>; 234 235 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 236 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 237 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 238 239 let DisableWQM = 1 in { 240 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 241 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 242 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 243 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 244 } 245} 246 247class MTBUF_Store_Pseudo <string opName, 248 int addrKind, 249 RegisterClass vdataClass, 250 int elems, 251 list<dag> pattern=[], 252 // Workaround bug bz30254 253 int addrKindCopy = addrKind, 254 RegisterClass vdataClassCopy = vdataClass> 255 : MTBUF_Pseudo<opName, 256 (outs), 257 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 258 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 259 pattern>, 260 MTBUF_SetupAddr<addrKindCopy> { 261 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 262 let mayLoad = 0; 263 let mayStore = 1; 264 let elements = elems; 265} 266 267multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 268 int elems, ValueType store_vt = i32, 269 SDPatternOperator st = null_frag> { 270 271 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 272 MTBUFAddr64Table<0, NAME>; 273 274 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 275 MTBUFAddr64Table<1, NAME>; 276 277 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 278 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 279 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 280 281 let DisableWQM = 1 in { 282 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 283 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 284 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 285 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 286 } 287} 288 289 290//===----------------------------------------------------------------------===// 291// MUBUF classes 292//===----------------------------------------------------------------------===// 293 294class MUBUFGetBaseOpcode<string Op> { 295 string ret = !subst("DWORDX2", "DWORD", 296 !subst("DWORDX3", "DWORD", 297 !subst("DWORDX4", "DWORD", Op))); 298} 299 300class MUBUF_Pseudo <string opName, dag outs, dag ins, 301 string asmOps, list<dag> pattern=[]> : 302 InstSI<outs, ins, "", pattern>, 303 SIMCInstr<opName, SIEncodingFamily.NONE> { 304 305 let isPseudo = 1; 306 let isCodeGenOnly = 1; 307 let Size = 8; 308 let UseNamedOperandTable = 1; 309 310 string Mnemonic = opName; 311 string AsmOperands = asmOps; 312 313 Instruction Opcode = !cast<Instruction>(NAME); 314 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 315 316 let VM_CNT = 1; 317 let EXP_CNT = 1; 318 let MUBUF = 1; 319 let Uses = [EXEC]; 320 let hasSideEffects = 0; 321 let SchedRW = [WriteVMEM]; 322 323 let AsmMatchConverter = "cvtMubuf"; 324 325 bits<1> offen = 0; 326 bits<1> idxen = 0; 327 bits<1> addr64 = 0; 328 bits<1> lds = 0; 329 bits<1> has_vdata = 1; 330 bits<1> has_vaddr = 1; 331 bits<1> has_glc = 1; 332 bits<1> has_dlc = 1; 333 bits<1> glc_value = 0; // the value for glc if no such operand 334 bits<1> dlc_value = 0; // the value for dlc if no such operand 335 bits<1> has_srsrc = 1; 336 bits<1> has_soffset = 1; 337 bits<1> has_offset = 1; 338 bits<1> has_slc = 1; 339 bits<1> has_tfe = 1; 340 bits<4> elements = 0; 341 bits<1> has_sccb = 1; 342 bits<1> sccb_value = 0; 343 bits<1> IsBufferInv = 0; 344} 345 346class MUBUF_Real <MUBUF_Pseudo ps> : 347 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 348 349 let isPseudo = 0; 350 let isCodeGenOnly = 0; 351 352 let VM_CNT = 1; 353 let EXP_CNT = 1; 354 let MUBUF = 1; 355 356 // copy relevant pseudo op flags 357 let SubtargetPredicate = ps.SubtargetPredicate; 358 let AsmMatchConverter = ps.AsmMatchConverter; 359 let OtherPredicates = ps.OtherPredicates; 360 let Constraints = ps.Constraints; 361 let DisableEncoding = ps.DisableEncoding; 362 let TSFlags = ps.TSFlags; 363 let UseNamedOperandTable = ps.UseNamedOperandTable; 364 let SchedRW = ps.SchedRW; 365 let mayLoad = ps.mayLoad; 366 let mayStore = ps.mayStore; 367 let IsAtomicRet = ps.IsAtomicRet; 368 let IsAtomicNoRet = ps.IsAtomicNoRet; 369 370 bits<12> offset; 371 bits<5> cpol; 372 bits<8> vaddr; 373 bits<10> vdata; 374 bits<7> srsrc; 375 bits<1> tfe; 376 bits<8> soffset; 377 378 // GFX90A+ only: instruction uses AccVGPR for data 379 // Bit superceedes tfe. 380 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 381} 382 383 384// For cache invalidation instructions. 385class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 386 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 387 388 let AsmMatchConverter = ""; 389 390 let hasSideEffects = 1; 391 let mayLoad = 0; 392 let mayStore = 0; 393 394 let IsBufferInv = 1; 395 // Set everything else to 0. 396 let offen = 0; 397 let idxen = 0; 398 let addr64 = 0; 399 let has_vdata = 0; 400 let has_vaddr = 0; 401 let has_glc = 0; 402 let has_dlc = 0; 403 let glc_value = 0; 404 let dlc_value = 0; 405 let has_srsrc = 0; 406 let has_soffset = 0; 407 let has_offset = 0; 408 let has_slc = 0; 409 let has_tfe = 0; 410 let has_sccb = 0; 411 let sccb_value = 0; 412} 413 414class getMUBUFInsDA<list<RegisterClass> vdataList, 415 list<RegisterClass> vaddrList=[], 416 bit isLds = 0> { 417 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 418 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 419 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 420 dag InsNoData = !if(!empty(vaddrList), 421 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 422 offset:$offset, CPol_0:$cpol), 423 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 424 offset:$offset, CPol_0:$cpol) 425 ); 426 dag InsData = !if(!empty(vaddrList), 427 (ins vdata_op:$vdata, SReg_128:$srsrc, 428 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 429 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 430 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 431 ); 432 dag ret = !con( 433 !if(!empty(vdataList), InsNoData, InsData), 434 !if(isLds, (ins SWZ_0:$swz), (ins TFE_0:$tfe, SWZ_0:$swz)) 435 ); 436} 437 438class getMUBUFElements<ValueType vt> { 439 int ret = 440 !if(!eq(vt, f16), 1, 441 !if(!eq(vt, v2f16), 2, 442 !if(!eq(vt, v3f16), 3, 443 !if(!eq(vt, v4f16), 4, 444 !if(!eq(vt.Size, 32), 1, 445 !if(!eq(vt.Size, 64), 2, 446 !if(!eq(vt.Size, 96), 3, 447 !if(!eq(vt.Size, 128), 4, 0) 448 ) 449 ) 450 ) 451 ) 452 ) 453 ) 454 ); 455} 456 457class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 458 dag ret = 459 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 460 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 461 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 462 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 463 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 464 (ins)))))); 465} 466 467class getMUBUFAsmOps<int addrKind> { 468 string Pfx = 469 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 470 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 471 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 472 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 473 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 474 ""))))); 475 string ret = Pfx # "$offset"; 476} 477 478class MUBUF_SetupAddr<int addrKind> { 479 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 480 !eq(addrKind, BUFAddrKind.BothEn)); 481 482 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 483 !eq(addrKind, BUFAddrKind.BothEn)); 484 485 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 486 487 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 488} 489 490class MUBUF_Load_Pseudo <string opName, 491 int addrKind, 492 ValueType vdata_vt, 493 bit HasTiedDest = 0, 494 bit isLds = 0, 495 list<dag> pattern=[], 496 // Workaround bug bz30254 497 int addrKindCopy = addrKind, 498 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 499 RegisterOperand vdata_op = getLdStRegisterOperand<vdata_rc>.ret> 500 : MUBUF_Pseudo<opName, 501 (outs vdata_op:$vdata), 502 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 503 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 504 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol" # 505 !if(isLds, " lds", "$tfe") # "$swz", 506 pattern>, 507 MUBUF_SetupAddr<addrKindCopy> { 508 let PseudoInstr = opName # !if(isLds, "_lds", "") # 509 "_" # getAddrName<addrKindCopy>.ret; 510 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 511 512 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 513 let mayLoad = 1; 514 let mayStore = 0; 515 let maybeAtomic = 1; 516 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 517 let has_tfe = !not(isLds); 518 let lds = isLds; 519 let elements = getMUBUFElements<vdata_vt>.ret; 520} 521 522class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 523 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 524 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset)) 525>; 526 527class MUBUF_Addr64_Load_Pat <Instruction inst, 528 ValueType load_vt = i32, 529 SDPatternOperator ld = null_frag> : Pat < 530 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 531 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset)) 532>; 533 534multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 535 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 536 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 537} 538 539 540// FIXME: tfe can't be an operand because it requires a separate 541// opcode because it needs an N+1 register class dest register. 542multiclass MUBUF_Pseudo_Loads<string opName, 543 ValueType load_vt = i32, 544 SDPatternOperator ld = null_frag, 545 bit TiedDest = 0, 546 bit isLds = 0> { 547 548 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 549 550 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>, 551 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 552 553 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>, 554 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 555 556 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 557 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 558 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 559 560 let DisableWQM = 1 in { 561 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>; 562 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 563 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 564 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 565 } 566} 567 568multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32, 569 SDPatternOperator ld_nolds = null_frag, 570 SDPatternOperator ld_lds = null_frag> { 571 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>; 572 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>; 573} 574 575class MUBUF_Store_Pseudo <string opName, 576 int addrKind, 577 ValueType store_vt, 578 list<dag> pattern=[], 579 // Workaround bug bz30254 580 int addrKindCopy = addrKind> 581 : MUBUF_Pseudo<opName, 582 (outs), 583 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 584 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 585 pattern>, 586 MUBUF_SetupAddr<addrKindCopy> { 587 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 588 let mayLoad = 0; 589 let mayStore = 1; 590 let maybeAtomic = 1; 591 let elements = getMUBUFElements<store_vt>.ret; 592} 593 594multiclass MUBUF_Pseudo_Stores<string opName, 595 ValueType store_vt = i32, 596 SDPatternOperator st = null_frag> { 597 598 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 599 600 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, 601 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 602 i16:$offset))]>, 603 MUBUFAddr64Table<0, NAME>; 604 605 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, 606 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 607 i16:$offset))]>, 608 MUBUFAddr64Table<1, NAME>; 609 610 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 611 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 612 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 613 614 let DisableWQM = 1 in { 615 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>; 616 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 617 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 618 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 619 } 620} 621 622class MUBUF_Pseudo_Store_Lds<string opName> 623 : MUBUF_Pseudo<opName, 624 (outs), 625 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz), 626 " $srsrc, $soffset$offset lds$cpol$swz"> { 627 let mayLoad = 0; 628 let mayStore = 1; 629 let maybeAtomic = 1; 630 631 let has_vdata = 0; 632 let has_vaddr = 0; 633 let has_tfe = 0; 634 let lds = 1; 635 636 let Uses = [EXEC, M0]; 637 let AsmMatchConverter = "cvtMubufLds"; 638} 639 640class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 641 list<RegisterClass> vaddrList=[]> { 642 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 643 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 644 dag ret = !if(vdata_in, 645 !if(!empty(vaddrList), 646 (ins vdata_op:$vdata_in, 647 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol), 648 (ins vdata_op:$vdata_in, vaddrClass:$vaddr, 649 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol) 650 ), 651 !if(!empty(vaddrList), 652 (ins vdata_op:$vdata, 653 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 654 (ins vdata_op:$vdata, vaddrClass:$vaddr, 655 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 656 )); 657} 658 659class getMUBUFAtomicIns<int addrKind, 660 RegisterClass vdataClass, 661 bit vdata_in, 662 // Workaround bug bz30254 663 RegisterClass vdataClassCopy=vdataClass> { 664 dag ret = 665 !if(!eq(addrKind, BUFAddrKind.Offset), 666 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 667 !if(!eq(addrKind, BUFAddrKind.OffEn), 668 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 669 !if(!eq(addrKind, BUFAddrKind.IdxEn), 670 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 671 !if(!eq(addrKind, BUFAddrKind.BothEn), 672 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 673 !if(!eq(addrKind, BUFAddrKind.Addr64), 674 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 675 (ins)))))); 676} 677 678class MUBUF_Atomic_Pseudo<string opName, 679 int addrKind, 680 dag outs, 681 dag ins, 682 string asmOps, 683 list<dag> pattern=[], 684 // Workaround bug bz30254 685 int addrKindCopy = addrKind> 686 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 687 MUBUF_SetupAddr<addrKindCopy> { 688 let mayStore = 1; 689 let mayLoad = 1; 690 let hasPostISelHook = 1; 691 let hasSideEffects = 1; 692 let DisableWQM = 1; 693 let has_glc = 0; 694 let has_dlc = 0; 695 let has_tfe = 0; 696 let has_sccb = 1; 697 let maybeAtomic = 1; 698 let AsmMatchConverter = "cvtMubufAtomic"; 699} 700 701class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 702 RegisterClass vdataClass, 703 list<dag> pattern=[], 704 // Workaround bug bz30254 705 int addrKindCopy = addrKind, 706 RegisterClass vdataClassCopy = vdataClass> 707 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 708 (outs), 709 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 710 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 711 pattern>, 712 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 713 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 714 let glc_value = 0; 715 let dlc_value = 0; 716 let sccb_value = 0; 717 let IsAtomicNoRet = 1; 718} 719 720class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 721 RegisterClass vdataClass, 722 list<dag> pattern=[], 723 // Workaround bug bz30254 724 int addrKindCopy = addrKind, 725 RegisterClass vdataClassCopy = vdataClass, 726 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 727 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 728 (outs vdata_op:$vdata), 729 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 730 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 731 pattern>, 732 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 733 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 734 let glc_value = 1; 735 let dlc_value = 0; 736 let sccb_value = 0; 737 let IsAtomicRet = 1; 738 let Constraints = "$vdata = $vdata_in"; 739 let DisableEncoding = "$vdata_in"; 740} 741 742multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 743 RegisterClass vdataClass, 744 ValueType vdataType, 745 SDPatternOperator atomic, 746 bit isFP = isFloatType<vdataType>.ret> { 747 let FPAtomic = isFP in 748 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 749 MUBUFAddr64Table <0, NAME>; 750 751 let FPAtomic = isFP in 752 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 753 MUBUFAddr64Table <1, NAME>; 754 755 let FPAtomic = isFP in 756 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 757 758 let FPAtomic = isFP in 759 760 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 761 762 let FPAtomic = isFP in 763 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 764} 765 766multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 767 RegisterClass vdataClass, 768 ValueType vdataType, 769 SDPatternOperator atomic, 770 bit isFP = isFloatType<vdataType>.ret> { 771 let FPAtomic = isFP in 772 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 773 [(set vdataType:$vdata, 774 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), 775 vdataType:$vdata_in))]>, 776 MUBUFAddr64Table <0, NAME # "_RTN">; 777 778 let FPAtomic = isFP in 779 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 780 [(set vdataType:$vdata, 781 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 782 vdataType:$vdata_in))]>, 783 MUBUFAddr64Table <1, NAME # "_RTN">; 784 785 let FPAtomic = isFP in 786 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 787 788 let FPAtomic = isFP in 789 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 790 791 let FPAtomic = isFP in 792 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 793} 794 795multiclass MUBUF_Pseudo_Atomics <string opName, 796 RegisterClass vdataClass, 797 ValueType vdataType, 798 SDPatternOperator atomic> : 799 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 800 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 801 802 803//===----------------------------------------------------------------------===// 804// MUBUF Instructions 805//===----------------------------------------------------------------------===// 806 807defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 808 "buffer_load_format_x", f32 809>; 810defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 811 "buffer_load_format_xy", v2f32 812>; 813defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 814 "buffer_load_format_xyz", v3f32 815>; 816defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 817 "buffer_load_format_xyzw", v4f32 818>; 819defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 820 "buffer_store_format_x", f32 821>; 822defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 823 "buffer_store_format_xy", v2f32 824>; 825defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 826 "buffer_store_format_xyz", v3f32 827>; 828defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 829 "buffer_store_format_xyzw", v4f32 830>; 831 832let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 833 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 834 "buffer_load_format_d16_x", i32 835 >; 836 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 837 "buffer_load_format_d16_xy", v2i32 838 >; 839 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 840 "buffer_load_format_d16_xyz", v3i32 841 >; 842 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 843 "buffer_load_format_d16_xyzw", v4i32 844 >; 845 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 846 "buffer_store_format_d16_x", i32 847 >; 848 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 849 "buffer_store_format_d16_xy", v2i32 850 >; 851 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 852 "buffer_store_format_d16_xyz", v3i32 853 >; 854 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 855 "buffer_store_format_d16_xyzw", v4i32 856 >; 857} // End HasUnpackedD16VMem. 858 859let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 860 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 861 "buffer_load_format_d16_x", f16 862 >; 863 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 864 "buffer_load_format_d16_xy", v2f16 865 >; 866 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 867 "buffer_load_format_d16_xyz", v3f16 868 >; 869 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 870 "buffer_load_format_d16_xyzw", v4f16 871 >; 872 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 873 "buffer_store_format_d16_x", f16 874 >; 875 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 876 "buffer_store_format_d16_xy", v2f16 877 >; 878 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 879 "buffer_store_format_d16_xyz", v3f16 880 >; 881 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 882 "buffer_store_format_d16_xyzw", v4f16 883 >; 884} // End HasPackedD16VMem. 885 886defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 887 "buffer_load_ubyte", i32 888>; 889defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 890 "buffer_load_sbyte", i32 891>; 892defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 893 "buffer_load_ushort", i32 894>; 895defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 896 "buffer_load_sshort", i32 897>; 898defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 899 "buffer_load_dword", i32 900>; 901defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 902 "buffer_load_dwordx2", v2i32 903>; 904defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 905 "buffer_load_dwordx3", v3i32 906>; 907defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 908 "buffer_load_dwordx4", v4i32 909>; 910 911defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 912defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 913defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 914defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 915defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 916defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 917defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 918defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 919defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 920defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 921 922// This is not described in AMD documentation, 923// but 'lds' versions of these opcodes are available 924// in at least GFX8+ chips. See Bug 37653. 925let SubtargetPredicate = isGFX8GFX9 in { 926defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 927 "buffer_load_dwordx2", v2i32, null_frag, 0, 1 928>; 929defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 930 "buffer_load_dwordx3", v3i32, null_frag, 0, 1 931>; 932defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 933 "buffer_load_dwordx4", v4i32, null_frag, 0, 1 934>; 935} 936 937defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 938 "buffer_store_byte", i32, truncstorei8_global 939>; 940defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 941 "buffer_store_short", i32, truncstorei16_global 942>; 943defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 944 "buffer_store_dword", i32, store_global 945>; 946defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 947 "buffer_store_dwordx2", v2i32, store_global 948>; 949defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 950 "buffer_store_dwordx3", v3i32, store_global 951>; 952defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 953 "buffer_store_dwordx4", v4i32, store_global 954>; 955defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 956 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32 957>; 958defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 959 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 960>; 961defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 962 "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32 963>; 964defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 965 "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32 966>; 967defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 968 "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32 969>; 970defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 971 "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32 972>; 973defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 974 "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32 975>; 976defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 977 "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32 978>; 979defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 980 "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32 981>; 982defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 983 "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32 984>; 985defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 986 "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32 987>; 988defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 989 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32 990>; 991defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 992 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32 993>; 994defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 995 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64 996>; 997defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 998 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 999>; 1000defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1001 "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64 1002>; 1003defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1004 "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64 1005>; 1006defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1007 "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64 1008>; 1009defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1010 "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64 1011>; 1012defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1013 "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64 1014>; 1015defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1016 "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64 1017>; 1018defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1019 "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64 1020>; 1021defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1022 "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64 1023>; 1024defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1025 "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64 1026>; 1027defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1028 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64 1029>; 1030defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1031 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64 1032>; 1033 1034let SubtargetPredicate = HasGFX10_BEncoding in 1035defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1036 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1037>; 1038 1039let SubtargetPredicate = isGFX8GFX9 in { 1040def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1041} 1042 1043let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1044/* 1045defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1046defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1047*/ 1048 1049def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1050 int_amdgcn_buffer_wbinvl1_sc>; 1051} 1052 1053let SubtargetPredicate = isGFX6GFX7GFX10 in { 1054 1055defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1056 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1057>; 1058defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1059 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1060>; 1061defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1062 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1063>; 1064defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1065 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1066>; 1067defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1068 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1069>; 1070defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1071 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1072>; 1073 1074} 1075 1076let SubtargetPredicate = HasD16LoadStore in { 1077 1078defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1079 "buffer_load_ubyte_d16", i32, null_frag, 1 1080>; 1081 1082defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1083 "buffer_load_ubyte_d16_hi", i32, null_frag, 1 1084>; 1085 1086defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1087 "buffer_load_sbyte_d16", i32, null_frag, 1 1088>; 1089 1090defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1091 "buffer_load_sbyte_d16_hi", i32, null_frag, 1 1092>; 1093 1094defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1095 "buffer_load_short_d16", i32, null_frag, 1 1096>; 1097 1098defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1099 "buffer_load_short_d16_hi", i32, null_frag, 1 1100>; 1101 1102defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1103 "buffer_store_byte_d16_hi", i32 1104>; 1105 1106defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1107 "buffer_store_short_d16_hi", i32 1108>; 1109 1110defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1111 "buffer_load_format_d16_hi_x", i32 1112>; 1113defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1114 "buffer_store_format_d16_hi_x", i32 1115>; 1116 1117} // End HasD16LoadStore 1118 1119def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1120 int_amdgcn_buffer_wbinvl1>; 1121 1122let SubtargetPredicate = HasAtomicFaddInsts in { 1123defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1124 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_noret_32 1125>; 1126defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1127 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_noret_32 1128>; 1129 1130let OtherPredicates = [isGFX90APlus] in { 1131defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN < 1132 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_32 1133>; 1134defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1135 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_32 1136>; 1137} 1138} // End SubtargetPredicate = HasAtomicFaddInsts 1139 1140//===----------------------------------------------------------------------===// 1141// MTBUF Instructions 1142//===----------------------------------------------------------------------===// 1143 1144defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1145defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1146defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1147defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1148defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1149defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1150defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1151defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1152 1153let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1154 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1155 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1156 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1157 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1158 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1159 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1160 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1161 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1162} // End HasUnpackedD16VMem. 1163 1164let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1165 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1166 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1167 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1168 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1169 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1170 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1171 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1172 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1173} // End HasPackedD16VMem. 1174 1175let SubtargetPredicate = isGFX7Plus in { 1176 1177//===----------------------------------------------------------------------===// 1178// Instruction definitions for CI and newer. 1179//===----------------------------------------------------------------------===// 1180 1181def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1182 int_amdgcn_buffer_wbinvl1_vol>; 1183 1184} // End let SubtargetPredicate = isGFX7Plus 1185 1186let SubtargetPredicate = isGFX90APlus in { 1187 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1188 } 1189 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1190 } 1191 1192 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64, int_amdgcn_global_atomic_fadd>; 1193 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64, int_amdgcn_global_atomic_fmin>; 1194 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64, int_amdgcn_global_atomic_fmax>; 1195} // End SubtargetPredicate = isGFX90APlus 1196 1197let SubtargetPredicate = isGFX10Plus in { 1198 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1199 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1200} // End SubtargetPredicate = isGFX10Plus 1201 1202//===----------------------------------------------------------------------===// 1203// MUBUF Patterns 1204//===----------------------------------------------------------------------===// 1205 1206//===----------------------------------------------------------------------===// 1207// buffer_load/store_format patterns 1208//===----------------------------------------------------------------------===// 1209 1210multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1211 string opcode, ValueType memoryVt = vt> { 1212 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1213 1214 def : GCNPat< 1215 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1216 timm:$auxiliary, 0)), 1217 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1218 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1219 >; 1220 1221 def : GCNPat< 1222 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1223 timm:$auxiliary, 0)), 1224 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1225 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1226 >; 1227 1228 def : GCNPat< 1229 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1230 timm:$auxiliary, timm)), 1231 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1232 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1233 >; 1234 1235 def : GCNPat< 1236 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1237 timm:$auxiliary, timm)), 1238 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1239 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1240 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1241 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1242 >; 1243} 1244 1245defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1246defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1247defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1248defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1249defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1250defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1251defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1252defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1253 1254let SubtargetPredicate = HasUnpackedD16VMem in { 1255 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1256 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1257 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1258 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1259 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1260 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1261} // End HasUnpackedD16VMem. 1262 1263let SubtargetPredicate = HasPackedD16VMem in { 1264 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1265 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1266 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1267 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1268 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1269 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1270 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1271 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1272 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1273} // End HasPackedD16VMem. 1274 1275defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1276defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1277defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1278defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1279defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1280defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1281defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1282defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1283defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1284defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1285defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1286defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1287defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1288defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1289defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1290defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1291 1292multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1293 string opcode, ValueType memoryVt = vt> { 1294 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1295 1296 def : GCNPat< 1297 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1298 timm:$auxiliary, 0), 1299 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1300 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1301 >; 1302 1303 def : GCNPat< 1304 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1305 timm:$auxiliary, 0), 1306 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1307 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1308 >; 1309 1310 def : GCNPat< 1311 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1312 timm:$auxiliary, timm), 1313 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1314 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1315 >; 1316 1317 def : GCNPat< 1318 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1319 timm:$auxiliary, timm), 1320 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1321 getVregSrcForVT<vt>.ret:$vdata, 1322 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1323 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_cpol $auxiliary), 1324 0, (extract_swz $auxiliary)) 1325 >; 1326} 1327 1328defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1329defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1330defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1331defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1332defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1333defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1334defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1335defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1336 1337let SubtargetPredicate = HasUnpackedD16VMem in { 1338 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1339 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1340 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1341 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1342 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1343 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1344} // End HasUnpackedD16VMem. 1345 1346let SubtargetPredicate = HasPackedD16VMem in { 1347 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1348 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1349 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1350 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1351 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1352 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1353 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1354 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1355 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1356} // End HasPackedD16VMem. 1357 1358defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1359defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1360defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1361defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1362defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1363defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1364defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1365defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1366defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1367defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1368defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1369defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1370defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1371defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1372 1373//===----------------------------------------------------------------------===// 1374// buffer_atomic patterns 1375//===----------------------------------------------------------------------===// 1376 1377multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt, 1378 string opcode> { 1379 def : GCNPat< 1380 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1381 timm:$offset, timm:$cachepolicy, 0)), 1382 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) 1383 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1384 (as_i16timm $offset), (set_glc $cachepolicy)) 1385 >; 1386 1387 def : GCNPat< 1388 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1389 timm:$offset, timm:$cachepolicy, timm)), 1390 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1391 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1392 (as_i16timm $offset), (set_glc $cachepolicy)) 1393 >; 1394 1395 def : GCNPat< 1396 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1397 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1398 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1399 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1400 (as_i16timm $offset), (set_glc $cachepolicy)) 1401 >; 1402 1403 def : GCNPat< 1404 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1405 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1406 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1407 getVregSrcForVT<vt>.ret:$vdata_in, 1408 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1409 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1410 (set_glc $cachepolicy)) 1411 >; 1412} 1413 1414defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">; 1415defm : BufferAtomicPatterns<SIbuffer_atomic_swap, f32, "BUFFER_ATOMIC_SWAP">; 1416defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">; 1417defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">; 1418defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">; 1419defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">; 1420defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">; 1421defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">; 1422defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">; 1423defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">; 1424defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">; 1425defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">; 1426defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">; 1427defm : BufferAtomicPatterns<SIbuffer_atomic_csub, i32, "BUFFER_ATOMIC_CSUB">; 1428defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">; 1429defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64, "BUFFER_ATOMIC_ADD_X2">; 1430defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">; 1431defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">; 1432defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">; 1433defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">; 1434defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">; 1435defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">; 1436defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">; 1437defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">; 1438defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">; 1439defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">; 1440 1441class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1442 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1443 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)), 1444 [{ return SDValue(N, 0).use_empty(); }]> { 1445 1446 let GISelPredicateCode = [{ 1447 return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); 1448 }]; 1449} 1450 1451multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1452 string opcode> { 1453 def : GCNPat< 1454 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1455 0, i32:$soffset, timm:$offset, 1456 timm:$cachepolicy, 0), 1457 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1458 (as_i16timm $offset), $cachepolicy) 1459 >; 1460 1461 def : GCNPat< 1462 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1463 0, i32:$soffset, timm:$offset, 1464 timm:$cachepolicy, timm), 1465 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1466 (as_i16timm $offset), $cachepolicy) 1467 >; 1468 1469 def : GCNPat< 1470 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1471 i32:$voffset, i32:$soffset, timm:$offset, 1472 timm:$cachepolicy, 0), 1473 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1474 (as_i16timm $offset), $cachepolicy) 1475 >; 1476 1477 def : GCNPat< 1478 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1479 i32:$voffset, i32:$soffset, timm:$offset, 1480 timm:$cachepolicy, timm), 1481 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1482 getVregSrcForVT<vt>.ret:$vdata_in, 1483 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1484 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), $cachepolicy) 1485 >; 1486} 1487 1488let SubtargetPredicate = HasAtomicFaddInsts in { 1489defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1490defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1491} 1492 1493let SubtargetPredicate = isGFX90APlus in { 1494 defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1495 defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1496 1497 defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, f64, "BUFFER_ATOMIC_ADD_F64">; 1498 defm : BufferAtomicPatterns<SIbuffer_atomic_fmin, f64, "BUFFER_ATOMIC_MIN_F64">; 1499 defm : BufferAtomicPatterns<SIbuffer_atomic_fmax, f64, "BUFFER_ATOMIC_MAX_F64">; 1500} // End SubtargetPredicate = isGFX90APlus 1501 1502def : GCNPat< 1503 (SIbuffer_atomic_cmpswap 1504 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1505 timm:$offset, timm:$cachepolicy, 0), 1506 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1507 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1508 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1509 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1510 (set_glc $cachepolicy)), VReg_64)), sub0) 1511>; 1512 1513def : GCNPat< 1514 (SIbuffer_atomic_cmpswap 1515 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1516 0, i32:$soffset, timm:$offset, 1517 timm:$cachepolicy, timm), 1518 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1519 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1520 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1521 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1522 (set_glc $cachepolicy)), VReg_64)), 1523 sub0) 1524>; 1525 1526def : GCNPat< 1527 (SIbuffer_atomic_cmpswap 1528 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1529 i32:$voffset, i32:$soffset, timm:$offset, 1530 timm:$cachepolicy, 0), 1531 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1532 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1533 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1534 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1535 (set_glc $cachepolicy)), VReg_64)), 1536 sub0) 1537>; 1538 1539def : GCNPat< 1540 (SIbuffer_atomic_cmpswap 1541 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1542 i32:$voffset, i32:$soffset, timm:$offset, 1543 timm:$cachepolicy, timm), 1544 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS 1545 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1546 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1547 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1548 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1549 (set_glc $cachepolicy)), VReg_64)), 1550 sub0) 1551>; 1552 1553class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1554 PatFrag constant_ld> : GCNPat < 1555 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1556 i16:$offset))), 1557 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1558 >; 1559 1560multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1561 ValueType vt, PatFrag atomic_ld> { 1562 def : GCNPat < 1563 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 1564 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1565 >; 1566 1567 def : GCNPat < 1568 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))), 1569 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1570 >; 1571} 1572 1573let SubtargetPredicate = isGFX6GFX7 in { 1574def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1575def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1576def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1577def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1578def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1579def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1580 1581defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1582defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1583} // End SubtargetPredicate = isGFX6GFX7 1584 1585multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1586 PatFrag ld> { 1587 1588 def : GCNPat < 1589 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 1590 (Instr_OFFSET $srsrc, $soffset, $offset) 1591 >; 1592} 1593 1594let OtherPredicates = [Has16BitInsts] in { 1595 1596defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1597defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1598defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1599defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1600defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1601defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1602 1603defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1604 1605} // End OtherPredicates = [Has16BitInsts] 1606 1607multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1608 MUBUF_Pseudo InstrOffset, 1609 ValueType vt, PatFrag ld> { 1610 def : GCNPat < 1611 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1612 i32:$soffset, u16imm:$offset))), 1613 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1614 >; 1615 1616 def : GCNPat < 1617 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1618 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1619 >; 1620} 1621 1622// XXX - Is it possible to have a complex pattern in a PatFrag? 1623multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1624 MUBUF_Pseudo InstrOffset, 1625 ValueType vt, PatFrag ld_frag> { 1626 def : GCNPat < 1627 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1628 (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) 1629 >; 1630 1631 def : GCNPat < 1632 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1633 (InstrOffset $srsrc, $soffset, $offset, $in) 1634 >; 1635} 1636 1637let OtherPredicates = [DisableFlatScratch] in { 1638defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1639defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1640defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1641defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1642defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1643defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1644defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1645defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1646defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1647defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1648 1649foreach vt = Reg32Types.types in { 1650defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1651} 1652defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1653defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1654defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1655 1656let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1657defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1658defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1659defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1660defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1661defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1662defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1663 1664defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1665defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1666defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1667defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1668defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1669defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1670} 1671 1672} // End OtherPredicates = [DisableFlatScratch] 1673 1674multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1675 ValueType vt, PatFrag atomic_st> { 1676 // Store follows atomic op convention so address is first 1677 def : GCNPat < 1678 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), 1679 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 1680 >; 1681 1682 def : GCNPat < 1683 (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1684 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 1685 >; 1686} 1687let SubtargetPredicate = isGFX6GFX7 in { 1688defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>; 1689defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>; 1690} // End Predicates = isGFX6GFX7 1691 1692 1693multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1694 PatFrag st> { 1695 1696 def : GCNPat < 1697 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)), 1698 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) 1699 >; 1700} 1701 1702defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1703defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1704 1705multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1706 MUBUF_Pseudo InstrOffset, 1707 ValueType vt, PatFrag st, 1708 RegisterClass rc = VGPR_32> { 1709 def : GCNPat < 1710 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1711 i32:$soffset, u16imm:$offset)), 1712 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1713 >; 1714 1715 def : GCNPat < 1716 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1717 u16imm:$offset)), 1718 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0) 1719 >; 1720} 1721 1722let OtherPredicates = [DisableFlatScratch] in { 1723defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1724defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1725defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1726defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1727 1728foreach vt = Reg32Types.types in { 1729defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1730} 1731 1732defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1733defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1734defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1735 1736 1737let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1738 // Hiding the extract high pattern in the PatFrag seems to not 1739 // automatically increase the complexity. 1740let AddedComplexity = 1 in { 1741defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1742defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1743} 1744} 1745} // End OtherPredicates = [DisableFlatScratch] 1746 1747//===----------------------------------------------------------------------===// 1748// MTBUF Patterns 1749//===----------------------------------------------------------------------===// 1750 1751//===----------------------------------------------------------------------===// 1752// tbuffer_load/store_format patterns 1753//===----------------------------------------------------------------------===// 1754 1755multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1756 string opcode, ValueType memoryVt = vt> { 1757 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 1758 1759 def : GCNPat< 1760 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1761 timm:$format, timm:$auxiliary, 0)), 1762 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1763 (as_i8timm $format), 1764 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1765 >; 1766 1767 def : GCNPat< 1768 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1769 timm:$format, timm:$auxiliary, timm)), 1770 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1771 (as_i8timm $format), 1772 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1773 >; 1774 1775 def : GCNPat< 1776 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1777 timm:$format, timm:$auxiliary, 0)), 1778 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1779 (as_i8timm $format), 1780 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1781 >; 1782 1783 def : GCNPat< 1784 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1785 timm:$format, timm:$auxiliary, timm)), 1786 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1787 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1788 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1789 (as_i8timm $format), 1790 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1791 >; 1792} 1793 1794defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1795defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1796defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1797defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1798defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1799defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1800defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1801defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1802 1803let SubtargetPredicate = HasUnpackedD16VMem in { 1804 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1805 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1806 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1807 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1808 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1809} // End HasUnpackedD16VMem. 1810 1811let SubtargetPredicate = HasPackedD16VMem in { 1812 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1813 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1814 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1815 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1816 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1817} // End HasPackedD16VMem. 1818 1819multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1820 string opcode, ValueType memoryVt = vt> { 1821 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 1822 1823 def : GCNPat< 1824 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1825 timm:$format, timm:$auxiliary, 0), 1826 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1827 (as_i16timm $offset), (as_i8timm $format), 1828 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1829 >; 1830 1831 def : GCNPat< 1832 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1833 timm:$format, timm:$auxiliary, timm), 1834 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1835 (as_i16timm $offset), (as_i8timm $format), 1836 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1837 >; 1838 1839 def : GCNPat< 1840 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1841 timm:$format, timm:$auxiliary, 0), 1842 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1843 (as_i16timm $offset), (as_i8timm $format), 1844 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1845 >; 1846 1847 def : GCNPat< 1848 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1849 timm:$offset, timm:$format, timm:$auxiliary, timm), 1850 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1851 getVregSrcForVT<vt>.ret:$vdata, 1852 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1853 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 1854 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1855 >; 1856} 1857 1858defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1859defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1860defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1861defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1862defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1863defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1864defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1865defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1866 1867let SubtargetPredicate = HasUnpackedD16VMem in { 1868 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1869 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1870 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1871 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1872 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1873} // End HasUnpackedD16VMem. 1874 1875let SubtargetPredicate = HasPackedD16VMem in { 1876 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1877 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 1878 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1879 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1880 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1881} // End HasPackedD16VMem. 1882 1883//===----------------------------------------------------------------------===// 1884// Target-specific instruction encodings. 1885//===----------------------------------------------------------------------===// 1886 1887//===----------------------------------------------------------------------===// 1888// Base ENC_MUBUF for GFX6, GFX7, GFX10. 1889//===----------------------------------------------------------------------===// 1890 1891class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 1892 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 1893 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1894 let Inst{12} = ps.offen; 1895 let Inst{13} = ps.idxen; 1896 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 1897 let Inst{16} = ps.lds; 1898 let Inst{24-18} = op; 1899 let Inst{31-26} = 0x38; 1900 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1901 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 1902 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1903 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 1904 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1905 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1906} 1907 1908class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 1909 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 1910 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 1911 let Inst{25} = op{7}; 1912} 1913 1914class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 1915 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 1916 let Inst{15} = ps.addr64; 1917} 1918 1919//===----------------------------------------------------------------------===// 1920// MUBUF - GFX10. 1921//===----------------------------------------------------------------------===// 1922 1923let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 1924 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 1925 def _BOTHEN_gfx10 : 1926 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1927 def _IDXEN_gfx10 : 1928 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1929 def _OFFEN_gfx10 : 1930 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1931 def _OFFSET_gfx10 : 1932 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1933 } 1934 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 1935 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1936 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 1937 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1938 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 1939 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1940 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 1941 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1942 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 1943 1944 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1945 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 1946 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1947 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 1948 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1949 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 1950 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1951 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 1952 } 1953 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 1954 def _BOTHEN_RTN_gfx10 : 1955 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 1956 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 1957 def _IDXEN_RTN_gfx10 : 1958 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 1959 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 1960 def _OFFEN_RTN_gfx10 : 1961 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 1962 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 1963 def _OFFSET_RTN_gfx10 : 1964 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 1965 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 1966 } 1967 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 1968 MUBUF_Real_Atomics_RTN_gfx10<op> { 1969 def _BOTHEN_gfx10 : 1970 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1971 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 1972 def _IDXEN_gfx10 : 1973 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1974 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 1975 def _OFFEN_gfx10 : 1976 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1977 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 1978 def _OFFSET_gfx10 : 1979 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1980 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 1981 } 1982} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 1983 1984defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 1985defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 1986defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 1987defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 1988defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 1989defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 1990defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 1991defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 1992// FIXME-GFX10: Add following instructions: 1993//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 1994//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 1995defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 1996defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 1997defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 1998defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 1999defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2000defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2001defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2002defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2003 2004def BUFFER_GL0_INV_gfx10 : 2005 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2006def BUFFER_GL1_INV_gfx10 : 2007 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2008 2009//===----------------------------------------------------------------------===// 2010// MUBUF - GFX6, GFX7, GFX10. 2011//===----------------------------------------------------------------------===// 2012 2013let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2014 multiclass MUBUF_Real_gfx6<bits<8> op> { 2015 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2016 } 2017} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2018 2019let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2020 multiclass MUBUF_Real_gfx7<bits<8> op> { 2021 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2022 } 2023} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2024 2025let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2026 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2027 def _ADDR64_gfx6_gfx7 : 2028 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2029 def _BOTHEN_gfx6_gfx7 : 2030 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2031 def _IDXEN_gfx6_gfx7 : 2032 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2033 def _OFFEN_gfx6_gfx7 : 2034 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2035 def _OFFSET_gfx6_gfx7 : 2036 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2037 } 2038 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 2039 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2040 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 2041 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2042 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 2043 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2044 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 2045 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2046 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 2047 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2048 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 2049 2050 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2051 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 2052 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 2053 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 2054 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2055 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 2056 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2057 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 2058 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2059 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 2060 } 2061 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2062 def _ADDR64_gfx6_gfx7 : 2063 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2064 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2065 def _BOTHEN_gfx6_gfx7 : 2066 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2067 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2068 def _IDXEN_gfx6_gfx7 : 2069 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2070 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2071 def _OFFEN_gfx6_gfx7 : 2072 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2073 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2074 def _OFFSET_gfx6_gfx7 : 2075 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2076 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2077 2078 def _ADDR64_RTN_gfx6_gfx7 : 2079 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2080 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2081 def _BOTHEN_RTN_gfx6_gfx7 : 2082 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2083 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2084 def _IDXEN_RTN_gfx6_gfx7 : 2085 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2086 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2087 def _OFFEN_RTN_gfx6_gfx7 : 2088 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2089 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2090 def _OFFSET_RTN_gfx6_gfx7 : 2091 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2092 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2093 } 2094} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2095 2096multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2097 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2098 2099multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2100 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2101 2102multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2103 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2104 2105// FIXME-GFX6: Following instructions are available only on GFX6. 2106//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2107//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2108 2109defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2110defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2111defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2112defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2113defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2114defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2115defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2116defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2117defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2118defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2119defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2120defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2121defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2122defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2123defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2124defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2125defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2126defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2127defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2128defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2129defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2130defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2131 2132defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2133defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2134defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2135defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2136defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2137defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2138defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2139defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2140defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2141defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2142defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2143defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2144defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2145defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2146defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2147defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2148defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2149defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2150defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2151defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2152defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2153defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2154defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2155defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2156defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2157defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2158defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2159defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2160defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2161// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2162defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2163defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2164defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2165 2166defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2167 2168defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2169defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2170def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2171 2172//===----------------------------------------------------------------------===// 2173// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2174//===----------------------------------------------------------------------===// 2175 2176class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2177 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2178 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2179 let Inst{12} = ps.offen; 2180 let Inst{13} = ps.idxen; 2181 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2182 let Inst{18-16} = op; 2183 let Inst{31-26} = 0x3a; //encoding 2184 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2185 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2186 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2187 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2188 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2189 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2190} 2191 2192//===----------------------------------------------------------------------===// 2193// MTBUF - GFX10. 2194//===----------------------------------------------------------------------===// 2195 2196class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2197 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2198 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2199 let Inst{25-19} = format; 2200 let Inst{53} = op{3}; 2201} 2202 2203let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2204 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2205 def _BOTHEN_gfx10 : 2206 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2207 def _IDXEN_gfx10 : 2208 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2209 def _OFFEN_gfx10 : 2210 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2211 def _OFFSET_gfx10 : 2212 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2213 } 2214} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2215 2216defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2217defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2218defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2219defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2220defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2221defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2222defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2223defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2224 2225//===----------------------------------------------------------------------===// 2226// MTBUF - GFX6, GFX7, GFX10. 2227//===----------------------------------------------------------------------===// 2228 2229class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2230 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2231 let Inst{15} = ps.addr64; 2232 let Inst{22-19} = dfmt; 2233 let Inst{25-23} = nfmt; 2234} 2235 2236let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2237 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2238 def _ADDR64_gfx6_gfx7 : 2239 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2240 def _BOTHEN_gfx6_gfx7 : 2241 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2242 def _IDXEN_gfx6_gfx7 : 2243 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2244 def _OFFEN_gfx6_gfx7 : 2245 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2246 def _OFFSET_gfx6_gfx7 : 2247 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2248 } 2249} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2250 2251multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2252 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2253 2254defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2255defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2256defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2257defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2258defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2259defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2260defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2261defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2262 2263//===----------------------------------------------------------------------===// 2264// GFX8, GFX9 (VI). 2265//===----------------------------------------------------------------------===// 2266 2267class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 2268 bit has_sccb = ps.has_sccb> : 2269 MUBUF_Real<ps>, 2270 Enc64, 2271 SIMCInstr<ps.PseudoInstr, Enc>, 2272 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 2273 !if(ps.IsAtomicRet, 1, ?))> { 2274 2275 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2276 let Inst{12} = ps.offen; 2277 let Inst{13} = ps.idxen; 2278 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2279 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2280 let Inst{16} = ps.lds; 2281 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2282 let Inst{24-18} = op; 2283 let Inst{31-26} = 0x38; //encoding 2284 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2285 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2286 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2287 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2288} 2289 2290class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 2291 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 2292 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2293 let DecoderNamespace = "GFX8"; 2294 2295 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2296} 2297 2298class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 2299 bit has_sccb = ps.has_sccb> : 2300 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 2301 let AssemblerPredicate = isGFX90APlus; 2302 let DecoderNamespace = "GFX90A"; 2303 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 2304 !subst("$tfe", "", ps.AsmOperands)); 2305 2306 let Inst{55} = acc; 2307} 2308 2309multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> { 2310 def _vi : MUBUF_Real_vi<op, ps>; 2311 def _gfx90a : MUBUF_Real_gfx90a<op, ps, !and(ps.has_sccb,!not(ps.FPAtomic))>; 2312} 2313 2314multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2315 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2316 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2317 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2318 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2319} 2320 2321multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2322 2323 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2324 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2325 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2326 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2327 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2328 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2329 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2330 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2331 2332 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2333 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2334 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2335 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2336 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2337 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2338 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2339 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2340 2341 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2342 MUBUFLdsTable<0, NAME # "_OFFSET_gfx90a">; 2343 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2344 MUBUFLdsTable<0, NAME # "_OFFEN_gfx90a">; 2345 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2346 MUBUFLdsTable<0, NAME # "_IDXEN_gfx90a">; 2347 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2348 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx90a">; 2349 2350 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2351 MUBUFLdsTable<1, NAME # "_OFFSET_gfx90a">; 2352 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2353 MUBUFLdsTable<1, NAME # "_OFFEN_gfx90a">; 2354 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2355 MUBUFLdsTable<1, NAME # "_IDXEN_gfx90a">; 2356 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2357 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx90a">; 2358} 2359 2360class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2361 MUBUF_Real<ps>, 2362 Enc64, 2363 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2364 let AssemblerPredicate=HasUnpackedD16VMem; 2365 let DecoderNamespace="GFX80_UNPACKED"; 2366 2367 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2368 let Inst{12} = ps.offen; 2369 let Inst{13} = ps.idxen; 2370 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2371 let Inst{16} = ps.lds; 2372 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2373 let Inst{24-18} = op; 2374 let Inst{31-26} = 0x38; //encoding 2375 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2376 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2377 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2378 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2379 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2380} 2381 2382multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2383 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2384 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2385 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2386 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2387} 2388 2389multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2390 MUBUF_Real_AllAddr_vi<op> { 2391 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2392 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2393 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2394 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2395} 2396 2397defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2398defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2399defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2400defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2401defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2402defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2403defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2404defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2405let SubtargetPredicate = HasUnpackedD16VMem in { 2406 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2407 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2408 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2409 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2410 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2411 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2412 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2413 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2414} // End HasUnpackedD16VMem. 2415let SubtargetPredicate = HasPackedD16VMem in { 2416 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2417 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2418 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2419 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2420 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2421 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2422 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2423 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2424} // End HasPackedD16VMem. 2425defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2426defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2427defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2428defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2429defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2430defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2431defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2432defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2433defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2434defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2435defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2436defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2437defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2438defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2439defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2440defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2441 2442defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2443defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2444defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2445defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2446defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2447defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2448 2449defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2450defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2451 2452defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2453defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2454defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2455defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2456defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2457defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2458defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2459defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2460defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2461defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2462defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2463defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2464defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2465 2466defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2467defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2468defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2469defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2470defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2471defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2472defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2473defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2474defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2475defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2476defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2477defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2478defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2479 2480defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2481 2482let AssemblerPredicate = isGFX8GFX9 in { 2483def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2484def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2485} // End AssemblerPredicate = isGFX8GFX9 2486 2487let SubtargetPredicate = HasAtomicFaddInsts in { 2488 2489defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2490defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2491 2492} // End SubtargetPredicate = HasAtomicFaddInsts 2493 2494let SubtargetPredicate = isGFX90APlus in { 2495 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2496 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2497 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2498} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2499 2500def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2501} 2502def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2503 2504class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2505 MTBUF_Real<ps>, 2506 Enc64, 2507 SIMCInstr<ps.PseudoInstr, Enc> { 2508 2509 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2510 let Inst{12} = ps.offen; 2511 let Inst{13} = ps.idxen; 2512 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2513 let Inst{18-15} = op; 2514 let Inst{22-19} = dfmt; 2515 let Inst{25-23} = nfmt; 2516 let Inst{31-26} = 0x3a; //encoding 2517 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2518 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2519 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2520 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2521 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2522 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2523 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2524} 2525 2526class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2527 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2528 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2529 let DecoderNamespace = "GFX8"; 2530 2531 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2532} 2533 2534class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2535 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2536 let AssemblerPredicate = isGFX90APlus; 2537 let DecoderNamespace = "GFX90A"; 2538 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2539 2540 let Inst{55} = acc; 2541} 2542 2543multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2544 def _vi : MTBUF_Real_vi<op, ps>; 2545 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2546} 2547 2548multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2549 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2550 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2551 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2552 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2553} 2554 2555class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2556 MTBUF_Real<ps>, 2557 Enc64, 2558 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2559 let AssemblerPredicate=HasUnpackedD16VMem; 2560 let DecoderNamespace="GFX80_UNPACKED"; 2561 2562 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2563 let Inst{12} = ps.offen; 2564 let Inst{13} = ps.idxen; 2565 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2566 let Inst{18-15} = op; 2567 let Inst{22-19} = dfmt; 2568 let Inst{25-23} = nfmt; 2569 let Inst{31-26} = 0x3a; //encoding 2570 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2571 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2572 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2573 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2574 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2575 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2576} 2577 2578multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2579 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2580 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2581 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2582 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2583} 2584 2585defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2586defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2587defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2588defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2589defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2590defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2591defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2592defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2593let SubtargetPredicate = HasUnpackedD16VMem in { 2594 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2595 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2596 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2597 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2598 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2599 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2600 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2601 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2602} // End HasUnpackedD16VMem. 2603let SubtargetPredicate = HasPackedD16VMem in { 2604 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2605 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2606 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2607 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2608 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2609 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2610 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2611 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2612} // End HasUnpackedD16VMem. 2613 2614def MUBUFInfoTable : GenericTable { 2615 let FilterClass = "MUBUF_Pseudo"; 2616 let CppTypeName = "MUBUFInfo"; 2617 let Fields = [ 2618 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 2619 "IsBufferInv" 2620 ]; 2621 2622 let PrimaryKey = ["Opcode"]; 2623 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2624} 2625 2626def getMUBUFInfoFromOpcode : SearchIndex { 2627 let Table = MUBUFInfoTable; 2628 let Key = ["Opcode"]; 2629} 2630 2631def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2632 let Table = MUBUFInfoTable; 2633 let Key = ["BaseOpcode", "elements"]; 2634} 2635 2636def MTBUFInfoTable : GenericTable { 2637 let FilterClass = "MTBUF_Pseudo"; 2638 let CppTypeName = "MTBUFInfo"; 2639 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2640 2641 let PrimaryKey = ["Opcode"]; 2642 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2643} 2644 2645def getMTBUFInfoFromOpcode : SearchIndex { 2646 let Table = MTBUFInfoTable; 2647 let Key = ["Opcode"]; 2648} 2649 2650def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2651 let Table = MTBUFInfoTable; 2652 let Key = ["BaseOpcode", "elements"]; 2653} 2654