1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI implementation of the TargetRegisterInfo class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPURegisterBankInfo.h" 16 #include "GCNSubtarget.h" 17 #include "MCTargetDesc/AMDGPUInstPrinter.h" 18 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 19 #include "SIMachineFunctionInfo.h" 20 #include "SIRegisterInfo.h" 21 #include "llvm/CodeGen/LiveIntervals.h" 22 #include "llvm/CodeGen/LivePhysRegs.h" 23 #include "llvm/CodeGen/MachineDominators.h" 24 #include "llvm/CodeGen/MachineFrameInfo.h" 25 #include "llvm/CodeGen/RegisterScavenging.h" 26 27 using namespace llvm; 28 29 #define GET_REGINFO_TARGET_DESC 30 #include "AMDGPUGenRegisterInfo.inc" 31 32 static cl::opt<bool> EnableSpillSGPRToVGPR( 33 "amdgpu-spill-sgpr-to-vgpr", 34 cl::desc("Enable spilling SGPRs to VGPRs"), 35 cl::ReallyHidden, 36 cl::init(true)); 37 38 std::array<std::vector<int16_t>, 16> SIRegisterInfo::RegSplitParts; 39 std::array<std::array<uint16_t, 32>, 9> SIRegisterInfo::SubRegFromChannelTable; 40 41 // Map numbers of DWORDs to indexes in SubRegFromChannelTable. 42 // Valid indexes are shifted 1, such that a 0 mapping means unsupported. 43 // e.g. for 8 DWORDs (256-bit), SubRegFromChannelTableWidthMap[8] = 8, 44 // meaning index 7 in SubRegFromChannelTable. 45 static const std::array<unsigned, 17> SubRegFromChannelTableWidthMap = { 46 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 9}; 47 48 namespace llvm { 49 50 // A temporary struct to spill SGPRs. 51 // This is mostly to spill SGPRs to memory. Spilling SGPRs into VGPR lanes emits 52 // just v_writelane and v_readlane. 53 // 54 // When spilling to memory, the SGPRs are written into VGPR lanes and the VGPR 55 // is saved to scratch (or the other way around for loads). 56 // For this, a VGPR is required where the needed lanes can be clobbered. The 57 // RegScavenger can provide a VGPR where currently active lanes can be 58 // clobbered, but we still need to save inactive lanes. 59 // The high-level steps are: 60 // - Try to scavenge SGPR(s) to save exec 61 // - Try to scavenge VGPR 62 // - Save needed, all or inactive lanes of a TmpVGPR 63 // - Spill/Restore SGPRs using TmpVGPR 64 // - Restore TmpVGPR 65 // 66 // To save all lanes of TmpVGPR, exec needs to be saved and modified. If we 67 // cannot scavenge temporary SGPRs to save exec, we use the following code: 68 // buffer_store_dword TmpVGPR ; only if active lanes need to be saved 69 // s_not exec, exec 70 // buffer_store_dword TmpVGPR ; save inactive lanes 71 // s_not exec, exec 72 struct SGPRSpillBuilder { 73 struct PerVGPRData { 74 unsigned PerVGPR; 75 unsigned NumVGPRs; 76 int64_t VGPRLanes; 77 }; 78 79 // The SGPR to save 80 Register SuperReg; 81 MachineBasicBlock::iterator MI; 82 ArrayRef<int16_t> SplitParts; 83 unsigned NumSubRegs; 84 bool IsKill; 85 const DebugLoc &DL; 86 87 /* When spilling to stack */ 88 // The SGPRs are written into this VGPR, which is then written to scratch 89 // (or vice versa for loads). 90 Register TmpVGPR = AMDGPU::NoRegister; 91 // Temporary spill slot to save TmpVGPR to. 92 int TmpVGPRIndex = 0; 93 // If TmpVGPR is live before the spill or if it is scavenged. 94 bool TmpVGPRLive = false; 95 // Scavenged SGPR to save EXEC. 96 Register SavedExecReg = AMDGPU::NoRegister; 97 // Stack index to write the SGPRs to. 98 int Index; 99 unsigned EltSize = 4; 100 101 RegScavenger *RS; 102 MachineBasicBlock *MBB; 103 MachineFunction &MF; 104 SIMachineFunctionInfo &MFI; 105 const SIInstrInfo &TII; 106 const SIRegisterInfo &TRI; 107 bool IsWave32; 108 Register ExecReg; 109 unsigned MovOpc; 110 unsigned NotOpc; 111 112 SGPRSpillBuilder(const SIRegisterInfo &TRI, const SIInstrInfo &TII, 113 bool IsWave32, MachineBasicBlock::iterator MI, int Index, 114 RegScavenger *RS) 115 : SGPRSpillBuilder(TRI, TII, IsWave32, MI, MI->getOperand(0).getReg(), 116 MI->getOperand(0).isKill(), Index, RS) {} 117 118 SGPRSpillBuilder(const SIRegisterInfo &TRI, const SIInstrInfo &TII, 119 bool IsWave32, MachineBasicBlock::iterator MI, Register Reg, 120 bool IsKill, int Index, RegScavenger *RS) 121 : SuperReg(Reg), MI(MI), IsKill(IsKill), DL(MI->getDebugLoc()), 122 Index(Index), RS(RS), MBB(MI->getParent()), MF(*MBB->getParent()), 123 MFI(*MF.getInfo<SIMachineFunctionInfo>()), TII(TII), TRI(TRI), 124 IsWave32(IsWave32) { 125 const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(SuperReg); 126 SplitParts = TRI.getRegSplitParts(RC, EltSize); 127 NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 128 129 if (IsWave32) { 130 ExecReg = AMDGPU::EXEC_LO; 131 MovOpc = AMDGPU::S_MOV_B32; 132 NotOpc = AMDGPU::S_NOT_B32; 133 } else { 134 ExecReg = AMDGPU::EXEC; 135 MovOpc = AMDGPU::S_MOV_B64; 136 NotOpc = AMDGPU::S_NOT_B64; 137 } 138 139 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 140 assert(SuperReg != AMDGPU::EXEC_LO && SuperReg != AMDGPU::EXEC_HI && 141 SuperReg != AMDGPU::EXEC && "exec should never spill"); 142 } 143 144 PerVGPRData getPerVGPRData() { 145 PerVGPRData Data; 146 Data.PerVGPR = IsWave32 ? 32 : 64; 147 Data.NumVGPRs = (NumSubRegs + (Data.PerVGPR - 1)) / Data.PerVGPR; 148 Data.VGPRLanes = (1LL << std::min(Data.PerVGPR, NumSubRegs)) - 1LL; 149 return Data; 150 } 151 152 // Tries to scavenge SGPRs to save EXEC and a VGPR. Uses v0 if no VGPR is 153 // free. 154 // Writes these instructions if an SGPR can be scavenged: 155 // s_mov_b64 s[6:7], exec ; Save exec 156 // s_mov_b64 exec, 3 ; Wanted lanemask 157 // buffer_store_dword v1 ; Write scavenged VGPR to emergency slot 158 // 159 // Writes these instructions if no SGPR can be scavenged: 160 // buffer_store_dword v0 ; Only if no free VGPR was found 161 // s_not_b64 exec, exec 162 // buffer_store_dword v0 ; Save inactive lanes 163 // ; exec stays inverted, it is flipped back in 164 // ; restore. 165 void prepare() { 166 // Scavenged temporary VGPR to use. It must be scavenged once for any number 167 // of spilled subregs. 168 // FIXME: The liveness analysis is limited and does not tell if a register 169 // is in use in lanes that are currently inactive. We can never be sure if 170 // a register as actually in use in another lane, so we need to save all 171 // used lanes of the chosen VGPR. 172 assert(RS && "Cannot spill SGPR to memory without RegScavenger"); 173 TmpVGPR = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, false, 174 0, false); 175 176 // Reserve temporary stack slot 177 TmpVGPRIndex = MFI.getScavengeFI(MF.getFrameInfo(), TRI); 178 if (TmpVGPR) { 179 // Found a register that is dead in the currently active lanes, we only 180 // need to spill inactive lanes. 181 TmpVGPRLive = false; 182 } else { 183 // Pick v0 because it doesn't make a difference. 184 TmpVGPR = AMDGPU::VGPR0; 185 TmpVGPRLive = true; 186 } 187 188 if (TmpVGPRLive) { 189 // We need to inform the scavenger that this index is already in use until 190 // we're done with the custom emergency spill. 191 RS->assignRegToScavengingIndex(TmpVGPRIndex, TmpVGPR); 192 } 193 194 // We may end up recursively calling the scavenger, and don't want to re-use 195 // the same register. 196 RS->setRegUsed(TmpVGPR); 197 198 // Try to scavenge SGPRs to save exec 199 assert(!SavedExecReg && "Exec is already saved, refuse to save again"); 200 const TargetRegisterClass &RC = 201 IsWave32 ? AMDGPU::SGPR_32RegClass : AMDGPU::SGPR_64RegClass; 202 RS->setRegUsed(SuperReg); 203 SavedExecReg = RS->scavengeRegisterBackwards(RC, MI, false, 0, false); 204 205 int64_t VGPRLanes = getPerVGPRData().VGPRLanes; 206 207 if (SavedExecReg) { 208 RS->setRegUsed(SavedExecReg); 209 // Set exec to needed lanes 210 BuildMI(*MBB, MI, DL, TII.get(MovOpc), SavedExecReg).addReg(ExecReg); 211 auto I = 212 BuildMI(*MBB, MI, DL, TII.get(MovOpc), ExecReg).addImm(VGPRLanes); 213 if (!TmpVGPRLive) 214 I.addReg(TmpVGPR, RegState::ImplicitDefine); 215 // Spill needed lanes 216 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false); 217 } else { 218 // The modify and restore of exec clobber SCC, which we would have to save 219 // and restore. FIXME: We probably would need to reserve a register for 220 // this. 221 if (RS->isRegUsed(AMDGPU::SCC)) 222 MI->emitError("unhandled SGPR spill to memory"); 223 224 // Spill active lanes 225 if (TmpVGPRLive) 226 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false, 227 /*IsKill*/ false); 228 // Spill inactive lanes 229 auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 230 if (!TmpVGPRLive) 231 I.addReg(TmpVGPR, RegState::ImplicitDefine); 232 I->getOperand(2).setIsDead(); // Mark SCC as dead. 233 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false); 234 } 235 } 236 237 // Writes these instructions if an SGPR can be scavenged: 238 // buffer_load_dword v1 ; Write scavenged VGPR to emergency slot 239 // s_waitcnt vmcnt(0) ; If a free VGPR was found 240 // s_mov_b64 exec, s[6:7] ; Save exec 241 // 242 // Writes these instructions if no SGPR can be scavenged: 243 // buffer_load_dword v0 ; Restore inactive lanes 244 // s_waitcnt vmcnt(0) ; If a free VGPR was found 245 // s_not_b64 exec, exec 246 // buffer_load_dword v0 ; Only if no free VGPR was found 247 void restore() { 248 if (SavedExecReg) { 249 // Restore used lanes 250 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true, 251 /*IsKill*/ false); 252 // Restore exec 253 auto I = BuildMI(*MBB, MI, DL, TII.get(MovOpc), ExecReg) 254 .addReg(SavedExecReg, RegState::Kill); 255 // Add an implicit use of the load so it is not dead. 256 // FIXME This inserts an unnecessary waitcnt 257 if (!TmpVGPRLive) { 258 I.addReg(TmpVGPR, RegState::ImplicitKill); 259 } 260 } else { 261 // Restore inactive lanes 262 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true, 263 /*IsKill*/ false); 264 auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 265 if (!TmpVGPRLive) 266 I.addReg(TmpVGPR, RegState::ImplicitKill); 267 I->getOperand(2).setIsDead(); // Mark SCC as dead. 268 269 // Restore active lanes 270 if (TmpVGPRLive) 271 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true); 272 } 273 274 // Inform the scavenger where we're releasing our custom scavenged register. 275 if (TmpVGPRLive) { 276 MachineBasicBlock::iterator RestorePt = std::prev(MI); 277 RS->assignRegToScavengingIndex(TmpVGPRIndex, TmpVGPR, &*RestorePt); 278 } 279 } 280 281 // Write TmpVGPR to memory or read TmpVGPR from memory. 282 // Either using a single buffer_load/store if exec is set to the needed mask 283 // or using 284 // buffer_load 285 // s_not exec, exec 286 // buffer_load 287 // s_not exec, exec 288 void readWriteTmpVGPR(unsigned Offset, bool IsLoad) { 289 if (SavedExecReg) { 290 // Spill needed lanes 291 TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad); 292 } else { 293 // The modify and restore of exec clobber SCC, which we would have to save 294 // and restore. FIXME: We probably would need to reserve a register for 295 // this. 296 if (RS->isRegUsed(AMDGPU::SCC)) 297 MI->emitError("unhandled SGPR spill to memory"); 298 299 // Spill active lanes 300 TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad, 301 /*IsKill*/ false); 302 // Spill inactive lanes 303 auto Not0 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 304 Not0->getOperand(2).setIsDead(); // Mark SCC as dead. 305 TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad); 306 auto Not1 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 307 Not1->getOperand(2).setIsDead(); // Mark SCC as dead. 308 } 309 } 310 311 void setMI(MachineBasicBlock *NewMBB, MachineBasicBlock::iterator NewMI) { 312 assert(MBB->getParent() == &MF); 313 MI = NewMI; 314 MBB = NewMBB; 315 } 316 }; 317 318 } // namespace llvm 319 320 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) 321 : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST), 322 SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) { 323 324 assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 && 325 getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) && 326 (getSubRegIndexLaneMask(AMDGPU::lo16) | 327 getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() == 328 getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() && 329 "getNumCoveredRegs() will not work with generated subreg masks!"); 330 331 RegPressureIgnoredUnits.resize(getNumRegUnits()); 332 RegPressureIgnoredUnits.set(*regunits(MCRegister::from(AMDGPU::M0)).begin()); 333 for (auto Reg : AMDGPU::VGPR_HI16RegClass) 334 RegPressureIgnoredUnits.set(*regunits(Reg).begin()); 335 336 // HACK: Until this is fully tablegen'd. 337 static llvm::once_flag InitializeRegSplitPartsFlag; 338 339 static auto InitializeRegSplitPartsOnce = [this]() { 340 for (unsigned Idx = 1, E = getNumSubRegIndices() - 1; Idx < E; ++Idx) { 341 unsigned Size = getSubRegIdxSize(Idx); 342 if (Size & 31) 343 continue; 344 std::vector<int16_t> &Vec = RegSplitParts[Size / 32 - 1]; 345 unsigned Pos = getSubRegIdxOffset(Idx); 346 if (Pos % Size) 347 continue; 348 Pos /= Size; 349 if (Vec.empty()) { 350 unsigned MaxNumParts = 1024 / Size; // Maximum register is 1024 bits. 351 Vec.resize(MaxNumParts); 352 } 353 Vec[Pos] = Idx; 354 } 355 }; 356 357 static llvm::once_flag InitializeSubRegFromChannelTableFlag; 358 359 static auto InitializeSubRegFromChannelTableOnce = [this]() { 360 for (auto &Row : SubRegFromChannelTable) 361 Row.fill(AMDGPU::NoSubRegister); 362 for (unsigned Idx = 1; Idx < getNumSubRegIndices(); ++Idx) { 363 unsigned Width = AMDGPUSubRegIdxRanges[Idx].Size / 32; 364 unsigned Offset = AMDGPUSubRegIdxRanges[Idx].Offset / 32; 365 assert(Width < SubRegFromChannelTableWidthMap.size()); 366 Width = SubRegFromChannelTableWidthMap[Width]; 367 if (Width == 0) 368 continue; 369 unsigned TableIdx = Width - 1; 370 assert(TableIdx < SubRegFromChannelTable.size()); 371 assert(Offset < SubRegFromChannelTable[TableIdx].size()); 372 SubRegFromChannelTable[TableIdx][Offset] = Idx; 373 } 374 }; 375 376 llvm::call_once(InitializeRegSplitPartsFlag, InitializeRegSplitPartsOnce); 377 llvm::call_once(InitializeSubRegFromChannelTableFlag, 378 InitializeSubRegFromChannelTableOnce); 379 } 380 381 void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved, 382 MCRegister Reg) const { 383 for (MCRegAliasIterator R(Reg, this, true); R.isValid(); ++R) 384 Reserved.set(*R); 385 } 386 387 // Forced to be here by one .inc 388 const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs( 389 const MachineFunction *MF) const { 390 CallingConv::ID CC = MF->getFunction().getCallingConv(); 391 switch (CC) { 392 case CallingConv::C: 393 case CallingConv::Fast: 394 case CallingConv::Cold: 395 return ST.hasGFX90AInsts() ? CSR_AMDGPU_GFX90AInsts_SaveList 396 : CSR_AMDGPU_SaveList; 397 case CallingConv::AMDGPU_Gfx: 398 return ST.hasGFX90AInsts() ? CSR_AMDGPU_SI_Gfx_GFX90AInsts_SaveList 399 : CSR_AMDGPU_SI_Gfx_SaveList; 400 default: { 401 // Dummy to not crash RegisterClassInfo. 402 static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister; 403 return &NoCalleeSavedReg; 404 } 405 } 406 } 407 408 const MCPhysReg * 409 SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const { 410 return nullptr; 411 } 412 413 const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF, 414 CallingConv::ID CC) const { 415 switch (CC) { 416 case CallingConv::C: 417 case CallingConv::Fast: 418 case CallingConv::Cold: 419 return ST.hasGFX90AInsts() ? CSR_AMDGPU_GFX90AInsts_RegMask 420 : CSR_AMDGPU_RegMask; 421 case CallingConv::AMDGPU_Gfx: 422 return ST.hasGFX90AInsts() ? CSR_AMDGPU_SI_Gfx_GFX90AInsts_RegMask 423 : CSR_AMDGPU_SI_Gfx_RegMask; 424 default: 425 return nullptr; 426 } 427 } 428 429 const uint32_t *SIRegisterInfo::getNoPreservedMask() const { 430 return CSR_AMDGPU_NoRegs_RegMask; 431 } 432 433 const TargetRegisterClass * 434 SIRegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC, 435 const MachineFunction &MF) const { 436 // FIXME: Should have a helper function like getEquivalentVGPRClass to get the 437 // equivalent AV class. If used one, the verifier will crash after 438 // RegBankSelect in the GISel flow. The aligned regclasses are not fully given 439 // until Instruction selection. 440 if (ST.hasMAIInsts() && (isVGPRClass(RC) || isAGPRClass(RC))) { 441 if (RC == &AMDGPU::VGPR_32RegClass || RC == &AMDGPU::AGPR_32RegClass) 442 return &AMDGPU::AV_32RegClass; 443 if (RC == &AMDGPU::VReg_64RegClass || RC == &AMDGPU::AReg_64RegClass) 444 return &AMDGPU::AV_64RegClass; 445 if (RC == &AMDGPU::VReg_64_Align2RegClass || 446 RC == &AMDGPU::AReg_64_Align2RegClass) 447 return &AMDGPU::AV_64_Align2RegClass; 448 if (RC == &AMDGPU::VReg_96RegClass || RC == &AMDGPU::AReg_96RegClass) 449 return &AMDGPU::AV_96RegClass; 450 if (RC == &AMDGPU::VReg_96_Align2RegClass || 451 RC == &AMDGPU::AReg_96_Align2RegClass) 452 return &AMDGPU::AV_96_Align2RegClass; 453 if (RC == &AMDGPU::VReg_128RegClass || RC == &AMDGPU::AReg_128RegClass) 454 return &AMDGPU::AV_128RegClass; 455 if (RC == &AMDGPU::VReg_128_Align2RegClass || 456 RC == &AMDGPU::AReg_128_Align2RegClass) 457 return &AMDGPU::AV_128_Align2RegClass; 458 if (RC == &AMDGPU::VReg_160RegClass || RC == &AMDGPU::AReg_160RegClass) 459 return &AMDGPU::AV_160RegClass; 460 if (RC == &AMDGPU::VReg_160_Align2RegClass || 461 RC == &AMDGPU::AReg_160_Align2RegClass) 462 return &AMDGPU::AV_160_Align2RegClass; 463 if (RC == &AMDGPU::VReg_192RegClass || RC == &AMDGPU::AReg_192RegClass) 464 return &AMDGPU::AV_192RegClass; 465 if (RC == &AMDGPU::VReg_192_Align2RegClass || 466 RC == &AMDGPU::AReg_192_Align2RegClass) 467 return &AMDGPU::AV_192_Align2RegClass; 468 if (RC == &AMDGPU::VReg_256RegClass || RC == &AMDGPU::AReg_256RegClass) 469 return &AMDGPU::AV_256RegClass; 470 if (RC == &AMDGPU::VReg_256_Align2RegClass || 471 RC == &AMDGPU::AReg_256_Align2RegClass) 472 return &AMDGPU::AV_256_Align2RegClass; 473 if (RC == &AMDGPU::VReg_512RegClass || RC == &AMDGPU::AReg_512RegClass) 474 return &AMDGPU::AV_512RegClass; 475 if (RC == &AMDGPU::VReg_512_Align2RegClass || 476 RC == &AMDGPU::AReg_512_Align2RegClass) 477 return &AMDGPU::AV_512_Align2RegClass; 478 if (RC == &AMDGPU::VReg_1024RegClass || RC == &AMDGPU::AReg_1024RegClass) 479 return &AMDGPU::AV_1024RegClass; 480 if (RC == &AMDGPU::VReg_1024_Align2RegClass || 481 RC == &AMDGPU::AReg_1024_Align2RegClass) 482 return &AMDGPU::AV_1024_Align2RegClass; 483 } 484 485 return TargetRegisterInfo::getLargestLegalSuperClass(RC, MF); 486 } 487 488 Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const { 489 const SIFrameLowering *TFI = ST.getFrameLowering(); 490 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); 491 // During ISel lowering we always reserve the stack pointer in entry 492 // functions, but never actually want to reference it when accessing our own 493 // frame. If we need a frame pointer we use it, but otherwise we can just use 494 // an immediate "0" which we represent by returning NoRegister. 495 if (FuncInfo->isEntryFunction()) { 496 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() : Register(); 497 } 498 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() 499 : FuncInfo->getStackPtrOffsetReg(); 500 } 501 502 bool SIRegisterInfo::hasBasePointer(const MachineFunction &MF) const { 503 // When we need stack realignment, we can't reference off of the 504 // stack pointer, so we reserve a base pointer. 505 const MachineFrameInfo &MFI = MF.getFrameInfo(); 506 return MFI.getNumFixedObjects() && shouldRealignStack(MF); 507 } 508 509 Register SIRegisterInfo::getBaseRegister() const { return AMDGPU::SGPR34; } 510 511 const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const { 512 return AMDGPU_AllVGPRs_RegMask; 513 } 514 515 const uint32_t *SIRegisterInfo::getAllAGPRRegMask() const { 516 return AMDGPU_AllAGPRs_RegMask; 517 } 518 519 const uint32_t *SIRegisterInfo::getAllVectorRegMask() const { 520 return AMDGPU_AllVectorRegs_RegMask; 521 } 522 523 const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const { 524 return AMDGPU_AllAllocatableSRegs_RegMask; 525 } 526 527 unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel, 528 unsigned NumRegs) { 529 assert(NumRegs < SubRegFromChannelTableWidthMap.size()); 530 unsigned NumRegIndex = SubRegFromChannelTableWidthMap[NumRegs]; 531 assert(NumRegIndex && "Not implemented"); 532 assert(Channel < SubRegFromChannelTable[NumRegIndex - 1].size()); 533 return SubRegFromChannelTable[NumRegIndex - 1][Channel]; 534 } 535 536 MCRegister 537 SIRegisterInfo::getAlignedHighSGPRForRC(const MachineFunction &MF, 538 const unsigned Align, 539 const TargetRegisterClass *RC) const { 540 unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), Align) - Align; 541 MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx)); 542 return getMatchingSuperReg(BaseReg, AMDGPU::sub0, RC); 543 } 544 545 MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg( 546 const MachineFunction &MF) const { 547 return getAlignedHighSGPRForRC(MF, /*Align=*/4, &AMDGPU::SGPR_128RegClass); 548 } 549 550 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const { 551 BitVector Reserved(getNumRegs()); 552 Reserved.set(AMDGPU::MODE); 553 554 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 555 556 // Reserve special purpose registers. 557 // 558 // EXEC_LO and EXEC_HI could be allocated and used as regular register, but 559 // this seems likely to result in bugs, so I'm marking them as reserved. 560 reserveRegisterTuples(Reserved, AMDGPU::EXEC); 561 reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR); 562 563 // M0 has to be reserved so that llvm accepts it as a live-in into a block. 564 reserveRegisterTuples(Reserved, AMDGPU::M0); 565 566 // Reserve src_vccz, src_execz, src_scc. 567 reserveRegisterTuples(Reserved, AMDGPU::SRC_VCCZ); 568 reserveRegisterTuples(Reserved, AMDGPU::SRC_EXECZ); 569 reserveRegisterTuples(Reserved, AMDGPU::SRC_SCC); 570 571 // Reserve the memory aperture registers 572 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE); 573 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT); 574 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE); 575 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT); 576 577 // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen. 578 reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID); 579 580 // Reserve xnack_mask registers - support is not implemented in Codegen. 581 reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK); 582 583 // Reserve lds_direct register - support is not implemented in Codegen. 584 reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT); 585 586 // Reserve Trap Handler registers - support is not implemented in Codegen. 587 reserveRegisterTuples(Reserved, AMDGPU::TBA); 588 reserveRegisterTuples(Reserved, AMDGPU::TMA); 589 reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1); 590 reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3); 591 reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5); 592 reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7); 593 reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9); 594 reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11); 595 reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13); 596 reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15); 597 598 // Reserve null register - it shall never be allocated 599 reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL64); 600 601 // Disallow vcc_hi allocation in wave32. It may be allocated but most likely 602 // will result in bugs. 603 if (isWave32) { 604 Reserved.set(AMDGPU::VCC); 605 Reserved.set(AMDGPU::VCC_HI); 606 } 607 608 // Reserve SGPRs. 609 // 610 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF); 611 unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); 612 for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) { 613 unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); 614 reserveRegisterTuples(Reserved, Reg); 615 } 616 617 Register ScratchRSrcReg = MFI->getScratchRSrcReg(); 618 if (ScratchRSrcReg != AMDGPU::NoRegister) { 619 // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we 620 // need to spill. 621 // TODO: May need to reserve a VGPR if doing LDS spilling. 622 reserveRegisterTuples(Reserved, ScratchRSrcReg); 623 } 624 625 Register LongBranchReservedReg = MFI->getLongBranchReservedReg(); 626 if (LongBranchReservedReg) 627 reserveRegisterTuples(Reserved, LongBranchReservedReg); 628 629 // We have to assume the SP is needed in case there are calls in the function, 630 // which is detected after the function is lowered. If we aren't really going 631 // to need SP, don't bother reserving it. 632 MCRegister StackPtrReg = MFI->getStackPtrOffsetReg(); 633 if (StackPtrReg) { 634 reserveRegisterTuples(Reserved, StackPtrReg); 635 assert(!isSubRegister(ScratchRSrcReg, StackPtrReg)); 636 } 637 638 MCRegister FrameReg = MFI->getFrameOffsetReg(); 639 if (FrameReg) { 640 reserveRegisterTuples(Reserved, FrameReg); 641 assert(!isSubRegister(ScratchRSrcReg, FrameReg)); 642 } 643 644 if (hasBasePointer(MF)) { 645 MCRegister BasePtrReg = getBaseRegister(); 646 reserveRegisterTuples(Reserved, BasePtrReg); 647 assert(!isSubRegister(ScratchRSrcReg, BasePtrReg)); 648 } 649 650 // FIXME: Use same reserved register introduced in D149775 651 // SGPR used to preserve EXEC MASK around WWM spill/copy instructions. 652 Register ExecCopyReg = MFI->getSGPRForEXECCopy(); 653 if (ExecCopyReg) 654 reserveRegisterTuples(Reserved, ExecCopyReg); 655 656 // Reserve VGPRs/AGPRs. 657 // 658 unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF); 659 unsigned MaxNumAGPRs = MaxNumVGPRs; 660 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs(); 661 662 // On GFX90A, the number of VGPRs and AGPRs need not be equal. Theoretically, 663 // a wave may have up to 512 total vector registers combining together both 664 // VGPRs and AGPRs. Hence, in an entry function without calls and without 665 // AGPRs used within it, it is possible to use the whole vector register 666 // budget for VGPRs. 667 // 668 // TODO: it shall be possible to estimate maximum AGPR/VGPR pressure and split 669 // register file accordingly. 670 if (ST.hasGFX90AInsts()) { 671 if (MFI->usesAGPRs(MF)) { 672 MaxNumVGPRs /= 2; 673 MaxNumAGPRs = MaxNumVGPRs; 674 } else { 675 if (MaxNumVGPRs > TotalNumVGPRs) { 676 MaxNumAGPRs = MaxNumVGPRs - TotalNumVGPRs; 677 MaxNumVGPRs = TotalNumVGPRs; 678 } else 679 MaxNumAGPRs = 0; 680 } 681 } 682 683 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) { 684 unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i); 685 reserveRegisterTuples(Reserved, Reg); 686 } 687 688 if (ST.hasMAIInsts()) { 689 for (unsigned i = MaxNumAGPRs; i < TotalNumVGPRs; ++i) { 690 unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 691 reserveRegisterTuples(Reserved, Reg); 692 } 693 } else { 694 // Reserve all the AGPRs if there are no instructions to use it. 695 for (MCRegister Reg : AMDGPU::AGPR_32RegClass) 696 reserveRegisterTuples(Reserved, Reg); 697 } 698 699 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch 700 // VGPR available at all times. 701 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) { 702 reserveRegisterTuples(Reserved, MFI->getVGPRForAGPRCopy()); 703 } 704 705 for (Register Reg : MFI->getWWMReservedRegs()) 706 reserveRegisterTuples(Reserved, Reg); 707 708 // FIXME: Stop using reserved registers for this. 709 for (MCPhysReg Reg : MFI->getAGPRSpillVGPRs()) 710 reserveRegisterTuples(Reserved, Reg); 711 712 for (MCPhysReg Reg : MFI->getVGPRSpillAGPRs()) 713 reserveRegisterTuples(Reserved, Reg); 714 715 return Reserved; 716 } 717 718 bool SIRegisterInfo::isAsmClobberable(const MachineFunction &MF, 719 MCRegister PhysReg) const { 720 return !MF.getRegInfo().isReserved(PhysReg); 721 } 722 723 bool SIRegisterInfo::shouldRealignStack(const MachineFunction &MF) const { 724 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 725 // On entry, the base address is 0, so it can't possibly need any more 726 // alignment. 727 728 // FIXME: Should be able to specify the entry frame alignment per calling 729 // convention instead. 730 if (Info->isEntryFunction()) 731 return false; 732 733 return TargetRegisterInfo::shouldRealignStack(MF); 734 } 735 736 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const { 737 const SIMachineFunctionInfo *Info = Fn.getInfo<SIMachineFunctionInfo>(); 738 if (Info->isEntryFunction()) { 739 const MachineFrameInfo &MFI = Fn.getFrameInfo(); 740 return MFI.hasStackObjects() || MFI.hasCalls(); 741 } 742 743 // May need scavenger for dealing with callee saved registers. 744 return true; 745 } 746 747 bool SIRegisterInfo::requiresFrameIndexScavenging( 748 const MachineFunction &MF) const { 749 // Do not use frame virtual registers. They used to be used for SGPRs, but 750 // once we reach PrologEpilogInserter, we can no longer spill SGPRs. If the 751 // scavenger fails, we can increment/decrement the necessary SGPRs to avoid a 752 // spill. 753 return false; 754 } 755 756 bool SIRegisterInfo::requiresFrameIndexReplacementScavenging( 757 const MachineFunction &MF) const { 758 const MachineFrameInfo &MFI = MF.getFrameInfo(); 759 return MFI.hasStackObjects(); 760 } 761 762 bool SIRegisterInfo::requiresVirtualBaseRegisters( 763 const MachineFunction &) const { 764 // There are no special dedicated stack or frame pointers. 765 return true; 766 } 767 768 int64_t SIRegisterInfo::getScratchInstrOffset(const MachineInstr *MI) const { 769 assert(SIInstrInfo::isMUBUF(*MI) || SIInstrInfo::isFLATScratch(*MI)); 770 771 int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 772 AMDGPU::OpName::offset); 773 return MI->getOperand(OffIdx).getImm(); 774 } 775 776 int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI, 777 int Idx) const { 778 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI)) 779 return 0; 780 781 assert((Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 782 AMDGPU::OpName::vaddr) || 783 (Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 784 AMDGPU::OpName::saddr))) && 785 "Should never see frame index on non-address operand"); 786 787 return getScratchInstrOffset(MI); 788 } 789 790 bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const { 791 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI)) 792 return false; 793 794 int64_t FullOffset = Offset + getScratchInstrOffset(MI); 795 796 if (SIInstrInfo::isMUBUF(*MI)) 797 return !SIInstrInfo::isLegalMUBUFImmOffset(FullOffset); 798 799 const SIInstrInfo *TII = ST.getInstrInfo(); 800 return !TII->isLegalFLATOffset(FullOffset, AMDGPUAS::PRIVATE_ADDRESS, 801 SIInstrFlags::FlatScratch); 802 } 803 804 Register SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, 805 int FrameIdx, 806 int64_t Offset) const { 807 MachineBasicBlock::iterator Ins = MBB->begin(); 808 DebugLoc DL; // Defaults to "unknown" 809 810 if (Ins != MBB->end()) 811 DL = Ins->getDebugLoc(); 812 813 MachineFunction *MF = MBB->getParent(); 814 const SIInstrInfo *TII = ST.getInstrInfo(); 815 MachineRegisterInfo &MRI = MF->getRegInfo(); 816 unsigned MovOpc = ST.enableFlatScratch() ? AMDGPU::S_MOV_B32 817 : AMDGPU::V_MOV_B32_e32; 818 819 Register BaseReg = MRI.createVirtualRegister( 820 ST.enableFlatScratch() ? &AMDGPU::SReg_32_XEXEC_HIRegClass 821 : &AMDGPU::VGPR_32RegClass); 822 823 if (Offset == 0) { 824 BuildMI(*MBB, Ins, DL, TII->get(MovOpc), BaseReg) 825 .addFrameIndex(FrameIdx); 826 return BaseReg; 827 } 828 829 Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 830 831 Register FIReg = MRI.createVirtualRegister( 832 ST.enableFlatScratch() ? &AMDGPU::SReg_32_XM0RegClass 833 : &AMDGPU::VGPR_32RegClass); 834 835 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) 836 .addImm(Offset); 837 BuildMI(*MBB, Ins, DL, TII->get(MovOpc), FIReg) 838 .addFrameIndex(FrameIdx); 839 840 if (ST.enableFlatScratch() ) { 841 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_ADD_I32), BaseReg) 842 .addReg(OffsetReg, RegState::Kill) 843 .addReg(FIReg); 844 return BaseReg; 845 } 846 847 TII->getAddNoCarry(*MBB, Ins, DL, BaseReg) 848 .addReg(OffsetReg, RegState::Kill) 849 .addReg(FIReg) 850 .addImm(0); // clamp bit 851 852 return BaseReg; 853 } 854 855 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg, 856 int64_t Offset) const { 857 const SIInstrInfo *TII = ST.getInstrInfo(); 858 bool IsFlat = TII->isFLATScratch(MI); 859 860 #ifndef NDEBUG 861 // FIXME: Is it possible to be storing a frame index to itself? 862 bool SeenFI = false; 863 for (const MachineOperand &MO: MI.operands()) { 864 if (MO.isFI()) { 865 if (SeenFI) 866 llvm_unreachable("should not see multiple frame indices"); 867 868 SeenFI = true; 869 } 870 } 871 #endif 872 873 MachineOperand *FIOp = 874 TII->getNamedOperand(MI, IsFlat ? AMDGPU::OpName::saddr 875 : AMDGPU::OpName::vaddr); 876 877 MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset); 878 int64_t NewOffset = OffsetOp->getImm() + Offset; 879 880 assert(FIOp && FIOp->isFI() && "frame index must be address operand"); 881 assert(TII->isMUBUF(MI) || TII->isFLATScratch(MI)); 882 883 if (IsFlat) { 884 assert(TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, 885 SIInstrFlags::FlatScratch) && 886 "offset should be legal"); 887 FIOp->ChangeToRegister(BaseReg, false); 888 OffsetOp->setImm(NewOffset); 889 return; 890 } 891 892 #ifndef NDEBUG 893 MachineOperand *SOffset = TII->getNamedOperand(MI, AMDGPU::OpName::soffset); 894 assert(SOffset->isImm() && SOffset->getImm() == 0); 895 #endif 896 897 assert(SIInstrInfo::isLegalMUBUFImmOffset(NewOffset) && 898 "offset should be legal"); 899 900 FIOp->ChangeToRegister(BaseReg, false); 901 OffsetOp->setImm(NewOffset); 902 } 903 904 bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, 905 Register BaseReg, 906 int64_t Offset) const { 907 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI)) 908 return false; 909 910 int64_t NewOffset = Offset + getScratchInstrOffset(MI); 911 912 if (SIInstrInfo::isMUBUF(*MI)) 913 return SIInstrInfo::isLegalMUBUFImmOffset(NewOffset); 914 915 const SIInstrInfo *TII = ST.getInstrInfo(); 916 return TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, 917 SIInstrFlags::FlatScratch); 918 } 919 920 const TargetRegisterClass *SIRegisterInfo::getPointerRegClass( 921 const MachineFunction &MF, unsigned Kind) const { 922 // This is inaccurate. It depends on the instruction and address space. The 923 // only place where we should hit this is for dealing with frame indexes / 924 // private accesses, so this is correct in that case. 925 return &AMDGPU::VGPR_32RegClass; 926 } 927 928 const TargetRegisterClass * 929 SIRegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const { 930 if (isAGPRClass(RC) && !ST.hasGFX90AInsts()) 931 return getEquivalentVGPRClass(RC); 932 if (RC == &AMDGPU::SCC_CLASSRegClass) 933 return getWaveMaskRegClass(); 934 935 return RC; 936 } 937 938 static unsigned getNumSubRegsForSpillOp(unsigned Op) { 939 940 switch (Op) { 941 case AMDGPU::SI_SPILL_S1024_SAVE: 942 case AMDGPU::SI_SPILL_S1024_RESTORE: 943 case AMDGPU::SI_SPILL_V1024_SAVE: 944 case AMDGPU::SI_SPILL_V1024_RESTORE: 945 case AMDGPU::SI_SPILL_A1024_SAVE: 946 case AMDGPU::SI_SPILL_A1024_RESTORE: 947 case AMDGPU::SI_SPILL_AV1024_SAVE: 948 case AMDGPU::SI_SPILL_AV1024_RESTORE: 949 return 32; 950 case AMDGPU::SI_SPILL_S512_SAVE: 951 case AMDGPU::SI_SPILL_S512_RESTORE: 952 case AMDGPU::SI_SPILL_V512_SAVE: 953 case AMDGPU::SI_SPILL_V512_RESTORE: 954 case AMDGPU::SI_SPILL_A512_SAVE: 955 case AMDGPU::SI_SPILL_A512_RESTORE: 956 case AMDGPU::SI_SPILL_AV512_SAVE: 957 case AMDGPU::SI_SPILL_AV512_RESTORE: 958 return 16; 959 case AMDGPU::SI_SPILL_S384_SAVE: 960 case AMDGPU::SI_SPILL_S384_RESTORE: 961 case AMDGPU::SI_SPILL_V384_SAVE: 962 case AMDGPU::SI_SPILL_V384_RESTORE: 963 case AMDGPU::SI_SPILL_A384_SAVE: 964 case AMDGPU::SI_SPILL_A384_RESTORE: 965 case AMDGPU::SI_SPILL_AV384_SAVE: 966 case AMDGPU::SI_SPILL_AV384_RESTORE: 967 return 12; 968 case AMDGPU::SI_SPILL_S352_SAVE: 969 case AMDGPU::SI_SPILL_S352_RESTORE: 970 case AMDGPU::SI_SPILL_V352_SAVE: 971 case AMDGPU::SI_SPILL_V352_RESTORE: 972 case AMDGPU::SI_SPILL_A352_SAVE: 973 case AMDGPU::SI_SPILL_A352_RESTORE: 974 case AMDGPU::SI_SPILL_AV352_SAVE: 975 case AMDGPU::SI_SPILL_AV352_RESTORE: 976 return 11; 977 case AMDGPU::SI_SPILL_S320_SAVE: 978 case AMDGPU::SI_SPILL_S320_RESTORE: 979 case AMDGPU::SI_SPILL_V320_SAVE: 980 case AMDGPU::SI_SPILL_V320_RESTORE: 981 case AMDGPU::SI_SPILL_A320_SAVE: 982 case AMDGPU::SI_SPILL_A320_RESTORE: 983 case AMDGPU::SI_SPILL_AV320_SAVE: 984 case AMDGPU::SI_SPILL_AV320_RESTORE: 985 return 10; 986 case AMDGPU::SI_SPILL_S288_SAVE: 987 case AMDGPU::SI_SPILL_S288_RESTORE: 988 case AMDGPU::SI_SPILL_V288_SAVE: 989 case AMDGPU::SI_SPILL_V288_RESTORE: 990 case AMDGPU::SI_SPILL_A288_SAVE: 991 case AMDGPU::SI_SPILL_A288_RESTORE: 992 case AMDGPU::SI_SPILL_AV288_SAVE: 993 case AMDGPU::SI_SPILL_AV288_RESTORE: 994 return 9; 995 case AMDGPU::SI_SPILL_S256_SAVE: 996 case AMDGPU::SI_SPILL_S256_RESTORE: 997 case AMDGPU::SI_SPILL_V256_SAVE: 998 case AMDGPU::SI_SPILL_V256_RESTORE: 999 case AMDGPU::SI_SPILL_A256_SAVE: 1000 case AMDGPU::SI_SPILL_A256_RESTORE: 1001 case AMDGPU::SI_SPILL_AV256_SAVE: 1002 case AMDGPU::SI_SPILL_AV256_RESTORE: 1003 return 8; 1004 case AMDGPU::SI_SPILL_S224_SAVE: 1005 case AMDGPU::SI_SPILL_S224_RESTORE: 1006 case AMDGPU::SI_SPILL_V224_SAVE: 1007 case AMDGPU::SI_SPILL_V224_RESTORE: 1008 case AMDGPU::SI_SPILL_A224_SAVE: 1009 case AMDGPU::SI_SPILL_A224_RESTORE: 1010 case AMDGPU::SI_SPILL_AV224_SAVE: 1011 case AMDGPU::SI_SPILL_AV224_RESTORE: 1012 return 7; 1013 case AMDGPU::SI_SPILL_S192_SAVE: 1014 case AMDGPU::SI_SPILL_S192_RESTORE: 1015 case AMDGPU::SI_SPILL_V192_SAVE: 1016 case AMDGPU::SI_SPILL_V192_RESTORE: 1017 case AMDGPU::SI_SPILL_A192_SAVE: 1018 case AMDGPU::SI_SPILL_A192_RESTORE: 1019 case AMDGPU::SI_SPILL_AV192_SAVE: 1020 case AMDGPU::SI_SPILL_AV192_RESTORE: 1021 return 6; 1022 case AMDGPU::SI_SPILL_S160_SAVE: 1023 case AMDGPU::SI_SPILL_S160_RESTORE: 1024 case AMDGPU::SI_SPILL_V160_SAVE: 1025 case AMDGPU::SI_SPILL_V160_RESTORE: 1026 case AMDGPU::SI_SPILL_A160_SAVE: 1027 case AMDGPU::SI_SPILL_A160_RESTORE: 1028 case AMDGPU::SI_SPILL_AV160_SAVE: 1029 case AMDGPU::SI_SPILL_AV160_RESTORE: 1030 return 5; 1031 case AMDGPU::SI_SPILL_S128_SAVE: 1032 case AMDGPU::SI_SPILL_S128_RESTORE: 1033 case AMDGPU::SI_SPILL_V128_SAVE: 1034 case AMDGPU::SI_SPILL_V128_RESTORE: 1035 case AMDGPU::SI_SPILL_A128_SAVE: 1036 case AMDGPU::SI_SPILL_A128_RESTORE: 1037 case AMDGPU::SI_SPILL_AV128_SAVE: 1038 case AMDGPU::SI_SPILL_AV128_RESTORE: 1039 return 4; 1040 case AMDGPU::SI_SPILL_S96_SAVE: 1041 case AMDGPU::SI_SPILL_S96_RESTORE: 1042 case AMDGPU::SI_SPILL_V96_SAVE: 1043 case AMDGPU::SI_SPILL_V96_RESTORE: 1044 case AMDGPU::SI_SPILL_A96_SAVE: 1045 case AMDGPU::SI_SPILL_A96_RESTORE: 1046 case AMDGPU::SI_SPILL_AV96_SAVE: 1047 case AMDGPU::SI_SPILL_AV96_RESTORE: 1048 return 3; 1049 case AMDGPU::SI_SPILL_S64_SAVE: 1050 case AMDGPU::SI_SPILL_S64_RESTORE: 1051 case AMDGPU::SI_SPILL_V64_SAVE: 1052 case AMDGPU::SI_SPILL_V64_RESTORE: 1053 case AMDGPU::SI_SPILL_A64_SAVE: 1054 case AMDGPU::SI_SPILL_A64_RESTORE: 1055 case AMDGPU::SI_SPILL_AV64_SAVE: 1056 case AMDGPU::SI_SPILL_AV64_RESTORE: 1057 return 2; 1058 case AMDGPU::SI_SPILL_S32_SAVE: 1059 case AMDGPU::SI_SPILL_S32_RESTORE: 1060 case AMDGPU::SI_SPILL_V32_SAVE: 1061 case AMDGPU::SI_SPILL_V32_RESTORE: 1062 case AMDGPU::SI_SPILL_A32_SAVE: 1063 case AMDGPU::SI_SPILL_A32_RESTORE: 1064 case AMDGPU::SI_SPILL_AV32_SAVE: 1065 case AMDGPU::SI_SPILL_AV32_RESTORE: 1066 case AMDGPU::SI_SPILL_WWM_V32_SAVE: 1067 case AMDGPU::SI_SPILL_WWM_V32_RESTORE: 1068 return 1; 1069 default: llvm_unreachable("Invalid spill opcode"); 1070 } 1071 } 1072 1073 static int getOffsetMUBUFStore(unsigned Opc) { 1074 switch (Opc) { 1075 case AMDGPU::BUFFER_STORE_DWORD_OFFEN: 1076 return AMDGPU::BUFFER_STORE_DWORD_OFFSET; 1077 case AMDGPU::BUFFER_STORE_BYTE_OFFEN: 1078 return AMDGPU::BUFFER_STORE_BYTE_OFFSET; 1079 case AMDGPU::BUFFER_STORE_SHORT_OFFEN: 1080 return AMDGPU::BUFFER_STORE_SHORT_OFFSET; 1081 case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN: 1082 return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET; 1083 case AMDGPU::BUFFER_STORE_DWORDX3_OFFEN: 1084 return AMDGPU::BUFFER_STORE_DWORDX3_OFFSET; 1085 case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN: 1086 return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET; 1087 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN: 1088 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET; 1089 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN: 1090 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET; 1091 default: 1092 return -1; 1093 } 1094 } 1095 1096 static int getOffsetMUBUFLoad(unsigned Opc) { 1097 switch (Opc) { 1098 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN: 1099 return AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 1100 case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN: 1101 return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET; 1102 case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN: 1103 return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET; 1104 case AMDGPU::BUFFER_LOAD_USHORT_OFFEN: 1105 return AMDGPU::BUFFER_LOAD_USHORT_OFFSET; 1106 case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN: 1107 return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET; 1108 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN: 1109 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET; 1110 case AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN: 1111 return AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET; 1112 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN: 1113 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET; 1114 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN: 1115 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET; 1116 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN: 1117 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET; 1118 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN: 1119 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET; 1120 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN: 1121 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET; 1122 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN: 1123 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET; 1124 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN: 1125 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET; 1126 default: 1127 return -1; 1128 } 1129 } 1130 1131 static int getOffenMUBUFStore(unsigned Opc) { 1132 switch (Opc) { 1133 case AMDGPU::BUFFER_STORE_DWORD_OFFSET: 1134 return AMDGPU::BUFFER_STORE_DWORD_OFFEN; 1135 case AMDGPU::BUFFER_STORE_BYTE_OFFSET: 1136 return AMDGPU::BUFFER_STORE_BYTE_OFFEN; 1137 case AMDGPU::BUFFER_STORE_SHORT_OFFSET: 1138 return AMDGPU::BUFFER_STORE_SHORT_OFFEN; 1139 case AMDGPU::BUFFER_STORE_DWORDX2_OFFSET: 1140 return AMDGPU::BUFFER_STORE_DWORDX2_OFFEN; 1141 case AMDGPU::BUFFER_STORE_DWORDX3_OFFSET: 1142 return AMDGPU::BUFFER_STORE_DWORDX3_OFFEN; 1143 case AMDGPU::BUFFER_STORE_DWORDX4_OFFSET: 1144 return AMDGPU::BUFFER_STORE_DWORDX4_OFFEN; 1145 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET: 1146 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN; 1147 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET: 1148 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN; 1149 default: 1150 return -1; 1151 } 1152 } 1153 1154 static int getOffenMUBUFLoad(unsigned Opc) { 1155 switch (Opc) { 1156 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET: 1157 return AMDGPU::BUFFER_LOAD_DWORD_OFFEN; 1158 case AMDGPU::BUFFER_LOAD_UBYTE_OFFSET: 1159 return AMDGPU::BUFFER_LOAD_UBYTE_OFFEN; 1160 case AMDGPU::BUFFER_LOAD_SBYTE_OFFSET: 1161 return AMDGPU::BUFFER_LOAD_SBYTE_OFFEN; 1162 case AMDGPU::BUFFER_LOAD_USHORT_OFFSET: 1163 return AMDGPU::BUFFER_LOAD_USHORT_OFFEN; 1164 case AMDGPU::BUFFER_LOAD_SSHORT_OFFSET: 1165 return AMDGPU::BUFFER_LOAD_SSHORT_OFFEN; 1166 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET: 1167 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN; 1168 case AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET: 1169 return AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN; 1170 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET: 1171 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN; 1172 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET: 1173 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN; 1174 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET: 1175 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN; 1176 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET: 1177 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN; 1178 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET: 1179 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN; 1180 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET: 1181 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN; 1182 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET: 1183 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN; 1184 default: 1185 return -1; 1186 } 1187 } 1188 1189 static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST, 1190 MachineBasicBlock &MBB, 1191 MachineBasicBlock::iterator MI, 1192 int Index, unsigned Lane, 1193 unsigned ValueReg, bool IsKill) { 1194 MachineFunction *MF = MBB.getParent(); 1195 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1196 const SIInstrInfo *TII = ST.getInstrInfo(); 1197 1198 MCPhysReg Reg = MFI->getVGPRToAGPRSpill(Index, Lane); 1199 1200 if (Reg == AMDGPU::NoRegister) 1201 return MachineInstrBuilder(); 1202 1203 bool IsStore = MI->mayStore(); 1204 MachineRegisterInfo &MRI = MF->getRegInfo(); 1205 auto *TRI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo()); 1206 1207 unsigned Dst = IsStore ? Reg : ValueReg; 1208 unsigned Src = IsStore ? ValueReg : Reg; 1209 bool IsVGPR = TRI->isVGPR(MRI, Reg); 1210 DebugLoc DL = MI->getDebugLoc(); 1211 if (IsVGPR == TRI->isVGPR(MRI, ValueReg)) { 1212 // Spiller during regalloc may restore a spilled register to its superclass. 1213 // It could result in AGPR spills restored to VGPRs or the other way around, 1214 // making the src and dst with identical regclasses at this point. It just 1215 // needs a copy in such cases. 1216 auto CopyMIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), Dst) 1217 .addReg(Src, getKillRegState(IsKill)); 1218 CopyMIB->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1219 return CopyMIB; 1220 } 1221 unsigned Opc = (IsStore ^ IsVGPR) ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 1222 : AMDGPU::V_ACCVGPR_READ_B32_e64; 1223 1224 auto MIB = BuildMI(MBB, MI, DL, TII->get(Opc), Dst) 1225 .addReg(Src, getKillRegState(IsKill)); 1226 MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1227 return MIB; 1228 } 1229 1230 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not 1231 // need to handle the case where an SGPR may need to be spilled while spilling. 1232 static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST, 1233 MachineFrameInfo &MFI, 1234 MachineBasicBlock::iterator MI, 1235 int Index, 1236 int64_t Offset) { 1237 const SIInstrInfo *TII = ST.getInstrInfo(); 1238 MachineBasicBlock *MBB = MI->getParent(); 1239 const DebugLoc &DL = MI->getDebugLoc(); 1240 bool IsStore = MI->mayStore(); 1241 1242 unsigned Opc = MI->getOpcode(); 1243 int LoadStoreOp = IsStore ? 1244 getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc); 1245 if (LoadStoreOp == -1) 1246 return false; 1247 1248 const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); 1249 if (spillVGPRtoAGPR(ST, *MBB, MI, Index, 0, Reg->getReg(), false).getInstr()) 1250 return true; 1251 1252 MachineInstrBuilder NewMI = 1253 BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp)) 1254 .add(*Reg) 1255 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)) 1256 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)) 1257 .addImm(Offset) 1258 .addImm(0) // cpol 1259 .addImm(0) // swz 1260 .cloneMemRefs(*MI); 1261 1262 const MachineOperand *VDataIn = TII->getNamedOperand(*MI, 1263 AMDGPU::OpName::vdata_in); 1264 if (VDataIn) 1265 NewMI.add(*VDataIn); 1266 return true; 1267 } 1268 1269 static unsigned getFlatScratchSpillOpcode(const SIInstrInfo *TII, 1270 unsigned LoadStoreOp, 1271 unsigned EltSize) { 1272 bool IsStore = TII->get(LoadStoreOp).mayStore(); 1273 bool HasVAddr = AMDGPU::hasNamedOperand(LoadStoreOp, AMDGPU::OpName::vaddr); 1274 bool UseST = 1275 !HasVAddr && !AMDGPU::hasNamedOperand(LoadStoreOp, AMDGPU::OpName::saddr); 1276 1277 switch (EltSize) { 1278 case 4: 1279 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORD_SADDR 1280 : AMDGPU::SCRATCH_LOAD_DWORD_SADDR; 1281 break; 1282 case 8: 1283 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX2_SADDR 1284 : AMDGPU::SCRATCH_LOAD_DWORDX2_SADDR; 1285 break; 1286 case 12: 1287 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX3_SADDR 1288 : AMDGPU::SCRATCH_LOAD_DWORDX3_SADDR; 1289 break; 1290 case 16: 1291 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX4_SADDR 1292 : AMDGPU::SCRATCH_LOAD_DWORDX4_SADDR; 1293 break; 1294 default: 1295 llvm_unreachable("Unexpected spill load/store size!"); 1296 } 1297 1298 if (HasVAddr) 1299 LoadStoreOp = AMDGPU::getFlatScratchInstSVfromSS(LoadStoreOp); 1300 else if (UseST) 1301 LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp); 1302 1303 return LoadStoreOp; 1304 } 1305 1306 void SIRegisterInfo::buildSpillLoadStore( 1307 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, 1308 unsigned LoadStoreOp, int Index, Register ValueReg, bool IsKill, 1309 MCRegister ScratchOffsetReg, int64_t InstOffset, MachineMemOperand *MMO, 1310 RegScavenger *RS, LivePhysRegs *LiveRegs) const { 1311 assert((!RS || !LiveRegs) && "Only RS or LiveRegs can be set but not both"); 1312 1313 MachineFunction *MF = MBB.getParent(); 1314 const SIInstrInfo *TII = ST.getInstrInfo(); 1315 const MachineFrameInfo &MFI = MF->getFrameInfo(); 1316 const SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>(); 1317 1318 const MCInstrDesc *Desc = &TII->get(LoadStoreOp); 1319 bool IsStore = Desc->mayStore(); 1320 bool IsFlat = TII->isFLATScratch(LoadStoreOp); 1321 1322 bool CanClobberSCC = false; 1323 bool Scavenged = false; 1324 MCRegister SOffset = ScratchOffsetReg; 1325 1326 const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg); 1327 // On gfx90a+ AGPR is a regular VGPR acceptable for loads and stores. 1328 const bool IsAGPR = !ST.hasGFX90AInsts() && isAGPRClass(RC); 1329 const unsigned RegWidth = AMDGPU::getRegBitWidth(*RC) / 8; 1330 1331 // Always use 4 byte operations for AGPRs because we need to scavenge 1332 // a temporary VGPR. 1333 unsigned EltSize = (IsFlat && !IsAGPR) ? std::min(RegWidth, 16u) : 4u; 1334 unsigned NumSubRegs = RegWidth / EltSize; 1335 unsigned Size = NumSubRegs * EltSize; 1336 unsigned RemSize = RegWidth - Size; 1337 unsigned NumRemSubRegs = RemSize ? 1 : 0; 1338 int64_t Offset = InstOffset + MFI.getObjectOffset(Index); 1339 int64_t MaterializedOffset = Offset; 1340 1341 int64_t MaxOffset = Offset + Size + RemSize - EltSize; 1342 int64_t ScratchOffsetRegDelta = 0; 1343 1344 if (IsFlat && EltSize > 4) { 1345 LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize); 1346 Desc = &TII->get(LoadStoreOp); 1347 } 1348 1349 Align Alignment = MFI.getObjectAlign(Index); 1350 const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo(); 1351 1352 assert((IsFlat || ((Offset % EltSize) == 0)) && 1353 "unexpected VGPR spill offset"); 1354 1355 // Track a VGPR to use for a constant offset we need to materialize. 1356 Register TmpOffsetVGPR; 1357 1358 // Track a VGPR to use as an intermediate value. 1359 Register TmpIntermediateVGPR; 1360 bool UseVGPROffset = false; 1361 1362 // Materialize a VGPR offset required for the given SGPR/VGPR/Immediate 1363 // combination. 1364 auto MaterializeVOffset = [&](Register SGPRBase, Register TmpVGPR, 1365 int64_t VOffset) { 1366 // We are using a VGPR offset 1367 if (IsFlat && SGPRBase) { 1368 // We only have 1 VGPR offset, or 1 SGPR offset. We don't have a free 1369 // SGPR, so perform the add as vector. 1370 // We don't need a base SGPR in the kernel. 1371 1372 if (ST.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) >= 2) { 1373 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64), TmpVGPR) 1374 .addReg(SGPRBase) 1375 .addImm(VOffset) 1376 .addImm(0); // clamp 1377 } else { 1378 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) 1379 .addReg(SGPRBase); 1380 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e32), TmpVGPR) 1381 .addImm(VOffset) 1382 .addReg(TmpOffsetVGPR); 1383 } 1384 } else { 1385 assert(TmpOffsetVGPR); 1386 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) 1387 .addImm(VOffset); 1388 } 1389 }; 1390 1391 bool IsOffsetLegal = 1392 IsFlat ? TII->isLegalFLATOffset(MaxOffset, AMDGPUAS::PRIVATE_ADDRESS, 1393 SIInstrFlags::FlatScratch) 1394 : SIInstrInfo::isLegalMUBUFImmOffset(MaxOffset); 1395 if (!IsOffsetLegal || (IsFlat && !SOffset && !ST.hasFlatScratchSTMode())) { 1396 SOffset = MCRegister(); 1397 1398 // We don't have access to the register scavenger if this function is called 1399 // during PEI::scavengeFrameVirtualRegs() so use LiveRegs in this case. 1400 // TODO: Clobbering SCC is not necessary for scratch instructions in the 1401 // entry. 1402 if (RS) { 1403 SOffset = RS->scavengeRegisterBackwards(AMDGPU::SGPR_32RegClass, MI, false, 0, false); 1404 1405 // Piggy back on the liveness scan we just did see if SCC is dead. 1406 CanClobberSCC = !RS->isRegUsed(AMDGPU::SCC); 1407 } else if (LiveRegs) { 1408 CanClobberSCC = !LiveRegs->contains(AMDGPU::SCC); 1409 for (MCRegister Reg : AMDGPU::SGPR_32RegClass) { 1410 if (LiveRegs->available(MF->getRegInfo(), Reg)) { 1411 SOffset = Reg; 1412 break; 1413 } 1414 } 1415 } 1416 1417 if (ScratchOffsetReg != AMDGPU::NoRegister && !CanClobberSCC) 1418 SOffset = Register(); 1419 1420 if (!SOffset) { 1421 UseVGPROffset = true; 1422 1423 if (RS) { 1424 TmpOffsetVGPR = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, false, 0); 1425 } else { 1426 assert(LiveRegs); 1427 for (MCRegister Reg : AMDGPU::VGPR_32RegClass) { 1428 if (LiveRegs->available(MF->getRegInfo(), Reg)) { 1429 TmpOffsetVGPR = Reg; 1430 break; 1431 } 1432 } 1433 } 1434 1435 assert(TmpOffsetVGPR); 1436 } else if (!SOffset && CanClobberSCC) { 1437 // There are no free SGPRs, and since we are in the process of spilling 1438 // VGPRs too. Since we need a VGPR in order to spill SGPRs (this is true 1439 // on SI/CI and on VI it is true until we implement spilling using scalar 1440 // stores), we have no way to free up an SGPR. Our solution here is to 1441 // add the offset directly to the ScratchOffset or StackPtrOffset 1442 // register, and then subtract the offset after the spill to return the 1443 // register to it's original value. 1444 1445 // TODO: If we don't have to do an emergency stack slot spill, converting 1446 // to use the VGPR offset is fewer instructions. 1447 if (!ScratchOffsetReg) 1448 ScratchOffsetReg = FuncInfo->getStackPtrOffsetReg(); 1449 SOffset = ScratchOffsetReg; 1450 ScratchOffsetRegDelta = Offset; 1451 } else { 1452 Scavenged = true; 1453 } 1454 1455 // We currently only support spilling VGPRs to EltSize boundaries, meaning 1456 // we can simplify the adjustment of Offset here to just scale with 1457 // WavefrontSize. 1458 if (!IsFlat && !UseVGPROffset) 1459 Offset *= ST.getWavefrontSize(); 1460 1461 if (!UseVGPROffset && !SOffset) 1462 report_fatal_error("could not scavenge SGPR to spill in entry function"); 1463 1464 if (UseVGPROffset) { 1465 // We are using a VGPR offset 1466 MaterializeVOffset(ScratchOffsetReg, TmpOffsetVGPR, Offset); 1467 } else if (ScratchOffsetReg == AMDGPU::NoRegister) { 1468 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset).addImm(Offset); 1469 } else { 1470 assert(Offset != 0); 1471 auto Add = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset) 1472 .addReg(ScratchOffsetReg) 1473 .addImm(Offset); 1474 Add->getOperand(3).setIsDead(); // Mark SCC as dead. 1475 } 1476 1477 Offset = 0; 1478 } 1479 1480 if (IsFlat && SOffset == AMDGPU::NoRegister) { 1481 assert(AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::vaddr) < 0 1482 && "Unexpected vaddr for flat scratch with a FI operand"); 1483 1484 if (UseVGPROffset) { 1485 LoadStoreOp = AMDGPU::getFlatScratchInstSVfromSS(LoadStoreOp); 1486 } else { 1487 assert(ST.hasFlatScratchSTMode()); 1488 LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp); 1489 } 1490 1491 Desc = &TII->get(LoadStoreOp); 1492 } 1493 1494 for (unsigned i = 0, e = NumSubRegs + NumRemSubRegs, RegOffset = 0; i != e; 1495 ++i, RegOffset += EltSize) { 1496 if (i == NumSubRegs) { 1497 EltSize = RemSize; 1498 LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize); 1499 } 1500 Desc = &TII->get(LoadStoreOp); 1501 1502 if (!IsFlat && UseVGPROffset) { 1503 int NewLoadStoreOp = IsStore ? getOffenMUBUFStore(LoadStoreOp) 1504 : getOffenMUBUFLoad(LoadStoreOp); 1505 Desc = &TII->get(NewLoadStoreOp); 1506 } 1507 1508 if (UseVGPROffset && TmpOffsetVGPR == TmpIntermediateVGPR) { 1509 // If we are spilling an AGPR beyond the range of the memory instruction 1510 // offset and need to use a VGPR offset, we ideally have at least 2 1511 // scratch VGPRs. If we don't have a second free VGPR without spilling, 1512 // recycle the VGPR used for the offset which requires resetting after 1513 // each subregister. 1514 1515 MaterializeVOffset(ScratchOffsetReg, TmpOffsetVGPR, MaterializedOffset); 1516 } 1517 1518 unsigned NumRegs = EltSize / 4; 1519 Register SubReg = e == 1 1520 ? ValueReg 1521 : Register(getSubReg(ValueReg, 1522 getSubRegFromChannel(RegOffset / 4, NumRegs))); 1523 1524 unsigned SOffsetRegState = 0; 1525 unsigned SrcDstRegState = getDefRegState(!IsStore); 1526 const bool IsLastSubReg = i + 1 == e; 1527 const bool IsFirstSubReg = i == 0; 1528 if (IsLastSubReg) { 1529 SOffsetRegState |= getKillRegState(Scavenged); 1530 // The last implicit use carries the "Kill" flag. 1531 SrcDstRegState |= getKillRegState(IsKill); 1532 } 1533 1534 // Make sure the whole register is defined if there are undef components by 1535 // adding an implicit def of the super-reg on the first instruction. 1536 bool NeedSuperRegDef = e > 1 && IsStore && IsFirstSubReg; 1537 bool NeedSuperRegImpOperand = e > 1; 1538 1539 // Remaining element size to spill into memory after some parts of it 1540 // spilled into either AGPRs or VGPRs. 1541 unsigned RemEltSize = EltSize; 1542 1543 // AGPRs to spill VGPRs and vice versa are allocated in a reverse order, 1544 // starting from the last lane. In case if a register cannot be completely 1545 // spilled into another register that will ensure its alignment does not 1546 // change. For targets with VGPR alignment requirement this is important 1547 // in case of flat scratch usage as we might get a scratch_load or 1548 // scratch_store of an unaligned register otherwise. 1549 for (int LaneS = (RegOffset + EltSize) / 4 - 1, Lane = LaneS, 1550 LaneE = RegOffset / 4; 1551 Lane >= LaneE; --Lane) { 1552 bool IsSubReg = e > 1 || EltSize > 4; 1553 Register Sub = IsSubReg 1554 ? Register(getSubReg(ValueReg, getSubRegFromChannel(Lane))) 1555 : ValueReg; 1556 auto MIB = spillVGPRtoAGPR(ST, MBB, MI, Index, Lane, Sub, IsKill); 1557 if (!MIB.getInstr()) 1558 break; 1559 if (NeedSuperRegDef || (IsSubReg && IsStore && Lane == LaneS && IsFirstSubReg)) { 1560 MIB.addReg(ValueReg, RegState::ImplicitDefine); 1561 NeedSuperRegDef = false; 1562 } 1563 if ((IsSubReg || NeedSuperRegImpOperand) && (IsFirstSubReg || IsLastSubReg)) { 1564 NeedSuperRegImpOperand = true; 1565 unsigned State = SrcDstRegState; 1566 if (!IsLastSubReg || (Lane != LaneE)) 1567 State &= ~RegState::Kill; 1568 if (!IsFirstSubReg || (Lane != LaneS)) 1569 State &= ~RegState::Define; 1570 MIB.addReg(ValueReg, RegState::Implicit | State); 1571 } 1572 RemEltSize -= 4; 1573 } 1574 1575 if (!RemEltSize) // Fully spilled into AGPRs. 1576 continue; 1577 1578 if (RemEltSize != EltSize) { // Partially spilled to AGPRs 1579 assert(IsFlat && EltSize > 4); 1580 1581 unsigned NumRegs = RemEltSize / 4; 1582 SubReg = Register(getSubReg(ValueReg, 1583 getSubRegFromChannel(RegOffset / 4, NumRegs))); 1584 unsigned Opc = getFlatScratchSpillOpcode(TII, LoadStoreOp, RemEltSize); 1585 Desc = &TII->get(Opc); 1586 } 1587 1588 unsigned FinalReg = SubReg; 1589 1590 if (IsAGPR) { 1591 assert(EltSize == 4); 1592 1593 if (!TmpIntermediateVGPR) { 1594 TmpIntermediateVGPR = FuncInfo->getVGPRForAGPRCopy(); 1595 assert(MF->getRegInfo().isReserved(TmpIntermediateVGPR)); 1596 } 1597 if (IsStore) { 1598 auto AccRead = BuildMI(MBB, MI, DL, 1599 TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64), 1600 TmpIntermediateVGPR) 1601 .addReg(SubReg, getKillRegState(IsKill)); 1602 if (NeedSuperRegDef) 1603 AccRead.addReg(ValueReg, RegState::ImplicitDefine); 1604 AccRead->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1605 } 1606 SubReg = TmpIntermediateVGPR; 1607 } else if (UseVGPROffset) { 1608 // FIXME: change to scavengeRegisterBackwards() 1609 if (!TmpOffsetVGPR) { 1610 TmpOffsetVGPR = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, 1611 MI, false, 0); 1612 RS->setRegUsed(TmpOffsetVGPR); 1613 } 1614 } 1615 1616 MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(RegOffset); 1617 MachineMemOperand *NewMMO = 1618 MF->getMachineMemOperand(PInfo, MMO->getFlags(), RemEltSize, 1619 commonAlignment(Alignment, RegOffset)); 1620 1621 auto MIB = 1622 BuildMI(MBB, MI, DL, *Desc) 1623 .addReg(SubReg, getDefRegState(!IsStore) | getKillRegState(IsKill)); 1624 1625 if (UseVGPROffset) { 1626 // For an AGPR spill, we reuse the same temp VGPR for the offset and the 1627 // intermediate accvgpr_write. 1628 MIB.addReg(TmpOffsetVGPR, getKillRegState(IsLastSubReg && !IsAGPR)); 1629 } 1630 1631 if (!IsFlat) 1632 MIB.addReg(FuncInfo->getScratchRSrcReg()); 1633 1634 if (SOffset == AMDGPU::NoRegister) { 1635 if (!IsFlat) { 1636 if (UseVGPROffset && ScratchOffsetReg) { 1637 MIB.addReg(ScratchOffsetReg); 1638 } else { 1639 assert(FuncInfo->isEntryFunction()); 1640 MIB.addImm(0); 1641 } 1642 } 1643 } else { 1644 MIB.addReg(SOffset, SOffsetRegState); 1645 } 1646 MIB.addImm(Offset + RegOffset) 1647 .addImm(0); // cpol 1648 if (!IsFlat) 1649 MIB.addImm(0); // swz 1650 MIB.addMemOperand(NewMMO); 1651 1652 if (!IsAGPR && NeedSuperRegDef) 1653 MIB.addReg(ValueReg, RegState::ImplicitDefine); 1654 1655 if (!IsStore && IsAGPR && TmpIntermediateVGPR != AMDGPU::NoRegister) { 1656 MIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), 1657 FinalReg) 1658 .addReg(TmpIntermediateVGPR, RegState::Kill); 1659 MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1660 } 1661 1662 if (NeedSuperRegImpOperand && (IsFirstSubReg || IsLastSubReg)) 1663 MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState); 1664 1665 // The epilog restore of a wwm-scratch register can cause undesired 1666 // optimization during machine-cp post PrologEpilogInserter if the same 1667 // register was assigned for return value ABI lowering with a COPY 1668 // instruction. As given below, with the epilog reload, the earlier COPY 1669 // appeared to be dead during machine-cp. 1670 // ... 1671 // v0 in WWM operation, needs the WWM spill at prolog/epilog. 1672 // $vgpr0 = V_WRITELANE_B32 $sgpr20, 0, $vgpr0 1673 // ... 1674 // Epilog block: 1675 // $vgpr0 = COPY $vgpr1 // outgoing value moved to v0 1676 // ... 1677 // WWM spill restore to preserve the inactive lanes of v0. 1678 // $sgpr4_sgpr5 = S_XOR_SAVEEXEC_B64 -1 1679 // $vgpr0 = BUFFER_LOAD $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0 1680 // $exec = S_MOV_B64 killed $sgpr4_sgpr5 1681 // ... 1682 // SI_RETURN implicit $vgpr0 1683 // ... 1684 // To fix it, mark the same reg as a tied op for such restore instructions 1685 // so that it marks a usage for the preceding COPY. 1686 if (!IsStore && MI != MBB.end() && MI->isReturn() && 1687 MI->readsRegister(SubReg, this)) { 1688 MIB.addReg(SubReg, RegState::Implicit); 1689 MIB->tieOperands(0, MIB->getNumOperands() - 1); 1690 } 1691 } 1692 1693 if (ScratchOffsetRegDelta != 0) { 1694 // Subtract the offset we added to the ScratchOffset register. 1695 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset) 1696 .addReg(SOffset) 1697 .addImm(-ScratchOffsetRegDelta); 1698 } 1699 } 1700 1701 void SIRegisterInfo::buildVGPRSpillLoadStore(SGPRSpillBuilder &SB, int Index, 1702 int Offset, bool IsLoad, 1703 bool IsKill) const { 1704 // Load/store VGPR 1705 MachineFrameInfo &FrameInfo = SB.MF.getFrameInfo(); 1706 assert(FrameInfo.getStackID(Index) != TargetStackID::SGPRSpill); 1707 1708 Register FrameReg = 1709 FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(SB.MF) 1710 ? getBaseRegister() 1711 : getFrameRegister(SB.MF); 1712 1713 Align Alignment = FrameInfo.getObjectAlign(Index); 1714 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SB.MF, Index); 1715 MachineMemOperand *MMO = SB.MF.getMachineMemOperand( 1716 PtrInfo, IsLoad ? MachineMemOperand::MOLoad : MachineMemOperand::MOStore, 1717 SB.EltSize, Alignment); 1718 1719 if (IsLoad) { 1720 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR 1721 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 1722 buildSpillLoadStore(*SB.MBB, SB.MI, SB.DL, Opc, Index, SB.TmpVGPR, false, 1723 FrameReg, Offset * SB.EltSize, MMO, SB.RS); 1724 } else { 1725 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR 1726 : AMDGPU::BUFFER_STORE_DWORD_OFFSET; 1727 buildSpillLoadStore(*SB.MBB, SB.MI, SB.DL, Opc, Index, SB.TmpVGPR, IsKill, 1728 FrameReg, Offset * SB.EltSize, MMO, SB.RS); 1729 // This only ever adds one VGPR spill 1730 SB.MFI.addToSpilledVGPRs(1); 1731 } 1732 } 1733 1734 bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, int Index, 1735 RegScavenger *RS, SlotIndexes *Indexes, 1736 LiveIntervals *LIS, bool OnlyToVGPR, 1737 bool SpillToPhysVGPRLane) const { 1738 SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, Index, RS); 1739 1740 ArrayRef<SpilledReg> VGPRSpills = 1741 SpillToPhysVGPRLane ? SB.MFI.getSGPRSpillToPhysicalVGPRLanes(Index) 1742 : SB.MFI.getSGPRSpillToVirtualVGPRLanes(Index); 1743 bool SpillToVGPR = !VGPRSpills.empty(); 1744 if (OnlyToVGPR && !SpillToVGPR) 1745 return false; 1746 1747 assert(SpillToVGPR || (SB.SuperReg != SB.MFI.getStackPtrOffsetReg() && 1748 SB.SuperReg != SB.MFI.getFrameOffsetReg())); 1749 1750 if (SpillToVGPR) { 1751 1752 assert(SB.NumSubRegs == VGPRSpills.size() && 1753 "Num of VGPR lanes should be equal to num of SGPRs spilled"); 1754 1755 for (unsigned i = 0, e = SB.NumSubRegs; i < e; ++i) { 1756 Register SubReg = 1757 SB.NumSubRegs == 1 1758 ? SB.SuperReg 1759 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1760 SpilledReg Spill = VGPRSpills[i]; 1761 1762 bool IsFirstSubreg = i == 0; 1763 bool IsLastSubreg = i == SB.NumSubRegs - 1; 1764 bool UseKill = SB.IsKill && IsLastSubreg; 1765 1766 1767 // Mark the "old value of vgpr" input undef only if this is the first sgpr 1768 // spill to this specific vgpr in the first basic block. 1769 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, 1770 SB.TII.get(AMDGPU::V_WRITELANE_B32), Spill.VGPR) 1771 .addReg(SubReg, getKillRegState(UseKill)) 1772 .addImm(Spill.Lane) 1773 .addReg(Spill.VGPR); 1774 if (Indexes) { 1775 if (IsFirstSubreg) 1776 Indexes->replaceMachineInstrInMaps(*MI, *MIB); 1777 else 1778 Indexes->insertMachineInstrInMaps(*MIB); 1779 } 1780 1781 if (IsFirstSubreg && SB.NumSubRegs > 1) { 1782 // We may be spilling a super-register which is only partially defined, 1783 // and need to ensure later spills think the value is defined. 1784 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1785 } 1786 1787 if (SB.NumSubRegs > 1 && (IsFirstSubreg || IsLastSubreg)) 1788 MIB.addReg(SB.SuperReg, getKillRegState(UseKill) | RegState::Implicit); 1789 1790 // FIXME: Since this spills to another register instead of an actual 1791 // frame index, we should delete the frame index when all references to 1792 // it are fixed. 1793 } 1794 } else { 1795 SB.prepare(); 1796 1797 // SubReg carries the "Kill" flag when SubReg == SB.SuperReg. 1798 unsigned SubKillState = getKillRegState((SB.NumSubRegs == 1) && SB.IsKill); 1799 1800 // Per VGPR helper data 1801 auto PVD = SB.getPerVGPRData(); 1802 1803 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1804 unsigned TmpVGPRFlags = RegState::Undef; 1805 1806 // Write sub registers into the VGPR 1807 for (unsigned i = Offset * PVD.PerVGPR, 1808 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1809 i < e; ++i) { 1810 Register SubReg = 1811 SB.NumSubRegs == 1 1812 ? SB.SuperReg 1813 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1814 1815 MachineInstrBuilder WriteLane = 1816 BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_WRITELANE_B32), 1817 SB.TmpVGPR) 1818 .addReg(SubReg, SubKillState) 1819 .addImm(i % PVD.PerVGPR) 1820 .addReg(SB.TmpVGPR, TmpVGPRFlags); 1821 TmpVGPRFlags = 0; 1822 1823 if (Indexes) { 1824 if (i == 0) 1825 Indexes->replaceMachineInstrInMaps(*MI, *WriteLane); 1826 else 1827 Indexes->insertMachineInstrInMaps(*WriteLane); 1828 } 1829 1830 // There could be undef components of a spilled super register. 1831 // TODO: Can we detect this and skip the spill? 1832 if (SB.NumSubRegs > 1) { 1833 // The last implicit use of the SB.SuperReg carries the "Kill" flag. 1834 unsigned SuperKillState = 0; 1835 if (i + 1 == SB.NumSubRegs) 1836 SuperKillState |= getKillRegState(SB.IsKill); 1837 WriteLane.addReg(SB.SuperReg, RegState::Implicit | SuperKillState); 1838 } 1839 } 1840 1841 // Write out VGPR 1842 SB.readWriteTmpVGPR(Offset, /*IsLoad*/ false); 1843 } 1844 1845 SB.restore(); 1846 } 1847 1848 MI->eraseFromParent(); 1849 SB.MFI.addToSpilledSGPRs(SB.NumSubRegs); 1850 1851 if (LIS) 1852 LIS->removeAllRegUnitsForPhysReg(SB.SuperReg); 1853 1854 return true; 1855 } 1856 1857 bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, int Index, 1858 RegScavenger *RS, SlotIndexes *Indexes, 1859 LiveIntervals *LIS, bool OnlyToVGPR, 1860 bool SpillToPhysVGPRLane) const { 1861 SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, Index, RS); 1862 1863 ArrayRef<SpilledReg> VGPRSpills = 1864 SpillToPhysVGPRLane ? SB.MFI.getSGPRSpillToPhysicalVGPRLanes(Index) 1865 : SB.MFI.getSGPRSpillToVirtualVGPRLanes(Index); 1866 bool SpillToVGPR = !VGPRSpills.empty(); 1867 if (OnlyToVGPR && !SpillToVGPR) 1868 return false; 1869 1870 if (SpillToVGPR) { 1871 for (unsigned i = 0, e = SB.NumSubRegs; i < e; ++i) { 1872 Register SubReg = 1873 SB.NumSubRegs == 1 1874 ? SB.SuperReg 1875 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1876 1877 SpilledReg Spill = VGPRSpills[i]; 1878 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_READLANE_B32), 1879 SubReg) 1880 .addReg(Spill.VGPR) 1881 .addImm(Spill.Lane); 1882 if (SB.NumSubRegs > 1 && i == 0) 1883 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1884 if (Indexes) { 1885 if (i == e - 1) 1886 Indexes->replaceMachineInstrInMaps(*MI, *MIB); 1887 else 1888 Indexes->insertMachineInstrInMaps(*MIB); 1889 } 1890 } 1891 } else { 1892 SB.prepare(); 1893 1894 // Per VGPR helper data 1895 auto PVD = SB.getPerVGPRData(); 1896 1897 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1898 // Load in VGPR data 1899 SB.readWriteTmpVGPR(Offset, /*IsLoad*/ true); 1900 1901 // Unpack lanes 1902 for (unsigned i = Offset * PVD.PerVGPR, 1903 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1904 i < e; ++i) { 1905 Register SubReg = 1906 SB.NumSubRegs == 1 1907 ? SB.SuperReg 1908 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1909 1910 bool LastSubReg = (i + 1 == e); 1911 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, 1912 SB.TII.get(AMDGPU::V_READLANE_B32), SubReg) 1913 .addReg(SB.TmpVGPR, getKillRegState(LastSubReg)) 1914 .addImm(i); 1915 if (SB.NumSubRegs > 1 && i == 0) 1916 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1917 if (Indexes) { 1918 if (i == e - 1) 1919 Indexes->replaceMachineInstrInMaps(*MI, *MIB); 1920 else 1921 Indexes->insertMachineInstrInMaps(*MIB); 1922 } 1923 } 1924 } 1925 1926 SB.restore(); 1927 } 1928 1929 MI->eraseFromParent(); 1930 1931 if (LIS) 1932 LIS->removeAllRegUnitsForPhysReg(SB.SuperReg); 1933 1934 return true; 1935 } 1936 1937 bool SIRegisterInfo::spillEmergencySGPR(MachineBasicBlock::iterator MI, 1938 MachineBasicBlock &RestoreMBB, 1939 Register SGPR, RegScavenger *RS) const { 1940 SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, SGPR, false, 0, 1941 RS); 1942 SB.prepare(); 1943 // Generate the spill of SGPR to SB.TmpVGPR. 1944 unsigned SubKillState = getKillRegState((SB.NumSubRegs == 1) && SB.IsKill); 1945 auto PVD = SB.getPerVGPRData(); 1946 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1947 unsigned TmpVGPRFlags = RegState::Undef; 1948 // Write sub registers into the VGPR 1949 for (unsigned i = Offset * PVD.PerVGPR, 1950 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1951 i < e; ++i) { 1952 Register SubReg = 1953 SB.NumSubRegs == 1 1954 ? SB.SuperReg 1955 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1956 1957 MachineInstrBuilder WriteLane = 1958 BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_WRITELANE_B32), 1959 SB.TmpVGPR) 1960 .addReg(SubReg, SubKillState) 1961 .addImm(i % PVD.PerVGPR) 1962 .addReg(SB.TmpVGPR, TmpVGPRFlags); 1963 TmpVGPRFlags = 0; 1964 // There could be undef components of a spilled super register. 1965 // TODO: Can we detect this and skip the spill? 1966 if (SB.NumSubRegs > 1) { 1967 // The last implicit use of the SB.SuperReg carries the "Kill" flag. 1968 unsigned SuperKillState = 0; 1969 if (i + 1 == SB.NumSubRegs) 1970 SuperKillState |= getKillRegState(SB.IsKill); 1971 WriteLane.addReg(SB.SuperReg, RegState::Implicit | SuperKillState); 1972 } 1973 } 1974 // Don't need to write VGPR out. 1975 } 1976 1977 // Restore clobbered registers in the specified restore block. 1978 MI = RestoreMBB.end(); 1979 SB.setMI(&RestoreMBB, MI); 1980 // Generate the restore of SGPR from SB.TmpVGPR. 1981 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1982 // Don't need to load VGPR in. 1983 // Unpack lanes 1984 for (unsigned i = Offset * PVD.PerVGPR, 1985 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1986 i < e; ++i) { 1987 Register SubReg = 1988 SB.NumSubRegs == 1 1989 ? SB.SuperReg 1990 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1991 bool LastSubReg = (i + 1 == e); 1992 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_READLANE_B32), 1993 SubReg) 1994 .addReg(SB.TmpVGPR, getKillRegState(LastSubReg)) 1995 .addImm(i); 1996 if (SB.NumSubRegs > 1 && i == 0) 1997 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1998 } 1999 } 2000 SB.restore(); 2001 2002 SB.MFI.addToSpilledSGPRs(SB.NumSubRegs); 2003 return false; 2004 } 2005 2006 /// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to 2007 /// a VGPR and the stack slot can be safely eliminated when all other users are 2008 /// handled. 2009 bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex( 2010 MachineBasicBlock::iterator MI, int FI, RegScavenger *RS, 2011 SlotIndexes *Indexes, LiveIntervals *LIS, bool SpillToPhysVGPRLane) const { 2012 switch (MI->getOpcode()) { 2013 case AMDGPU::SI_SPILL_S1024_SAVE: 2014 case AMDGPU::SI_SPILL_S512_SAVE: 2015 case AMDGPU::SI_SPILL_S384_SAVE: 2016 case AMDGPU::SI_SPILL_S352_SAVE: 2017 case AMDGPU::SI_SPILL_S320_SAVE: 2018 case AMDGPU::SI_SPILL_S288_SAVE: 2019 case AMDGPU::SI_SPILL_S256_SAVE: 2020 case AMDGPU::SI_SPILL_S224_SAVE: 2021 case AMDGPU::SI_SPILL_S192_SAVE: 2022 case AMDGPU::SI_SPILL_S160_SAVE: 2023 case AMDGPU::SI_SPILL_S128_SAVE: 2024 case AMDGPU::SI_SPILL_S96_SAVE: 2025 case AMDGPU::SI_SPILL_S64_SAVE: 2026 case AMDGPU::SI_SPILL_S32_SAVE: 2027 return spillSGPR(MI, FI, RS, Indexes, LIS, true, SpillToPhysVGPRLane); 2028 case AMDGPU::SI_SPILL_S1024_RESTORE: 2029 case AMDGPU::SI_SPILL_S512_RESTORE: 2030 case AMDGPU::SI_SPILL_S384_RESTORE: 2031 case AMDGPU::SI_SPILL_S352_RESTORE: 2032 case AMDGPU::SI_SPILL_S320_RESTORE: 2033 case AMDGPU::SI_SPILL_S288_RESTORE: 2034 case AMDGPU::SI_SPILL_S256_RESTORE: 2035 case AMDGPU::SI_SPILL_S224_RESTORE: 2036 case AMDGPU::SI_SPILL_S192_RESTORE: 2037 case AMDGPU::SI_SPILL_S160_RESTORE: 2038 case AMDGPU::SI_SPILL_S128_RESTORE: 2039 case AMDGPU::SI_SPILL_S96_RESTORE: 2040 case AMDGPU::SI_SPILL_S64_RESTORE: 2041 case AMDGPU::SI_SPILL_S32_RESTORE: 2042 return restoreSGPR(MI, FI, RS, Indexes, LIS, true, SpillToPhysVGPRLane); 2043 default: 2044 llvm_unreachable("not an SGPR spill instruction"); 2045 } 2046 } 2047 2048 bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, 2049 int SPAdj, unsigned FIOperandNum, 2050 RegScavenger *RS) const { 2051 MachineFunction *MF = MI->getParent()->getParent(); 2052 MachineBasicBlock *MBB = MI->getParent(); 2053 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 2054 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 2055 const SIInstrInfo *TII = ST.getInstrInfo(); 2056 DebugLoc DL = MI->getDebugLoc(); 2057 2058 assert(SPAdj == 0 && "unhandled SP adjustment in call sequence?"); 2059 2060 MachineOperand &FIOp = MI->getOperand(FIOperandNum); 2061 int Index = MI->getOperand(FIOperandNum).getIndex(); 2062 2063 Register FrameReg = FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(*MF) 2064 ? getBaseRegister() 2065 : getFrameRegister(*MF); 2066 2067 switch (MI->getOpcode()) { 2068 // SGPR register spill 2069 case AMDGPU::SI_SPILL_S1024_SAVE: 2070 case AMDGPU::SI_SPILL_S512_SAVE: 2071 case AMDGPU::SI_SPILL_S384_SAVE: 2072 case AMDGPU::SI_SPILL_S352_SAVE: 2073 case AMDGPU::SI_SPILL_S320_SAVE: 2074 case AMDGPU::SI_SPILL_S288_SAVE: 2075 case AMDGPU::SI_SPILL_S256_SAVE: 2076 case AMDGPU::SI_SPILL_S224_SAVE: 2077 case AMDGPU::SI_SPILL_S192_SAVE: 2078 case AMDGPU::SI_SPILL_S160_SAVE: 2079 case AMDGPU::SI_SPILL_S128_SAVE: 2080 case AMDGPU::SI_SPILL_S96_SAVE: 2081 case AMDGPU::SI_SPILL_S64_SAVE: 2082 case AMDGPU::SI_SPILL_S32_SAVE: { 2083 return spillSGPR(MI, Index, RS); 2084 } 2085 2086 // SGPR register restore 2087 case AMDGPU::SI_SPILL_S1024_RESTORE: 2088 case AMDGPU::SI_SPILL_S512_RESTORE: 2089 case AMDGPU::SI_SPILL_S384_RESTORE: 2090 case AMDGPU::SI_SPILL_S352_RESTORE: 2091 case AMDGPU::SI_SPILL_S320_RESTORE: 2092 case AMDGPU::SI_SPILL_S288_RESTORE: 2093 case AMDGPU::SI_SPILL_S256_RESTORE: 2094 case AMDGPU::SI_SPILL_S224_RESTORE: 2095 case AMDGPU::SI_SPILL_S192_RESTORE: 2096 case AMDGPU::SI_SPILL_S160_RESTORE: 2097 case AMDGPU::SI_SPILL_S128_RESTORE: 2098 case AMDGPU::SI_SPILL_S96_RESTORE: 2099 case AMDGPU::SI_SPILL_S64_RESTORE: 2100 case AMDGPU::SI_SPILL_S32_RESTORE: { 2101 return restoreSGPR(MI, Index, RS); 2102 } 2103 2104 // VGPR register spill 2105 case AMDGPU::SI_SPILL_V1024_SAVE: 2106 case AMDGPU::SI_SPILL_V512_SAVE: 2107 case AMDGPU::SI_SPILL_V384_SAVE: 2108 case AMDGPU::SI_SPILL_V352_SAVE: 2109 case AMDGPU::SI_SPILL_V320_SAVE: 2110 case AMDGPU::SI_SPILL_V288_SAVE: 2111 case AMDGPU::SI_SPILL_V256_SAVE: 2112 case AMDGPU::SI_SPILL_V224_SAVE: 2113 case AMDGPU::SI_SPILL_V192_SAVE: 2114 case AMDGPU::SI_SPILL_V160_SAVE: 2115 case AMDGPU::SI_SPILL_V128_SAVE: 2116 case AMDGPU::SI_SPILL_V96_SAVE: 2117 case AMDGPU::SI_SPILL_V64_SAVE: 2118 case AMDGPU::SI_SPILL_V32_SAVE: 2119 case AMDGPU::SI_SPILL_A1024_SAVE: 2120 case AMDGPU::SI_SPILL_A512_SAVE: 2121 case AMDGPU::SI_SPILL_A384_SAVE: 2122 case AMDGPU::SI_SPILL_A352_SAVE: 2123 case AMDGPU::SI_SPILL_A320_SAVE: 2124 case AMDGPU::SI_SPILL_A288_SAVE: 2125 case AMDGPU::SI_SPILL_A256_SAVE: 2126 case AMDGPU::SI_SPILL_A224_SAVE: 2127 case AMDGPU::SI_SPILL_A192_SAVE: 2128 case AMDGPU::SI_SPILL_A160_SAVE: 2129 case AMDGPU::SI_SPILL_A128_SAVE: 2130 case AMDGPU::SI_SPILL_A96_SAVE: 2131 case AMDGPU::SI_SPILL_A64_SAVE: 2132 case AMDGPU::SI_SPILL_A32_SAVE: 2133 case AMDGPU::SI_SPILL_AV1024_SAVE: 2134 case AMDGPU::SI_SPILL_AV512_SAVE: 2135 case AMDGPU::SI_SPILL_AV384_SAVE: 2136 case AMDGPU::SI_SPILL_AV352_SAVE: 2137 case AMDGPU::SI_SPILL_AV320_SAVE: 2138 case AMDGPU::SI_SPILL_AV288_SAVE: 2139 case AMDGPU::SI_SPILL_AV256_SAVE: 2140 case AMDGPU::SI_SPILL_AV224_SAVE: 2141 case AMDGPU::SI_SPILL_AV192_SAVE: 2142 case AMDGPU::SI_SPILL_AV160_SAVE: 2143 case AMDGPU::SI_SPILL_AV128_SAVE: 2144 case AMDGPU::SI_SPILL_AV96_SAVE: 2145 case AMDGPU::SI_SPILL_AV64_SAVE: 2146 case AMDGPU::SI_SPILL_AV32_SAVE: 2147 case AMDGPU::SI_SPILL_WWM_V32_SAVE: { 2148 const MachineOperand *VData = TII->getNamedOperand(*MI, 2149 AMDGPU::OpName::vdata); 2150 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 2151 MFI->getStackPtrOffsetReg()); 2152 2153 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR 2154 : AMDGPU::BUFFER_STORE_DWORD_OFFSET; 2155 auto *MBB = MI->getParent(); 2156 bool IsWWMRegSpill = TII->isWWMRegSpillOpcode(MI->getOpcode()); 2157 if (IsWWMRegSpill) { 2158 TII->insertScratchExecCopy(*MF, *MBB, MI, DL, MFI->getSGPRForEXECCopy(), 2159 RS->isRegUsed(AMDGPU::SCC)); 2160 } 2161 buildSpillLoadStore( 2162 *MBB, MI, DL, Opc, Index, VData->getReg(), VData->isKill(), FrameReg, 2163 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 2164 *MI->memoperands_begin(), RS); 2165 MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode())); 2166 if (IsWWMRegSpill) 2167 TII->restoreExec(*MF, *MBB, MI, DL, MFI->getSGPRForEXECCopy()); 2168 2169 MI->eraseFromParent(); 2170 return true; 2171 } 2172 case AMDGPU::SI_SPILL_V32_RESTORE: 2173 case AMDGPU::SI_SPILL_V64_RESTORE: 2174 case AMDGPU::SI_SPILL_V96_RESTORE: 2175 case AMDGPU::SI_SPILL_V128_RESTORE: 2176 case AMDGPU::SI_SPILL_V160_RESTORE: 2177 case AMDGPU::SI_SPILL_V192_RESTORE: 2178 case AMDGPU::SI_SPILL_V224_RESTORE: 2179 case AMDGPU::SI_SPILL_V256_RESTORE: 2180 case AMDGPU::SI_SPILL_V288_RESTORE: 2181 case AMDGPU::SI_SPILL_V320_RESTORE: 2182 case AMDGPU::SI_SPILL_V352_RESTORE: 2183 case AMDGPU::SI_SPILL_V384_RESTORE: 2184 case AMDGPU::SI_SPILL_V512_RESTORE: 2185 case AMDGPU::SI_SPILL_V1024_RESTORE: 2186 case AMDGPU::SI_SPILL_A32_RESTORE: 2187 case AMDGPU::SI_SPILL_A64_RESTORE: 2188 case AMDGPU::SI_SPILL_A96_RESTORE: 2189 case AMDGPU::SI_SPILL_A128_RESTORE: 2190 case AMDGPU::SI_SPILL_A160_RESTORE: 2191 case AMDGPU::SI_SPILL_A192_RESTORE: 2192 case AMDGPU::SI_SPILL_A224_RESTORE: 2193 case AMDGPU::SI_SPILL_A256_RESTORE: 2194 case AMDGPU::SI_SPILL_A288_RESTORE: 2195 case AMDGPU::SI_SPILL_A320_RESTORE: 2196 case AMDGPU::SI_SPILL_A352_RESTORE: 2197 case AMDGPU::SI_SPILL_A384_RESTORE: 2198 case AMDGPU::SI_SPILL_A512_RESTORE: 2199 case AMDGPU::SI_SPILL_A1024_RESTORE: 2200 case AMDGPU::SI_SPILL_AV32_RESTORE: 2201 case AMDGPU::SI_SPILL_AV64_RESTORE: 2202 case AMDGPU::SI_SPILL_AV96_RESTORE: 2203 case AMDGPU::SI_SPILL_AV128_RESTORE: 2204 case AMDGPU::SI_SPILL_AV160_RESTORE: 2205 case AMDGPU::SI_SPILL_AV192_RESTORE: 2206 case AMDGPU::SI_SPILL_AV224_RESTORE: 2207 case AMDGPU::SI_SPILL_AV256_RESTORE: 2208 case AMDGPU::SI_SPILL_AV288_RESTORE: 2209 case AMDGPU::SI_SPILL_AV320_RESTORE: 2210 case AMDGPU::SI_SPILL_AV352_RESTORE: 2211 case AMDGPU::SI_SPILL_AV384_RESTORE: 2212 case AMDGPU::SI_SPILL_AV512_RESTORE: 2213 case AMDGPU::SI_SPILL_AV1024_RESTORE: 2214 case AMDGPU::SI_SPILL_WWM_V32_RESTORE: { 2215 const MachineOperand *VData = TII->getNamedOperand(*MI, 2216 AMDGPU::OpName::vdata); 2217 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 2218 MFI->getStackPtrOffsetReg()); 2219 2220 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR 2221 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 2222 auto *MBB = MI->getParent(); 2223 bool IsWWMRegSpill = TII->isWWMRegSpillOpcode(MI->getOpcode()); 2224 if (IsWWMRegSpill) { 2225 TII->insertScratchExecCopy(*MF, *MBB, MI, DL, MFI->getSGPRForEXECCopy(), 2226 RS->isRegUsed(AMDGPU::SCC)); 2227 } 2228 buildSpillLoadStore( 2229 *MBB, MI, DL, Opc, Index, VData->getReg(), VData->isKill(), FrameReg, 2230 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 2231 *MI->memoperands_begin(), RS); 2232 2233 if (IsWWMRegSpill) 2234 TII->restoreExec(*MF, *MBB, MI, DL, MFI->getSGPRForEXECCopy()); 2235 2236 MI->eraseFromParent(); 2237 return true; 2238 } 2239 2240 default: { 2241 // Other access to frame index 2242 const DebugLoc &DL = MI->getDebugLoc(); 2243 2244 int64_t Offset = FrameInfo.getObjectOffset(Index); 2245 if (ST.enableFlatScratch()) { 2246 if (TII->isFLATScratch(*MI)) { 2247 assert((int16_t)FIOperandNum == 2248 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 2249 AMDGPU::OpName::saddr)); 2250 2251 // The offset is always swizzled, just replace it 2252 if (FrameReg) 2253 FIOp.ChangeToRegister(FrameReg, false); 2254 2255 if (!Offset) 2256 return false; 2257 2258 MachineOperand *OffsetOp = 2259 TII->getNamedOperand(*MI, AMDGPU::OpName::offset); 2260 int64_t NewOffset = Offset + OffsetOp->getImm(); 2261 if (TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, 2262 SIInstrFlags::FlatScratch)) { 2263 OffsetOp->setImm(NewOffset); 2264 if (FrameReg) 2265 return false; 2266 Offset = 0; 2267 } 2268 2269 if (!Offset) { 2270 unsigned Opc = MI->getOpcode(); 2271 int NewOpc = -1; 2272 if (AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr)) { 2273 NewOpc = AMDGPU::getFlatScratchInstSVfromSVS(Opc); 2274 } else if (ST.hasFlatScratchSTMode()) { 2275 // On GFX10 we have ST mode to use no registers for an address. 2276 // Otherwise we need to materialize 0 into an SGPR. 2277 NewOpc = AMDGPU::getFlatScratchInstSTfromSS(Opc); 2278 } 2279 2280 if (NewOpc != -1) { 2281 // removeOperand doesn't fixup tied operand indexes as it goes, so 2282 // it asserts. Untie vdst_in for now and retie them afterwards. 2283 int VDstIn = AMDGPU::getNamedOperandIdx(Opc, 2284 AMDGPU::OpName::vdst_in); 2285 bool TiedVDst = VDstIn != -1 && 2286 MI->getOperand(VDstIn).isReg() && 2287 MI->getOperand(VDstIn).isTied(); 2288 if (TiedVDst) 2289 MI->untieRegOperand(VDstIn); 2290 2291 MI->removeOperand( 2292 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr)); 2293 2294 if (TiedVDst) { 2295 int NewVDst = 2296 AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst); 2297 int NewVDstIn = 2298 AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst_in); 2299 assert (NewVDst != -1 && NewVDstIn != -1 && "Must be tied!"); 2300 MI->tieOperands(NewVDst, NewVDstIn); 2301 } 2302 MI->setDesc(TII->get(NewOpc)); 2303 return false; 2304 } 2305 } 2306 } 2307 2308 if (!FrameReg) { 2309 FIOp.ChangeToImmediate(Offset); 2310 if (TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) 2311 return false; 2312 } 2313 2314 // We need to use register here. Check if we can use an SGPR or need 2315 // a VGPR. 2316 FIOp.ChangeToRegister(AMDGPU::M0, false); 2317 bool UseSGPR = TII->isOperandLegal(*MI, FIOperandNum, &FIOp); 2318 2319 if (!Offset && FrameReg && UseSGPR) { 2320 FIOp.setReg(FrameReg); 2321 return false; 2322 } 2323 2324 const TargetRegisterClass *RC = UseSGPR ? &AMDGPU::SReg_32_XM0RegClass 2325 : &AMDGPU::VGPR_32RegClass; 2326 2327 Register TmpReg = 2328 RS->scavengeRegisterBackwards(*RC, MI, false, 0, !UseSGPR); 2329 FIOp.setReg(TmpReg); 2330 FIOp.setIsKill(); 2331 2332 if ((!FrameReg || !Offset) && TmpReg) { 2333 unsigned Opc = UseSGPR ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 2334 auto MIB = BuildMI(*MBB, MI, DL, TII->get(Opc), TmpReg); 2335 if (FrameReg) 2336 MIB.addReg(FrameReg); 2337 else 2338 MIB.addImm(Offset); 2339 2340 return false; 2341 } 2342 2343 bool NeedSaveSCC = 2344 RS->isRegUsed(AMDGPU::SCC) && !MI->definesRegister(AMDGPU::SCC); 2345 2346 Register TmpSReg = 2347 UseSGPR ? TmpReg 2348 : RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass, 2349 MI, false, 0, !UseSGPR); 2350 2351 // TODO: for flat scratch another attempt can be made with a VGPR index 2352 // if no SGPRs can be scavenged. 2353 if ((!TmpSReg && !FrameReg) || (!TmpReg && !UseSGPR)) 2354 report_fatal_error("Cannot scavenge register in FI elimination!"); 2355 2356 if (!TmpSReg) { 2357 // Use frame register and restore it after. 2358 TmpSReg = FrameReg; 2359 FIOp.setReg(FrameReg); 2360 FIOp.setIsKill(false); 2361 } 2362 2363 if (NeedSaveSCC) { 2364 assert(!(Offset & 0x1) && "Flat scratch offset must be aligned!"); 2365 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADDC_U32), TmpSReg) 2366 .addReg(FrameReg) 2367 .addImm(Offset); 2368 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_BITCMP1_B32)) 2369 .addReg(TmpSReg) 2370 .addImm(0); 2371 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_BITSET0_B32), TmpSReg) 2372 .addImm(0) 2373 .addReg(TmpSReg); 2374 } else { 2375 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), TmpSReg) 2376 .addReg(FrameReg) 2377 .addImm(Offset); 2378 } 2379 2380 if (!UseSGPR) 2381 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 2382 .addReg(TmpSReg, RegState::Kill); 2383 2384 if (TmpSReg == FrameReg) { 2385 // Undo frame register modification. 2386 if (NeedSaveSCC && !MI->registerDefIsDead(AMDGPU::SCC)) { 2387 MachineBasicBlock::iterator I = 2388 BuildMI(*MBB, std::next(MI), DL, TII->get(AMDGPU::S_ADDC_U32), 2389 TmpSReg) 2390 .addReg(FrameReg) 2391 .addImm(-Offset); 2392 I = BuildMI(*MBB, std::next(I), DL, TII->get(AMDGPU::S_BITCMP1_B32)) 2393 .addReg(TmpSReg) 2394 .addImm(0); 2395 BuildMI(*MBB, std::next(I), DL, TII->get(AMDGPU::S_BITSET0_B32), 2396 TmpSReg) 2397 .addImm(0) 2398 .addReg(TmpSReg); 2399 } else { 2400 BuildMI(*MBB, std::next(MI), DL, TII->get(AMDGPU::S_ADD_I32), 2401 FrameReg) 2402 .addReg(FrameReg) 2403 .addImm(-Offset); 2404 } 2405 } 2406 2407 return false; 2408 } 2409 2410 bool IsMUBUF = TII->isMUBUF(*MI); 2411 2412 if (!IsMUBUF && !MFI->isEntryFunction()) { 2413 // Convert to a swizzled stack address by scaling by the wave size. 2414 // In an entry function/kernel the offset is already swizzled. 2415 bool IsSALU = isSGPRClass(TII->getOpRegClass(*MI, FIOperandNum)); 2416 bool LiveSCC = 2417 RS->isRegUsed(AMDGPU::SCC) && !MI->definesRegister(AMDGPU::SCC); 2418 const TargetRegisterClass *RC = IsSALU && !LiveSCC 2419 ? &AMDGPU::SReg_32RegClass 2420 : &AMDGPU::VGPR_32RegClass; 2421 bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32 || 2422 MI->getOpcode() == AMDGPU::V_MOV_B32_e64; 2423 Register ResultReg = 2424 IsCopy ? MI->getOperand(0).getReg() 2425 : RS->scavengeRegisterBackwards(*RC, MI, false, 0); 2426 2427 int64_t Offset = FrameInfo.getObjectOffset(Index); 2428 if (Offset == 0) { 2429 unsigned OpCode = IsSALU && !LiveSCC ? AMDGPU::S_LSHR_B32 2430 : AMDGPU::V_LSHRREV_B32_e64; 2431 // XXX - This never happens because of emergency scavenging slot at 0? 2432 auto Shift = BuildMI(*MBB, MI, DL, TII->get(OpCode), ResultReg) 2433 .addImm(ST.getWavefrontSizeLog2()) 2434 .addReg(FrameReg); 2435 if (IsSALU && !LiveSCC) 2436 Shift.getInstr()->getOperand(3).setIsDead(); // Mark SCC as dead. 2437 if (IsSALU && LiveSCC) { 2438 Register NewDest = RS->scavengeRegisterBackwards( 2439 AMDGPU::SReg_32RegClass, Shift, false, 0); 2440 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), 2441 NewDest) 2442 .addReg(ResultReg); 2443 ResultReg = NewDest; 2444 } 2445 } else { 2446 MachineInstrBuilder MIB; 2447 if (!IsSALU) { 2448 if ((MIB = TII->getAddNoCarry(*MBB, MI, DL, ResultReg, *RS)) != 2449 nullptr) { 2450 // Reuse ResultReg in intermediate step. 2451 Register ScaledReg = ResultReg; 2452 2453 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), 2454 ScaledReg) 2455 .addImm(ST.getWavefrontSizeLog2()) 2456 .addReg(FrameReg); 2457 2458 const bool IsVOP2 = MIB->getOpcode() == AMDGPU::V_ADD_U32_e32; 2459 2460 // TODO: Fold if use instruction is another add of a constant. 2461 if (IsVOP2 || AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) { 2462 // FIXME: This can fail 2463 MIB.addImm(Offset); 2464 MIB.addReg(ScaledReg, RegState::Kill); 2465 if (!IsVOP2) 2466 MIB.addImm(0); // clamp bit 2467 } else { 2468 assert(MIB->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 && 2469 "Need to reuse carry out register"); 2470 2471 // Use scavenged unused carry out as offset register. 2472 Register ConstOffsetReg; 2473 if (!isWave32) 2474 ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0); 2475 else 2476 ConstOffsetReg = MIB.getReg(1); 2477 2478 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg) 2479 .addImm(Offset); 2480 MIB.addReg(ConstOffsetReg, RegState::Kill); 2481 MIB.addReg(ScaledReg, RegState::Kill); 2482 MIB.addImm(0); // clamp bit 2483 } 2484 } 2485 } 2486 if (!MIB || IsSALU) { 2487 // We have to produce a carry out, and there isn't a free SGPR pair 2488 // for it. We can keep the whole computation on the SALU to avoid 2489 // clobbering an additional register at the cost of an extra mov. 2490 2491 // We may have 1 free scratch SGPR even though a carry out is 2492 // unavailable. Only one additional mov is needed. 2493 Register TmpScaledReg = RS->scavengeRegisterBackwards( 2494 AMDGPU::SReg_32_XM0RegClass, MI, false, 0, false); 2495 Register ScaledReg = TmpScaledReg.isValid() ? TmpScaledReg : FrameReg; 2496 2497 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHR_B32), ScaledReg) 2498 .addReg(FrameReg) 2499 .addImm(ST.getWavefrontSizeLog2()); 2500 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), ScaledReg) 2501 .addReg(ScaledReg, RegState::Kill) 2502 .addImm(Offset); 2503 if (!IsSALU) 2504 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), ResultReg) 2505 .addReg(ScaledReg, RegState::Kill); 2506 else 2507 ResultReg = ScaledReg; 2508 2509 // If there were truly no free SGPRs, we need to undo everything. 2510 if (!TmpScaledReg.isValid()) { 2511 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), ScaledReg) 2512 .addReg(ScaledReg, RegState::Kill) 2513 .addImm(-Offset); 2514 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg) 2515 .addReg(FrameReg) 2516 .addImm(ST.getWavefrontSizeLog2()); 2517 } 2518 } 2519 } 2520 2521 // Don't introduce an extra copy if we're just materializing in a mov. 2522 if (IsCopy) { 2523 MI->eraseFromParent(); 2524 return true; 2525 } 2526 FIOp.ChangeToRegister(ResultReg, false, false, true); 2527 return false; 2528 } 2529 2530 if (IsMUBUF) { 2531 // Disable offen so we don't need a 0 vgpr base. 2532 assert(static_cast<int>(FIOperandNum) == 2533 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 2534 AMDGPU::OpName::vaddr)); 2535 2536 auto &SOffset = *TII->getNamedOperand(*MI, AMDGPU::OpName::soffset); 2537 assert((SOffset.isImm() && SOffset.getImm() == 0)); 2538 2539 if (FrameReg != AMDGPU::NoRegister) 2540 SOffset.ChangeToRegister(FrameReg, false); 2541 2542 int64_t Offset = FrameInfo.getObjectOffset(Index); 2543 int64_t OldImm 2544 = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(); 2545 int64_t NewOffset = OldImm + Offset; 2546 2547 if (SIInstrInfo::isLegalMUBUFImmOffset(NewOffset) && 2548 buildMUBUFOffsetLoadStore(ST, FrameInfo, MI, Index, NewOffset)) { 2549 MI->eraseFromParent(); 2550 return true; 2551 } 2552 } 2553 2554 // If the offset is simply too big, don't convert to a scratch wave offset 2555 // relative index. 2556 2557 FIOp.ChangeToImmediate(Offset); 2558 if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) { 2559 Register TmpReg = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, 2560 MI, false, 0); 2561 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 2562 .addImm(Offset); 2563 FIOp.ChangeToRegister(TmpReg, false, false, true); 2564 } 2565 } 2566 } 2567 return false; 2568 } 2569 2570 StringRef SIRegisterInfo::getRegAsmName(MCRegister Reg) const { 2571 return AMDGPUInstPrinter::getRegisterName(Reg); 2572 } 2573 2574 static const TargetRegisterClass * 2575 getAnyVGPRClassForBitWidth(unsigned BitWidth) { 2576 if (BitWidth == 64) 2577 return &AMDGPU::VReg_64RegClass; 2578 if (BitWidth == 96) 2579 return &AMDGPU::VReg_96RegClass; 2580 if (BitWidth == 128) 2581 return &AMDGPU::VReg_128RegClass; 2582 if (BitWidth == 160) 2583 return &AMDGPU::VReg_160RegClass; 2584 if (BitWidth == 192) 2585 return &AMDGPU::VReg_192RegClass; 2586 if (BitWidth == 224) 2587 return &AMDGPU::VReg_224RegClass; 2588 if (BitWidth == 256) 2589 return &AMDGPU::VReg_256RegClass; 2590 if (BitWidth == 288) 2591 return &AMDGPU::VReg_288RegClass; 2592 if (BitWidth == 320) 2593 return &AMDGPU::VReg_320RegClass; 2594 if (BitWidth == 352) 2595 return &AMDGPU::VReg_352RegClass; 2596 if (BitWidth == 384) 2597 return &AMDGPU::VReg_384RegClass; 2598 if (BitWidth == 512) 2599 return &AMDGPU::VReg_512RegClass; 2600 if (BitWidth == 1024) 2601 return &AMDGPU::VReg_1024RegClass; 2602 2603 return nullptr; 2604 } 2605 2606 static const TargetRegisterClass * 2607 getAlignedVGPRClassForBitWidth(unsigned BitWidth) { 2608 if (BitWidth == 64) 2609 return &AMDGPU::VReg_64_Align2RegClass; 2610 if (BitWidth == 96) 2611 return &AMDGPU::VReg_96_Align2RegClass; 2612 if (BitWidth == 128) 2613 return &AMDGPU::VReg_128_Align2RegClass; 2614 if (BitWidth == 160) 2615 return &AMDGPU::VReg_160_Align2RegClass; 2616 if (BitWidth == 192) 2617 return &AMDGPU::VReg_192_Align2RegClass; 2618 if (BitWidth == 224) 2619 return &AMDGPU::VReg_224_Align2RegClass; 2620 if (BitWidth == 256) 2621 return &AMDGPU::VReg_256_Align2RegClass; 2622 if (BitWidth == 288) 2623 return &AMDGPU::VReg_288_Align2RegClass; 2624 if (BitWidth == 320) 2625 return &AMDGPU::VReg_320_Align2RegClass; 2626 if (BitWidth == 352) 2627 return &AMDGPU::VReg_352_Align2RegClass; 2628 if (BitWidth == 384) 2629 return &AMDGPU::VReg_384_Align2RegClass; 2630 if (BitWidth == 512) 2631 return &AMDGPU::VReg_512_Align2RegClass; 2632 if (BitWidth == 1024) 2633 return &AMDGPU::VReg_1024_Align2RegClass; 2634 2635 return nullptr; 2636 } 2637 2638 const TargetRegisterClass * 2639 SIRegisterInfo::getVGPRClassForBitWidth(unsigned BitWidth) const { 2640 if (BitWidth == 1) 2641 return &AMDGPU::VReg_1RegClass; 2642 if (BitWidth == 16) 2643 return &AMDGPU::VGPR_LO16RegClass; 2644 if (BitWidth == 32) 2645 return &AMDGPU::VGPR_32RegClass; 2646 return ST.needsAlignedVGPRs() ? getAlignedVGPRClassForBitWidth(BitWidth) 2647 : getAnyVGPRClassForBitWidth(BitWidth); 2648 } 2649 2650 static const TargetRegisterClass * 2651 getAnyAGPRClassForBitWidth(unsigned BitWidth) { 2652 if (BitWidth == 64) 2653 return &AMDGPU::AReg_64RegClass; 2654 if (BitWidth == 96) 2655 return &AMDGPU::AReg_96RegClass; 2656 if (BitWidth == 128) 2657 return &AMDGPU::AReg_128RegClass; 2658 if (BitWidth == 160) 2659 return &AMDGPU::AReg_160RegClass; 2660 if (BitWidth == 192) 2661 return &AMDGPU::AReg_192RegClass; 2662 if (BitWidth == 224) 2663 return &AMDGPU::AReg_224RegClass; 2664 if (BitWidth == 256) 2665 return &AMDGPU::AReg_256RegClass; 2666 if (BitWidth == 288) 2667 return &AMDGPU::AReg_288RegClass; 2668 if (BitWidth == 320) 2669 return &AMDGPU::AReg_320RegClass; 2670 if (BitWidth == 352) 2671 return &AMDGPU::AReg_352RegClass; 2672 if (BitWidth == 384) 2673 return &AMDGPU::AReg_384RegClass; 2674 if (BitWidth == 512) 2675 return &AMDGPU::AReg_512RegClass; 2676 if (BitWidth == 1024) 2677 return &AMDGPU::AReg_1024RegClass; 2678 2679 return nullptr; 2680 } 2681 2682 static const TargetRegisterClass * 2683 getAlignedAGPRClassForBitWidth(unsigned BitWidth) { 2684 if (BitWidth == 64) 2685 return &AMDGPU::AReg_64_Align2RegClass; 2686 if (BitWidth == 96) 2687 return &AMDGPU::AReg_96_Align2RegClass; 2688 if (BitWidth == 128) 2689 return &AMDGPU::AReg_128_Align2RegClass; 2690 if (BitWidth == 160) 2691 return &AMDGPU::AReg_160_Align2RegClass; 2692 if (BitWidth == 192) 2693 return &AMDGPU::AReg_192_Align2RegClass; 2694 if (BitWidth == 224) 2695 return &AMDGPU::AReg_224_Align2RegClass; 2696 if (BitWidth == 256) 2697 return &AMDGPU::AReg_256_Align2RegClass; 2698 if (BitWidth == 288) 2699 return &AMDGPU::AReg_288_Align2RegClass; 2700 if (BitWidth == 320) 2701 return &AMDGPU::AReg_320_Align2RegClass; 2702 if (BitWidth == 352) 2703 return &AMDGPU::AReg_352_Align2RegClass; 2704 if (BitWidth == 384) 2705 return &AMDGPU::AReg_384_Align2RegClass; 2706 if (BitWidth == 512) 2707 return &AMDGPU::AReg_512_Align2RegClass; 2708 if (BitWidth == 1024) 2709 return &AMDGPU::AReg_1024_Align2RegClass; 2710 2711 return nullptr; 2712 } 2713 2714 const TargetRegisterClass * 2715 SIRegisterInfo::getAGPRClassForBitWidth(unsigned BitWidth) const { 2716 if (BitWidth == 16) 2717 return &AMDGPU::AGPR_LO16RegClass; 2718 if (BitWidth == 32) 2719 return &AMDGPU::AGPR_32RegClass; 2720 return ST.needsAlignedVGPRs() ? getAlignedAGPRClassForBitWidth(BitWidth) 2721 : getAnyAGPRClassForBitWidth(BitWidth); 2722 } 2723 2724 static const TargetRegisterClass * 2725 getAnyVectorSuperClassForBitWidth(unsigned BitWidth) { 2726 if (BitWidth == 64) 2727 return &AMDGPU::AV_64RegClass; 2728 if (BitWidth == 96) 2729 return &AMDGPU::AV_96RegClass; 2730 if (BitWidth == 128) 2731 return &AMDGPU::AV_128RegClass; 2732 if (BitWidth == 160) 2733 return &AMDGPU::AV_160RegClass; 2734 if (BitWidth == 192) 2735 return &AMDGPU::AV_192RegClass; 2736 if (BitWidth == 224) 2737 return &AMDGPU::AV_224RegClass; 2738 if (BitWidth == 256) 2739 return &AMDGPU::AV_256RegClass; 2740 if (BitWidth == 288) 2741 return &AMDGPU::AV_288RegClass; 2742 if (BitWidth == 320) 2743 return &AMDGPU::AV_320RegClass; 2744 if (BitWidth == 352) 2745 return &AMDGPU::AV_352RegClass; 2746 if (BitWidth == 384) 2747 return &AMDGPU::AV_384RegClass; 2748 if (BitWidth == 512) 2749 return &AMDGPU::AV_512RegClass; 2750 if (BitWidth == 1024) 2751 return &AMDGPU::AV_1024RegClass; 2752 2753 return nullptr; 2754 } 2755 2756 static const TargetRegisterClass * 2757 getAlignedVectorSuperClassForBitWidth(unsigned BitWidth) { 2758 if (BitWidth == 64) 2759 return &AMDGPU::AV_64_Align2RegClass; 2760 if (BitWidth == 96) 2761 return &AMDGPU::AV_96_Align2RegClass; 2762 if (BitWidth == 128) 2763 return &AMDGPU::AV_128_Align2RegClass; 2764 if (BitWidth == 160) 2765 return &AMDGPU::AV_160_Align2RegClass; 2766 if (BitWidth == 192) 2767 return &AMDGPU::AV_192_Align2RegClass; 2768 if (BitWidth == 224) 2769 return &AMDGPU::AV_224_Align2RegClass; 2770 if (BitWidth == 256) 2771 return &AMDGPU::AV_256_Align2RegClass; 2772 if (BitWidth == 288) 2773 return &AMDGPU::AV_288_Align2RegClass; 2774 if (BitWidth == 320) 2775 return &AMDGPU::AV_320_Align2RegClass; 2776 if (BitWidth == 352) 2777 return &AMDGPU::AV_352_Align2RegClass; 2778 if (BitWidth == 384) 2779 return &AMDGPU::AV_384_Align2RegClass; 2780 if (BitWidth == 512) 2781 return &AMDGPU::AV_512_Align2RegClass; 2782 if (BitWidth == 1024) 2783 return &AMDGPU::AV_1024_Align2RegClass; 2784 2785 return nullptr; 2786 } 2787 2788 const TargetRegisterClass * 2789 SIRegisterInfo::getVectorSuperClassForBitWidth(unsigned BitWidth) const { 2790 if (BitWidth == 16) 2791 return &AMDGPU::VGPR_LO16RegClass; 2792 if (BitWidth == 32) 2793 return &AMDGPU::AV_32RegClass; 2794 return ST.needsAlignedVGPRs() 2795 ? getAlignedVectorSuperClassForBitWidth(BitWidth) 2796 : getAnyVectorSuperClassForBitWidth(BitWidth); 2797 } 2798 2799 const TargetRegisterClass * 2800 SIRegisterInfo::getSGPRClassForBitWidth(unsigned BitWidth) { 2801 if (BitWidth == 16) 2802 return &AMDGPU::SGPR_LO16RegClass; 2803 if (BitWidth == 32) 2804 return &AMDGPU::SReg_32RegClass; 2805 if (BitWidth == 64) 2806 return &AMDGPU::SReg_64RegClass; 2807 if (BitWidth == 96) 2808 return &AMDGPU::SGPR_96RegClass; 2809 if (BitWidth == 128) 2810 return &AMDGPU::SGPR_128RegClass; 2811 if (BitWidth == 160) 2812 return &AMDGPU::SGPR_160RegClass; 2813 if (BitWidth == 192) 2814 return &AMDGPU::SGPR_192RegClass; 2815 if (BitWidth == 224) 2816 return &AMDGPU::SGPR_224RegClass; 2817 if (BitWidth == 256) 2818 return &AMDGPU::SGPR_256RegClass; 2819 if (BitWidth == 288) 2820 return &AMDGPU::SGPR_288RegClass; 2821 if (BitWidth == 320) 2822 return &AMDGPU::SGPR_320RegClass; 2823 if (BitWidth == 352) 2824 return &AMDGPU::SGPR_352RegClass; 2825 if (BitWidth == 384) 2826 return &AMDGPU::SGPR_384RegClass; 2827 if (BitWidth == 512) 2828 return &AMDGPU::SGPR_512RegClass; 2829 if (BitWidth == 1024) 2830 return &AMDGPU::SGPR_1024RegClass; 2831 2832 return nullptr; 2833 } 2834 2835 bool SIRegisterInfo::isSGPRReg(const MachineRegisterInfo &MRI, 2836 Register Reg) const { 2837 const TargetRegisterClass *RC; 2838 if (Reg.isVirtual()) 2839 RC = MRI.getRegClass(Reg); 2840 else 2841 RC = getPhysRegBaseClass(Reg); 2842 return RC ? isSGPRClass(RC) : false; 2843 } 2844 2845 const TargetRegisterClass * 2846 SIRegisterInfo::getEquivalentVGPRClass(const TargetRegisterClass *SRC) const { 2847 unsigned Size = getRegSizeInBits(*SRC); 2848 const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size); 2849 assert(VRC && "Invalid register class size"); 2850 return VRC; 2851 } 2852 2853 const TargetRegisterClass * 2854 SIRegisterInfo::getEquivalentAGPRClass(const TargetRegisterClass *SRC) const { 2855 unsigned Size = getRegSizeInBits(*SRC); 2856 const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size); 2857 assert(ARC && "Invalid register class size"); 2858 return ARC; 2859 } 2860 2861 const TargetRegisterClass * 2862 SIRegisterInfo::getEquivalentSGPRClass(const TargetRegisterClass *VRC) const { 2863 unsigned Size = getRegSizeInBits(*VRC); 2864 if (Size == 32) 2865 return &AMDGPU::SGPR_32RegClass; 2866 const TargetRegisterClass *SRC = getSGPRClassForBitWidth(Size); 2867 assert(SRC && "Invalid register class size"); 2868 return SRC; 2869 } 2870 2871 const TargetRegisterClass * 2872 SIRegisterInfo::getCompatibleSubRegClass(const TargetRegisterClass *SuperRC, 2873 const TargetRegisterClass *SubRC, 2874 unsigned SubIdx) const { 2875 // Ensure this subregister index is aligned in the super register. 2876 const TargetRegisterClass *MatchRC = 2877 getMatchingSuperRegClass(SuperRC, SubRC, SubIdx); 2878 return MatchRC && MatchRC->hasSubClassEq(SuperRC) ? MatchRC : nullptr; 2879 } 2880 2881 bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const { 2882 if (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST && 2883 OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST) 2884 return !ST.hasMFMAInlineLiteralBug(); 2885 2886 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 2887 OpType <= AMDGPU::OPERAND_SRC_LAST; 2888 } 2889 2890 bool SIRegisterInfo::shouldRewriteCopySrc( 2891 const TargetRegisterClass *DefRC, 2892 unsigned DefSubReg, 2893 const TargetRegisterClass *SrcRC, 2894 unsigned SrcSubReg) const { 2895 // We want to prefer the smallest register class possible, so we don't want to 2896 // stop and rewrite on anything that looks like a subregister 2897 // extract. Operations mostly don't care about the super register class, so we 2898 // only want to stop on the most basic of copies between the same register 2899 // class. 2900 // 2901 // e.g. if we have something like 2902 // %0 = ... 2903 // %1 = ... 2904 // %2 = REG_SEQUENCE %0, sub0, %1, sub1, %2, sub2 2905 // %3 = COPY %2, sub0 2906 // 2907 // We want to look through the COPY to find: 2908 // => %3 = COPY %0 2909 2910 // Plain copy. 2911 return getCommonSubClass(DefRC, SrcRC) != nullptr; 2912 } 2913 2914 bool SIRegisterInfo::opCanUseLiteralConstant(unsigned OpType) const { 2915 // TODO: 64-bit operands have extending behavior from 32-bit literal. 2916 return OpType >= AMDGPU::OPERAND_REG_IMM_FIRST && 2917 OpType <= AMDGPU::OPERAND_REG_IMM_LAST; 2918 } 2919 2920 /// Returns a lowest register that is not used at any point in the function. 2921 /// If all registers are used, then this function will return 2922 /// AMDGPU::NoRegister. If \p ReserveHighestRegister = true, then return 2923 /// highest unused register. 2924 MCRegister SIRegisterInfo::findUnusedRegister( 2925 const MachineRegisterInfo &MRI, const TargetRegisterClass *RC, 2926 const MachineFunction &MF, bool ReserveHighestRegister) const { 2927 if (ReserveHighestRegister) { 2928 for (MCRegister Reg : reverse(*RC)) 2929 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 2930 return Reg; 2931 } else { 2932 for (MCRegister Reg : *RC) 2933 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 2934 return Reg; 2935 } 2936 return MCRegister(); 2937 } 2938 2939 bool SIRegisterInfo::isUniformReg(const MachineRegisterInfo &MRI, 2940 const RegisterBankInfo &RBI, 2941 Register Reg) const { 2942 auto *RB = RBI.getRegBank(Reg, MRI, *MRI.getTargetRegisterInfo()); 2943 if (!RB) 2944 return false; 2945 2946 return !RBI.isDivergentRegBank(RB); 2947 } 2948 2949 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC, 2950 unsigned EltSize) const { 2951 const unsigned RegBitWidth = AMDGPU::getRegBitWidth(*RC); 2952 assert(RegBitWidth >= 32 && RegBitWidth <= 1024); 2953 2954 const unsigned RegDWORDs = RegBitWidth / 32; 2955 const unsigned EltDWORDs = EltSize / 4; 2956 assert(RegSplitParts.size() + 1 >= EltDWORDs); 2957 2958 const std::vector<int16_t> &Parts = RegSplitParts[EltDWORDs - 1]; 2959 const unsigned NumParts = RegDWORDs / EltDWORDs; 2960 2961 return ArrayRef(Parts.data(), NumParts); 2962 } 2963 2964 const TargetRegisterClass* 2965 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI, 2966 Register Reg) const { 2967 return Reg.isVirtual() ? MRI.getRegClass(Reg) : getPhysRegBaseClass(Reg); 2968 } 2969 2970 const TargetRegisterClass * 2971 SIRegisterInfo::getRegClassForOperandReg(const MachineRegisterInfo &MRI, 2972 const MachineOperand &MO) const { 2973 const TargetRegisterClass *SrcRC = getRegClassForReg(MRI, MO.getReg()); 2974 return getSubRegisterClass(SrcRC, MO.getSubReg()); 2975 } 2976 2977 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI, 2978 Register Reg) const { 2979 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 2980 // Registers without classes are unaddressable, SGPR-like registers. 2981 return RC && isVGPRClass(RC); 2982 } 2983 2984 bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI, 2985 Register Reg) const { 2986 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 2987 2988 // Registers without classes are unaddressable, SGPR-like registers. 2989 return RC && isAGPRClass(RC); 2990 } 2991 2992 bool SIRegisterInfo::shouldCoalesce(MachineInstr *MI, 2993 const TargetRegisterClass *SrcRC, 2994 unsigned SubReg, 2995 const TargetRegisterClass *DstRC, 2996 unsigned DstSubReg, 2997 const TargetRegisterClass *NewRC, 2998 LiveIntervals &LIS) const { 2999 unsigned SrcSize = getRegSizeInBits(*SrcRC); 3000 unsigned DstSize = getRegSizeInBits(*DstRC); 3001 unsigned NewSize = getRegSizeInBits(*NewRC); 3002 3003 // Do not increase size of registers beyond dword, we would need to allocate 3004 // adjacent registers and constraint regalloc more than needed. 3005 3006 // Always allow dword coalescing. 3007 if (SrcSize <= 32 || DstSize <= 32) 3008 return true; 3009 3010 return NewSize <= DstSize || NewSize <= SrcSize; 3011 } 3012 3013 unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, 3014 MachineFunction &MF) const { 3015 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 3016 3017 unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(), 3018 MF.getFunction()); 3019 switch (RC->getID()) { 3020 default: 3021 return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF); 3022 case AMDGPU::VGPR_32RegClassID: 3023 case AMDGPU::VGPR_LO16RegClassID: 3024 case AMDGPU::VGPR_HI16RegClassID: 3025 return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF)); 3026 case AMDGPU::SGPR_32RegClassID: 3027 case AMDGPU::SGPR_LO16RegClassID: 3028 return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF)); 3029 } 3030 } 3031 3032 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF, 3033 unsigned Idx) const { 3034 if (Idx == AMDGPU::RegisterPressureSets::VGPR_32 || 3035 Idx == AMDGPU::RegisterPressureSets::AGPR_32) 3036 return getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 3037 const_cast<MachineFunction &>(MF)); 3038 3039 if (Idx == AMDGPU::RegisterPressureSets::SReg_32) 3040 return getRegPressureLimit(&AMDGPU::SGPR_32RegClass, 3041 const_cast<MachineFunction &>(MF)); 3042 3043 llvm_unreachable("Unexpected register pressure set!"); 3044 } 3045 3046 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const { 3047 static const int Empty[] = { -1 }; 3048 3049 if (RegPressureIgnoredUnits[RegUnit]) 3050 return Empty; 3051 3052 return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit); 3053 } 3054 3055 MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const { 3056 // Not a callee saved register. 3057 return AMDGPU::SGPR30_SGPR31; 3058 } 3059 3060 const TargetRegisterClass * 3061 SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size, 3062 const RegisterBank &RB) const { 3063 switch (RB.getID()) { 3064 case AMDGPU::VGPRRegBankID: 3065 return getVGPRClassForBitWidth(std::max(32u, Size)); 3066 case AMDGPU::VCCRegBankID: 3067 assert(Size == 1); 3068 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 3069 : &AMDGPU::SReg_64_XEXECRegClass; 3070 case AMDGPU::SGPRRegBankID: 3071 return getSGPRClassForBitWidth(std::max(32u, Size)); 3072 case AMDGPU::AGPRRegBankID: 3073 return getAGPRClassForBitWidth(std::max(32u, Size)); 3074 default: 3075 llvm_unreachable("unknown register bank"); 3076 } 3077 } 3078 3079 const TargetRegisterClass * 3080 SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO, 3081 const MachineRegisterInfo &MRI) const { 3082 const RegClassOrRegBank &RCOrRB = MRI.getRegClassOrRegBank(MO.getReg()); 3083 if (const RegisterBank *RB = RCOrRB.dyn_cast<const RegisterBank*>()) 3084 return getRegClassForTypeOnBank(MRI.getType(MO.getReg()), *RB); 3085 3086 if (const auto *RC = RCOrRB.dyn_cast<const TargetRegisterClass *>()) 3087 return getAllocatableClass(RC); 3088 3089 return nullptr; 3090 } 3091 3092 MCRegister SIRegisterInfo::getVCC() const { 3093 return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; 3094 } 3095 3096 MCRegister SIRegisterInfo::getExec() const { 3097 return isWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 3098 } 3099 3100 const TargetRegisterClass *SIRegisterInfo::getVGPR64Class() const { 3101 // VGPR tuples have an alignment requirement on gfx90a variants. 3102 return ST.needsAlignedVGPRs() ? &AMDGPU::VReg_64_Align2RegClass 3103 : &AMDGPU::VReg_64RegClass; 3104 } 3105 3106 const TargetRegisterClass * 3107 SIRegisterInfo::getRegClass(unsigned RCID) const { 3108 switch ((int)RCID) { 3109 case AMDGPU::SReg_1RegClassID: 3110 return getBoolRC(); 3111 case AMDGPU::SReg_1_XEXECRegClassID: 3112 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 3113 : &AMDGPU::SReg_64_XEXECRegClass; 3114 case -1: 3115 return nullptr; 3116 default: 3117 return AMDGPUGenRegisterInfo::getRegClass(RCID); 3118 } 3119 } 3120 3121 // Find reaching register definition 3122 MachineInstr *SIRegisterInfo::findReachingDef(Register Reg, unsigned SubReg, 3123 MachineInstr &Use, 3124 MachineRegisterInfo &MRI, 3125 LiveIntervals *LIS) const { 3126 auto &MDT = LIS->getAnalysis<MachineDominatorTree>(); 3127 SlotIndex UseIdx = LIS->getInstructionIndex(Use); 3128 SlotIndex DefIdx; 3129 3130 if (Reg.isVirtual()) { 3131 if (!LIS->hasInterval(Reg)) 3132 return nullptr; 3133 LiveInterval &LI = LIS->getInterval(Reg); 3134 LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask(SubReg) 3135 : MRI.getMaxLaneMaskForVReg(Reg); 3136 VNInfo *V = nullptr; 3137 if (LI.hasSubRanges()) { 3138 for (auto &S : LI.subranges()) { 3139 if ((S.LaneMask & SubLanes) == SubLanes) { 3140 V = S.getVNInfoAt(UseIdx); 3141 break; 3142 } 3143 } 3144 } else { 3145 V = LI.getVNInfoAt(UseIdx); 3146 } 3147 if (!V) 3148 return nullptr; 3149 DefIdx = V->def; 3150 } else { 3151 // Find last def. 3152 for (MCRegUnit Unit : regunits(Reg.asMCReg())) { 3153 LiveRange &LR = LIS->getRegUnit(Unit); 3154 if (VNInfo *V = LR.getVNInfoAt(UseIdx)) { 3155 if (!DefIdx.isValid() || 3156 MDT.dominates(LIS->getInstructionFromIndex(DefIdx), 3157 LIS->getInstructionFromIndex(V->def))) 3158 DefIdx = V->def; 3159 } else { 3160 return nullptr; 3161 } 3162 } 3163 } 3164 3165 MachineInstr *Def = LIS->getInstructionFromIndex(DefIdx); 3166 3167 if (!Def || !MDT.dominates(Def, &Use)) 3168 return nullptr; 3169 3170 assert(Def->modifiesRegister(Reg, this)); 3171 3172 return Def; 3173 } 3174 3175 MCPhysReg SIRegisterInfo::get32BitRegister(MCPhysReg Reg) const { 3176 assert(getRegSizeInBits(*getPhysRegBaseClass(Reg)) <= 32); 3177 3178 for (const TargetRegisterClass &RC : { AMDGPU::VGPR_32RegClass, 3179 AMDGPU::SReg_32RegClass, 3180 AMDGPU::AGPR_32RegClass } ) { 3181 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::lo16, &RC)) 3182 return Super; 3183 } 3184 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::hi16, 3185 &AMDGPU::VGPR_32RegClass)) { 3186 return Super; 3187 } 3188 3189 return AMDGPU::NoRegister; 3190 } 3191 3192 bool SIRegisterInfo::isProperlyAlignedRC(const TargetRegisterClass &RC) const { 3193 if (!ST.needsAlignedVGPRs()) 3194 return true; 3195 3196 if (isVGPRClass(&RC)) 3197 return RC.hasSuperClassEq(getVGPRClassForBitWidth(getRegSizeInBits(RC))); 3198 if (isAGPRClass(&RC)) 3199 return RC.hasSuperClassEq(getAGPRClassForBitWidth(getRegSizeInBits(RC))); 3200 if (isVectorSuperClass(&RC)) 3201 return RC.hasSuperClassEq( 3202 getVectorSuperClassForBitWidth(getRegSizeInBits(RC))); 3203 3204 return true; 3205 } 3206 3207 const TargetRegisterClass * 3208 SIRegisterInfo::getProperlyAlignedRC(const TargetRegisterClass *RC) const { 3209 if (!RC || !ST.needsAlignedVGPRs()) 3210 return RC; 3211 3212 unsigned Size = getRegSizeInBits(*RC); 3213 if (Size <= 32) 3214 return RC; 3215 3216 if (isVGPRClass(RC)) 3217 return getAlignedVGPRClassForBitWidth(Size); 3218 if (isAGPRClass(RC)) 3219 return getAlignedAGPRClassForBitWidth(Size); 3220 if (isVectorSuperClass(RC)) 3221 return getAlignedVectorSuperClassForBitWidth(Size); 3222 3223 return RC; 3224 } 3225 3226 ArrayRef<MCPhysReg> 3227 SIRegisterInfo::getAllSGPR128(const MachineFunction &MF) const { 3228 return ArrayRef(AMDGPU::SGPR_128RegClass.begin(), ST.getMaxNumSGPRs(MF) / 4); 3229 } 3230 3231 ArrayRef<MCPhysReg> 3232 SIRegisterInfo::getAllSGPR64(const MachineFunction &MF) const { 3233 return ArrayRef(AMDGPU::SGPR_64RegClass.begin(), ST.getMaxNumSGPRs(MF) / 2); 3234 } 3235 3236 ArrayRef<MCPhysReg> 3237 SIRegisterInfo::getAllSGPR32(const MachineFunction &MF) const { 3238 return ArrayRef(AMDGPU::SGPR_32RegClass.begin(), ST.getMaxNumSGPRs(MF)); 3239 } 3240 3241 unsigned 3242 SIRegisterInfo::getSubRegAlignmentNumBits(const TargetRegisterClass *RC, 3243 unsigned SubReg) const { 3244 switch (RC->TSFlags & SIRCFlags::RegKindMask) { 3245 case SIRCFlags::HasSGPR: 3246 return std::min(128u, getSubRegIdxSize(SubReg)); 3247 case SIRCFlags::HasAGPR: 3248 case SIRCFlags::HasVGPR: 3249 case SIRCFlags::HasVGPR | SIRCFlags::HasAGPR: 3250 return std::min(32u, getSubRegIdxSize(SubReg)); 3251 default: 3252 break; 3253 } 3254 return 0; 3255 } 3256