1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI implementation of the TargetRegisterInfo class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIRegisterInfo.h" 15 #include "AMDGPU.h" 16 #include "AMDGPURegisterBankInfo.h" 17 #include "GCNSubtarget.h" 18 #include "MCTargetDesc/AMDGPUInstPrinter.h" 19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 20 #include "SIMachineFunctionInfo.h" 21 #include "llvm/CodeGen/LiveIntervals.h" 22 #include "llvm/CodeGen/LivePhysRegs.h" 23 #include "llvm/CodeGen/MachineDominators.h" 24 #include "llvm/CodeGen/MachineFrameInfo.h" 25 #include "llvm/CodeGen/RegisterScavenging.h" 26 27 using namespace llvm; 28 29 #define GET_REGINFO_TARGET_DESC 30 #include "AMDGPUGenRegisterInfo.inc" 31 32 static cl::opt<bool> EnableSpillSGPRToVGPR( 33 "amdgpu-spill-sgpr-to-vgpr", 34 cl::desc("Enable spilling VGPRs to SGPRs"), 35 cl::ReallyHidden, 36 cl::init(true)); 37 38 std::array<std::vector<int16_t>, 16> SIRegisterInfo::RegSplitParts; 39 std::array<std::array<uint16_t, 32>, 9> SIRegisterInfo::SubRegFromChannelTable; 40 41 // Map numbers of DWORDs to indexes in SubRegFromChannelTable. 42 // Valid indexes are shifted 1, such that a 0 mapping means unsupported. 43 // e.g. for 8 DWORDs (256-bit), SubRegFromChannelTableWidthMap[8] = 8, 44 // meaning index 7 in SubRegFromChannelTable. 45 static const std::array<unsigned, 17> SubRegFromChannelTableWidthMap = { 46 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 9}; 47 48 namespace llvm { 49 50 // A temporary struct to spill SGPRs. 51 // This is mostly to spill SGPRs to memory. Spilling SGPRs into VGPR lanes emits 52 // just v_writelane and v_readlane. 53 // 54 // When spilling to memory, the SGPRs are written into VGPR lanes and the VGPR 55 // is saved to scratch (or the other way around for loads). 56 // For this, a VGPR is required where the needed lanes can be clobbered. The 57 // RegScavenger can provide a VGPR where currently active lanes can be 58 // clobbered, but we still need to save inactive lanes. 59 // The high-level steps are: 60 // - Try to scavenge SGPR(s) to save exec 61 // - Try to scavenge VGPR 62 // - Save needed, all or inactive lanes of a TmpVGPR 63 // - Spill/Restore SGPRs using TmpVGPR 64 // - Restore TmpVGPR 65 // 66 // To save all lanes of TmpVGPR, exec needs to be saved and modified. If we 67 // cannot scavenge temporary SGPRs to save exec, we use the following code: 68 // buffer_store_dword TmpVGPR ; only if active lanes need to be saved 69 // s_not exec, exec 70 // buffer_store_dword TmpVGPR ; save inactive lanes 71 // s_not exec, exec 72 struct SGPRSpillBuilder { 73 struct PerVGPRData { 74 unsigned PerVGPR; 75 unsigned NumVGPRs; 76 int64_t VGPRLanes; 77 }; 78 79 // The SGPR to save 80 Register SuperReg; 81 MachineBasicBlock::iterator MI; 82 ArrayRef<int16_t> SplitParts; 83 unsigned NumSubRegs; 84 bool IsKill; 85 const DebugLoc &DL; 86 87 /* When spilling to stack */ 88 // The SGPRs are written into this VGPR, which is then written to scratch 89 // (or vice versa for loads). 90 Register TmpVGPR = AMDGPU::NoRegister; 91 // Temporary spill slot to save TmpVGPR to. 92 int TmpVGPRIndex = 0; 93 // If TmpVGPR is live before the spill or if it is scavenged. 94 bool TmpVGPRLive = false; 95 // Scavenged SGPR to save EXEC. 96 Register SavedExecReg = AMDGPU::NoRegister; 97 // Stack index to write the SGPRs to. 98 int Index; 99 unsigned EltSize = 4; 100 101 RegScavenger *RS; 102 MachineBasicBlock *MBB; 103 MachineFunction &MF; 104 SIMachineFunctionInfo &MFI; 105 const SIInstrInfo &TII; 106 const SIRegisterInfo &TRI; 107 bool IsWave32; 108 Register ExecReg; 109 unsigned MovOpc; 110 unsigned NotOpc; 111 112 SGPRSpillBuilder(const SIRegisterInfo &TRI, const SIInstrInfo &TII, 113 bool IsWave32, MachineBasicBlock::iterator MI, int Index, 114 RegScavenger *RS) 115 : SGPRSpillBuilder(TRI, TII, IsWave32, MI, MI->getOperand(0).getReg(), 116 MI->getOperand(0).isKill(), Index, RS) {} 117 118 SGPRSpillBuilder(const SIRegisterInfo &TRI, const SIInstrInfo &TII, 119 bool IsWave32, MachineBasicBlock::iterator MI, Register Reg, 120 bool IsKill, int Index, RegScavenger *RS) 121 : SuperReg(Reg), MI(MI), IsKill(IsKill), DL(MI->getDebugLoc()), 122 Index(Index), RS(RS), MBB(MI->getParent()), MF(*MBB->getParent()), 123 MFI(*MF.getInfo<SIMachineFunctionInfo>()), TII(TII), TRI(TRI), 124 IsWave32(IsWave32) { 125 const TargetRegisterClass *RC = TRI.getPhysRegClass(SuperReg); 126 SplitParts = TRI.getRegSplitParts(RC, EltSize); 127 NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 128 129 if (IsWave32) { 130 ExecReg = AMDGPU::EXEC_LO; 131 MovOpc = AMDGPU::S_MOV_B32; 132 NotOpc = AMDGPU::S_NOT_B32; 133 } else { 134 ExecReg = AMDGPU::EXEC; 135 MovOpc = AMDGPU::S_MOV_B64; 136 NotOpc = AMDGPU::S_NOT_B64; 137 } 138 139 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 140 assert(SuperReg != AMDGPU::EXEC_LO && SuperReg != AMDGPU::EXEC_HI && 141 SuperReg != AMDGPU::EXEC && "exec should never spill"); 142 } 143 144 PerVGPRData getPerVGPRData() { 145 PerVGPRData Data; 146 Data.PerVGPR = IsWave32 ? 32 : 64; 147 Data.NumVGPRs = (NumSubRegs + (Data.PerVGPR - 1)) / Data.PerVGPR; 148 Data.VGPRLanes = (1LL << std::min(Data.PerVGPR, NumSubRegs)) - 1LL; 149 return Data; 150 } 151 152 // Tries to scavenge SGPRs to save EXEC and a VGPR. Uses v0 if no VGPR is 153 // free. 154 // Writes these instructions if an SGPR can be scavenged: 155 // s_mov_b64 s[6:7], exec ; Save exec 156 // s_mov_b64 exec, 3 ; Wanted lanemask 157 // buffer_store_dword v1 ; Write scavenged VGPR to emergency slot 158 // 159 // Writes these instructions if no SGPR can be scavenged: 160 // buffer_store_dword v0 ; Only if no free VGPR was found 161 // s_not_b64 exec, exec 162 // buffer_store_dword v0 ; Save inactive lanes 163 // ; exec stays inverted, it is flipped back in 164 // ; restore. 165 void prepare() { 166 // Scavenged temporary VGPR to use. It must be scavenged once for any number 167 // of spilled subregs. 168 // FIXME: The liveness analysis is limited and does not tell if a register 169 // is in use in lanes that are currently inactive. We can never be sure if 170 // a register as actually in use in another lane, so we need to save all 171 // used lanes of the chosen VGPR. 172 assert(RS && "Cannot spill SGPR to memory without RegScavenger"); 173 TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0, false); 174 175 // Reserve temporary stack slot 176 TmpVGPRIndex = MFI.getScavengeFI(MF.getFrameInfo(), TRI); 177 if (TmpVGPR) { 178 // Found a register that is dead in the currently active lanes, we only 179 // need to spill inactive lanes. 180 TmpVGPRLive = false; 181 } else { 182 // Pick v0 because it doesn't make a difference. 183 TmpVGPR = AMDGPU::VGPR0; 184 TmpVGPRLive = true; 185 } 186 187 if (TmpVGPRLive) { 188 // We need to inform the scavenger that this index is already in use until 189 // we're done with the custom emergency spill. 190 RS->assignRegToScavengingIndex(TmpVGPRIndex, TmpVGPR); 191 } 192 193 // We may end up recursively calling the scavenger, and don't want to re-use 194 // the same register. 195 RS->setRegUsed(TmpVGPR); 196 197 // Try to scavenge SGPRs to save exec 198 assert(!SavedExecReg && "Exec is already saved, refuse to save again"); 199 const TargetRegisterClass &RC = 200 IsWave32 ? AMDGPU::SGPR_32RegClass : AMDGPU::SGPR_64RegClass; 201 RS->setRegUsed(SuperReg); 202 SavedExecReg = RS->scavengeRegister(&RC, MI, 0, false); 203 204 int64_t VGPRLanes = getPerVGPRData().VGPRLanes; 205 206 if (SavedExecReg) { 207 RS->setRegUsed(SavedExecReg); 208 // Set exec to needed lanes 209 BuildMI(*MBB, MI, DL, TII.get(MovOpc), SavedExecReg).addReg(ExecReg); 210 auto I = 211 BuildMI(*MBB, MI, DL, TII.get(MovOpc), ExecReg).addImm(VGPRLanes); 212 if (!TmpVGPRLive) 213 I.addReg(TmpVGPR, RegState::ImplicitDefine); 214 // Spill needed lanes 215 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false); 216 } else { 217 // The modify and restore of exec clobber SCC, which we would have to save 218 // and restore. FIXME: We probably would need to reserve a register for 219 // this. 220 if (RS->isRegUsed(AMDGPU::SCC)) 221 MI->emitError("unhandled SGPR spill to memory"); 222 223 // Spill active lanes 224 if (TmpVGPRLive) 225 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false, 226 /*IsKill*/ false); 227 // Spill inactive lanes 228 auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 229 if (!TmpVGPRLive) 230 I.addReg(TmpVGPR, RegState::ImplicitDefine); 231 I->getOperand(2).setIsDead(true); // Mark SCC as dead. 232 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false); 233 } 234 } 235 236 // Writes these instructions if an SGPR can be scavenged: 237 // buffer_load_dword v1 ; Write scavenged VGPR to emergency slot 238 // s_waitcnt vmcnt(0) ; If a free VGPR was found 239 // s_mov_b64 exec, s[6:7] ; Save exec 240 // 241 // Writes these instructions if no SGPR can be scavenged: 242 // buffer_load_dword v0 ; Restore inactive lanes 243 // s_waitcnt vmcnt(0) ; If a free VGPR was found 244 // s_not_b64 exec, exec 245 // buffer_load_dword v0 ; Only if no free VGPR was found 246 void restore() { 247 if (SavedExecReg) { 248 // Restore used lanes 249 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true, 250 /*IsKill*/ false); 251 // Restore exec 252 auto I = BuildMI(*MBB, MI, DL, TII.get(MovOpc), ExecReg) 253 .addReg(SavedExecReg, RegState::Kill); 254 // Add an implicit use of the load so it is not dead. 255 // FIXME This inserts an unnecessary waitcnt 256 if (!TmpVGPRLive) { 257 I.addReg(TmpVGPR, RegState::ImplicitKill); 258 } 259 } else { 260 // Restore inactive lanes 261 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true, 262 /*IsKill*/ false); 263 auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 264 if (!TmpVGPRLive) 265 I.addReg(TmpVGPR, RegState::ImplicitKill); 266 I->getOperand(2).setIsDead(true); // Mark SCC as dead. 267 268 // Restore active lanes 269 if (TmpVGPRLive) 270 TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true); 271 } 272 273 // Inform the scavenger where we're releasing our custom scavenged register. 274 if (TmpVGPRLive) { 275 MachineBasicBlock::iterator RestorePt = std::prev(MI); 276 RS->assignRegToScavengingIndex(TmpVGPRIndex, TmpVGPR, &*RestorePt); 277 } 278 } 279 280 // Write TmpVGPR to memory or read TmpVGPR from memory. 281 // Either using a single buffer_load/store if exec is set to the needed mask 282 // or using 283 // buffer_load 284 // s_not exec, exec 285 // buffer_load 286 // s_not exec, exec 287 void readWriteTmpVGPR(unsigned Offset, bool IsLoad) { 288 if (SavedExecReg) { 289 // Spill needed lanes 290 TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad); 291 } else { 292 // The modify and restore of exec clobber SCC, which we would have to save 293 // and restore. FIXME: We probably would need to reserve a register for 294 // this. 295 if (RS->isRegUsed(AMDGPU::SCC)) 296 MI->emitError("unhandled SGPR spill to memory"); 297 298 // Spill active lanes 299 TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad, 300 /*IsKill*/ false); 301 // Spill inactive lanes 302 auto Not0 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 303 Not0->getOperand(2).setIsDead(); // Mark SCC as dead. 304 TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad); 305 auto Not1 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); 306 Not1->getOperand(2).setIsDead(); // Mark SCC as dead. 307 } 308 } 309 310 void setMI(MachineBasicBlock *NewMBB, MachineBasicBlock::iterator NewMI) { 311 assert(MBB->getParent() == &MF); 312 MI = NewMI; 313 MBB = NewMBB; 314 } 315 }; 316 317 } // namespace llvm 318 319 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) 320 : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST), 321 SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) { 322 323 assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 && 324 getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) && 325 (getSubRegIndexLaneMask(AMDGPU::lo16) | 326 getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() == 327 getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() && 328 "getNumCoveredRegs() will not work with generated subreg masks!"); 329 330 RegPressureIgnoredUnits.resize(getNumRegUnits()); 331 RegPressureIgnoredUnits.set( 332 *MCRegUnitIterator(MCRegister::from(AMDGPU::M0), this)); 333 for (auto Reg : AMDGPU::VGPR_HI16RegClass) 334 RegPressureIgnoredUnits.set(*MCRegUnitIterator(Reg, this)); 335 336 // HACK: Until this is fully tablegen'd. 337 static llvm::once_flag InitializeRegSplitPartsFlag; 338 339 static auto InitializeRegSplitPartsOnce = [this]() { 340 for (unsigned Idx = 1, E = getNumSubRegIndices() - 1; Idx < E; ++Idx) { 341 unsigned Size = getSubRegIdxSize(Idx); 342 if (Size & 31) 343 continue; 344 std::vector<int16_t> &Vec = RegSplitParts[Size / 32 - 1]; 345 unsigned Pos = getSubRegIdxOffset(Idx); 346 if (Pos % Size) 347 continue; 348 Pos /= Size; 349 if (Vec.empty()) { 350 unsigned MaxNumParts = 1024 / Size; // Maximum register is 1024 bits. 351 Vec.resize(MaxNumParts); 352 } 353 Vec[Pos] = Idx; 354 } 355 }; 356 357 static llvm::once_flag InitializeSubRegFromChannelTableFlag; 358 359 static auto InitializeSubRegFromChannelTableOnce = [this]() { 360 for (auto &Row : SubRegFromChannelTable) 361 Row.fill(AMDGPU::NoSubRegister); 362 for (unsigned Idx = 1; Idx < getNumSubRegIndices(); ++Idx) { 363 unsigned Width = AMDGPUSubRegIdxRanges[Idx].Size / 32; 364 unsigned Offset = AMDGPUSubRegIdxRanges[Idx].Offset / 32; 365 assert(Width < SubRegFromChannelTableWidthMap.size()); 366 Width = SubRegFromChannelTableWidthMap[Width]; 367 if (Width == 0) 368 continue; 369 unsigned TableIdx = Width - 1; 370 assert(TableIdx < SubRegFromChannelTable.size()); 371 assert(Offset < SubRegFromChannelTable[TableIdx].size()); 372 SubRegFromChannelTable[TableIdx][Offset] = Idx; 373 } 374 }; 375 376 llvm::call_once(InitializeRegSplitPartsFlag, InitializeRegSplitPartsOnce); 377 llvm::call_once(InitializeSubRegFromChannelTableFlag, 378 InitializeSubRegFromChannelTableOnce); 379 } 380 381 void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved, 382 MCRegister Reg) const { 383 MCRegAliasIterator R(Reg, this, true); 384 385 for (; R.isValid(); ++R) 386 Reserved.set(*R); 387 } 388 389 // Forced to be here by one .inc 390 const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs( 391 const MachineFunction *MF) const { 392 CallingConv::ID CC = MF->getFunction().getCallingConv(); 393 switch (CC) { 394 case CallingConv::C: 395 case CallingConv::Fast: 396 case CallingConv::Cold: 397 return ST.hasGFX90AInsts() ? CSR_AMDGPU_GFX90AInsts_SaveList 398 : CSR_AMDGPU_SaveList; 399 case CallingConv::AMDGPU_Gfx: 400 return ST.hasGFX90AInsts() ? CSR_AMDGPU_SI_Gfx_GFX90AInsts_SaveList 401 : CSR_AMDGPU_SI_Gfx_SaveList; 402 default: { 403 // Dummy to not crash RegisterClassInfo. 404 static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister; 405 return &NoCalleeSavedReg; 406 } 407 } 408 } 409 410 const MCPhysReg * 411 SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const { 412 return nullptr; 413 } 414 415 const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF, 416 CallingConv::ID CC) const { 417 switch (CC) { 418 case CallingConv::C: 419 case CallingConv::Fast: 420 case CallingConv::Cold: 421 return ST.hasGFX90AInsts() ? CSR_AMDGPU_GFX90AInsts_RegMask 422 : CSR_AMDGPU_RegMask; 423 case CallingConv::AMDGPU_Gfx: 424 return ST.hasGFX90AInsts() ? CSR_AMDGPU_SI_Gfx_GFX90AInsts_RegMask 425 : CSR_AMDGPU_SI_Gfx_RegMask; 426 default: 427 return nullptr; 428 } 429 } 430 431 const uint32_t *SIRegisterInfo::getNoPreservedMask() const { 432 return CSR_AMDGPU_NoRegs_RegMask; 433 } 434 435 const TargetRegisterClass * 436 SIRegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC, 437 const MachineFunction &MF) const { 438 // FIXME: Should have a helper function like getEquivalentVGPRClass to get the 439 // equivalent AV class. If used one, the verifier will crash after 440 // RegBankSelect in the GISel flow. The aligned regclasses are not fully given 441 // until Instruction selection. 442 if (ST.hasMAIInsts() && (isVGPRClass(RC) || isAGPRClass(RC))) { 443 if (RC == &AMDGPU::VGPR_32RegClass || RC == &AMDGPU::AGPR_32RegClass) 444 return &AMDGPU::AV_32RegClass; 445 if (RC == &AMDGPU::VReg_64RegClass || RC == &AMDGPU::AReg_64RegClass) 446 return &AMDGPU::AV_64RegClass; 447 if (RC == &AMDGPU::VReg_64_Align2RegClass || 448 RC == &AMDGPU::AReg_64_Align2RegClass) 449 return &AMDGPU::AV_64_Align2RegClass; 450 if (RC == &AMDGPU::VReg_96RegClass || RC == &AMDGPU::AReg_96RegClass) 451 return &AMDGPU::AV_96RegClass; 452 if (RC == &AMDGPU::VReg_96_Align2RegClass || 453 RC == &AMDGPU::AReg_96_Align2RegClass) 454 return &AMDGPU::AV_96_Align2RegClass; 455 if (RC == &AMDGPU::VReg_128RegClass || RC == &AMDGPU::AReg_128RegClass) 456 return &AMDGPU::AV_128RegClass; 457 if (RC == &AMDGPU::VReg_128_Align2RegClass || 458 RC == &AMDGPU::AReg_128_Align2RegClass) 459 return &AMDGPU::AV_128_Align2RegClass; 460 if (RC == &AMDGPU::VReg_160RegClass || RC == &AMDGPU::AReg_160RegClass) 461 return &AMDGPU::AV_160RegClass; 462 if (RC == &AMDGPU::VReg_160_Align2RegClass || 463 RC == &AMDGPU::AReg_160_Align2RegClass) 464 return &AMDGPU::AV_160_Align2RegClass; 465 if (RC == &AMDGPU::VReg_192RegClass || RC == &AMDGPU::AReg_192RegClass) 466 return &AMDGPU::AV_192RegClass; 467 if (RC == &AMDGPU::VReg_192_Align2RegClass || 468 RC == &AMDGPU::AReg_192_Align2RegClass) 469 return &AMDGPU::AV_192_Align2RegClass; 470 if (RC == &AMDGPU::VReg_256RegClass || RC == &AMDGPU::AReg_256RegClass) 471 return &AMDGPU::AV_256RegClass; 472 if (RC == &AMDGPU::VReg_256_Align2RegClass || 473 RC == &AMDGPU::AReg_256_Align2RegClass) 474 return &AMDGPU::AV_256_Align2RegClass; 475 if (RC == &AMDGPU::VReg_512RegClass || RC == &AMDGPU::AReg_512RegClass) 476 return &AMDGPU::AV_512RegClass; 477 if (RC == &AMDGPU::VReg_512_Align2RegClass || 478 RC == &AMDGPU::AReg_512_Align2RegClass) 479 return &AMDGPU::AV_512_Align2RegClass; 480 if (RC == &AMDGPU::VReg_1024RegClass || RC == &AMDGPU::AReg_1024RegClass) 481 return &AMDGPU::AV_1024RegClass; 482 if (RC == &AMDGPU::VReg_1024_Align2RegClass || 483 RC == &AMDGPU::AReg_1024_Align2RegClass) 484 return &AMDGPU::AV_1024_Align2RegClass; 485 } 486 487 return TargetRegisterInfo::getLargestLegalSuperClass(RC, MF); 488 } 489 490 Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const { 491 const SIFrameLowering *TFI = ST.getFrameLowering(); 492 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); 493 // During ISel lowering we always reserve the stack pointer in entry 494 // functions, but never actually want to reference it when accessing our own 495 // frame. If we need a frame pointer we use it, but otherwise we can just use 496 // an immediate "0" which we represent by returning NoRegister. 497 if (FuncInfo->isEntryFunction()) { 498 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() : Register(); 499 } 500 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() 501 : FuncInfo->getStackPtrOffsetReg(); 502 } 503 504 bool SIRegisterInfo::hasBasePointer(const MachineFunction &MF) const { 505 // When we need stack realignment, we can't reference off of the 506 // stack pointer, so we reserve a base pointer. 507 const MachineFrameInfo &MFI = MF.getFrameInfo(); 508 return MFI.getNumFixedObjects() && shouldRealignStack(MF); 509 } 510 511 Register SIRegisterInfo::getBaseRegister() const { return AMDGPU::SGPR34; } 512 513 const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const { 514 return AMDGPU_AllVGPRs_RegMask; 515 } 516 517 const uint32_t *SIRegisterInfo::getAllAGPRRegMask() const { 518 return AMDGPU_AllAGPRs_RegMask; 519 } 520 521 const uint32_t *SIRegisterInfo::getAllVectorRegMask() const { 522 return AMDGPU_AllVectorRegs_RegMask; 523 } 524 525 const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const { 526 return AMDGPU_AllAllocatableSRegs_RegMask; 527 } 528 529 unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel, 530 unsigned NumRegs) { 531 assert(NumRegs < SubRegFromChannelTableWidthMap.size()); 532 unsigned NumRegIndex = SubRegFromChannelTableWidthMap[NumRegs]; 533 assert(NumRegIndex && "Not implemented"); 534 assert(Channel < SubRegFromChannelTable[NumRegIndex - 1].size()); 535 return SubRegFromChannelTable[NumRegIndex - 1][Channel]; 536 } 537 538 MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg( 539 const MachineFunction &MF) const { 540 unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4; 541 MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx)); 542 return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SGPR_128RegClass); 543 } 544 545 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const { 546 BitVector Reserved(getNumRegs()); 547 Reserved.set(AMDGPU::MODE); 548 549 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 550 551 // Reserve special purpose registers. 552 // 553 // EXEC_LO and EXEC_HI could be allocated and used as regular register, but 554 // this seems likely to result in bugs, so I'm marking them as reserved. 555 reserveRegisterTuples(Reserved, AMDGPU::EXEC); 556 reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR); 557 558 // M0 has to be reserved so that llvm accepts it as a live-in into a block. 559 reserveRegisterTuples(Reserved, AMDGPU::M0); 560 561 // Reserve src_vccz, src_execz, src_scc. 562 reserveRegisterTuples(Reserved, AMDGPU::SRC_VCCZ); 563 reserveRegisterTuples(Reserved, AMDGPU::SRC_EXECZ); 564 reserveRegisterTuples(Reserved, AMDGPU::SRC_SCC); 565 566 // Reserve the memory aperture registers. 567 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE); 568 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT); 569 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE); 570 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT); 571 572 // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen. 573 reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID); 574 575 // Reserve xnack_mask registers - support is not implemented in Codegen. 576 reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK); 577 578 // Reserve lds_direct register - support is not implemented in Codegen. 579 reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT); 580 581 // Reserve Trap Handler registers - support is not implemented in Codegen. 582 reserveRegisterTuples(Reserved, AMDGPU::TBA); 583 reserveRegisterTuples(Reserved, AMDGPU::TMA); 584 reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1); 585 reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3); 586 reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5); 587 reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7); 588 reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9); 589 reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11); 590 reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13); 591 reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15); 592 593 // Reserve null register - it shall never be allocated 594 reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL64); 595 596 // Disallow vcc_hi allocation in wave32. It may be allocated but most likely 597 // will result in bugs. 598 if (isWave32) { 599 Reserved.set(AMDGPU::VCC); 600 Reserved.set(AMDGPU::VCC_HI); 601 } 602 603 // Reserve SGPRs. 604 // 605 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF); 606 unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); 607 for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) { 608 unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); 609 reserveRegisterTuples(Reserved, Reg); 610 } 611 612 for (auto Reg : AMDGPU::SReg_32RegClass) { 613 Reserved.set(getSubReg(Reg, AMDGPU::hi16)); 614 Register Low = getSubReg(Reg, AMDGPU::lo16); 615 // This is to prevent BB vcc liveness errors. 616 if (!AMDGPU::SGPR_LO16RegClass.contains(Low)) 617 Reserved.set(Low); 618 } 619 620 Register ScratchRSrcReg = MFI->getScratchRSrcReg(); 621 if (ScratchRSrcReg != AMDGPU::NoRegister) { 622 // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we 623 // need to spill. 624 // TODO: May need to reserve a VGPR if doing LDS spilling. 625 reserveRegisterTuples(Reserved, ScratchRSrcReg); 626 } 627 628 // We have to assume the SP is needed in case there are calls in the function, 629 // which is detected after the function is lowered. If we aren't really going 630 // to need SP, don't bother reserving it. 631 MCRegister StackPtrReg = MFI->getStackPtrOffsetReg(); 632 if (StackPtrReg) { 633 reserveRegisterTuples(Reserved, StackPtrReg); 634 assert(!isSubRegister(ScratchRSrcReg, StackPtrReg)); 635 } 636 637 MCRegister FrameReg = MFI->getFrameOffsetReg(); 638 if (FrameReg) { 639 reserveRegisterTuples(Reserved, FrameReg); 640 assert(!isSubRegister(ScratchRSrcReg, FrameReg)); 641 } 642 643 if (hasBasePointer(MF)) { 644 MCRegister BasePtrReg = getBaseRegister(); 645 reserveRegisterTuples(Reserved, BasePtrReg); 646 assert(!isSubRegister(ScratchRSrcReg, BasePtrReg)); 647 } 648 649 // Reserve VGPRs/AGPRs. 650 // 651 unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF); 652 unsigned MaxNumAGPRs = MaxNumVGPRs; 653 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs(); 654 655 // Reserve all the AGPRs if there are no instructions to use it. 656 if (!ST.hasMAIInsts()) { 657 for (unsigned i = 0; i < MaxNumAGPRs; ++i) { 658 unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 659 reserveRegisterTuples(Reserved, Reg); 660 } 661 } 662 663 for (auto Reg : AMDGPU::AGPR_32RegClass) { 664 Reserved.set(getSubReg(Reg, AMDGPU::hi16)); 665 } 666 667 // On GFX90A, the number of VGPRs and AGPRs need not be equal. Theoretically, 668 // a wave may have up to 512 total vector registers combining together both 669 // VGPRs and AGPRs. Hence, in an entry function without calls and without 670 // AGPRs used within it, it is possible to use the whole vector register 671 // budget for VGPRs. 672 // 673 // TODO: it shall be possible to estimate maximum AGPR/VGPR pressure and split 674 // register file accordingly. 675 if (ST.hasGFX90AInsts()) { 676 if (MFI->usesAGPRs(MF)) { 677 MaxNumVGPRs /= 2; 678 MaxNumAGPRs = MaxNumVGPRs; 679 } else { 680 if (MaxNumVGPRs > TotalNumVGPRs) { 681 MaxNumAGPRs = MaxNumVGPRs - TotalNumVGPRs; 682 MaxNumVGPRs = TotalNumVGPRs; 683 } else 684 MaxNumAGPRs = 0; 685 } 686 } 687 688 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) { 689 unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i); 690 reserveRegisterTuples(Reserved, Reg); 691 } 692 693 for (unsigned i = MaxNumAGPRs; i < TotalNumVGPRs; ++i) { 694 unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 695 reserveRegisterTuples(Reserved, Reg); 696 } 697 698 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch 699 // VGPR available at all times. 700 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) { 701 reserveRegisterTuples(Reserved, MFI->getVGPRForAGPRCopy()); 702 } 703 704 for (Register Reg : MFI->WWMReservedRegs) 705 reserveRegisterTuples(Reserved, Reg); 706 707 // FIXME: Stop using reserved registers for this. 708 for (MCPhysReg Reg : MFI->getAGPRSpillVGPRs()) 709 reserveRegisterTuples(Reserved, Reg); 710 711 for (MCPhysReg Reg : MFI->getVGPRSpillAGPRs()) 712 reserveRegisterTuples(Reserved, Reg); 713 714 for (auto SSpill : MFI->getSGPRSpillVGPRs()) 715 reserveRegisterTuples(Reserved, SSpill.VGPR); 716 717 return Reserved; 718 } 719 720 bool SIRegisterInfo::isAsmClobberable(const MachineFunction &MF, 721 MCRegister PhysReg) const { 722 return !MF.getRegInfo().isReserved(PhysReg); 723 } 724 725 bool SIRegisterInfo::shouldRealignStack(const MachineFunction &MF) const { 726 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 727 // On entry, the base address is 0, so it can't possibly need any more 728 // alignment. 729 730 // FIXME: Should be able to specify the entry frame alignment per calling 731 // convention instead. 732 if (Info->isEntryFunction()) 733 return false; 734 735 return TargetRegisterInfo::shouldRealignStack(MF); 736 } 737 738 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const { 739 const SIMachineFunctionInfo *Info = Fn.getInfo<SIMachineFunctionInfo>(); 740 if (Info->isEntryFunction()) { 741 const MachineFrameInfo &MFI = Fn.getFrameInfo(); 742 return MFI.hasStackObjects() || MFI.hasCalls(); 743 } 744 745 // May need scavenger for dealing with callee saved registers. 746 return true; 747 } 748 749 bool SIRegisterInfo::requiresFrameIndexScavenging( 750 const MachineFunction &MF) const { 751 // Do not use frame virtual registers. They used to be used for SGPRs, but 752 // once we reach PrologEpilogInserter, we can no longer spill SGPRs. If the 753 // scavenger fails, we can increment/decrement the necessary SGPRs to avoid a 754 // spill. 755 return false; 756 } 757 758 bool SIRegisterInfo::requiresFrameIndexReplacementScavenging( 759 const MachineFunction &MF) const { 760 const MachineFrameInfo &MFI = MF.getFrameInfo(); 761 return MFI.hasStackObjects(); 762 } 763 764 bool SIRegisterInfo::requiresVirtualBaseRegisters( 765 const MachineFunction &) const { 766 // There are no special dedicated stack or frame pointers. 767 return true; 768 } 769 770 int64_t SIRegisterInfo::getScratchInstrOffset(const MachineInstr *MI) const { 771 assert(SIInstrInfo::isMUBUF(*MI) || SIInstrInfo::isFLATScratch(*MI)); 772 773 int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 774 AMDGPU::OpName::offset); 775 return MI->getOperand(OffIdx).getImm(); 776 } 777 778 int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI, 779 int Idx) const { 780 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI)) 781 return 0; 782 783 assert((Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 784 AMDGPU::OpName::vaddr) || 785 (Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 786 AMDGPU::OpName::saddr))) && 787 "Should never see frame index on non-address operand"); 788 789 return getScratchInstrOffset(MI); 790 } 791 792 bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const { 793 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI)) 794 return false; 795 796 int64_t FullOffset = Offset + getScratchInstrOffset(MI); 797 798 if (SIInstrInfo::isMUBUF(*MI)) 799 return !SIInstrInfo::isLegalMUBUFImmOffset(FullOffset); 800 801 const SIInstrInfo *TII = ST.getInstrInfo(); 802 return !TII->isLegalFLATOffset(FullOffset, AMDGPUAS::PRIVATE_ADDRESS, 803 SIInstrFlags::FlatScratch); 804 } 805 806 Register SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, 807 int FrameIdx, 808 int64_t Offset) const { 809 MachineBasicBlock::iterator Ins = MBB->begin(); 810 DebugLoc DL; // Defaults to "unknown" 811 812 if (Ins != MBB->end()) 813 DL = Ins->getDebugLoc(); 814 815 MachineFunction *MF = MBB->getParent(); 816 const SIInstrInfo *TII = ST.getInstrInfo(); 817 MachineRegisterInfo &MRI = MF->getRegInfo(); 818 unsigned MovOpc = ST.enableFlatScratch() ? AMDGPU::S_MOV_B32 819 : AMDGPU::V_MOV_B32_e32; 820 821 Register BaseReg = MRI.createVirtualRegister( 822 ST.enableFlatScratch() ? &AMDGPU::SReg_32_XEXEC_HIRegClass 823 : &AMDGPU::VGPR_32RegClass); 824 825 if (Offset == 0) { 826 BuildMI(*MBB, Ins, DL, TII->get(MovOpc), BaseReg) 827 .addFrameIndex(FrameIdx); 828 return BaseReg; 829 } 830 831 Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 832 833 Register FIReg = MRI.createVirtualRegister( 834 ST.enableFlatScratch() ? &AMDGPU::SReg_32_XM0RegClass 835 : &AMDGPU::VGPR_32RegClass); 836 837 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) 838 .addImm(Offset); 839 BuildMI(*MBB, Ins, DL, TII->get(MovOpc), FIReg) 840 .addFrameIndex(FrameIdx); 841 842 if (ST.enableFlatScratch() ) { 843 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_ADD_I32), BaseReg) 844 .addReg(OffsetReg, RegState::Kill) 845 .addReg(FIReg); 846 return BaseReg; 847 } 848 849 TII->getAddNoCarry(*MBB, Ins, DL, BaseReg) 850 .addReg(OffsetReg, RegState::Kill) 851 .addReg(FIReg) 852 .addImm(0); // clamp bit 853 854 return BaseReg; 855 } 856 857 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg, 858 int64_t Offset) const { 859 const SIInstrInfo *TII = ST.getInstrInfo(); 860 bool IsFlat = TII->isFLATScratch(MI); 861 862 #ifndef NDEBUG 863 // FIXME: Is it possible to be storing a frame index to itself? 864 bool SeenFI = false; 865 for (const MachineOperand &MO: MI.operands()) { 866 if (MO.isFI()) { 867 if (SeenFI) 868 llvm_unreachable("should not see multiple frame indices"); 869 870 SeenFI = true; 871 } 872 } 873 #endif 874 875 MachineOperand *FIOp = 876 TII->getNamedOperand(MI, IsFlat ? AMDGPU::OpName::saddr 877 : AMDGPU::OpName::vaddr); 878 879 MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset); 880 int64_t NewOffset = OffsetOp->getImm() + Offset; 881 882 assert(FIOp && FIOp->isFI() && "frame index must be address operand"); 883 assert(TII->isMUBUF(MI) || TII->isFLATScratch(MI)); 884 885 if (IsFlat) { 886 assert(TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, 887 SIInstrFlags::FlatScratch) && 888 "offset should be legal"); 889 FIOp->ChangeToRegister(BaseReg, false); 890 OffsetOp->setImm(NewOffset); 891 return; 892 } 893 894 #ifndef NDEBUG 895 MachineOperand *SOffset = TII->getNamedOperand(MI, AMDGPU::OpName::soffset); 896 assert(SOffset->isImm() && SOffset->getImm() == 0); 897 #endif 898 899 assert(SIInstrInfo::isLegalMUBUFImmOffset(NewOffset) && 900 "offset should be legal"); 901 902 FIOp->ChangeToRegister(BaseReg, false); 903 OffsetOp->setImm(NewOffset); 904 } 905 906 bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, 907 Register BaseReg, 908 int64_t Offset) const { 909 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isFLATScratch(*MI)) 910 return false; 911 912 int64_t NewOffset = Offset + getScratchInstrOffset(MI); 913 914 if (SIInstrInfo::isMUBUF(*MI)) 915 return SIInstrInfo::isLegalMUBUFImmOffset(NewOffset); 916 917 const SIInstrInfo *TII = ST.getInstrInfo(); 918 return TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, 919 SIInstrFlags::FlatScratch); 920 } 921 922 const TargetRegisterClass *SIRegisterInfo::getPointerRegClass( 923 const MachineFunction &MF, unsigned Kind) const { 924 // This is inaccurate. It depends on the instruction and address space. The 925 // only place where we should hit this is for dealing with frame indexes / 926 // private accesses, so this is correct in that case. 927 return &AMDGPU::VGPR_32RegClass; 928 } 929 930 const TargetRegisterClass * 931 SIRegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const { 932 if (isAGPRClass(RC) && !ST.hasGFX90AInsts()) 933 return getEquivalentVGPRClass(RC); 934 935 return RC; 936 } 937 938 static unsigned getNumSubRegsForSpillOp(unsigned Op) { 939 940 switch (Op) { 941 case AMDGPU::SI_SPILL_S1024_SAVE: 942 case AMDGPU::SI_SPILL_S1024_RESTORE: 943 case AMDGPU::SI_SPILL_V1024_SAVE: 944 case AMDGPU::SI_SPILL_V1024_RESTORE: 945 case AMDGPU::SI_SPILL_A1024_SAVE: 946 case AMDGPU::SI_SPILL_A1024_RESTORE: 947 case AMDGPU::SI_SPILL_AV1024_SAVE: 948 case AMDGPU::SI_SPILL_AV1024_RESTORE: 949 return 32; 950 case AMDGPU::SI_SPILL_S512_SAVE: 951 case AMDGPU::SI_SPILL_S512_RESTORE: 952 case AMDGPU::SI_SPILL_V512_SAVE: 953 case AMDGPU::SI_SPILL_V512_RESTORE: 954 case AMDGPU::SI_SPILL_A512_SAVE: 955 case AMDGPU::SI_SPILL_A512_RESTORE: 956 case AMDGPU::SI_SPILL_AV512_SAVE: 957 case AMDGPU::SI_SPILL_AV512_RESTORE: 958 return 16; 959 case AMDGPU::SI_SPILL_S256_SAVE: 960 case AMDGPU::SI_SPILL_S256_RESTORE: 961 case AMDGPU::SI_SPILL_V256_SAVE: 962 case AMDGPU::SI_SPILL_V256_RESTORE: 963 case AMDGPU::SI_SPILL_A256_SAVE: 964 case AMDGPU::SI_SPILL_A256_RESTORE: 965 case AMDGPU::SI_SPILL_AV256_SAVE: 966 case AMDGPU::SI_SPILL_AV256_RESTORE: 967 return 8; 968 case AMDGPU::SI_SPILL_S224_SAVE: 969 case AMDGPU::SI_SPILL_S224_RESTORE: 970 case AMDGPU::SI_SPILL_V224_SAVE: 971 case AMDGPU::SI_SPILL_V224_RESTORE: 972 case AMDGPU::SI_SPILL_A224_SAVE: 973 case AMDGPU::SI_SPILL_A224_RESTORE: 974 case AMDGPU::SI_SPILL_AV224_SAVE: 975 case AMDGPU::SI_SPILL_AV224_RESTORE: 976 return 7; 977 case AMDGPU::SI_SPILL_S192_SAVE: 978 case AMDGPU::SI_SPILL_S192_RESTORE: 979 case AMDGPU::SI_SPILL_V192_SAVE: 980 case AMDGPU::SI_SPILL_V192_RESTORE: 981 case AMDGPU::SI_SPILL_A192_SAVE: 982 case AMDGPU::SI_SPILL_A192_RESTORE: 983 case AMDGPU::SI_SPILL_AV192_SAVE: 984 case AMDGPU::SI_SPILL_AV192_RESTORE: 985 return 6; 986 case AMDGPU::SI_SPILL_S160_SAVE: 987 case AMDGPU::SI_SPILL_S160_RESTORE: 988 case AMDGPU::SI_SPILL_V160_SAVE: 989 case AMDGPU::SI_SPILL_V160_RESTORE: 990 case AMDGPU::SI_SPILL_A160_SAVE: 991 case AMDGPU::SI_SPILL_A160_RESTORE: 992 case AMDGPU::SI_SPILL_AV160_SAVE: 993 case AMDGPU::SI_SPILL_AV160_RESTORE: 994 return 5; 995 case AMDGPU::SI_SPILL_S128_SAVE: 996 case AMDGPU::SI_SPILL_S128_RESTORE: 997 case AMDGPU::SI_SPILL_V128_SAVE: 998 case AMDGPU::SI_SPILL_V128_RESTORE: 999 case AMDGPU::SI_SPILL_A128_SAVE: 1000 case AMDGPU::SI_SPILL_A128_RESTORE: 1001 case AMDGPU::SI_SPILL_AV128_SAVE: 1002 case AMDGPU::SI_SPILL_AV128_RESTORE: 1003 return 4; 1004 case AMDGPU::SI_SPILL_S96_SAVE: 1005 case AMDGPU::SI_SPILL_S96_RESTORE: 1006 case AMDGPU::SI_SPILL_V96_SAVE: 1007 case AMDGPU::SI_SPILL_V96_RESTORE: 1008 case AMDGPU::SI_SPILL_A96_SAVE: 1009 case AMDGPU::SI_SPILL_A96_RESTORE: 1010 case AMDGPU::SI_SPILL_AV96_SAVE: 1011 case AMDGPU::SI_SPILL_AV96_RESTORE: 1012 return 3; 1013 case AMDGPU::SI_SPILL_S64_SAVE: 1014 case AMDGPU::SI_SPILL_S64_RESTORE: 1015 case AMDGPU::SI_SPILL_V64_SAVE: 1016 case AMDGPU::SI_SPILL_V64_RESTORE: 1017 case AMDGPU::SI_SPILL_A64_SAVE: 1018 case AMDGPU::SI_SPILL_A64_RESTORE: 1019 case AMDGPU::SI_SPILL_AV64_SAVE: 1020 case AMDGPU::SI_SPILL_AV64_RESTORE: 1021 return 2; 1022 case AMDGPU::SI_SPILL_S32_SAVE: 1023 case AMDGPU::SI_SPILL_S32_RESTORE: 1024 case AMDGPU::SI_SPILL_V32_SAVE: 1025 case AMDGPU::SI_SPILL_V32_RESTORE: 1026 case AMDGPU::SI_SPILL_A32_SAVE: 1027 case AMDGPU::SI_SPILL_A32_RESTORE: 1028 case AMDGPU::SI_SPILL_AV32_SAVE: 1029 case AMDGPU::SI_SPILL_AV32_RESTORE: 1030 return 1; 1031 default: llvm_unreachable("Invalid spill opcode"); 1032 } 1033 } 1034 1035 static int getOffsetMUBUFStore(unsigned Opc) { 1036 switch (Opc) { 1037 case AMDGPU::BUFFER_STORE_DWORD_OFFEN: 1038 return AMDGPU::BUFFER_STORE_DWORD_OFFSET; 1039 case AMDGPU::BUFFER_STORE_BYTE_OFFEN: 1040 return AMDGPU::BUFFER_STORE_BYTE_OFFSET; 1041 case AMDGPU::BUFFER_STORE_SHORT_OFFEN: 1042 return AMDGPU::BUFFER_STORE_SHORT_OFFSET; 1043 case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN: 1044 return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET; 1045 case AMDGPU::BUFFER_STORE_DWORDX3_OFFEN: 1046 return AMDGPU::BUFFER_STORE_DWORDX3_OFFSET; 1047 case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN: 1048 return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET; 1049 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN: 1050 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET; 1051 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN: 1052 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET; 1053 default: 1054 return -1; 1055 } 1056 } 1057 1058 static int getOffsetMUBUFLoad(unsigned Opc) { 1059 switch (Opc) { 1060 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN: 1061 return AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 1062 case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN: 1063 return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET; 1064 case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN: 1065 return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET; 1066 case AMDGPU::BUFFER_LOAD_USHORT_OFFEN: 1067 return AMDGPU::BUFFER_LOAD_USHORT_OFFSET; 1068 case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN: 1069 return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET; 1070 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN: 1071 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET; 1072 case AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN: 1073 return AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET; 1074 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN: 1075 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET; 1076 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN: 1077 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET; 1078 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN: 1079 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET; 1080 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN: 1081 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET; 1082 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN: 1083 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET; 1084 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN: 1085 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET; 1086 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN: 1087 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET; 1088 default: 1089 return -1; 1090 } 1091 } 1092 1093 static int getOffenMUBUFStore(unsigned Opc) { 1094 switch (Opc) { 1095 case AMDGPU::BUFFER_STORE_DWORD_OFFSET: 1096 return AMDGPU::BUFFER_STORE_DWORD_OFFEN; 1097 case AMDGPU::BUFFER_STORE_BYTE_OFFSET: 1098 return AMDGPU::BUFFER_STORE_BYTE_OFFEN; 1099 case AMDGPU::BUFFER_STORE_SHORT_OFFSET: 1100 return AMDGPU::BUFFER_STORE_SHORT_OFFEN; 1101 case AMDGPU::BUFFER_STORE_DWORDX2_OFFSET: 1102 return AMDGPU::BUFFER_STORE_DWORDX2_OFFEN; 1103 case AMDGPU::BUFFER_STORE_DWORDX3_OFFSET: 1104 return AMDGPU::BUFFER_STORE_DWORDX3_OFFEN; 1105 case AMDGPU::BUFFER_STORE_DWORDX4_OFFSET: 1106 return AMDGPU::BUFFER_STORE_DWORDX4_OFFEN; 1107 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET: 1108 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN; 1109 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET: 1110 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN; 1111 default: 1112 return -1; 1113 } 1114 } 1115 1116 static int getOffenMUBUFLoad(unsigned Opc) { 1117 switch (Opc) { 1118 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET: 1119 return AMDGPU::BUFFER_LOAD_DWORD_OFFEN; 1120 case AMDGPU::BUFFER_LOAD_UBYTE_OFFSET: 1121 return AMDGPU::BUFFER_LOAD_UBYTE_OFFEN; 1122 case AMDGPU::BUFFER_LOAD_SBYTE_OFFSET: 1123 return AMDGPU::BUFFER_LOAD_SBYTE_OFFEN; 1124 case AMDGPU::BUFFER_LOAD_USHORT_OFFSET: 1125 return AMDGPU::BUFFER_LOAD_USHORT_OFFEN; 1126 case AMDGPU::BUFFER_LOAD_SSHORT_OFFSET: 1127 return AMDGPU::BUFFER_LOAD_SSHORT_OFFEN; 1128 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET: 1129 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN; 1130 case AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET: 1131 return AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN; 1132 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET: 1133 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN; 1134 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET: 1135 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN; 1136 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET: 1137 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN; 1138 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET: 1139 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN; 1140 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET: 1141 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN; 1142 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET: 1143 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN; 1144 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET: 1145 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN; 1146 default: 1147 return -1; 1148 } 1149 } 1150 1151 static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST, 1152 MachineBasicBlock &MBB, 1153 MachineBasicBlock::iterator MI, 1154 int Index, unsigned Lane, 1155 unsigned ValueReg, bool IsKill) { 1156 MachineFunction *MF = MBB.getParent(); 1157 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1158 const SIInstrInfo *TII = ST.getInstrInfo(); 1159 1160 MCPhysReg Reg = MFI->getVGPRToAGPRSpill(Index, Lane); 1161 1162 if (Reg == AMDGPU::NoRegister) 1163 return MachineInstrBuilder(); 1164 1165 bool IsStore = MI->mayStore(); 1166 MachineRegisterInfo &MRI = MF->getRegInfo(); 1167 auto *TRI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo()); 1168 1169 unsigned Dst = IsStore ? Reg : ValueReg; 1170 unsigned Src = IsStore ? ValueReg : Reg; 1171 bool IsVGPR = TRI->isVGPR(MRI, Reg); 1172 DebugLoc DL = MI->getDebugLoc(); 1173 if (IsVGPR == TRI->isVGPR(MRI, ValueReg)) { 1174 // Spiller during regalloc may restore a spilled register to its superclass. 1175 // It could result in AGPR spills restored to VGPRs or the other way around, 1176 // making the src and dst with identical regclasses at this point. It just 1177 // needs a copy in such cases. 1178 auto CopyMIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), Dst) 1179 .addReg(Src, getKillRegState(IsKill)); 1180 CopyMIB->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1181 return CopyMIB; 1182 } 1183 unsigned Opc = (IsStore ^ IsVGPR) ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 1184 : AMDGPU::V_ACCVGPR_READ_B32_e64; 1185 1186 auto MIB = BuildMI(MBB, MI, DL, TII->get(Opc), Dst) 1187 .addReg(Src, getKillRegState(IsKill)); 1188 MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1189 return MIB; 1190 } 1191 1192 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not 1193 // need to handle the case where an SGPR may need to be spilled while spilling. 1194 static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST, 1195 MachineFrameInfo &MFI, 1196 MachineBasicBlock::iterator MI, 1197 int Index, 1198 int64_t Offset) { 1199 const SIInstrInfo *TII = ST.getInstrInfo(); 1200 MachineBasicBlock *MBB = MI->getParent(); 1201 const DebugLoc &DL = MI->getDebugLoc(); 1202 bool IsStore = MI->mayStore(); 1203 1204 unsigned Opc = MI->getOpcode(); 1205 int LoadStoreOp = IsStore ? 1206 getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc); 1207 if (LoadStoreOp == -1) 1208 return false; 1209 1210 const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); 1211 if (spillVGPRtoAGPR(ST, *MBB, MI, Index, 0, Reg->getReg(), false).getInstr()) 1212 return true; 1213 1214 MachineInstrBuilder NewMI = 1215 BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp)) 1216 .add(*Reg) 1217 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)) 1218 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)) 1219 .addImm(Offset) 1220 .addImm(0) // cpol 1221 .addImm(0) // tfe 1222 .addImm(0) // swz 1223 .cloneMemRefs(*MI); 1224 1225 const MachineOperand *VDataIn = TII->getNamedOperand(*MI, 1226 AMDGPU::OpName::vdata_in); 1227 if (VDataIn) 1228 NewMI.add(*VDataIn); 1229 return true; 1230 } 1231 1232 static unsigned getFlatScratchSpillOpcode(const SIInstrInfo *TII, 1233 unsigned LoadStoreOp, 1234 unsigned EltSize) { 1235 bool IsStore = TII->get(LoadStoreOp).mayStore(); 1236 bool HasVAddr = AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::vaddr) != -1; 1237 bool UseST = 1238 !HasVAddr && 1239 AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::saddr) < 0; 1240 1241 switch (EltSize) { 1242 case 4: 1243 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORD_SADDR 1244 : AMDGPU::SCRATCH_LOAD_DWORD_SADDR; 1245 break; 1246 case 8: 1247 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX2_SADDR 1248 : AMDGPU::SCRATCH_LOAD_DWORDX2_SADDR; 1249 break; 1250 case 12: 1251 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX3_SADDR 1252 : AMDGPU::SCRATCH_LOAD_DWORDX3_SADDR; 1253 break; 1254 case 16: 1255 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX4_SADDR 1256 : AMDGPU::SCRATCH_LOAD_DWORDX4_SADDR; 1257 break; 1258 default: 1259 llvm_unreachable("Unexpected spill load/store size!"); 1260 } 1261 1262 if (HasVAddr) 1263 LoadStoreOp = AMDGPU::getFlatScratchInstSVfromSS(LoadStoreOp); 1264 else if (UseST) 1265 LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp); 1266 1267 return LoadStoreOp; 1268 } 1269 1270 void SIRegisterInfo::buildSpillLoadStore( 1271 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, 1272 unsigned LoadStoreOp, int Index, Register ValueReg, bool IsKill, 1273 MCRegister ScratchOffsetReg, int64_t InstOffset, MachineMemOperand *MMO, 1274 RegScavenger *RS, LivePhysRegs *LiveRegs) const { 1275 assert((!RS || !LiveRegs) && "Only RS or LiveRegs can be set but not both"); 1276 1277 MachineFunction *MF = MBB.getParent(); 1278 const SIInstrInfo *TII = ST.getInstrInfo(); 1279 const MachineFrameInfo &MFI = MF->getFrameInfo(); 1280 const SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>(); 1281 1282 const MCInstrDesc *Desc = &TII->get(LoadStoreOp); 1283 bool IsStore = Desc->mayStore(); 1284 bool IsFlat = TII->isFLATScratch(LoadStoreOp); 1285 1286 bool CanClobberSCC = false; 1287 bool Scavenged = false; 1288 MCRegister SOffset = ScratchOffsetReg; 1289 1290 const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg); 1291 // On gfx90a+ AGPR is a regular VGPR acceptable for loads and stores. 1292 const bool IsAGPR = !ST.hasGFX90AInsts() && isAGPRClass(RC); 1293 const unsigned RegWidth = AMDGPU::getRegBitWidth(RC->getID()) / 8; 1294 1295 // Always use 4 byte operations for AGPRs because we need to scavenge 1296 // a temporary VGPR. 1297 unsigned EltSize = (IsFlat && !IsAGPR) ? std::min(RegWidth, 16u) : 4u; 1298 unsigned NumSubRegs = RegWidth / EltSize; 1299 unsigned Size = NumSubRegs * EltSize; 1300 unsigned RemSize = RegWidth - Size; 1301 unsigned NumRemSubRegs = RemSize ? 1 : 0; 1302 int64_t Offset = InstOffset + MFI.getObjectOffset(Index); 1303 int64_t MaterializedOffset = Offset; 1304 1305 int64_t MaxOffset = Offset + Size + RemSize - EltSize; 1306 int64_t ScratchOffsetRegDelta = 0; 1307 1308 if (IsFlat && EltSize > 4) { 1309 LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize); 1310 Desc = &TII->get(LoadStoreOp); 1311 } 1312 1313 Align Alignment = MFI.getObjectAlign(Index); 1314 const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo(); 1315 1316 assert((IsFlat || ((Offset % EltSize) == 0)) && 1317 "unexpected VGPR spill offset"); 1318 1319 // Track a VGPR to use for a constant offset we need to materialize. 1320 Register TmpOffsetVGPR; 1321 1322 // Track a VGPR to use as an intermediate value. 1323 Register TmpIntermediateVGPR; 1324 bool UseVGPROffset = false; 1325 1326 // Materialize a VGPR offset required for the given SGPR/VGPR/Immediate 1327 // combination. 1328 auto MaterializeVOffset = [&](Register SGPRBase, Register TmpVGPR, 1329 int64_t VOffset) { 1330 // We are using a VGPR offset 1331 if (IsFlat && SGPRBase) { 1332 // We only have 1 VGPR offset, or 1 SGPR offset. We don't have a free 1333 // SGPR, so perform the add as vector. 1334 // We don't need a base SGPR in the kernel. 1335 1336 if (ST.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) >= 2) { 1337 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64), TmpVGPR) 1338 .addReg(SGPRBase) 1339 .addImm(VOffset) 1340 .addImm(0); // clamp 1341 } else { 1342 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) 1343 .addReg(SGPRBase); 1344 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e32), TmpVGPR) 1345 .addImm(VOffset) 1346 .addReg(TmpOffsetVGPR); 1347 } 1348 } else { 1349 assert(TmpOffsetVGPR); 1350 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) 1351 .addImm(VOffset); 1352 } 1353 }; 1354 1355 bool IsOffsetLegal = 1356 IsFlat ? TII->isLegalFLATOffset(MaxOffset, AMDGPUAS::PRIVATE_ADDRESS, 1357 SIInstrFlags::FlatScratch) 1358 : SIInstrInfo::isLegalMUBUFImmOffset(MaxOffset); 1359 if (!IsOffsetLegal || (IsFlat && !SOffset && !ST.hasFlatScratchSTMode())) { 1360 SOffset = MCRegister(); 1361 1362 // We don't have access to the register scavenger if this function is called 1363 // during PEI::scavengeFrameVirtualRegs() so use LiveRegs in this case. 1364 // TODO: Clobbering SCC is not necessary for scratch instructions in the 1365 // entry. 1366 if (RS) { 1367 SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0, false); 1368 1369 // Piggy back on the liveness scan we just did see if SCC is dead. 1370 CanClobberSCC = !RS->isRegUsed(AMDGPU::SCC); 1371 } else if (LiveRegs) { 1372 CanClobberSCC = !LiveRegs->contains(AMDGPU::SCC); 1373 for (MCRegister Reg : AMDGPU::SGPR_32RegClass) { 1374 if (LiveRegs->available(MF->getRegInfo(), Reg)) { 1375 SOffset = Reg; 1376 break; 1377 } 1378 } 1379 } 1380 1381 if (ScratchOffsetReg != AMDGPU::NoRegister && !CanClobberSCC) 1382 SOffset = Register(); 1383 1384 if (!SOffset) { 1385 UseVGPROffset = true; 1386 1387 if (RS) { 1388 TmpOffsetVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1389 } else { 1390 assert(LiveRegs); 1391 for (MCRegister Reg : AMDGPU::VGPR_32RegClass) { 1392 if (LiveRegs->available(MF->getRegInfo(), Reg)) { 1393 TmpOffsetVGPR = Reg; 1394 break; 1395 } 1396 } 1397 } 1398 1399 assert(TmpOffsetVGPR); 1400 } else if (!SOffset && CanClobberSCC) { 1401 // There are no free SGPRs, and since we are in the process of spilling 1402 // VGPRs too. Since we need a VGPR in order to spill SGPRs (this is true 1403 // on SI/CI and on VI it is true until we implement spilling using scalar 1404 // stores), we have no way to free up an SGPR. Our solution here is to 1405 // add the offset directly to the ScratchOffset or StackPtrOffset 1406 // register, and then subtract the offset after the spill to return the 1407 // register to it's original value. 1408 1409 // TODO: If we don't have to do an emergency stack slot spill, converting 1410 // to use the VGPR offset is fewer instructions. 1411 if (!ScratchOffsetReg) 1412 ScratchOffsetReg = FuncInfo->getStackPtrOffsetReg(); 1413 SOffset = ScratchOffsetReg; 1414 ScratchOffsetRegDelta = Offset; 1415 } else { 1416 Scavenged = true; 1417 } 1418 1419 // We currently only support spilling VGPRs to EltSize boundaries, meaning 1420 // we can simplify the adjustment of Offset here to just scale with 1421 // WavefrontSize. 1422 if (!IsFlat && !UseVGPROffset) 1423 Offset *= ST.getWavefrontSize(); 1424 1425 if (!UseVGPROffset && !SOffset) 1426 report_fatal_error("could not scavenge SGPR to spill in entry function"); 1427 1428 if (UseVGPROffset) { 1429 // We are using a VGPR offset 1430 MaterializeVOffset(ScratchOffsetReg, TmpOffsetVGPR, Offset); 1431 } else if (ScratchOffsetReg == AMDGPU::NoRegister) { 1432 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset).addImm(Offset); 1433 } else { 1434 assert(Offset != 0); 1435 auto Add = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset) 1436 .addReg(ScratchOffsetReg) 1437 .addImm(Offset); 1438 Add->getOperand(3).setIsDead(); // Mark SCC as dead. 1439 } 1440 1441 Offset = 0; 1442 } 1443 1444 if (IsFlat && SOffset == AMDGPU::NoRegister) { 1445 assert(AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::vaddr) < 0 1446 && "Unexpected vaddr for flat scratch with a FI operand"); 1447 1448 if (UseVGPROffset) { 1449 LoadStoreOp = AMDGPU::getFlatScratchInstSVfromSS(LoadStoreOp); 1450 } else { 1451 assert(ST.hasFlatScratchSTMode()); 1452 LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp); 1453 } 1454 1455 Desc = &TII->get(LoadStoreOp); 1456 } 1457 1458 for (unsigned i = 0, e = NumSubRegs + NumRemSubRegs, RegOffset = 0; i != e; 1459 ++i, RegOffset += EltSize) { 1460 if (i == NumSubRegs) { 1461 EltSize = RemSize; 1462 LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize); 1463 } 1464 Desc = &TII->get(LoadStoreOp); 1465 1466 if (!IsFlat && UseVGPROffset) { 1467 int NewLoadStoreOp = IsStore ? getOffenMUBUFStore(LoadStoreOp) 1468 : getOffenMUBUFLoad(LoadStoreOp); 1469 Desc = &TII->get(NewLoadStoreOp); 1470 } 1471 1472 if (UseVGPROffset && TmpOffsetVGPR == TmpIntermediateVGPR) { 1473 // If we are spilling an AGPR beyond the range of the memory instruction 1474 // offset and need to use a VGPR offset, we ideally have at least 2 1475 // scratch VGPRs. If we don't have a second free VGPR without spilling, 1476 // recycle the VGPR used for the offset which requires resetting after 1477 // each subregister. 1478 1479 MaterializeVOffset(ScratchOffsetReg, TmpOffsetVGPR, MaterializedOffset); 1480 } 1481 1482 unsigned NumRegs = EltSize / 4; 1483 Register SubReg = e == 1 1484 ? ValueReg 1485 : Register(getSubReg(ValueReg, 1486 getSubRegFromChannel(RegOffset / 4, NumRegs))); 1487 1488 unsigned SOffsetRegState = 0; 1489 unsigned SrcDstRegState = getDefRegState(!IsStore); 1490 const bool IsLastSubReg = i + 1 == e; 1491 if (IsLastSubReg) { 1492 SOffsetRegState |= getKillRegState(Scavenged); 1493 // The last implicit use carries the "Kill" flag. 1494 SrcDstRegState |= getKillRegState(IsKill); 1495 } 1496 1497 // Make sure the whole register is defined if there are undef components by 1498 // adding an implicit def of the super-reg on the first instruction. 1499 bool NeedSuperRegDef = e > 1 && IsStore && i == 0; 1500 bool NeedSuperRegImpOperand = e > 1; 1501 1502 // Remaining element size to spill into memory after some parts of it 1503 // spilled into either AGPRs or VGPRs. 1504 unsigned RemEltSize = EltSize; 1505 1506 // AGPRs to spill VGPRs and vice versa are allocated in a reverse order, 1507 // starting from the last lane. In case if a register cannot be completely 1508 // spilled into another register that will ensure its alignment does not 1509 // change. For targets with VGPR alignment requirement this is important 1510 // in case of flat scratch usage as we might get a scratch_load or 1511 // scratch_store of an unaligned register otherwise. 1512 for (int LaneS = (RegOffset + EltSize) / 4 - 1, Lane = LaneS, 1513 LaneE = RegOffset / 4; 1514 Lane >= LaneE; --Lane) { 1515 bool IsSubReg = e > 1 || EltSize > 4; 1516 Register Sub = IsSubReg 1517 ? Register(getSubReg(ValueReg, getSubRegFromChannel(Lane))) 1518 : ValueReg; 1519 auto MIB = spillVGPRtoAGPR(ST, MBB, MI, Index, Lane, Sub, IsKill); 1520 if (!MIB.getInstr()) 1521 break; 1522 if (NeedSuperRegDef || (IsSubReg && IsStore && Lane == LaneS && !i)) { 1523 MIB.addReg(ValueReg, RegState::ImplicitDefine); 1524 NeedSuperRegDef = false; 1525 } 1526 if (IsSubReg || NeedSuperRegImpOperand) { 1527 NeedSuperRegImpOperand = true; 1528 unsigned State = SrcDstRegState; 1529 if (Lane != LaneE) 1530 State &= ~RegState::Kill; 1531 MIB.addReg(ValueReg, RegState::Implicit | State); 1532 } 1533 RemEltSize -= 4; 1534 } 1535 1536 if (!RemEltSize) // Fully spilled into AGPRs. 1537 continue; 1538 1539 if (RemEltSize != EltSize) { // Partially spilled to AGPRs 1540 assert(IsFlat && EltSize > 4); 1541 1542 unsigned NumRegs = RemEltSize / 4; 1543 SubReg = Register(getSubReg(ValueReg, 1544 getSubRegFromChannel(RegOffset / 4, NumRegs))); 1545 unsigned Opc = getFlatScratchSpillOpcode(TII, LoadStoreOp, RemEltSize); 1546 Desc = &TII->get(Opc); 1547 } 1548 1549 unsigned FinalReg = SubReg; 1550 1551 if (IsAGPR) { 1552 assert(EltSize == 4); 1553 1554 if (!TmpIntermediateVGPR) { 1555 TmpIntermediateVGPR = FuncInfo->getVGPRForAGPRCopy(); 1556 assert(MF->getRegInfo().isReserved(TmpIntermediateVGPR)); 1557 } 1558 if (IsStore) { 1559 auto AccRead = BuildMI(MBB, MI, DL, 1560 TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64), 1561 TmpIntermediateVGPR) 1562 .addReg(SubReg, getKillRegState(IsKill)); 1563 if (NeedSuperRegDef) 1564 AccRead.addReg(ValueReg, RegState::ImplicitDefine); 1565 AccRead->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1566 } 1567 SubReg = TmpIntermediateVGPR; 1568 } else if (UseVGPROffset) { 1569 // FIXME: change to scavengeRegisterBackwards() 1570 if (!TmpOffsetVGPR) { 1571 TmpOffsetVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1572 RS->setRegUsed(TmpOffsetVGPR); 1573 } 1574 } 1575 1576 MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(RegOffset); 1577 MachineMemOperand *NewMMO = 1578 MF->getMachineMemOperand(PInfo, MMO->getFlags(), RemEltSize, 1579 commonAlignment(Alignment, RegOffset)); 1580 1581 auto MIB = 1582 BuildMI(MBB, MI, DL, *Desc) 1583 .addReg(SubReg, getDefRegState(!IsStore) | getKillRegState(IsKill)); 1584 1585 if (UseVGPROffset) { 1586 // For an AGPR spill, we reuse the same temp VGPR for the offset and the 1587 // intermediate accvgpr_write. 1588 MIB.addReg(TmpOffsetVGPR, getKillRegState(IsLastSubReg && !IsAGPR)); 1589 } 1590 1591 if (!IsFlat) 1592 MIB.addReg(FuncInfo->getScratchRSrcReg()); 1593 1594 if (SOffset == AMDGPU::NoRegister) { 1595 if (!IsFlat) { 1596 if (UseVGPROffset && ScratchOffsetReg) { 1597 assert(!FuncInfo->isEntryFunction()); 1598 MIB.addReg(ScratchOffsetReg); 1599 } else { 1600 assert(FuncInfo->isEntryFunction()); 1601 MIB.addImm(0); 1602 } 1603 } 1604 } else { 1605 MIB.addReg(SOffset, SOffsetRegState); 1606 } 1607 MIB.addImm(Offset + RegOffset) 1608 .addImm(0); // cpol 1609 if (!IsFlat) 1610 MIB.addImm(0) // tfe 1611 .addImm(0); // swz 1612 MIB.addMemOperand(NewMMO); 1613 1614 if (!IsAGPR && NeedSuperRegDef) 1615 MIB.addReg(ValueReg, RegState::ImplicitDefine); 1616 1617 if (!IsStore && IsAGPR && TmpIntermediateVGPR != AMDGPU::NoRegister) { 1618 MIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), 1619 FinalReg) 1620 .addReg(TmpIntermediateVGPR, RegState::Kill); 1621 MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse); 1622 } 1623 1624 if (NeedSuperRegImpOperand) 1625 MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState); 1626 } 1627 1628 if (ScratchOffsetRegDelta != 0) { 1629 // Subtract the offset we added to the ScratchOffset register. 1630 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset) 1631 .addReg(SOffset) 1632 .addImm(-ScratchOffsetRegDelta); 1633 } 1634 } 1635 1636 void SIRegisterInfo::buildVGPRSpillLoadStore(SGPRSpillBuilder &SB, int Index, 1637 int Offset, bool IsLoad, 1638 bool IsKill) const { 1639 // Load/store VGPR 1640 MachineFrameInfo &FrameInfo = SB.MF.getFrameInfo(); 1641 assert(FrameInfo.getStackID(Index) != TargetStackID::SGPRSpill); 1642 1643 Register FrameReg = 1644 FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(SB.MF) 1645 ? getBaseRegister() 1646 : getFrameRegister(SB.MF); 1647 1648 Align Alignment = FrameInfo.getObjectAlign(Index); 1649 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SB.MF, Index); 1650 MachineMemOperand *MMO = SB.MF.getMachineMemOperand( 1651 PtrInfo, IsLoad ? MachineMemOperand::MOLoad : MachineMemOperand::MOStore, 1652 SB.EltSize, Alignment); 1653 1654 if (IsLoad) { 1655 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR 1656 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 1657 buildSpillLoadStore(*SB.MBB, SB.MI, SB.DL, Opc, Index, SB.TmpVGPR, false, 1658 FrameReg, Offset * SB.EltSize, MMO, SB.RS); 1659 } else { 1660 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR 1661 : AMDGPU::BUFFER_STORE_DWORD_OFFSET; 1662 buildSpillLoadStore(*SB.MBB, SB.MI, SB.DL, Opc, Index, SB.TmpVGPR, IsKill, 1663 FrameReg, Offset * SB.EltSize, MMO, SB.RS); 1664 // This only ever adds one VGPR spill 1665 SB.MFI.addToSpilledVGPRs(1); 1666 } 1667 } 1668 1669 bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, 1670 int Index, 1671 RegScavenger *RS, 1672 LiveIntervals *LIS, 1673 bool OnlyToVGPR) const { 1674 SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, Index, RS); 1675 1676 ArrayRef<SpilledReg> VGPRSpills = SB.MFI.getSGPRToVGPRSpills(Index); 1677 bool SpillToVGPR = !VGPRSpills.empty(); 1678 if (OnlyToVGPR && !SpillToVGPR) 1679 return false; 1680 1681 assert(SpillToVGPR || (SB.SuperReg != SB.MFI.getStackPtrOffsetReg() && 1682 SB.SuperReg != SB.MFI.getFrameOffsetReg())); 1683 1684 if (SpillToVGPR) { 1685 1686 assert(SB.NumSubRegs == VGPRSpills.size() && 1687 "Num of VGPR lanes should be equal to num of SGPRs spilled"); 1688 1689 for (unsigned i = 0, e = SB.NumSubRegs; i < e; ++i) { 1690 Register SubReg = 1691 SB.NumSubRegs == 1 1692 ? SB.SuperReg 1693 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1694 SpilledReg Spill = VGPRSpills[i]; 1695 1696 bool UseKill = SB.IsKill && i == SB.NumSubRegs - 1; 1697 1698 // Mark the "old value of vgpr" input undef only if this is the first sgpr 1699 // spill to this specific vgpr in the first basic block. 1700 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, 1701 SB.TII.get(AMDGPU::V_WRITELANE_B32), Spill.VGPR) 1702 .addReg(SubReg, getKillRegState(UseKill)) 1703 .addImm(Spill.Lane) 1704 .addReg(Spill.VGPR); 1705 if (LIS) { 1706 if (i == 0) 1707 LIS->ReplaceMachineInstrInMaps(*MI, *MIB); 1708 else 1709 LIS->InsertMachineInstrInMaps(*MIB); 1710 } 1711 1712 if (i == 0 && SB.NumSubRegs > 1) { 1713 // We may be spilling a super-register which is only partially defined, 1714 // and need to ensure later spills think the value is defined. 1715 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1716 } 1717 1718 if (SB.NumSubRegs > 1) 1719 MIB.addReg(SB.SuperReg, getKillRegState(UseKill) | RegState::Implicit); 1720 1721 // FIXME: Since this spills to another register instead of an actual 1722 // frame index, we should delete the frame index when all references to 1723 // it are fixed. 1724 } 1725 } else { 1726 SB.prepare(); 1727 1728 // SubReg carries the "Kill" flag when SubReg == SB.SuperReg. 1729 unsigned SubKillState = getKillRegState((SB.NumSubRegs == 1) && SB.IsKill); 1730 1731 // Per VGPR helper data 1732 auto PVD = SB.getPerVGPRData(); 1733 1734 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1735 unsigned TmpVGPRFlags = RegState::Undef; 1736 1737 // Write sub registers into the VGPR 1738 for (unsigned i = Offset * PVD.PerVGPR, 1739 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1740 i < e; ++i) { 1741 Register SubReg = 1742 SB.NumSubRegs == 1 1743 ? SB.SuperReg 1744 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1745 1746 MachineInstrBuilder WriteLane = 1747 BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_WRITELANE_B32), 1748 SB.TmpVGPR) 1749 .addReg(SubReg, SubKillState) 1750 .addImm(i % PVD.PerVGPR) 1751 .addReg(SB.TmpVGPR, TmpVGPRFlags); 1752 TmpVGPRFlags = 0; 1753 1754 if (LIS) { 1755 if (i == 0) 1756 LIS->ReplaceMachineInstrInMaps(*MI, *WriteLane); 1757 else 1758 LIS->InsertMachineInstrInMaps(*WriteLane); 1759 } 1760 1761 // There could be undef components of a spilled super register. 1762 // TODO: Can we detect this and skip the spill? 1763 if (SB.NumSubRegs > 1) { 1764 // The last implicit use of the SB.SuperReg carries the "Kill" flag. 1765 unsigned SuperKillState = 0; 1766 if (i + 1 == SB.NumSubRegs) 1767 SuperKillState |= getKillRegState(SB.IsKill); 1768 WriteLane.addReg(SB.SuperReg, RegState::Implicit | SuperKillState); 1769 } 1770 } 1771 1772 // Write out VGPR 1773 SB.readWriteTmpVGPR(Offset, /*IsLoad*/ false); 1774 } 1775 1776 SB.restore(); 1777 } 1778 1779 MI->eraseFromParent(); 1780 SB.MFI.addToSpilledSGPRs(SB.NumSubRegs); 1781 1782 if (LIS) 1783 LIS->removeAllRegUnitsForPhysReg(SB.SuperReg); 1784 1785 return true; 1786 } 1787 1788 bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, 1789 int Index, 1790 RegScavenger *RS, 1791 LiveIntervals *LIS, 1792 bool OnlyToVGPR) const { 1793 SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, Index, RS); 1794 1795 ArrayRef<SpilledReg> VGPRSpills = SB.MFI.getSGPRToVGPRSpills(Index); 1796 bool SpillToVGPR = !VGPRSpills.empty(); 1797 if (OnlyToVGPR && !SpillToVGPR) 1798 return false; 1799 1800 if (SpillToVGPR) { 1801 for (unsigned i = 0, e = SB.NumSubRegs; i < e; ++i) { 1802 Register SubReg = 1803 SB.NumSubRegs == 1 1804 ? SB.SuperReg 1805 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1806 1807 SpilledReg Spill = VGPRSpills[i]; 1808 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_READLANE_B32), 1809 SubReg) 1810 .addReg(Spill.VGPR) 1811 .addImm(Spill.Lane); 1812 if (SB.NumSubRegs > 1 && i == 0) 1813 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1814 if (LIS) { 1815 if (i == e - 1) 1816 LIS->ReplaceMachineInstrInMaps(*MI, *MIB); 1817 else 1818 LIS->InsertMachineInstrInMaps(*MIB); 1819 } 1820 1821 } 1822 } else { 1823 SB.prepare(); 1824 1825 // Per VGPR helper data 1826 auto PVD = SB.getPerVGPRData(); 1827 1828 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1829 // Load in VGPR data 1830 SB.readWriteTmpVGPR(Offset, /*IsLoad*/ true); 1831 1832 // Unpack lanes 1833 for (unsigned i = Offset * PVD.PerVGPR, 1834 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1835 i < e; ++i) { 1836 Register SubReg = 1837 SB.NumSubRegs == 1 1838 ? SB.SuperReg 1839 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1840 1841 bool LastSubReg = (i + 1 == e); 1842 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, 1843 SB.TII.get(AMDGPU::V_READLANE_B32), SubReg) 1844 .addReg(SB.TmpVGPR, getKillRegState(LastSubReg)) 1845 .addImm(i); 1846 if (SB.NumSubRegs > 1 && i == 0) 1847 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1848 if (LIS) { 1849 if (i == e - 1) 1850 LIS->ReplaceMachineInstrInMaps(*MI, *MIB); 1851 else 1852 LIS->InsertMachineInstrInMaps(*MIB); 1853 } 1854 } 1855 } 1856 1857 SB.restore(); 1858 } 1859 1860 MI->eraseFromParent(); 1861 1862 if (LIS) 1863 LIS->removeAllRegUnitsForPhysReg(SB.SuperReg); 1864 1865 return true; 1866 } 1867 1868 bool SIRegisterInfo::spillEmergencySGPR(MachineBasicBlock::iterator MI, 1869 MachineBasicBlock &RestoreMBB, 1870 Register SGPR, RegScavenger *RS) const { 1871 SGPRSpillBuilder SB(*this, *ST.getInstrInfo(), isWave32, MI, SGPR, false, 0, 1872 RS); 1873 SB.prepare(); 1874 // Generate the spill of SGPR to SB.TmpVGPR. 1875 unsigned SubKillState = getKillRegState((SB.NumSubRegs == 1) && SB.IsKill); 1876 auto PVD = SB.getPerVGPRData(); 1877 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1878 unsigned TmpVGPRFlags = RegState::Undef; 1879 // Write sub registers into the VGPR 1880 for (unsigned i = Offset * PVD.PerVGPR, 1881 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1882 i < e; ++i) { 1883 Register SubReg = 1884 SB.NumSubRegs == 1 1885 ? SB.SuperReg 1886 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1887 1888 MachineInstrBuilder WriteLane = 1889 BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_WRITELANE_B32), 1890 SB.TmpVGPR) 1891 .addReg(SubReg, SubKillState) 1892 .addImm(i % PVD.PerVGPR) 1893 .addReg(SB.TmpVGPR, TmpVGPRFlags); 1894 TmpVGPRFlags = 0; 1895 // There could be undef components of a spilled super register. 1896 // TODO: Can we detect this and skip the spill? 1897 if (SB.NumSubRegs > 1) { 1898 // The last implicit use of the SB.SuperReg carries the "Kill" flag. 1899 unsigned SuperKillState = 0; 1900 if (i + 1 == SB.NumSubRegs) 1901 SuperKillState |= getKillRegState(SB.IsKill); 1902 WriteLane.addReg(SB.SuperReg, RegState::Implicit | SuperKillState); 1903 } 1904 } 1905 // Don't need to write VGPR out. 1906 } 1907 1908 // Restore clobbered registers in the specified restore block. 1909 MI = RestoreMBB.end(); 1910 SB.setMI(&RestoreMBB, MI); 1911 // Generate the restore of SGPR from SB.TmpVGPR. 1912 for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) { 1913 // Don't need to load VGPR in. 1914 // Unpack lanes 1915 for (unsigned i = Offset * PVD.PerVGPR, 1916 e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs); 1917 i < e; ++i) { 1918 Register SubReg = 1919 SB.NumSubRegs == 1 1920 ? SB.SuperReg 1921 : Register(getSubReg(SB.SuperReg, SB.SplitParts[i])); 1922 bool LastSubReg = (i + 1 == e); 1923 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_READLANE_B32), 1924 SubReg) 1925 .addReg(SB.TmpVGPR, getKillRegState(LastSubReg)) 1926 .addImm(i); 1927 if (SB.NumSubRegs > 1 && i == 0) 1928 MIB.addReg(SB.SuperReg, RegState::ImplicitDefine); 1929 } 1930 } 1931 SB.restore(); 1932 1933 SB.MFI.addToSpilledSGPRs(SB.NumSubRegs); 1934 return false; 1935 } 1936 1937 /// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to 1938 /// a VGPR and the stack slot can be safely eliminated when all other users are 1939 /// handled. 1940 bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex( 1941 MachineBasicBlock::iterator MI, 1942 int FI, 1943 RegScavenger *RS, 1944 LiveIntervals *LIS) const { 1945 switch (MI->getOpcode()) { 1946 case AMDGPU::SI_SPILL_S1024_SAVE: 1947 case AMDGPU::SI_SPILL_S512_SAVE: 1948 case AMDGPU::SI_SPILL_S256_SAVE: 1949 case AMDGPU::SI_SPILL_S224_SAVE: 1950 case AMDGPU::SI_SPILL_S192_SAVE: 1951 case AMDGPU::SI_SPILL_S160_SAVE: 1952 case AMDGPU::SI_SPILL_S128_SAVE: 1953 case AMDGPU::SI_SPILL_S96_SAVE: 1954 case AMDGPU::SI_SPILL_S64_SAVE: 1955 case AMDGPU::SI_SPILL_S32_SAVE: 1956 return spillSGPR(MI, FI, RS, LIS, true); 1957 case AMDGPU::SI_SPILL_S1024_RESTORE: 1958 case AMDGPU::SI_SPILL_S512_RESTORE: 1959 case AMDGPU::SI_SPILL_S256_RESTORE: 1960 case AMDGPU::SI_SPILL_S224_RESTORE: 1961 case AMDGPU::SI_SPILL_S192_RESTORE: 1962 case AMDGPU::SI_SPILL_S160_RESTORE: 1963 case AMDGPU::SI_SPILL_S128_RESTORE: 1964 case AMDGPU::SI_SPILL_S96_RESTORE: 1965 case AMDGPU::SI_SPILL_S64_RESTORE: 1966 case AMDGPU::SI_SPILL_S32_RESTORE: 1967 return restoreSGPR(MI, FI, RS, LIS, true); 1968 default: 1969 llvm_unreachable("not an SGPR spill instruction"); 1970 } 1971 } 1972 1973 void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, 1974 int SPAdj, unsigned FIOperandNum, 1975 RegScavenger *RS) const { 1976 MachineFunction *MF = MI->getParent()->getParent(); 1977 MachineBasicBlock *MBB = MI->getParent(); 1978 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1979 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1980 const SIInstrInfo *TII = ST.getInstrInfo(); 1981 DebugLoc DL = MI->getDebugLoc(); 1982 1983 assert(SPAdj == 0 && "unhandled SP adjustment in call sequence?"); 1984 1985 MachineOperand &FIOp = MI->getOperand(FIOperandNum); 1986 int Index = MI->getOperand(FIOperandNum).getIndex(); 1987 1988 Register FrameReg = FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(*MF) 1989 ? getBaseRegister() 1990 : getFrameRegister(*MF); 1991 1992 switch (MI->getOpcode()) { 1993 // SGPR register spill 1994 case AMDGPU::SI_SPILL_S1024_SAVE: 1995 case AMDGPU::SI_SPILL_S512_SAVE: 1996 case AMDGPU::SI_SPILL_S256_SAVE: 1997 case AMDGPU::SI_SPILL_S224_SAVE: 1998 case AMDGPU::SI_SPILL_S192_SAVE: 1999 case AMDGPU::SI_SPILL_S160_SAVE: 2000 case AMDGPU::SI_SPILL_S128_SAVE: 2001 case AMDGPU::SI_SPILL_S96_SAVE: 2002 case AMDGPU::SI_SPILL_S64_SAVE: 2003 case AMDGPU::SI_SPILL_S32_SAVE: { 2004 spillSGPR(MI, Index, RS); 2005 break; 2006 } 2007 2008 // SGPR register restore 2009 case AMDGPU::SI_SPILL_S1024_RESTORE: 2010 case AMDGPU::SI_SPILL_S512_RESTORE: 2011 case AMDGPU::SI_SPILL_S256_RESTORE: 2012 case AMDGPU::SI_SPILL_S224_RESTORE: 2013 case AMDGPU::SI_SPILL_S192_RESTORE: 2014 case AMDGPU::SI_SPILL_S160_RESTORE: 2015 case AMDGPU::SI_SPILL_S128_RESTORE: 2016 case AMDGPU::SI_SPILL_S96_RESTORE: 2017 case AMDGPU::SI_SPILL_S64_RESTORE: 2018 case AMDGPU::SI_SPILL_S32_RESTORE: { 2019 restoreSGPR(MI, Index, RS); 2020 break; 2021 } 2022 2023 // VGPR register spill 2024 case AMDGPU::SI_SPILL_V1024_SAVE: 2025 case AMDGPU::SI_SPILL_V512_SAVE: 2026 case AMDGPU::SI_SPILL_V256_SAVE: 2027 case AMDGPU::SI_SPILL_V224_SAVE: 2028 case AMDGPU::SI_SPILL_V192_SAVE: 2029 case AMDGPU::SI_SPILL_V160_SAVE: 2030 case AMDGPU::SI_SPILL_V128_SAVE: 2031 case AMDGPU::SI_SPILL_V96_SAVE: 2032 case AMDGPU::SI_SPILL_V64_SAVE: 2033 case AMDGPU::SI_SPILL_V32_SAVE: 2034 case AMDGPU::SI_SPILL_A1024_SAVE: 2035 case AMDGPU::SI_SPILL_A512_SAVE: 2036 case AMDGPU::SI_SPILL_A256_SAVE: 2037 case AMDGPU::SI_SPILL_A224_SAVE: 2038 case AMDGPU::SI_SPILL_A192_SAVE: 2039 case AMDGPU::SI_SPILL_A160_SAVE: 2040 case AMDGPU::SI_SPILL_A128_SAVE: 2041 case AMDGPU::SI_SPILL_A96_SAVE: 2042 case AMDGPU::SI_SPILL_A64_SAVE: 2043 case AMDGPU::SI_SPILL_A32_SAVE: 2044 case AMDGPU::SI_SPILL_AV1024_SAVE: 2045 case AMDGPU::SI_SPILL_AV512_SAVE: 2046 case AMDGPU::SI_SPILL_AV256_SAVE: 2047 case AMDGPU::SI_SPILL_AV224_SAVE: 2048 case AMDGPU::SI_SPILL_AV192_SAVE: 2049 case AMDGPU::SI_SPILL_AV160_SAVE: 2050 case AMDGPU::SI_SPILL_AV128_SAVE: 2051 case AMDGPU::SI_SPILL_AV96_SAVE: 2052 case AMDGPU::SI_SPILL_AV64_SAVE: 2053 case AMDGPU::SI_SPILL_AV32_SAVE: { 2054 const MachineOperand *VData = TII->getNamedOperand(*MI, 2055 AMDGPU::OpName::vdata); 2056 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 2057 MFI->getStackPtrOffsetReg()); 2058 2059 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR 2060 : AMDGPU::BUFFER_STORE_DWORD_OFFSET; 2061 auto *MBB = MI->getParent(); 2062 buildSpillLoadStore( 2063 *MBB, MI, DL, Opc, Index, VData->getReg(), VData->isKill(), FrameReg, 2064 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 2065 *MI->memoperands_begin(), RS); 2066 MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode())); 2067 MI->eraseFromParent(); 2068 break; 2069 } 2070 case AMDGPU::SI_SPILL_V32_RESTORE: 2071 case AMDGPU::SI_SPILL_V64_RESTORE: 2072 case AMDGPU::SI_SPILL_V96_RESTORE: 2073 case AMDGPU::SI_SPILL_V128_RESTORE: 2074 case AMDGPU::SI_SPILL_V160_RESTORE: 2075 case AMDGPU::SI_SPILL_V192_RESTORE: 2076 case AMDGPU::SI_SPILL_V224_RESTORE: 2077 case AMDGPU::SI_SPILL_V256_RESTORE: 2078 case AMDGPU::SI_SPILL_V512_RESTORE: 2079 case AMDGPU::SI_SPILL_V1024_RESTORE: 2080 case AMDGPU::SI_SPILL_A32_RESTORE: 2081 case AMDGPU::SI_SPILL_A64_RESTORE: 2082 case AMDGPU::SI_SPILL_A96_RESTORE: 2083 case AMDGPU::SI_SPILL_A128_RESTORE: 2084 case AMDGPU::SI_SPILL_A160_RESTORE: 2085 case AMDGPU::SI_SPILL_A192_RESTORE: 2086 case AMDGPU::SI_SPILL_A224_RESTORE: 2087 case AMDGPU::SI_SPILL_A256_RESTORE: 2088 case AMDGPU::SI_SPILL_A512_RESTORE: 2089 case AMDGPU::SI_SPILL_A1024_RESTORE: 2090 case AMDGPU::SI_SPILL_AV32_RESTORE: 2091 case AMDGPU::SI_SPILL_AV64_RESTORE: 2092 case AMDGPU::SI_SPILL_AV96_RESTORE: 2093 case AMDGPU::SI_SPILL_AV128_RESTORE: 2094 case AMDGPU::SI_SPILL_AV160_RESTORE: 2095 case AMDGPU::SI_SPILL_AV192_RESTORE: 2096 case AMDGPU::SI_SPILL_AV224_RESTORE: 2097 case AMDGPU::SI_SPILL_AV256_RESTORE: 2098 case AMDGPU::SI_SPILL_AV512_RESTORE: 2099 case AMDGPU::SI_SPILL_AV1024_RESTORE: { 2100 const MachineOperand *VData = TII->getNamedOperand(*MI, 2101 AMDGPU::OpName::vdata); 2102 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 2103 MFI->getStackPtrOffsetReg()); 2104 2105 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR 2106 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 2107 auto *MBB = MI->getParent(); 2108 buildSpillLoadStore( 2109 *MBB, MI, DL, Opc, Index, VData->getReg(), VData->isKill(), FrameReg, 2110 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 2111 *MI->memoperands_begin(), RS); 2112 MI->eraseFromParent(); 2113 break; 2114 } 2115 2116 default: { 2117 // Other access to frame index 2118 const DebugLoc &DL = MI->getDebugLoc(); 2119 2120 int64_t Offset = FrameInfo.getObjectOffset(Index); 2121 if (ST.enableFlatScratch()) { 2122 if (TII->isFLATScratch(*MI)) { 2123 assert((int16_t)FIOperandNum == 2124 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 2125 AMDGPU::OpName::saddr)); 2126 2127 // The offset is always swizzled, just replace it 2128 if (FrameReg) 2129 FIOp.ChangeToRegister(FrameReg, false); 2130 2131 if (!Offset) 2132 return; 2133 2134 MachineOperand *OffsetOp = 2135 TII->getNamedOperand(*MI, AMDGPU::OpName::offset); 2136 int64_t NewOffset = Offset + OffsetOp->getImm(); 2137 if (TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS, 2138 SIInstrFlags::FlatScratch)) { 2139 OffsetOp->setImm(NewOffset); 2140 if (FrameReg) 2141 return; 2142 Offset = 0; 2143 } 2144 2145 if (!Offset) { 2146 unsigned Opc = MI->getOpcode(); 2147 int NewOpc = -1; 2148 if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr) != -1) { 2149 NewOpc = AMDGPU::getFlatScratchInstSVfromSVS(Opc); 2150 } else if (ST.hasFlatScratchSTMode()) { 2151 // On GFX10 we have ST mode to use no registers for an address. 2152 // Otherwise we need to materialize 0 into an SGPR. 2153 NewOpc = AMDGPU::getFlatScratchInstSTfromSS(Opc); 2154 } 2155 2156 if (NewOpc != -1) { 2157 MI->removeOperand( 2158 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr)); 2159 MI->setDesc(TII->get(NewOpc)); 2160 return; 2161 } 2162 } 2163 } 2164 2165 if (!FrameReg) { 2166 FIOp.ChangeToImmediate(Offset); 2167 if (TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) 2168 return; 2169 } 2170 2171 // We need to use register here. Check if we can use an SGPR or need 2172 // a VGPR. 2173 FIOp.ChangeToRegister(AMDGPU::M0, false); 2174 bool UseSGPR = TII->isOperandLegal(*MI, FIOperandNum, &FIOp); 2175 2176 if (!Offset && FrameReg && UseSGPR) { 2177 FIOp.setReg(FrameReg); 2178 return; 2179 } 2180 2181 const TargetRegisterClass *RC = UseSGPR ? &AMDGPU::SReg_32_XM0RegClass 2182 : &AMDGPU::VGPR_32RegClass; 2183 2184 Register TmpReg = RS->scavengeRegister(RC, MI, 0, !UseSGPR); 2185 FIOp.setReg(TmpReg); 2186 FIOp.setIsKill(true); 2187 2188 if ((!FrameReg || !Offset) && TmpReg) { 2189 unsigned Opc = UseSGPR ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 2190 auto MIB = BuildMI(*MBB, MI, DL, TII->get(Opc), TmpReg); 2191 if (FrameReg) 2192 MIB.addReg(FrameReg); 2193 else 2194 MIB.addImm(Offset); 2195 2196 return; 2197 } 2198 2199 Register TmpSReg = 2200 UseSGPR ? TmpReg 2201 : RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0, 2202 !UseSGPR); 2203 2204 // TODO: for flat scratch another attempt can be made with a VGPR index 2205 // if no SGPRs can be scavenged. 2206 if ((!TmpSReg && !FrameReg) || (!TmpReg && !UseSGPR)) 2207 report_fatal_error("Cannot scavenge register in FI elimination!"); 2208 2209 if (!TmpSReg) { 2210 // Use frame register and restore it after. 2211 TmpSReg = FrameReg; 2212 FIOp.setReg(FrameReg); 2213 FIOp.setIsKill(false); 2214 } 2215 2216 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), TmpSReg) 2217 .addReg(FrameReg) 2218 .addImm(Offset); 2219 2220 if (!UseSGPR) 2221 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 2222 .addReg(TmpSReg, RegState::Kill); 2223 2224 if (TmpSReg == FrameReg) { 2225 // Undo frame register modification. 2226 BuildMI(*MBB, std::next(MI), DL, TII->get(AMDGPU::S_ADD_I32), 2227 FrameReg) 2228 .addReg(FrameReg) 2229 .addImm(-Offset); 2230 } 2231 2232 return; 2233 } 2234 2235 bool IsMUBUF = TII->isMUBUF(*MI); 2236 2237 if (!IsMUBUF && !MFI->isEntryFunction()) { 2238 // Convert to a swizzled stack address by scaling by the wave size. 2239 // In an entry function/kernel the offset is already swizzled. 2240 bool IsSALU = isSGPRClass(TII->getOpRegClass(*MI, FIOperandNum)); 2241 bool LiveSCC = RS->isRegUsed(AMDGPU::SCC); 2242 const TargetRegisterClass *RC = IsSALU && !LiveSCC 2243 ? &AMDGPU::SReg_32RegClass 2244 : &AMDGPU::VGPR_32RegClass; 2245 bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32 || 2246 MI->getOpcode() == AMDGPU::V_MOV_B32_e64; 2247 Register ResultReg = IsCopy ? MI->getOperand(0).getReg() 2248 : RS->scavengeRegister(RC, MI, 0); 2249 2250 int64_t Offset = FrameInfo.getObjectOffset(Index); 2251 if (Offset == 0) { 2252 unsigned OpCode = IsSALU && !LiveSCC ? AMDGPU::S_LSHR_B32 2253 : AMDGPU::V_LSHRREV_B32_e64; 2254 // XXX - This never happens because of emergency scavenging slot at 0? 2255 auto Shift = BuildMI(*MBB, MI, DL, TII->get(OpCode), ResultReg) 2256 .addImm(ST.getWavefrontSizeLog2()) 2257 .addReg(FrameReg); 2258 if (IsSALU && !LiveSCC) 2259 Shift.getInstr()->getOperand(3).setIsDead( 2260 true); // Mark SCC as dead. 2261 if (IsSALU && LiveSCC) { 2262 Register NewDest = 2263 RS->scavengeRegister(&AMDGPU::SReg_32RegClass, Shift, 0); 2264 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), 2265 NewDest) 2266 .addReg(ResultReg); 2267 ResultReg = NewDest; 2268 } 2269 } else { 2270 MachineInstrBuilder MIB; 2271 if (!IsSALU) { 2272 if ((MIB = TII->getAddNoCarry(*MBB, MI, DL, ResultReg, *RS)) != 2273 nullptr) { 2274 // Reuse ResultReg in intermediate step. 2275 Register ScaledReg = ResultReg; 2276 2277 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), 2278 ScaledReg) 2279 .addImm(ST.getWavefrontSizeLog2()) 2280 .addReg(FrameReg); 2281 2282 const bool IsVOP2 = MIB->getOpcode() == AMDGPU::V_ADD_U32_e32; 2283 2284 // TODO: Fold if use instruction is another add of a constant. 2285 if (IsVOP2 || AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) { 2286 // FIXME: This can fail 2287 MIB.addImm(Offset); 2288 MIB.addReg(ScaledReg, RegState::Kill); 2289 if (!IsVOP2) 2290 MIB.addImm(0); // clamp bit 2291 } else { 2292 assert(MIB->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 && 2293 "Need to reuse carry out register"); 2294 2295 // Use scavenged unused carry out as offset register. 2296 Register ConstOffsetReg; 2297 if (!isWave32) 2298 ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0); 2299 else 2300 ConstOffsetReg = MIB.getReg(1); 2301 2302 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg) 2303 .addImm(Offset); 2304 MIB.addReg(ConstOffsetReg, RegState::Kill); 2305 MIB.addReg(ScaledReg, RegState::Kill); 2306 MIB.addImm(0); // clamp bit 2307 } 2308 } 2309 } 2310 if (!MIB || IsSALU) { 2311 // We have to produce a carry out, and there isn't a free SGPR pair 2312 // for it. We can keep the whole computation on the SALU to avoid 2313 // clobbering an additional register at the cost of an extra mov. 2314 2315 // We may have 1 free scratch SGPR even though a carry out is 2316 // unavailable. Only one additional mov is needed. 2317 Register TmpScaledReg = 2318 RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0, false); 2319 Register ScaledReg = TmpScaledReg.isValid() ? TmpScaledReg : FrameReg; 2320 2321 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHR_B32), ScaledReg) 2322 .addReg(FrameReg) 2323 .addImm(ST.getWavefrontSizeLog2()); 2324 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), ScaledReg) 2325 .addReg(ScaledReg, RegState::Kill) 2326 .addImm(Offset); 2327 if (!IsSALU) 2328 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), ResultReg) 2329 .addReg(ScaledReg, RegState::Kill); 2330 else 2331 ResultReg = ScaledReg; 2332 2333 // If there were truly no free SGPRs, we need to undo everything. 2334 if (!TmpScaledReg.isValid()) { 2335 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), ScaledReg) 2336 .addReg(ScaledReg, RegState::Kill) 2337 .addImm(-Offset); 2338 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg) 2339 .addReg(FrameReg) 2340 .addImm(ST.getWavefrontSizeLog2()); 2341 } 2342 } 2343 } 2344 2345 // Don't introduce an extra copy if we're just materializing in a mov. 2346 if (IsCopy) 2347 MI->eraseFromParent(); 2348 else 2349 FIOp.ChangeToRegister(ResultReg, false, false, true); 2350 return; 2351 } 2352 2353 if (IsMUBUF) { 2354 // Disable offen so we don't need a 0 vgpr base. 2355 assert(static_cast<int>(FIOperandNum) == 2356 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 2357 AMDGPU::OpName::vaddr)); 2358 2359 auto &SOffset = *TII->getNamedOperand(*MI, AMDGPU::OpName::soffset); 2360 assert((SOffset.isImm() && SOffset.getImm() == 0)); 2361 2362 if (FrameReg != AMDGPU::NoRegister) 2363 SOffset.ChangeToRegister(FrameReg, false); 2364 2365 int64_t Offset = FrameInfo.getObjectOffset(Index); 2366 int64_t OldImm 2367 = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(); 2368 int64_t NewOffset = OldImm + Offset; 2369 2370 if (SIInstrInfo::isLegalMUBUFImmOffset(NewOffset) && 2371 buildMUBUFOffsetLoadStore(ST, FrameInfo, MI, Index, NewOffset)) { 2372 MI->eraseFromParent(); 2373 return; 2374 } 2375 } 2376 2377 // If the offset is simply too big, don't convert to a scratch wave offset 2378 // relative index. 2379 2380 FIOp.ChangeToImmediate(Offset); 2381 if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) { 2382 Register TmpReg = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 2383 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 2384 .addImm(Offset); 2385 FIOp.ChangeToRegister(TmpReg, false, false, true); 2386 } 2387 } 2388 } 2389 } 2390 2391 StringRef SIRegisterInfo::getRegAsmName(MCRegister Reg) const { 2392 return AMDGPUInstPrinter::getRegisterName(Reg); 2393 } 2394 2395 static const TargetRegisterClass * 2396 getAnyVGPRClassForBitWidth(unsigned BitWidth) { 2397 if (BitWidth <= 64) 2398 return &AMDGPU::VReg_64RegClass; 2399 if (BitWidth <= 96) 2400 return &AMDGPU::VReg_96RegClass; 2401 if (BitWidth <= 128) 2402 return &AMDGPU::VReg_128RegClass; 2403 if (BitWidth <= 160) 2404 return &AMDGPU::VReg_160RegClass; 2405 if (BitWidth <= 192) 2406 return &AMDGPU::VReg_192RegClass; 2407 if (BitWidth <= 224) 2408 return &AMDGPU::VReg_224RegClass; 2409 if (BitWidth <= 256) 2410 return &AMDGPU::VReg_256RegClass; 2411 if (BitWidth <= 512) 2412 return &AMDGPU::VReg_512RegClass; 2413 if (BitWidth <= 1024) 2414 return &AMDGPU::VReg_1024RegClass; 2415 2416 return nullptr; 2417 } 2418 2419 static const TargetRegisterClass * 2420 getAlignedVGPRClassForBitWidth(unsigned BitWidth) { 2421 if (BitWidth <= 64) 2422 return &AMDGPU::VReg_64_Align2RegClass; 2423 if (BitWidth <= 96) 2424 return &AMDGPU::VReg_96_Align2RegClass; 2425 if (BitWidth <= 128) 2426 return &AMDGPU::VReg_128_Align2RegClass; 2427 if (BitWidth <= 160) 2428 return &AMDGPU::VReg_160_Align2RegClass; 2429 if (BitWidth <= 192) 2430 return &AMDGPU::VReg_192_Align2RegClass; 2431 if (BitWidth <= 224) 2432 return &AMDGPU::VReg_224_Align2RegClass; 2433 if (BitWidth <= 256) 2434 return &AMDGPU::VReg_256_Align2RegClass; 2435 if (BitWidth <= 512) 2436 return &AMDGPU::VReg_512_Align2RegClass; 2437 if (BitWidth <= 1024) 2438 return &AMDGPU::VReg_1024_Align2RegClass; 2439 2440 return nullptr; 2441 } 2442 2443 const TargetRegisterClass * 2444 SIRegisterInfo::getVGPRClassForBitWidth(unsigned BitWidth) const { 2445 if (BitWidth == 1) 2446 return &AMDGPU::VReg_1RegClass; 2447 if (BitWidth <= 16) 2448 return &AMDGPU::VGPR_LO16RegClass; 2449 if (BitWidth <= 32) 2450 return &AMDGPU::VGPR_32RegClass; 2451 return ST.needsAlignedVGPRs() ? getAlignedVGPRClassForBitWidth(BitWidth) 2452 : getAnyVGPRClassForBitWidth(BitWidth); 2453 } 2454 2455 static const TargetRegisterClass * 2456 getAnyAGPRClassForBitWidth(unsigned BitWidth) { 2457 if (BitWidth <= 64) 2458 return &AMDGPU::AReg_64RegClass; 2459 if (BitWidth <= 96) 2460 return &AMDGPU::AReg_96RegClass; 2461 if (BitWidth <= 128) 2462 return &AMDGPU::AReg_128RegClass; 2463 if (BitWidth <= 160) 2464 return &AMDGPU::AReg_160RegClass; 2465 if (BitWidth <= 192) 2466 return &AMDGPU::AReg_192RegClass; 2467 if (BitWidth <= 224) 2468 return &AMDGPU::AReg_224RegClass; 2469 if (BitWidth <= 256) 2470 return &AMDGPU::AReg_256RegClass; 2471 if (BitWidth <= 512) 2472 return &AMDGPU::AReg_512RegClass; 2473 if (BitWidth <= 1024) 2474 return &AMDGPU::AReg_1024RegClass; 2475 2476 return nullptr; 2477 } 2478 2479 static const TargetRegisterClass * 2480 getAlignedAGPRClassForBitWidth(unsigned BitWidth) { 2481 if (BitWidth <= 64) 2482 return &AMDGPU::AReg_64_Align2RegClass; 2483 if (BitWidth <= 96) 2484 return &AMDGPU::AReg_96_Align2RegClass; 2485 if (BitWidth <= 128) 2486 return &AMDGPU::AReg_128_Align2RegClass; 2487 if (BitWidth <= 160) 2488 return &AMDGPU::AReg_160_Align2RegClass; 2489 if (BitWidth <= 192) 2490 return &AMDGPU::AReg_192_Align2RegClass; 2491 if (BitWidth <= 224) 2492 return &AMDGPU::AReg_224_Align2RegClass; 2493 if (BitWidth <= 256) 2494 return &AMDGPU::AReg_256_Align2RegClass; 2495 if (BitWidth <= 512) 2496 return &AMDGPU::AReg_512_Align2RegClass; 2497 if (BitWidth <= 1024) 2498 return &AMDGPU::AReg_1024_Align2RegClass; 2499 2500 return nullptr; 2501 } 2502 2503 const TargetRegisterClass * 2504 SIRegisterInfo::getAGPRClassForBitWidth(unsigned BitWidth) const { 2505 if (BitWidth <= 16) 2506 return &AMDGPU::AGPR_LO16RegClass; 2507 if (BitWidth <= 32) 2508 return &AMDGPU::AGPR_32RegClass; 2509 return ST.needsAlignedVGPRs() ? getAlignedAGPRClassForBitWidth(BitWidth) 2510 : getAnyAGPRClassForBitWidth(BitWidth); 2511 } 2512 2513 static const TargetRegisterClass * 2514 getAnyVectorSuperClassForBitWidth(unsigned BitWidth) { 2515 if (BitWidth <= 64) 2516 return &AMDGPU::AV_64RegClass; 2517 if (BitWidth <= 96) 2518 return &AMDGPU::AV_96RegClass; 2519 if (BitWidth <= 128) 2520 return &AMDGPU::AV_128RegClass; 2521 if (BitWidth <= 160) 2522 return &AMDGPU::AV_160RegClass; 2523 if (BitWidth <= 192) 2524 return &AMDGPU::AV_192RegClass; 2525 if (BitWidth <= 224) 2526 return &AMDGPU::AV_224RegClass; 2527 if (BitWidth <= 256) 2528 return &AMDGPU::AV_256RegClass; 2529 if (BitWidth <= 512) 2530 return &AMDGPU::AV_512RegClass; 2531 if (BitWidth <= 1024) 2532 return &AMDGPU::AV_1024RegClass; 2533 2534 return nullptr; 2535 } 2536 2537 static const TargetRegisterClass * 2538 getAlignedVectorSuperClassForBitWidth(unsigned BitWidth) { 2539 if (BitWidth <= 64) 2540 return &AMDGPU::AV_64_Align2RegClass; 2541 if (BitWidth <= 96) 2542 return &AMDGPU::AV_96_Align2RegClass; 2543 if (BitWidth <= 128) 2544 return &AMDGPU::AV_128_Align2RegClass; 2545 if (BitWidth <= 160) 2546 return &AMDGPU::AV_160_Align2RegClass; 2547 if (BitWidth <= 192) 2548 return &AMDGPU::AV_192_Align2RegClass; 2549 if (BitWidth <= 224) 2550 return &AMDGPU::AV_224_Align2RegClass; 2551 if (BitWidth <= 256) 2552 return &AMDGPU::AV_256_Align2RegClass; 2553 if (BitWidth <= 512) 2554 return &AMDGPU::AV_512_Align2RegClass; 2555 if (BitWidth <= 1024) 2556 return &AMDGPU::AV_1024_Align2RegClass; 2557 2558 return nullptr; 2559 } 2560 2561 const TargetRegisterClass * 2562 SIRegisterInfo::getVectorSuperClassForBitWidth(unsigned BitWidth) const { 2563 if (BitWidth <= 16) 2564 return &AMDGPU::VGPR_LO16RegClass; 2565 if (BitWidth <= 32) 2566 return &AMDGPU::AV_32RegClass; 2567 return ST.needsAlignedVGPRs() 2568 ? getAlignedVectorSuperClassForBitWidth(BitWidth) 2569 : getAnyVectorSuperClassForBitWidth(BitWidth); 2570 } 2571 2572 const TargetRegisterClass * 2573 SIRegisterInfo::getSGPRClassForBitWidth(unsigned BitWidth) { 2574 if (BitWidth <= 16) 2575 return &AMDGPU::SGPR_LO16RegClass; 2576 if (BitWidth <= 32) 2577 return &AMDGPU::SReg_32RegClass; 2578 if (BitWidth <= 64) 2579 return &AMDGPU::SReg_64RegClass; 2580 if (BitWidth <= 96) 2581 return &AMDGPU::SGPR_96RegClass; 2582 if (BitWidth <= 128) 2583 return &AMDGPU::SGPR_128RegClass; 2584 if (BitWidth <= 160) 2585 return &AMDGPU::SGPR_160RegClass; 2586 if (BitWidth <= 192) 2587 return &AMDGPU::SGPR_192RegClass; 2588 if (BitWidth <= 224) 2589 return &AMDGPU::SGPR_224RegClass; 2590 if (BitWidth <= 256) 2591 return &AMDGPU::SGPR_256RegClass; 2592 if (BitWidth <= 512) 2593 return &AMDGPU::SGPR_512RegClass; 2594 if (BitWidth <= 1024) 2595 return &AMDGPU::SGPR_1024RegClass; 2596 2597 return nullptr; 2598 } 2599 2600 // FIXME: This is very slow. It might be worth creating a map from physreg to 2601 // register class. 2602 const TargetRegisterClass * 2603 SIRegisterInfo::getPhysRegClass(MCRegister Reg) const { 2604 static const TargetRegisterClass *const BaseClasses[] = { 2605 &AMDGPU::VGPR_LO16RegClass, 2606 &AMDGPU::VGPR_HI16RegClass, 2607 &AMDGPU::SReg_LO16RegClass, 2608 &AMDGPU::AGPR_LO16RegClass, 2609 &AMDGPU::VGPR_32RegClass, 2610 &AMDGPU::SReg_32RegClass, 2611 &AMDGPU::AGPR_32RegClass, 2612 &AMDGPU::AGPR_32RegClass, 2613 &AMDGPU::VReg_64_Align2RegClass, 2614 &AMDGPU::VReg_64RegClass, 2615 &AMDGPU::SReg_64RegClass, 2616 &AMDGPU::AReg_64_Align2RegClass, 2617 &AMDGPU::AReg_64RegClass, 2618 &AMDGPU::VReg_96_Align2RegClass, 2619 &AMDGPU::VReg_96RegClass, 2620 &AMDGPU::SReg_96RegClass, 2621 &AMDGPU::AReg_96_Align2RegClass, 2622 &AMDGPU::AReg_96RegClass, 2623 &AMDGPU::VReg_128_Align2RegClass, 2624 &AMDGPU::VReg_128RegClass, 2625 &AMDGPU::SReg_128RegClass, 2626 &AMDGPU::AReg_128_Align2RegClass, 2627 &AMDGPU::AReg_128RegClass, 2628 &AMDGPU::VReg_160_Align2RegClass, 2629 &AMDGPU::VReg_160RegClass, 2630 &AMDGPU::SReg_160RegClass, 2631 &AMDGPU::AReg_160_Align2RegClass, 2632 &AMDGPU::AReg_160RegClass, 2633 &AMDGPU::VReg_192_Align2RegClass, 2634 &AMDGPU::VReg_192RegClass, 2635 &AMDGPU::SReg_192RegClass, 2636 &AMDGPU::AReg_192_Align2RegClass, 2637 &AMDGPU::AReg_192RegClass, 2638 &AMDGPU::VReg_224_Align2RegClass, 2639 &AMDGPU::VReg_224RegClass, 2640 &AMDGPU::SReg_224RegClass, 2641 &AMDGPU::AReg_224_Align2RegClass, 2642 &AMDGPU::AReg_224RegClass, 2643 &AMDGPU::VReg_256_Align2RegClass, 2644 &AMDGPU::VReg_256RegClass, 2645 &AMDGPU::SReg_256RegClass, 2646 &AMDGPU::AReg_256_Align2RegClass, 2647 &AMDGPU::AReg_256RegClass, 2648 &AMDGPU::VReg_512_Align2RegClass, 2649 &AMDGPU::VReg_512RegClass, 2650 &AMDGPU::SReg_512RegClass, 2651 &AMDGPU::AReg_512_Align2RegClass, 2652 &AMDGPU::AReg_512RegClass, 2653 &AMDGPU::SReg_1024RegClass, 2654 &AMDGPU::VReg_1024_Align2RegClass, 2655 &AMDGPU::VReg_1024RegClass, 2656 &AMDGPU::AReg_1024_Align2RegClass, 2657 &AMDGPU::AReg_1024RegClass, 2658 &AMDGPU::SCC_CLASSRegClass, 2659 &AMDGPU::Pseudo_SReg_32RegClass, 2660 &AMDGPU::Pseudo_SReg_128RegClass, 2661 }; 2662 2663 for (const TargetRegisterClass *BaseClass : BaseClasses) { 2664 if (BaseClass->contains(Reg)) { 2665 return BaseClass; 2666 } 2667 } 2668 return nullptr; 2669 } 2670 2671 bool SIRegisterInfo::isSGPRReg(const MachineRegisterInfo &MRI, 2672 Register Reg) const { 2673 const TargetRegisterClass *RC; 2674 if (Reg.isVirtual()) 2675 RC = MRI.getRegClass(Reg); 2676 else 2677 RC = getPhysRegClass(Reg); 2678 return isSGPRClass(RC); 2679 } 2680 2681 const TargetRegisterClass * 2682 SIRegisterInfo::getEquivalentVGPRClass(const TargetRegisterClass *SRC) const { 2683 unsigned Size = getRegSizeInBits(*SRC); 2684 const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size); 2685 assert(VRC && "Invalid register class size"); 2686 return VRC; 2687 } 2688 2689 const TargetRegisterClass * 2690 SIRegisterInfo::getEquivalentAGPRClass(const TargetRegisterClass *SRC) const { 2691 unsigned Size = getRegSizeInBits(*SRC); 2692 const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size); 2693 assert(ARC && "Invalid register class size"); 2694 return ARC; 2695 } 2696 2697 const TargetRegisterClass * 2698 SIRegisterInfo::getEquivalentSGPRClass(const TargetRegisterClass *VRC) const { 2699 unsigned Size = getRegSizeInBits(*VRC); 2700 if (Size == 32) 2701 return &AMDGPU::SGPR_32RegClass; 2702 const TargetRegisterClass *SRC = getSGPRClassForBitWidth(Size); 2703 assert(SRC && "Invalid register class size"); 2704 return SRC; 2705 } 2706 2707 const TargetRegisterClass *SIRegisterInfo::getSubRegClass( 2708 const TargetRegisterClass *RC, unsigned SubIdx) const { 2709 if (SubIdx == AMDGPU::NoSubRegister) 2710 return RC; 2711 2712 // We can assume that each lane corresponds to one 32-bit register. 2713 unsigned Size = getNumChannelsFromSubReg(SubIdx) * 32; 2714 if (isAGPRClass(RC)) { 2715 RC = getAGPRClassForBitWidth(Size); 2716 } else if (isVGPRClass(RC)) { 2717 RC = getVGPRClassForBitWidth(Size); 2718 } else if (isVectorSuperClass(RC)) { 2719 RC = getVectorSuperClassForBitWidth(Size); 2720 } else { 2721 RC = getSGPRClassForBitWidth(Size); 2722 } 2723 assert(RC && "Invalid sub-register class size"); 2724 return RC; 2725 } 2726 2727 const TargetRegisterClass * 2728 SIRegisterInfo::getCompatibleSubRegClass(const TargetRegisterClass *SuperRC, 2729 const TargetRegisterClass *SubRC, 2730 unsigned SubIdx) const { 2731 // Ensure this subregister index is aligned in the super register. 2732 const TargetRegisterClass *MatchRC = 2733 getMatchingSuperRegClass(SuperRC, SubRC, SubIdx); 2734 return MatchRC && MatchRC->hasSubClassEq(SuperRC) ? MatchRC : nullptr; 2735 } 2736 2737 bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const { 2738 if (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST && 2739 OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST) 2740 return !ST.hasMFMAInlineLiteralBug(); 2741 2742 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 2743 OpType <= AMDGPU::OPERAND_SRC_LAST; 2744 } 2745 2746 bool SIRegisterInfo::shouldRewriteCopySrc( 2747 const TargetRegisterClass *DefRC, 2748 unsigned DefSubReg, 2749 const TargetRegisterClass *SrcRC, 2750 unsigned SrcSubReg) const { 2751 // We want to prefer the smallest register class possible, so we don't want to 2752 // stop and rewrite on anything that looks like a subregister 2753 // extract. Operations mostly don't care about the super register class, so we 2754 // only want to stop on the most basic of copies between the same register 2755 // class. 2756 // 2757 // e.g. if we have something like 2758 // %0 = ... 2759 // %1 = ... 2760 // %2 = REG_SEQUENCE %0, sub0, %1, sub1, %2, sub2 2761 // %3 = COPY %2, sub0 2762 // 2763 // We want to look through the COPY to find: 2764 // => %3 = COPY %0 2765 2766 // Plain copy. 2767 return getCommonSubClass(DefRC, SrcRC) != nullptr; 2768 } 2769 2770 bool SIRegisterInfo::opCanUseLiteralConstant(unsigned OpType) const { 2771 // TODO: 64-bit operands have extending behavior from 32-bit literal. 2772 return OpType >= AMDGPU::OPERAND_REG_IMM_FIRST && 2773 OpType <= AMDGPU::OPERAND_REG_IMM_LAST; 2774 } 2775 2776 /// Returns a lowest register that is not used at any point in the function. 2777 /// If all registers are used, then this function will return 2778 /// AMDGPU::NoRegister. If \p ReserveHighestVGPR = true, then return 2779 /// highest unused register. 2780 MCRegister SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI, 2781 const TargetRegisterClass *RC, 2782 const MachineFunction &MF, 2783 bool ReserveHighestVGPR) const { 2784 if (ReserveHighestVGPR) { 2785 for (MCRegister Reg : reverse(*RC)) 2786 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 2787 return Reg; 2788 } else { 2789 for (MCRegister Reg : *RC) 2790 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 2791 return Reg; 2792 } 2793 return MCRegister(); 2794 } 2795 2796 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC, 2797 unsigned EltSize) const { 2798 const unsigned RegBitWidth = AMDGPU::getRegBitWidth(*RC->MC); 2799 assert(RegBitWidth >= 32 && RegBitWidth <= 1024); 2800 2801 const unsigned RegDWORDs = RegBitWidth / 32; 2802 const unsigned EltDWORDs = EltSize / 4; 2803 assert(RegSplitParts.size() + 1 >= EltDWORDs); 2804 2805 const std::vector<int16_t> &Parts = RegSplitParts[EltDWORDs - 1]; 2806 const unsigned NumParts = RegDWORDs / EltDWORDs; 2807 2808 return makeArrayRef(Parts.data(), NumParts); 2809 } 2810 2811 const TargetRegisterClass* 2812 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI, 2813 Register Reg) const { 2814 return Reg.isVirtual() ? MRI.getRegClass(Reg) : getPhysRegClass(Reg); 2815 } 2816 2817 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI, 2818 Register Reg) const { 2819 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 2820 // Registers without classes are unaddressable, SGPR-like registers. 2821 return RC && isVGPRClass(RC); 2822 } 2823 2824 bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI, 2825 Register Reg) const { 2826 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 2827 2828 // Registers without classes are unaddressable, SGPR-like registers. 2829 return RC && isAGPRClass(RC); 2830 } 2831 2832 bool SIRegisterInfo::shouldCoalesce(MachineInstr *MI, 2833 const TargetRegisterClass *SrcRC, 2834 unsigned SubReg, 2835 const TargetRegisterClass *DstRC, 2836 unsigned DstSubReg, 2837 const TargetRegisterClass *NewRC, 2838 LiveIntervals &LIS) const { 2839 unsigned SrcSize = getRegSizeInBits(*SrcRC); 2840 unsigned DstSize = getRegSizeInBits(*DstRC); 2841 unsigned NewSize = getRegSizeInBits(*NewRC); 2842 2843 // Do not increase size of registers beyond dword, we would need to allocate 2844 // adjacent registers and constraint regalloc more than needed. 2845 2846 // Always allow dword coalescing. 2847 if (SrcSize <= 32 || DstSize <= 32) 2848 return true; 2849 2850 return NewSize <= DstSize || NewSize <= SrcSize; 2851 } 2852 2853 unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, 2854 MachineFunction &MF) const { 2855 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 2856 2857 unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(), 2858 MF.getFunction()); 2859 switch (RC->getID()) { 2860 default: 2861 return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF); 2862 case AMDGPU::VGPR_32RegClassID: 2863 case AMDGPU::VGPR_LO16RegClassID: 2864 case AMDGPU::VGPR_HI16RegClassID: 2865 return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF)); 2866 case AMDGPU::SGPR_32RegClassID: 2867 case AMDGPU::SGPR_LO16RegClassID: 2868 return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF)); 2869 } 2870 } 2871 2872 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF, 2873 unsigned Idx) const { 2874 if (Idx == AMDGPU::RegisterPressureSets::VGPR_32 || 2875 Idx == AMDGPU::RegisterPressureSets::AGPR_32) 2876 return getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 2877 const_cast<MachineFunction &>(MF)); 2878 2879 if (Idx == AMDGPU::RegisterPressureSets::SReg_32) 2880 return getRegPressureLimit(&AMDGPU::SGPR_32RegClass, 2881 const_cast<MachineFunction &>(MF)); 2882 2883 llvm_unreachable("Unexpected register pressure set!"); 2884 } 2885 2886 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const { 2887 static const int Empty[] = { -1 }; 2888 2889 if (RegPressureIgnoredUnits[RegUnit]) 2890 return Empty; 2891 2892 return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit); 2893 } 2894 2895 MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const { 2896 // Not a callee saved register. 2897 return AMDGPU::SGPR30_SGPR31; 2898 } 2899 2900 const TargetRegisterClass * 2901 SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size, 2902 const RegisterBank &RB) const { 2903 switch (RB.getID()) { 2904 case AMDGPU::VGPRRegBankID: 2905 return getVGPRClassForBitWidth(std::max(32u, Size)); 2906 case AMDGPU::VCCRegBankID: 2907 assert(Size == 1); 2908 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 2909 : &AMDGPU::SReg_64_XEXECRegClass; 2910 case AMDGPU::SGPRRegBankID: 2911 return getSGPRClassForBitWidth(std::max(32u, Size)); 2912 case AMDGPU::AGPRRegBankID: 2913 return getAGPRClassForBitWidth(std::max(32u, Size)); 2914 default: 2915 llvm_unreachable("unknown register bank"); 2916 } 2917 } 2918 2919 const TargetRegisterClass * 2920 SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO, 2921 const MachineRegisterInfo &MRI) const { 2922 const RegClassOrRegBank &RCOrRB = MRI.getRegClassOrRegBank(MO.getReg()); 2923 if (const RegisterBank *RB = RCOrRB.dyn_cast<const RegisterBank*>()) 2924 return getRegClassForTypeOnBank(MRI.getType(MO.getReg()), *RB); 2925 2926 if (const auto *RC = RCOrRB.dyn_cast<const TargetRegisterClass *>()) 2927 return getAllocatableClass(RC); 2928 2929 return nullptr; 2930 } 2931 2932 MCRegister SIRegisterInfo::getVCC() const { 2933 return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; 2934 } 2935 2936 MCRegister SIRegisterInfo::getExec() const { 2937 return isWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 2938 } 2939 2940 const TargetRegisterClass *SIRegisterInfo::getVGPR64Class() const { 2941 // VGPR tuples have an alignment requirement on gfx90a variants. 2942 return ST.needsAlignedVGPRs() ? &AMDGPU::VReg_64_Align2RegClass 2943 : &AMDGPU::VReg_64RegClass; 2944 } 2945 2946 const TargetRegisterClass * 2947 SIRegisterInfo::getRegClass(unsigned RCID) const { 2948 switch ((int)RCID) { 2949 case AMDGPU::SReg_1RegClassID: 2950 return getBoolRC(); 2951 case AMDGPU::SReg_1_XEXECRegClassID: 2952 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 2953 : &AMDGPU::SReg_64_XEXECRegClass; 2954 case -1: 2955 return nullptr; 2956 default: 2957 return AMDGPUGenRegisterInfo::getRegClass(RCID); 2958 } 2959 } 2960 2961 // Find reaching register definition 2962 MachineInstr *SIRegisterInfo::findReachingDef(Register Reg, unsigned SubReg, 2963 MachineInstr &Use, 2964 MachineRegisterInfo &MRI, 2965 LiveIntervals *LIS) const { 2966 auto &MDT = LIS->getAnalysis<MachineDominatorTree>(); 2967 SlotIndex UseIdx = LIS->getInstructionIndex(Use); 2968 SlotIndex DefIdx; 2969 2970 if (Reg.isVirtual()) { 2971 if (!LIS->hasInterval(Reg)) 2972 return nullptr; 2973 LiveInterval &LI = LIS->getInterval(Reg); 2974 LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask(SubReg) 2975 : MRI.getMaxLaneMaskForVReg(Reg); 2976 VNInfo *V = nullptr; 2977 if (LI.hasSubRanges()) { 2978 for (auto &S : LI.subranges()) { 2979 if ((S.LaneMask & SubLanes) == SubLanes) { 2980 V = S.getVNInfoAt(UseIdx); 2981 break; 2982 } 2983 } 2984 } else { 2985 V = LI.getVNInfoAt(UseIdx); 2986 } 2987 if (!V) 2988 return nullptr; 2989 DefIdx = V->def; 2990 } else { 2991 // Find last def. 2992 for (MCRegUnitIterator Units(Reg.asMCReg(), this); Units.isValid(); 2993 ++Units) { 2994 LiveRange &LR = LIS->getRegUnit(*Units); 2995 if (VNInfo *V = LR.getVNInfoAt(UseIdx)) { 2996 if (!DefIdx.isValid() || 2997 MDT.dominates(LIS->getInstructionFromIndex(DefIdx), 2998 LIS->getInstructionFromIndex(V->def))) 2999 DefIdx = V->def; 3000 } else { 3001 return nullptr; 3002 } 3003 } 3004 } 3005 3006 MachineInstr *Def = LIS->getInstructionFromIndex(DefIdx); 3007 3008 if (!Def || !MDT.dominates(Def, &Use)) 3009 return nullptr; 3010 3011 assert(Def->modifiesRegister(Reg, this)); 3012 3013 return Def; 3014 } 3015 3016 MCPhysReg SIRegisterInfo::get32BitRegister(MCPhysReg Reg) const { 3017 assert(getRegSizeInBits(*getPhysRegClass(Reg)) <= 32); 3018 3019 for (const TargetRegisterClass &RC : { AMDGPU::VGPR_32RegClass, 3020 AMDGPU::SReg_32RegClass, 3021 AMDGPU::AGPR_32RegClass } ) { 3022 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::lo16, &RC)) 3023 return Super; 3024 } 3025 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::hi16, 3026 &AMDGPU::VGPR_32RegClass)) { 3027 return Super; 3028 } 3029 3030 return AMDGPU::NoRegister; 3031 } 3032 3033 bool SIRegisterInfo::isProperlyAlignedRC(const TargetRegisterClass &RC) const { 3034 if (!ST.needsAlignedVGPRs()) 3035 return true; 3036 3037 if (isVGPRClass(&RC)) 3038 return RC.hasSuperClassEq(getVGPRClassForBitWidth(getRegSizeInBits(RC))); 3039 if (isAGPRClass(&RC)) 3040 return RC.hasSuperClassEq(getAGPRClassForBitWidth(getRegSizeInBits(RC))); 3041 if (isVectorSuperClass(&RC)) 3042 return RC.hasSuperClassEq( 3043 getVectorSuperClassForBitWidth(getRegSizeInBits(RC))); 3044 3045 return true; 3046 } 3047 3048 const TargetRegisterClass * 3049 SIRegisterInfo::getProperlyAlignedRC(const TargetRegisterClass *RC) const { 3050 if (!RC || !ST.needsAlignedVGPRs()) 3051 return RC; 3052 3053 unsigned Size = getRegSizeInBits(*RC); 3054 if (Size <= 32) 3055 return RC; 3056 3057 if (isVGPRClass(RC)) 3058 return getAlignedVGPRClassForBitWidth(Size); 3059 if (isAGPRClass(RC)) 3060 return getAlignedAGPRClassForBitWidth(Size); 3061 if (isVectorSuperClass(RC)) 3062 return getAlignedVectorSuperClassForBitWidth(Size); 3063 3064 return RC; 3065 } 3066 3067 bool SIRegisterInfo::isConstantPhysReg(MCRegister PhysReg) const { 3068 switch (PhysReg) { 3069 case AMDGPU::SGPR_NULL: 3070 case AMDGPU::SGPR_NULL64: 3071 case AMDGPU::SRC_SHARED_BASE: 3072 case AMDGPU::SRC_PRIVATE_BASE: 3073 case AMDGPU::SRC_SHARED_LIMIT: 3074 case AMDGPU::SRC_PRIVATE_LIMIT: 3075 return true; 3076 default: 3077 return false; 3078 } 3079 } 3080 3081 ArrayRef<MCPhysReg> 3082 SIRegisterInfo::getAllSGPR128(const MachineFunction &MF) const { 3083 return makeArrayRef(AMDGPU::SGPR_128RegClass.begin(), 3084 ST.getMaxNumSGPRs(MF) / 4); 3085 } 3086 3087 ArrayRef<MCPhysReg> 3088 SIRegisterInfo::getAllSGPR64(const MachineFunction &MF) const { 3089 return makeArrayRef(AMDGPU::SGPR_64RegClass.begin(), 3090 ST.getMaxNumSGPRs(MF) / 2); 3091 } 3092 3093 ArrayRef<MCPhysReg> 3094 SIRegisterInfo::getAllSGPR32(const MachineFunction &MF) const { 3095 return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(), ST.getMaxNumSGPRs(MF)); 3096 } 3097