1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "SIMachineFunctionInfo.h" 10 #include "AMDGPUSubtarget.h" 11 #include "AMDGPUTargetMachine.h" 12 #include "GCNSubtarget.h" 13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 14 #include "SIRegisterInfo.h" 15 #include "Utils/AMDGPUBaseInfo.h" 16 #include "llvm/CodeGen/LiveIntervals.h" 17 #include "llvm/CodeGen/MIRParser/MIParser.h" 18 #include "llvm/CodeGen/MachineBasicBlock.h" 19 #include "llvm/CodeGen/MachineFrameInfo.h" 20 #include "llvm/CodeGen/MachineFunction.h" 21 #include "llvm/CodeGen/MachineRegisterInfo.h" 22 #include "llvm/IR/CallingConv.h" 23 #include "llvm/IR/DiagnosticInfo.h" 24 #include "llvm/IR/Function.h" 25 #include <cassert> 26 #include <optional> 27 #include <vector> 28 29 #define MAX_LANES 64 30 31 using namespace llvm; 32 33 const GCNTargetMachine &getTM(const GCNSubtarget *STI) { 34 const SITargetLowering *TLI = STI->getTargetLowering(); 35 return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine()); 36 } 37 38 SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F, 39 const GCNSubtarget *STI) 40 : AMDGPUMachineFunction(F, *STI), Mode(F, *STI), GWSResourcePSV(getTM(STI)), 41 UserSGPRInfo(F, *STI), WorkGroupIDX(false), WorkGroupIDY(false), 42 WorkGroupIDZ(false), WorkGroupInfo(false), LDSKernelId(false), 43 PrivateSegmentWaveByteOffset(false), WorkItemIDX(false), 44 WorkItemIDY(false), WorkItemIDZ(false), ImplicitArgPtr(false), 45 GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0) { 46 const GCNSubtarget &ST = *static_cast<const GCNSubtarget *>(STI); 47 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F); 48 WavesPerEU = ST.getWavesPerEU(F); 49 50 Occupancy = ST.computeOccupancy(F, getLDSSize()); 51 CallingConv::ID CC = F.getCallingConv(); 52 53 VRegFlags.reserve(1024); 54 55 const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL || 56 CC == CallingConv::SPIR_KERNEL; 57 58 if (IsKernel) { 59 WorkGroupIDX = true; 60 WorkItemIDX = true; 61 } else if (CC == CallingConv::AMDGPU_PS) { 62 PSInputAddr = AMDGPU::getInitialPSInputAddr(F); 63 } 64 65 MayNeedAGPRs = ST.hasMAIInsts(); 66 67 if (AMDGPU::isChainCC(CC)) { 68 // Chain functions don't receive an SP from their caller, but are free to 69 // set one up. For now, we can use s32 to match what amdgpu_gfx functions 70 // would use if called, but this can be revisited. 71 // FIXME: Only reserve this if we actually need it. 72 StackPtrOffsetReg = AMDGPU::SGPR32; 73 74 ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51; 75 76 ArgInfo.PrivateSegmentBuffer = 77 ArgDescriptor::createRegister(ScratchRSrcReg); 78 79 ImplicitArgPtr = false; 80 } else if (!isEntryFunction()) { 81 if (CC != CallingConv::AMDGPU_Gfx) 82 ArgInfo = AMDGPUArgumentUsageInfo::FixedABIFunctionInfo; 83 84 // TODO: Pick a high register, and shift down, similar to a kernel. 85 FrameOffsetReg = AMDGPU::SGPR33; 86 StackPtrOffsetReg = AMDGPU::SGPR32; 87 88 if (!ST.enableFlatScratch()) { 89 // Non-entry functions have no special inputs for now, other registers 90 // required for scratch access. 91 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3; 92 93 ArgInfo.PrivateSegmentBuffer = 94 ArgDescriptor::createRegister(ScratchRSrcReg); 95 } 96 97 if (!F.hasFnAttribute("amdgpu-no-implicitarg-ptr")) 98 ImplicitArgPtr = true; 99 } else { 100 ImplicitArgPtr = false; 101 MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(), 102 MaxKernArgAlign); 103 104 if (ST.hasGFX90AInsts() && 105 ST.getMaxNumVGPRs(F) <= AMDGPU::VGPR_32RegClass.getNumRegs() && 106 !mayUseAGPRs(F)) 107 MayNeedAGPRs = false; // We will select all MAI with VGPR operands. 108 } 109 110 if (!AMDGPU::isGraphics(CC) || 111 (CC == CallingConv::AMDGPU_CS && ST.hasArchitectedSGPRs())) { 112 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x")) 113 WorkGroupIDX = true; 114 115 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-y")) 116 WorkGroupIDY = true; 117 118 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-z")) 119 WorkGroupIDZ = true; 120 } 121 122 if (!AMDGPU::isGraphics(CC)) { 123 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workitem-id-x")) 124 WorkItemIDX = true; 125 126 if (!F.hasFnAttribute("amdgpu-no-workitem-id-y") && 127 ST.getMaxWorkitemID(F, 1) != 0) 128 WorkItemIDY = true; 129 130 if (!F.hasFnAttribute("amdgpu-no-workitem-id-z") && 131 ST.getMaxWorkitemID(F, 2) != 0) 132 WorkItemIDZ = true; 133 134 if (!IsKernel && !F.hasFnAttribute("amdgpu-no-lds-kernel-id")) 135 LDSKernelId = true; 136 } 137 138 if (isEntryFunction()) { 139 // X, XY, and XYZ are the only supported combinations, so make sure Y is 140 // enabled if Z is. 141 if (WorkItemIDZ) 142 WorkItemIDY = true; 143 144 if (!ST.flatScratchIsArchitected()) { 145 PrivateSegmentWaveByteOffset = true; 146 147 // HS and GS always have the scratch wave offset in SGPR5 on GFX9. 148 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 && 149 (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS)) 150 ArgInfo.PrivateSegmentWaveByteOffset = 151 ArgDescriptor::createRegister(AMDGPU::SGPR5); 152 } 153 } 154 155 Attribute A = F.getFnAttribute("amdgpu-git-ptr-high"); 156 StringRef S = A.getValueAsString(); 157 if (!S.empty()) 158 S.consumeInteger(0, GITPtrHigh); 159 160 A = F.getFnAttribute("amdgpu-32bit-address-high-bits"); 161 S = A.getValueAsString(); 162 if (!S.empty()) 163 S.consumeInteger(0, HighBitsOf32BitAddress); 164 165 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch 166 // VGPR available at all times. For now, reserve highest available VGPR. After 167 // RA, shift it to the lowest available unused VGPR if the one exist. 168 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) { 169 VGPRForAGPRCopy = 170 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(F) - 1); 171 } 172 } 173 174 MachineFunctionInfo *SIMachineFunctionInfo::clone( 175 BumpPtrAllocator &Allocator, MachineFunction &DestMF, 176 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB) 177 const { 178 return DestMF.cloneInfo<SIMachineFunctionInfo>(*this); 179 } 180 181 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) { 182 limitOccupancy(getMaxWavesPerEU()); 183 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>(); 184 limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(), 185 MF.getFunction())); 186 } 187 188 Register SIMachineFunctionInfo::addPrivateSegmentBuffer( 189 const SIRegisterInfo &TRI) { 190 ArgInfo.PrivateSegmentBuffer = 191 ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 192 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass)); 193 NumUserSGPRs += 4; 194 return ArgInfo.PrivateSegmentBuffer.getRegister(); 195 } 196 197 Register SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) { 198 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 199 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 200 NumUserSGPRs += 2; 201 return ArgInfo.DispatchPtr.getRegister(); 202 } 203 204 Register SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) { 205 ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 206 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 207 NumUserSGPRs += 2; 208 return ArgInfo.QueuePtr.getRegister(); 209 } 210 211 Register SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) { 212 ArgInfo.KernargSegmentPtr 213 = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 214 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 215 NumUserSGPRs += 2; 216 return ArgInfo.KernargSegmentPtr.getRegister(); 217 } 218 219 Register SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) { 220 ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 221 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 222 NumUserSGPRs += 2; 223 return ArgInfo.DispatchID.getRegister(); 224 } 225 226 Register SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) { 227 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 228 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 229 NumUserSGPRs += 2; 230 return ArgInfo.FlatScratchInit.getRegister(); 231 } 232 233 Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) { 234 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 235 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 236 NumUserSGPRs += 2; 237 return ArgInfo.ImplicitBufferPtr.getRegister(); 238 } 239 240 Register SIMachineFunctionInfo::addLDSKernelId() { 241 ArgInfo.LDSKernelId = ArgDescriptor::createRegister(getNextUserSGPR()); 242 NumUserSGPRs += 1; 243 return ArgInfo.LDSKernelId.getRegister(); 244 } 245 246 SmallVectorImpl<MCRegister> *SIMachineFunctionInfo::addPreloadedKernArg( 247 const SIRegisterInfo &TRI, const TargetRegisterClass *RC, 248 unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs) { 249 assert(!ArgInfo.PreloadKernArgs.count(KernArgIdx) && 250 "Preload kernel argument allocated twice."); 251 NumUserSGPRs += PaddingSGPRs; 252 // If the available register tuples are aligned with the kernarg to be 253 // preloaded use that register, otherwise we need to use a set of SGPRs and 254 // merge them. 255 Register PreloadReg = 256 TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC); 257 if (PreloadReg && 258 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) { 259 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(PreloadReg); 260 NumUserSGPRs += AllocSizeDWord; 261 } else { 262 for (unsigned I = 0; I < AllocSizeDWord; ++I) { 263 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(getNextUserSGPR()); 264 NumUserSGPRs++; 265 } 266 } 267 268 // Track the actual number of SGPRs that HW will preload to. 269 UserSGPRInfo.allocKernargPreloadSGPRs(AllocSizeDWord + PaddingSGPRs); 270 return &ArgInfo.PreloadKernArgs[KernArgIdx].Regs; 271 } 272 273 void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR, 274 uint64_t Size, Align Alignment) { 275 // Skip if it is an entry function or the register is already added. 276 if (isEntryFunction() || WWMSpills.count(VGPR)) 277 return; 278 279 // Skip if this is a function with the amdgpu_cs_chain or 280 // amdgpu_cs_chain_preserve calling convention and this is a scratch register. 281 // We never need to allocate a spill for these because we don't even need to 282 // restore the inactive lanes for them (they're scratchier than the usual 283 // scratch registers). 284 if (isChainFunction() && SIRegisterInfo::isChainScratchRegister(VGPR)) 285 return; 286 287 WWMSpills.insert(std::make_pair( 288 VGPR, MF.getFrameInfo().CreateSpillStackObject(Size, Alignment))); 289 } 290 291 // Separate out the callee-saved and scratch registers. 292 void SIMachineFunctionInfo::splitWWMSpillRegisters( 293 MachineFunction &MF, 294 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs, 295 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const { 296 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs(); 297 for (auto &Reg : WWMSpills) { 298 if (isCalleeSavedReg(CSRegs, Reg.first)) 299 CalleeSavedRegs.push_back(Reg); 300 else 301 ScratchRegs.push_back(Reg); 302 } 303 } 304 305 bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs, 306 MCPhysReg Reg) const { 307 for (unsigned I = 0; CSRegs[I]; ++I) { 308 if (CSRegs[I] == Reg) 309 return true; 310 } 311 312 return false; 313 } 314 315 bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills( 316 MachineFunction &MF, int FI, unsigned LaneIndex) { 317 MachineRegisterInfo &MRI = MF.getRegInfo(); 318 Register LaneVGPR; 319 if (!LaneIndex) { 320 LaneVGPR = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 321 SpillVGPRs.push_back(LaneVGPR); 322 } else { 323 LaneVGPR = SpillVGPRs.back(); 324 } 325 326 SGPRSpillsToVirtualVGPRLanes[FI].push_back( 327 SIRegisterInfo::SpilledReg(LaneVGPR, LaneIndex)); 328 return true; 329 } 330 331 bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills( 332 MachineFunction &MF, int FI, unsigned LaneIndex) { 333 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 334 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 335 MachineRegisterInfo &MRI = MF.getRegInfo(); 336 Register LaneVGPR; 337 if (!LaneIndex) { 338 LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF); 339 if (LaneVGPR == AMDGPU::NoRegister) { 340 // We have no VGPRs left for spilling SGPRs. Reset because we will not 341 // partially spill the SGPR to VGPRs. 342 SGPRSpillsToPhysicalVGPRLanes.erase(FI); 343 return false; 344 } 345 346 allocateWWMSpill(MF, LaneVGPR); 347 reserveWWMRegister(LaneVGPR); 348 for (MachineBasicBlock &MBB : MF) { 349 MBB.addLiveIn(LaneVGPR); 350 MBB.sortUniqueLiveIns(); 351 } 352 SpillPhysVGPRs.push_back(LaneVGPR); 353 } else { 354 LaneVGPR = SpillPhysVGPRs.back(); 355 } 356 357 SGPRSpillsToPhysicalVGPRLanes[FI].push_back( 358 SIRegisterInfo::SpilledReg(LaneVGPR, LaneIndex)); 359 return true; 360 } 361 362 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPRLane(MachineFunction &MF, 363 int FI, 364 bool IsPrologEpilog) { 365 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes = 366 IsPrologEpilog ? SGPRSpillsToPhysicalVGPRLanes[FI] 367 : SGPRSpillsToVirtualVGPRLanes[FI]; 368 369 // This has already been allocated. 370 if (!SpillLanes.empty()) 371 return true; 372 373 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 374 MachineFrameInfo &FrameInfo = MF.getFrameInfo(); 375 unsigned WaveSize = ST.getWavefrontSize(); 376 377 unsigned Size = FrameInfo.getObjectSize(FI); 378 unsigned NumLanes = Size / 4; 379 380 if (NumLanes > WaveSize) 381 return false; 382 383 assert(Size >= 4 && "invalid sgpr spill size"); 384 assert(ST.getRegisterInfo()->spillSGPRToVGPR() && 385 "not spilling SGPRs to VGPRs"); 386 387 unsigned &NumSpillLanes = 388 IsPrologEpilog ? NumPhysicalVGPRSpillLanes : NumVirtualVGPRSpillLanes; 389 390 for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) { 391 unsigned LaneIndex = (NumSpillLanes % WaveSize); 392 393 bool Allocated = IsPrologEpilog 394 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex) 395 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex); 396 if (!Allocated) { 397 NumSpillLanes -= I; 398 return false; 399 } 400 } 401 402 return true; 403 } 404 405 /// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI. 406 /// Either AGPR is spilled to VGPR to vice versa. 407 /// Returns true if a \p FI can be eliminated completely. 408 bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF, 409 int FI, 410 bool isAGPRtoVGPR) { 411 MachineRegisterInfo &MRI = MF.getRegInfo(); 412 MachineFrameInfo &FrameInfo = MF.getFrameInfo(); 413 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 414 415 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI)); 416 417 auto &Spill = VGPRToAGPRSpills[FI]; 418 419 // This has already been allocated. 420 if (!Spill.Lanes.empty()) 421 return Spill.FullyAllocated; 422 423 unsigned Size = FrameInfo.getObjectSize(FI); 424 unsigned NumLanes = Size / 4; 425 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister); 426 427 const TargetRegisterClass &RC = 428 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass; 429 auto Regs = RC.getRegisters(); 430 431 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR; 432 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 433 Spill.FullyAllocated = true; 434 435 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize 436 // once. 437 BitVector OtherUsedRegs; 438 OtherUsedRegs.resize(TRI->getNumRegs()); 439 440 const uint32_t *CSRMask = 441 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv()); 442 if (CSRMask) 443 OtherUsedRegs.setBitsInMask(CSRMask); 444 445 // TODO: Should include register tuples, but doesn't matter with current 446 // usage. 447 for (MCPhysReg Reg : SpillAGPR) 448 OtherUsedRegs.set(Reg); 449 for (MCPhysReg Reg : SpillVGPR) 450 OtherUsedRegs.set(Reg); 451 452 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin(); 453 for (int I = NumLanes - 1; I >= 0; --I) { 454 NextSpillReg = std::find_if( 455 NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) { 456 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) && 457 !OtherUsedRegs[Reg]; 458 }); 459 460 if (NextSpillReg == Regs.end()) { // Registers exhausted 461 Spill.FullyAllocated = false; 462 break; 463 } 464 465 OtherUsedRegs.set(*NextSpillReg); 466 SpillRegs.push_back(*NextSpillReg); 467 MRI.reserveReg(*NextSpillReg, TRI); 468 Spill.Lanes[I] = *NextSpillReg++; 469 } 470 471 return Spill.FullyAllocated; 472 } 473 474 bool SIMachineFunctionInfo::removeDeadFrameIndices( 475 MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) { 476 // Remove dead frame indices from function frame, however keep FP & BP since 477 // spills for them haven't been inserted yet. And also make sure to remove the 478 // frame indices from `SGPRSpillsToVirtualVGPRLanes` data structure, 479 // otherwise, it could result in an unexpected side effect and bug, in case of 480 // any re-mapping of freed frame indices by later pass(es) like "stack slot 481 // coloring". 482 for (auto &R : make_early_inc_range(SGPRSpillsToVirtualVGPRLanes)) { 483 MFI.RemoveStackObject(R.first); 484 SGPRSpillsToVirtualVGPRLanes.erase(R.first); 485 } 486 487 // Remove the dead frame indices of CSR SGPRs which are spilled to physical 488 // VGPR lanes during SILowerSGPRSpills pass. 489 if (!ResetSGPRSpillStackIDs) { 490 for (auto &R : make_early_inc_range(SGPRSpillsToPhysicalVGPRLanes)) { 491 MFI.RemoveStackObject(R.first); 492 SGPRSpillsToPhysicalVGPRLanes.erase(R.first); 493 } 494 } 495 bool HaveSGPRToMemory = false; 496 497 if (ResetSGPRSpillStackIDs) { 498 // All other SGPRs must be allocated on the default stack, so reset the 499 // stack ID. 500 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E; 501 ++I) { 502 if (!checkIndexInPrologEpilogSGPRSpills(I)) { 503 if (MFI.getStackID(I) == TargetStackID::SGPRSpill) { 504 MFI.setStackID(I, TargetStackID::Default); 505 HaveSGPRToMemory = true; 506 } 507 } 508 } 509 } 510 511 for (auto &R : VGPRToAGPRSpills) { 512 if (R.second.IsDead) 513 MFI.RemoveStackObject(R.first); 514 } 515 516 return HaveSGPRToMemory; 517 } 518 519 int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI, 520 const SIRegisterInfo &TRI) { 521 if (ScavengeFI) 522 return *ScavengeFI; 523 if (isBottomOfStack()) { 524 ScavengeFI = MFI.CreateFixedObject( 525 TRI.getSpillSize(AMDGPU::SGPR_32RegClass), 0, false); 526 } else { 527 ScavengeFI = MFI.CreateStackObject( 528 TRI.getSpillSize(AMDGPU::SGPR_32RegClass), 529 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass), false); 530 } 531 return *ScavengeFI; 532 } 533 534 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const { 535 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs"); 536 return AMDGPU::SGPR0 + NumUserSGPRs; 537 } 538 539 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const { 540 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs; 541 } 542 543 void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) { 544 VRegFlags.grow(Reg); 545 } 546 547 void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg, 548 Register SrcReg) { 549 VRegFlags.grow(NewReg); 550 VRegFlags[NewReg] = VRegFlags[SrcReg]; 551 } 552 553 Register 554 SIMachineFunctionInfo::getGITPtrLoReg(const MachineFunction &MF) const { 555 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 556 if (!ST.isAmdPalOS()) 557 return Register(); 558 Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in 559 if (ST.hasMergedShaders()) { 560 switch (MF.getFunction().getCallingConv()) { 561 case CallingConv::AMDGPU_HS: 562 case CallingConv::AMDGPU_GS: 563 // Low GIT address is passed in s8 rather than s0 for an LS+HS or 564 // ES+GS merged shader on gfx9+. 565 GitPtrLo = AMDGPU::SGPR8; 566 return GitPtrLo; 567 default: 568 return GitPtrLo; 569 } 570 } 571 return GitPtrLo; 572 } 573 574 static yaml::StringValue regToString(Register Reg, 575 const TargetRegisterInfo &TRI) { 576 yaml::StringValue Dest; 577 { 578 raw_string_ostream OS(Dest.Value); 579 OS << printReg(Reg, &TRI); 580 } 581 return Dest; 582 } 583 584 static std::optional<yaml::SIArgumentInfo> 585 convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo, 586 const TargetRegisterInfo &TRI) { 587 yaml::SIArgumentInfo AI; 588 589 auto convertArg = [&](std::optional<yaml::SIArgument> &A, 590 const ArgDescriptor &Arg) { 591 if (!Arg) 592 return false; 593 594 // Create a register or stack argument. 595 yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister()); 596 if (Arg.isRegister()) { 597 raw_string_ostream OS(SA.RegisterName.Value); 598 OS << printReg(Arg.getRegister(), &TRI); 599 } else 600 SA.StackOffset = Arg.getStackOffset(); 601 // Check and update the optional mask. 602 if (Arg.isMasked()) 603 SA.Mask = Arg.getMask(); 604 605 A = SA; 606 return true; 607 }; 608 609 // TODO: Need to serialize kernarg preloads. 610 bool Any = false; 611 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer); 612 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr); 613 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr); 614 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr); 615 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID); 616 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit); 617 Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId); 618 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize); 619 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX); 620 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY); 621 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ); 622 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo); 623 Any |= convertArg(AI.PrivateSegmentWaveByteOffset, 624 ArgInfo.PrivateSegmentWaveByteOffset); 625 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr); 626 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr); 627 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX); 628 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY); 629 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ); 630 631 if (Any) 632 return AI; 633 634 return std::nullopt; 635 } 636 637 yaml::SIMachineFunctionInfo::SIMachineFunctionInfo( 638 const llvm::SIMachineFunctionInfo &MFI, const TargetRegisterInfo &TRI, 639 const llvm::MachineFunction &MF) 640 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()), 641 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()), 642 GDSSize(MFI.getGDSSize()), 643 DynLDSAlign(MFI.getDynLDSAlign()), IsEntryFunction(MFI.isEntryFunction()), 644 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()), 645 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()), 646 HasSpilledSGPRs(MFI.hasSpilledSGPRs()), 647 HasSpilledVGPRs(MFI.hasSpilledVGPRs()), 648 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()), 649 Occupancy(MFI.getOccupancy()), 650 ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)), 651 FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)), 652 StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)), 653 BytesInStackArgArea(MFI.getBytesInStackArgArea()), 654 ReturnsVoid(MFI.returnsVoid()), 655 ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)), 656 PSInputAddr(MFI.getPSInputAddr()), 657 PSInputEnable(MFI.getPSInputEnable()), 658 Mode(MFI.getMode()) { 659 for (Register Reg : MFI.getWWMReservedRegs()) 660 WWMReservedRegs.push_back(regToString(Reg, TRI)); 661 662 if (MFI.getLongBranchReservedReg()) 663 LongBranchReservedReg = regToString(MFI.getLongBranchReservedReg(), TRI); 664 if (MFI.getVGPRForAGPRCopy()) 665 VGPRForAGPRCopy = regToString(MFI.getVGPRForAGPRCopy(), TRI); 666 667 if (MFI.getSGPRForEXECCopy()) 668 SGPRForEXECCopy = regToString(MFI.getSGPRForEXECCopy(), TRI); 669 670 auto SFI = MFI.getOptionalScavengeFI(); 671 if (SFI) 672 ScavengeFI = yaml::FrameIndex(*SFI, MF.getFrameInfo()); 673 } 674 675 void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) { 676 MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this); 677 } 678 679 bool SIMachineFunctionInfo::initializeBaseYamlFields( 680 const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, 681 PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) { 682 ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize; 683 MaxKernArgAlign = YamlMFI.MaxKernArgAlign; 684 LDSSize = YamlMFI.LDSSize; 685 GDSSize = YamlMFI.GDSSize; 686 DynLDSAlign = YamlMFI.DynLDSAlign; 687 PSInputAddr = YamlMFI.PSInputAddr; 688 PSInputEnable = YamlMFI.PSInputEnable; 689 HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress; 690 Occupancy = YamlMFI.Occupancy; 691 IsEntryFunction = YamlMFI.IsEntryFunction; 692 NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath; 693 MemoryBound = YamlMFI.MemoryBound; 694 WaveLimiter = YamlMFI.WaveLimiter; 695 HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs; 696 HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs; 697 BytesInStackArgArea = YamlMFI.BytesInStackArgArea; 698 ReturnsVoid = YamlMFI.ReturnsVoid; 699 700 if (YamlMFI.ScavengeFI) { 701 auto FIOrErr = YamlMFI.ScavengeFI->getFI(MF.getFrameInfo()); 702 if (!FIOrErr) { 703 // Create a diagnostic for a the frame index. 704 const MemoryBuffer &Buffer = 705 *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID()); 706 707 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1, 708 SourceMgr::DK_Error, toString(FIOrErr.takeError()), 709 "", std::nullopt, std::nullopt); 710 SourceRange = YamlMFI.ScavengeFI->SourceRange; 711 return true; 712 } 713 ScavengeFI = *FIOrErr; 714 } else { 715 ScavengeFI = std::nullopt; 716 } 717 return false; 718 } 719 720 bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const { 721 for (const BasicBlock &BB : F) { 722 for (const Instruction &I : BB) { 723 const auto *CB = dyn_cast<CallBase>(&I); 724 if (!CB) 725 continue; 726 727 if (CB->isInlineAsm()) { 728 const InlineAsm *IA = dyn_cast<InlineAsm>(CB->getCalledOperand()); 729 for (const auto &CI : IA->ParseConstraints()) { 730 for (StringRef Code : CI.Codes) { 731 Code.consume_front("{"); 732 if (Code.starts_with("a")) 733 return true; 734 } 735 } 736 continue; 737 } 738 739 const Function *Callee = 740 dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts()); 741 if (!Callee) 742 return true; 743 744 if (Callee->getIntrinsicID() == Intrinsic::not_intrinsic) 745 return true; 746 } 747 } 748 749 return false; 750 } 751 752 bool SIMachineFunctionInfo::usesAGPRs(const MachineFunction &MF) const { 753 if (UsesAGPRs) 754 return *UsesAGPRs; 755 756 if (!mayNeedAGPRs()) { 757 UsesAGPRs = false; 758 return false; 759 } 760 761 if (!AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv()) || 762 MF.getFrameInfo().hasCalls()) { 763 UsesAGPRs = true; 764 return true; 765 } 766 767 const MachineRegisterInfo &MRI = MF.getRegInfo(); 768 769 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 770 const Register Reg = Register::index2VirtReg(I); 771 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg); 772 if (RC && SIRegisterInfo::isAGPRClass(RC)) { 773 UsesAGPRs = true; 774 return true; 775 } else if (!RC && !MRI.use_empty(Reg) && MRI.getType(Reg).isValid()) { 776 // Defer caching UsesAGPRs, function might not yet been regbank selected. 777 return true; 778 } 779 } 780 781 for (MCRegister Reg : AMDGPU::AGPR_32RegClass) { 782 if (MRI.isPhysRegUsed(Reg)) { 783 UsesAGPRs = true; 784 return true; 785 } 786 } 787 788 UsesAGPRs = false; 789 return false; 790 } 791