1 //==- SIMachineFunctionInfo.h - SIMachineFunctionInfo interface --*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifndef LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H 14 #define LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H 15 16 #include "AMDGPUArgumentUsageInfo.h" 17 #include "AMDGPUMachineFunction.h" 18 #include "AMDGPUTargetMachine.h" 19 #include "GCNSubtarget.h" 20 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 21 #include "SIInstrInfo.h" 22 #include "SIModeRegisterDefaults.h" 23 #include "llvm/ADT/SetVector.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/CodeGen/MIRYamlMapping.h" 26 #include "llvm/CodeGen/PseudoSourceValue.h" 27 #include "llvm/Support/raw_ostream.h" 28 #include <optional> 29 30 namespace llvm { 31 32 class MachineFrameInfo; 33 class MachineFunction; 34 class SIMachineFunctionInfo; 35 class SIRegisterInfo; 36 class TargetRegisterClass; 37 38 class AMDGPUPseudoSourceValue : public PseudoSourceValue { 39 public: 40 enum AMDGPUPSVKind : unsigned { 41 PSVImage = PseudoSourceValue::TargetCustom, 42 GWSResource 43 }; 44 45 protected: 46 AMDGPUPseudoSourceValue(unsigned Kind, const AMDGPUTargetMachine &TM) 47 : PseudoSourceValue(Kind, TM) {} 48 49 public: 50 bool isConstant(const MachineFrameInfo *) const override { 51 // This should probably be true for most images, but we will start by being 52 // conservative. 53 return false; 54 } 55 56 bool isAliased(const MachineFrameInfo *) const override { 57 return true; 58 } 59 60 bool mayAlias(const MachineFrameInfo *) const override { 61 return true; 62 } 63 }; 64 65 class AMDGPUGWSResourcePseudoSourceValue final : public AMDGPUPseudoSourceValue { 66 public: 67 explicit AMDGPUGWSResourcePseudoSourceValue(const AMDGPUTargetMachine &TM) 68 : AMDGPUPseudoSourceValue(GWSResource, TM) {} 69 70 static bool classof(const PseudoSourceValue *V) { 71 return V->kind() == GWSResource; 72 } 73 74 // These are inaccessible memory from IR. 75 bool isAliased(const MachineFrameInfo *) const override { 76 return false; 77 } 78 79 // These are inaccessible memory from IR. 80 bool mayAlias(const MachineFrameInfo *) const override { 81 return false; 82 } 83 84 void printCustom(raw_ostream &OS) const override { 85 OS << "GWSResource"; 86 } 87 }; 88 89 namespace yaml { 90 91 struct SIArgument { 92 bool IsRegister; 93 union { 94 StringValue RegisterName; 95 unsigned StackOffset; 96 }; 97 std::optional<unsigned> Mask; 98 99 // Default constructor, which creates a stack argument. 100 SIArgument() : IsRegister(false), StackOffset(0) {} 101 SIArgument(const SIArgument &Other) { 102 IsRegister = Other.IsRegister; 103 if (IsRegister) { 104 ::new ((void *)std::addressof(RegisterName)) 105 StringValue(Other.RegisterName); 106 } else 107 StackOffset = Other.StackOffset; 108 Mask = Other.Mask; 109 } 110 SIArgument &operator=(const SIArgument &Other) { 111 IsRegister = Other.IsRegister; 112 if (IsRegister) { 113 ::new ((void *)std::addressof(RegisterName)) 114 StringValue(Other.RegisterName); 115 } else 116 StackOffset = Other.StackOffset; 117 Mask = Other.Mask; 118 return *this; 119 } 120 ~SIArgument() { 121 if (IsRegister) 122 RegisterName.~StringValue(); 123 } 124 125 // Helper to create a register or stack argument. 126 static inline SIArgument createArgument(bool IsReg) { 127 if (IsReg) 128 return SIArgument(IsReg); 129 return SIArgument(); 130 } 131 132 private: 133 // Construct a register argument. 134 SIArgument(bool) : IsRegister(true), RegisterName() {} 135 }; 136 137 template <> struct MappingTraits<SIArgument> { 138 static void mapping(IO &YamlIO, SIArgument &A) { 139 if (YamlIO.outputting()) { 140 if (A.IsRegister) 141 YamlIO.mapRequired("reg", A.RegisterName); 142 else 143 YamlIO.mapRequired("offset", A.StackOffset); 144 } else { 145 auto Keys = YamlIO.keys(); 146 if (is_contained(Keys, "reg")) { 147 A = SIArgument::createArgument(true); 148 YamlIO.mapRequired("reg", A.RegisterName); 149 } else if (is_contained(Keys, "offset")) 150 YamlIO.mapRequired("offset", A.StackOffset); 151 else 152 YamlIO.setError("missing required key 'reg' or 'offset'"); 153 } 154 YamlIO.mapOptional("mask", A.Mask); 155 } 156 static const bool flow = true; 157 }; 158 159 struct SIArgumentInfo { 160 std::optional<SIArgument> PrivateSegmentBuffer; 161 std::optional<SIArgument> DispatchPtr; 162 std::optional<SIArgument> QueuePtr; 163 std::optional<SIArgument> KernargSegmentPtr; 164 std::optional<SIArgument> DispatchID; 165 std::optional<SIArgument> FlatScratchInit; 166 std::optional<SIArgument> PrivateSegmentSize; 167 168 std::optional<SIArgument> WorkGroupIDX; 169 std::optional<SIArgument> WorkGroupIDY; 170 std::optional<SIArgument> WorkGroupIDZ; 171 std::optional<SIArgument> WorkGroupInfo; 172 std::optional<SIArgument> LDSKernelId; 173 std::optional<SIArgument> PrivateSegmentWaveByteOffset; 174 175 std::optional<SIArgument> ImplicitArgPtr; 176 std::optional<SIArgument> ImplicitBufferPtr; 177 178 std::optional<SIArgument> WorkItemIDX; 179 std::optional<SIArgument> WorkItemIDY; 180 std::optional<SIArgument> WorkItemIDZ; 181 }; 182 183 template <> struct MappingTraits<SIArgumentInfo> { 184 static void mapping(IO &YamlIO, SIArgumentInfo &AI) { 185 YamlIO.mapOptional("privateSegmentBuffer", AI.PrivateSegmentBuffer); 186 YamlIO.mapOptional("dispatchPtr", AI.DispatchPtr); 187 YamlIO.mapOptional("queuePtr", AI.QueuePtr); 188 YamlIO.mapOptional("kernargSegmentPtr", AI.KernargSegmentPtr); 189 YamlIO.mapOptional("dispatchID", AI.DispatchID); 190 YamlIO.mapOptional("flatScratchInit", AI.FlatScratchInit); 191 YamlIO.mapOptional("privateSegmentSize", AI.PrivateSegmentSize); 192 193 YamlIO.mapOptional("workGroupIDX", AI.WorkGroupIDX); 194 YamlIO.mapOptional("workGroupIDY", AI.WorkGroupIDY); 195 YamlIO.mapOptional("workGroupIDZ", AI.WorkGroupIDZ); 196 YamlIO.mapOptional("workGroupInfo", AI.WorkGroupInfo); 197 YamlIO.mapOptional("LDSKernelId", AI.LDSKernelId); 198 YamlIO.mapOptional("privateSegmentWaveByteOffset", 199 AI.PrivateSegmentWaveByteOffset); 200 201 YamlIO.mapOptional("implicitArgPtr", AI.ImplicitArgPtr); 202 YamlIO.mapOptional("implicitBufferPtr", AI.ImplicitBufferPtr); 203 204 YamlIO.mapOptional("workItemIDX", AI.WorkItemIDX); 205 YamlIO.mapOptional("workItemIDY", AI.WorkItemIDY); 206 YamlIO.mapOptional("workItemIDZ", AI.WorkItemIDZ); 207 } 208 }; 209 210 // Default to default mode for default calling convention. 211 struct SIMode { 212 bool IEEE = true; 213 bool DX10Clamp = true; 214 bool FP32InputDenormals = true; 215 bool FP32OutputDenormals = true; 216 bool FP64FP16InputDenormals = true; 217 bool FP64FP16OutputDenormals = true; 218 219 SIMode() = default; 220 221 SIMode(const SIModeRegisterDefaults &Mode) { 222 IEEE = Mode.IEEE; 223 DX10Clamp = Mode.DX10Clamp; 224 FP32InputDenormals = Mode.FP32Denormals.Input != DenormalMode::PreserveSign; 225 FP32OutputDenormals = 226 Mode.FP32Denormals.Output != DenormalMode::PreserveSign; 227 FP64FP16InputDenormals = 228 Mode.FP64FP16Denormals.Input != DenormalMode::PreserveSign; 229 FP64FP16OutputDenormals = 230 Mode.FP64FP16Denormals.Output != DenormalMode::PreserveSign; 231 } 232 233 bool operator ==(const SIMode Other) const { 234 return IEEE == Other.IEEE && 235 DX10Clamp == Other.DX10Clamp && 236 FP32InputDenormals == Other.FP32InputDenormals && 237 FP32OutputDenormals == Other.FP32OutputDenormals && 238 FP64FP16InputDenormals == Other.FP64FP16InputDenormals && 239 FP64FP16OutputDenormals == Other.FP64FP16OutputDenormals; 240 } 241 }; 242 243 template <> struct MappingTraits<SIMode> { 244 static void mapping(IO &YamlIO, SIMode &Mode) { 245 YamlIO.mapOptional("ieee", Mode.IEEE, true); 246 YamlIO.mapOptional("dx10-clamp", Mode.DX10Clamp, true); 247 YamlIO.mapOptional("fp32-input-denormals", Mode.FP32InputDenormals, true); 248 YamlIO.mapOptional("fp32-output-denormals", Mode.FP32OutputDenormals, true); 249 YamlIO.mapOptional("fp64-fp16-input-denormals", Mode.FP64FP16InputDenormals, true); 250 YamlIO.mapOptional("fp64-fp16-output-denormals", Mode.FP64FP16OutputDenormals, true); 251 } 252 }; 253 254 struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo { 255 uint64_t ExplicitKernArgSize = 0; 256 Align MaxKernArgAlign; 257 uint32_t LDSSize = 0; 258 uint32_t GDSSize = 0; 259 Align DynLDSAlign; 260 bool IsEntryFunction = false; 261 bool IsChainFunction = false; 262 bool NoSignedZerosFPMath = false; 263 bool MemoryBound = false; 264 bool WaveLimiter = false; 265 bool HasSpilledSGPRs = false; 266 bool HasSpilledVGPRs = false; 267 uint32_t HighBitsOf32BitAddress = 0; 268 269 // TODO: 10 may be a better default since it's the maximum. 270 unsigned Occupancy = 0; 271 272 SmallVector<StringValue> WWMReservedRegs; 273 274 StringValue ScratchRSrcReg = "$private_rsrc_reg"; 275 StringValue FrameOffsetReg = "$fp_reg"; 276 StringValue StackPtrOffsetReg = "$sp_reg"; 277 278 unsigned BytesInStackArgArea = 0; 279 bool ReturnsVoid = true; 280 281 std::optional<SIArgumentInfo> ArgInfo; 282 283 unsigned PSInputAddr = 0; 284 unsigned PSInputEnable = 0; 285 286 SIMode Mode; 287 std::optional<FrameIndex> ScavengeFI; 288 StringValue VGPRForAGPRCopy; 289 StringValue SGPRForEXECCopy; 290 StringValue LongBranchReservedReg; 291 292 SIMachineFunctionInfo() = default; 293 SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo &, 294 const TargetRegisterInfo &TRI, 295 const llvm::MachineFunction &MF); 296 297 void mappingImpl(yaml::IO &YamlIO) override; 298 ~SIMachineFunctionInfo() = default; 299 }; 300 301 template <> struct MappingTraits<SIMachineFunctionInfo> { 302 static void mapping(IO &YamlIO, SIMachineFunctionInfo &MFI) { 303 YamlIO.mapOptional("explicitKernArgSize", MFI.ExplicitKernArgSize, 304 UINT64_C(0)); 305 YamlIO.mapOptional("maxKernArgAlign", MFI.MaxKernArgAlign); 306 YamlIO.mapOptional("ldsSize", MFI.LDSSize, 0u); 307 YamlIO.mapOptional("gdsSize", MFI.GDSSize, 0u); 308 YamlIO.mapOptional("dynLDSAlign", MFI.DynLDSAlign, Align()); 309 YamlIO.mapOptional("isEntryFunction", MFI.IsEntryFunction, false); 310 YamlIO.mapOptional("isChainFunction", MFI.IsChainFunction, false); 311 YamlIO.mapOptional("noSignedZerosFPMath", MFI.NoSignedZerosFPMath, false); 312 YamlIO.mapOptional("memoryBound", MFI.MemoryBound, false); 313 YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false); 314 YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false); 315 YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false); 316 YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg, 317 StringValue("$private_rsrc_reg")); 318 YamlIO.mapOptional("frameOffsetReg", MFI.FrameOffsetReg, 319 StringValue("$fp_reg")); 320 YamlIO.mapOptional("stackPtrOffsetReg", MFI.StackPtrOffsetReg, 321 StringValue("$sp_reg")); 322 YamlIO.mapOptional("bytesInStackArgArea", MFI.BytesInStackArgArea, 0u); 323 YamlIO.mapOptional("returnsVoid", MFI.ReturnsVoid, true); 324 YamlIO.mapOptional("argumentInfo", MFI.ArgInfo); 325 YamlIO.mapOptional("psInputAddr", MFI.PSInputAddr, 0u); 326 YamlIO.mapOptional("psInputEnable", MFI.PSInputEnable, 0u); 327 YamlIO.mapOptional("mode", MFI.Mode, SIMode()); 328 YamlIO.mapOptional("highBitsOf32BitAddress", 329 MFI.HighBitsOf32BitAddress, 0u); 330 YamlIO.mapOptional("occupancy", MFI.Occupancy, 0); 331 YamlIO.mapOptional("wwmReservedRegs", MFI.WWMReservedRegs); 332 YamlIO.mapOptional("scavengeFI", MFI.ScavengeFI); 333 YamlIO.mapOptional("vgprForAGPRCopy", MFI.VGPRForAGPRCopy, 334 StringValue()); // Don't print out when it's empty. 335 YamlIO.mapOptional("sgprForEXECCopy", MFI.SGPRForEXECCopy, 336 StringValue()); // Don't print out when it's empty. 337 YamlIO.mapOptional("longBranchReservedReg", MFI.LongBranchReservedReg, 338 StringValue()); 339 } 340 }; 341 342 } // end namespace yaml 343 344 // A CSR SGPR value can be preserved inside a callee using one of the following 345 // methods. 346 // 1. Copy to an unused scratch SGPR. 347 // 2. Spill to a VGPR lane. 348 // 3. Spill to memory via. a scratch VGPR. 349 // class PrologEpilogSGPRSaveRestoreInfo represents the save/restore method used 350 // for an SGPR at function prolog/epilog. 351 enum class SGPRSaveKind : uint8_t { 352 COPY_TO_SCRATCH_SGPR, 353 SPILL_TO_VGPR_LANE, 354 SPILL_TO_MEM 355 }; 356 357 class PrologEpilogSGPRSaveRestoreInfo { 358 SGPRSaveKind Kind; 359 union { 360 int Index; 361 Register Reg; 362 }; 363 364 public: 365 PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, int I) : Kind(K), Index(I) {} 366 PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, Register R) 367 : Kind(K), Reg(R) {} 368 Register getReg() const { return Reg; } 369 int getIndex() const { return Index; } 370 SGPRSaveKind getKind() const { return Kind; } 371 }; 372 373 /// This class keeps track of the SPI_SP_INPUT_ADDR config register, which 374 /// tells the hardware which interpolation parameters to load. 375 class SIMachineFunctionInfo final : public AMDGPUMachineFunction, 376 private MachineRegisterInfo::Delegate { 377 friend class GCNTargetMachine; 378 379 // State of MODE register, assumed FP mode. 380 SIModeRegisterDefaults Mode; 381 382 // Registers that may be reserved for spilling purposes. These may be the same 383 // as the input registers. 384 Register ScratchRSrcReg = AMDGPU::PRIVATE_RSRC_REG; 385 386 // This is the unswizzled offset from the current dispatch's scratch wave 387 // base to the beginning of the current function's frame. 388 Register FrameOffsetReg = AMDGPU::FP_REG; 389 390 // This is an ABI register used in the non-entry calling convention to 391 // communicate the unswizzled offset from the current dispatch's scratch wave 392 // base to the beginning of the new function's frame. 393 Register StackPtrOffsetReg = AMDGPU::SP_REG; 394 395 // Registers that may be reserved when RA doesn't allocate enough 396 // registers to plan for the case where an indirect branch ends up 397 // being needed during branch relaxation. 398 Register LongBranchReservedReg; 399 400 AMDGPUFunctionArgInfo ArgInfo; 401 402 // Graphics info. 403 unsigned PSInputAddr = 0; 404 unsigned PSInputEnable = 0; 405 406 /// Number of bytes of arguments this function has on the stack. If the callee 407 /// is expected to restore the argument stack this should be a multiple of 16, 408 /// all usable during a tail call. 409 /// 410 /// The alternative would forbid tail call optimisation in some cases: if we 411 /// want to transfer control from a function with 8-bytes of stack-argument 412 /// space to a function with 16-bytes then misalignment of this value would 413 /// make a stack adjustment necessary, which could not be undone by the 414 /// callee. 415 unsigned BytesInStackArgArea = 0; 416 417 bool ReturnsVoid = true; 418 419 // A pair of default/requested minimum/maximum flat work group sizes. 420 // Minimum - first, maximum - second. 421 std::pair<unsigned, unsigned> FlatWorkGroupSizes = {0, 0}; 422 423 // A pair of default/requested minimum/maximum number of waves per execution 424 // unit. Minimum - first, maximum - second. 425 std::pair<unsigned, unsigned> WavesPerEU = {0, 0}; 426 427 const AMDGPUGWSResourcePseudoSourceValue GWSResourcePSV; 428 429 private: 430 unsigned NumUserSGPRs = 0; 431 unsigned NumSystemSGPRs = 0; 432 433 bool HasSpilledSGPRs = false; 434 bool HasSpilledVGPRs = false; 435 bool HasNonSpillStackObjects = false; 436 bool IsStackRealigned = false; 437 438 unsigned NumSpilledSGPRs = 0; 439 unsigned NumSpilledVGPRs = 0; 440 441 // Tracks information about user SGPRs that will be setup by hardware which 442 // will apply to all wavefronts of the grid. 443 GCNUserSGPRUsageInfo UserSGPRInfo; 444 445 // Feature bits required for inputs passed in system SGPRs. 446 bool WorkGroupIDX : 1; // Always initialized. 447 bool WorkGroupIDY : 1; 448 bool WorkGroupIDZ : 1; 449 bool WorkGroupInfo : 1; 450 bool LDSKernelId : 1; 451 bool PrivateSegmentWaveByteOffset : 1; 452 453 bool WorkItemIDX : 1; // Always initialized. 454 bool WorkItemIDY : 1; 455 bool WorkItemIDZ : 1; 456 457 // Pointer to where the ABI inserts special kernel arguments separate from the 458 // user arguments. This is an offset from the KernargSegmentPtr. 459 bool ImplicitArgPtr : 1; 460 461 bool MayNeedAGPRs : 1; 462 463 // The hard-wired high half of the address of the global information table 464 // for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since 465 // current hardware only allows a 16 bit value. 466 unsigned GITPtrHigh; 467 468 unsigned HighBitsOf32BitAddress; 469 470 // Flags associated with the virtual registers. 471 IndexedMap<uint8_t, VirtReg2IndexFunctor> VRegFlags; 472 473 // Current recorded maximum possible occupancy. 474 unsigned Occupancy; 475 476 mutable std::optional<bool> UsesAGPRs; 477 478 MCPhysReg getNextUserSGPR() const; 479 480 MCPhysReg getNextSystemSGPR() const; 481 482 // MachineRegisterInfo callback functions to notify events. 483 void MRI_NoteNewVirtualRegister(Register Reg) override; 484 void MRI_NoteCloneVirtualRegister(Register NewReg, Register SrcReg) override; 485 486 public: 487 struct VGPRSpillToAGPR { 488 SmallVector<MCPhysReg, 32> Lanes; 489 bool FullyAllocated = false; 490 bool IsDead = false; 491 }; 492 493 private: 494 // To track virtual VGPR + lane index for each subregister of the SGPR spilled 495 // to frameindex key during SILowerSGPRSpills pass. 496 DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>> 497 SGPRSpillsToVirtualVGPRLanes; 498 // To track physical VGPR + lane index for CSR SGPR spills and special SGPRs 499 // like Frame Pointer identified during PrologEpilogInserter. 500 DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>> 501 SGPRSpillsToPhysicalVGPRLanes; 502 unsigned NumVirtualVGPRSpillLanes = 0; 503 unsigned NumPhysicalVGPRSpillLanes = 0; 504 SmallVector<Register, 2> SpillVGPRs; 505 SmallVector<Register, 2> SpillPhysVGPRs; 506 using WWMSpillsMap = MapVector<Register, int>; 507 // To track the registers used in instructions that can potentially modify the 508 // inactive lanes. The WWM instructions and the writelane instructions for 509 // spilling SGPRs to VGPRs fall under such category of operations. The VGPRs 510 // modified by them should be spilled/restored at function prolog/epilog to 511 // avoid any undesired outcome. Each entry in this map holds a pair of values, 512 // the VGPR and its stack slot index. 513 WWMSpillsMap WWMSpills; 514 515 using ReservedRegSet = SmallSetVector<Register, 8>; 516 // To track the VGPRs reserved for WWM instructions. They get stack slots 517 // later during PrologEpilogInserter and get added into the superset WWMSpills 518 // for actual spilling. A separate set makes the register reserved part and 519 // the serialization easier. 520 ReservedRegSet WWMReservedRegs; 521 522 using PrologEpilogSGPRSpillsMap = 523 DenseMap<Register, PrologEpilogSGPRSaveRestoreInfo>; 524 // To track the SGPR spill method used for a CSR SGPR register during 525 // frame lowering. Even though the SGPR spills are handled during 526 // SILowerSGPRSpills pass, some special handling needed later during the 527 // PrologEpilogInserter. 528 PrologEpilogSGPRSpillsMap PrologEpilogSGPRSpills; 529 530 // To save/restore EXEC MASK around WWM spills and copies. 531 Register SGPRForEXECCopy; 532 533 DenseMap<int, VGPRSpillToAGPR> VGPRToAGPRSpills; 534 535 // AGPRs used for VGPR spills. 536 SmallVector<MCPhysReg, 32> SpillAGPR; 537 538 // VGPRs used for AGPR spills. 539 SmallVector<MCPhysReg, 32> SpillVGPR; 540 541 // Emergency stack slot. Sometimes, we create this before finalizing the stack 542 // frame, so save it here and add it to the RegScavenger later. 543 std::optional<int> ScavengeFI; 544 545 private: 546 Register VGPRForAGPRCopy; 547 548 bool allocateVirtualVGPRForSGPRSpills(MachineFunction &MF, int FI, 549 unsigned LaneIndex); 550 bool allocatePhysicalVGPRForSGPRSpills(MachineFunction &MF, int FI, 551 unsigned LaneIndex); 552 553 public: 554 Register getVGPRForAGPRCopy() const { 555 return VGPRForAGPRCopy; 556 } 557 558 void setVGPRForAGPRCopy(Register NewVGPRForAGPRCopy) { 559 VGPRForAGPRCopy = NewVGPRForAGPRCopy; 560 } 561 562 bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const; 563 564 public: 565 SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI) = default; 566 SIMachineFunctionInfo(const Function &F, const GCNSubtarget *STI); 567 568 MachineFunctionInfo * 569 clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF, 570 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB) 571 const override; 572 573 bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, 574 const MachineFunction &MF, 575 PerFunctionMIParsingState &PFS, 576 SMDiagnostic &Error, SMRange &SourceRange); 577 578 void reserveWWMRegister(Register Reg) { WWMReservedRegs.insert(Reg); } 579 580 SIModeRegisterDefaults getMode() const { return Mode; } 581 582 ArrayRef<SIRegisterInfo::SpilledReg> 583 getSGPRSpillToVirtualVGPRLanes(int FrameIndex) const { 584 auto I = SGPRSpillsToVirtualVGPRLanes.find(FrameIndex); 585 return (I == SGPRSpillsToVirtualVGPRLanes.end()) 586 ? ArrayRef<SIRegisterInfo::SpilledReg>() 587 : ArrayRef(I->second); 588 } 589 590 ArrayRef<Register> getSGPRSpillVGPRs() const { return SpillVGPRs; } 591 const WWMSpillsMap &getWWMSpills() const { return WWMSpills; } 592 const ReservedRegSet &getWWMReservedRegs() const { return WWMReservedRegs; } 593 594 const PrologEpilogSGPRSpillsMap &getPrologEpilogSGPRSpills() const { 595 return PrologEpilogSGPRSpills; 596 } 597 598 GCNUserSGPRUsageInfo &getUserSGPRInfo() { return UserSGPRInfo; } 599 600 const GCNUserSGPRUsageInfo &getUserSGPRInfo() const { return UserSGPRInfo; } 601 602 void addToPrologEpilogSGPRSpills(Register Reg, 603 PrologEpilogSGPRSaveRestoreInfo SI) { 604 PrologEpilogSGPRSpills.insert(std::make_pair(Reg, SI)); 605 } 606 607 // Check if an entry created for \p Reg in PrologEpilogSGPRSpills. Return true 608 // on success and false otherwise. 609 bool hasPrologEpilogSGPRSpillEntry(Register Reg) const { 610 return PrologEpilogSGPRSpills.contains(Reg); 611 } 612 613 // Get the scratch SGPR if allocated to save/restore \p Reg. 614 Register getScratchSGPRCopyDstReg(Register Reg) const { 615 auto I = PrologEpilogSGPRSpills.find(Reg); 616 if (I != PrologEpilogSGPRSpills.end() && 617 I->second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR) 618 return I->second.getReg(); 619 620 return AMDGPU::NoRegister; 621 } 622 623 // Get all scratch SGPRs allocated to copy/restore the SGPR spills. 624 void getAllScratchSGPRCopyDstRegs(SmallVectorImpl<Register> &Regs) const { 625 for (const auto &SI : PrologEpilogSGPRSpills) { 626 if (SI.second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR) 627 Regs.push_back(SI.second.getReg()); 628 } 629 } 630 631 // Check if \p FI is allocated for any SGPR spill to a VGPR lane during PEI. 632 bool checkIndexInPrologEpilogSGPRSpills(int FI) const { 633 return find_if(PrologEpilogSGPRSpills, 634 [FI](const std::pair<Register, 635 PrologEpilogSGPRSaveRestoreInfo> &SI) { 636 return SI.second.getKind() == 637 SGPRSaveKind::SPILL_TO_VGPR_LANE && 638 SI.second.getIndex() == FI; 639 }) != PrologEpilogSGPRSpills.end(); 640 } 641 642 const PrologEpilogSGPRSaveRestoreInfo & 643 getPrologEpilogSGPRSaveRestoreInfo(Register Reg) const { 644 auto I = PrologEpilogSGPRSpills.find(Reg); 645 assert(I != PrologEpilogSGPRSpills.end()); 646 647 return I->second; 648 } 649 650 ArrayRef<SIRegisterInfo::SpilledReg> 651 getSGPRSpillToPhysicalVGPRLanes(int FrameIndex) const { 652 auto I = SGPRSpillsToPhysicalVGPRLanes.find(FrameIndex); 653 return (I == SGPRSpillsToPhysicalVGPRLanes.end()) 654 ? ArrayRef<SIRegisterInfo::SpilledReg>() 655 : ArrayRef(I->second); 656 } 657 658 void setFlag(Register Reg, uint8_t Flag) { 659 assert(Reg.isVirtual()); 660 if (VRegFlags.inBounds(Reg)) 661 VRegFlags[Reg] |= Flag; 662 } 663 664 bool checkFlag(Register Reg, uint8_t Flag) const { 665 if (Reg.isPhysical()) 666 return false; 667 668 return VRegFlags.inBounds(Reg) && VRegFlags[Reg] & Flag; 669 } 670 671 bool hasVRegFlags() { return VRegFlags.size(); } 672 673 void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size = 4, 674 Align Alignment = Align(4)); 675 676 void splitWWMSpillRegisters( 677 MachineFunction &MF, 678 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs, 679 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const; 680 681 ArrayRef<MCPhysReg> getAGPRSpillVGPRs() const { 682 return SpillAGPR; 683 } 684 685 Register getSGPRForEXECCopy() const { return SGPRForEXECCopy; } 686 687 void setSGPRForEXECCopy(Register Reg) { SGPRForEXECCopy = Reg; } 688 689 ArrayRef<MCPhysReg> getVGPRSpillAGPRs() const { 690 return SpillVGPR; 691 } 692 693 MCPhysReg getVGPRToAGPRSpill(int FrameIndex, unsigned Lane) const { 694 auto I = VGPRToAGPRSpills.find(FrameIndex); 695 return (I == VGPRToAGPRSpills.end()) ? (MCPhysReg)AMDGPU::NoRegister 696 : I->second.Lanes[Lane]; 697 } 698 699 void setVGPRToAGPRSpillDead(int FrameIndex) { 700 auto I = VGPRToAGPRSpills.find(FrameIndex); 701 if (I != VGPRToAGPRSpills.end()) 702 I->second.IsDead = true; 703 } 704 705 bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, 706 bool IsPrologEpilog = false); 707 bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR); 708 709 /// If \p ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill 710 /// to the default stack. 711 bool removeDeadFrameIndices(MachineFrameInfo &MFI, 712 bool ResetSGPRSpillStackIDs); 713 714 int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI); 715 std::optional<int> getOptionalScavengeFI() const { return ScavengeFI; } 716 717 unsigned getBytesInStackArgArea() const { 718 return BytesInStackArgArea; 719 } 720 721 void setBytesInStackArgArea(unsigned Bytes) { 722 BytesInStackArgArea = Bytes; 723 } 724 725 // Add user SGPRs. 726 Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI); 727 Register addDispatchPtr(const SIRegisterInfo &TRI); 728 Register addQueuePtr(const SIRegisterInfo &TRI); 729 Register addKernargSegmentPtr(const SIRegisterInfo &TRI); 730 Register addDispatchID(const SIRegisterInfo &TRI); 731 Register addFlatScratchInit(const SIRegisterInfo &TRI); 732 Register addImplicitBufferPtr(const SIRegisterInfo &TRI); 733 Register addLDSKernelId(); 734 SmallVectorImpl<MCRegister> * 735 addPreloadedKernArg(const SIRegisterInfo &TRI, const TargetRegisterClass *RC, 736 unsigned AllocSizeDWord, int KernArgIdx, 737 int PaddingSGPRs); 738 739 /// Increment user SGPRs used for padding the argument list only. 740 Register addReservedUserSGPR() { 741 Register Next = getNextUserSGPR(); 742 ++NumUserSGPRs; 743 return Next; 744 } 745 746 // Add system SGPRs. 747 Register addWorkGroupIDX(bool HasArchitectedSGPRs) { 748 Register Reg = 749 HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP9 : getNextSystemSGPR(); 750 ArgInfo.WorkGroupIDX = ArgDescriptor::createRegister(Reg); 751 if (!HasArchitectedSGPRs) 752 NumSystemSGPRs += 1; 753 754 return ArgInfo.WorkGroupIDX.getRegister(); 755 } 756 757 Register addWorkGroupIDY(bool HasArchitectedSGPRs) { 758 Register Reg = 759 HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP7 : getNextSystemSGPR(); 760 unsigned Mask = HasArchitectedSGPRs && hasWorkGroupIDZ() ? 0xffff : ~0u; 761 ArgInfo.WorkGroupIDY = ArgDescriptor::createRegister(Reg, Mask); 762 if (!HasArchitectedSGPRs) 763 NumSystemSGPRs += 1; 764 765 return ArgInfo.WorkGroupIDY.getRegister(); 766 } 767 768 Register addWorkGroupIDZ(bool HasArchitectedSGPRs) { 769 Register Reg = 770 HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP7 : getNextSystemSGPR(); 771 unsigned Mask = HasArchitectedSGPRs ? 0xffff << 16 : ~0u; 772 ArgInfo.WorkGroupIDZ = ArgDescriptor::createRegister(Reg, Mask); 773 if (!HasArchitectedSGPRs) 774 NumSystemSGPRs += 1; 775 776 return ArgInfo.WorkGroupIDZ.getRegister(); 777 } 778 779 Register addWorkGroupInfo() { 780 ArgInfo.WorkGroupInfo = ArgDescriptor::createRegister(getNextSystemSGPR()); 781 NumSystemSGPRs += 1; 782 return ArgInfo.WorkGroupInfo.getRegister(); 783 } 784 785 bool hasLDSKernelId() const { return LDSKernelId; } 786 787 // Add special VGPR inputs 788 void setWorkItemIDX(ArgDescriptor Arg) { 789 ArgInfo.WorkItemIDX = Arg; 790 } 791 792 void setWorkItemIDY(ArgDescriptor Arg) { 793 ArgInfo.WorkItemIDY = Arg; 794 } 795 796 void setWorkItemIDZ(ArgDescriptor Arg) { 797 ArgInfo.WorkItemIDZ = Arg; 798 } 799 800 Register addPrivateSegmentWaveByteOffset() { 801 ArgInfo.PrivateSegmentWaveByteOffset 802 = ArgDescriptor::createRegister(getNextSystemSGPR()); 803 NumSystemSGPRs += 1; 804 return ArgInfo.PrivateSegmentWaveByteOffset.getRegister(); 805 } 806 807 void setPrivateSegmentWaveByteOffset(Register Reg) { 808 ArgInfo.PrivateSegmentWaveByteOffset = ArgDescriptor::createRegister(Reg); 809 } 810 811 bool hasWorkGroupIDX() const { 812 return WorkGroupIDX; 813 } 814 815 bool hasWorkGroupIDY() const { 816 return WorkGroupIDY; 817 } 818 819 bool hasWorkGroupIDZ() const { 820 return WorkGroupIDZ; 821 } 822 823 bool hasWorkGroupInfo() const { 824 return WorkGroupInfo; 825 } 826 827 bool hasPrivateSegmentWaveByteOffset() const { 828 return PrivateSegmentWaveByteOffset; 829 } 830 831 bool hasWorkItemIDX() const { 832 return WorkItemIDX; 833 } 834 835 bool hasWorkItemIDY() const { 836 return WorkItemIDY; 837 } 838 839 bool hasWorkItemIDZ() const { 840 return WorkItemIDZ; 841 } 842 843 bool hasImplicitArgPtr() const { 844 return ImplicitArgPtr; 845 } 846 847 AMDGPUFunctionArgInfo &getArgInfo() { 848 return ArgInfo; 849 } 850 851 const AMDGPUFunctionArgInfo &getArgInfo() const { 852 return ArgInfo; 853 } 854 855 std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT> 856 getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const { 857 return ArgInfo.getPreloadedValue(Value); 858 } 859 860 MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const { 861 auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value)); 862 return Arg ? Arg->getRegister() : MCRegister(); 863 } 864 865 unsigned getGITPtrHigh() const { 866 return GITPtrHigh; 867 } 868 869 Register getGITPtrLoReg(const MachineFunction &MF) const; 870 871 uint32_t get32BitAddressHighBits() const { 872 return HighBitsOf32BitAddress; 873 } 874 875 unsigned getNumUserSGPRs() const { 876 return NumUserSGPRs; 877 } 878 879 unsigned getNumPreloadedSGPRs() const { 880 return NumUserSGPRs + NumSystemSGPRs; 881 } 882 883 unsigned getNumKernargPreloadedSGPRs() const { 884 return UserSGPRInfo.getNumKernargPreloadSGPRs(); 885 } 886 887 Register getPrivateSegmentWaveByteOffsetSystemSGPR() const { 888 return ArgInfo.PrivateSegmentWaveByteOffset.getRegister(); 889 } 890 891 /// Returns the physical register reserved for use as the resource 892 /// descriptor for scratch accesses. 893 Register getScratchRSrcReg() const { 894 return ScratchRSrcReg; 895 } 896 897 void setScratchRSrcReg(Register Reg) { 898 assert(Reg != 0 && "Should never be unset"); 899 ScratchRSrcReg = Reg; 900 } 901 902 Register getFrameOffsetReg() const { 903 return FrameOffsetReg; 904 } 905 906 void setFrameOffsetReg(Register Reg) { 907 assert(Reg != 0 && "Should never be unset"); 908 FrameOffsetReg = Reg; 909 } 910 911 void setStackPtrOffsetReg(Register Reg) { 912 assert(Reg != 0 && "Should never be unset"); 913 StackPtrOffsetReg = Reg; 914 } 915 916 void setLongBranchReservedReg(Register Reg) { LongBranchReservedReg = Reg; } 917 918 // Note the unset value for this is AMDGPU::SP_REG rather than 919 // NoRegister. This is mostly a workaround for MIR tests where state that 920 // can't be directly computed from the function is not preserved in serialized 921 // MIR. 922 Register getStackPtrOffsetReg() const { 923 return StackPtrOffsetReg; 924 } 925 926 Register getLongBranchReservedReg() const { return LongBranchReservedReg; } 927 928 Register getQueuePtrUserSGPR() const { 929 return ArgInfo.QueuePtr.getRegister(); 930 } 931 932 Register getImplicitBufferPtrUserSGPR() const { 933 return ArgInfo.ImplicitBufferPtr.getRegister(); 934 } 935 936 bool hasSpilledSGPRs() const { 937 return HasSpilledSGPRs; 938 } 939 940 void setHasSpilledSGPRs(bool Spill = true) { 941 HasSpilledSGPRs = Spill; 942 } 943 944 bool hasSpilledVGPRs() const { 945 return HasSpilledVGPRs; 946 } 947 948 void setHasSpilledVGPRs(bool Spill = true) { 949 HasSpilledVGPRs = Spill; 950 } 951 952 bool hasNonSpillStackObjects() const { 953 return HasNonSpillStackObjects; 954 } 955 956 void setHasNonSpillStackObjects(bool StackObject = true) { 957 HasNonSpillStackObjects = StackObject; 958 } 959 960 bool isStackRealigned() const { 961 return IsStackRealigned; 962 } 963 964 void setIsStackRealigned(bool Realigned = true) { 965 IsStackRealigned = Realigned; 966 } 967 968 unsigned getNumSpilledSGPRs() const { 969 return NumSpilledSGPRs; 970 } 971 972 unsigned getNumSpilledVGPRs() const { 973 return NumSpilledVGPRs; 974 } 975 976 void addToSpilledSGPRs(unsigned num) { 977 NumSpilledSGPRs += num; 978 } 979 980 void addToSpilledVGPRs(unsigned num) { 981 NumSpilledVGPRs += num; 982 } 983 984 unsigned getPSInputAddr() const { 985 return PSInputAddr; 986 } 987 988 unsigned getPSInputEnable() const { 989 return PSInputEnable; 990 } 991 992 bool isPSInputAllocated(unsigned Index) const { 993 return PSInputAddr & (1 << Index); 994 } 995 996 void markPSInputAllocated(unsigned Index) { 997 PSInputAddr |= 1 << Index; 998 } 999 1000 void markPSInputEnabled(unsigned Index) { 1001 PSInputEnable |= 1 << Index; 1002 } 1003 1004 bool returnsVoid() const { 1005 return ReturnsVoid; 1006 } 1007 1008 void setIfReturnsVoid(bool Value) { 1009 ReturnsVoid = Value; 1010 } 1011 1012 /// \returns A pair of default/requested minimum/maximum flat work group sizes 1013 /// for this function. 1014 std::pair<unsigned, unsigned> getFlatWorkGroupSizes() const { 1015 return FlatWorkGroupSizes; 1016 } 1017 1018 /// \returns Default/requested minimum flat work group size for this function. 1019 unsigned getMinFlatWorkGroupSize() const { 1020 return FlatWorkGroupSizes.first; 1021 } 1022 1023 /// \returns Default/requested maximum flat work group size for this function. 1024 unsigned getMaxFlatWorkGroupSize() const { 1025 return FlatWorkGroupSizes.second; 1026 } 1027 1028 /// \returns A pair of default/requested minimum/maximum number of waves per 1029 /// execution unit. 1030 std::pair<unsigned, unsigned> getWavesPerEU() const { 1031 return WavesPerEU; 1032 } 1033 1034 /// \returns Default/requested minimum number of waves per execution unit. 1035 unsigned getMinWavesPerEU() const { 1036 return WavesPerEU.first; 1037 } 1038 1039 /// \returns Default/requested maximum number of waves per execution unit. 1040 unsigned getMaxWavesPerEU() const { 1041 return WavesPerEU.second; 1042 } 1043 1044 /// \returns SGPR used for \p Dim's work group ID. 1045 Register getWorkGroupIDSGPR(unsigned Dim) const { 1046 switch (Dim) { 1047 case 0: 1048 assert(hasWorkGroupIDX()); 1049 return ArgInfo.WorkGroupIDX.getRegister(); 1050 case 1: 1051 assert(hasWorkGroupIDY()); 1052 return ArgInfo.WorkGroupIDY.getRegister(); 1053 case 2: 1054 assert(hasWorkGroupIDZ()); 1055 return ArgInfo.WorkGroupIDZ.getRegister(); 1056 } 1057 llvm_unreachable("unexpected dimension"); 1058 } 1059 1060 const AMDGPUGWSResourcePseudoSourceValue * 1061 getGWSPSV(const AMDGPUTargetMachine &TM) { 1062 return &GWSResourcePSV; 1063 } 1064 1065 unsigned getOccupancy() const { 1066 return Occupancy; 1067 } 1068 1069 unsigned getMinAllowedOccupancy() const { 1070 if (!isMemoryBound() && !needsWaveLimiter()) 1071 return Occupancy; 1072 return (Occupancy < 4) ? Occupancy : 4; 1073 } 1074 1075 void limitOccupancy(const MachineFunction &MF); 1076 1077 void limitOccupancy(unsigned Limit) { 1078 if (Occupancy > Limit) 1079 Occupancy = Limit; 1080 } 1081 1082 void increaseOccupancy(const MachineFunction &MF, unsigned Limit) { 1083 if (Occupancy < Limit) 1084 Occupancy = Limit; 1085 limitOccupancy(MF); 1086 } 1087 1088 bool mayNeedAGPRs() const { 1089 return MayNeedAGPRs; 1090 } 1091 1092 // \returns true if a function has a use of AGPRs via inline asm or 1093 // has a call which may use it. 1094 bool mayUseAGPRs(const Function &F) const; 1095 1096 // \returns true if a function needs or may need AGPRs. 1097 bool usesAGPRs(const MachineFunction &MF) const; 1098 }; 1099 1100 } // end namespace llvm 1101 1102 #endif // LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H 1103