1 //==- SIMachineFunctionInfo.h - SIMachineFunctionInfo interface --*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifndef LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H 14 #define LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H 15 16 #include "AMDGPUArgumentUsageInfo.h" 17 #include "AMDGPUMachineFunction.h" 18 #include "AMDGPUTargetMachine.h" 19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 20 #include "SIInstrInfo.h" 21 #include "SIModeRegisterDefaults.h" 22 #include "llvm/ADT/SetVector.h" 23 #include "llvm/CodeGen/MIRYamlMapping.h" 24 #include "llvm/CodeGen/PseudoSourceValue.h" 25 #include "llvm/Support/raw_ostream.h" 26 #include <optional> 27 28 namespace llvm { 29 30 class MachineFrameInfo; 31 class MachineFunction; 32 class SIMachineFunctionInfo; 33 class SIRegisterInfo; 34 class TargetRegisterClass; 35 36 class AMDGPUPseudoSourceValue : public PseudoSourceValue { 37 public: 38 enum AMDGPUPSVKind : unsigned { 39 PSVImage = PseudoSourceValue::TargetCustom, 40 GWSResource 41 }; 42 43 protected: 44 AMDGPUPseudoSourceValue(unsigned Kind, const AMDGPUTargetMachine &TM) 45 : PseudoSourceValue(Kind, TM) {} 46 47 public: 48 bool isConstant(const MachineFrameInfo *) const override { 49 // This should probably be true for most images, but we will start by being 50 // conservative. 51 return false; 52 } 53 54 bool isAliased(const MachineFrameInfo *) const override { 55 return true; 56 } 57 58 bool mayAlias(const MachineFrameInfo *) const override { 59 return true; 60 } 61 }; 62 63 class AMDGPUGWSResourcePseudoSourceValue final : public AMDGPUPseudoSourceValue { 64 public: 65 explicit AMDGPUGWSResourcePseudoSourceValue(const AMDGPUTargetMachine &TM) 66 : AMDGPUPseudoSourceValue(GWSResource, TM) {} 67 68 static bool classof(const PseudoSourceValue *V) { 69 return V->kind() == GWSResource; 70 } 71 72 // These are inaccessible memory from IR. 73 bool isAliased(const MachineFrameInfo *) const override { 74 return false; 75 } 76 77 // These are inaccessible memory from IR. 78 bool mayAlias(const MachineFrameInfo *) const override { 79 return false; 80 } 81 82 void printCustom(raw_ostream &OS) const override { 83 OS << "GWSResource"; 84 } 85 }; 86 87 namespace yaml { 88 89 struct SIArgument { 90 bool IsRegister; 91 union { 92 StringValue RegisterName; 93 unsigned StackOffset; 94 }; 95 std::optional<unsigned> Mask; 96 97 // Default constructor, which creates a stack argument. 98 SIArgument() : IsRegister(false), StackOffset(0) {} 99 SIArgument(const SIArgument &Other) { 100 IsRegister = Other.IsRegister; 101 if (IsRegister) { 102 ::new ((void *)std::addressof(RegisterName)) 103 StringValue(Other.RegisterName); 104 } else 105 StackOffset = Other.StackOffset; 106 Mask = Other.Mask; 107 } 108 SIArgument &operator=(const SIArgument &Other) { 109 IsRegister = Other.IsRegister; 110 if (IsRegister) { 111 ::new ((void *)std::addressof(RegisterName)) 112 StringValue(Other.RegisterName); 113 } else 114 StackOffset = Other.StackOffset; 115 Mask = Other.Mask; 116 return *this; 117 } 118 ~SIArgument() { 119 if (IsRegister) 120 RegisterName.~StringValue(); 121 } 122 123 // Helper to create a register or stack argument. 124 static inline SIArgument createArgument(bool IsReg) { 125 if (IsReg) 126 return SIArgument(IsReg); 127 return SIArgument(); 128 } 129 130 private: 131 // Construct a register argument. 132 SIArgument(bool) : IsRegister(true), RegisterName() {} 133 }; 134 135 template <> struct MappingTraits<SIArgument> { 136 static void mapping(IO &YamlIO, SIArgument &A) { 137 if (YamlIO.outputting()) { 138 if (A.IsRegister) 139 YamlIO.mapRequired("reg", A.RegisterName); 140 else 141 YamlIO.mapRequired("offset", A.StackOffset); 142 } else { 143 auto Keys = YamlIO.keys(); 144 if (is_contained(Keys, "reg")) { 145 A = SIArgument::createArgument(true); 146 YamlIO.mapRequired("reg", A.RegisterName); 147 } else if (is_contained(Keys, "offset")) 148 YamlIO.mapRequired("offset", A.StackOffset); 149 else 150 YamlIO.setError("missing required key 'reg' or 'offset'"); 151 } 152 YamlIO.mapOptional("mask", A.Mask); 153 } 154 static const bool flow = true; 155 }; 156 157 struct SIArgumentInfo { 158 std::optional<SIArgument> PrivateSegmentBuffer; 159 std::optional<SIArgument> DispatchPtr; 160 std::optional<SIArgument> QueuePtr; 161 std::optional<SIArgument> KernargSegmentPtr; 162 std::optional<SIArgument> DispatchID; 163 std::optional<SIArgument> FlatScratchInit; 164 std::optional<SIArgument> PrivateSegmentSize; 165 166 std::optional<SIArgument> WorkGroupIDX; 167 std::optional<SIArgument> WorkGroupIDY; 168 std::optional<SIArgument> WorkGroupIDZ; 169 std::optional<SIArgument> WorkGroupInfo; 170 std::optional<SIArgument> LDSKernelId; 171 std::optional<SIArgument> PrivateSegmentWaveByteOffset; 172 173 std::optional<SIArgument> ImplicitArgPtr; 174 std::optional<SIArgument> ImplicitBufferPtr; 175 176 std::optional<SIArgument> WorkItemIDX; 177 std::optional<SIArgument> WorkItemIDY; 178 std::optional<SIArgument> WorkItemIDZ; 179 }; 180 181 template <> struct MappingTraits<SIArgumentInfo> { 182 static void mapping(IO &YamlIO, SIArgumentInfo &AI) { 183 YamlIO.mapOptional("privateSegmentBuffer", AI.PrivateSegmentBuffer); 184 YamlIO.mapOptional("dispatchPtr", AI.DispatchPtr); 185 YamlIO.mapOptional("queuePtr", AI.QueuePtr); 186 YamlIO.mapOptional("kernargSegmentPtr", AI.KernargSegmentPtr); 187 YamlIO.mapOptional("dispatchID", AI.DispatchID); 188 YamlIO.mapOptional("flatScratchInit", AI.FlatScratchInit); 189 YamlIO.mapOptional("privateSegmentSize", AI.PrivateSegmentSize); 190 191 YamlIO.mapOptional("workGroupIDX", AI.WorkGroupIDX); 192 YamlIO.mapOptional("workGroupIDY", AI.WorkGroupIDY); 193 YamlIO.mapOptional("workGroupIDZ", AI.WorkGroupIDZ); 194 YamlIO.mapOptional("workGroupInfo", AI.WorkGroupInfo); 195 YamlIO.mapOptional("LDSKernelId", AI.LDSKernelId); 196 YamlIO.mapOptional("privateSegmentWaveByteOffset", 197 AI.PrivateSegmentWaveByteOffset); 198 199 YamlIO.mapOptional("implicitArgPtr", AI.ImplicitArgPtr); 200 YamlIO.mapOptional("implicitBufferPtr", AI.ImplicitBufferPtr); 201 202 YamlIO.mapOptional("workItemIDX", AI.WorkItemIDX); 203 YamlIO.mapOptional("workItemIDY", AI.WorkItemIDY); 204 YamlIO.mapOptional("workItemIDZ", AI.WorkItemIDZ); 205 } 206 }; 207 208 // Default to default mode for default calling convention. 209 struct SIMode { 210 bool IEEE = true; 211 bool DX10Clamp = true; 212 bool FP32InputDenormals = true; 213 bool FP32OutputDenormals = true; 214 bool FP64FP16InputDenormals = true; 215 bool FP64FP16OutputDenormals = true; 216 217 SIMode() = default; 218 219 SIMode(const SIModeRegisterDefaults &Mode) { 220 IEEE = Mode.IEEE; 221 DX10Clamp = Mode.DX10Clamp; 222 FP32InputDenormals = Mode.FP32Denormals.Input != DenormalMode::PreserveSign; 223 FP32OutputDenormals = 224 Mode.FP32Denormals.Output != DenormalMode::PreserveSign; 225 FP64FP16InputDenormals = 226 Mode.FP64FP16Denormals.Input != DenormalMode::PreserveSign; 227 FP64FP16OutputDenormals = 228 Mode.FP64FP16Denormals.Output != DenormalMode::PreserveSign; 229 } 230 231 bool operator ==(const SIMode Other) const { 232 return IEEE == Other.IEEE && 233 DX10Clamp == Other.DX10Clamp && 234 FP32InputDenormals == Other.FP32InputDenormals && 235 FP32OutputDenormals == Other.FP32OutputDenormals && 236 FP64FP16InputDenormals == Other.FP64FP16InputDenormals && 237 FP64FP16OutputDenormals == Other.FP64FP16OutputDenormals; 238 } 239 }; 240 241 template <> struct MappingTraits<SIMode> { 242 static void mapping(IO &YamlIO, SIMode &Mode) { 243 YamlIO.mapOptional("ieee", Mode.IEEE, true); 244 YamlIO.mapOptional("dx10-clamp", Mode.DX10Clamp, true); 245 YamlIO.mapOptional("fp32-input-denormals", Mode.FP32InputDenormals, true); 246 YamlIO.mapOptional("fp32-output-denormals", Mode.FP32OutputDenormals, true); 247 YamlIO.mapOptional("fp64-fp16-input-denormals", Mode.FP64FP16InputDenormals, true); 248 YamlIO.mapOptional("fp64-fp16-output-denormals", Mode.FP64FP16OutputDenormals, true); 249 } 250 }; 251 252 struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo { 253 uint64_t ExplicitKernArgSize = 0; 254 Align MaxKernArgAlign; 255 uint32_t LDSSize = 0; 256 uint32_t GDSSize = 0; 257 Align DynLDSAlign; 258 bool IsEntryFunction = false; 259 bool NoSignedZerosFPMath = false; 260 bool MemoryBound = false; 261 bool WaveLimiter = false; 262 bool HasSpilledSGPRs = false; 263 bool HasSpilledVGPRs = false; 264 uint32_t HighBitsOf32BitAddress = 0; 265 266 // TODO: 10 may be a better default since it's the maximum. 267 unsigned Occupancy = 0; 268 269 SmallVector<StringValue> WWMReservedRegs; 270 271 StringValue ScratchRSrcReg = "$private_rsrc_reg"; 272 StringValue FrameOffsetReg = "$fp_reg"; 273 StringValue StackPtrOffsetReg = "$sp_reg"; 274 275 unsigned BytesInStackArgArea = 0; 276 bool ReturnsVoid = true; 277 278 std::optional<SIArgumentInfo> ArgInfo; 279 280 unsigned PSInputAddr = 0; 281 unsigned PSInputEnable = 0; 282 283 SIMode Mode; 284 std::optional<FrameIndex> ScavengeFI; 285 StringValue VGPRForAGPRCopy; 286 StringValue SGPRForEXECCopy; 287 StringValue LongBranchReservedReg; 288 289 SIMachineFunctionInfo() = default; 290 SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo &, 291 const TargetRegisterInfo &TRI, 292 const llvm::MachineFunction &MF); 293 294 void mappingImpl(yaml::IO &YamlIO) override; 295 ~SIMachineFunctionInfo() = default; 296 }; 297 298 template <> struct MappingTraits<SIMachineFunctionInfo> { 299 static void mapping(IO &YamlIO, SIMachineFunctionInfo &MFI) { 300 YamlIO.mapOptional("explicitKernArgSize", MFI.ExplicitKernArgSize, 301 UINT64_C(0)); 302 YamlIO.mapOptional("maxKernArgAlign", MFI.MaxKernArgAlign); 303 YamlIO.mapOptional("ldsSize", MFI.LDSSize, 0u); 304 YamlIO.mapOptional("gdsSize", MFI.GDSSize, 0u); 305 YamlIO.mapOptional("dynLDSAlign", MFI.DynLDSAlign, Align()); 306 YamlIO.mapOptional("isEntryFunction", MFI.IsEntryFunction, false); 307 YamlIO.mapOptional("noSignedZerosFPMath", MFI.NoSignedZerosFPMath, false); 308 YamlIO.mapOptional("memoryBound", MFI.MemoryBound, false); 309 YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false); 310 YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false); 311 YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false); 312 YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg, 313 StringValue("$private_rsrc_reg")); 314 YamlIO.mapOptional("frameOffsetReg", MFI.FrameOffsetReg, 315 StringValue("$fp_reg")); 316 YamlIO.mapOptional("stackPtrOffsetReg", MFI.StackPtrOffsetReg, 317 StringValue("$sp_reg")); 318 YamlIO.mapOptional("bytesInStackArgArea", MFI.BytesInStackArgArea, 0u); 319 YamlIO.mapOptional("returnsVoid", MFI.ReturnsVoid, true); 320 YamlIO.mapOptional("argumentInfo", MFI.ArgInfo); 321 YamlIO.mapOptional("psInputAddr", MFI.PSInputAddr, 0u); 322 YamlIO.mapOptional("psInputEnable", MFI.PSInputEnable, 0u); 323 YamlIO.mapOptional("mode", MFI.Mode, SIMode()); 324 YamlIO.mapOptional("highBitsOf32BitAddress", 325 MFI.HighBitsOf32BitAddress, 0u); 326 YamlIO.mapOptional("occupancy", MFI.Occupancy, 0); 327 YamlIO.mapOptional("wwmReservedRegs", MFI.WWMReservedRegs); 328 YamlIO.mapOptional("scavengeFI", MFI.ScavengeFI); 329 YamlIO.mapOptional("vgprForAGPRCopy", MFI.VGPRForAGPRCopy, 330 StringValue()); // Don't print out when it's empty. 331 YamlIO.mapOptional("sgprForEXECCopy", MFI.SGPRForEXECCopy, 332 StringValue()); // Don't print out when it's empty. 333 YamlIO.mapOptional("longBranchReservedReg", MFI.LongBranchReservedReg, 334 StringValue()); 335 } 336 }; 337 338 } // end namespace yaml 339 340 // A CSR SGPR value can be preserved inside a callee using one of the following 341 // methods. 342 // 1. Copy to an unused scratch SGPR. 343 // 2. Spill to a VGPR lane. 344 // 3. Spill to memory via. a scratch VGPR. 345 // class PrologEpilogSGPRSaveRestoreInfo represents the save/restore method used 346 // for an SGPR at function prolog/epilog. 347 enum class SGPRSaveKind : uint8_t { 348 COPY_TO_SCRATCH_SGPR, 349 SPILL_TO_VGPR_LANE, 350 SPILL_TO_MEM 351 }; 352 353 class PrologEpilogSGPRSaveRestoreInfo { 354 SGPRSaveKind Kind; 355 union { 356 int Index; 357 Register Reg; 358 }; 359 360 public: 361 PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, int I) : Kind(K), Index(I) {} 362 PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, Register R) 363 : Kind(K), Reg(R) {} 364 Register getReg() const { return Reg; } 365 int getIndex() const { return Index; } 366 SGPRSaveKind getKind() const { return Kind; } 367 }; 368 369 /// This class keeps track of the SPI_SP_INPUT_ADDR config register, which 370 /// tells the hardware which interpolation parameters to load. 371 class SIMachineFunctionInfo final : public AMDGPUMachineFunction, 372 private MachineRegisterInfo::Delegate { 373 friend class GCNTargetMachine; 374 375 // State of MODE register, assumed FP mode. 376 SIModeRegisterDefaults Mode; 377 378 // Registers that may be reserved for spilling purposes. These may be the same 379 // as the input registers. 380 Register ScratchRSrcReg = AMDGPU::PRIVATE_RSRC_REG; 381 382 // This is the unswizzled offset from the current dispatch's scratch wave 383 // base to the beginning of the current function's frame. 384 Register FrameOffsetReg = AMDGPU::FP_REG; 385 386 // This is an ABI register used in the non-entry calling convention to 387 // communicate the unswizzled offset from the current dispatch's scratch wave 388 // base to the beginning of the new function's frame. 389 Register StackPtrOffsetReg = AMDGPU::SP_REG; 390 391 // Registers that may be reserved when RA doesn't allocate enough 392 // registers to plan for the case where an indirect branch ends up 393 // being needed during branch relaxation. 394 Register LongBranchReservedReg; 395 396 AMDGPUFunctionArgInfo ArgInfo; 397 398 // Graphics info. 399 unsigned PSInputAddr = 0; 400 unsigned PSInputEnable = 0; 401 402 /// Number of bytes of arguments this function has on the stack. If the callee 403 /// is expected to restore the argument stack this should be a multiple of 16, 404 /// all usable during a tail call. 405 /// 406 /// The alternative would forbid tail call optimisation in some cases: if we 407 /// want to transfer control from a function with 8-bytes of stack-argument 408 /// space to a function with 16-bytes then misalignment of this value would 409 /// make a stack adjustment necessary, which could not be undone by the 410 /// callee. 411 unsigned BytesInStackArgArea = 0; 412 413 bool ReturnsVoid = true; 414 415 // A pair of default/requested minimum/maximum flat work group sizes. 416 // Minimum - first, maximum - second. 417 std::pair<unsigned, unsigned> FlatWorkGroupSizes = {0, 0}; 418 419 // A pair of default/requested minimum/maximum number of waves per execution 420 // unit. Minimum - first, maximum - second. 421 std::pair<unsigned, unsigned> WavesPerEU = {0, 0}; 422 423 const AMDGPUGWSResourcePseudoSourceValue GWSResourcePSV; 424 425 private: 426 unsigned NumUserSGPRs = 0; 427 unsigned NumSystemSGPRs = 0; 428 429 bool HasSpilledSGPRs = false; 430 bool HasSpilledVGPRs = false; 431 bool HasNonSpillStackObjects = false; 432 bool IsStackRealigned = false; 433 434 unsigned NumSpilledSGPRs = 0; 435 unsigned NumSpilledVGPRs = 0; 436 437 // Feature bits required for inputs passed in user SGPRs. 438 bool PrivateSegmentBuffer : 1; 439 bool DispatchPtr : 1; 440 bool QueuePtr : 1; 441 bool KernargSegmentPtr : 1; 442 bool DispatchID : 1; 443 bool FlatScratchInit : 1; 444 445 // Feature bits required for inputs passed in system SGPRs. 446 bool WorkGroupIDX : 1; // Always initialized. 447 bool WorkGroupIDY : 1; 448 bool WorkGroupIDZ : 1; 449 bool WorkGroupInfo : 1; 450 bool LDSKernelId : 1; 451 bool PrivateSegmentWaveByteOffset : 1; 452 453 bool WorkItemIDX : 1; // Always initialized. 454 bool WorkItemIDY : 1; 455 bool WorkItemIDZ : 1; 456 457 // Private memory buffer 458 // Compute directly in sgpr[0:1] 459 // Other shaders indirect 64-bits at sgpr[0:1] 460 bool ImplicitBufferPtr : 1; 461 462 // Pointer to where the ABI inserts special kernel arguments separate from the 463 // user arguments. This is an offset from the KernargSegmentPtr. 464 bool ImplicitArgPtr : 1; 465 466 bool MayNeedAGPRs : 1; 467 468 // The hard-wired high half of the address of the global information table 469 // for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since 470 // current hardware only allows a 16 bit value. 471 unsigned GITPtrHigh; 472 473 unsigned HighBitsOf32BitAddress; 474 475 // Flags associated with the virtual registers. 476 IndexedMap<uint8_t, VirtReg2IndexFunctor> VRegFlags; 477 478 // Current recorded maximum possible occupancy. 479 unsigned Occupancy; 480 481 mutable std::optional<bool> UsesAGPRs; 482 483 MCPhysReg getNextUserSGPR() const; 484 485 MCPhysReg getNextSystemSGPR() const; 486 487 // MachineRegisterInfo callback functions to notify events. 488 void MRI_NoteNewVirtualRegister(Register Reg) override; 489 void MRI_NoteCloneVirtualRegister(Register NewReg, Register SrcReg) override; 490 491 public: 492 struct VGPRSpillToAGPR { 493 SmallVector<MCPhysReg, 32> Lanes; 494 bool FullyAllocated = false; 495 bool IsDead = false; 496 }; 497 498 private: 499 // To track virtual VGPR + lane index for each subregister of the SGPR spilled 500 // to frameindex key during SILowerSGPRSpills pass. 501 DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>> 502 SGPRSpillsToVirtualVGPRLanes; 503 // To track physical VGPR + lane index for CSR SGPR spills and special SGPRs 504 // like Frame Pointer identified during PrologEpilogInserter. 505 DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>> 506 SGPRSpillsToPhysicalVGPRLanes; 507 unsigned NumVirtualVGPRSpillLanes = 0; 508 unsigned NumPhysicalVGPRSpillLanes = 0; 509 SmallVector<Register, 2> SpillVGPRs; 510 using WWMSpillsMap = MapVector<Register, int>; 511 // To track the registers used in instructions that can potentially modify the 512 // inactive lanes. The WWM instructions and the writelane instructions for 513 // spilling SGPRs to VGPRs fall under such category of operations. The VGPRs 514 // modified by them should be spilled/restored at function prolog/epilog to 515 // avoid any undesired outcome. Each entry in this map holds a pair of values, 516 // the VGPR and its stack slot index. 517 WWMSpillsMap WWMSpills; 518 519 using ReservedRegSet = SmallSetVector<Register, 8>; 520 // To track the VGPRs reserved for WWM instructions. They get stack slots 521 // later during PrologEpilogInserter and get added into the superset WWMSpills 522 // for actual spilling. A separate set makes the register reserved part and 523 // the serialization easier. 524 ReservedRegSet WWMReservedRegs; 525 526 using PrologEpilogSGPRSpillsMap = 527 DenseMap<Register, PrologEpilogSGPRSaveRestoreInfo>; 528 // To track the SGPR spill method used for a CSR SGPR register during 529 // frame lowering. Even though the SGPR spills are handled during 530 // SILowerSGPRSpills pass, some special handling needed later during the 531 // PrologEpilogInserter. 532 PrologEpilogSGPRSpillsMap PrologEpilogSGPRSpills; 533 534 // To save/restore EXEC MASK around WWM spills and copies. 535 Register SGPRForEXECCopy; 536 537 DenseMap<int, VGPRSpillToAGPR> VGPRToAGPRSpills; 538 539 // AGPRs used for VGPR spills. 540 SmallVector<MCPhysReg, 32> SpillAGPR; 541 542 // VGPRs used for AGPR spills. 543 SmallVector<MCPhysReg, 32> SpillVGPR; 544 545 // Emergency stack slot. Sometimes, we create this before finalizing the stack 546 // frame, so save it here and add it to the RegScavenger later. 547 std::optional<int> ScavengeFI; 548 549 private: 550 Register VGPRForAGPRCopy; 551 552 bool allocateVirtualVGPRForSGPRSpills(MachineFunction &MF, int FI, 553 unsigned LaneIndex); 554 bool allocatePhysicalVGPRForSGPRSpills(MachineFunction &MF, int FI, 555 unsigned LaneIndex); 556 557 public: 558 Register getVGPRForAGPRCopy() const { 559 return VGPRForAGPRCopy; 560 } 561 562 void setVGPRForAGPRCopy(Register NewVGPRForAGPRCopy) { 563 VGPRForAGPRCopy = NewVGPRForAGPRCopy; 564 } 565 566 bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const; 567 568 public: 569 SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI) = default; 570 SIMachineFunctionInfo(const Function &F, const GCNSubtarget *STI); 571 572 MachineFunctionInfo * 573 clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF, 574 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB) 575 const override; 576 577 bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, 578 const MachineFunction &MF, 579 PerFunctionMIParsingState &PFS, 580 SMDiagnostic &Error, SMRange &SourceRange); 581 582 void reserveWWMRegister(Register Reg) { WWMReservedRegs.insert(Reg); } 583 584 SIModeRegisterDefaults getMode() const { return Mode; } 585 586 ArrayRef<SIRegisterInfo::SpilledReg> 587 getSGPRSpillToVirtualVGPRLanes(int FrameIndex) const { 588 auto I = SGPRSpillsToVirtualVGPRLanes.find(FrameIndex); 589 return (I == SGPRSpillsToVirtualVGPRLanes.end()) 590 ? ArrayRef<SIRegisterInfo::SpilledReg>() 591 : ArrayRef(I->second); 592 } 593 594 ArrayRef<Register> getSGPRSpillVGPRs() const { return SpillVGPRs; } 595 const WWMSpillsMap &getWWMSpills() const { return WWMSpills; } 596 const ReservedRegSet &getWWMReservedRegs() const { return WWMReservedRegs; } 597 598 const PrologEpilogSGPRSpillsMap &getPrologEpilogSGPRSpills() const { 599 return PrologEpilogSGPRSpills; 600 } 601 602 void addToPrologEpilogSGPRSpills(Register Reg, 603 PrologEpilogSGPRSaveRestoreInfo SI) { 604 PrologEpilogSGPRSpills.insert(std::make_pair(Reg, SI)); 605 } 606 607 // Check if an entry created for \p Reg in PrologEpilogSGPRSpills. Return true 608 // on success and false otherwise. 609 bool hasPrologEpilogSGPRSpillEntry(Register Reg) const { 610 return PrologEpilogSGPRSpills.contains(Reg); 611 } 612 613 // Get the scratch SGPR if allocated to save/restore \p Reg. 614 Register getScratchSGPRCopyDstReg(Register Reg) const { 615 auto I = PrologEpilogSGPRSpills.find(Reg); 616 if (I != PrologEpilogSGPRSpills.end() && 617 I->second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR) 618 return I->second.getReg(); 619 620 return AMDGPU::NoRegister; 621 } 622 623 // Get all scratch SGPRs allocated to copy/restore the SGPR spills. 624 void getAllScratchSGPRCopyDstRegs(SmallVectorImpl<Register> &Regs) const { 625 for (const auto &SI : PrologEpilogSGPRSpills) { 626 if (SI.second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR) 627 Regs.push_back(SI.second.getReg()); 628 } 629 } 630 631 // Check if \p FI is allocated for any SGPR spill to a VGPR lane during PEI. 632 bool checkIndexInPrologEpilogSGPRSpills(int FI) const { 633 return find_if(PrologEpilogSGPRSpills, 634 [FI](const std::pair<Register, 635 PrologEpilogSGPRSaveRestoreInfo> &SI) { 636 return SI.second.getKind() == 637 SGPRSaveKind::SPILL_TO_VGPR_LANE && 638 SI.second.getIndex() == FI; 639 }) != PrologEpilogSGPRSpills.end(); 640 } 641 642 const PrologEpilogSGPRSaveRestoreInfo & 643 getPrologEpilogSGPRSaveRestoreInfo(Register Reg) const { 644 auto I = PrologEpilogSGPRSpills.find(Reg); 645 assert(I != PrologEpilogSGPRSpills.end()); 646 647 return I->second; 648 } 649 650 ArrayRef<SIRegisterInfo::SpilledReg> 651 getSGPRSpillToPhysicalVGPRLanes(int FrameIndex) const { 652 auto I = SGPRSpillsToPhysicalVGPRLanes.find(FrameIndex); 653 return (I == SGPRSpillsToPhysicalVGPRLanes.end()) 654 ? ArrayRef<SIRegisterInfo::SpilledReg>() 655 : ArrayRef(I->second); 656 } 657 658 void setFlag(Register Reg, uint8_t Flag) { 659 assert(Reg.isVirtual()); 660 if (VRegFlags.inBounds(Reg)) 661 VRegFlags[Reg] |= Flag; 662 } 663 664 bool checkFlag(Register Reg, uint8_t Flag) const { 665 if (Reg.isPhysical()) 666 return false; 667 668 return VRegFlags.inBounds(Reg) && VRegFlags[Reg] & Flag; 669 } 670 671 bool hasVRegFlags() { return VRegFlags.size(); } 672 673 void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size = 4, 674 Align Alignment = Align(4)); 675 676 void splitWWMSpillRegisters( 677 MachineFunction &MF, 678 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs, 679 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const; 680 681 ArrayRef<MCPhysReg> getAGPRSpillVGPRs() const { 682 return SpillAGPR; 683 } 684 685 Register getSGPRForEXECCopy() const { return SGPRForEXECCopy; } 686 687 void setSGPRForEXECCopy(Register Reg) { SGPRForEXECCopy = Reg; } 688 689 ArrayRef<MCPhysReg> getVGPRSpillAGPRs() const { 690 return SpillVGPR; 691 } 692 693 MCPhysReg getVGPRToAGPRSpill(int FrameIndex, unsigned Lane) const { 694 auto I = VGPRToAGPRSpills.find(FrameIndex); 695 return (I == VGPRToAGPRSpills.end()) ? (MCPhysReg)AMDGPU::NoRegister 696 : I->second.Lanes[Lane]; 697 } 698 699 void setVGPRToAGPRSpillDead(int FrameIndex) { 700 auto I = VGPRToAGPRSpills.find(FrameIndex); 701 if (I != VGPRToAGPRSpills.end()) 702 I->second.IsDead = true; 703 } 704 705 bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, 706 bool IsPrologEpilog = false); 707 bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR); 708 709 /// If \p ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill 710 /// to the default stack. 711 bool removeDeadFrameIndices(MachineFrameInfo &MFI, 712 bool ResetSGPRSpillStackIDs); 713 714 int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI); 715 std::optional<int> getOptionalScavengeFI() const { return ScavengeFI; } 716 717 unsigned getBytesInStackArgArea() const { 718 return BytesInStackArgArea; 719 } 720 721 void setBytesInStackArgArea(unsigned Bytes) { 722 BytesInStackArgArea = Bytes; 723 } 724 725 // Add user SGPRs. 726 Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI); 727 Register addDispatchPtr(const SIRegisterInfo &TRI); 728 Register addQueuePtr(const SIRegisterInfo &TRI); 729 Register addKernargSegmentPtr(const SIRegisterInfo &TRI); 730 Register addDispatchID(const SIRegisterInfo &TRI); 731 Register addFlatScratchInit(const SIRegisterInfo &TRI); 732 Register addImplicitBufferPtr(const SIRegisterInfo &TRI); 733 Register addLDSKernelId(); 734 735 /// Increment user SGPRs used for padding the argument list only. 736 Register addReservedUserSGPR() { 737 Register Next = getNextUserSGPR(); 738 ++NumUserSGPRs; 739 return Next; 740 } 741 742 // Add system SGPRs. 743 Register addWorkGroupIDX(bool HasArchitectedSGPRs) { 744 Register Reg = 745 HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP9 : getNextSystemSGPR(); 746 ArgInfo.WorkGroupIDX = ArgDescriptor::createRegister(Reg); 747 if (!HasArchitectedSGPRs) 748 NumSystemSGPRs += 1; 749 750 return ArgInfo.WorkGroupIDX.getRegister(); 751 } 752 753 Register addWorkGroupIDY(bool HasArchitectedSGPRs) { 754 Register Reg = 755 HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP7 : getNextSystemSGPR(); 756 unsigned Mask = HasArchitectedSGPRs && hasWorkGroupIDZ() ? 0xffff : ~0u; 757 ArgInfo.WorkGroupIDY = ArgDescriptor::createRegister(Reg, Mask); 758 if (!HasArchitectedSGPRs) 759 NumSystemSGPRs += 1; 760 761 return ArgInfo.WorkGroupIDY.getRegister(); 762 } 763 764 Register addWorkGroupIDZ(bool HasArchitectedSGPRs) { 765 Register Reg = 766 HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP7 : getNextSystemSGPR(); 767 unsigned Mask = HasArchitectedSGPRs ? 0xffff << 16 : ~0u; 768 ArgInfo.WorkGroupIDZ = ArgDescriptor::createRegister(Reg, Mask); 769 if (!HasArchitectedSGPRs) 770 NumSystemSGPRs += 1; 771 772 return ArgInfo.WorkGroupIDZ.getRegister(); 773 } 774 775 Register addWorkGroupInfo() { 776 ArgInfo.WorkGroupInfo = ArgDescriptor::createRegister(getNextSystemSGPR()); 777 NumSystemSGPRs += 1; 778 return ArgInfo.WorkGroupInfo.getRegister(); 779 } 780 781 // Add special VGPR inputs 782 void setWorkItemIDX(ArgDescriptor Arg) { 783 ArgInfo.WorkItemIDX = Arg; 784 } 785 786 void setWorkItemIDY(ArgDescriptor Arg) { 787 ArgInfo.WorkItemIDY = Arg; 788 } 789 790 void setWorkItemIDZ(ArgDescriptor Arg) { 791 ArgInfo.WorkItemIDZ = Arg; 792 } 793 794 Register addPrivateSegmentWaveByteOffset() { 795 ArgInfo.PrivateSegmentWaveByteOffset 796 = ArgDescriptor::createRegister(getNextSystemSGPR()); 797 NumSystemSGPRs += 1; 798 return ArgInfo.PrivateSegmentWaveByteOffset.getRegister(); 799 } 800 801 void setPrivateSegmentWaveByteOffset(Register Reg) { 802 ArgInfo.PrivateSegmentWaveByteOffset = ArgDescriptor::createRegister(Reg); 803 } 804 805 bool hasPrivateSegmentBuffer() const { 806 return PrivateSegmentBuffer; 807 } 808 809 bool hasDispatchPtr() const { 810 return DispatchPtr; 811 } 812 813 bool hasQueuePtr() const { 814 return QueuePtr; 815 } 816 817 bool hasKernargSegmentPtr() const { 818 return KernargSegmentPtr; 819 } 820 821 bool hasDispatchID() const { 822 return DispatchID; 823 } 824 825 bool hasFlatScratchInit() const { 826 return FlatScratchInit; 827 } 828 829 bool hasWorkGroupIDX() const { 830 return WorkGroupIDX; 831 } 832 833 bool hasWorkGroupIDY() const { 834 return WorkGroupIDY; 835 } 836 837 bool hasWorkGroupIDZ() const { 838 return WorkGroupIDZ; 839 } 840 841 bool hasWorkGroupInfo() const { 842 return WorkGroupInfo; 843 } 844 845 bool hasLDSKernelId() const { return LDSKernelId; } 846 847 bool hasPrivateSegmentWaveByteOffset() const { 848 return PrivateSegmentWaveByteOffset; 849 } 850 851 bool hasWorkItemIDX() const { 852 return WorkItemIDX; 853 } 854 855 bool hasWorkItemIDY() const { 856 return WorkItemIDY; 857 } 858 859 bool hasWorkItemIDZ() const { 860 return WorkItemIDZ; 861 } 862 863 bool hasImplicitArgPtr() const { 864 return ImplicitArgPtr; 865 } 866 867 bool hasImplicitBufferPtr() const { 868 return ImplicitBufferPtr; 869 } 870 871 AMDGPUFunctionArgInfo &getArgInfo() { 872 return ArgInfo; 873 } 874 875 const AMDGPUFunctionArgInfo &getArgInfo() const { 876 return ArgInfo; 877 } 878 879 std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT> 880 getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const { 881 return ArgInfo.getPreloadedValue(Value); 882 } 883 884 MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const { 885 auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value)); 886 return Arg ? Arg->getRegister() : MCRegister(); 887 } 888 889 unsigned getGITPtrHigh() const { 890 return GITPtrHigh; 891 } 892 893 Register getGITPtrLoReg(const MachineFunction &MF) const; 894 895 uint32_t get32BitAddressHighBits() const { 896 return HighBitsOf32BitAddress; 897 } 898 899 unsigned getNumUserSGPRs() const { 900 return NumUserSGPRs; 901 } 902 903 unsigned getNumPreloadedSGPRs() const { 904 return NumUserSGPRs + NumSystemSGPRs; 905 } 906 907 Register getPrivateSegmentWaveByteOffsetSystemSGPR() const { 908 return ArgInfo.PrivateSegmentWaveByteOffset.getRegister(); 909 } 910 911 /// Returns the physical register reserved for use as the resource 912 /// descriptor for scratch accesses. 913 Register getScratchRSrcReg() const { 914 return ScratchRSrcReg; 915 } 916 917 void setScratchRSrcReg(Register Reg) { 918 assert(Reg != 0 && "Should never be unset"); 919 ScratchRSrcReg = Reg; 920 } 921 922 Register getFrameOffsetReg() const { 923 return FrameOffsetReg; 924 } 925 926 void setFrameOffsetReg(Register Reg) { 927 assert(Reg != 0 && "Should never be unset"); 928 FrameOffsetReg = Reg; 929 } 930 931 void setStackPtrOffsetReg(Register Reg) { 932 assert(Reg != 0 && "Should never be unset"); 933 StackPtrOffsetReg = Reg; 934 } 935 936 void setLongBranchReservedReg(Register Reg) { LongBranchReservedReg = Reg; } 937 938 // Note the unset value for this is AMDGPU::SP_REG rather than 939 // NoRegister. This is mostly a workaround for MIR tests where state that 940 // can't be directly computed from the function is not preserved in serialized 941 // MIR. 942 Register getStackPtrOffsetReg() const { 943 return StackPtrOffsetReg; 944 } 945 946 Register getLongBranchReservedReg() const { return LongBranchReservedReg; } 947 948 Register getQueuePtrUserSGPR() const { 949 return ArgInfo.QueuePtr.getRegister(); 950 } 951 952 Register getImplicitBufferPtrUserSGPR() const { 953 return ArgInfo.ImplicitBufferPtr.getRegister(); 954 } 955 956 bool hasSpilledSGPRs() const { 957 return HasSpilledSGPRs; 958 } 959 960 void setHasSpilledSGPRs(bool Spill = true) { 961 HasSpilledSGPRs = Spill; 962 } 963 964 bool hasSpilledVGPRs() const { 965 return HasSpilledVGPRs; 966 } 967 968 void setHasSpilledVGPRs(bool Spill = true) { 969 HasSpilledVGPRs = Spill; 970 } 971 972 bool hasNonSpillStackObjects() const { 973 return HasNonSpillStackObjects; 974 } 975 976 void setHasNonSpillStackObjects(bool StackObject = true) { 977 HasNonSpillStackObjects = StackObject; 978 } 979 980 bool isStackRealigned() const { 981 return IsStackRealigned; 982 } 983 984 void setIsStackRealigned(bool Realigned = true) { 985 IsStackRealigned = Realigned; 986 } 987 988 unsigned getNumSpilledSGPRs() const { 989 return NumSpilledSGPRs; 990 } 991 992 unsigned getNumSpilledVGPRs() const { 993 return NumSpilledVGPRs; 994 } 995 996 void addToSpilledSGPRs(unsigned num) { 997 NumSpilledSGPRs += num; 998 } 999 1000 void addToSpilledVGPRs(unsigned num) { 1001 NumSpilledVGPRs += num; 1002 } 1003 1004 unsigned getPSInputAddr() const { 1005 return PSInputAddr; 1006 } 1007 1008 unsigned getPSInputEnable() const { 1009 return PSInputEnable; 1010 } 1011 1012 bool isPSInputAllocated(unsigned Index) const { 1013 return PSInputAddr & (1 << Index); 1014 } 1015 1016 void markPSInputAllocated(unsigned Index) { 1017 PSInputAddr |= 1 << Index; 1018 } 1019 1020 void markPSInputEnabled(unsigned Index) { 1021 PSInputEnable |= 1 << Index; 1022 } 1023 1024 bool returnsVoid() const { 1025 return ReturnsVoid; 1026 } 1027 1028 void setIfReturnsVoid(bool Value) { 1029 ReturnsVoid = Value; 1030 } 1031 1032 /// \returns A pair of default/requested minimum/maximum flat work group sizes 1033 /// for this function. 1034 std::pair<unsigned, unsigned> getFlatWorkGroupSizes() const { 1035 return FlatWorkGroupSizes; 1036 } 1037 1038 /// \returns Default/requested minimum flat work group size for this function. 1039 unsigned getMinFlatWorkGroupSize() const { 1040 return FlatWorkGroupSizes.first; 1041 } 1042 1043 /// \returns Default/requested maximum flat work group size for this function. 1044 unsigned getMaxFlatWorkGroupSize() const { 1045 return FlatWorkGroupSizes.second; 1046 } 1047 1048 /// \returns A pair of default/requested minimum/maximum number of waves per 1049 /// execution unit. 1050 std::pair<unsigned, unsigned> getWavesPerEU() const { 1051 return WavesPerEU; 1052 } 1053 1054 /// \returns Default/requested minimum number of waves per execution unit. 1055 unsigned getMinWavesPerEU() const { 1056 return WavesPerEU.first; 1057 } 1058 1059 /// \returns Default/requested maximum number of waves per execution unit. 1060 unsigned getMaxWavesPerEU() const { 1061 return WavesPerEU.second; 1062 } 1063 1064 /// \returns SGPR used for \p Dim's work group ID. 1065 Register getWorkGroupIDSGPR(unsigned Dim) const { 1066 switch (Dim) { 1067 case 0: 1068 assert(hasWorkGroupIDX()); 1069 return ArgInfo.WorkGroupIDX.getRegister(); 1070 case 1: 1071 assert(hasWorkGroupIDY()); 1072 return ArgInfo.WorkGroupIDY.getRegister(); 1073 case 2: 1074 assert(hasWorkGroupIDZ()); 1075 return ArgInfo.WorkGroupIDZ.getRegister(); 1076 } 1077 llvm_unreachable("unexpected dimension"); 1078 } 1079 1080 const AMDGPUGWSResourcePseudoSourceValue * 1081 getGWSPSV(const AMDGPUTargetMachine &TM) { 1082 return &GWSResourcePSV; 1083 } 1084 1085 unsigned getOccupancy() const { 1086 return Occupancy; 1087 } 1088 1089 unsigned getMinAllowedOccupancy() const { 1090 if (!isMemoryBound() && !needsWaveLimiter()) 1091 return Occupancy; 1092 return (Occupancy < 4) ? Occupancy : 4; 1093 } 1094 1095 void limitOccupancy(const MachineFunction &MF); 1096 1097 void limitOccupancy(unsigned Limit) { 1098 if (Occupancy > Limit) 1099 Occupancy = Limit; 1100 } 1101 1102 void increaseOccupancy(const MachineFunction &MF, unsigned Limit) { 1103 if (Occupancy < Limit) 1104 Occupancy = Limit; 1105 limitOccupancy(MF); 1106 } 1107 1108 bool mayNeedAGPRs() const { 1109 return MayNeedAGPRs; 1110 } 1111 1112 // \returns true if a function has a use of AGPRs via inline asm or 1113 // has a call which may use it. 1114 bool mayUseAGPRs(const Function &F) const; 1115 1116 // \returns true if a function needs or may need AGPRs. 1117 bool usesAGPRs(const MachineFunction &MF) const; 1118 }; 1119 1120 } // end namespace llvm 1121 1122 #endif // LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H 1123