1 //==- SIMachineFunctionInfo.h - SIMachineFunctionInfo interface --*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
14 #define LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
15 
16 #include "AMDGPUArgumentUsageInfo.h"
17 #include "AMDGPUMachineFunction.h"
18 #include "AMDGPUTargetMachine.h"
19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20 #include "SIInstrInfo.h"
21 #include "llvm/ADT/SetVector.h"
22 #include "llvm/CodeGen/MIRYamlMapping.h"
23 #include "llvm/CodeGen/PseudoSourceValue.h"
24 #include "llvm/Support/raw_ostream.h"
25 #include <optional>
26 
27 namespace llvm {
28 
29 class MachineFrameInfo;
30 class MachineFunction;
31 class SIMachineFunctionInfo;
32 class SIRegisterInfo;
33 class TargetRegisterClass;
34 
35 class AMDGPUPseudoSourceValue : public PseudoSourceValue {
36 public:
37   enum AMDGPUPSVKind : unsigned {
38     PSVImage = PseudoSourceValue::TargetCustom,
39     GWSResource
40   };
41 
42 protected:
43   AMDGPUPseudoSourceValue(unsigned Kind, const AMDGPUTargetMachine &TM)
44       : PseudoSourceValue(Kind, TM) {}
45 
46 public:
47   bool isConstant(const MachineFrameInfo *) const override {
48     // This should probably be true for most images, but we will start by being
49     // conservative.
50     return false;
51   }
52 
53   bool isAliased(const MachineFrameInfo *) const override {
54     return true;
55   }
56 
57   bool mayAlias(const MachineFrameInfo *) const override {
58     return true;
59   }
60 };
61 
62 class AMDGPUGWSResourcePseudoSourceValue final : public AMDGPUPseudoSourceValue {
63 public:
64   explicit AMDGPUGWSResourcePseudoSourceValue(const AMDGPUTargetMachine &TM)
65       : AMDGPUPseudoSourceValue(GWSResource, TM) {}
66 
67   static bool classof(const PseudoSourceValue *V) {
68     return V->kind() == GWSResource;
69   }
70 
71   // These are inaccessible memory from IR.
72   bool isAliased(const MachineFrameInfo *) const override {
73     return false;
74   }
75 
76   // These are inaccessible memory from IR.
77   bool mayAlias(const MachineFrameInfo *) const override {
78     return false;
79   }
80 
81   void printCustom(raw_ostream &OS) const override {
82     OS << "GWSResource";
83   }
84 };
85 
86 namespace yaml {
87 
88 struct SIArgument {
89   bool IsRegister;
90   union {
91     StringValue RegisterName;
92     unsigned StackOffset;
93   };
94   std::optional<unsigned> Mask;
95 
96   // Default constructor, which creates a stack argument.
97   SIArgument() : IsRegister(false), StackOffset(0) {}
98   SIArgument(const SIArgument &Other) {
99     IsRegister = Other.IsRegister;
100     if (IsRegister) {
101       ::new ((void *)std::addressof(RegisterName))
102           StringValue(Other.RegisterName);
103     } else
104       StackOffset = Other.StackOffset;
105     Mask = Other.Mask;
106   }
107   SIArgument &operator=(const SIArgument &Other) {
108     IsRegister = Other.IsRegister;
109     if (IsRegister) {
110       ::new ((void *)std::addressof(RegisterName))
111           StringValue(Other.RegisterName);
112     } else
113       StackOffset = Other.StackOffset;
114     Mask = Other.Mask;
115     return *this;
116   }
117   ~SIArgument() {
118     if (IsRegister)
119       RegisterName.~StringValue();
120   }
121 
122   // Helper to create a register or stack argument.
123   static inline SIArgument createArgument(bool IsReg) {
124     if (IsReg)
125       return SIArgument(IsReg);
126     return SIArgument();
127   }
128 
129 private:
130   // Construct a register argument.
131   SIArgument(bool) : IsRegister(true), RegisterName() {}
132 };
133 
134 template <> struct MappingTraits<SIArgument> {
135   static void mapping(IO &YamlIO, SIArgument &A) {
136     if (YamlIO.outputting()) {
137       if (A.IsRegister)
138         YamlIO.mapRequired("reg", A.RegisterName);
139       else
140         YamlIO.mapRequired("offset", A.StackOffset);
141     } else {
142       auto Keys = YamlIO.keys();
143       if (is_contained(Keys, "reg")) {
144         A = SIArgument::createArgument(true);
145         YamlIO.mapRequired("reg", A.RegisterName);
146       } else if (is_contained(Keys, "offset"))
147         YamlIO.mapRequired("offset", A.StackOffset);
148       else
149         YamlIO.setError("missing required key 'reg' or 'offset'");
150     }
151     YamlIO.mapOptional("mask", A.Mask);
152   }
153   static const bool flow = true;
154 };
155 
156 struct SIArgumentInfo {
157   std::optional<SIArgument> PrivateSegmentBuffer;
158   std::optional<SIArgument> DispatchPtr;
159   std::optional<SIArgument> QueuePtr;
160   std::optional<SIArgument> KernargSegmentPtr;
161   std::optional<SIArgument> DispatchID;
162   std::optional<SIArgument> FlatScratchInit;
163   std::optional<SIArgument> PrivateSegmentSize;
164 
165   std::optional<SIArgument> WorkGroupIDX;
166   std::optional<SIArgument> WorkGroupIDY;
167   std::optional<SIArgument> WorkGroupIDZ;
168   std::optional<SIArgument> WorkGroupInfo;
169   std::optional<SIArgument> LDSKernelId;
170   std::optional<SIArgument> PrivateSegmentWaveByteOffset;
171 
172   std::optional<SIArgument> ImplicitArgPtr;
173   std::optional<SIArgument> ImplicitBufferPtr;
174 
175   std::optional<SIArgument> WorkItemIDX;
176   std::optional<SIArgument> WorkItemIDY;
177   std::optional<SIArgument> WorkItemIDZ;
178 };
179 
180 template <> struct MappingTraits<SIArgumentInfo> {
181   static void mapping(IO &YamlIO, SIArgumentInfo &AI) {
182     YamlIO.mapOptional("privateSegmentBuffer", AI.PrivateSegmentBuffer);
183     YamlIO.mapOptional("dispatchPtr", AI.DispatchPtr);
184     YamlIO.mapOptional("queuePtr", AI.QueuePtr);
185     YamlIO.mapOptional("kernargSegmentPtr", AI.KernargSegmentPtr);
186     YamlIO.mapOptional("dispatchID", AI.DispatchID);
187     YamlIO.mapOptional("flatScratchInit", AI.FlatScratchInit);
188     YamlIO.mapOptional("privateSegmentSize", AI.PrivateSegmentSize);
189 
190     YamlIO.mapOptional("workGroupIDX", AI.WorkGroupIDX);
191     YamlIO.mapOptional("workGroupIDY", AI.WorkGroupIDY);
192     YamlIO.mapOptional("workGroupIDZ", AI.WorkGroupIDZ);
193     YamlIO.mapOptional("workGroupInfo", AI.WorkGroupInfo);
194     YamlIO.mapOptional("LDSKernelId", AI.LDSKernelId);
195     YamlIO.mapOptional("privateSegmentWaveByteOffset",
196                        AI.PrivateSegmentWaveByteOffset);
197 
198     YamlIO.mapOptional("implicitArgPtr", AI.ImplicitArgPtr);
199     YamlIO.mapOptional("implicitBufferPtr", AI.ImplicitBufferPtr);
200 
201     YamlIO.mapOptional("workItemIDX", AI.WorkItemIDX);
202     YamlIO.mapOptional("workItemIDY", AI.WorkItemIDY);
203     YamlIO.mapOptional("workItemIDZ", AI.WorkItemIDZ);
204   }
205 };
206 
207 // Default to default mode for default calling convention.
208 struct SIMode {
209   bool IEEE = true;
210   bool DX10Clamp = true;
211   bool FP32InputDenormals = true;
212   bool FP32OutputDenormals = true;
213   bool FP64FP16InputDenormals = true;
214   bool FP64FP16OutputDenormals = true;
215 
216   SIMode() = default;
217 
218   SIMode(const AMDGPU::SIModeRegisterDefaults &Mode) {
219     IEEE = Mode.IEEE;
220     DX10Clamp = Mode.DX10Clamp;
221     FP32InputDenormals = Mode.FP32Denormals.Input != DenormalMode::PreserveSign;
222     FP32OutputDenormals =
223         Mode.FP32Denormals.Output != DenormalMode::PreserveSign;
224     FP64FP16InputDenormals =
225         Mode.FP64FP16Denormals.Input != DenormalMode::PreserveSign;
226     FP64FP16OutputDenormals =
227         Mode.FP64FP16Denormals.Output != DenormalMode::PreserveSign;
228   }
229 
230   bool operator ==(const SIMode Other) const {
231     return IEEE == Other.IEEE &&
232            DX10Clamp == Other.DX10Clamp &&
233            FP32InputDenormals == Other.FP32InputDenormals &&
234            FP32OutputDenormals == Other.FP32OutputDenormals &&
235            FP64FP16InputDenormals == Other.FP64FP16InputDenormals &&
236            FP64FP16OutputDenormals == Other.FP64FP16OutputDenormals;
237   }
238 };
239 
240 template <> struct MappingTraits<SIMode> {
241   static void mapping(IO &YamlIO, SIMode &Mode) {
242     YamlIO.mapOptional("ieee", Mode.IEEE, true);
243     YamlIO.mapOptional("dx10-clamp", Mode.DX10Clamp, true);
244     YamlIO.mapOptional("fp32-input-denormals", Mode.FP32InputDenormals, true);
245     YamlIO.mapOptional("fp32-output-denormals", Mode.FP32OutputDenormals, true);
246     YamlIO.mapOptional("fp64-fp16-input-denormals", Mode.FP64FP16InputDenormals, true);
247     YamlIO.mapOptional("fp64-fp16-output-denormals", Mode.FP64FP16OutputDenormals, true);
248   }
249 };
250 
251 struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
252   uint64_t ExplicitKernArgSize = 0;
253   Align MaxKernArgAlign;
254   uint32_t LDSSize = 0;
255   uint32_t GDSSize = 0;
256   Align DynLDSAlign;
257   bool IsEntryFunction = false;
258   bool NoSignedZerosFPMath = false;
259   bool MemoryBound = false;
260   bool WaveLimiter = false;
261   bool HasSpilledSGPRs = false;
262   bool HasSpilledVGPRs = false;
263   uint32_t HighBitsOf32BitAddress = 0;
264 
265   // TODO: 10 may be a better default since it's the maximum.
266   unsigned Occupancy = 0;
267 
268   SmallVector<StringValue> WWMReservedRegs;
269 
270   StringValue ScratchRSrcReg = "$private_rsrc_reg";
271   StringValue FrameOffsetReg = "$fp_reg";
272   StringValue StackPtrOffsetReg = "$sp_reg";
273 
274   unsigned BytesInStackArgArea = 0;
275   bool ReturnsVoid = true;
276 
277   std::optional<SIArgumentInfo> ArgInfo;
278   SIMode Mode;
279   std::optional<FrameIndex> ScavengeFI;
280   StringValue VGPRForAGPRCopy;
281 
282   SIMachineFunctionInfo() = default;
283   SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo &,
284                         const TargetRegisterInfo &TRI,
285                         const llvm::MachineFunction &MF);
286 
287   void mappingImpl(yaml::IO &YamlIO) override;
288   ~SIMachineFunctionInfo() = default;
289 };
290 
291 template <> struct MappingTraits<SIMachineFunctionInfo> {
292   static void mapping(IO &YamlIO, SIMachineFunctionInfo &MFI) {
293     YamlIO.mapOptional("explicitKernArgSize", MFI.ExplicitKernArgSize,
294                        UINT64_C(0));
295     YamlIO.mapOptional("maxKernArgAlign", MFI.MaxKernArgAlign);
296     YamlIO.mapOptional("ldsSize", MFI.LDSSize, 0u);
297     YamlIO.mapOptional("gdsSize", MFI.GDSSize, 0u);
298     YamlIO.mapOptional("dynLDSAlign", MFI.DynLDSAlign, Align());
299     YamlIO.mapOptional("isEntryFunction", MFI.IsEntryFunction, false);
300     YamlIO.mapOptional("noSignedZerosFPMath", MFI.NoSignedZerosFPMath, false);
301     YamlIO.mapOptional("memoryBound", MFI.MemoryBound, false);
302     YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
303     YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
304     YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
305     YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg,
306                        StringValue("$private_rsrc_reg"));
307     YamlIO.mapOptional("frameOffsetReg", MFI.FrameOffsetReg,
308                        StringValue("$fp_reg"));
309     YamlIO.mapOptional("stackPtrOffsetReg", MFI.StackPtrOffsetReg,
310                        StringValue("$sp_reg"));
311     YamlIO.mapOptional("bytesInStackArgArea", MFI.BytesInStackArgArea, 0u);
312     YamlIO.mapOptional("returnsVoid", MFI.ReturnsVoid, true);
313     YamlIO.mapOptional("argumentInfo", MFI.ArgInfo);
314     YamlIO.mapOptional("mode", MFI.Mode, SIMode());
315     YamlIO.mapOptional("highBitsOf32BitAddress",
316                        MFI.HighBitsOf32BitAddress, 0u);
317     YamlIO.mapOptional("occupancy", MFI.Occupancy, 0);
318     YamlIO.mapOptional("wwmReservedRegs", MFI.WWMReservedRegs);
319     YamlIO.mapOptional("scavengeFI", MFI.ScavengeFI);
320     YamlIO.mapOptional("vgprForAGPRCopy", MFI.VGPRForAGPRCopy,
321                        StringValue()); // Don't print out when it's empty.
322   }
323 };
324 
325 } // end namespace yaml
326 
327 // A CSR SGPR value can be preserved inside a callee using one of the following
328 // methods.
329 //   1. Copy to an unused scratch SGPR.
330 //   2. Spill to a VGPR lane.
331 //   3. Spill to memory via. a scratch VGPR.
332 // class PrologEpilogSGPRSaveRestoreInfo represents the save/restore method used
333 // for an SGPR at function prolog/epilog.
334 enum class SGPRSaveKind : uint8_t {
335   COPY_TO_SCRATCH_SGPR,
336   SPILL_TO_VGPR_LANE,
337   SPILL_TO_MEM
338 };
339 
340 class PrologEpilogSGPRSaveRestoreInfo {
341   SGPRSaveKind Kind;
342   union {
343     int Index;
344     Register Reg;
345   };
346 
347 public:
348   PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, int I) : Kind(K), Index(I) {}
349   PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, Register R)
350       : Kind(K), Reg(R) {}
351   Register getReg() const { return Reg; }
352   int getIndex() const { return Index; }
353   SGPRSaveKind getKind() const { return Kind; }
354 };
355 
356 /// This class keeps track of the SPI_SP_INPUT_ADDR config register, which
357 /// tells the hardware which interpolation parameters to load.
358 class SIMachineFunctionInfo final : public AMDGPUMachineFunction {
359   friend class GCNTargetMachine;
360 
361   // State of MODE register, assumed FP mode.
362   AMDGPU::SIModeRegisterDefaults Mode;
363 
364   // Registers that may be reserved for spilling purposes. These may be the same
365   // as the input registers.
366   Register ScratchRSrcReg = AMDGPU::PRIVATE_RSRC_REG;
367 
368   // This is the unswizzled offset from the current dispatch's scratch wave
369   // base to the beginning of the current function's frame.
370   Register FrameOffsetReg = AMDGPU::FP_REG;
371 
372   // This is an ABI register used in the non-entry calling convention to
373   // communicate the unswizzled offset from the current dispatch's scratch wave
374   // base to the beginning of the new function's frame.
375   Register StackPtrOffsetReg = AMDGPU::SP_REG;
376 
377   AMDGPUFunctionArgInfo ArgInfo;
378 
379   // Graphics info.
380   unsigned PSInputAddr = 0;
381   unsigned PSInputEnable = 0;
382 
383   /// Number of bytes of arguments this function has on the stack. If the callee
384   /// is expected to restore the argument stack this should be a multiple of 16,
385   /// all usable during a tail call.
386   ///
387   /// The alternative would forbid tail call optimisation in some cases: if we
388   /// want to transfer control from a function with 8-bytes of stack-argument
389   /// space to a function with 16-bytes then misalignment of this value would
390   /// make a stack adjustment necessary, which could not be undone by the
391   /// callee.
392   unsigned BytesInStackArgArea = 0;
393 
394   bool ReturnsVoid = true;
395 
396   // A pair of default/requested minimum/maximum flat work group sizes.
397   // Minimum - first, maximum - second.
398   std::pair<unsigned, unsigned> FlatWorkGroupSizes = {0, 0};
399 
400   // A pair of default/requested minimum/maximum number of waves per execution
401   // unit. Minimum - first, maximum - second.
402   std::pair<unsigned, unsigned> WavesPerEU = {0, 0};
403 
404   const AMDGPUGWSResourcePseudoSourceValue GWSResourcePSV;
405 
406 private:
407   unsigned NumUserSGPRs = 0;
408   unsigned NumSystemSGPRs = 0;
409 
410   bool HasSpilledSGPRs = false;
411   bool HasSpilledVGPRs = false;
412   bool HasNonSpillStackObjects = false;
413   bool IsStackRealigned = false;
414 
415   unsigned NumSpilledSGPRs = 0;
416   unsigned NumSpilledVGPRs = 0;
417 
418   // Feature bits required for inputs passed in user SGPRs.
419   bool PrivateSegmentBuffer : 1;
420   bool DispatchPtr : 1;
421   bool QueuePtr : 1;
422   bool KernargSegmentPtr : 1;
423   bool DispatchID : 1;
424   bool FlatScratchInit : 1;
425 
426   // Feature bits required for inputs passed in system SGPRs.
427   bool WorkGroupIDX : 1; // Always initialized.
428   bool WorkGroupIDY : 1;
429   bool WorkGroupIDZ : 1;
430   bool WorkGroupInfo : 1;
431   bool LDSKernelId : 1;
432   bool PrivateSegmentWaveByteOffset : 1;
433 
434   bool WorkItemIDX : 1; // Always initialized.
435   bool WorkItemIDY : 1;
436   bool WorkItemIDZ : 1;
437 
438   // Private memory buffer
439   // Compute directly in sgpr[0:1]
440   // Other shaders indirect 64-bits at sgpr[0:1]
441   bool ImplicitBufferPtr : 1;
442 
443   // Pointer to where the ABI inserts special kernel arguments separate from the
444   // user arguments. This is an offset from the KernargSegmentPtr.
445   bool ImplicitArgPtr : 1;
446 
447   bool MayNeedAGPRs : 1;
448 
449   // The hard-wired high half of the address of the global information table
450   // for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since
451   // current hardware only allows a 16 bit value.
452   unsigned GITPtrHigh;
453 
454   unsigned HighBitsOf32BitAddress;
455 
456   // Current recorded maximum possible occupancy.
457   unsigned Occupancy;
458 
459   mutable std::optional<bool> UsesAGPRs;
460 
461   MCPhysReg getNextUserSGPR() const;
462 
463   MCPhysReg getNextSystemSGPR() const;
464 
465 public:
466   struct VGPRSpillToAGPR {
467     SmallVector<MCPhysReg, 32> Lanes;
468     bool FullyAllocated = false;
469     bool IsDead = false;
470   };
471 
472 private:
473   // To track VGPR + lane index for each subregister of the SGPR spilled to
474   // frameindex key during SILowerSGPRSpills pass.
475   DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>> SGPRSpillToVGPRLanes;
476   // To track VGPR + lane index for spilling special SGPRs like Frame Pointer
477   // identified during PrologEpilogInserter.
478   DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>>
479       PrologEpilogSGPRSpillToVGPRLanes;
480   unsigned NumVGPRSpillLanes = 0;
481   unsigned NumVGPRPrologEpilogSpillLanes = 0;
482   SmallVector<Register, 2> SpillVGPRs;
483   using WWMSpillsMap = MapVector<Register, int>;
484   // To track the registers used in instructions that can potentially modify the
485   // inactive lanes. The WWM instructions and the writelane instructions for
486   // spilling SGPRs to VGPRs fall under such category of operations. The VGPRs
487   // modified by them should be spilled/restored at function prolog/epilog to
488   // avoid any undesired outcome. Each entry in this map holds a pair of values,
489   // the VGPR and its stack slot index.
490   WWMSpillsMap WWMSpills;
491 
492   using ReservedRegSet = SmallSetVector<Register, 8>;
493   // To track the VGPRs reserved for WWM instructions. They get stack slots
494   // later during PrologEpilogInserter and get added into the superset WWMSpills
495   // for actual spilling. A separate set makes the register reserved part and
496   // the serialization easier.
497   ReservedRegSet WWMReservedRegs;
498 
499   using PrologEpilogSGPRSpillsMap =
500       DenseMap<Register, PrologEpilogSGPRSaveRestoreInfo>;
501   // To track the SGPR spill method used for a CSR SGPR register during
502   // frame lowering. Even though the SGPR spills are handled during
503   // SILowerSGPRSpills pass, some special handling needed later during the
504   // PrologEpilogInserter.
505   PrologEpilogSGPRSpillsMap PrologEpilogSGPRSpills;
506 
507   DenseMap<int, VGPRSpillToAGPR> VGPRToAGPRSpills;
508 
509   // AGPRs used for VGPR spills.
510   SmallVector<MCPhysReg, 32> SpillAGPR;
511 
512   // VGPRs used for AGPR spills.
513   SmallVector<MCPhysReg, 32> SpillVGPR;
514 
515   // Emergency stack slot. Sometimes, we create this before finalizing the stack
516   // frame, so save it here and add it to the RegScavenger later.
517   std::optional<int> ScavengeFI;
518 
519 private:
520   Register VGPRForAGPRCopy;
521 
522   bool allocateVGPRForSGPRSpills(MachineFunction &MF, int FI,
523                                  unsigned LaneIndex);
524   bool allocateVGPRForPrologEpilogSGPRSpills(MachineFunction &MF, int FI,
525                                              unsigned LaneIndex);
526 
527 public:
528   Register getVGPRForAGPRCopy() const {
529     return VGPRForAGPRCopy;
530   }
531 
532   void setVGPRForAGPRCopy(Register NewVGPRForAGPRCopy) {
533     VGPRForAGPRCopy = NewVGPRForAGPRCopy;
534   }
535 
536   bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const;
537 
538 public:
539   SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI) = default;
540   SIMachineFunctionInfo(const Function &F, const GCNSubtarget *STI);
541 
542   MachineFunctionInfo *
543   clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF,
544         const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
545       const override;
546 
547   bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI,
548                                 const MachineFunction &MF,
549                                 PerFunctionMIParsingState &PFS,
550                                 SMDiagnostic &Error, SMRange &SourceRange);
551 
552   void reserveWWMRegister(Register Reg) { WWMReservedRegs.insert(Reg); }
553 
554   AMDGPU::SIModeRegisterDefaults getMode() const {
555     return Mode;
556   }
557 
558   ArrayRef<SIRegisterInfo::SpilledReg>
559   getSGPRSpillToVGPRLanes(int FrameIndex) const {
560     auto I = SGPRSpillToVGPRLanes.find(FrameIndex);
561     return (I == SGPRSpillToVGPRLanes.end())
562                ? ArrayRef<SIRegisterInfo::SpilledReg>()
563                : ArrayRef(I->second);
564   }
565 
566   ArrayRef<Register> getSGPRSpillVGPRs() const { return SpillVGPRs; }
567   const WWMSpillsMap &getWWMSpills() const { return WWMSpills; }
568   const ReservedRegSet &getWWMReservedRegs() const { return WWMReservedRegs; }
569 
570   const PrologEpilogSGPRSpillsMap &getPrologEpilogSGPRSpills() const {
571     return PrologEpilogSGPRSpills;
572   }
573 
574   void addToPrologEpilogSGPRSpills(Register Reg,
575                                    PrologEpilogSGPRSaveRestoreInfo SI) {
576     PrologEpilogSGPRSpills.insert(std::make_pair(Reg, SI));
577   }
578 
579   // Check if an entry created for \p Reg in PrologEpilogSGPRSpills. Return true
580   // on success and false otherwise.
581   bool hasPrologEpilogSGPRSpillEntry(Register Reg) const {
582     return PrologEpilogSGPRSpills.find(Reg) != PrologEpilogSGPRSpills.end();
583   }
584 
585   // Get the scratch SGPR if allocated to save/restore \p Reg.
586   Register getScratchSGPRCopyDstReg(Register Reg) const {
587     auto I = PrologEpilogSGPRSpills.find(Reg);
588     if (I != PrologEpilogSGPRSpills.end() &&
589         I->second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR)
590       return I->second.getReg();
591 
592     return AMDGPU::NoRegister;
593   }
594 
595   // Get all scratch SGPRs allocated to copy/restore the SGPR spills.
596   void getAllScratchSGPRCopyDstRegs(SmallVectorImpl<Register> &Regs) const {
597     for (const auto &SI : PrologEpilogSGPRSpills) {
598       if (SI.second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR)
599         Regs.push_back(SI.second.getReg());
600     }
601   }
602 
603   // Check if \p FI is allocated for any SGPR spill to a VGPR lane during PEI.
604   bool checkIndexInPrologEpilogSGPRSpills(int FI) const {
605     return find_if(PrologEpilogSGPRSpills,
606                    [FI](const std::pair<Register,
607                                         PrologEpilogSGPRSaveRestoreInfo> &SI) {
608                      return SI.second.getKind() ==
609                                 SGPRSaveKind::SPILL_TO_VGPR_LANE &&
610                             SI.second.getIndex() == FI;
611                    }) != PrologEpilogSGPRSpills.end();
612   }
613 
614   const PrologEpilogSGPRSaveRestoreInfo &
615   getPrologEpilogSGPRSaveRestoreInfo(Register Reg) const {
616     auto I = PrologEpilogSGPRSpills.find(Reg);
617     assert(I != PrologEpilogSGPRSpills.end());
618 
619     return I->second;
620   }
621 
622   ArrayRef<SIRegisterInfo::SpilledReg>
623   getPrologEpilogSGPRSpillToVGPRLanes(int FrameIndex) const {
624     auto I = PrologEpilogSGPRSpillToVGPRLanes.find(FrameIndex);
625     return (I == PrologEpilogSGPRSpillToVGPRLanes.end())
626                ? ArrayRef<SIRegisterInfo::SpilledReg>()
627                : ArrayRef(I->second);
628   }
629 
630   void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size = 4,
631                         Align Alignment = Align(4));
632 
633   void splitWWMSpillRegisters(
634       MachineFunction &MF,
635       SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
636       SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const;
637 
638   ArrayRef<MCPhysReg> getAGPRSpillVGPRs() const {
639     return SpillAGPR;
640   }
641 
642   ArrayRef<MCPhysReg> getVGPRSpillAGPRs() const {
643     return SpillVGPR;
644   }
645 
646   MCPhysReg getVGPRToAGPRSpill(int FrameIndex, unsigned Lane) const {
647     auto I = VGPRToAGPRSpills.find(FrameIndex);
648     return (I == VGPRToAGPRSpills.end()) ? (MCPhysReg)AMDGPU::NoRegister
649                                          : I->second.Lanes[Lane];
650   }
651 
652   void setVGPRToAGPRSpillDead(int FrameIndex) {
653     auto I = VGPRToAGPRSpills.find(FrameIndex);
654     if (I != VGPRToAGPRSpills.end())
655       I->second.IsDead = true;
656   }
657 
658   bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI,
659                                    bool IsPrologEpilog = false);
660   bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR);
661 
662   /// If \p ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill
663   /// to the default stack.
664   bool removeDeadFrameIndices(MachineFrameInfo &MFI,
665                               bool ResetSGPRSpillStackIDs);
666 
667   int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI);
668   std::optional<int> getOptionalScavengeFI() const { return ScavengeFI; }
669 
670   unsigned getBytesInStackArgArea() const {
671     return BytesInStackArgArea;
672   }
673 
674   void setBytesInStackArgArea(unsigned Bytes) {
675     BytesInStackArgArea = Bytes;
676   }
677 
678   // Add user SGPRs.
679   Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI);
680   Register addDispatchPtr(const SIRegisterInfo &TRI);
681   Register addQueuePtr(const SIRegisterInfo &TRI);
682   Register addKernargSegmentPtr(const SIRegisterInfo &TRI);
683   Register addDispatchID(const SIRegisterInfo &TRI);
684   Register addFlatScratchInit(const SIRegisterInfo &TRI);
685   Register addImplicitBufferPtr(const SIRegisterInfo &TRI);
686   Register addLDSKernelId();
687 
688   /// Increment user SGPRs used for padding the argument list only.
689   Register addReservedUserSGPR() {
690     Register Next = getNextUserSGPR();
691     ++NumUserSGPRs;
692     return Next;
693   }
694 
695   // Add system SGPRs.
696   Register addWorkGroupIDX() {
697     ArgInfo.WorkGroupIDX = ArgDescriptor::createRegister(getNextSystemSGPR());
698     NumSystemSGPRs += 1;
699     return ArgInfo.WorkGroupIDX.getRegister();
700   }
701 
702   Register addWorkGroupIDY() {
703     ArgInfo.WorkGroupIDY = ArgDescriptor::createRegister(getNextSystemSGPR());
704     NumSystemSGPRs += 1;
705     return ArgInfo.WorkGroupIDY.getRegister();
706   }
707 
708   Register addWorkGroupIDZ() {
709     ArgInfo.WorkGroupIDZ = ArgDescriptor::createRegister(getNextSystemSGPR());
710     NumSystemSGPRs += 1;
711     return ArgInfo.WorkGroupIDZ.getRegister();
712   }
713 
714   Register addWorkGroupInfo() {
715     ArgInfo.WorkGroupInfo = ArgDescriptor::createRegister(getNextSystemSGPR());
716     NumSystemSGPRs += 1;
717     return ArgInfo.WorkGroupInfo.getRegister();
718   }
719 
720   // Add special VGPR inputs
721   void setWorkItemIDX(ArgDescriptor Arg) {
722     ArgInfo.WorkItemIDX = Arg;
723   }
724 
725   void setWorkItemIDY(ArgDescriptor Arg) {
726     ArgInfo.WorkItemIDY = Arg;
727   }
728 
729   void setWorkItemIDZ(ArgDescriptor Arg) {
730     ArgInfo.WorkItemIDZ = Arg;
731   }
732 
733   Register addPrivateSegmentWaveByteOffset() {
734     ArgInfo.PrivateSegmentWaveByteOffset
735       = ArgDescriptor::createRegister(getNextSystemSGPR());
736     NumSystemSGPRs += 1;
737     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
738   }
739 
740   void setPrivateSegmentWaveByteOffset(Register Reg) {
741     ArgInfo.PrivateSegmentWaveByteOffset = ArgDescriptor::createRegister(Reg);
742   }
743 
744   bool hasPrivateSegmentBuffer() const {
745     return PrivateSegmentBuffer;
746   }
747 
748   bool hasDispatchPtr() const {
749     return DispatchPtr;
750   }
751 
752   bool hasQueuePtr() const {
753     return QueuePtr;
754   }
755 
756   bool hasKernargSegmentPtr() const {
757     return KernargSegmentPtr;
758   }
759 
760   bool hasDispatchID() const {
761     return DispatchID;
762   }
763 
764   bool hasFlatScratchInit() const {
765     return FlatScratchInit;
766   }
767 
768   bool hasWorkGroupIDX() const {
769     return WorkGroupIDX;
770   }
771 
772   bool hasWorkGroupIDY() const {
773     return WorkGroupIDY;
774   }
775 
776   bool hasWorkGroupIDZ() const {
777     return WorkGroupIDZ;
778   }
779 
780   bool hasWorkGroupInfo() const {
781     return WorkGroupInfo;
782   }
783 
784   bool hasLDSKernelId() const { return LDSKernelId; }
785 
786   bool hasPrivateSegmentWaveByteOffset() const {
787     return PrivateSegmentWaveByteOffset;
788   }
789 
790   bool hasWorkItemIDX() const {
791     return WorkItemIDX;
792   }
793 
794   bool hasWorkItemIDY() const {
795     return WorkItemIDY;
796   }
797 
798   bool hasWorkItemIDZ() const {
799     return WorkItemIDZ;
800   }
801 
802   bool hasImplicitArgPtr() const {
803     return ImplicitArgPtr;
804   }
805 
806   bool hasImplicitBufferPtr() const {
807     return ImplicitBufferPtr;
808   }
809 
810   AMDGPUFunctionArgInfo &getArgInfo() {
811     return ArgInfo;
812   }
813 
814   const AMDGPUFunctionArgInfo &getArgInfo() const {
815     return ArgInfo;
816   }
817 
818   std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
819   getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
820     return ArgInfo.getPreloadedValue(Value);
821   }
822 
823   MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
824     auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value));
825     return Arg ? Arg->getRegister() : MCRegister();
826   }
827 
828   unsigned getGITPtrHigh() const {
829     return GITPtrHigh;
830   }
831 
832   Register getGITPtrLoReg(const MachineFunction &MF) const;
833 
834   uint32_t get32BitAddressHighBits() const {
835     return HighBitsOf32BitAddress;
836   }
837 
838   unsigned getNumUserSGPRs() const {
839     return NumUserSGPRs;
840   }
841 
842   unsigned getNumPreloadedSGPRs() const {
843     return NumUserSGPRs + NumSystemSGPRs;
844   }
845 
846   Register getPrivateSegmentWaveByteOffsetSystemSGPR() const {
847     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
848   }
849 
850   /// Returns the physical register reserved for use as the resource
851   /// descriptor for scratch accesses.
852   Register getScratchRSrcReg() const {
853     return ScratchRSrcReg;
854   }
855 
856   void setScratchRSrcReg(Register Reg) {
857     assert(Reg != 0 && "Should never be unset");
858     ScratchRSrcReg = Reg;
859   }
860 
861   Register getFrameOffsetReg() const {
862     return FrameOffsetReg;
863   }
864 
865   void setFrameOffsetReg(Register Reg) {
866     assert(Reg != 0 && "Should never be unset");
867     FrameOffsetReg = Reg;
868   }
869 
870   void setStackPtrOffsetReg(Register Reg) {
871     assert(Reg != 0 && "Should never be unset");
872     StackPtrOffsetReg = Reg;
873   }
874 
875   // Note the unset value for this is AMDGPU::SP_REG rather than
876   // NoRegister. This is mostly a workaround for MIR tests where state that
877   // can't be directly computed from the function is not preserved in serialized
878   // MIR.
879   Register getStackPtrOffsetReg() const {
880     return StackPtrOffsetReg;
881   }
882 
883   Register getQueuePtrUserSGPR() const {
884     return ArgInfo.QueuePtr.getRegister();
885   }
886 
887   Register getImplicitBufferPtrUserSGPR() const {
888     return ArgInfo.ImplicitBufferPtr.getRegister();
889   }
890 
891   bool hasSpilledSGPRs() const {
892     return HasSpilledSGPRs;
893   }
894 
895   void setHasSpilledSGPRs(bool Spill = true) {
896     HasSpilledSGPRs = Spill;
897   }
898 
899   bool hasSpilledVGPRs() const {
900     return HasSpilledVGPRs;
901   }
902 
903   void setHasSpilledVGPRs(bool Spill = true) {
904     HasSpilledVGPRs = Spill;
905   }
906 
907   bool hasNonSpillStackObjects() const {
908     return HasNonSpillStackObjects;
909   }
910 
911   void setHasNonSpillStackObjects(bool StackObject = true) {
912     HasNonSpillStackObjects = StackObject;
913   }
914 
915   bool isStackRealigned() const {
916     return IsStackRealigned;
917   }
918 
919   void setIsStackRealigned(bool Realigned = true) {
920     IsStackRealigned = Realigned;
921   }
922 
923   unsigned getNumSpilledSGPRs() const {
924     return NumSpilledSGPRs;
925   }
926 
927   unsigned getNumSpilledVGPRs() const {
928     return NumSpilledVGPRs;
929   }
930 
931   void addToSpilledSGPRs(unsigned num) {
932     NumSpilledSGPRs += num;
933   }
934 
935   void addToSpilledVGPRs(unsigned num) {
936     NumSpilledVGPRs += num;
937   }
938 
939   unsigned getPSInputAddr() const {
940     return PSInputAddr;
941   }
942 
943   unsigned getPSInputEnable() const {
944     return PSInputEnable;
945   }
946 
947   bool isPSInputAllocated(unsigned Index) const {
948     return PSInputAddr & (1 << Index);
949   }
950 
951   void markPSInputAllocated(unsigned Index) {
952     PSInputAddr |= 1 << Index;
953   }
954 
955   void markPSInputEnabled(unsigned Index) {
956     PSInputEnable |= 1 << Index;
957   }
958 
959   bool returnsVoid() const {
960     return ReturnsVoid;
961   }
962 
963   void setIfReturnsVoid(bool Value) {
964     ReturnsVoid = Value;
965   }
966 
967   /// \returns A pair of default/requested minimum/maximum flat work group sizes
968   /// for this function.
969   std::pair<unsigned, unsigned> getFlatWorkGroupSizes() const {
970     return FlatWorkGroupSizes;
971   }
972 
973   /// \returns Default/requested minimum flat work group size for this function.
974   unsigned getMinFlatWorkGroupSize() const {
975     return FlatWorkGroupSizes.first;
976   }
977 
978   /// \returns Default/requested maximum flat work group size for this function.
979   unsigned getMaxFlatWorkGroupSize() const {
980     return FlatWorkGroupSizes.second;
981   }
982 
983   /// \returns A pair of default/requested minimum/maximum number of waves per
984   /// execution unit.
985   std::pair<unsigned, unsigned> getWavesPerEU() const {
986     return WavesPerEU;
987   }
988 
989   /// \returns Default/requested minimum number of waves per execution unit.
990   unsigned getMinWavesPerEU() const {
991     return WavesPerEU.first;
992   }
993 
994   /// \returns Default/requested maximum number of waves per execution unit.
995   unsigned getMaxWavesPerEU() const {
996     return WavesPerEU.second;
997   }
998 
999   /// \returns SGPR used for \p Dim's work group ID.
1000   Register getWorkGroupIDSGPR(unsigned Dim) const {
1001     switch (Dim) {
1002     case 0:
1003       assert(hasWorkGroupIDX());
1004       return ArgInfo.WorkGroupIDX.getRegister();
1005     case 1:
1006       assert(hasWorkGroupIDY());
1007       return ArgInfo.WorkGroupIDY.getRegister();
1008     case 2:
1009       assert(hasWorkGroupIDZ());
1010       return ArgInfo.WorkGroupIDZ.getRegister();
1011     }
1012     llvm_unreachable("unexpected dimension");
1013   }
1014 
1015   const AMDGPUGWSResourcePseudoSourceValue *
1016   getGWSPSV(const AMDGPUTargetMachine &TM) {
1017     return &GWSResourcePSV;
1018   }
1019 
1020   unsigned getOccupancy() const {
1021     return Occupancy;
1022   }
1023 
1024   unsigned getMinAllowedOccupancy() const {
1025     if (!isMemoryBound() && !needsWaveLimiter())
1026       return Occupancy;
1027     return (Occupancy < 4) ? Occupancy : 4;
1028   }
1029 
1030   void limitOccupancy(const MachineFunction &MF);
1031 
1032   void limitOccupancy(unsigned Limit) {
1033     if (Occupancy > Limit)
1034       Occupancy = Limit;
1035   }
1036 
1037   void increaseOccupancy(const MachineFunction &MF, unsigned Limit) {
1038     if (Occupancy < Limit)
1039       Occupancy = Limit;
1040     limitOccupancy(MF);
1041   }
1042 
1043   bool mayNeedAGPRs() const {
1044     return MayNeedAGPRs;
1045   }
1046 
1047   // \returns true if a function has a use of AGPRs via inline asm or
1048   // has a call which may use it.
1049   bool mayUseAGPRs(const Function &F) const;
1050 
1051   // \returns true if a function needs or may need AGPRs.
1052   bool usesAGPRs(const MachineFunction &MF) const;
1053 };
1054 
1055 } // end namespace llvm
1056 
1057 #endif // LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
1058