1 //==- SIMachineFunctionInfo.h - SIMachineFunctionInfo interface --*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
14 #define LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
15 
16 #include "AMDGPUArgumentUsageInfo.h"
17 #include "AMDGPUMachineFunction.h"
18 #include "AMDGPUTargetMachine.h"
19 #include "GCNSubtarget.h"
20 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
21 #include "SIInstrInfo.h"
22 #include "SIModeRegisterDefaults.h"
23 #include "llvm/ADT/SetVector.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/CodeGen/MIRYamlMapping.h"
26 #include "llvm/CodeGen/PseudoSourceValue.h"
27 #include "llvm/Support/raw_ostream.h"
28 #include <optional>
29 
30 namespace llvm {
31 
32 class MachineFrameInfo;
33 class MachineFunction;
34 class SIMachineFunctionInfo;
35 class SIRegisterInfo;
36 class TargetRegisterClass;
37 
38 class AMDGPUPseudoSourceValue : public PseudoSourceValue {
39 public:
40   enum AMDGPUPSVKind : unsigned {
41     PSVImage = PseudoSourceValue::TargetCustom,
42     GWSResource
43   };
44 
45 protected:
46   AMDGPUPseudoSourceValue(unsigned Kind, const AMDGPUTargetMachine &TM)
47       : PseudoSourceValue(Kind, TM) {}
48 
49 public:
50   bool isConstant(const MachineFrameInfo *) const override {
51     // This should probably be true for most images, but we will start by being
52     // conservative.
53     return false;
54   }
55 
56   bool isAliased(const MachineFrameInfo *) const override {
57     return true;
58   }
59 
60   bool mayAlias(const MachineFrameInfo *) const override {
61     return true;
62   }
63 };
64 
65 class AMDGPUGWSResourcePseudoSourceValue final : public AMDGPUPseudoSourceValue {
66 public:
67   explicit AMDGPUGWSResourcePseudoSourceValue(const AMDGPUTargetMachine &TM)
68       : AMDGPUPseudoSourceValue(GWSResource, TM) {}
69 
70   static bool classof(const PseudoSourceValue *V) {
71     return V->kind() == GWSResource;
72   }
73 
74   // These are inaccessible memory from IR.
75   bool isAliased(const MachineFrameInfo *) const override {
76     return false;
77   }
78 
79   // These are inaccessible memory from IR.
80   bool mayAlias(const MachineFrameInfo *) const override {
81     return false;
82   }
83 
84   void printCustom(raw_ostream &OS) const override {
85     OS << "GWSResource";
86   }
87 };
88 
89 namespace yaml {
90 
91 struct SIArgument {
92   bool IsRegister;
93   union {
94     StringValue RegisterName;
95     unsigned StackOffset;
96   };
97   std::optional<unsigned> Mask;
98 
99   // Default constructor, which creates a stack argument.
100   SIArgument() : IsRegister(false), StackOffset(0) {}
101   SIArgument(const SIArgument &Other) {
102     IsRegister = Other.IsRegister;
103     if (IsRegister) {
104       ::new ((void *)std::addressof(RegisterName))
105           StringValue(Other.RegisterName);
106     } else
107       StackOffset = Other.StackOffset;
108     Mask = Other.Mask;
109   }
110   SIArgument &operator=(const SIArgument &Other) {
111     IsRegister = Other.IsRegister;
112     if (IsRegister) {
113       ::new ((void *)std::addressof(RegisterName))
114           StringValue(Other.RegisterName);
115     } else
116       StackOffset = Other.StackOffset;
117     Mask = Other.Mask;
118     return *this;
119   }
120   ~SIArgument() {
121     if (IsRegister)
122       RegisterName.~StringValue();
123   }
124 
125   // Helper to create a register or stack argument.
126   static inline SIArgument createArgument(bool IsReg) {
127     if (IsReg)
128       return SIArgument(IsReg);
129     return SIArgument();
130   }
131 
132 private:
133   // Construct a register argument.
134   SIArgument(bool) : IsRegister(true), RegisterName() {}
135 };
136 
137 template <> struct MappingTraits<SIArgument> {
138   static void mapping(IO &YamlIO, SIArgument &A) {
139     if (YamlIO.outputting()) {
140       if (A.IsRegister)
141         YamlIO.mapRequired("reg", A.RegisterName);
142       else
143         YamlIO.mapRequired("offset", A.StackOffset);
144     } else {
145       auto Keys = YamlIO.keys();
146       if (is_contained(Keys, "reg")) {
147         A = SIArgument::createArgument(true);
148         YamlIO.mapRequired("reg", A.RegisterName);
149       } else if (is_contained(Keys, "offset"))
150         YamlIO.mapRequired("offset", A.StackOffset);
151       else
152         YamlIO.setError("missing required key 'reg' or 'offset'");
153     }
154     YamlIO.mapOptional("mask", A.Mask);
155   }
156   static const bool flow = true;
157 };
158 
159 struct SIArgumentInfo {
160   std::optional<SIArgument> PrivateSegmentBuffer;
161   std::optional<SIArgument> DispatchPtr;
162   std::optional<SIArgument> QueuePtr;
163   std::optional<SIArgument> KernargSegmentPtr;
164   std::optional<SIArgument> DispatchID;
165   std::optional<SIArgument> FlatScratchInit;
166   std::optional<SIArgument> PrivateSegmentSize;
167 
168   std::optional<SIArgument> WorkGroupIDX;
169   std::optional<SIArgument> WorkGroupIDY;
170   std::optional<SIArgument> WorkGroupIDZ;
171   std::optional<SIArgument> WorkGroupInfo;
172   std::optional<SIArgument> LDSKernelId;
173   std::optional<SIArgument> PrivateSegmentWaveByteOffset;
174 
175   std::optional<SIArgument> ImplicitArgPtr;
176   std::optional<SIArgument> ImplicitBufferPtr;
177 
178   std::optional<SIArgument> WorkItemIDX;
179   std::optional<SIArgument> WorkItemIDY;
180   std::optional<SIArgument> WorkItemIDZ;
181 };
182 
183 template <> struct MappingTraits<SIArgumentInfo> {
184   static void mapping(IO &YamlIO, SIArgumentInfo &AI) {
185     YamlIO.mapOptional("privateSegmentBuffer", AI.PrivateSegmentBuffer);
186     YamlIO.mapOptional("dispatchPtr", AI.DispatchPtr);
187     YamlIO.mapOptional("queuePtr", AI.QueuePtr);
188     YamlIO.mapOptional("kernargSegmentPtr", AI.KernargSegmentPtr);
189     YamlIO.mapOptional("dispatchID", AI.DispatchID);
190     YamlIO.mapOptional("flatScratchInit", AI.FlatScratchInit);
191     YamlIO.mapOptional("privateSegmentSize", AI.PrivateSegmentSize);
192 
193     YamlIO.mapOptional("workGroupIDX", AI.WorkGroupIDX);
194     YamlIO.mapOptional("workGroupIDY", AI.WorkGroupIDY);
195     YamlIO.mapOptional("workGroupIDZ", AI.WorkGroupIDZ);
196     YamlIO.mapOptional("workGroupInfo", AI.WorkGroupInfo);
197     YamlIO.mapOptional("LDSKernelId", AI.LDSKernelId);
198     YamlIO.mapOptional("privateSegmentWaveByteOffset",
199                        AI.PrivateSegmentWaveByteOffset);
200 
201     YamlIO.mapOptional("implicitArgPtr", AI.ImplicitArgPtr);
202     YamlIO.mapOptional("implicitBufferPtr", AI.ImplicitBufferPtr);
203 
204     YamlIO.mapOptional("workItemIDX", AI.WorkItemIDX);
205     YamlIO.mapOptional("workItemIDY", AI.WorkItemIDY);
206     YamlIO.mapOptional("workItemIDZ", AI.WorkItemIDZ);
207   }
208 };
209 
210 // Default to default mode for default calling convention.
211 struct SIMode {
212   bool IEEE = true;
213   bool DX10Clamp = true;
214   bool FP32InputDenormals = true;
215   bool FP32OutputDenormals = true;
216   bool FP64FP16InputDenormals = true;
217   bool FP64FP16OutputDenormals = true;
218 
219   SIMode() = default;
220 
221   SIMode(const SIModeRegisterDefaults &Mode) {
222     IEEE = Mode.IEEE;
223     DX10Clamp = Mode.DX10Clamp;
224     FP32InputDenormals = Mode.FP32Denormals.Input != DenormalMode::PreserveSign;
225     FP32OutputDenormals =
226         Mode.FP32Denormals.Output != DenormalMode::PreserveSign;
227     FP64FP16InputDenormals =
228         Mode.FP64FP16Denormals.Input != DenormalMode::PreserveSign;
229     FP64FP16OutputDenormals =
230         Mode.FP64FP16Denormals.Output != DenormalMode::PreserveSign;
231   }
232 
233   bool operator ==(const SIMode Other) const {
234     return IEEE == Other.IEEE &&
235            DX10Clamp == Other.DX10Clamp &&
236            FP32InputDenormals == Other.FP32InputDenormals &&
237            FP32OutputDenormals == Other.FP32OutputDenormals &&
238            FP64FP16InputDenormals == Other.FP64FP16InputDenormals &&
239            FP64FP16OutputDenormals == Other.FP64FP16OutputDenormals;
240   }
241 };
242 
243 template <> struct MappingTraits<SIMode> {
244   static void mapping(IO &YamlIO, SIMode &Mode) {
245     YamlIO.mapOptional("ieee", Mode.IEEE, true);
246     YamlIO.mapOptional("dx10-clamp", Mode.DX10Clamp, true);
247     YamlIO.mapOptional("fp32-input-denormals", Mode.FP32InputDenormals, true);
248     YamlIO.mapOptional("fp32-output-denormals", Mode.FP32OutputDenormals, true);
249     YamlIO.mapOptional("fp64-fp16-input-denormals", Mode.FP64FP16InputDenormals, true);
250     YamlIO.mapOptional("fp64-fp16-output-denormals", Mode.FP64FP16OutputDenormals, true);
251   }
252 };
253 
254 struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
255   uint64_t ExplicitKernArgSize = 0;
256   Align MaxKernArgAlign;
257   uint32_t LDSSize = 0;
258   uint32_t GDSSize = 0;
259   Align DynLDSAlign;
260   bool IsEntryFunction = false;
261   bool IsChainFunction = false;
262   bool NoSignedZerosFPMath = false;
263   bool MemoryBound = false;
264   bool WaveLimiter = false;
265   bool HasSpilledSGPRs = false;
266   bool HasSpilledVGPRs = false;
267   uint32_t HighBitsOf32BitAddress = 0;
268 
269   // TODO: 10 may be a better default since it's the maximum.
270   unsigned Occupancy = 0;
271 
272   SmallVector<StringValue> WWMReservedRegs;
273 
274   StringValue ScratchRSrcReg = "$private_rsrc_reg";
275   StringValue FrameOffsetReg = "$fp_reg";
276   StringValue StackPtrOffsetReg = "$sp_reg";
277 
278   unsigned BytesInStackArgArea = 0;
279   bool ReturnsVoid = true;
280 
281   std::optional<SIArgumentInfo> ArgInfo;
282 
283   unsigned PSInputAddr = 0;
284   unsigned PSInputEnable = 0;
285 
286   SIMode Mode;
287   std::optional<FrameIndex> ScavengeFI;
288   StringValue VGPRForAGPRCopy;
289   StringValue SGPRForEXECCopy;
290   StringValue LongBranchReservedReg;
291 
292   SIMachineFunctionInfo() = default;
293   SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo &,
294                         const TargetRegisterInfo &TRI,
295                         const llvm::MachineFunction &MF);
296 
297   void mappingImpl(yaml::IO &YamlIO) override;
298   ~SIMachineFunctionInfo() = default;
299 };
300 
301 template <> struct MappingTraits<SIMachineFunctionInfo> {
302   static void mapping(IO &YamlIO, SIMachineFunctionInfo &MFI) {
303     YamlIO.mapOptional("explicitKernArgSize", MFI.ExplicitKernArgSize,
304                        UINT64_C(0));
305     YamlIO.mapOptional("maxKernArgAlign", MFI.MaxKernArgAlign);
306     YamlIO.mapOptional("ldsSize", MFI.LDSSize, 0u);
307     YamlIO.mapOptional("gdsSize", MFI.GDSSize, 0u);
308     YamlIO.mapOptional("dynLDSAlign", MFI.DynLDSAlign, Align());
309     YamlIO.mapOptional("isEntryFunction", MFI.IsEntryFunction, false);
310     YamlIO.mapOptional("isChainFunction", MFI.IsChainFunction, false);
311     YamlIO.mapOptional("noSignedZerosFPMath", MFI.NoSignedZerosFPMath, false);
312     YamlIO.mapOptional("memoryBound", MFI.MemoryBound, false);
313     YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
314     YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
315     YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
316     YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg,
317                        StringValue("$private_rsrc_reg"));
318     YamlIO.mapOptional("frameOffsetReg", MFI.FrameOffsetReg,
319                        StringValue("$fp_reg"));
320     YamlIO.mapOptional("stackPtrOffsetReg", MFI.StackPtrOffsetReg,
321                        StringValue("$sp_reg"));
322     YamlIO.mapOptional("bytesInStackArgArea", MFI.BytesInStackArgArea, 0u);
323     YamlIO.mapOptional("returnsVoid", MFI.ReturnsVoid, true);
324     YamlIO.mapOptional("argumentInfo", MFI.ArgInfo);
325     YamlIO.mapOptional("psInputAddr", MFI.PSInputAddr, 0u);
326     YamlIO.mapOptional("psInputEnable", MFI.PSInputEnable, 0u);
327     YamlIO.mapOptional("mode", MFI.Mode, SIMode());
328     YamlIO.mapOptional("highBitsOf32BitAddress",
329                        MFI.HighBitsOf32BitAddress, 0u);
330     YamlIO.mapOptional("occupancy", MFI.Occupancy, 0);
331     YamlIO.mapOptional("wwmReservedRegs", MFI.WWMReservedRegs);
332     YamlIO.mapOptional("scavengeFI", MFI.ScavengeFI);
333     YamlIO.mapOptional("vgprForAGPRCopy", MFI.VGPRForAGPRCopy,
334                        StringValue()); // Don't print out when it's empty.
335     YamlIO.mapOptional("sgprForEXECCopy", MFI.SGPRForEXECCopy,
336                        StringValue()); // Don't print out when it's empty.
337     YamlIO.mapOptional("longBranchReservedReg", MFI.LongBranchReservedReg,
338                        StringValue());
339   }
340 };
341 
342 } // end namespace yaml
343 
344 // A CSR SGPR value can be preserved inside a callee using one of the following
345 // methods.
346 //   1. Copy to an unused scratch SGPR.
347 //   2. Spill to a VGPR lane.
348 //   3. Spill to memory via. a scratch VGPR.
349 // class PrologEpilogSGPRSaveRestoreInfo represents the save/restore method used
350 // for an SGPR at function prolog/epilog.
351 enum class SGPRSaveKind : uint8_t {
352   COPY_TO_SCRATCH_SGPR,
353   SPILL_TO_VGPR_LANE,
354   SPILL_TO_MEM
355 };
356 
357 class PrologEpilogSGPRSaveRestoreInfo {
358   SGPRSaveKind Kind;
359   union {
360     int Index;
361     Register Reg;
362   };
363 
364 public:
365   PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, int I) : Kind(K), Index(I) {}
366   PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, Register R)
367       : Kind(K), Reg(R) {}
368   Register getReg() const { return Reg; }
369   int getIndex() const { return Index; }
370   SGPRSaveKind getKind() const { return Kind; }
371 };
372 
373 /// This class keeps track of the SPI_SP_INPUT_ADDR config register, which
374 /// tells the hardware which interpolation parameters to load.
375 class SIMachineFunctionInfo final : public AMDGPUMachineFunction,
376                                     private MachineRegisterInfo::Delegate {
377   friend class GCNTargetMachine;
378 
379   // State of MODE register, assumed FP mode.
380   SIModeRegisterDefaults Mode;
381 
382   // Registers that may be reserved for spilling purposes. These may be the same
383   // as the input registers.
384   Register ScratchRSrcReg = AMDGPU::PRIVATE_RSRC_REG;
385 
386   // This is the unswizzled offset from the current dispatch's scratch wave
387   // base to the beginning of the current function's frame.
388   Register FrameOffsetReg = AMDGPU::FP_REG;
389 
390   // This is an ABI register used in the non-entry calling convention to
391   // communicate the unswizzled offset from the current dispatch's scratch wave
392   // base to the beginning of the new function's frame.
393   Register StackPtrOffsetReg = AMDGPU::SP_REG;
394 
395   // Registers that may be reserved when RA doesn't allocate enough
396   // registers to plan for the case where an indirect branch ends up
397   // being needed during branch relaxation.
398   Register LongBranchReservedReg;
399 
400   AMDGPUFunctionArgInfo ArgInfo;
401 
402   // Graphics info.
403   unsigned PSInputAddr = 0;
404   unsigned PSInputEnable = 0;
405 
406   /// Number of bytes of arguments this function has on the stack. If the callee
407   /// is expected to restore the argument stack this should be a multiple of 16,
408   /// all usable during a tail call.
409   ///
410   /// The alternative would forbid tail call optimisation in some cases: if we
411   /// want to transfer control from a function with 8-bytes of stack-argument
412   /// space to a function with 16-bytes then misalignment of this value would
413   /// make a stack adjustment necessary, which could not be undone by the
414   /// callee.
415   unsigned BytesInStackArgArea = 0;
416 
417   bool ReturnsVoid = true;
418 
419   // A pair of default/requested minimum/maximum flat work group sizes.
420   // Minimum - first, maximum - second.
421   std::pair<unsigned, unsigned> FlatWorkGroupSizes = {0, 0};
422 
423   // A pair of default/requested minimum/maximum number of waves per execution
424   // unit. Minimum - first, maximum - second.
425   std::pair<unsigned, unsigned> WavesPerEU = {0, 0};
426 
427   const AMDGPUGWSResourcePseudoSourceValue GWSResourcePSV;
428 
429 private:
430   unsigned NumUserSGPRs = 0;
431   unsigned NumSystemSGPRs = 0;
432 
433   bool HasSpilledSGPRs = false;
434   bool HasSpilledVGPRs = false;
435   bool HasNonSpillStackObjects = false;
436   bool IsStackRealigned = false;
437 
438   unsigned NumSpilledSGPRs = 0;
439   unsigned NumSpilledVGPRs = 0;
440 
441   // Tracks information about user SGPRs that will be setup by hardware which
442   // will apply to all wavefronts of the grid.
443   GCNUserSGPRUsageInfo UserSGPRInfo;
444 
445   // Feature bits required for inputs passed in system SGPRs.
446   bool WorkGroupIDX : 1; // Always initialized.
447   bool WorkGroupIDY : 1;
448   bool WorkGroupIDZ : 1;
449   bool WorkGroupInfo : 1;
450   bool LDSKernelId : 1;
451   bool PrivateSegmentWaveByteOffset : 1;
452 
453   bool WorkItemIDX : 1; // Always initialized.
454   bool WorkItemIDY : 1;
455   bool WorkItemIDZ : 1;
456 
457   // Pointer to where the ABI inserts special kernel arguments separate from the
458   // user arguments. This is an offset from the KernargSegmentPtr.
459   bool ImplicitArgPtr : 1;
460 
461   bool MayNeedAGPRs : 1;
462 
463   // The hard-wired high half of the address of the global information table
464   // for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since
465   // current hardware only allows a 16 bit value.
466   unsigned GITPtrHigh;
467 
468   unsigned HighBitsOf32BitAddress;
469 
470   // Flags associated with the virtual registers.
471   IndexedMap<uint8_t, VirtReg2IndexFunctor> VRegFlags;
472 
473   // Current recorded maximum possible occupancy.
474   unsigned Occupancy;
475 
476   mutable std::optional<bool> UsesAGPRs;
477 
478   MCPhysReg getNextUserSGPR() const;
479 
480   MCPhysReg getNextSystemSGPR() const;
481 
482   // MachineRegisterInfo callback functions to notify events.
483   void MRI_NoteNewVirtualRegister(Register Reg) override;
484   void MRI_NoteCloneVirtualRegister(Register NewReg, Register SrcReg) override;
485 
486 public:
487   struct VGPRSpillToAGPR {
488     SmallVector<MCPhysReg, 32> Lanes;
489     bool FullyAllocated = false;
490     bool IsDead = false;
491   };
492 
493 private:
494   // To track virtual VGPR + lane index for each subregister of the SGPR spilled
495   // to frameindex key during SILowerSGPRSpills pass.
496   DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>>
497       SGPRSpillsToVirtualVGPRLanes;
498   // To track physical VGPR + lane index for CSR SGPR spills and special SGPRs
499   // like Frame Pointer identified during PrologEpilogInserter.
500   DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>>
501       SGPRSpillsToPhysicalVGPRLanes;
502   unsigned NumVirtualVGPRSpillLanes = 0;
503   unsigned NumPhysicalVGPRSpillLanes = 0;
504   SmallVector<Register, 2> SpillVGPRs;
505   SmallVector<Register, 2> SpillPhysVGPRs;
506   using WWMSpillsMap = MapVector<Register, int>;
507   // To track the registers used in instructions that can potentially modify the
508   // inactive lanes. The WWM instructions and the writelane instructions for
509   // spilling SGPRs to VGPRs fall under such category of operations. The VGPRs
510   // modified by them should be spilled/restored at function prolog/epilog to
511   // avoid any undesired outcome. Each entry in this map holds a pair of values,
512   // the VGPR and its stack slot index.
513   WWMSpillsMap WWMSpills;
514 
515   using ReservedRegSet = SmallSetVector<Register, 8>;
516   // To track the VGPRs reserved for WWM instructions. They get stack slots
517   // later during PrologEpilogInserter and get added into the superset WWMSpills
518   // for actual spilling. A separate set makes the register reserved part and
519   // the serialization easier.
520   ReservedRegSet WWMReservedRegs;
521 
522   using PrologEpilogSGPRSpillsMap =
523       DenseMap<Register, PrologEpilogSGPRSaveRestoreInfo>;
524   // To track the SGPR spill method used for a CSR SGPR register during
525   // frame lowering. Even though the SGPR spills are handled during
526   // SILowerSGPRSpills pass, some special handling needed later during the
527   // PrologEpilogInserter.
528   PrologEpilogSGPRSpillsMap PrologEpilogSGPRSpills;
529 
530   // To save/restore EXEC MASK around WWM spills and copies.
531   Register SGPRForEXECCopy;
532 
533   DenseMap<int, VGPRSpillToAGPR> VGPRToAGPRSpills;
534 
535   // AGPRs used for VGPR spills.
536   SmallVector<MCPhysReg, 32> SpillAGPR;
537 
538   // VGPRs used for AGPR spills.
539   SmallVector<MCPhysReg, 32> SpillVGPR;
540 
541   // Emergency stack slot. Sometimes, we create this before finalizing the stack
542   // frame, so save it here and add it to the RegScavenger later.
543   std::optional<int> ScavengeFI;
544 
545 private:
546   Register VGPRForAGPRCopy;
547 
548   bool allocateVirtualVGPRForSGPRSpills(MachineFunction &MF, int FI,
549                                         unsigned LaneIndex);
550   bool allocatePhysicalVGPRForSGPRSpills(MachineFunction &MF, int FI,
551                                          unsigned LaneIndex);
552 
553 public:
554   Register getVGPRForAGPRCopy() const {
555     return VGPRForAGPRCopy;
556   }
557 
558   void setVGPRForAGPRCopy(Register NewVGPRForAGPRCopy) {
559     VGPRForAGPRCopy = NewVGPRForAGPRCopy;
560   }
561 
562   bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const;
563 
564 public:
565   SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI) = default;
566   SIMachineFunctionInfo(const Function &F, const GCNSubtarget *STI);
567 
568   MachineFunctionInfo *
569   clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF,
570         const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
571       const override;
572 
573   bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI,
574                                 const MachineFunction &MF,
575                                 PerFunctionMIParsingState &PFS,
576                                 SMDiagnostic &Error, SMRange &SourceRange);
577 
578   void reserveWWMRegister(Register Reg) { WWMReservedRegs.insert(Reg); }
579 
580   SIModeRegisterDefaults getMode() const { return Mode; }
581 
582   ArrayRef<SIRegisterInfo::SpilledReg>
583   getSGPRSpillToVirtualVGPRLanes(int FrameIndex) const {
584     auto I = SGPRSpillsToVirtualVGPRLanes.find(FrameIndex);
585     return (I == SGPRSpillsToVirtualVGPRLanes.end())
586                ? ArrayRef<SIRegisterInfo::SpilledReg>()
587                : ArrayRef(I->second);
588   }
589 
590   ArrayRef<Register> getSGPRSpillVGPRs() const { return SpillVGPRs; }
591   const WWMSpillsMap &getWWMSpills() const { return WWMSpills; }
592   const ReservedRegSet &getWWMReservedRegs() const { return WWMReservedRegs; }
593 
594   const PrologEpilogSGPRSpillsMap &getPrologEpilogSGPRSpills() const {
595     return PrologEpilogSGPRSpills;
596   }
597 
598   GCNUserSGPRUsageInfo &getUserSGPRInfo() { return UserSGPRInfo; }
599 
600   const GCNUserSGPRUsageInfo &getUserSGPRInfo() const { return UserSGPRInfo; }
601 
602   void addToPrologEpilogSGPRSpills(Register Reg,
603                                    PrologEpilogSGPRSaveRestoreInfo SI) {
604     PrologEpilogSGPRSpills.insert(std::make_pair(Reg, SI));
605   }
606 
607   // Check if an entry created for \p Reg in PrologEpilogSGPRSpills. Return true
608   // on success and false otherwise.
609   bool hasPrologEpilogSGPRSpillEntry(Register Reg) const {
610     return PrologEpilogSGPRSpills.contains(Reg);
611   }
612 
613   // Get the scratch SGPR if allocated to save/restore \p Reg.
614   Register getScratchSGPRCopyDstReg(Register Reg) const {
615     auto I = PrologEpilogSGPRSpills.find(Reg);
616     if (I != PrologEpilogSGPRSpills.end() &&
617         I->second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR)
618       return I->second.getReg();
619 
620     return AMDGPU::NoRegister;
621   }
622 
623   // Get all scratch SGPRs allocated to copy/restore the SGPR spills.
624   void getAllScratchSGPRCopyDstRegs(SmallVectorImpl<Register> &Regs) const {
625     for (const auto &SI : PrologEpilogSGPRSpills) {
626       if (SI.second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR)
627         Regs.push_back(SI.second.getReg());
628     }
629   }
630 
631   // Check if \p FI is allocated for any SGPR spill to a VGPR lane during PEI.
632   bool checkIndexInPrologEpilogSGPRSpills(int FI) const {
633     return find_if(PrologEpilogSGPRSpills,
634                    [FI](const std::pair<Register,
635                                         PrologEpilogSGPRSaveRestoreInfo> &SI) {
636                      return SI.second.getKind() ==
637                                 SGPRSaveKind::SPILL_TO_VGPR_LANE &&
638                             SI.second.getIndex() == FI;
639                    }) != PrologEpilogSGPRSpills.end();
640   }
641 
642   const PrologEpilogSGPRSaveRestoreInfo &
643   getPrologEpilogSGPRSaveRestoreInfo(Register Reg) const {
644     auto I = PrologEpilogSGPRSpills.find(Reg);
645     assert(I != PrologEpilogSGPRSpills.end());
646 
647     return I->second;
648   }
649 
650   ArrayRef<SIRegisterInfo::SpilledReg>
651   getSGPRSpillToPhysicalVGPRLanes(int FrameIndex) const {
652     auto I = SGPRSpillsToPhysicalVGPRLanes.find(FrameIndex);
653     return (I == SGPRSpillsToPhysicalVGPRLanes.end())
654                ? ArrayRef<SIRegisterInfo::SpilledReg>()
655                : ArrayRef(I->second);
656   }
657 
658   void setFlag(Register Reg, uint8_t Flag) {
659     assert(Reg.isVirtual());
660     if (VRegFlags.inBounds(Reg))
661       VRegFlags[Reg] |= Flag;
662   }
663 
664   bool checkFlag(Register Reg, uint8_t Flag) const {
665     if (Reg.isPhysical())
666       return false;
667 
668     return VRegFlags.inBounds(Reg) && VRegFlags[Reg] & Flag;
669   }
670 
671   bool hasVRegFlags() { return VRegFlags.size(); }
672 
673   void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size = 4,
674                         Align Alignment = Align(4));
675 
676   void splitWWMSpillRegisters(
677       MachineFunction &MF,
678       SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
679       SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const;
680 
681   ArrayRef<MCPhysReg> getAGPRSpillVGPRs() const {
682     return SpillAGPR;
683   }
684 
685   Register getSGPRForEXECCopy() const { return SGPRForEXECCopy; }
686 
687   void setSGPRForEXECCopy(Register Reg) { SGPRForEXECCopy = Reg; }
688 
689   ArrayRef<MCPhysReg> getVGPRSpillAGPRs() const {
690     return SpillVGPR;
691   }
692 
693   MCPhysReg getVGPRToAGPRSpill(int FrameIndex, unsigned Lane) const {
694     auto I = VGPRToAGPRSpills.find(FrameIndex);
695     return (I == VGPRToAGPRSpills.end()) ? (MCPhysReg)AMDGPU::NoRegister
696                                          : I->second.Lanes[Lane];
697   }
698 
699   void setVGPRToAGPRSpillDead(int FrameIndex) {
700     auto I = VGPRToAGPRSpills.find(FrameIndex);
701     if (I != VGPRToAGPRSpills.end())
702       I->second.IsDead = true;
703   }
704 
705   bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI,
706                                    bool IsPrologEpilog = false);
707   bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR);
708 
709   /// If \p ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill
710   /// to the default stack.
711   bool removeDeadFrameIndices(MachineFrameInfo &MFI,
712                               bool ResetSGPRSpillStackIDs);
713 
714   int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI);
715   std::optional<int> getOptionalScavengeFI() const { return ScavengeFI; }
716 
717   unsigned getBytesInStackArgArea() const {
718     return BytesInStackArgArea;
719   }
720 
721   void setBytesInStackArgArea(unsigned Bytes) {
722     BytesInStackArgArea = Bytes;
723   }
724 
725   // Add user SGPRs.
726   Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI);
727   Register addDispatchPtr(const SIRegisterInfo &TRI);
728   Register addQueuePtr(const SIRegisterInfo &TRI);
729   Register addKernargSegmentPtr(const SIRegisterInfo &TRI);
730   Register addDispatchID(const SIRegisterInfo &TRI);
731   Register addFlatScratchInit(const SIRegisterInfo &TRI);
732   Register addImplicitBufferPtr(const SIRegisterInfo &TRI);
733   Register addLDSKernelId();
734   SmallVectorImpl<MCRegister> *
735   addPreloadedKernArg(const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
736                       unsigned AllocSizeDWord, int KernArgIdx,
737                       int PaddingSGPRs);
738 
739   /// Increment user SGPRs used for padding the argument list only.
740   Register addReservedUserSGPR() {
741     Register Next = getNextUserSGPR();
742     ++NumUserSGPRs;
743     return Next;
744   }
745 
746   // Add system SGPRs.
747   Register addWorkGroupIDX(bool HasArchitectedSGPRs) {
748     Register Reg =
749         HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP9 : getNextSystemSGPR();
750     ArgInfo.WorkGroupIDX = ArgDescriptor::createRegister(Reg);
751     if (!HasArchitectedSGPRs)
752       NumSystemSGPRs += 1;
753 
754     return ArgInfo.WorkGroupIDX.getRegister();
755   }
756 
757   Register addWorkGroupIDY(bool HasArchitectedSGPRs) {
758     Register Reg =
759         HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP7 : getNextSystemSGPR();
760     unsigned Mask = HasArchitectedSGPRs && hasWorkGroupIDZ() ? 0xffff : ~0u;
761     ArgInfo.WorkGroupIDY = ArgDescriptor::createRegister(Reg, Mask);
762     if (!HasArchitectedSGPRs)
763       NumSystemSGPRs += 1;
764 
765     return ArgInfo.WorkGroupIDY.getRegister();
766   }
767 
768   Register addWorkGroupIDZ(bool HasArchitectedSGPRs) {
769     Register Reg =
770         HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP7 : getNextSystemSGPR();
771     unsigned Mask = HasArchitectedSGPRs ? 0xffff << 16 : ~0u;
772     ArgInfo.WorkGroupIDZ = ArgDescriptor::createRegister(Reg, Mask);
773     if (!HasArchitectedSGPRs)
774       NumSystemSGPRs += 1;
775 
776     return ArgInfo.WorkGroupIDZ.getRegister();
777   }
778 
779   Register addWorkGroupInfo() {
780     ArgInfo.WorkGroupInfo = ArgDescriptor::createRegister(getNextSystemSGPR());
781     NumSystemSGPRs += 1;
782     return ArgInfo.WorkGroupInfo.getRegister();
783   }
784 
785   bool hasLDSKernelId() const { return LDSKernelId; }
786 
787   // Add special VGPR inputs
788   void setWorkItemIDX(ArgDescriptor Arg) {
789     ArgInfo.WorkItemIDX = Arg;
790   }
791 
792   void setWorkItemIDY(ArgDescriptor Arg) {
793     ArgInfo.WorkItemIDY = Arg;
794   }
795 
796   void setWorkItemIDZ(ArgDescriptor Arg) {
797     ArgInfo.WorkItemIDZ = Arg;
798   }
799 
800   Register addPrivateSegmentWaveByteOffset() {
801     ArgInfo.PrivateSegmentWaveByteOffset
802       = ArgDescriptor::createRegister(getNextSystemSGPR());
803     NumSystemSGPRs += 1;
804     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
805   }
806 
807   void setPrivateSegmentWaveByteOffset(Register Reg) {
808     ArgInfo.PrivateSegmentWaveByteOffset = ArgDescriptor::createRegister(Reg);
809   }
810 
811   bool hasWorkGroupIDX() const {
812     return WorkGroupIDX;
813   }
814 
815   bool hasWorkGroupIDY() const {
816     return WorkGroupIDY;
817   }
818 
819   bool hasWorkGroupIDZ() const {
820     return WorkGroupIDZ;
821   }
822 
823   bool hasWorkGroupInfo() const {
824     return WorkGroupInfo;
825   }
826 
827   bool hasPrivateSegmentWaveByteOffset() const {
828     return PrivateSegmentWaveByteOffset;
829   }
830 
831   bool hasWorkItemIDX() const {
832     return WorkItemIDX;
833   }
834 
835   bool hasWorkItemIDY() const {
836     return WorkItemIDY;
837   }
838 
839   bool hasWorkItemIDZ() const {
840     return WorkItemIDZ;
841   }
842 
843   bool hasImplicitArgPtr() const {
844     return ImplicitArgPtr;
845   }
846 
847   AMDGPUFunctionArgInfo &getArgInfo() {
848     return ArgInfo;
849   }
850 
851   const AMDGPUFunctionArgInfo &getArgInfo() const {
852     return ArgInfo;
853   }
854 
855   std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
856   getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
857     return ArgInfo.getPreloadedValue(Value);
858   }
859 
860   MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
861     auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value));
862     return Arg ? Arg->getRegister() : MCRegister();
863   }
864 
865   unsigned getGITPtrHigh() const {
866     return GITPtrHigh;
867   }
868 
869   Register getGITPtrLoReg(const MachineFunction &MF) const;
870 
871   uint32_t get32BitAddressHighBits() const {
872     return HighBitsOf32BitAddress;
873   }
874 
875   unsigned getNumUserSGPRs() const {
876     return NumUserSGPRs;
877   }
878 
879   unsigned getNumPreloadedSGPRs() const {
880     return NumUserSGPRs + NumSystemSGPRs;
881   }
882 
883   unsigned getNumKernargPreloadedSGPRs() const {
884     return UserSGPRInfo.getNumKernargPreloadSGPRs();
885   }
886 
887   Register getPrivateSegmentWaveByteOffsetSystemSGPR() const {
888     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
889   }
890 
891   /// Returns the physical register reserved for use as the resource
892   /// descriptor for scratch accesses.
893   Register getScratchRSrcReg() const {
894     return ScratchRSrcReg;
895   }
896 
897   void setScratchRSrcReg(Register Reg) {
898     assert(Reg != 0 && "Should never be unset");
899     ScratchRSrcReg = Reg;
900   }
901 
902   Register getFrameOffsetReg() const {
903     return FrameOffsetReg;
904   }
905 
906   void setFrameOffsetReg(Register Reg) {
907     assert(Reg != 0 && "Should never be unset");
908     FrameOffsetReg = Reg;
909   }
910 
911   void setStackPtrOffsetReg(Register Reg) {
912     assert(Reg != 0 && "Should never be unset");
913     StackPtrOffsetReg = Reg;
914   }
915 
916   void setLongBranchReservedReg(Register Reg) { LongBranchReservedReg = Reg; }
917 
918   // Note the unset value for this is AMDGPU::SP_REG rather than
919   // NoRegister. This is mostly a workaround for MIR tests where state that
920   // can't be directly computed from the function is not preserved in serialized
921   // MIR.
922   Register getStackPtrOffsetReg() const {
923     return StackPtrOffsetReg;
924   }
925 
926   Register getLongBranchReservedReg() const { return LongBranchReservedReg; }
927 
928   Register getQueuePtrUserSGPR() const {
929     return ArgInfo.QueuePtr.getRegister();
930   }
931 
932   Register getImplicitBufferPtrUserSGPR() const {
933     return ArgInfo.ImplicitBufferPtr.getRegister();
934   }
935 
936   bool hasSpilledSGPRs() const {
937     return HasSpilledSGPRs;
938   }
939 
940   void setHasSpilledSGPRs(bool Spill = true) {
941     HasSpilledSGPRs = Spill;
942   }
943 
944   bool hasSpilledVGPRs() const {
945     return HasSpilledVGPRs;
946   }
947 
948   void setHasSpilledVGPRs(bool Spill = true) {
949     HasSpilledVGPRs = Spill;
950   }
951 
952   bool hasNonSpillStackObjects() const {
953     return HasNonSpillStackObjects;
954   }
955 
956   void setHasNonSpillStackObjects(bool StackObject = true) {
957     HasNonSpillStackObjects = StackObject;
958   }
959 
960   bool isStackRealigned() const {
961     return IsStackRealigned;
962   }
963 
964   void setIsStackRealigned(bool Realigned = true) {
965     IsStackRealigned = Realigned;
966   }
967 
968   unsigned getNumSpilledSGPRs() const {
969     return NumSpilledSGPRs;
970   }
971 
972   unsigned getNumSpilledVGPRs() const {
973     return NumSpilledVGPRs;
974   }
975 
976   void addToSpilledSGPRs(unsigned num) {
977     NumSpilledSGPRs += num;
978   }
979 
980   void addToSpilledVGPRs(unsigned num) {
981     NumSpilledVGPRs += num;
982   }
983 
984   unsigned getPSInputAddr() const {
985     return PSInputAddr;
986   }
987 
988   unsigned getPSInputEnable() const {
989     return PSInputEnable;
990   }
991 
992   bool isPSInputAllocated(unsigned Index) const {
993     return PSInputAddr & (1 << Index);
994   }
995 
996   void markPSInputAllocated(unsigned Index) {
997     PSInputAddr |= 1 << Index;
998   }
999 
1000   void markPSInputEnabled(unsigned Index) {
1001     PSInputEnable |= 1 << Index;
1002   }
1003 
1004   bool returnsVoid() const {
1005     return ReturnsVoid;
1006   }
1007 
1008   void setIfReturnsVoid(bool Value) {
1009     ReturnsVoid = Value;
1010   }
1011 
1012   /// \returns A pair of default/requested minimum/maximum flat work group sizes
1013   /// for this function.
1014   std::pair<unsigned, unsigned> getFlatWorkGroupSizes() const {
1015     return FlatWorkGroupSizes;
1016   }
1017 
1018   /// \returns Default/requested minimum flat work group size for this function.
1019   unsigned getMinFlatWorkGroupSize() const {
1020     return FlatWorkGroupSizes.first;
1021   }
1022 
1023   /// \returns Default/requested maximum flat work group size for this function.
1024   unsigned getMaxFlatWorkGroupSize() const {
1025     return FlatWorkGroupSizes.second;
1026   }
1027 
1028   /// \returns A pair of default/requested minimum/maximum number of waves per
1029   /// execution unit.
1030   std::pair<unsigned, unsigned> getWavesPerEU() const {
1031     return WavesPerEU;
1032   }
1033 
1034   /// \returns Default/requested minimum number of waves per execution unit.
1035   unsigned getMinWavesPerEU() const {
1036     return WavesPerEU.first;
1037   }
1038 
1039   /// \returns Default/requested maximum number of waves per execution unit.
1040   unsigned getMaxWavesPerEU() const {
1041     return WavesPerEU.second;
1042   }
1043 
1044   /// \returns SGPR used for \p Dim's work group ID.
1045   Register getWorkGroupIDSGPR(unsigned Dim) const {
1046     switch (Dim) {
1047     case 0:
1048       assert(hasWorkGroupIDX());
1049       return ArgInfo.WorkGroupIDX.getRegister();
1050     case 1:
1051       assert(hasWorkGroupIDY());
1052       return ArgInfo.WorkGroupIDY.getRegister();
1053     case 2:
1054       assert(hasWorkGroupIDZ());
1055       return ArgInfo.WorkGroupIDZ.getRegister();
1056     }
1057     llvm_unreachable("unexpected dimension");
1058   }
1059 
1060   const AMDGPUGWSResourcePseudoSourceValue *
1061   getGWSPSV(const AMDGPUTargetMachine &TM) {
1062     return &GWSResourcePSV;
1063   }
1064 
1065   unsigned getOccupancy() const {
1066     return Occupancy;
1067   }
1068 
1069   unsigned getMinAllowedOccupancy() const {
1070     if (!isMemoryBound() && !needsWaveLimiter())
1071       return Occupancy;
1072     return (Occupancy < 4) ? Occupancy : 4;
1073   }
1074 
1075   void limitOccupancy(const MachineFunction &MF);
1076 
1077   void limitOccupancy(unsigned Limit) {
1078     if (Occupancy > Limit)
1079       Occupancy = Limit;
1080   }
1081 
1082   void increaseOccupancy(const MachineFunction &MF, unsigned Limit) {
1083     if (Occupancy < Limit)
1084       Occupancy = Limit;
1085     limitOccupancy(MF);
1086   }
1087 
1088   bool mayNeedAGPRs() const {
1089     return MayNeedAGPRs;
1090   }
1091 
1092   // \returns true if a function has a use of AGPRs via inline asm or
1093   // has a call which may use it.
1094   bool mayUseAGPRs(const Function &F) const;
1095 
1096   // \returns true if a function needs or may need AGPRs.
1097   bool usesAGPRs(const MachineFunction &MF) const;
1098 };
1099 
1100 } // end namespace llvm
1101 
1102 #endif // LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
1103