1 //==- SIMachineFunctionInfo.h - SIMachineFunctionInfo interface --*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
14 #define LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
15 
16 #include "AMDGPUArgumentUsageInfo.h"
17 #include "AMDGPUMachineFunction.h"
18 #include "AMDGPUTargetMachine.h"
19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20 #include "SIInstrInfo.h"
21 #include "SIModeRegisterDefaults.h"
22 #include "llvm/ADT/SetVector.h"
23 #include "llvm/CodeGen/MIRYamlMapping.h"
24 #include "llvm/CodeGen/PseudoSourceValue.h"
25 #include "llvm/Support/raw_ostream.h"
26 #include <optional>
27 
28 namespace llvm {
29 
30 class MachineFrameInfo;
31 class MachineFunction;
32 class SIMachineFunctionInfo;
33 class SIRegisterInfo;
34 class TargetRegisterClass;
35 
36 class AMDGPUPseudoSourceValue : public PseudoSourceValue {
37 public:
38   enum AMDGPUPSVKind : unsigned {
39     PSVImage = PseudoSourceValue::TargetCustom,
40     GWSResource
41   };
42 
43 protected:
44   AMDGPUPseudoSourceValue(unsigned Kind, const AMDGPUTargetMachine &TM)
45       : PseudoSourceValue(Kind, TM) {}
46 
47 public:
48   bool isConstant(const MachineFrameInfo *) const override {
49     // This should probably be true for most images, but we will start by being
50     // conservative.
51     return false;
52   }
53 
54   bool isAliased(const MachineFrameInfo *) const override {
55     return true;
56   }
57 
58   bool mayAlias(const MachineFrameInfo *) const override {
59     return true;
60   }
61 };
62 
63 class AMDGPUGWSResourcePseudoSourceValue final : public AMDGPUPseudoSourceValue {
64 public:
65   explicit AMDGPUGWSResourcePseudoSourceValue(const AMDGPUTargetMachine &TM)
66       : AMDGPUPseudoSourceValue(GWSResource, TM) {}
67 
68   static bool classof(const PseudoSourceValue *V) {
69     return V->kind() == GWSResource;
70   }
71 
72   // These are inaccessible memory from IR.
73   bool isAliased(const MachineFrameInfo *) const override {
74     return false;
75   }
76 
77   // These are inaccessible memory from IR.
78   bool mayAlias(const MachineFrameInfo *) const override {
79     return false;
80   }
81 
82   void printCustom(raw_ostream &OS) const override {
83     OS << "GWSResource";
84   }
85 };
86 
87 namespace yaml {
88 
89 struct SIArgument {
90   bool IsRegister;
91   union {
92     StringValue RegisterName;
93     unsigned StackOffset;
94   };
95   std::optional<unsigned> Mask;
96 
97   // Default constructor, which creates a stack argument.
98   SIArgument() : IsRegister(false), StackOffset(0) {}
99   SIArgument(const SIArgument &Other) {
100     IsRegister = Other.IsRegister;
101     if (IsRegister) {
102       ::new ((void *)std::addressof(RegisterName))
103           StringValue(Other.RegisterName);
104     } else
105       StackOffset = Other.StackOffset;
106     Mask = Other.Mask;
107   }
108   SIArgument &operator=(const SIArgument &Other) {
109     IsRegister = Other.IsRegister;
110     if (IsRegister) {
111       ::new ((void *)std::addressof(RegisterName))
112           StringValue(Other.RegisterName);
113     } else
114       StackOffset = Other.StackOffset;
115     Mask = Other.Mask;
116     return *this;
117   }
118   ~SIArgument() {
119     if (IsRegister)
120       RegisterName.~StringValue();
121   }
122 
123   // Helper to create a register or stack argument.
124   static inline SIArgument createArgument(bool IsReg) {
125     if (IsReg)
126       return SIArgument(IsReg);
127     return SIArgument();
128   }
129 
130 private:
131   // Construct a register argument.
132   SIArgument(bool) : IsRegister(true), RegisterName() {}
133 };
134 
135 template <> struct MappingTraits<SIArgument> {
136   static void mapping(IO &YamlIO, SIArgument &A) {
137     if (YamlIO.outputting()) {
138       if (A.IsRegister)
139         YamlIO.mapRequired("reg", A.RegisterName);
140       else
141         YamlIO.mapRequired("offset", A.StackOffset);
142     } else {
143       auto Keys = YamlIO.keys();
144       if (is_contained(Keys, "reg")) {
145         A = SIArgument::createArgument(true);
146         YamlIO.mapRequired("reg", A.RegisterName);
147       } else if (is_contained(Keys, "offset"))
148         YamlIO.mapRequired("offset", A.StackOffset);
149       else
150         YamlIO.setError("missing required key 'reg' or 'offset'");
151     }
152     YamlIO.mapOptional("mask", A.Mask);
153   }
154   static const bool flow = true;
155 };
156 
157 struct SIArgumentInfo {
158   std::optional<SIArgument> PrivateSegmentBuffer;
159   std::optional<SIArgument> DispatchPtr;
160   std::optional<SIArgument> QueuePtr;
161   std::optional<SIArgument> KernargSegmentPtr;
162   std::optional<SIArgument> DispatchID;
163   std::optional<SIArgument> FlatScratchInit;
164   std::optional<SIArgument> PrivateSegmentSize;
165 
166   std::optional<SIArgument> WorkGroupIDX;
167   std::optional<SIArgument> WorkGroupIDY;
168   std::optional<SIArgument> WorkGroupIDZ;
169   std::optional<SIArgument> WorkGroupInfo;
170   std::optional<SIArgument> LDSKernelId;
171   std::optional<SIArgument> PrivateSegmentWaveByteOffset;
172 
173   std::optional<SIArgument> ImplicitArgPtr;
174   std::optional<SIArgument> ImplicitBufferPtr;
175 
176   std::optional<SIArgument> WorkItemIDX;
177   std::optional<SIArgument> WorkItemIDY;
178   std::optional<SIArgument> WorkItemIDZ;
179 };
180 
181 template <> struct MappingTraits<SIArgumentInfo> {
182   static void mapping(IO &YamlIO, SIArgumentInfo &AI) {
183     YamlIO.mapOptional("privateSegmentBuffer", AI.PrivateSegmentBuffer);
184     YamlIO.mapOptional("dispatchPtr", AI.DispatchPtr);
185     YamlIO.mapOptional("queuePtr", AI.QueuePtr);
186     YamlIO.mapOptional("kernargSegmentPtr", AI.KernargSegmentPtr);
187     YamlIO.mapOptional("dispatchID", AI.DispatchID);
188     YamlIO.mapOptional("flatScratchInit", AI.FlatScratchInit);
189     YamlIO.mapOptional("privateSegmentSize", AI.PrivateSegmentSize);
190 
191     YamlIO.mapOptional("workGroupIDX", AI.WorkGroupIDX);
192     YamlIO.mapOptional("workGroupIDY", AI.WorkGroupIDY);
193     YamlIO.mapOptional("workGroupIDZ", AI.WorkGroupIDZ);
194     YamlIO.mapOptional("workGroupInfo", AI.WorkGroupInfo);
195     YamlIO.mapOptional("LDSKernelId", AI.LDSKernelId);
196     YamlIO.mapOptional("privateSegmentWaveByteOffset",
197                        AI.PrivateSegmentWaveByteOffset);
198 
199     YamlIO.mapOptional("implicitArgPtr", AI.ImplicitArgPtr);
200     YamlIO.mapOptional("implicitBufferPtr", AI.ImplicitBufferPtr);
201 
202     YamlIO.mapOptional("workItemIDX", AI.WorkItemIDX);
203     YamlIO.mapOptional("workItemIDY", AI.WorkItemIDY);
204     YamlIO.mapOptional("workItemIDZ", AI.WorkItemIDZ);
205   }
206 };
207 
208 // Default to default mode for default calling convention.
209 struct SIMode {
210   bool IEEE = true;
211   bool DX10Clamp = true;
212   bool FP32InputDenormals = true;
213   bool FP32OutputDenormals = true;
214   bool FP64FP16InputDenormals = true;
215   bool FP64FP16OutputDenormals = true;
216 
217   SIMode() = default;
218 
219   SIMode(const SIModeRegisterDefaults &Mode) {
220     IEEE = Mode.IEEE;
221     DX10Clamp = Mode.DX10Clamp;
222     FP32InputDenormals = Mode.FP32Denormals.Input != DenormalMode::PreserveSign;
223     FP32OutputDenormals =
224         Mode.FP32Denormals.Output != DenormalMode::PreserveSign;
225     FP64FP16InputDenormals =
226         Mode.FP64FP16Denormals.Input != DenormalMode::PreserveSign;
227     FP64FP16OutputDenormals =
228         Mode.FP64FP16Denormals.Output != DenormalMode::PreserveSign;
229   }
230 
231   bool operator ==(const SIMode Other) const {
232     return IEEE == Other.IEEE &&
233            DX10Clamp == Other.DX10Clamp &&
234            FP32InputDenormals == Other.FP32InputDenormals &&
235            FP32OutputDenormals == Other.FP32OutputDenormals &&
236            FP64FP16InputDenormals == Other.FP64FP16InputDenormals &&
237            FP64FP16OutputDenormals == Other.FP64FP16OutputDenormals;
238   }
239 };
240 
241 template <> struct MappingTraits<SIMode> {
242   static void mapping(IO &YamlIO, SIMode &Mode) {
243     YamlIO.mapOptional("ieee", Mode.IEEE, true);
244     YamlIO.mapOptional("dx10-clamp", Mode.DX10Clamp, true);
245     YamlIO.mapOptional("fp32-input-denormals", Mode.FP32InputDenormals, true);
246     YamlIO.mapOptional("fp32-output-denormals", Mode.FP32OutputDenormals, true);
247     YamlIO.mapOptional("fp64-fp16-input-denormals", Mode.FP64FP16InputDenormals, true);
248     YamlIO.mapOptional("fp64-fp16-output-denormals", Mode.FP64FP16OutputDenormals, true);
249   }
250 };
251 
252 struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
253   uint64_t ExplicitKernArgSize = 0;
254   Align MaxKernArgAlign;
255   uint32_t LDSSize = 0;
256   uint32_t GDSSize = 0;
257   Align DynLDSAlign;
258   bool IsEntryFunction = false;
259   bool NoSignedZerosFPMath = false;
260   bool MemoryBound = false;
261   bool WaveLimiter = false;
262   bool HasSpilledSGPRs = false;
263   bool HasSpilledVGPRs = false;
264   uint32_t HighBitsOf32BitAddress = 0;
265 
266   // TODO: 10 may be a better default since it's the maximum.
267   unsigned Occupancy = 0;
268 
269   SmallVector<StringValue> WWMReservedRegs;
270 
271   StringValue ScratchRSrcReg = "$private_rsrc_reg";
272   StringValue FrameOffsetReg = "$fp_reg";
273   StringValue StackPtrOffsetReg = "$sp_reg";
274 
275   unsigned BytesInStackArgArea = 0;
276   bool ReturnsVoid = true;
277 
278   std::optional<SIArgumentInfo> ArgInfo;
279 
280   unsigned PSInputAddr = 0;
281   unsigned PSInputEnable = 0;
282 
283   SIMode Mode;
284   std::optional<FrameIndex> ScavengeFI;
285   StringValue VGPRForAGPRCopy;
286   StringValue SGPRForEXECCopy;
287   StringValue LongBranchReservedReg;
288 
289   SIMachineFunctionInfo() = default;
290   SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo &,
291                         const TargetRegisterInfo &TRI,
292                         const llvm::MachineFunction &MF);
293 
294   void mappingImpl(yaml::IO &YamlIO) override;
295   ~SIMachineFunctionInfo() = default;
296 };
297 
298 template <> struct MappingTraits<SIMachineFunctionInfo> {
299   static void mapping(IO &YamlIO, SIMachineFunctionInfo &MFI) {
300     YamlIO.mapOptional("explicitKernArgSize", MFI.ExplicitKernArgSize,
301                        UINT64_C(0));
302     YamlIO.mapOptional("maxKernArgAlign", MFI.MaxKernArgAlign);
303     YamlIO.mapOptional("ldsSize", MFI.LDSSize, 0u);
304     YamlIO.mapOptional("gdsSize", MFI.GDSSize, 0u);
305     YamlIO.mapOptional("dynLDSAlign", MFI.DynLDSAlign, Align());
306     YamlIO.mapOptional("isEntryFunction", MFI.IsEntryFunction, false);
307     YamlIO.mapOptional("noSignedZerosFPMath", MFI.NoSignedZerosFPMath, false);
308     YamlIO.mapOptional("memoryBound", MFI.MemoryBound, false);
309     YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
310     YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
311     YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
312     YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg,
313                        StringValue("$private_rsrc_reg"));
314     YamlIO.mapOptional("frameOffsetReg", MFI.FrameOffsetReg,
315                        StringValue("$fp_reg"));
316     YamlIO.mapOptional("stackPtrOffsetReg", MFI.StackPtrOffsetReg,
317                        StringValue("$sp_reg"));
318     YamlIO.mapOptional("bytesInStackArgArea", MFI.BytesInStackArgArea, 0u);
319     YamlIO.mapOptional("returnsVoid", MFI.ReturnsVoid, true);
320     YamlIO.mapOptional("argumentInfo", MFI.ArgInfo);
321     YamlIO.mapOptional("psInputAddr", MFI.PSInputAddr, 0u);
322     YamlIO.mapOptional("psInputEnable", MFI.PSInputEnable, 0u);
323     YamlIO.mapOptional("mode", MFI.Mode, SIMode());
324     YamlIO.mapOptional("highBitsOf32BitAddress",
325                        MFI.HighBitsOf32BitAddress, 0u);
326     YamlIO.mapOptional("occupancy", MFI.Occupancy, 0);
327     YamlIO.mapOptional("wwmReservedRegs", MFI.WWMReservedRegs);
328     YamlIO.mapOptional("scavengeFI", MFI.ScavengeFI);
329     YamlIO.mapOptional("vgprForAGPRCopy", MFI.VGPRForAGPRCopy,
330                        StringValue()); // Don't print out when it's empty.
331     YamlIO.mapOptional("sgprForEXECCopy", MFI.SGPRForEXECCopy,
332                        StringValue()); // Don't print out when it's empty.
333     YamlIO.mapOptional("longBranchReservedReg", MFI.LongBranchReservedReg,
334                        StringValue());
335   }
336 };
337 
338 } // end namespace yaml
339 
340 // A CSR SGPR value can be preserved inside a callee using one of the following
341 // methods.
342 //   1. Copy to an unused scratch SGPR.
343 //   2. Spill to a VGPR lane.
344 //   3. Spill to memory via. a scratch VGPR.
345 // class PrologEpilogSGPRSaveRestoreInfo represents the save/restore method used
346 // for an SGPR at function prolog/epilog.
347 enum class SGPRSaveKind : uint8_t {
348   COPY_TO_SCRATCH_SGPR,
349   SPILL_TO_VGPR_LANE,
350   SPILL_TO_MEM
351 };
352 
353 class PrologEpilogSGPRSaveRestoreInfo {
354   SGPRSaveKind Kind;
355   union {
356     int Index;
357     Register Reg;
358   };
359 
360 public:
361   PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, int I) : Kind(K), Index(I) {}
362   PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, Register R)
363       : Kind(K), Reg(R) {}
364   Register getReg() const { return Reg; }
365   int getIndex() const { return Index; }
366   SGPRSaveKind getKind() const { return Kind; }
367 };
368 
369 /// This class keeps track of the SPI_SP_INPUT_ADDR config register, which
370 /// tells the hardware which interpolation parameters to load.
371 class SIMachineFunctionInfo final : public AMDGPUMachineFunction,
372                                     private MachineRegisterInfo::Delegate {
373   friend class GCNTargetMachine;
374 
375   // State of MODE register, assumed FP mode.
376   SIModeRegisterDefaults Mode;
377 
378   // Registers that may be reserved for spilling purposes. These may be the same
379   // as the input registers.
380   Register ScratchRSrcReg = AMDGPU::PRIVATE_RSRC_REG;
381 
382   // This is the unswizzled offset from the current dispatch's scratch wave
383   // base to the beginning of the current function's frame.
384   Register FrameOffsetReg = AMDGPU::FP_REG;
385 
386   // This is an ABI register used in the non-entry calling convention to
387   // communicate the unswizzled offset from the current dispatch's scratch wave
388   // base to the beginning of the new function's frame.
389   Register StackPtrOffsetReg = AMDGPU::SP_REG;
390 
391   // Registers that may be reserved when RA doesn't allocate enough
392   // registers to plan for the case where an indirect branch ends up
393   // being needed during branch relaxation.
394   Register LongBranchReservedReg;
395 
396   AMDGPUFunctionArgInfo ArgInfo;
397 
398   // Graphics info.
399   unsigned PSInputAddr = 0;
400   unsigned PSInputEnable = 0;
401 
402   /// Number of bytes of arguments this function has on the stack. If the callee
403   /// is expected to restore the argument stack this should be a multiple of 16,
404   /// all usable during a tail call.
405   ///
406   /// The alternative would forbid tail call optimisation in some cases: if we
407   /// want to transfer control from a function with 8-bytes of stack-argument
408   /// space to a function with 16-bytes then misalignment of this value would
409   /// make a stack adjustment necessary, which could not be undone by the
410   /// callee.
411   unsigned BytesInStackArgArea = 0;
412 
413   bool ReturnsVoid = true;
414 
415   // A pair of default/requested minimum/maximum flat work group sizes.
416   // Minimum - first, maximum - second.
417   std::pair<unsigned, unsigned> FlatWorkGroupSizes = {0, 0};
418 
419   // A pair of default/requested minimum/maximum number of waves per execution
420   // unit. Minimum - first, maximum - second.
421   std::pair<unsigned, unsigned> WavesPerEU = {0, 0};
422 
423   const AMDGPUGWSResourcePseudoSourceValue GWSResourcePSV;
424 
425 private:
426   unsigned NumUserSGPRs = 0;
427   unsigned NumSystemSGPRs = 0;
428 
429   bool HasSpilledSGPRs = false;
430   bool HasSpilledVGPRs = false;
431   bool HasNonSpillStackObjects = false;
432   bool IsStackRealigned = false;
433 
434   unsigned NumSpilledSGPRs = 0;
435   unsigned NumSpilledVGPRs = 0;
436 
437   // Feature bits required for inputs passed in user SGPRs.
438   bool PrivateSegmentBuffer : 1;
439   bool DispatchPtr : 1;
440   bool QueuePtr : 1;
441   bool KernargSegmentPtr : 1;
442   bool DispatchID : 1;
443   bool FlatScratchInit : 1;
444 
445   // Feature bits required for inputs passed in system SGPRs.
446   bool WorkGroupIDX : 1; // Always initialized.
447   bool WorkGroupIDY : 1;
448   bool WorkGroupIDZ : 1;
449   bool WorkGroupInfo : 1;
450   bool LDSKernelId : 1;
451   bool PrivateSegmentWaveByteOffset : 1;
452 
453   bool WorkItemIDX : 1; // Always initialized.
454   bool WorkItemIDY : 1;
455   bool WorkItemIDZ : 1;
456 
457   // Private memory buffer
458   // Compute directly in sgpr[0:1]
459   // Other shaders indirect 64-bits at sgpr[0:1]
460   bool ImplicitBufferPtr : 1;
461 
462   // Pointer to where the ABI inserts special kernel arguments separate from the
463   // user arguments. This is an offset from the KernargSegmentPtr.
464   bool ImplicitArgPtr : 1;
465 
466   bool MayNeedAGPRs : 1;
467 
468   // The hard-wired high half of the address of the global information table
469   // for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since
470   // current hardware only allows a 16 bit value.
471   unsigned GITPtrHigh;
472 
473   unsigned HighBitsOf32BitAddress;
474 
475   // Flags associated with the virtual registers.
476   IndexedMap<uint8_t, VirtReg2IndexFunctor> VRegFlags;
477 
478   // Current recorded maximum possible occupancy.
479   unsigned Occupancy;
480 
481   mutable std::optional<bool> UsesAGPRs;
482 
483   MCPhysReg getNextUserSGPR() const;
484 
485   MCPhysReg getNextSystemSGPR() const;
486 
487   // MachineRegisterInfo callback functions to notify events.
488   void MRI_NoteNewVirtualRegister(Register Reg) override;
489   void MRI_NoteCloneVirtualRegister(Register NewReg, Register SrcReg) override;
490 
491 public:
492   struct VGPRSpillToAGPR {
493     SmallVector<MCPhysReg, 32> Lanes;
494     bool FullyAllocated = false;
495     bool IsDead = false;
496   };
497 
498 private:
499   // To track VGPR + lane index for each subregister of the SGPR spilled to
500   // frameindex key during SILowerSGPRSpills pass.
501   DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>> SGPRSpillToVGPRLanes;
502   // To track VGPR + lane index for spilling special SGPRs like Frame Pointer
503   // identified during PrologEpilogInserter.
504   DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>>
505       PrologEpilogSGPRSpillToVGPRLanes;
506   unsigned NumVGPRSpillLanes = 0;
507   unsigned NumVGPRPrologEpilogSpillLanes = 0;
508   SmallVector<Register, 2> SpillVGPRs;
509   using WWMSpillsMap = MapVector<Register, int>;
510   // To track the registers used in instructions that can potentially modify the
511   // inactive lanes. The WWM instructions and the writelane instructions for
512   // spilling SGPRs to VGPRs fall under such category of operations. The VGPRs
513   // modified by them should be spilled/restored at function prolog/epilog to
514   // avoid any undesired outcome. Each entry in this map holds a pair of values,
515   // the VGPR and its stack slot index.
516   WWMSpillsMap WWMSpills;
517 
518   using ReservedRegSet = SmallSetVector<Register, 8>;
519   // To track the VGPRs reserved for WWM instructions. They get stack slots
520   // later during PrologEpilogInserter and get added into the superset WWMSpills
521   // for actual spilling. A separate set makes the register reserved part and
522   // the serialization easier.
523   ReservedRegSet WWMReservedRegs;
524 
525   using PrologEpilogSGPRSpillsMap =
526       DenseMap<Register, PrologEpilogSGPRSaveRestoreInfo>;
527   // To track the SGPR spill method used for a CSR SGPR register during
528   // frame lowering. Even though the SGPR spills are handled during
529   // SILowerSGPRSpills pass, some special handling needed later during the
530   // PrologEpilogInserter.
531   PrologEpilogSGPRSpillsMap PrologEpilogSGPRSpills;
532 
533   // To save/restore EXEC MASK around WWM spills and copies.
534   Register SGPRForEXECCopy;
535 
536   DenseMap<int, VGPRSpillToAGPR> VGPRToAGPRSpills;
537 
538   // AGPRs used for VGPR spills.
539   SmallVector<MCPhysReg, 32> SpillAGPR;
540 
541   // VGPRs used for AGPR spills.
542   SmallVector<MCPhysReg, 32> SpillVGPR;
543 
544   // Emergency stack slot. Sometimes, we create this before finalizing the stack
545   // frame, so save it here and add it to the RegScavenger later.
546   std::optional<int> ScavengeFI;
547 
548 private:
549   Register VGPRForAGPRCopy;
550 
551   bool allocateVGPRForSGPRSpills(MachineFunction &MF, int FI,
552                                  unsigned LaneIndex);
553   bool allocateVGPRForPrologEpilogSGPRSpills(MachineFunction &MF, int FI,
554                                              unsigned LaneIndex);
555 
556 public:
557   Register getVGPRForAGPRCopy() const {
558     return VGPRForAGPRCopy;
559   }
560 
561   void setVGPRForAGPRCopy(Register NewVGPRForAGPRCopy) {
562     VGPRForAGPRCopy = NewVGPRForAGPRCopy;
563   }
564 
565   bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const;
566 
567 public:
568   SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI) = default;
569   SIMachineFunctionInfo(const Function &F, const GCNSubtarget *STI);
570 
571   MachineFunctionInfo *
572   clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF,
573         const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
574       const override;
575 
576   bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI,
577                                 const MachineFunction &MF,
578                                 PerFunctionMIParsingState &PFS,
579                                 SMDiagnostic &Error, SMRange &SourceRange);
580 
581   void reserveWWMRegister(Register Reg) { WWMReservedRegs.insert(Reg); }
582 
583   SIModeRegisterDefaults getMode() const { return Mode; }
584 
585   ArrayRef<SIRegisterInfo::SpilledReg>
586   getSGPRSpillToVGPRLanes(int FrameIndex) const {
587     auto I = SGPRSpillToVGPRLanes.find(FrameIndex);
588     return (I == SGPRSpillToVGPRLanes.end())
589                ? ArrayRef<SIRegisterInfo::SpilledReg>()
590                : ArrayRef(I->second);
591   }
592 
593   ArrayRef<Register> getSGPRSpillVGPRs() const { return SpillVGPRs; }
594   const WWMSpillsMap &getWWMSpills() const { return WWMSpills; }
595   const ReservedRegSet &getWWMReservedRegs() const { return WWMReservedRegs; }
596 
597   const PrologEpilogSGPRSpillsMap &getPrologEpilogSGPRSpills() const {
598     return PrologEpilogSGPRSpills;
599   }
600 
601   void addToPrologEpilogSGPRSpills(Register Reg,
602                                    PrologEpilogSGPRSaveRestoreInfo SI) {
603     PrologEpilogSGPRSpills.insert(std::make_pair(Reg, SI));
604   }
605 
606   // Check if an entry created for \p Reg in PrologEpilogSGPRSpills. Return true
607   // on success and false otherwise.
608   bool hasPrologEpilogSGPRSpillEntry(Register Reg) const {
609     return PrologEpilogSGPRSpills.contains(Reg);
610   }
611 
612   // Get the scratch SGPR if allocated to save/restore \p Reg.
613   Register getScratchSGPRCopyDstReg(Register Reg) const {
614     auto I = PrologEpilogSGPRSpills.find(Reg);
615     if (I != PrologEpilogSGPRSpills.end() &&
616         I->second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR)
617       return I->second.getReg();
618 
619     return AMDGPU::NoRegister;
620   }
621 
622   // Get all scratch SGPRs allocated to copy/restore the SGPR spills.
623   void getAllScratchSGPRCopyDstRegs(SmallVectorImpl<Register> &Regs) const {
624     for (const auto &SI : PrologEpilogSGPRSpills) {
625       if (SI.second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR)
626         Regs.push_back(SI.second.getReg());
627     }
628   }
629 
630   // Check if \p FI is allocated for any SGPR spill to a VGPR lane during PEI.
631   bool checkIndexInPrologEpilogSGPRSpills(int FI) const {
632     return find_if(PrologEpilogSGPRSpills,
633                    [FI](const std::pair<Register,
634                                         PrologEpilogSGPRSaveRestoreInfo> &SI) {
635                      return SI.second.getKind() ==
636                                 SGPRSaveKind::SPILL_TO_VGPR_LANE &&
637                             SI.second.getIndex() == FI;
638                    }) != PrologEpilogSGPRSpills.end();
639   }
640 
641   const PrologEpilogSGPRSaveRestoreInfo &
642   getPrologEpilogSGPRSaveRestoreInfo(Register Reg) const {
643     auto I = PrologEpilogSGPRSpills.find(Reg);
644     assert(I != PrologEpilogSGPRSpills.end());
645 
646     return I->second;
647   }
648 
649   ArrayRef<SIRegisterInfo::SpilledReg>
650   getPrologEpilogSGPRSpillToVGPRLanes(int FrameIndex) const {
651     auto I = PrologEpilogSGPRSpillToVGPRLanes.find(FrameIndex);
652     return (I == PrologEpilogSGPRSpillToVGPRLanes.end())
653                ? ArrayRef<SIRegisterInfo::SpilledReg>()
654                : ArrayRef(I->second);
655   }
656 
657   void setFlag(Register Reg, uint8_t Flag) {
658     assert(Reg.isVirtual());
659     if (VRegFlags.inBounds(Reg))
660       VRegFlags[Reg] |= Flag;
661   }
662 
663   bool checkFlag(Register Reg, uint8_t Flag) const {
664     if (Reg.isPhysical())
665       return false;
666 
667     return VRegFlags.inBounds(Reg) && VRegFlags[Reg] & Flag;
668   }
669 
670   void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size = 4,
671                         Align Alignment = Align(4));
672 
673   void splitWWMSpillRegisters(
674       MachineFunction &MF,
675       SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
676       SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const;
677 
678   ArrayRef<MCPhysReg> getAGPRSpillVGPRs() const {
679     return SpillAGPR;
680   }
681 
682   Register getSGPRForEXECCopy() const { return SGPRForEXECCopy; }
683 
684   void setSGPRForEXECCopy(Register Reg) { SGPRForEXECCopy = Reg; }
685 
686   ArrayRef<MCPhysReg> getVGPRSpillAGPRs() const {
687     return SpillVGPR;
688   }
689 
690   MCPhysReg getVGPRToAGPRSpill(int FrameIndex, unsigned Lane) const {
691     auto I = VGPRToAGPRSpills.find(FrameIndex);
692     return (I == VGPRToAGPRSpills.end()) ? (MCPhysReg)AMDGPU::NoRegister
693                                          : I->second.Lanes[Lane];
694   }
695 
696   void setVGPRToAGPRSpillDead(int FrameIndex) {
697     auto I = VGPRToAGPRSpills.find(FrameIndex);
698     if (I != VGPRToAGPRSpills.end())
699       I->second.IsDead = true;
700   }
701 
702   bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI,
703                                    bool IsPrologEpilog = false);
704   bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR);
705 
706   /// If \p ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill
707   /// to the default stack.
708   bool removeDeadFrameIndices(MachineFrameInfo &MFI,
709                               bool ResetSGPRSpillStackIDs);
710 
711   int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI);
712   std::optional<int> getOptionalScavengeFI() const { return ScavengeFI; }
713 
714   unsigned getBytesInStackArgArea() const {
715     return BytesInStackArgArea;
716   }
717 
718   void setBytesInStackArgArea(unsigned Bytes) {
719     BytesInStackArgArea = Bytes;
720   }
721 
722   // Add user SGPRs.
723   Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI);
724   Register addDispatchPtr(const SIRegisterInfo &TRI);
725   Register addQueuePtr(const SIRegisterInfo &TRI);
726   Register addKernargSegmentPtr(const SIRegisterInfo &TRI);
727   Register addDispatchID(const SIRegisterInfo &TRI);
728   Register addFlatScratchInit(const SIRegisterInfo &TRI);
729   Register addImplicitBufferPtr(const SIRegisterInfo &TRI);
730   Register addLDSKernelId();
731 
732   /// Increment user SGPRs used for padding the argument list only.
733   Register addReservedUserSGPR() {
734     Register Next = getNextUserSGPR();
735     ++NumUserSGPRs;
736     return Next;
737   }
738 
739   // Add system SGPRs.
740   Register addWorkGroupIDX(bool HasArchitectedSGPRs) {
741     Register Reg =
742         HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP9 : getNextSystemSGPR();
743     ArgInfo.WorkGroupIDX = ArgDescriptor::createRegister(Reg);
744     if (!HasArchitectedSGPRs)
745       NumSystemSGPRs += 1;
746 
747     return ArgInfo.WorkGroupIDX.getRegister();
748   }
749 
750   Register addWorkGroupIDY(bool HasArchitectedSGPRs) {
751     Register Reg =
752         HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP7 : getNextSystemSGPR();
753     unsigned Mask = HasArchitectedSGPRs && hasWorkGroupIDZ() ? 0xffff : ~0u;
754     ArgInfo.WorkGroupIDY = ArgDescriptor::createRegister(Reg, Mask);
755     if (!HasArchitectedSGPRs)
756       NumSystemSGPRs += 1;
757 
758     return ArgInfo.WorkGroupIDY.getRegister();
759   }
760 
761   Register addWorkGroupIDZ(bool HasArchitectedSGPRs) {
762     Register Reg =
763         HasArchitectedSGPRs ? (MCPhysReg)AMDGPU::TTMP7 : getNextSystemSGPR();
764     unsigned Mask = HasArchitectedSGPRs ? 0xffff << 16 : ~0u;
765     ArgInfo.WorkGroupIDZ = ArgDescriptor::createRegister(Reg, Mask);
766     if (!HasArchitectedSGPRs)
767       NumSystemSGPRs += 1;
768 
769     return ArgInfo.WorkGroupIDZ.getRegister();
770   }
771 
772   Register addWorkGroupInfo() {
773     ArgInfo.WorkGroupInfo = ArgDescriptor::createRegister(getNextSystemSGPR());
774     NumSystemSGPRs += 1;
775     return ArgInfo.WorkGroupInfo.getRegister();
776   }
777 
778   // Add special VGPR inputs
779   void setWorkItemIDX(ArgDescriptor Arg) {
780     ArgInfo.WorkItemIDX = Arg;
781   }
782 
783   void setWorkItemIDY(ArgDescriptor Arg) {
784     ArgInfo.WorkItemIDY = Arg;
785   }
786 
787   void setWorkItemIDZ(ArgDescriptor Arg) {
788     ArgInfo.WorkItemIDZ = Arg;
789   }
790 
791   Register addPrivateSegmentWaveByteOffset() {
792     ArgInfo.PrivateSegmentWaveByteOffset
793       = ArgDescriptor::createRegister(getNextSystemSGPR());
794     NumSystemSGPRs += 1;
795     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
796   }
797 
798   void setPrivateSegmentWaveByteOffset(Register Reg) {
799     ArgInfo.PrivateSegmentWaveByteOffset = ArgDescriptor::createRegister(Reg);
800   }
801 
802   bool hasPrivateSegmentBuffer() const {
803     return PrivateSegmentBuffer;
804   }
805 
806   bool hasDispatchPtr() const {
807     return DispatchPtr;
808   }
809 
810   bool hasQueuePtr() const {
811     return QueuePtr;
812   }
813 
814   bool hasKernargSegmentPtr() const {
815     return KernargSegmentPtr;
816   }
817 
818   bool hasDispatchID() const {
819     return DispatchID;
820   }
821 
822   bool hasFlatScratchInit() const {
823     return FlatScratchInit;
824   }
825 
826   bool hasWorkGroupIDX() const {
827     return WorkGroupIDX;
828   }
829 
830   bool hasWorkGroupIDY() const {
831     return WorkGroupIDY;
832   }
833 
834   bool hasWorkGroupIDZ() const {
835     return WorkGroupIDZ;
836   }
837 
838   bool hasWorkGroupInfo() const {
839     return WorkGroupInfo;
840   }
841 
842   bool hasLDSKernelId() const { return LDSKernelId; }
843 
844   bool hasPrivateSegmentWaveByteOffset() const {
845     return PrivateSegmentWaveByteOffset;
846   }
847 
848   bool hasWorkItemIDX() const {
849     return WorkItemIDX;
850   }
851 
852   bool hasWorkItemIDY() const {
853     return WorkItemIDY;
854   }
855 
856   bool hasWorkItemIDZ() const {
857     return WorkItemIDZ;
858   }
859 
860   bool hasImplicitArgPtr() const {
861     return ImplicitArgPtr;
862   }
863 
864   bool hasImplicitBufferPtr() const {
865     return ImplicitBufferPtr;
866   }
867 
868   AMDGPUFunctionArgInfo &getArgInfo() {
869     return ArgInfo;
870   }
871 
872   const AMDGPUFunctionArgInfo &getArgInfo() const {
873     return ArgInfo;
874   }
875 
876   std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
877   getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
878     return ArgInfo.getPreloadedValue(Value);
879   }
880 
881   MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
882     auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value));
883     return Arg ? Arg->getRegister() : MCRegister();
884   }
885 
886   unsigned getGITPtrHigh() const {
887     return GITPtrHigh;
888   }
889 
890   Register getGITPtrLoReg(const MachineFunction &MF) const;
891 
892   uint32_t get32BitAddressHighBits() const {
893     return HighBitsOf32BitAddress;
894   }
895 
896   unsigned getNumUserSGPRs() const {
897     return NumUserSGPRs;
898   }
899 
900   unsigned getNumPreloadedSGPRs() const {
901     return NumUserSGPRs + NumSystemSGPRs;
902   }
903 
904   Register getPrivateSegmentWaveByteOffsetSystemSGPR() const {
905     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
906   }
907 
908   /// Returns the physical register reserved for use as the resource
909   /// descriptor for scratch accesses.
910   Register getScratchRSrcReg() const {
911     return ScratchRSrcReg;
912   }
913 
914   void setScratchRSrcReg(Register Reg) {
915     assert(Reg != 0 && "Should never be unset");
916     ScratchRSrcReg = Reg;
917   }
918 
919   Register getFrameOffsetReg() const {
920     return FrameOffsetReg;
921   }
922 
923   void setFrameOffsetReg(Register Reg) {
924     assert(Reg != 0 && "Should never be unset");
925     FrameOffsetReg = Reg;
926   }
927 
928   void setStackPtrOffsetReg(Register Reg) {
929     assert(Reg != 0 && "Should never be unset");
930     StackPtrOffsetReg = Reg;
931   }
932 
933   void setLongBranchReservedReg(Register Reg) { LongBranchReservedReg = Reg; }
934 
935   // Note the unset value for this is AMDGPU::SP_REG rather than
936   // NoRegister. This is mostly a workaround for MIR tests where state that
937   // can't be directly computed from the function is not preserved in serialized
938   // MIR.
939   Register getStackPtrOffsetReg() const {
940     return StackPtrOffsetReg;
941   }
942 
943   Register getLongBranchReservedReg() const { return LongBranchReservedReg; }
944 
945   Register getQueuePtrUserSGPR() const {
946     return ArgInfo.QueuePtr.getRegister();
947   }
948 
949   Register getImplicitBufferPtrUserSGPR() const {
950     return ArgInfo.ImplicitBufferPtr.getRegister();
951   }
952 
953   bool hasSpilledSGPRs() const {
954     return HasSpilledSGPRs;
955   }
956 
957   void setHasSpilledSGPRs(bool Spill = true) {
958     HasSpilledSGPRs = Spill;
959   }
960 
961   bool hasSpilledVGPRs() const {
962     return HasSpilledVGPRs;
963   }
964 
965   void setHasSpilledVGPRs(bool Spill = true) {
966     HasSpilledVGPRs = Spill;
967   }
968 
969   bool hasNonSpillStackObjects() const {
970     return HasNonSpillStackObjects;
971   }
972 
973   void setHasNonSpillStackObjects(bool StackObject = true) {
974     HasNonSpillStackObjects = StackObject;
975   }
976 
977   bool isStackRealigned() const {
978     return IsStackRealigned;
979   }
980 
981   void setIsStackRealigned(bool Realigned = true) {
982     IsStackRealigned = Realigned;
983   }
984 
985   unsigned getNumSpilledSGPRs() const {
986     return NumSpilledSGPRs;
987   }
988 
989   unsigned getNumSpilledVGPRs() const {
990     return NumSpilledVGPRs;
991   }
992 
993   void addToSpilledSGPRs(unsigned num) {
994     NumSpilledSGPRs += num;
995   }
996 
997   void addToSpilledVGPRs(unsigned num) {
998     NumSpilledVGPRs += num;
999   }
1000 
1001   unsigned getPSInputAddr() const {
1002     return PSInputAddr;
1003   }
1004 
1005   unsigned getPSInputEnable() const {
1006     return PSInputEnable;
1007   }
1008 
1009   bool isPSInputAllocated(unsigned Index) const {
1010     return PSInputAddr & (1 << Index);
1011   }
1012 
1013   void markPSInputAllocated(unsigned Index) {
1014     PSInputAddr |= 1 << Index;
1015   }
1016 
1017   void markPSInputEnabled(unsigned Index) {
1018     PSInputEnable |= 1 << Index;
1019   }
1020 
1021   bool returnsVoid() const {
1022     return ReturnsVoid;
1023   }
1024 
1025   void setIfReturnsVoid(bool Value) {
1026     ReturnsVoid = Value;
1027   }
1028 
1029   /// \returns A pair of default/requested minimum/maximum flat work group sizes
1030   /// for this function.
1031   std::pair<unsigned, unsigned> getFlatWorkGroupSizes() const {
1032     return FlatWorkGroupSizes;
1033   }
1034 
1035   /// \returns Default/requested minimum flat work group size for this function.
1036   unsigned getMinFlatWorkGroupSize() const {
1037     return FlatWorkGroupSizes.first;
1038   }
1039 
1040   /// \returns Default/requested maximum flat work group size for this function.
1041   unsigned getMaxFlatWorkGroupSize() const {
1042     return FlatWorkGroupSizes.second;
1043   }
1044 
1045   /// \returns A pair of default/requested minimum/maximum number of waves per
1046   /// execution unit.
1047   std::pair<unsigned, unsigned> getWavesPerEU() const {
1048     return WavesPerEU;
1049   }
1050 
1051   /// \returns Default/requested minimum number of waves per execution unit.
1052   unsigned getMinWavesPerEU() const {
1053     return WavesPerEU.first;
1054   }
1055 
1056   /// \returns Default/requested maximum number of waves per execution unit.
1057   unsigned getMaxWavesPerEU() const {
1058     return WavesPerEU.second;
1059   }
1060 
1061   /// \returns SGPR used for \p Dim's work group ID.
1062   Register getWorkGroupIDSGPR(unsigned Dim) const {
1063     switch (Dim) {
1064     case 0:
1065       assert(hasWorkGroupIDX());
1066       return ArgInfo.WorkGroupIDX.getRegister();
1067     case 1:
1068       assert(hasWorkGroupIDY());
1069       return ArgInfo.WorkGroupIDY.getRegister();
1070     case 2:
1071       assert(hasWorkGroupIDZ());
1072       return ArgInfo.WorkGroupIDZ.getRegister();
1073     }
1074     llvm_unreachable("unexpected dimension");
1075   }
1076 
1077   const AMDGPUGWSResourcePseudoSourceValue *
1078   getGWSPSV(const AMDGPUTargetMachine &TM) {
1079     return &GWSResourcePSV;
1080   }
1081 
1082   unsigned getOccupancy() const {
1083     return Occupancy;
1084   }
1085 
1086   unsigned getMinAllowedOccupancy() const {
1087     if (!isMemoryBound() && !needsWaveLimiter())
1088       return Occupancy;
1089     return (Occupancy < 4) ? Occupancy : 4;
1090   }
1091 
1092   void limitOccupancy(const MachineFunction &MF);
1093 
1094   void limitOccupancy(unsigned Limit) {
1095     if (Occupancy > Limit)
1096       Occupancy = Limit;
1097   }
1098 
1099   void increaseOccupancy(const MachineFunction &MF, unsigned Limit) {
1100     if (Occupancy < Limit)
1101       Occupancy = Limit;
1102     limitOccupancy(MF);
1103   }
1104 
1105   bool mayNeedAGPRs() const {
1106     return MayNeedAGPRs;
1107   }
1108 
1109   // \returns true if a function has a use of AGPRs via inline asm or
1110   // has a call which may use it.
1111   bool mayUseAGPRs(const Function &F) const;
1112 
1113   // \returns true if a function needs or may need AGPRs.
1114   bool usesAGPRs(const MachineFunction &MF) const;
1115 };
1116 
1117 } // end namespace llvm
1118 
1119 #endif // LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
1120