1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "SIMachineFunctionInfo.h"
10 #include "AMDGPUTargetMachine.h"
11 #include "AMDGPUSubtarget.h"
12 #include "SIRegisterInfo.h"
13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "llvm/CodeGen/LiveIntervals.h"
16 #include "llvm/CodeGen/MachineBasicBlock.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/CodeGen/MIRParser/MIParser.h"
21 #include "llvm/IR/CallingConv.h"
22 #include "llvm/IR/DiagnosticInfo.h"
23 #include "llvm/IR/Function.h"
24 #include <cassert>
25 #include <optional>
26 #include <vector>
27
28 #define MAX_LANES 64
29
30 using namespace llvm;
31
getTM(const GCNSubtarget * STI)32 const GCNTargetMachine &getTM(const GCNSubtarget *STI) {
33 const SITargetLowering *TLI = STI->getTargetLowering();
34 return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine());
35 }
36
SIMachineFunctionInfo(const Function & F,const GCNSubtarget * STI)37 SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F,
38 const GCNSubtarget *STI)
39 : AMDGPUMachineFunction(F, *STI),
40 Mode(F),
41 GWSResourcePSV(getTM(STI)),
42 PrivateSegmentBuffer(false),
43 DispatchPtr(false),
44 QueuePtr(false),
45 KernargSegmentPtr(false),
46 DispatchID(false),
47 FlatScratchInit(false),
48 WorkGroupIDX(false),
49 WorkGroupIDY(false),
50 WorkGroupIDZ(false),
51 WorkGroupInfo(false),
52 LDSKernelId(false),
53 PrivateSegmentWaveByteOffset(false),
54 WorkItemIDX(false),
55 WorkItemIDY(false),
56 WorkItemIDZ(false),
57 ImplicitBufferPtr(false),
58 ImplicitArgPtr(false),
59 GITPtrHigh(0xffffffff),
60 HighBitsOf32BitAddress(0) {
61 const GCNSubtarget &ST = *static_cast<const GCNSubtarget *>(STI);
62 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
63 WavesPerEU = ST.getWavesPerEU(F);
64
65 Occupancy = ST.computeOccupancy(F, getLDSSize());
66 CallingConv::ID CC = F.getCallingConv();
67
68 // FIXME: Should have analysis or something rather than attribute to detect
69 // calls.
70 const bool HasCalls = F.hasFnAttribute("amdgpu-calls");
71
72 const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||
73 CC == CallingConv::SPIR_KERNEL;
74
75 if (IsKernel) {
76 if (!F.arg_empty() || ST.getImplicitArgNumBytes(F) != 0)
77 KernargSegmentPtr = true;
78 WorkGroupIDX = true;
79 WorkItemIDX = true;
80 } else if (CC == CallingConv::AMDGPU_PS) {
81 PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
82 }
83
84 MayNeedAGPRs = ST.hasMAIInsts();
85
86 if (!isEntryFunction()) {
87 if (CC != CallingConv::AMDGPU_Gfx)
88 ArgInfo = AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
89
90 // TODO: Pick a high register, and shift down, similar to a kernel.
91 FrameOffsetReg = AMDGPU::SGPR33;
92 StackPtrOffsetReg = AMDGPU::SGPR32;
93
94 if (!ST.enableFlatScratch()) {
95 // Non-entry functions have no special inputs for now, other registers
96 // required for scratch access.
97 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
98
99 ArgInfo.PrivateSegmentBuffer =
100 ArgDescriptor::createRegister(ScratchRSrcReg);
101 }
102
103 if (!F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))
104 ImplicitArgPtr = true;
105 } else {
106 ImplicitArgPtr = false;
107 MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
108 MaxKernArgAlign);
109
110 if (ST.hasGFX90AInsts() &&
111 ST.getMaxNumVGPRs(F) <= AMDGPU::VGPR_32RegClass.getNumRegs() &&
112 !mayUseAGPRs(F))
113 MayNeedAGPRs = false; // We will select all MAI with VGPR operands.
114 }
115
116 bool isAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
117 if (isAmdHsaOrMesa && !ST.enableFlatScratch())
118 PrivateSegmentBuffer = true;
119 else if (ST.isMesaGfxShader(F))
120 ImplicitBufferPtr = true;
121
122 if (!AMDGPU::isGraphics(CC)) {
123 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x"))
124 WorkGroupIDX = true;
125
126 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-y"))
127 WorkGroupIDY = true;
128
129 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-z"))
130 WorkGroupIDZ = true;
131
132 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workitem-id-x"))
133 WorkItemIDX = true;
134
135 if (!F.hasFnAttribute("amdgpu-no-workitem-id-y") &&
136 ST.getMaxWorkitemID(F, 1) != 0)
137 WorkItemIDY = true;
138
139 if (!F.hasFnAttribute("amdgpu-no-workitem-id-z") &&
140 ST.getMaxWorkitemID(F, 2) != 0)
141 WorkItemIDZ = true;
142
143 if (!F.hasFnAttribute("amdgpu-no-dispatch-ptr"))
144 DispatchPtr = true;
145
146 if (!F.hasFnAttribute("amdgpu-no-queue-ptr"))
147 QueuePtr = true;
148
149 if (!F.hasFnAttribute("amdgpu-no-dispatch-id"))
150 DispatchID = true;
151
152 if (!IsKernel && !F.hasFnAttribute("amdgpu-no-lds-kernel-id"))
153 LDSKernelId = true;
154 }
155
156 // FIXME: This attribute is a hack, we just need an analysis on the function
157 // to look for allocas.
158 bool HasStackObjects = F.hasFnAttribute("amdgpu-stack-objects");
159
160 // TODO: This could be refined a lot. The attribute is a poor way of
161 // detecting calls or stack objects that may require it before argument
162 // lowering.
163 if (ST.hasFlatAddressSpace() && isEntryFunction() &&
164 (isAmdHsaOrMesa || ST.enableFlatScratch()) &&
165 (HasCalls || HasStackObjects || ST.enableFlatScratch()) &&
166 !ST.flatScratchIsArchitected()) {
167 FlatScratchInit = true;
168 }
169
170 if (isEntryFunction()) {
171 // X, XY, and XYZ are the only supported combinations, so make sure Y is
172 // enabled if Z is.
173 if (WorkItemIDZ)
174 WorkItemIDY = true;
175
176 if (!ST.flatScratchIsArchitected()) {
177 PrivateSegmentWaveByteOffset = true;
178
179 // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
180 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
181 (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
182 ArgInfo.PrivateSegmentWaveByteOffset =
183 ArgDescriptor::createRegister(AMDGPU::SGPR5);
184 }
185 }
186
187 Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
188 StringRef S = A.getValueAsString();
189 if (!S.empty())
190 S.consumeInteger(0, GITPtrHigh);
191
192 A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
193 S = A.getValueAsString();
194 if (!S.empty())
195 S.consumeInteger(0, HighBitsOf32BitAddress);
196
197 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch
198 // VGPR available at all times. For now, reserve highest available VGPR. After
199 // RA, shift it to the lowest available unused VGPR if the one exist.
200 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
201 VGPRForAGPRCopy =
202 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(F) - 1);
203 }
204 }
205
clone(BumpPtrAllocator & Allocator,MachineFunction & DestMF,const DenseMap<MachineBasicBlock *,MachineBasicBlock * > & Src2DstMBB) const206 MachineFunctionInfo *SIMachineFunctionInfo::clone(
207 BumpPtrAllocator &Allocator, MachineFunction &DestMF,
208 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
209 const {
210 return DestMF.cloneInfo<SIMachineFunctionInfo>(*this);
211 }
212
limitOccupancy(const MachineFunction & MF)213 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
214 limitOccupancy(getMaxWavesPerEU());
215 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
216 limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
217 MF.getFunction()));
218 }
219
addPrivateSegmentBuffer(const SIRegisterInfo & TRI)220 Register SIMachineFunctionInfo::addPrivateSegmentBuffer(
221 const SIRegisterInfo &TRI) {
222 ArgInfo.PrivateSegmentBuffer =
223 ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
224 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
225 NumUserSGPRs += 4;
226 return ArgInfo.PrivateSegmentBuffer.getRegister();
227 }
228
addDispatchPtr(const SIRegisterInfo & TRI)229 Register SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
230 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
231 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
232 NumUserSGPRs += 2;
233 return ArgInfo.DispatchPtr.getRegister();
234 }
235
addQueuePtr(const SIRegisterInfo & TRI)236 Register SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
237 ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
238 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
239 NumUserSGPRs += 2;
240 return ArgInfo.QueuePtr.getRegister();
241 }
242
addKernargSegmentPtr(const SIRegisterInfo & TRI)243 Register SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
244 ArgInfo.KernargSegmentPtr
245 = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
246 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
247 NumUserSGPRs += 2;
248 return ArgInfo.KernargSegmentPtr.getRegister();
249 }
250
addDispatchID(const SIRegisterInfo & TRI)251 Register SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
252 ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
253 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
254 NumUserSGPRs += 2;
255 return ArgInfo.DispatchID.getRegister();
256 }
257
addFlatScratchInit(const SIRegisterInfo & TRI)258 Register SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
259 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
260 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
261 NumUserSGPRs += 2;
262 return ArgInfo.FlatScratchInit.getRegister();
263 }
264
addImplicitBufferPtr(const SIRegisterInfo & TRI)265 Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
266 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
267 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
268 NumUserSGPRs += 2;
269 return ArgInfo.ImplicitBufferPtr.getRegister();
270 }
271
addLDSKernelId()272 Register SIMachineFunctionInfo::addLDSKernelId() {
273 ArgInfo.LDSKernelId = ArgDescriptor::createRegister(getNextUserSGPR());
274 NumUserSGPRs += 1;
275 return ArgInfo.LDSKernelId.getRegister();
276 }
277
allocateWWMSpill(MachineFunction & MF,Register VGPR,uint64_t Size,Align Alignment)278 void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR,
279 uint64_t Size, Align Alignment) {
280 // Skip if it is an entry function or the register is already added.
281 if (isEntryFunction() || WWMSpills.count(VGPR))
282 return;
283
284 WWMSpills.insert(std::make_pair(
285 VGPR, MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));
286 }
287
288 // Separate out the callee-saved and scratch registers.
splitWWMSpillRegisters(MachineFunction & MF,SmallVectorImpl<std::pair<Register,int>> & CalleeSavedRegs,SmallVectorImpl<std::pair<Register,int>> & ScratchRegs) const289 void SIMachineFunctionInfo::splitWWMSpillRegisters(
290 MachineFunction &MF,
291 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
292 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const {
293 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
294 for (auto &Reg : WWMSpills) {
295 if (isCalleeSavedReg(CSRegs, Reg.first))
296 CalleeSavedRegs.push_back(Reg);
297 else
298 ScratchRegs.push_back(Reg);
299 }
300 }
301
isCalleeSavedReg(const MCPhysReg * CSRegs,MCPhysReg Reg) const302 bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs,
303 MCPhysReg Reg) const {
304 for (unsigned I = 0; CSRegs[I]; ++I) {
305 if (CSRegs[I] == Reg)
306 return true;
307 }
308
309 return false;
310 }
311
allocateVGPRForSGPRSpills(MachineFunction & MF,int FI,unsigned LaneIndex)312 bool SIMachineFunctionInfo::allocateVGPRForSGPRSpills(MachineFunction &MF,
313 int FI,
314 unsigned LaneIndex) {
315 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
316 const SIRegisterInfo *TRI = ST.getRegisterInfo();
317 MachineRegisterInfo &MRI = MF.getRegInfo();
318 Register LaneVGPR;
319 if (!LaneIndex) {
320 LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
321 if (LaneVGPR == AMDGPU::NoRegister) {
322 // We have no VGPRs left for spilling SGPRs. Reset because we will not
323 // partially spill the SGPR to VGPRs.
324 SGPRSpillToVGPRLanes.erase(FI);
325 return false;
326 }
327
328 SpillVGPRs.push_back(LaneVGPR);
329 // Add this register as live-in to all blocks to avoid machine verifier
330 // complaining about use of an undefined physical register.
331 for (MachineBasicBlock &BB : MF)
332 BB.addLiveIn(LaneVGPR);
333 } else {
334 LaneVGPR = SpillVGPRs.back();
335 }
336
337 SGPRSpillToVGPRLanes[FI].push_back(
338 SIRegisterInfo::SpilledReg(LaneVGPR, LaneIndex));
339 return true;
340 }
341
allocateVGPRForPrologEpilogSGPRSpills(MachineFunction & MF,int FI,unsigned LaneIndex)342 bool SIMachineFunctionInfo::allocateVGPRForPrologEpilogSGPRSpills(
343 MachineFunction &MF, int FI, unsigned LaneIndex) {
344 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
345 const SIRegisterInfo *TRI = ST.getRegisterInfo();
346 MachineRegisterInfo &MRI = MF.getRegInfo();
347 Register LaneVGPR;
348 if (!LaneIndex) {
349 LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
350 if (LaneVGPR == AMDGPU::NoRegister) {
351 // We have no VGPRs left for spilling SGPRs. Reset because we will not
352 // partially spill the SGPR to VGPRs.
353 PrologEpilogSGPRSpillToVGPRLanes.erase(FI);
354 return false;
355 }
356
357 allocateWWMSpill(MF, LaneVGPR);
358 } else {
359 LaneVGPR = WWMSpills.back().first;
360 }
361
362 PrologEpilogSGPRSpillToVGPRLanes[FI].push_back(
363 SIRegisterInfo::SpilledReg(LaneVGPR, LaneIndex));
364 return true;
365 }
366
allocateSGPRSpillToVGPRLane(MachineFunction & MF,int FI,bool IsPrologEpilog)367 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPRLane(MachineFunction &MF,
368 int FI,
369 bool IsPrologEpilog) {
370 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
371 IsPrologEpilog ? PrologEpilogSGPRSpillToVGPRLanes[FI]
372 : SGPRSpillToVGPRLanes[FI];
373
374 // This has already been allocated.
375 if (!SpillLanes.empty())
376 return true;
377
378 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
379 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
380 unsigned WaveSize = ST.getWavefrontSize();
381
382 unsigned Size = FrameInfo.getObjectSize(FI);
383 unsigned NumLanes = Size / 4;
384
385 if (NumLanes > WaveSize)
386 return false;
387
388 assert(Size >= 4 && "invalid sgpr spill size");
389 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
390 "not spilling SGPRs to VGPRs");
391
392 unsigned &NumSpillLanes =
393 IsPrologEpilog ? NumVGPRPrologEpilogSpillLanes : NumVGPRSpillLanes;
394
395 for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) {
396 unsigned LaneIndex = (NumSpillLanes % WaveSize);
397
398 bool Allocated =
399 IsPrologEpilog
400 ? allocateVGPRForPrologEpilogSGPRSpills(MF, FI, LaneIndex)
401 : allocateVGPRForSGPRSpills(MF, FI, LaneIndex);
402 if (!Allocated) {
403 NumSpillLanes -= I;
404 return false;
405 }
406 }
407
408 return true;
409 }
410
411 /// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
412 /// Either AGPR is spilled to VGPR to vice versa.
413 /// Returns true if a \p FI can be eliminated completely.
allocateVGPRSpillToAGPR(MachineFunction & MF,int FI,bool isAGPRtoVGPR)414 bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF,
415 int FI,
416 bool isAGPRtoVGPR) {
417 MachineRegisterInfo &MRI = MF.getRegInfo();
418 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
419 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
420
421 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
422
423 auto &Spill = VGPRToAGPRSpills[FI];
424
425 // This has already been allocated.
426 if (!Spill.Lanes.empty())
427 return Spill.FullyAllocated;
428
429 unsigned Size = FrameInfo.getObjectSize(FI);
430 unsigned NumLanes = Size / 4;
431 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
432
433 const TargetRegisterClass &RC =
434 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
435 auto Regs = RC.getRegisters();
436
437 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
438 const SIRegisterInfo *TRI = ST.getRegisterInfo();
439 Spill.FullyAllocated = true;
440
441 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
442 // once.
443 BitVector OtherUsedRegs;
444 OtherUsedRegs.resize(TRI->getNumRegs());
445
446 const uint32_t *CSRMask =
447 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
448 if (CSRMask)
449 OtherUsedRegs.setBitsInMask(CSRMask);
450
451 // TODO: Should include register tuples, but doesn't matter with current
452 // usage.
453 for (MCPhysReg Reg : SpillAGPR)
454 OtherUsedRegs.set(Reg);
455 for (MCPhysReg Reg : SpillVGPR)
456 OtherUsedRegs.set(Reg);
457
458 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
459 for (int I = NumLanes - 1; I >= 0; --I) {
460 NextSpillReg = std::find_if(
461 NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
462 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
463 !OtherUsedRegs[Reg];
464 });
465
466 if (NextSpillReg == Regs.end()) { // Registers exhausted
467 Spill.FullyAllocated = false;
468 break;
469 }
470
471 OtherUsedRegs.set(*NextSpillReg);
472 SpillRegs.push_back(*NextSpillReg);
473 MRI.reserveReg(*NextSpillReg, TRI);
474 Spill.Lanes[I] = *NextSpillReg++;
475 }
476
477 return Spill.FullyAllocated;
478 }
479
removeDeadFrameIndices(MachineFrameInfo & MFI,bool ResetSGPRSpillStackIDs)480 bool SIMachineFunctionInfo::removeDeadFrameIndices(
481 MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) {
482 // Remove dead frame indices from function frame. And also make sure to remove
483 // the frame indices from `SGPRSpillToVGPRLanes` data structure, otherwise, it
484 // could result in an unexpected side effect and bug, in case of any
485 // re-mapping of freed frame indices by later pass(es) like "stack slot
486 // coloring".
487 for (auto &R : make_early_inc_range(SGPRSpillToVGPRLanes)) {
488 MFI.RemoveStackObject(R.first);
489 SGPRSpillToVGPRLanes.erase(R.first);
490 }
491
492 bool HaveSGPRToMemory = false;
493
494 if (ResetSGPRSpillStackIDs) {
495 // All other SGPRs must be allocated on the default stack, so reset the
496 // stack ID.
497 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E;
498 ++I) {
499 if (!checkIndexInPrologEpilogSGPRSpills(I)) {
500 if (MFI.getStackID(I) == TargetStackID::SGPRSpill) {
501 MFI.setStackID(I, TargetStackID::Default);
502 HaveSGPRToMemory = true;
503 }
504 }
505 }
506 }
507
508 for (auto &R : VGPRToAGPRSpills) {
509 if (R.second.IsDead)
510 MFI.RemoveStackObject(R.first);
511 }
512
513 return HaveSGPRToMemory;
514 }
515
getScavengeFI(MachineFrameInfo & MFI,const SIRegisterInfo & TRI)516 int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI,
517 const SIRegisterInfo &TRI) {
518 if (ScavengeFI)
519 return *ScavengeFI;
520 if (isEntryFunction()) {
521 ScavengeFI = MFI.CreateFixedObject(
522 TRI.getSpillSize(AMDGPU::SGPR_32RegClass), 0, false);
523 } else {
524 ScavengeFI = MFI.CreateStackObject(
525 TRI.getSpillSize(AMDGPU::SGPR_32RegClass),
526 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass), false);
527 }
528 return *ScavengeFI;
529 }
530
getNextUserSGPR() const531 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
532 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
533 return AMDGPU::SGPR0 + NumUserSGPRs;
534 }
535
getNextSystemSGPR() const536 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
537 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
538 }
539
540 Register
getGITPtrLoReg(const MachineFunction & MF) const541 SIMachineFunctionInfo::getGITPtrLoReg(const MachineFunction &MF) const {
542 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
543 if (!ST.isAmdPalOS())
544 return Register();
545 Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
546 if (ST.hasMergedShaders()) {
547 switch (MF.getFunction().getCallingConv()) {
548 case CallingConv::AMDGPU_HS:
549 case CallingConv::AMDGPU_GS:
550 // Low GIT address is passed in s8 rather than s0 for an LS+HS or
551 // ES+GS merged shader on gfx9+.
552 GitPtrLo = AMDGPU::SGPR8;
553 return GitPtrLo;
554 default:
555 return GitPtrLo;
556 }
557 }
558 return GitPtrLo;
559 }
560
regToString(Register Reg,const TargetRegisterInfo & TRI)561 static yaml::StringValue regToString(Register Reg,
562 const TargetRegisterInfo &TRI) {
563 yaml::StringValue Dest;
564 {
565 raw_string_ostream OS(Dest.Value);
566 OS << printReg(Reg, &TRI);
567 }
568 return Dest;
569 }
570
571 static std::optional<yaml::SIArgumentInfo>
convertArgumentInfo(const AMDGPUFunctionArgInfo & ArgInfo,const TargetRegisterInfo & TRI)572 convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
573 const TargetRegisterInfo &TRI) {
574 yaml::SIArgumentInfo AI;
575
576 auto convertArg = [&](std::optional<yaml::SIArgument> &A,
577 const ArgDescriptor &Arg) {
578 if (!Arg)
579 return false;
580
581 // Create a register or stack argument.
582 yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister());
583 if (Arg.isRegister()) {
584 raw_string_ostream OS(SA.RegisterName.Value);
585 OS << printReg(Arg.getRegister(), &TRI);
586 } else
587 SA.StackOffset = Arg.getStackOffset();
588 // Check and update the optional mask.
589 if (Arg.isMasked())
590 SA.Mask = Arg.getMask();
591
592 A = SA;
593 return true;
594 };
595
596 bool Any = false;
597 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
598 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
599 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
600 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
601 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
602 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
603 Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);
604 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
605 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
606 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
607 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
608 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
609 Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
610 ArgInfo.PrivateSegmentWaveByteOffset);
611 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
612 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
613 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
614 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
615 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
616
617 if (Any)
618 return AI;
619
620 return std::nullopt;
621 }
622
SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo & MFI,const TargetRegisterInfo & TRI,const llvm::MachineFunction & MF)623 yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
624 const llvm::SIMachineFunctionInfo &MFI, const TargetRegisterInfo &TRI,
625 const llvm::MachineFunction &MF)
626 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
627 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
628 GDSSize(MFI.getGDSSize()),
629 DynLDSAlign(MFI.getDynLDSAlign()), IsEntryFunction(MFI.isEntryFunction()),
630 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
631 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
632 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
633 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
634 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
635 Occupancy(MFI.getOccupancy()),
636 ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
637 FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
638 StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
639 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
640 ReturnsVoid(MFI.returnsVoid()),
641 ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)), Mode(MFI.getMode()) {
642 for (Register Reg : MFI.getWWMReservedRegs())
643 WWMReservedRegs.push_back(regToString(Reg, TRI));
644
645 if (MFI.getVGPRForAGPRCopy())
646 VGPRForAGPRCopy = regToString(MFI.getVGPRForAGPRCopy(), TRI);
647 auto SFI = MFI.getOptionalScavengeFI();
648 if (SFI)
649 ScavengeFI = yaml::FrameIndex(*SFI, MF.getFrameInfo());
650 }
651
mappingImpl(yaml::IO & YamlIO)652 void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
653 MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this);
654 }
655
initializeBaseYamlFields(const yaml::SIMachineFunctionInfo & YamlMFI,const MachineFunction & MF,PerFunctionMIParsingState & PFS,SMDiagnostic & Error,SMRange & SourceRange)656 bool SIMachineFunctionInfo::initializeBaseYamlFields(
657 const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF,
658 PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) {
659 ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
660 MaxKernArgAlign = YamlMFI.MaxKernArgAlign;
661 LDSSize = YamlMFI.LDSSize;
662 GDSSize = YamlMFI.GDSSize;
663 DynLDSAlign = YamlMFI.DynLDSAlign;
664 HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress;
665 Occupancy = YamlMFI.Occupancy;
666 IsEntryFunction = YamlMFI.IsEntryFunction;
667 NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;
668 MemoryBound = YamlMFI.MemoryBound;
669 WaveLimiter = YamlMFI.WaveLimiter;
670 HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
671 HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
672 BytesInStackArgArea = YamlMFI.BytesInStackArgArea;
673 ReturnsVoid = YamlMFI.ReturnsVoid;
674
675 if (YamlMFI.ScavengeFI) {
676 auto FIOrErr = YamlMFI.ScavengeFI->getFI(MF.getFrameInfo());
677 if (!FIOrErr) {
678 // Create a diagnostic for a the frame index.
679 const MemoryBuffer &Buffer =
680 *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());
681
682 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1,
683 SourceMgr::DK_Error, toString(FIOrErr.takeError()),
684 "", std::nullopt, std::nullopt);
685 SourceRange = YamlMFI.ScavengeFI->SourceRange;
686 return true;
687 }
688 ScavengeFI = *FIOrErr;
689 } else {
690 ScavengeFI = std::nullopt;
691 }
692 return false;
693 }
694
mayUseAGPRs(const Function & F) const695 bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const {
696 for (const BasicBlock &BB : F) {
697 for (const Instruction &I : BB) {
698 const auto *CB = dyn_cast<CallBase>(&I);
699 if (!CB)
700 continue;
701
702 if (CB->isInlineAsm()) {
703 const InlineAsm *IA = dyn_cast<InlineAsm>(CB->getCalledOperand());
704 for (const auto &CI : IA->ParseConstraints()) {
705 for (StringRef Code : CI.Codes) {
706 Code.consume_front("{");
707 if (Code.startswith("a"))
708 return true;
709 }
710 }
711 continue;
712 }
713
714 const Function *Callee =
715 dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());
716 if (!Callee)
717 return true;
718
719 if (Callee->getIntrinsicID() == Intrinsic::not_intrinsic)
720 return true;
721 }
722 }
723
724 return false;
725 }
726
usesAGPRs(const MachineFunction & MF) const727 bool SIMachineFunctionInfo::usesAGPRs(const MachineFunction &MF) const {
728 if (UsesAGPRs)
729 return *UsesAGPRs;
730
731 if (!mayNeedAGPRs()) {
732 UsesAGPRs = false;
733 return false;
734 }
735
736 if (!AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv()) ||
737 MF.getFrameInfo().hasCalls()) {
738 UsesAGPRs = true;
739 return true;
740 }
741
742 const MachineRegisterInfo &MRI = MF.getRegInfo();
743
744 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
745 const Register Reg = Register::index2VirtReg(I);
746 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
747 if (RC && SIRegisterInfo::isAGPRClass(RC)) {
748 UsesAGPRs = true;
749 return true;
750 } else if (!RC && !MRI.use_empty(Reg) && MRI.getType(Reg).isValid()) {
751 // Defer caching UsesAGPRs, function might not yet been regbank selected.
752 return true;
753 }
754 }
755
756 for (MCRegister Reg : AMDGPU::AGPR_32RegClass) {
757 if (MRI.isPhysRegUsed(Reg)) {
758 UsesAGPRs = true;
759 return true;
760 }
761 }
762
763 UsesAGPRs = false;
764 return false;
765 }
766