1 //===----------------------- SIFrameLowering.cpp --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8
9 #include "SIFrameLowering.h"
10 #include "AMDGPUSubtarget.h"
11 #include "SIInstrInfo.h"
12 #include "SIMachineFunctionInfo.h"
13 #include "SIRegisterInfo.h"
14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
15
16 #include "llvm/CodeGen/LivePhysRegs.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 #include "llvm/CodeGen/RegisterScavenging.h"
21
22 using namespace llvm;
23
24 #define DEBUG_TYPE "frame-info"
25
26
27 // Find a scratch register that we can use at the start of the prologue to
28 // re-align the stack pointer. We avoid using callee-save registers since they
29 // may appear to be free when this is called from canUseAsPrologue (during
30 // shrink wrapping), but then no longer be free when this is called from
31 // emitPrologue.
32 //
33 // FIXME: This is a bit conservative, since in the above case we could use one
34 // of the callee-save registers as a scratch temp to re-align the stack pointer,
35 // but we would then have to make sure that we were in fact saving at least one
36 // callee-save register in the prologue, which is additional complexity that
37 // doesn't seem worth the benefit.
findScratchNonCalleeSaveRegister(MachineRegisterInfo & MRI,LivePhysRegs & LiveRegs,const TargetRegisterClass & RC,bool Unused=false)38 static MCRegister findScratchNonCalleeSaveRegister(MachineRegisterInfo &MRI,
39 LivePhysRegs &LiveRegs,
40 const TargetRegisterClass &RC,
41 bool Unused = false) {
42 // Mark callee saved registers as used so we will not choose them.
43 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
44 for (unsigned i = 0; CSRegs[i]; ++i)
45 LiveRegs.addReg(CSRegs[i]);
46
47 if (Unused) {
48 // We are looking for a register that can be used throughout the entire
49 // function, so any use is unacceptable.
50 for (MCRegister Reg : RC) {
51 if (!MRI.isPhysRegUsed(Reg) && LiveRegs.available(MRI, Reg))
52 return Reg;
53 }
54 } else {
55 for (MCRegister Reg : RC) {
56 if (LiveRegs.available(MRI, Reg))
57 return Reg;
58 }
59 }
60
61 // If we require an unused register, this is used in contexts where failure is
62 // an option and has an alternative plan. In other contexts, this must
63 // succeed0.
64 if (!Unused)
65 report_fatal_error("failed to find free scratch register");
66
67 return MCRegister();
68 }
69
getVGPRSpillLaneOrTempRegister(MachineFunction & MF,LivePhysRegs & LiveRegs,Register & TempSGPR,Optional<int> & FrameIndex,bool IsFP)70 static void getVGPRSpillLaneOrTempRegister(MachineFunction &MF,
71 LivePhysRegs &LiveRegs,
72 Register &TempSGPR,
73 Optional<int> &FrameIndex,
74 bool IsFP) {
75 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
76 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
77
78 #ifndef NDEBUG
79 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
80 const SIRegisterInfo *TRI = ST.getRegisterInfo();
81 #endif
82
83 // We need to save and restore the current FP/BP.
84
85 // 1: If there is already a VGPR with free lanes, use it. We
86 // may already have to pay the penalty for spilling a CSR VGPR.
87 if (MFI->haveFreeLanesForSGPRSpill(MF, 1)) {
88 int NewFI = FrameInfo.CreateStackObject(4, Align(4), true, nullptr,
89 TargetStackID::SGPRSpill);
90
91 if (!MFI->allocateSGPRSpillToVGPR(MF, NewFI))
92 llvm_unreachable("allocate SGPR spill should have worked");
93
94 FrameIndex = NewFI;
95
96 LLVM_DEBUG(auto Spill = MFI->getSGPRToVGPRSpills(NewFI).front();
97 dbgs() << "Spilling " << (IsFP ? "FP" : "BP") << " to "
98 << printReg(Spill.VGPR, TRI) << ':' << Spill.Lane
99 << '\n');
100 return;
101 }
102
103 // 2: Next, try to save the FP/BP in an unused SGPR.
104 TempSGPR = findScratchNonCalleeSaveRegister(
105 MF.getRegInfo(), LiveRegs, AMDGPU::SReg_32_XM0_XEXECRegClass, true);
106
107 if (!TempSGPR) {
108 int NewFI = FrameInfo.CreateStackObject(4, Align(4), true, nullptr,
109 TargetStackID::SGPRSpill);
110
111 if (MFI->allocateSGPRSpillToVGPR(MF, NewFI)) {
112 // 3: There's no free lane to spill, and no free register to save FP/BP,
113 // so we're forced to spill another VGPR to use for the spill.
114 FrameIndex = NewFI;
115 } else {
116 // 4: If all else fails, spill the FP/BP to memory.
117 FrameIndex = FrameInfo.CreateSpillStackObject(4, Align(4));
118 }
119
120 LLVM_DEBUG(auto Spill = MFI->getSGPRToVGPRSpills(NewFI).front();
121 dbgs() << (IsFP ? "FP" : "BP") << " requires fallback spill to "
122 << printReg(Spill.VGPR, TRI) << ':' << Spill.Lane
123 << '\n';);
124 } else {
125 LLVM_DEBUG(dbgs() << "Saving " << (IsFP ? "FP" : "BP") << " with copy to "
126 << printReg(TempSGPR, TRI) << '\n');
127 }
128 }
129
130 // We need to specially emit stack operations here because a different frame
131 // register is used than in the rest of the function, as getFrameRegister would
132 // use.
buildPrologSpill(LivePhysRegs & LiveRegs,MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const SIInstrInfo * TII,Register SpillReg,Register ScratchRsrcReg,Register SPReg,int FI)133 static void buildPrologSpill(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
134 MachineBasicBlock::iterator I,
135 const SIInstrInfo *TII, Register SpillReg,
136 Register ScratchRsrcReg, Register SPReg, int FI) {
137 MachineFunction *MF = MBB.getParent();
138 MachineFrameInfo &MFI = MF->getFrameInfo();
139
140 int64_t Offset = MFI.getObjectOffset(FI);
141
142 MachineMemOperand *MMO = MF->getMachineMemOperand(
143 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 4,
144 MFI.getObjectAlign(FI));
145
146 if (isUInt<12>(Offset)) {
147 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::BUFFER_STORE_DWORD_OFFSET))
148 .addReg(SpillReg, RegState::Kill)
149 .addReg(ScratchRsrcReg)
150 .addReg(SPReg)
151 .addImm(Offset)
152 .addImm(0) // glc
153 .addImm(0) // slc
154 .addImm(0) // tfe
155 .addImm(0) // dlc
156 .addImm(0) // swz
157 .addMemOperand(MMO);
158 return;
159 }
160
161 MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister(
162 MF->getRegInfo(), LiveRegs, AMDGPU::VGPR_32RegClass);
163
164 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::V_MOV_B32_e32), OffsetReg)
165 .addImm(Offset);
166
167 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::BUFFER_STORE_DWORD_OFFEN))
168 .addReg(SpillReg, RegState::Kill)
169 .addReg(OffsetReg, RegState::Kill)
170 .addReg(ScratchRsrcReg)
171 .addReg(SPReg)
172 .addImm(0)
173 .addImm(0) // glc
174 .addImm(0) // slc
175 .addImm(0) // tfe
176 .addImm(0) // dlc
177 .addImm(0) // swz
178 .addMemOperand(MMO);
179 }
180
buildEpilogReload(LivePhysRegs & LiveRegs,MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const SIInstrInfo * TII,Register SpillReg,Register ScratchRsrcReg,Register SPReg,int FI)181 static void buildEpilogReload(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
182 MachineBasicBlock::iterator I,
183 const SIInstrInfo *TII, Register SpillReg,
184 Register ScratchRsrcReg, Register SPReg, int FI) {
185 MachineFunction *MF = MBB.getParent();
186 MachineFrameInfo &MFI = MF->getFrameInfo();
187 int64_t Offset = MFI.getObjectOffset(FI);
188
189 MachineMemOperand *MMO = MF->getMachineMemOperand(
190 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad, 4,
191 MFI.getObjectAlign(FI));
192
193 if (isUInt<12>(Offset)) {
194 BuildMI(MBB, I, DebugLoc(),
195 TII->get(AMDGPU::BUFFER_LOAD_DWORD_OFFSET), SpillReg)
196 .addReg(ScratchRsrcReg)
197 .addReg(SPReg)
198 .addImm(Offset)
199 .addImm(0) // glc
200 .addImm(0) // slc
201 .addImm(0) // tfe
202 .addImm(0) // dlc
203 .addImm(0) // swz
204 .addMemOperand(MMO);
205 return;
206 }
207
208 MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister(
209 MF->getRegInfo(), LiveRegs, AMDGPU::VGPR_32RegClass);
210
211 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::V_MOV_B32_e32), OffsetReg)
212 .addImm(Offset);
213
214 BuildMI(MBB, I, DebugLoc(),
215 TII->get(AMDGPU::BUFFER_LOAD_DWORD_OFFEN), SpillReg)
216 .addReg(OffsetReg, RegState::Kill)
217 .addReg(ScratchRsrcReg)
218 .addReg(SPReg)
219 .addImm(0)
220 .addImm(0) // glc
221 .addImm(0) // slc
222 .addImm(0) // tfe
223 .addImm(0) // dlc
224 .addImm(0) // swz
225 .addMemOperand(MMO);
226 }
227
228 // Emit flat scratch setup code, assuming `MFI->hasFlatScratchInit()`
emitEntryFunctionFlatScratchInit(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const DebugLoc & DL,Register ScratchWaveOffsetReg) const229 void SIFrameLowering::emitEntryFunctionFlatScratchInit(
230 MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
231 const DebugLoc &DL, Register ScratchWaveOffsetReg) const {
232 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
233 const SIInstrInfo *TII = ST.getInstrInfo();
234 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
235 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
236
237 // We don't need this if we only have spills since there is no user facing
238 // scratch.
239
240 // TODO: If we know we don't have flat instructions earlier, we can omit
241 // this from the input registers.
242 //
243 // TODO: We only need to know if we access scratch space through a flat
244 // pointer. Because we only detect if flat instructions are used at all,
245 // this will be used more often than necessary on VI.
246
247 Register FlatScratchInitReg =
248 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::FLAT_SCRATCH_INIT);
249
250 MachineRegisterInfo &MRI = MF.getRegInfo();
251 MRI.addLiveIn(FlatScratchInitReg);
252 MBB.addLiveIn(FlatScratchInitReg);
253
254 Register FlatScrInitLo = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub0);
255 Register FlatScrInitHi = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub1);
256
257 // Do a 64-bit pointer add.
258 if (ST.flatScratchIsPointer()) {
259 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
260 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), FlatScrInitLo)
261 .addReg(FlatScrInitLo)
262 .addReg(ScratchWaveOffsetReg);
263 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32), FlatScrInitHi)
264 .addReg(FlatScrInitHi)
265 .addImm(0);
266 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SETREG_B32)).
267 addReg(FlatScrInitLo).
268 addImm(int16_t(AMDGPU::Hwreg::ID_FLAT_SCR_LO |
269 (31 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_)));
270 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SETREG_B32)).
271 addReg(FlatScrInitHi).
272 addImm(int16_t(AMDGPU::Hwreg::ID_FLAT_SCR_HI |
273 (31 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_)));
274 return;
275 }
276
277 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), AMDGPU::FLAT_SCR_LO)
278 .addReg(FlatScrInitLo)
279 .addReg(ScratchWaveOffsetReg);
280 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32), AMDGPU::FLAT_SCR_HI)
281 .addReg(FlatScrInitHi)
282 .addImm(0);
283
284 return;
285 }
286
287 assert(ST.getGeneration() < AMDGPUSubtarget::GFX10);
288
289 // Copy the size in bytes.
290 BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::FLAT_SCR_LO)
291 .addReg(FlatScrInitHi, RegState::Kill);
292
293 // Add wave offset in bytes to private base offset.
294 // See comment in AMDKernelCodeT.h for enable_sgpr_flat_scratch_init.
295 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), FlatScrInitLo)
296 .addReg(FlatScrInitLo)
297 .addReg(ScratchWaveOffsetReg);
298
299 // Convert offset to 256-byte units.
300 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_LSHR_B32), AMDGPU::FLAT_SCR_HI)
301 .addReg(FlatScrInitLo, RegState::Kill)
302 .addImm(8);
303 }
304
305 // Shift down registers reserved for the scratch RSRC.
getEntryFunctionReservedScratchRsrcReg(MachineFunction & MF) const306 Register SIFrameLowering::getEntryFunctionReservedScratchRsrcReg(
307 MachineFunction &MF) const {
308
309 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
310 const SIInstrInfo *TII = ST.getInstrInfo();
311 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
312 MachineRegisterInfo &MRI = MF.getRegInfo();
313 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
314
315 assert(MFI->isEntryFunction());
316
317 Register ScratchRsrcReg = MFI->getScratchRSrcReg();
318
319 if (!ScratchRsrcReg || !MRI.isPhysRegUsed(ScratchRsrcReg))
320 return Register();
321
322 if (ST.hasSGPRInitBug() ||
323 ScratchRsrcReg != TRI->reservedPrivateSegmentBufferReg(MF))
324 return ScratchRsrcReg;
325
326 // We reserved the last registers for this. Shift it down to the end of those
327 // which were actually used.
328 //
329 // FIXME: It might be safer to use a pseudoregister before replacement.
330
331 // FIXME: We should be able to eliminate unused input registers. We only
332 // cannot do this for the resources required for scratch access. For now we
333 // skip over user SGPRs and may leave unused holes.
334
335 unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 3) / 4;
336 ArrayRef<MCPhysReg> AllSGPR128s = TRI->getAllSGPR128(MF);
337 AllSGPR128s = AllSGPR128s.slice(std::min(static_cast<unsigned>(AllSGPR128s.size()), NumPreloaded));
338
339 // Skip the last N reserved elements because they should have already been
340 // reserved for VCC etc.
341 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
342 for (MCPhysReg Reg : AllSGPR128s) {
343 // Pick the first unallocated one. Make sure we don't clobber the other
344 // reserved input we needed. Also for PAL, make sure we don't clobber
345 // the GIT pointer passed in SGPR0 or SGPR8.
346 if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg) &&
347 !TRI->isSubRegisterEq(Reg, GITPtrLoReg)) {
348 MRI.replaceRegWith(ScratchRsrcReg, Reg);
349 MFI->setScratchRSrcReg(Reg);
350 return Reg;
351 }
352 }
353
354 return ScratchRsrcReg;
355 }
356
emitEntryFunctionPrologue(MachineFunction & MF,MachineBasicBlock & MBB) const357 void SIFrameLowering::emitEntryFunctionPrologue(MachineFunction &MF,
358 MachineBasicBlock &MBB) const {
359 assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
360
361 // FIXME: If we only have SGPR spills, we won't actually be using scratch
362 // memory since these spill to VGPRs. We should be cleaning up these unused
363 // SGPR spill frame indices somewhere.
364
365 // FIXME: We still have implicit uses on SGPR spill instructions in case they
366 // need to spill to vector memory. It's likely that will not happen, but at
367 // this point it appears we need the setup. This part of the prolog should be
368 // emitted after frame indices are eliminated.
369
370 // FIXME: Remove all of the isPhysRegUsed checks
371
372 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
373 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
374 const SIInstrInfo *TII = ST.getInstrInfo();
375 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
376 MachineRegisterInfo &MRI = MF.getRegInfo();
377 const Function &F = MF.getFunction();
378
379 assert(MFI->isEntryFunction());
380
381 Register PreloadedScratchWaveOffsetReg = MFI->getPreloadedReg(
382 AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET);
383 // FIXME: Hack to not crash in situations which emitted an error.
384 if (!PreloadedScratchWaveOffsetReg)
385 return;
386
387 // We need to do the replacement of the private segment buffer register even
388 // if there are no stack objects. There could be stores to undef or a
389 // constant without an associated object.
390 //
391 // This will return `Register()` in cases where there are no actual
392 // uses of the SRSRC.
393 Register ScratchRsrcReg = getEntryFunctionReservedScratchRsrcReg(MF);
394
395 // Make the selected register live throughout the function.
396 if (ScratchRsrcReg) {
397 for (MachineBasicBlock &OtherBB : MF) {
398 if (&OtherBB != &MBB) {
399 OtherBB.addLiveIn(ScratchRsrcReg);
400 }
401 }
402 }
403
404 // Now that we have fixed the reserved SRSRC we need to locate the
405 // (potentially) preloaded SRSRC.
406 Register PreloadedScratchRsrcReg;
407 if (ST.isAmdHsaOrMesa(F)) {
408 PreloadedScratchRsrcReg =
409 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
410 if (ScratchRsrcReg && PreloadedScratchRsrcReg) {
411 // We added live-ins during argument lowering, but since they were not
412 // used they were deleted. We're adding the uses now, so add them back.
413 MRI.addLiveIn(PreloadedScratchRsrcReg);
414 MBB.addLiveIn(PreloadedScratchRsrcReg);
415 }
416 }
417
418 // Debug location must be unknown since the first debug location is used to
419 // determine the end of the prologue.
420 DebugLoc DL;
421 MachineBasicBlock::iterator I = MBB.begin();
422
423 // We found the SRSRC first because it needs four registers and has an
424 // alignment requirement. If the SRSRC that we found is clobbering with
425 // the scratch wave offset, which may be in a fixed SGPR or a free SGPR
426 // chosen by SITargetLowering::allocateSystemSGPRs, COPY the scratch
427 // wave offset to a free SGPR.
428 Register ScratchWaveOffsetReg;
429 if (TRI->isSubRegisterEq(ScratchRsrcReg, PreloadedScratchWaveOffsetReg)) {
430 ArrayRef<MCPhysReg> AllSGPRs = TRI->getAllSGPR32(MF);
431 unsigned NumPreloaded = MFI->getNumPreloadedSGPRs();
432 AllSGPRs = AllSGPRs.slice(
433 std::min(static_cast<unsigned>(AllSGPRs.size()), NumPreloaded));
434 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
435 for (MCPhysReg Reg : AllSGPRs) {
436 if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg) &&
437 !TRI->isSubRegisterEq(ScratchRsrcReg, Reg) && GITPtrLoReg != Reg) {
438 ScratchWaveOffsetReg = Reg;
439 BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchWaveOffsetReg)
440 .addReg(PreloadedScratchWaveOffsetReg, RegState::Kill);
441 break;
442 }
443 }
444 } else {
445 ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg;
446 }
447 assert(ScratchWaveOffsetReg);
448
449 if (MF.getFrameInfo().hasCalls()) {
450 Register SPReg = MFI->getStackPtrOffsetReg();
451 assert(SPReg != AMDGPU::SP_REG);
452 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), SPReg)
453 .addImm(MF.getFrameInfo().getStackSize() * ST.getWavefrontSize());
454 }
455
456 if (hasFP(MF)) {
457 Register FPReg = MFI->getFrameOffsetReg();
458 assert(FPReg != AMDGPU::FP_REG);
459 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), FPReg).addImm(0);
460 }
461
462 if (MFI->hasFlatScratchInit() || ScratchRsrcReg) {
463 MRI.addLiveIn(PreloadedScratchWaveOffsetReg);
464 MBB.addLiveIn(PreloadedScratchWaveOffsetReg);
465 }
466
467 if (MFI->hasFlatScratchInit()) {
468 emitEntryFunctionFlatScratchInit(MF, MBB, I, DL, ScratchWaveOffsetReg);
469 }
470
471 if (ScratchRsrcReg) {
472 emitEntryFunctionScratchRsrcRegSetup(MF, MBB, I, DL,
473 PreloadedScratchRsrcReg,
474 ScratchRsrcReg, ScratchWaveOffsetReg);
475 }
476 }
477
478 // Emit scratch RSRC setup code, assuming `ScratchRsrcReg != AMDGPU::NoReg`
emitEntryFunctionScratchRsrcRegSetup(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const DebugLoc & DL,Register PreloadedScratchRsrcReg,Register ScratchRsrcReg,Register ScratchWaveOffsetReg) const479 void SIFrameLowering::emitEntryFunctionScratchRsrcRegSetup(
480 MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
481 const DebugLoc &DL, Register PreloadedScratchRsrcReg,
482 Register ScratchRsrcReg, Register ScratchWaveOffsetReg) const {
483
484 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
485 const SIInstrInfo *TII = ST.getInstrInfo();
486 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
487 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
488 const Function &Fn = MF.getFunction();
489
490 if (ST.isAmdPalOS()) {
491 // The pointer to the GIT is formed from the offset passed in and either
492 // the amdgpu-git-ptr-high function attribute or the top part of the PC
493 Register RsrcLo = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
494 Register RsrcHi = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
495 Register Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
496
497 const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
498
499 if (MFI->getGITPtrHigh() != 0xffffffff) {
500 BuildMI(MBB, I, DL, SMovB32, RsrcHi)
501 .addImm(MFI->getGITPtrHigh())
502 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
503 } else {
504 const MCInstrDesc &GetPC64 = TII->get(AMDGPU::S_GETPC_B64);
505 BuildMI(MBB, I, DL, GetPC64, Rsrc01);
506 }
507 Register GitPtrLo = MFI->getGITPtrLoReg(MF);
508 MF.getRegInfo().addLiveIn(GitPtrLo);
509 MBB.addLiveIn(GitPtrLo);
510 BuildMI(MBB, I, DL, SMovB32, RsrcLo)
511 .addReg(GitPtrLo)
512 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
513
514 // We now have the GIT ptr - now get the scratch descriptor from the entry
515 // at offset 0 (or offset 16 for a compute shader).
516 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
517 const MCInstrDesc &LoadDwordX4 = TII->get(AMDGPU::S_LOAD_DWORDX4_IMM);
518 auto MMO = MF.getMachineMemOperand(PtrInfo,
519 MachineMemOperand::MOLoad |
520 MachineMemOperand::MOInvariant |
521 MachineMemOperand::MODereferenceable,
522 16, Align(4));
523 unsigned Offset = Fn.getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
524 const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
525 unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(Subtarget, Offset);
526 BuildMI(MBB, I, DL, LoadDwordX4, ScratchRsrcReg)
527 .addReg(Rsrc01)
528 .addImm(EncodedOffset) // offset
529 .addImm(0) // glc
530 .addImm(0) // dlc
531 .addReg(ScratchRsrcReg, RegState::ImplicitDefine)
532 .addMemOperand(MMO);
533 } else if (ST.isMesaGfxShader(Fn) || !PreloadedScratchRsrcReg) {
534 assert(!ST.isAmdHsaOrMesa(Fn));
535 const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
536
537 Register Rsrc2 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2);
538 Register Rsrc3 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
539
540 // Use relocations to get the pointer, and setup the other bits manually.
541 uint64_t Rsrc23 = TII->getScratchRsrcWords23();
542
543 if (MFI->hasImplicitBufferPtr()) {
544 Register Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
545
546 if (AMDGPU::isCompute(MF.getFunction().getCallingConv())) {
547 const MCInstrDesc &Mov64 = TII->get(AMDGPU::S_MOV_B64);
548
549 BuildMI(MBB, I, DL, Mov64, Rsrc01)
550 .addReg(MFI->getImplicitBufferPtrUserSGPR())
551 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
552 } else {
553 const MCInstrDesc &LoadDwordX2 = TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
554
555 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
556 auto MMO = MF.getMachineMemOperand(
557 PtrInfo,
558 MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
559 MachineMemOperand::MODereferenceable,
560 8, Align(4));
561 BuildMI(MBB, I, DL, LoadDwordX2, Rsrc01)
562 .addReg(MFI->getImplicitBufferPtrUserSGPR())
563 .addImm(0) // offset
564 .addImm(0) // glc
565 .addImm(0) // dlc
566 .addMemOperand(MMO)
567 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
568
569 MF.getRegInfo().addLiveIn(MFI->getImplicitBufferPtrUserSGPR());
570 MBB.addLiveIn(MFI->getImplicitBufferPtrUserSGPR());
571 }
572 } else {
573 Register Rsrc0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
574 Register Rsrc1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
575
576 BuildMI(MBB, I, DL, SMovB32, Rsrc0)
577 .addExternalSymbol("SCRATCH_RSRC_DWORD0")
578 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
579
580 BuildMI(MBB, I, DL, SMovB32, Rsrc1)
581 .addExternalSymbol("SCRATCH_RSRC_DWORD1")
582 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
583
584 }
585
586 BuildMI(MBB, I, DL, SMovB32, Rsrc2)
587 .addImm(Rsrc23 & 0xffffffff)
588 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
589
590 BuildMI(MBB, I, DL, SMovB32, Rsrc3)
591 .addImm(Rsrc23 >> 32)
592 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
593 } else if (ST.isAmdHsaOrMesa(Fn)) {
594 assert(PreloadedScratchRsrcReg);
595
596 if (ScratchRsrcReg != PreloadedScratchRsrcReg) {
597 BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchRsrcReg)
598 .addReg(PreloadedScratchRsrcReg, RegState::Kill);
599 }
600 }
601
602 // Add the scratch wave offset into the scratch RSRC.
603 //
604 // We only want to update the first 48 bits, which is the base address
605 // pointer, without touching the adjacent 16 bits of flags. We know this add
606 // cannot carry-out from bit 47, otherwise the scratch allocation would be
607 // impossible to fit in the 48-bit global address space.
608 //
609 // TODO: Evaluate if it is better to just construct an SRD using the flat
610 // scratch init and some constants rather than update the one we are passed.
611 Register ScratchRsrcSub0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
612 Register ScratchRsrcSub1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
613
614 // We cannot Kill ScratchWaveOffsetReg here because we allow it to be used in
615 // the kernel body via inreg arguments.
616 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), ScratchRsrcSub0)
617 .addReg(ScratchRsrcSub0)
618 .addReg(ScratchWaveOffsetReg)
619 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
620 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32), ScratchRsrcSub1)
621 .addReg(ScratchRsrcSub1)
622 .addImm(0)
623 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
624 }
625
isSupportedStackID(TargetStackID::Value ID) const626 bool SIFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
627 switch (ID) {
628 case TargetStackID::Default:
629 case TargetStackID::NoAlloc:
630 case TargetStackID::SGPRSpill:
631 return true;
632 case TargetStackID::SVEVector:
633 return false;
634 }
635 llvm_unreachable("Invalid TargetStackID::Value");
636 }
637
638 // Activate all lanes, returns saved exec.
buildScratchExecCopy(LivePhysRegs & LiveRegs,MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,bool IsProlog)639 static Register buildScratchExecCopy(LivePhysRegs &LiveRegs,
640 MachineFunction &MF,
641 MachineBasicBlock &MBB,
642 MachineBasicBlock::iterator MBBI,
643 bool IsProlog) {
644 Register ScratchExecCopy;
645 MachineRegisterInfo &MRI = MF.getRegInfo();
646 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
647 const SIInstrInfo *TII = ST.getInstrInfo();
648 const SIRegisterInfo &TRI = TII->getRegisterInfo();
649 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
650 DebugLoc DL;
651
652 if (LiveRegs.empty()) {
653 if (IsProlog) {
654 LiveRegs.init(TRI);
655 LiveRegs.addLiveIns(MBB);
656 if (FuncInfo->SGPRForFPSaveRestoreCopy)
657 LiveRegs.removeReg(FuncInfo->SGPRForFPSaveRestoreCopy);
658
659 if (FuncInfo->SGPRForBPSaveRestoreCopy)
660 LiveRegs.removeReg(FuncInfo->SGPRForBPSaveRestoreCopy);
661 } else {
662 // In epilog.
663 LiveRegs.init(*ST.getRegisterInfo());
664 LiveRegs.addLiveOuts(MBB);
665 LiveRegs.stepBackward(*MBBI);
666 }
667 }
668
669 ScratchExecCopy = findScratchNonCalleeSaveRegister(
670 MRI, LiveRegs, *TRI.getWaveMaskRegClass());
671
672 if (!IsProlog)
673 LiveRegs.removeReg(ScratchExecCopy);
674
675 const unsigned OrSaveExec =
676 ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 : AMDGPU::S_OR_SAVEEXEC_B64;
677 BuildMI(MBB, MBBI, DL, TII->get(OrSaveExec), ScratchExecCopy).addImm(-1);
678
679 return ScratchExecCopy;
680 }
681
emitPrologue(MachineFunction & MF,MachineBasicBlock & MBB) const682 void SIFrameLowering::emitPrologue(MachineFunction &MF,
683 MachineBasicBlock &MBB) const {
684 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
685 if (FuncInfo->isEntryFunction()) {
686 emitEntryFunctionPrologue(MF, MBB);
687 return;
688 }
689
690 const MachineFrameInfo &MFI = MF.getFrameInfo();
691 MachineRegisterInfo &MRI = MF.getRegInfo();
692 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
693 const SIInstrInfo *TII = ST.getInstrInfo();
694 const SIRegisterInfo &TRI = TII->getRegisterInfo();
695
696 Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
697 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
698 Register BasePtrReg =
699 TRI.hasBasePointer(MF) ? TRI.getBaseRegister() : Register();
700 LivePhysRegs LiveRegs;
701
702 MachineBasicBlock::iterator MBBI = MBB.begin();
703 DebugLoc DL;
704
705 bool HasFP = false;
706 bool HasBP = false;
707 uint32_t NumBytes = MFI.getStackSize();
708 uint32_t RoundedSize = NumBytes;
709 // To avoid clobbering VGPRs in lanes that weren't active on function entry,
710 // turn on all lanes before doing the spill to memory.
711 Register ScratchExecCopy;
712
713 bool HasFPSaveIndex = FuncInfo->FramePointerSaveIndex.hasValue();
714 bool SpillFPToMemory = false;
715 // A StackID of SGPRSpill implies that this is a spill from SGPR to VGPR.
716 // Otherwise we are spilling the FP to memory.
717 if (HasFPSaveIndex) {
718 SpillFPToMemory = MFI.getStackID(*FuncInfo->FramePointerSaveIndex) !=
719 TargetStackID::SGPRSpill;
720 }
721
722 bool HasBPSaveIndex = FuncInfo->BasePointerSaveIndex.hasValue();
723 bool SpillBPToMemory = false;
724 // A StackID of SGPRSpill implies that this is a spill from SGPR to VGPR.
725 // Otherwise we are spilling the BP to memory.
726 if (HasBPSaveIndex) {
727 SpillBPToMemory = MFI.getStackID(*FuncInfo->BasePointerSaveIndex) !=
728 TargetStackID::SGPRSpill;
729 }
730
731 // Emit the copy if we need an FP, and are using a free SGPR to save it.
732 if (FuncInfo->SGPRForFPSaveRestoreCopy) {
733 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FuncInfo->SGPRForFPSaveRestoreCopy)
734 .addReg(FramePtrReg)
735 .setMIFlag(MachineInstr::FrameSetup);
736 }
737
738 // Emit the copy if we need a BP, and are using a free SGPR to save it.
739 if (FuncInfo->SGPRForBPSaveRestoreCopy) {
740 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY),
741 FuncInfo->SGPRForBPSaveRestoreCopy)
742 .addReg(BasePtrReg)
743 .setMIFlag(MachineInstr::FrameSetup);
744 }
745
746 // If a copy has been emitted for FP and/or BP, Make the SGPRs
747 // used in the copy instructions live throughout the function.
748 SmallVector<MCPhysReg, 2> TempSGPRs;
749 if (FuncInfo->SGPRForFPSaveRestoreCopy)
750 TempSGPRs.push_back(FuncInfo->SGPRForFPSaveRestoreCopy);
751
752 if (FuncInfo->SGPRForBPSaveRestoreCopy)
753 TempSGPRs.push_back(FuncInfo->SGPRForBPSaveRestoreCopy);
754
755 if (!TempSGPRs.empty()) {
756 for (MachineBasicBlock &MBB : MF) {
757 for (MCPhysReg Reg : TempSGPRs)
758 MBB.addLiveIn(Reg);
759
760 MBB.sortUniqueLiveIns();
761 }
762 }
763
764 for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg
765 : FuncInfo->getSGPRSpillVGPRs()) {
766 if (!Reg.FI.hasValue())
767 continue;
768
769 if (!ScratchExecCopy)
770 ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
771
772 buildPrologSpill(LiveRegs, MBB, MBBI, TII, Reg.VGPR,
773 FuncInfo->getScratchRSrcReg(),
774 StackPtrReg,
775 Reg.FI.getValue());
776 }
777
778 if (HasFPSaveIndex && SpillFPToMemory) {
779 assert(!MFI.isDeadObjectIndex(FuncInfo->FramePointerSaveIndex.getValue()));
780
781 if (!ScratchExecCopy)
782 ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
783
784 MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister(
785 MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
786
787 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
788 .addReg(FramePtrReg);
789
790 buildPrologSpill(LiveRegs, MBB, MBBI, TII, TmpVGPR,
791 FuncInfo->getScratchRSrcReg(), StackPtrReg,
792 FuncInfo->FramePointerSaveIndex.getValue());
793 }
794
795 if (HasBPSaveIndex && SpillBPToMemory) {
796 assert(!MFI.isDeadObjectIndex(*FuncInfo->BasePointerSaveIndex));
797
798 if (!ScratchExecCopy)
799 ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
800
801 MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister(
802 MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
803
804 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
805 .addReg(BasePtrReg);
806
807 buildPrologSpill(LiveRegs, MBB, MBBI, TII, TmpVGPR,
808 FuncInfo->getScratchRSrcReg(), StackPtrReg,
809 *FuncInfo->BasePointerSaveIndex);
810 }
811
812 if (ScratchExecCopy) {
813 // FIXME: Split block and make terminator.
814 unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
815 MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
816 BuildMI(MBB, MBBI, DL, TII->get(ExecMov), Exec)
817 .addReg(ScratchExecCopy, RegState::Kill);
818 LiveRegs.addReg(ScratchExecCopy);
819 }
820
821 // In this case, spill the FP to a reserved VGPR.
822 if (HasFPSaveIndex && !SpillFPToMemory) {
823 const int FI = FuncInfo->FramePointerSaveIndex.getValue();
824 assert(!MFI.isDeadObjectIndex(FI));
825
826 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
827 ArrayRef<SIMachineFunctionInfo::SpilledReg> Spill =
828 FuncInfo->getSGPRToVGPRSpills(FI);
829 assert(Spill.size() == 1);
830
831 // Save FP before setting it up.
832 // FIXME: This should respect spillSGPRToVGPR;
833 BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
834 Spill[0].VGPR)
835 .addReg(FramePtrReg)
836 .addImm(Spill[0].Lane)
837 .addReg(Spill[0].VGPR, RegState::Undef);
838 }
839
840 // In this case, spill the BP to a reserved VGPR.
841 if (HasBPSaveIndex && !SpillBPToMemory) {
842 const int BasePtrFI = *FuncInfo->BasePointerSaveIndex;
843 assert(!MFI.isDeadObjectIndex(BasePtrFI));
844
845 assert(MFI.getStackID(BasePtrFI) == TargetStackID::SGPRSpill);
846 ArrayRef<SIMachineFunctionInfo::SpilledReg> Spill =
847 FuncInfo->getSGPRToVGPRSpills(BasePtrFI);
848 assert(Spill.size() == 1);
849
850 // Save BP before setting it up.
851 // FIXME: This should respect spillSGPRToVGPR;
852 BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
853 Spill[0].VGPR)
854 .addReg(BasePtrReg)
855 .addImm(Spill[0].Lane)
856 .addReg(Spill[0].VGPR, RegState::Undef);
857 }
858
859 if (TRI.needsStackRealignment(MF)) {
860 HasFP = true;
861 const unsigned Alignment = MFI.getMaxAlign().value();
862
863 RoundedSize += Alignment;
864 if (LiveRegs.empty()) {
865 LiveRegs.init(TRI);
866 LiveRegs.addLiveIns(MBB);
867 LiveRegs.addReg(FuncInfo->SGPRForFPSaveRestoreCopy);
868 LiveRegs.addReg(FuncInfo->SGPRForBPSaveRestoreCopy);
869 }
870
871 Register ScratchSPReg = findScratchNonCalleeSaveRegister(
872 MRI, LiveRegs, AMDGPU::SReg_32_XM0RegClass);
873 assert(ScratchSPReg && ScratchSPReg != FuncInfo->SGPRForFPSaveRestoreCopy &&
874 ScratchSPReg != FuncInfo->SGPRForBPSaveRestoreCopy);
875
876 // s_add_u32 tmp_reg, s32, NumBytes
877 // s_and_b32 s32, tmp_reg, 0b111...0000
878 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_U32), ScratchSPReg)
879 .addReg(StackPtrReg)
880 .addImm((Alignment - 1) * ST.getWavefrontSize())
881 .setMIFlag(MachineInstr::FrameSetup);
882 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_AND_B32), FramePtrReg)
883 .addReg(ScratchSPReg, RegState::Kill)
884 .addImm(-Alignment * ST.getWavefrontSize())
885 .setMIFlag(MachineInstr::FrameSetup);
886 FuncInfo->setIsStackRealigned(true);
887 } else if ((HasFP = hasFP(MF))) {
888 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrReg)
889 .addReg(StackPtrReg)
890 .setMIFlag(MachineInstr::FrameSetup);
891 }
892
893 // If we need a base pointer, set it up here. It's whatever the value of
894 // the stack pointer is at this point. Any variable size objects will be
895 // allocated after this, so we can still use the base pointer to reference
896 // the incoming arguments.
897 if ((HasBP = TRI.hasBasePointer(MF))) {
898 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), BasePtrReg)
899 .addReg(StackPtrReg)
900 .setMIFlag(MachineInstr::FrameSetup);
901 }
902
903 if (HasFP && RoundedSize != 0) {
904 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_U32), StackPtrReg)
905 .addReg(StackPtrReg)
906 .addImm(RoundedSize * ST.getWavefrontSize())
907 .setMIFlag(MachineInstr::FrameSetup);
908 }
909
910 assert((!HasFP || (FuncInfo->SGPRForFPSaveRestoreCopy ||
911 FuncInfo->FramePointerSaveIndex)) &&
912 "Needed to save FP but didn't save it anywhere");
913
914 assert((HasFP || (!FuncInfo->SGPRForFPSaveRestoreCopy &&
915 !FuncInfo->FramePointerSaveIndex)) &&
916 "Saved FP but didn't need it");
917
918 assert((!HasBP || (FuncInfo->SGPRForBPSaveRestoreCopy ||
919 FuncInfo->BasePointerSaveIndex)) &&
920 "Needed to save BP but didn't save it anywhere");
921
922 assert((HasBP || (!FuncInfo->SGPRForBPSaveRestoreCopy &&
923 !FuncInfo->BasePointerSaveIndex)) &&
924 "Saved BP but didn't need it");
925 }
926
emitEpilogue(MachineFunction & MF,MachineBasicBlock & MBB) const927 void SIFrameLowering::emitEpilogue(MachineFunction &MF,
928 MachineBasicBlock &MBB) const {
929 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
930 if (FuncInfo->isEntryFunction())
931 return;
932
933 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
934 const SIInstrInfo *TII = ST.getInstrInfo();
935 MachineRegisterInfo &MRI = MF.getRegInfo();
936 const SIRegisterInfo &TRI = TII->getRegisterInfo();
937 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
938 LivePhysRegs LiveRegs;
939 DebugLoc DL;
940
941 const MachineFrameInfo &MFI = MF.getFrameInfo();
942 uint32_t NumBytes = MFI.getStackSize();
943 uint32_t RoundedSize = FuncInfo->isStackRealigned()
944 ? NumBytes + MFI.getMaxAlign().value()
945 : NumBytes;
946 const Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
947 const Register FramePtrReg = FuncInfo->getFrameOffsetReg();
948 const Register BasePtrReg =
949 TRI.hasBasePointer(MF) ? TRI.getBaseRegister() : Register();
950
951 bool HasFPSaveIndex = FuncInfo->FramePointerSaveIndex.hasValue();
952 bool SpillFPToMemory = false;
953 if (HasFPSaveIndex) {
954 SpillFPToMemory = MFI.getStackID(*FuncInfo->FramePointerSaveIndex) !=
955 TargetStackID::SGPRSpill;
956 }
957
958 bool HasBPSaveIndex = FuncInfo->BasePointerSaveIndex.hasValue();
959 bool SpillBPToMemory = false;
960 if (HasBPSaveIndex) {
961 SpillBPToMemory = MFI.getStackID(*FuncInfo->BasePointerSaveIndex) !=
962 TargetStackID::SGPRSpill;
963 }
964
965 if (RoundedSize != 0 && hasFP(MF)) {
966 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_SUB_U32), StackPtrReg)
967 .addReg(StackPtrReg)
968 .addImm(RoundedSize * ST.getWavefrontSize())
969 .setMIFlag(MachineInstr::FrameDestroy);
970 }
971
972 if (FuncInfo->SGPRForFPSaveRestoreCopy) {
973 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrReg)
974 .addReg(FuncInfo->SGPRForFPSaveRestoreCopy)
975 .setMIFlag(MachineInstr::FrameSetup);
976 }
977
978 if (FuncInfo->SGPRForBPSaveRestoreCopy) {
979 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), BasePtrReg)
980 .addReg(FuncInfo->SGPRForBPSaveRestoreCopy)
981 .setMIFlag(MachineInstr::FrameSetup);
982 }
983
984 Register ScratchExecCopy;
985 if (HasFPSaveIndex) {
986 const int FI = FuncInfo->FramePointerSaveIndex.getValue();
987 assert(!MFI.isDeadObjectIndex(FI));
988 if (SpillFPToMemory) {
989 if (!ScratchExecCopy)
990 ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false);
991
992 MCPhysReg TempVGPR = findScratchNonCalleeSaveRegister(
993 MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
994 buildEpilogReload(LiveRegs, MBB, MBBI, TII, TempVGPR,
995 FuncInfo->getScratchRSrcReg(), StackPtrReg, FI);
996 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), FramePtrReg)
997 .addReg(TempVGPR, RegState::Kill);
998 } else {
999 // Reload from VGPR spill.
1000 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
1001 ArrayRef<SIMachineFunctionInfo::SpilledReg> Spill =
1002 FuncInfo->getSGPRToVGPRSpills(FI);
1003 assert(Spill.size() == 1);
1004 BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32),
1005 FramePtrReg)
1006 .addReg(Spill[0].VGPR)
1007 .addImm(Spill[0].Lane);
1008 }
1009 }
1010
1011 if (HasBPSaveIndex) {
1012 const int BasePtrFI = *FuncInfo->BasePointerSaveIndex;
1013 assert(!MFI.isDeadObjectIndex(BasePtrFI));
1014 if (SpillBPToMemory) {
1015 if (!ScratchExecCopy)
1016 ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false);
1017
1018 MCPhysReg TempVGPR = findScratchNonCalleeSaveRegister(
1019 MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
1020 buildEpilogReload(LiveRegs, MBB, MBBI, TII, TempVGPR,
1021 FuncInfo->getScratchRSrcReg(), StackPtrReg, BasePtrFI);
1022 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), BasePtrReg)
1023 .addReg(TempVGPR, RegState::Kill);
1024 } else {
1025 // Reload from VGPR spill.
1026 assert(MFI.getStackID(BasePtrFI) == TargetStackID::SGPRSpill);
1027 ArrayRef<SIMachineFunctionInfo::SpilledReg> Spill =
1028 FuncInfo->getSGPRToVGPRSpills(BasePtrFI);
1029 assert(Spill.size() == 1);
1030 BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32),
1031 BasePtrReg)
1032 .addReg(Spill[0].VGPR)
1033 .addImm(Spill[0].Lane);
1034 }
1035 }
1036
1037 for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg :
1038 FuncInfo->getSGPRSpillVGPRs()) {
1039 if (!Reg.FI.hasValue())
1040 continue;
1041
1042 if (!ScratchExecCopy)
1043 ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false);
1044
1045 buildEpilogReload(LiveRegs, MBB, MBBI, TII, Reg.VGPR,
1046 FuncInfo->getScratchRSrcReg(), StackPtrReg,
1047 Reg.FI.getValue());
1048 }
1049
1050 if (ScratchExecCopy) {
1051 // FIXME: Split block and make terminator.
1052 unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
1053 MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1054 BuildMI(MBB, MBBI, DL, TII->get(ExecMov), Exec)
1055 .addReg(ScratchExecCopy, RegState::Kill);
1056 }
1057 }
1058
1059 // Note SGPRSpill stack IDs should only be used for SGPR spilling to VGPRs, not
1060 // memory. They should have been removed by now.
allStackObjectsAreDead(const MachineFrameInfo & MFI)1061 static bool allStackObjectsAreDead(const MachineFrameInfo &MFI) {
1062 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
1063 I != E; ++I) {
1064 if (!MFI.isDeadObjectIndex(I))
1065 return false;
1066 }
1067
1068 return true;
1069 }
1070
1071 #ifndef NDEBUG
allSGPRSpillsAreDead(const MachineFrameInfo & MFI,Optional<int> FramePointerSaveIndex,Optional<int> BasePointerSaveIndex)1072 static bool allSGPRSpillsAreDead(const MachineFrameInfo &MFI,
1073 Optional<int> FramePointerSaveIndex,
1074 Optional<int> BasePointerSaveIndex) {
1075 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
1076 I != E; ++I) {
1077 if (!MFI.isDeadObjectIndex(I) &&
1078 MFI.getStackID(I) == TargetStackID::SGPRSpill &&
1079 ((FramePointerSaveIndex && I != FramePointerSaveIndex) ||
1080 (BasePointerSaveIndex && I != BasePointerSaveIndex))) {
1081 return false;
1082 }
1083 }
1084
1085 return true;
1086 }
1087 #endif
1088
getFrameIndexReference(const MachineFunction & MF,int FI,Register & FrameReg) const1089 int SIFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
1090 Register &FrameReg) const {
1091 const SIRegisterInfo *RI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
1092
1093 FrameReg = RI->getFrameRegister(MF);
1094 return MF.getFrameInfo().getObjectOffset(FI);
1095 }
1096
processFunctionBeforeFrameFinalized(MachineFunction & MF,RegScavenger * RS) const1097 void SIFrameLowering::processFunctionBeforeFrameFinalized(
1098 MachineFunction &MF,
1099 RegScavenger *RS) const {
1100 MachineFrameInfo &MFI = MF.getFrameInfo();
1101
1102 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1103 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1104 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1105
1106 FuncInfo->removeDeadFrameIndices(MFI);
1107 assert(allSGPRSpillsAreDead(MFI, None, None) &&
1108 "SGPR spill should have been removed in SILowerSGPRSpills");
1109
1110 // FIXME: The other checks should be redundant with allStackObjectsAreDead,
1111 // but currently hasNonSpillStackObjects is set only from source
1112 // allocas. Stack temps produced from legalization are not counted currently.
1113 if (!allStackObjectsAreDead(MFI)) {
1114 assert(RS && "RegScavenger required if spilling");
1115
1116 if (FuncInfo->isEntryFunction()) {
1117 int ScavengeFI = MFI.CreateFixedObject(
1118 TRI->getSpillSize(AMDGPU::SGPR_32RegClass), 0, false);
1119 RS->addScavengingFrameIndex(ScavengeFI);
1120 } else {
1121 int ScavengeFI = MFI.CreateStackObject(
1122 TRI->getSpillSize(AMDGPU::SGPR_32RegClass),
1123 TRI->getSpillAlign(AMDGPU::SGPR_32RegClass), false);
1124 RS->addScavengingFrameIndex(ScavengeFI);
1125 }
1126 }
1127 }
1128
1129 // Only report VGPRs to generic code.
determineCalleeSaves(MachineFunction & MF,BitVector & SavedVGPRs,RegScavenger * RS) const1130 void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
1131 BitVector &SavedVGPRs,
1132 RegScavenger *RS) const {
1133 TargetFrameLowering::determineCalleeSaves(MF, SavedVGPRs, RS);
1134 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1135 if (MFI->isEntryFunction())
1136 return;
1137
1138 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
1139 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1140 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1141
1142 // Ignore the SGPRs the default implementation found.
1143 SavedVGPRs.clearBitsNotInMask(TRI->getAllVGPRRegMask());
1144
1145 // hasFP only knows about stack objects that already exist. We're now
1146 // determining the stack slots that will be created, so we have to predict
1147 // them. Stack objects force FP usage with calls.
1148 //
1149 // Note a new VGPR CSR may be introduced if one is used for the spill, but we
1150 // don't want to report it here.
1151 //
1152 // FIXME: Is this really hasReservedCallFrame?
1153 const bool WillHaveFP =
1154 FrameInfo.hasCalls() &&
1155 (SavedVGPRs.any() || !allStackObjectsAreDead(FrameInfo));
1156
1157 // VGPRs used for SGPR spilling need to be specially inserted in the prolog,
1158 // so don't allow the default insertion to handle them.
1159 for (auto SSpill : MFI->getSGPRSpillVGPRs())
1160 SavedVGPRs.reset(SSpill.VGPR);
1161
1162 LivePhysRegs LiveRegs;
1163 LiveRegs.init(*TRI);
1164
1165 if (WillHaveFP || hasFP(MF)) {
1166 getVGPRSpillLaneOrTempRegister(MF, LiveRegs, MFI->SGPRForFPSaveRestoreCopy,
1167 MFI->FramePointerSaveIndex, true);
1168 }
1169
1170 if (TRI->hasBasePointer(MF)) {
1171 if (MFI->SGPRForFPSaveRestoreCopy)
1172 LiveRegs.addReg(MFI->SGPRForFPSaveRestoreCopy);
1173 getVGPRSpillLaneOrTempRegister(MF, LiveRegs, MFI->SGPRForBPSaveRestoreCopy,
1174 MFI->BasePointerSaveIndex, false);
1175 }
1176 }
1177
determineCalleeSavesSGPR(MachineFunction & MF,BitVector & SavedRegs,RegScavenger * RS) const1178 void SIFrameLowering::determineCalleeSavesSGPR(MachineFunction &MF,
1179 BitVector &SavedRegs,
1180 RegScavenger *RS) const {
1181 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1182 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1183 if (MFI->isEntryFunction())
1184 return;
1185
1186 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1187 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1188
1189 // The SP is specifically managed and we don't want extra spills of it.
1190 SavedRegs.reset(MFI->getStackPtrOffsetReg());
1191 SavedRegs.clearBitsInMask(TRI->getAllVGPRRegMask());
1192 }
1193
assignCalleeSavedSpillSlots(MachineFunction & MF,const TargetRegisterInfo * TRI,std::vector<CalleeSavedInfo> & CSI) const1194 bool SIFrameLowering::assignCalleeSavedSpillSlots(
1195 MachineFunction &MF, const TargetRegisterInfo *TRI,
1196 std::vector<CalleeSavedInfo> &CSI) const {
1197 if (CSI.empty())
1198 return true; // Early exit if no callee saved registers are modified!
1199
1200 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1201 if (!FuncInfo->SGPRForFPSaveRestoreCopy &&
1202 !FuncInfo->SGPRForBPSaveRestoreCopy)
1203 return false;
1204
1205 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1206 const SIRegisterInfo *RI = ST.getRegisterInfo();
1207 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1208 Register BasePtrReg = RI->getBaseRegister();
1209 unsigned NumModifiedRegs = 0;
1210
1211 if (FuncInfo->SGPRForFPSaveRestoreCopy)
1212 NumModifiedRegs++;
1213 if (FuncInfo->SGPRForBPSaveRestoreCopy)
1214 NumModifiedRegs++;
1215
1216 for (auto &CS : CSI) {
1217 if (CS.getReg() == FramePtrReg && FuncInfo->SGPRForFPSaveRestoreCopy) {
1218 CS.setDstReg(FuncInfo->SGPRForFPSaveRestoreCopy);
1219 if (--NumModifiedRegs)
1220 break;
1221 } else if (CS.getReg() == BasePtrReg &&
1222 FuncInfo->SGPRForBPSaveRestoreCopy) {
1223 CS.setDstReg(FuncInfo->SGPRForBPSaveRestoreCopy);
1224 if (--NumModifiedRegs)
1225 break;
1226 }
1227 }
1228
1229 return false;
1230 }
1231
eliminateCallFramePseudoInstr(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator I) const1232 MachineBasicBlock::iterator SIFrameLowering::eliminateCallFramePseudoInstr(
1233 MachineFunction &MF,
1234 MachineBasicBlock &MBB,
1235 MachineBasicBlock::iterator I) const {
1236 int64_t Amount = I->getOperand(0).getImm();
1237 if (Amount == 0)
1238 return MBB.erase(I);
1239
1240 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1241 const SIInstrInfo *TII = ST.getInstrInfo();
1242 const DebugLoc &DL = I->getDebugLoc();
1243 unsigned Opc = I->getOpcode();
1244 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
1245 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
1246
1247 if (!hasReservedCallFrame(MF)) {
1248 Amount = alignTo(Amount, getStackAlign());
1249 assert(isUInt<32>(Amount) && "exceeded stack address space size");
1250 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1251 Register SPReg = MFI->getStackPtrOffsetReg();
1252
1253 unsigned Op = IsDestroy ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
1254 BuildMI(MBB, I, DL, TII->get(Op), SPReg)
1255 .addReg(SPReg)
1256 .addImm(Amount * ST.getWavefrontSize());
1257 } else if (CalleePopAmount != 0) {
1258 llvm_unreachable("is this used?");
1259 }
1260
1261 return MBB.erase(I);
1262 }
1263
hasFP(const MachineFunction & MF) const1264 bool SIFrameLowering::hasFP(const MachineFunction &MF) const {
1265 const MachineFrameInfo &MFI = MF.getFrameInfo();
1266
1267 // For entry functions we can use an immediate offset in most cases, so the
1268 // presence of calls doesn't imply we need a distinct frame pointer.
1269 if (MFI.hasCalls() &&
1270 !MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
1271 // All offsets are unsigned, so need to be addressed in the same direction
1272 // as stack growth.
1273
1274 // FIXME: This function is pretty broken, since it can be called before the
1275 // frame layout is determined or CSR spills are inserted.
1276 return MFI.getStackSize() != 0;
1277 }
1278
1279 return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
1280 MFI.hasStackMap() || MFI.hasPatchPoint() ||
1281 MF.getSubtarget<GCNSubtarget>().getRegisterInfo()->needsStackRealignment(MF) ||
1282 MF.getTarget().Options.DisableFramePointerElim(MF);
1283 }
1284