1 //===-- SystemZFrameLowering.cpp - Frame lowering for SystemZ -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "SystemZFrameLowering.h"
10 #include "SystemZCallingConv.h"
11 #include "SystemZInstrBuilder.h"
12 #include "SystemZInstrInfo.h"
13 #include "SystemZMachineFunctionInfo.h"
14 #include "SystemZRegisterInfo.h"
15 #include "SystemZSubtarget.h"
16 #include "llvm/CodeGen/MachineModuleInfo.h"
17 #include "llvm/CodeGen/MachineRegisterInfo.h"
18 #include "llvm/CodeGen/RegisterScavenging.h"
19 #include "llvm/IR/Function.h"
20 #include "llvm/Target/TargetMachine.h"
21
22 using namespace llvm;
23
24 namespace {
25 // The ABI-defined register save slots, relative to the CFA (i.e.
26 // incoming stack pointer + SystemZMC::CallFrameSize).
27 static const TargetFrameLowering::SpillSlot SpillOffsetTable[] = {
28 { SystemZ::R2D, 0x10 },
29 { SystemZ::R3D, 0x18 },
30 { SystemZ::R4D, 0x20 },
31 { SystemZ::R5D, 0x28 },
32 { SystemZ::R6D, 0x30 },
33 { SystemZ::R7D, 0x38 },
34 { SystemZ::R8D, 0x40 },
35 { SystemZ::R9D, 0x48 },
36 { SystemZ::R10D, 0x50 },
37 { SystemZ::R11D, 0x58 },
38 { SystemZ::R12D, 0x60 },
39 { SystemZ::R13D, 0x68 },
40 { SystemZ::R14D, 0x70 },
41 { SystemZ::R15D, 0x78 },
42 { SystemZ::F0D, 0x80 },
43 { SystemZ::F2D, 0x88 },
44 { SystemZ::F4D, 0x90 },
45 { SystemZ::F6D, 0x98 }
46 };
47 } // end anonymous namespace
48
SystemZFrameLowering()49 SystemZFrameLowering::SystemZFrameLowering()
50 : TargetFrameLowering(TargetFrameLowering::StackGrowsDown, Align(8),
51 0, Align(8), false /* StackRealignable */),
52 RegSpillOffsets(0) {
53 // Due to the SystemZ ABI, the DWARF CFA (Canonical Frame Address) is not
54 // equal to the incoming stack pointer, but to incoming stack pointer plus
55 // 160. Instead of using a Local Area Offset, the Register save area will
56 // be occupied by fixed frame objects, and all offsets are actually
57 // relative to CFA.
58
59 // Create a mapping from register number to save slot offset.
60 // These offsets are relative to the start of the register save area.
61 RegSpillOffsets.grow(SystemZ::NUM_TARGET_REGS);
62 for (unsigned I = 0, E = array_lengthof(SpillOffsetTable); I != E; ++I)
63 RegSpillOffsets[SpillOffsetTable[I].Reg] = SpillOffsetTable[I].Offset;
64 }
65
66 bool SystemZFrameLowering::
assignCalleeSavedSpillSlots(MachineFunction & MF,const TargetRegisterInfo * TRI,std::vector<CalleeSavedInfo> & CSI) const67 assignCalleeSavedSpillSlots(MachineFunction &MF,
68 const TargetRegisterInfo *TRI,
69 std::vector<CalleeSavedInfo> &CSI) const {
70 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
71 MachineFrameInfo &MFFrame = MF.getFrameInfo();
72 bool IsVarArg = MF.getFunction().isVarArg();
73 if (CSI.empty())
74 return true; // Early exit if no callee saved registers are modified!
75
76 unsigned LowGPR = 0;
77 unsigned HighGPR = SystemZ::R15D;
78 int StartSPOffset = SystemZMC::CallFrameSize;
79 for (auto &CS : CSI) {
80 unsigned Reg = CS.getReg();
81 int Offset = getRegSpillOffset(MF, Reg);
82 if (Offset) {
83 if (SystemZ::GR64BitRegClass.contains(Reg) && StartSPOffset > Offset) {
84 LowGPR = Reg;
85 StartSPOffset = Offset;
86 }
87 Offset -= SystemZMC::CallFrameSize;
88 int FrameIdx = MFFrame.CreateFixedSpillStackObject(8, Offset);
89 CS.setFrameIdx(FrameIdx);
90 } else
91 CS.setFrameIdx(INT32_MAX);
92 }
93
94 // Save the range of call-saved registers, for use by the
95 // prologue/epilogue inserters.
96 ZFI->setRestoreGPRRegs(LowGPR, HighGPR, StartSPOffset);
97 if (IsVarArg) {
98 // Also save the GPR varargs, if any. R6D is call-saved, so would
99 // already be included, but we also need to handle the call-clobbered
100 // argument registers.
101 unsigned FirstGPR = ZFI->getVarArgsFirstGPR();
102 if (FirstGPR < SystemZ::NumArgGPRs) {
103 unsigned Reg = SystemZ::ArgGPRs[FirstGPR];
104 int Offset = getRegSpillOffset(MF, Reg);
105 if (StartSPOffset > Offset) {
106 LowGPR = Reg; StartSPOffset = Offset;
107 }
108 }
109 }
110 ZFI->setSpillGPRRegs(LowGPR, HighGPR, StartSPOffset);
111
112 // Create fixed stack objects for the remaining registers.
113 int CurrOffset = -SystemZMC::CallFrameSize;
114 if (usePackedStack(MF))
115 CurrOffset += StartSPOffset;
116
117 for (auto &CS : CSI) {
118 if (CS.getFrameIdx() != INT32_MAX)
119 continue;
120 unsigned Reg = CS.getReg();
121 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
122 unsigned Size = TRI->getSpillSize(*RC);
123 CurrOffset -= Size;
124 assert(CurrOffset % 8 == 0 &&
125 "8-byte alignment required for for all register save slots");
126 int FrameIdx = MFFrame.CreateFixedSpillStackObject(Size, CurrOffset);
127 CS.setFrameIdx(FrameIdx);
128 }
129
130 return true;
131 }
132
determineCalleeSaves(MachineFunction & MF,BitVector & SavedRegs,RegScavenger * RS) const133 void SystemZFrameLowering::determineCalleeSaves(MachineFunction &MF,
134 BitVector &SavedRegs,
135 RegScavenger *RS) const {
136 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
137
138 MachineFrameInfo &MFFrame = MF.getFrameInfo();
139 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
140 bool HasFP = hasFP(MF);
141 SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
142 bool IsVarArg = MF.getFunction().isVarArg();
143
144 // va_start stores incoming FPR varargs in the normal way, but delegates
145 // the saving of incoming GPR varargs to spillCalleeSavedRegisters().
146 // Record these pending uses, which typically include the call-saved
147 // argument register R6D.
148 if (IsVarArg)
149 for (unsigned I = MFI->getVarArgsFirstGPR(); I < SystemZ::NumArgGPRs; ++I)
150 SavedRegs.set(SystemZ::ArgGPRs[I]);
151
152 // If there are any landing pads, entering them will modify r6/r7.
153 if (!MF.getLandingPads().empty()) {
154 SavedRegs.set(SystemZ::R6D);
155 SavedRegs.set(SystemZ::R7D);
156 }
157
158 // If the function requires a frame pointer, record that the hard
159 // frame pointer will be clobbered.
160 if (HasFP)
161 SavedRegs.set(SystemZ::R11D);
162
163 // If the function calls other functions, record that the return
164 // address register will be clobbered.
165 if (MFFrame.hasCalls())
166 SavedRegs.set(SystemZ::R14D);
167
168 // If we are saving GPRs other than the stack pointer, we might as well
169 // save and restore the stack pointer at the same time, via STMG and LMG.
170 // This allows the deallocation to be done by the LMG, rather than needing
171 // a separate %r15 addition.
172 const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
173 for (unsigned I = 0; CSRegs[I]; ++I) {
174 unsigned Reg = CSRegs[I];
175 if (SystemZ::GR64BitRegClass.contains(Reg) && SavedRegs.test(Reg)) {
176 SavedRegs.set(SystemZ::R15D);
177 break;
178 }
179 }
180 }
181
182 // Add GPR64 to the save instruction being built by MIB, which is in basic
183 // block MBB. IsImplicit says whether this is an explicit operand to the
184 // instruction, or an implicit one that comes between the explicit start
185 // and end registers.
addSavedGPR(MachineBasicBlock & MBB,MachineInstrBuilder & MIB,unsigned GPR64,bool IsImplicit)186 static void addSavedGPR(MachineBasicBlock &MBB, MachineInstrBuilder &MIB,
187 unsigned GPR64, bool IsImplicit) {
188 const TargetRegisterInfo *RI =
189 MBB.getParent()->getSubtarget().getRegisterInfo();
190 Register GPR32 = RI->getSubReg(GPR64, SystemZ::subreg_l32);
191 bool IsLive = MBB.isLiveIn(GPR64) || MBB.isLiveIn(GPR32);
192 if (!IsLive || !IsImplicit) {
193 MIB.addReg(GPR64, getImplRegState(IsImplicit) | getKillRegState(!IsLive));
194 if (!IsLive)
195 MBB.addLiveIn(GPR64);
196 }
197 }
198
spillCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,ArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const199 bool SystemZFrameLowering::spillCalleeSavedRegisters(
200 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
201 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
202 if (CSI.empty())
203 return false;
204
205 MachineFunction &MF = *MBB.getParent();
206 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
207 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
208 bool IsVarArg = MF.getFunction().isVarArg();
209 DebugLoc DL;
210
211 // Save GPRs
212 SystemZ::GPRRegs SpillGPRs = ZFI->getSpillGPRRegs();
213 if (SpillGPRs.LowGPR) {
214 assert(SpillGPRs.LowGPR != SpillGPRs.HighGPR &&
215 "Should be saving %r15 and something else");
216
217 // Build an STMG instruction.
218 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(SystemZ::STMG));
219
220 // Add the explicit register operands.
221 addSavedGPR(MBB, MIB, SpillGPRs.LowGPR, false);
222 addSavedGPR(MBB, MIB, SpillGPRs.HighGPR, false);
223
224 // Add the address.
225 MIB.addReg(SystemZ::R15D).addImm(SpillGPRs.GPROffset);
226
227 // Make sure all call-saved GPRs are included as operands and are
228 // marked as live on entry.
229 for (unsigned I = 0, E = CSI.size(); I != E; ++I) {
230 unsigned Reg = CSI[I].getReg();
231 if (SystemZ::GR64BitRegClass.contains(Reg))
232 addSavedGPR(MBB, MIB, Reg, true);
233 }
234
235 // ...likewise GPR varargs.
236 if (IsVarArg)
237 for (unsigned I = ZFI->getVarArgsFirstGPR(); I < SystemZ::NumArgGPRs; ++I)
238 addSavedGPR(MBB, MIB, SystemZ::ArgGPRs[I], true);
239 }
240
241 // Save FPRs/VRs in the normal TargetInstrInfo way.
242 for (unsigned I = 0, E = CSI.size(); I != E; ++I) {
243 unsigned Reg = CSI[I].getReg();
244 if (SystemZ::FP64BitRegClass.contains(Reg)) {
245 MBB.addLiveIn(Reg);
246 TII->storeRegToStackSlot(MBB, MBBI, Reg, true, CSI[I].getFrameIdx(),
247 &SystemZ::FP64BitRegClass, TRI);
248 }
249 if (SystemZ::VR128BitRegClass.contains(Reg)) {
250 MBB.addLiveIn(Reg);
251 TII->storeRegToStackSlot(MBB, MBBI, Reg, true, CSI[I].getFrameIdx(),
252 &SystemZ::VR128BitRegClass, TRI);
253 }
254 }
255
256 return true;
257 }
258
restoreCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MutableArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const259 bool SystemZFrameLowering::restoreCalleeSavedRegisters(
260 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
261 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
262 if (CSI.empty())
263 return false;
264
265 MachineFunction &MF = *MBB.getParent();
266 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
267 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
268 bool HasFP = hasFP(MF);
269 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
270
271 // Restore FPRs/VRs in the normal TargetInstrInfo way.
272 for (unsigned I = 0, E = CSI.size(); I != E; ++I) {
273 unsigned Reg = CSI[I].getReg();
274 if (SystemZ::FP64BitRegClass.contains(Reg))
275 TII->loadRegFromStackSlot(MBB, MBBI, Reg, CSI[I].getFrameIdx(),
276 &SystemZ::FP64BitRegClass, TRI);
277 if (SystemZ::VR128BitRegClass.contains(Reg))
278 TII->loadRegFromStackSlot(MBB, MBBI, Reg, CSI[I].getFrameIdx(),
279 &SystemZ::VR128BitRegClass, TRI);
280 }
281
282 // Restore call-saved GPRs (but not call-clobbered varargs, which at
283 // this point might hold return values).
284 SystemZ::GPRRegs RestoreGPRs = ZFI->getRestoreGPRRegs();
285 if (RestoreGPRs.LowGPR) {
286 // If we saved any of %r2-%r5 as varargs, we should also be saving
287 // and restoring %r6. If we're saving %r6 or above, we should be
288 // restoring it too.
289 assert(RestoreGPRs.LowGPR != RestoreGPRs.HighGPR &&
290 "Should be loading %r15 and something else");
291
292 // Build an LMG instruction.
293 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(SystemZ::LMG));
294
295 // Add the explicit register operands.
296 MIB.addReg(RestoreGPRs.LowGPR, RegState::Define);
297 MIB.addReg(RestoreGPRs.HighGPR, RegState::Define);
298
299 // Add the address.
300 MIB.addReg(HasFP ? SystemZ::R11D : SystemZ::R15D);
301 MIB.addImm(RestoreGPRs.GPROffset);
302
303 // Do a second scan adding regs as being defined by instruction
304 for (unsigned I = 0, E = CSI.size(); I != E; ++I) {
305 unsigned Reg = CSI[I].getReg();
306 if (Reg != RestoreGPRs.LowGPR && Reg != RestoreGPRs.HighGPR &&
307 SystemZ::GR64BitRegClass.contains(Reg))
308 MIB.addReg(Reg, RegState::ImplicitDefine);
309 }
310 }
311
312 return true;
313 }
314
315 void SystemZFrameLowering::
processFunctionBeforeFrameFinalized(MachineFunction & MF,RegScavenger * RS) const316 processFunctionBeforeFrameFinalized(MachineFunction &MF,
317 RegScavenger *RS) const {
318 MachineFrameInfo &MFFrame = MF.getFrameInfo();
319 bool BackChain = MF.getFunction().hasFnAttribute("backchain");
320
321 if (!usePackedStack(MF) || BackChain)
322 // Create the incoming register save area.
323 getOrCreateFramePointerSaveIndex(MF);
324
325 // Get the size of our stack frame to be allocated ...
326 uint64_t StackSize = (MFFrame.estimateStackSize(MF) +
327 SystemZMC::CallFrameSize);
328 // ... and the maximum offset we may need to reach into the
329 // caller's frame to access the save area or stack arguments.
330 int64_t MaxArgOffset = 0;
331 for (int I = MFFrame.getObjectIndexBegin(); I != 0; ++I)
332 if (MFFrame.getObjectOffset(I) >= 0) {
333 int64_t ArgOffset = MFFrame.getObjectOffset(I) +
334 MFFrame.getObjectSize(I);
335 MaxArgOffset = std::max(MaxArgOffset, ArgOffset);
336 }
337
338 uint64_t MaxReach = StackSize + MaxArgOffset;
339 if (!isUInt<12>(MaxReach)) {
340 // We may need register scavenging slots if some parts of the frame
341 // are outside the reach of an unsigned 12-bit displacement.
342 // Create 2 for the case where both addresses in an MVC are
343 // out of range.
344 RS->addScavengingFrameIndex(MFFrame.CreateStackObject(8, Align(8), false));
345 RS->addScavengingFrameIndex(MFFrame.CreateStackObject(8, Align(8), false));
346 }
347 }
348
349 // Emit instructions before MBBI (in MBB) to add NumBytes to Reg.
emitIncrement(MachineBasicBlock & MBB,MachineBasicBlock::iterator & MBBI,const DebugLoc & DL,Register Reg,int64_t NumBytes,const TargetInstrInfo * TII)350 static void emitIncrement(MachineBasicBlock &MBB,
351 MachineBasicBlock::iterator &MBBI, const DebugLoc &DL,
352 Register Reg, int64_t NumBytes,
353 const TargetInstrInfo *TII) {
354 while (NumBytes) {
355 unsigned Opcode;
356 int64_t ThisVal = NumBytes;
357 if (isInt<16>(NumBytes))
358 Opcode = SystemZ::AGHI;
359 else {
360 Opcode = SystemZ::AGFI;
361 // Make sure we maintain 8-byte stack alignment.
362 int64_t MinVal = -uint64_t(1) << 31;
363 int64_t MaxVal = (int64_t(1) << 31) - 8;
364 if (ThisVal < MinVal)
365 ThisVal = MinVal;
366 else if (ThisVal > MaxVal)
367 ThisVal = MaxVal;
368 }
369 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII->get(Opcode), Reg)
370 .addReg(Reg).addImm(ThisVal);
371 // The CC implicit def is dead.
372 MI->getOperand(3).setIsDead();
373 NumBytes -= ThisVal;
374 }
375 }
376
377 // Add CFI for the new CFA offset.
buildCFAOffs(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,int Offset,const SystemZInstrInfo * ZII)378 static void buildCFAOffs(MachineBasicBlock &MBB,
379 MachineBasicBlock::iterator MBBI,
380 const DebugLoc &DL, int Offset,
381 const SystemZInstrInfo *ZII) {
382 unsigned CFIIndex = MBB.getParent()->addFrameInst(
383 MCCFIInstruction::cfiDefCfaOffset(nullptr, -Offset));
384 BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION))
385 .addCFIIndex(CFIIndex);
386 }
387
388 // Add CFI for the new frame location.
buildDefCFAReg(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,unsigned Reg,const SystemZInstrInfo * ZII)389 static void buildDefCFAReg(MachineBasicBlock &MBB,
390 MachineBasicBlock::iterator MBBI,
391 const DebugLoc &DL, unsigned Reg,
392 const SystemZInstrInfo *ZII) {
393 MachineFunction &MF = *MBB.getParent();
394 MachineModuleInfo &MMI = MF.getMMI();
395 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
396 unsigned RegNum = MRI->getDwarfRegNum(Reg, true);
397 unsigned CFIIndex = MF.addFrameInst(
398 MCCFIInstruction::createDefCfaRegister(nullptr, RegNum));
399 BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION))
400 .addCFIIndex(CFIIndex);
401 }
402
emitPrologue(MachineFunction & MF,MachineBasicBlock & MBB) const403 void SystemZFrameLowering::emitPrologue(MachineFunction &MF,
404 MachineBasicBlock &MBB) const {
405 assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
406 const SystemZSubtarget &STI = MF.getSubtarget<SystemZSubtarget>();
407 const SystemZTargetLowering &TLI = *STI.getTargetLowering();
408 MachineFrameInfo &MFFrame = MF.getFrameInfo();
409 auto *ZII = static_cast<const SystemZInstrInfo *>(STI.getInstrInfo());
410 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
411 MachineBasicBlock::iterator MBBI = MBB.begin();
412 MachineModuleInfo &MMI = MF.getMMI();
413 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
414 const std::vector<CalleeSavedInfo> &CSI = MFFrame.getCalleeSavedInfo();
415 bool HasFP = hasFP(MF);
416
417 // In GHC calling convention C stack space, including the ABI-defined
418 // 160-byte base area, is (de)allocated by GHC itself. This stack space may
419 // be used by LLVM as spill slots for the tail recursive GHC functions. Thus
420 // do not allocate stack space here, too.
421 if (MF.getFunction().getCallingConv() == CallingConv::GHC) {
422 if (MFFrame.getStackSize() > 2048 * sizeof(long)) {
423 report_fatal_error(
424 "Pre allocated stack space for GHC function is too small");
425 }
426 if (HasFP) {
427 report_fatal_error(
428 "In GHC calling convention a frame pointer is not supported");
429 }
430 MFFrame.setStackSize(MFFrame.getStackSize() + SystemZMC::CallFrameSize);
431 return;
432 }
433
434 // Debug location must be unknown since the first debug location is used
435 // to determine the end of the prologue.
436 DebugLoc DL;
437
438 // The current offset of the stack pointer from the CFA.
439 int64_t SPOffsetFromCFA = -SystemZMC::CFAOffsetFromInitialSP;
440
441 if (ZFI->getSpillGPRRegs().LowGPR) {
442 // Skip over the GPR saves.
443 if (MBBI != MBB.end() && MBBI->getOpcode() == SystemZ::STMG)
444 ++MBBI;
445 else
446 llvm_unreachable("Couldn't skip over GPR saves");
447
448 // Add CFI for the GPR saves.
449 for (auto &Save : CSI) {
450 unsigned Reg = Save.getReg();
451 if (SystemZ::GR64BitRegClass.contains(Reg)) {
452 int FI = Save.getFrameIdx();
453 int64_t Offset = MFFrame.getObjectOffset(FI);
454 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
455 nullptr, MRI->getDwarfRegNum(Reg, true), Offset));
456 BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION))
457 .addCFIIndex(CFIIndex);
458 }
459 }
460 }
461
462 uint64_t StackSize = MFFrame.getStackSize();
463 // We need to allocate the ABI-defined 160-byte base area whenever
464 // we allocate stack space for our own use and whenever we call another
465 // function.
466 bool HasStackObject = false;
467 for (unsigned i = 0, e = MFFrame.getObjectIndexEnd(); i != e; ++i)
468 if (!MFFrame.isDeadObjectIndex(i)) {
469 HasStackObject = true;
470 break;
471 }
472 if (HasStackObject || MFFrame.hasCalls())
473 StackSize += SystemZMC::CallFrameSize;
474 // Don't allocate the incoming reg save area.
475 StackSize = StackSize > SystemZMC::CallFrameSize
476 ? StackSize - SystemZMC::CallFrameSize
477 : 0;
478 MFFrame.setStackSize(StackSize);
479
480 if (StackSize) {
481 // Determine if we want to store a backchain.
482 bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain");
483
484 // If we need backchain, save current stack pointer. R1 is free at this
485 // point.
486 if (StoreBackchain)
487 BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR))
488 .addReg(SystemZ::R1D, RegState::Define).addReg(SystemZ::R15D);
489
490 // Allocate StackSize bytes.
491 int64_t Delta = -int64_t(StackSize);
492 const unsigned ProbeSize = TLI.getStackProbeSize(MF);
493 bool FreeProbe = (ZFI->getSpillGPRRegs().GPROffset &&
494 (ZFI->getSpillGPRRegs().GPROffset + StackSize) < ProbeSize);
495 if (!FreeProbe &&
496 MF.getSubtarget().getTargetLowering()->hasInlineStackProbe(MF)) {
497 // Stack probing may involve looping, but splitting the prologue block
498 // is not possible at this point since it would invalidate the
499 // SaveBlocks / RestoreBlocks sets of PEI in the single block function
500 // case. Build a pseudo to be handled later by inlineStackProbe().
501 BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::PROBED_STACKALLOC))
502 .addImm(StackSize);
503 }
504 else {
505 emitIncrement(MBB, MBBI, DL, SystemZ::R15D, Delta, ZII);
506 buildCFAOffs(MBB, MBBI, DL, SPOffsetFromCFA + Delta, ZII);
507 }
508 SPOffsetFromCFA += Delta;
509
510 if (StoreBackchain) {
511 // The back chain is stored topmost with packed-stack.
512 int Offset = usePackedStack(MF) ? SystemZMC::CallFrameSize - 8 : 0;
513 BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::STG))
514 .addReg(SystemZ::R1D, RegState::Kill).addReg(SystemZ::R15D)
515 .addImm(Offset).addReg(0);
516 }
517 }
518
519 if (HasFP) {
520 // Copy the base of the frame to R11.
521 BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR), SystemZ::R11D)
522 .addReg(SystemZ::R15D);
523
524 // Add CFI for the new frame location.
525 buildDefCFAReg(MBB, MBBI, DL, SystemZ::R11D, ZII);
526
527 // Mark the FramePtr as live at the beginning of every block except
528 // the entry block. (We'll have marked R11 as live on entry when
529 // saving the GPRs.)
530 for (auto I = std::next(MF.begin()), E = MF.end(); I != E; ++I)
531 I->addLiveIn(SystemZ::R11D);
532 }
533
534 // Skip over the FPR/VR saves.
535 SmallVector<unsigned, 8> CFIIndexes;
536 for (auto &Save : CSI) {
537 unsigned Reg = Save.getReg();
538 if (SystemZ::FP64BitRegClass.contains(Reg)) {
539 if (MBBI != MBB.end() &&
540 (MBBI->getOpcode() == SystemZ::STD ||
541 MBBI->getOpcode() == SystemZ::STDY))
542 ++MBBI;
543 else
544 llvm_unreachable("Couldn't skip over FPR save");
545 } else if (SystemZ::VR128BitRegClass.contains(Reg)) {
546 if (MBBI != MBB.end() &&
547 MBBI->getOpcode() == SystemZ::VST)
548 ++MBBI;
549 else
550 llvm_unreachable("Couldn't skip over VR save");
551 } else
552 continue;
553
554 // Add CFI for the this save.
555 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
556 Register IgnoredFrameReg;
557 int64_t Offset =
558 getFrameIndexReference(MF, Save.getFrameIdx(), IgnoredFrameReg);
559
560 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
561 nullptr, DwarfReg, SPOffsetFromCFA + Offset));
562 CFIIndexes.push_back(CFIIndex);
563 }
564 // Complete the CFI for the FPR/VR saves, modelling them as taking effect
565 // after the last save.
566 for (auto CFIIndex : CFIIndexes) {
567 BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION))
568 .addCFIIndex(CFIIndex);
569 }
570 }
571
emitEpilogue(MachineFunction & MF,MachineBasicBlock & MBB) const572 void SystemZFrameLowering::emitEpilogue(MachineFunction &MF,
573 MachineBasicBlock &MBB) const {
574 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
575 auto *ZII =
576 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
577 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
578 MachineFrameInfo &MFFrame = MF.getFrameInfo();
579
580 // See SystemZFrameLowering::emitPrologue
581 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
582 return;
583
584 // Skip the return instruction.
585 assert(MBBI->isReturn() && "Can only insert epilogue into returning blocks");
586
587 uint64_t StackSize = MFFrame.getStackSize();
588 if (ZFI->getRestoreGPRRegs().LowGPR) {
589 --MBBI;
590 unsigned Opcode = MBBI->getOpcode();
591 if (Opcode != SystemZ::LMG)
592 llvm_unreachable("Expected to see callee-save register restore code");
593
594 unsigned AddrOpNo = 2;
595 DebugLoc DL = MBBI->getDebugLoc();
596 uint64_t Offset = StackSize + MBBI->getOperand(AddrOpNo + 1).getImm();
597 unsigned NewOpcode = ZII->getOpcodeForOffset(Opcode, Offset);
598
599 // If the offset is too large, use the largest stack-aligned offset
600 // and add the rest to the base register (the stack or frame pointer).
601 if (!NewOpcode) {
602 uint64_t NumBytes = Offset - 0x7fff8;
603 emitIncrement(MBB, MBBI, DL, MBBI->getOperand(AddrOpNo).getReg(),
604 NumBytes, ZII);
605 Offset -= NumBytes;
606 NewOpcode = ZII->getOpcodeForOffset(Opcode, Offset);
607 assert(NewOpcode && "No restore instruction available");
608 }
609
610 MBBI->setDesc(ZII->get(NewOpcode));
611 MBBI->getOperand(AddrOpNo + 1).ChangeToImmediate(Offset);
612 } else if (StackSize) {
613 DebugLoc DL = MBBI->getDebugLoc();
614 emitIncrement(MBB, MBBI, DL, SystemZ::R15D, StackSize, ZII);
615 }
616 }
617
inlineStackProbe(MachineFunction & MF,MachineBasicBlock & PrologMBB) const618 void SystemZFrameLowering::inlineStackProbe(MachineFunction &MF,
619 MachineBasicBlock &PrologMBB) const {
620 auto *ZII =
621 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
622 const SystemZSubtarget &STI = MF.getSubtarget<SystemZSubtarget>();
623 const SystemZTargetLowering &TLI = *STI.getTargetLowering();
624
625 MachineInstr *StackAllocMI = nullptr;
626 for (MachineInstr &MI : PrologMBB)
627 if (MI.getOpcode() == SystemZ::PROBED_STACKALLOC) {
628 StackAllocMI = &MI;
629 break;
630 }
631 if (StackAllocMI == nullptr)
632 return;
633 uint64_t StackSize = StackAllocMI->getOperand(0).getImm();
634 const unsigned ProbeSize = TLI.getStackProbeSize(MF);
635 uint64_t NumFullBlocks = StackSize / ProbeSize;
636 uint64_t Residual = StackSize % ProbeSize;
637 int64_t SPOffsetFromCFA = -SystemZMC::CFAOffsetFromInitialSP;
638 MachineBasicBlock *MBB = &PrologMBB;
639 MachineBasicBlock::iterator MBBI = StackAllocMI;
640 const DebugLoc DL = StackAllocMI->getDebugLoc();
641
642 // Allocate a block of Size bytes on the stack and probe it.
643 auto allocateAndProbe = [&](MachineBasicBlock &InsMBB,
644 MachineBasicBlock::iterator InsPt, unsigned Size,
645 bool EmitCFI) -> void {
646 emitIncrement(InsMBB, InsPt, DL, SystemZ::R15D, -int64_t(Size), ZII);
647 if (EmitCFI) {
648 SPOffsetFromCFA -= Size;
649 buildCFAOffs(InsMBB, InsPt, DL, SPOffsetFromCFA, ZII);
650 }
651 // Probe by means of a volatile compare.
652 MachineMemOperand *MMO = MF.getMachineMemOperand(MachinePointerInfo(),
653 MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, 8, Align(1));
654 BuildMI(InsMBB, InsPt, DL, ZII->get(SystemZ::CG))
655 .addReg(SystemZ::R0D, RegState::Undef)
656 .addReg(SystemZ::R15D).addImm(Size - 8).addReg(0)
657 .addMemOperand(MMO);
658 };
659
660 if (NumFullBlocks < 3) {
661 // Emit unrolled probe statements.
662 for (unsigned int i = 0; i < NumFullBlocks; i++)
663 allocateAndProbe(*MBB, MBBI, ProbeSize, true/*EmitCFI*/);
664 } else {
665 // Emit a loop probing the pages.
666 uint64_t LoopAlloc = ProbeSize * NumFullBlocks;
667 SPOffsetFromCFA -= LoopAlloc;
668
669 BuildMI(*MBB, MBBI, DL, ZII->get(SystemZ::LGR), SystemZ::R1D)
670 .addReg(SystemZ::R15D);
671 buildDefCFAReg(*MBB, MBBI, DL, SystemZ::R1D, ZII);
672 emitIncrement(*MBB, MBBI, DL, SystemZ::R1D, -int64_t(LoopAlloc), ZII);
673 buildCFAOffs(*MBB, MBBI, DL, -int64_t(SystemZMC::CallFrameSize + LoopAlloc),
674 ZII);
675
676 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MBBI, MBB);
677 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB);
678 MBB->addSuccessor(LoopMBB);
679 LoopMBB->addSuccessor(LoopMBB);
680 LoopMBB->addSuccessor(DoneMBB);
681
682 MBB = LoopMBB;
683 allocateAndProbe(*MBB, MBB->end(), ProbeSize, false/*EmitCFI*/);
684 BuildMI(*MBB, MBB->end(), DL, ZII->get(SystemZ::CLGR))
685 .addReg(SystemZ::R15D).addReg(SystemZ::R1D);
686 BuildMI(*MBB, MBB->end(), DL, ZII->get(SystemZ::BRC))
687 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_GT).addMBB(MBB);
688
689 MBB = DoneMBB;
690 MBBI = DoneMBB->begin();
691 buildDefCFAReg(*MBB, MBBI, DL, SystemZ::R15D, ZII);
692
693 recomputeLiveIns(*DoneMBB);
694 recomputeLiveIns(*LoopMBB);
695 }
696
697 if (Residual)
698 allocateAndProbe(*MBB, MBBI, Residual, true/*EmitCFI*/);
699
700 StackAllocMI->eraseFromParent();
701 }
702
hasFP(const MachineFunction & MF) const703 bool SystemZFrameLowering::hasFP(const MachineFunction &MF) const {
704 return (MF.getTarget().Options.DisableFramePointerElim(MF) ||
705 MF.getFrameInfo().hasVarSizedObjects() ||
706 MF.getInfo<SystemZMachineFunctionInfo>()->getManipulatesSP());
707 }
708
709 bool
hasReservedCallFrame(const MachineFunction & MF) const710 SystemZFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
711 // The ABI requires us to allocate 160 bytes of stack space for the callee,
712 // with any outgoing stack arguments being placed above that. It seems
713 // better to make that area a permanent feature of the frame even if
714 // we're using a frame pointer.
715 return true;
716 }
717
getFrameIndexReference(const MachineFunction & MF,int FI,Register & FrameReg) const718 int SystemZFrameLowering::getFrameIndexReference(const MachineFunction &MF,
719 int FI,
720 Register &FrameReg) const {
721 // Our incoming SP is actually SystemZMC::CallFrameSize below the CFA, so
722 // add that difference here.
723 int64_t Offset =
724 TargetFrameLowering::getFrameIndexReference(MF, FI, FrameReg);
725 return Offset + SystemZMC::CallFrameSize;
726 }
727
728 MachineBasicBlock::iterator SystemZFrameLowering::
eliminateCallFramePseudoInstr(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MI) const729 eliminateCallFramePseudoInstr(MachineFunction &MF,
730 MachineBasicBlock &MBB,
731 MachineBasicBlock::iterator MI) const {
732 switch (MI->getOpcode()) {
733 case SystemZ::ADJCALLSTACKDOWN:
734 case SystemZ::ADJCALLSTACKUP:
735 assert(hasReservedCallFrame(MF) &&
736 "ADJSTACKDOWN and ADJSTACKUP should be no-ops");
737 return MBB.erase(MI);
738 break;
739
740 default:
741 llvm_unreachable("Unexpected call frame instruction");
742 }
743 }
744
getRegSpillOffset(MachineFunction & MF,Register Reg) const745 unsigned SystemZFrameLowering::getRegSpillOffset(MachineFunction &MF,
746 Register Reg) const {
747 bool IsVarArg = MF.getFunction().isVarArg();
748 bool BackChain = MF.getFunction().hasFnAttribute("backchain");
749 bool SoftFloat = MF.getSubtarget<SystemZSubtarget>().hasSoftFloat();
750 unsigned Offset = RegSpillOffsets[Reg];
751 if (usePackedStack(MF) && !(IsVarArg && !SoftFloat)) {
752 if (SystemZ::GR64BitRegClass.contains(Reg))
753 // Put all GPRs at the top of the Register save area with packed
754 // stack. Make room for the backchain if needed.
755 Offset += BackChain ? 24 : 32;
756 else
757 Offset = 0;
758 }
759 return Offset;
760 }
761
762 int SystemZFrameLowering::
getOrCreateFramePointerSaveIndex(MachineFunction & MF) const763 getOrCreateFramePointerSaveIndex(MachineFunction &MF) const {
764 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
765 int FI = ZFI->getFramePointerSaveIndex();
766 if (!FI) {
767 MachineFrameInfo &MFFrame = MF.getFrameInfo();
768 // The back chain is stored topmost with packed-stack.
769 int Offset = usePackedStack(MF) ? -8 : -SystemZMC::CallFrameSize;
770 FI = MFFrame.CreateFixedObject(8, Offset, false);
771 ZFI->setFramePointerSaveIndex(FI);
772 }
773 return FI;
774 }
775
usePackedStack(MachineFunction & MF) const776 bool SystemZFrameLowering::usePackedStack(MachineFunction &MF) const {
777 bool HasPackedStackAttr = MF.getFunction().hasFnAttribute("packed-stack");
778 bool BackChain = MF.getFunction().hasFnAttribute("backchain");
779 bool SoftFloat = MF.getSubtarget<SystemZSubtarget>().hasSoftFloat();
780 if (HasPackedStackAttr && BackChain && !SoftFloat)
781 report_fatal_error("packed-stack + backchain + hard-float is unsupported.");
782 bool CallConv = MF.getFunction().getCallingConv() != CallingConv::GHC;
783 return HasPackedStackAttr && CallConv;
784 }
785