1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of TargetFrameLowering class.
10 //
11 // On AArch64, stack frames are structured as follows:
12 //
13 // The stack grows downward.
14 //
15 // All of the individual frame areas on the frame below are optional, i.e. it's
16 // possible to create a function so that the particular area isn't present
17 // in the frame.
18 //
19 // At function entry, the "frame" looks as follows:
20 //
21 // | | Higher address
22 // |-----------------------------------|
23 // | |
24 // | arguments passed on the stack |
25 // | |
26 // |-----------------------------------| <- sp
27 // | | Lower address
28 //
29 //
30 // After the prologue has run, the frame has the following general structure.
31 // Note that this doesn't depict the case where a red-zone is used. Also,
32 // technically the last frame area (VLAs) doesn't get created until in the
33 // main function body, after the prologue is run. However, it's depicted here
34 // for completeness.
35 //
36 // | | Higher address
37 // |-----------------------------------|
38 // | |
39 // | arguments passed on the stack |
40 // | |
41 // |-----------------------------------|
42 // | |
43 // | (Win64 only) varargs from reg |
44 // | |
45 // |-----------------------------------|
46 // | |
47 // | callee-saved gpr registers | <--.
48 // | | | On Darwin platforms these
49 // |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
50 // | | | (frame record first)
51 // | prev_fp, prev_lr | <--'
52 // | (a.k.a. "frame record") |
53 // |-----------------------------------| <- fp(=x29)
54 // | |
55 // | callee-saved fp/simd/SVE regs |
56 // | |
57 // |-----------------------------------|
58 // | |
59 // | SVE stack objects |
60 // | |
61 // |-----------------------------------|
62 // |.empty.space.to.make.part.below....|
63 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
64 // |.the.standard.16-byte.alignment....| compile time; if present)
65 // |-----------------------------------|
66 // | |
67 // | local variables of fixed size |
68 // | including spill slots |
69 // |-----------------------------------| <- bp(not defined by ABI,
70 // |.variable-sized.local.variables....| LLVM chooses X19)
71 // |.(VLAs)............................| (size of this area is unknown at
72 // |...................................| compile time)
73 // |-----------------------------------| <- sp
74 // | | Lower address
75 //
76 //
77 // To access the data in a frame, at-compile time, a constant offset must be
78 // computable from one of the pointers (fp, bp, sp) to access it. The size
79 // of the areas with a dotted background cannot be computed at compile-time
80 // if they are present, making it required to have all three of fp, bp and
81 // sp to be set up to be able to access all contents in the frame areas,
82 // assuming all of the frame areas are non-empty.
83 //
84 // For most functions, some of the frame areas are empty. For those functions,
85 // it may not be necessary to set up fp or bp:
86 // * A base pointer is definitely needed when there are both VLAs and local
87 // variables with more-than-default alignment requirements.
88 // * A frame pointer is definitely needed when there are local variables with
89 // more-than-default alignment requirements.
90 //
91 // For Darwin platforms the frame-record (fp, lr) is stored at the top of the
92 // callee-saved area, since the unwind encoding does not allow for encoding
93 // this dynamically and existing tools depend on this layout. For other
94 // platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
95 // area to allow SVE stack objects (allocated directly below the callee-saves,
96 // if available) to be accessed directly from the framepointer.
97 // The SVE spill/fill instructions have VL-scaled addressing modes such
98 // as:
99 // ldr z8, [fp, #-7 mul vl]
100 // For SVE the size of the vector length (VL) is not known at compile-time, so
101 // '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
102 // layout, we don't need to add an unscaled offset to the framepointer before
103 // accessing the SVE object in the frame.
104 //
105 // In some cases when a base pointer is not strictly needed, it is generated
106 // anyway when offsets from the frame pointer to access local variables become
107 // so large that the offset can't be encoded in the immediate fields of loads
108 // or stores.
109 //
110 // FIXME: also explain the redzone concept.
111 // FIXME: also explain the concept of reserved call frames.
112 //
113 //===----------------------------------------------------------------------===//
114
115 #include "AArch64FrameLowering.h"
116 #include "AArch64InstrInfo.h"
117 #include "AArch64MachineFunctionInfo.h"
118 #include "AArch64RegisterInfo.h"
119 #include "AArch64StackOffset.h"
120 #include "AArch64Subtarget.h"
121 #include "AArch64TargetMachine.h"
122 #include "MCTargetDesc/AArch64AddressingModes.h"
123 #include "llvm/ADT/ScopeExit.h"
124 #include "llvm/ADT/SmallVector.h"
125 #include "llvm/ADT/Statistic.h"
126 #include "llvm/CodeGen/LivePhysRegs.h"
127 #include "llvm/CodeGen/MachineBasicBlock.h"
128 #include "llvm/CodeGen/MachineFrameInfo.h"
129 #include "llvm/CodeGen/MachineFunction.h"
130 #include "llvm/CodeGen/MachineInstr.h"
131 #include "llvm/CodeGen/MachineInstrBuilder.h"
132 #include "llvm/CodeGen/MachineMemOperand.h"
133 #include "llvm/CodeGen/MachineModuleInfo.h"
134 #include "llvm/CodeGen/MachineOperand.h"
135 #include "llvm/CodeGen/MachineRegisterInfo.h"
136 #include "llvm/CodeGen/RegisterScavenging.h"
137 #include "llvm/CodeGen/TargetInstrInfo.h"
138 #include "llvm/CodeGen/TargetRegisterInfo.h"
139 #include "llvm/CodeGen/TargetSubtargetInfo.h"
140 #include "llvm/CodeGen/WinEHFuncInfo.h"
141 #include "llvm/IR/Attributes.h"
142 #include "llvm/IR/CallingConv.h"
143 #include "llvm/IR/DataLayout.h"
144 #include "llvm/IR/DebugLoc.h"
145 #include "llvm/IR/Function.h"
146 #include "llvm/MC/MCAsmInfo.h"
147 #include "llvm/MC/MCDwarf.h"
148 #include "llvm/Support/CommandLine.h"
149 #include "llvm/Support/Debug.h"
150 #include "llvm/Support/ErrorHandling.h"
151 #include "llvm/Support/LEB128.h"
152 #include "llvm/Support/MathExtras.h"
153 #include "llvm/Support/raw_ostream.h"
154 #include "llvm/Target/TargetMachine.h"
155 #include "llvm/Target/TargetOptions.h"
156 #include <cassert>
157 #include <cstdint>
158 #include <iterator>
159 #include <vector>
160
161 using namespace llvm;
162
163 #define DEBUG_TYPE "frame-info"
164
165 static cl::opt<bool> EnableRedZone("aarch64-redzone",
166 cl::desc("enable use of redzone on AArch64"),
167 cl::init(false), cl::Hidden);
168
169 static cl::opt<bool>
170 ReverseCSRRestoreSeq("reverse-csr-restore-seq",
171 cl::desc("reverse the CSR restore sequence"),
172 cl::init(false), cl::Hidden);
173
174 static cl::opt<bool> StackTaggingMergeSetTag(
175 "stack-tagging-merge-settag",
176 cl::desc("merge settag instruction in function epilog"), cl::init(true),
177 cl::Hidden);
178
179 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
180
181 /// Returns the argument pop size.
getArgumentPopSize(MachineFunction & MF,MachineBasicBlock & MBB)182 static uint64_t getArgumentPopSize(MachineFunction &MF,
183 MachineBasicBlock &MBB) {
184 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
185 bool IsTailCallReturn = false;
186 if (MBB.end() != MBBI) {
187 unsigned RetOpcode = MBBI->getOpcode();
188 IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
189 RetOpcode == AArch64::TCRETURNri ||
190 RetOpcode == AArch64::TCRETURNriBTI;
191 }
192 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
193
194 uint64_t ArgumentPopSize = 0;
195 if (IsTailCallReturn) {
196 MachineOperand &StackAdjust = MBBI->getOperand(1);
197
198 // For a tail-call in a callee-pops-arguments environment, some or all of
199 // the stack may actually be in use for the call's arguments, this is
200 // calculated during LowerCall and consumed here...
201 ArgumentPopSize = StackAdjust.getImm();
202 } else {
203 // ... otherwise the amount to pop is *all* of the argument space,
204 // conveniently stored in the MachineFunctionInfo by
205 // LowerFormalArguments. This will, of course, be zero for the C calling
206 // convention.
207 ArgumentPopSize = AFI->getArgumentStackToRestore();
208 }
209
210 return ArgumentPopSize;
211 }
212
213 /// This is the biggest offset to the stack pointer we can encode in aarch64
214 /// instructions (without using a separate calculation and a temp register).
215 /// Note that the exception here are vector stores/loads which cannot encode any
216 /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
217 static const unsigned DefaultSafeSPDisplacement = 255;
218
219 /// Look at each instruction that references stack frames and return the stack
220 /// size limit beyond which some of these instructions will require a scratch
221 /// register during their expansion later.
estimateRSStackSizeLimit(MachineFunction & MF)222 static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
223 // FIXME: For now, just conservatively guestimate based on unscaled indexing
224 // range. We'll end up allocating an unnecessary spill slot a lot, but
225 // realistically that's not a big deal at this stage of the game.
226 for (MachineBasicBlock &MBB : MF) {
227 for (MachineInstr &MI : MBB) {
228 if (MI.isDebugInstr() || MI.isPseudo() ||
229 MI.getOpcode() == AArch64::ADDXri ||
230 MI.getOpcode() == AArch64::ADDSXri)
231 continue;
232
233 for (const MachineOperand &MO : MI.operands()) {
234 if (!MO.isFI())
235 continue;
236
237 StackOffset Offset;
238 if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
239 AArch64FrameOffsetCannotUpdate)
240 return 0;
241 }
242 }
243 }
244 return DefaultSafeSPDisplacement;
245 }
246
247 TargetStackID::Value
getStackIDForScalableVectors() const248 AArch64FrameLowering::getStackIDForScalableVectors() const {
249 return TargetStackID::SVEVector;
250 }
251
252 /// Returns the size of the fixed object area (allocated next to sp on entry)
253 /// On Win64 this may include a var args area and an UnwindHelp object for EH.
getFixedObjectSize(const MachineFunction & MF,const AArch64FunctionInfo * AFI,bool IsWin64,bool IsFunclet)254 static unsigned getFixedObjectSize(const MachineFunction &MF,
255 const AArch64FunctionInfo *AFI, bool IsWin64,
256 bool IsFunclet) {
257 if (!IsWin64 || IsFunclet) {
258 // Only Win64 uses fixed objects, and then only for the function (not
259 // funclets)
260 return 0;
261 } else {
262 // Var args are stored here in the primary function.
263 const unsigned VarArgsArea = AFI->getVarArgsGPRSize();
264 // To support EH funclets we allocate an UnwindHelp object
265 const unsigned UnwindHelpObject = (MF.hasEHFunclets() ? 8 : 0);
266 return alignTo(VarArgsArea + UnwindHelpObject, 16);
267 }
268 }
269
270 /// Returns the size of the entire SVE stackframe (calleesaves + spills).
getSVEStackSize(const MachineFunction & MF)271 static StackOffset getSVEStackSize(const MachineFunction &MF) {
272 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
273 return {(int64_t)AFI->getStackSizeSVE(), MVT::nxv1i8};
274 }
275
canUseRedZone(const MachineFunction & MF) const276 bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
277 if (!EnableRedZone)
278 return false;
279 // Don't use the red zone if the function explicitly asks us not to.
280 // This is typically used for kernel code.
281 if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
282 return false;
283
284 const MachineFrameInfo &MFI = MF.getFrameInfo();
285 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
286 uint64_t NumBytes = AFI->getLocalStackSize();
287
288 return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128 ||
289 getSVEStackSize(MF));
290 }
291
292 /// hasFP - Return true if the specified function should have a dedicated frame
293 /// pointer register.
hasFP(const MachineFunction & MF) const294 bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
295 const MachineFrameInfo &MFI = MF.getFrameInfo();
296 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
297 // Win64 EH requires a frame pointer if funclets are present, as the locals
298 // are accessed off the frame pointer in both the parent function and the
299 // funclets.
300 if (MF.hasEHFunclets())
301 return true;
302 // Retain behavior of always omitting the FP for leaf functions when possible.
303 if (MF.getTarget().Options.DisableFramePointerElim(MF))
304 return true;
305 if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
306 MFI.hasStackMap() || MFI.hasPatchPoint() ||
307 RegInfo->needsStackRealignment(MF))
308 return true;
309 // With large callframes around we may need to use FP to access the scavenging
310 // emergency spillslot.
311 //
312 // Unfortunately some calls to hasFP() like machine verifier ->
313 // getReservedReg() -> hasFP in the middle of global isel are too early
314 // to know the max call frame size. Hopefully conservatively returning "true"
315 // in those cases is fine.
316 // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
317 if (!MFI.isMaxCallFrameSizeComputed() ||
318 MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
319 return true;
320
321 return false;
322 }
323
324 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
325 /// not required, we reserve argument space for call sites in the function
326 /// immediately on entry to the current function. This eliminates the need for
327 /// add/sub sp brackets around call sites. Returns true if the call frame is
328 /// included as part of the stack frame.
329 bool
hasReservedCallFrame(const MachineFunction & MF) const330 AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
331 return !MF.getFrameInfo().hasVarSizedObjects();
332 }
333
eliminateCallFramePseudoInstr(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator I) const334 MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
335 MachineFunction &MF, MachineBasicBlock &MBB,
336 MachineBasicBlock::iterator I) const {
337 const AArch64InstrInfo *TII =
338 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
339 DebugLoc DL = I->getDebugLoc();
340 unsigned Opc = I->getOpcode();
341 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
342 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
343
344 if (!hasReservedCallFrame(MF)) {
345 int64_t Amount = I->getOperand(0).getImm();
346 Amount = alignTo(Amount, getStackAlign());
347 if (!IsDestroy)
348 Amount = -Amount;
349
350 // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
351 // doesn't have to pop anything), then the first operand will be zero too so
352 // this adjustment is a no-op.
353 if (CalleePopAmount == 0) {
354 // FIXME: in-function stack adjustment for calls is limited to 24-bits
355 // because there's no guaranteed temporary register available.
356 //
357 // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
358 // 1) For offset <= 12-bit, we use LSL #0
359 // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
360 // LSL #0, and the other uses LSL #12.
361 //
362 // Most call frames will be allocated at the start of a function so
363 // this is OK, but it is a limitation that needs dealing with.
364 assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
365 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, {Amount, MVT::i8},
366 TII);
367 }
368 } else if (CalleePopAmount != 0) {
369 // If the calling convention demands that the callee pops arguments from the
370 // stack, we want to add it back if we have a reserved call frame.
371 assert(CalleePopAmount < 0xffffff && "call frame too large");
372 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
373 {-(int64_t)CalleePopAmount, MVT::i8}, TII);
374 }
375 return MBB.erase(I);
376 }
377
ShouldSignReturnAddress(MachineFunction & MF)378 static bool ShouldSignReturnAddress(MachineFunction &MF) {
379 // The function should be signed in the following situations:
380 // - sign-return-address=all
381 // - sign-return-address=non-leaf and the functions spills the LR
382
383 const Function &F = MF.getFunction();
384 if (!F.hasFnAttribute("sign-return-address"))
385 return false;
386
387 StringRef Scope = F.getFnAttribute("sign-return-address").getValueAsString();
388 if (Scope.equals("none"))
389 return false;
390
391 if (Scope.equals("all"))
392 return true;
393
394 assert(Scope.equals("non-leaf") && "Expected all, none or non-leaf");
395
396 for (const auto &Info : MF.getFrameInfo().getCalleeSavedInfo())
397 if (Info.getReg() == AArch64::LR)
398 return true;
399
400 return false;
401 }
402
403 // Convenience function to create a DWARF expression for
404 // Expr + NumBytes + NumVGScaledBytes * AArch64::VG
appendVGScaledOffsetExpr(SmallVectorImpl<char> & Expr,int NumBytes,int NumVGScaledBytes,unsigned VG,llvm::raw_string_ostream & Comment)405 static void appendVGScaledOffsetExpr(SmallVectorImpl<char> &Expr,
406 int NumBytes, int NumVGScaledBytes, unsigned VG,
407 llvm::raw_string_ostream &Comment) {
408 uint8_t buffer[16];
409
410 if (NumBytes) {
411 Expr.push_back(dwarf::DW_OP_consts);
412 Expr.append(buffer, buffer + encodeSLEB128(NumBytes, buffer));
413 Expr.push_back((uint8_t)dwarf::DW_OP_plus);
414 Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(NumBytes);
415 }
416
417 if (NumVGScaledBytes) {
418 Expr.push_back((uint8_t)dwarf::DW_OP_consts);
419 Expr.append(buffer, buffer + encodeSLEB128(NumVGScaledBytes, buffer));
420
421 Expr.push_back((uint8_t)dwarf::DW_OP_bregx);
422 Expr.append(buffer, buffer + encodeULEB128(VG, buffer));
423 Expr.push_back(0);
424
425 Expr.push_back((uint8_t)dwarf::DW_OP_mul);
426 Expr.push_back((uint8_t)dwarf::DW_OP_plus);
427
428 Comment << (NumVGScaledBytes < 0 ? " - " : " + ")
429 << std::abs(NumVGScaledBytes) << " * VG";
430 }
431 }
432
433 // Creates an MCCFIInstruction:
434 // { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
createDefCFAExpressionFromSP(const TargetRegisterInfo & TRI,const StackOffset & OffsetFromSP) const435 MCCFIInstruction AArch64FrameLowering::createDefCFAExpressionFromSP(
436 const TargetRegisterInfo &TRI, const StackOffset &OffsetFromSP) const {
437 int64_t NumBytes, NumVGScaledBytes;
438 OffsetFromSP.getForDwarfOffset(NumBytes, NumVGScaledBytes);
439
440 std::string CommentBuffer = "sp";
441 llvm::raw_string_ostream Comment(CommentBuffer);
442
443 // Build up the expression (SP + NumBytes + NumVGScaledBytes * AArch64::VG)
444 SmallString<64> Expr;
445 Expr.push_back((uint8_t)(dwarf::DW_OP_breg0 + /*SP*/ 31));
446 Expr.push_back(0);
447 appendVGScaledOffsetExpr(Expr, NumBytes, NumVGScaledBytes,
448 TRI.getDwarfRegNum(AArch64::VG, true), Comment);
449
450 // Wrap this into DW_CFA_def_cfa.
451 SmallString<64> DefCfaExpr;
452 DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression);
453 uint8_t buffer[16];
454 DefCfaExpr.append(buffer,
455 buffer + encodeULEB128(Expr.size(), buffer));
456 DefCfaExpr.append(Expr.str());
457 return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(),
458 Comment.str());
459 }
460
createCfaOffset(const TargetRegisterInfo & TRI,unsigned Reg,const StackOffset & OffsetFromDefCFA) const461 MCCFIInstruction AArch64FrameLowering::createCfaOffset(
462 const TargetRegisterInfo &TRI, unsigned Reg,
463 const StackOffset &OffsetFromDefCFA) const {
464 int64_t NumBytes, NumVGScaledBytes;
465 OffsetFromDefCFA.getForDwarfOffset(NumBytes, NumVGScaledBytes);
466
467 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
468
469 // Non-scalable offsets can use DW_CFA_offset directly.
470 if (!NumVGScaledBytes)
471 return MCCFIInstruction::createOffset(nullptr, DwarfReg, NumBytes);
472
473 std::string CommentBuffer;
474 llvm::raw_string_ostream Comment(CommentBuffer);
475 Comment << printReg(Reg, &TRI) << " @ cfa";
476
477 // Build up expression (NumBytes + NumVGScaledBytes * AArch64::VG)
478 SmallString<64> OffsetExpr;
479 appendVGScaledOffsetExpr(OffsetExpr, NumBytes, NumVGScaledBytes,
480 TRI.getDwarfRegNum(AArch64::VG, true), Comment);
481
482 // Wrap this into DW_CFA_expression
483 SmallString<64> CfaExpr;
484 CfaExpr.push_back(dwarf::DW_CFA_expression);
485 uint8_t buffer[16];
486 CfaExpr.append(buffer, buffer + encodeULEB128(DwarfReg, buffer));
487 CfaExpr.append(buffer, buffer + encodeULEB128(OffsetExpr.size(), buffer));
488 CfaExpr.append(OffsetExpr.str());
489
490 return MCCFIInstruction::createEscape(nullptr, CfaExpr.str(), Comment.str());
491 }
492
emitCalleeSavedFrameMoves(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI) const493 void AArch64FrameLowering::emitCalleeSavedFrameMoves(
494 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
495 MachineFunction &MF = *MBB.getParent();
496 MachineFrameInfo &MFI = MF.getFrameInfo();
497 const TargetSubtargetInfo &STI = MF.getSubtarget();
498 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
499 const TargetInstrInfo *TII = STI.getInstrInfo();
500 DebugLoc DL = MBB.findDebugLoc(MBBI);
501
502 // Add callee saved registers to move list.
503 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
504 if (CSI.empty())
505 return;
506
507 for (const auto &Info : CSI) {
508 unsigned Reg = Info.getReg();
509
510 // Not all unwinders may know about SVE registers, so assume the lowest
511 // common demoninator.
512 unsigned NewReg;
513 if (static_cast<const AArch64RegisterInfo *>(TRI)->regNeedsCFI(Reg, NewReg))
514 Reg = NewReg;
515 else
516 continue;
517
518 StackOffset Offset;
519 if (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::SVEVector) {
520 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
521 Offset = StackOffset(MFI.getObjectOffset(Info.getFrameIdx()), MVT::nxv1i8) -
522 StackOffset(AFI->getCalleeSavedStackSize(MFI), MVT::i8);
523 } else {
524 Offset = {MFI.getObjectOffset(Info.getFrameIdx()) -
525 getOffsetOfLocalArea(),
526 MVT::i8};
527 }
528 unsigned CFIIndex = MF.addFrameInst(createCfaOffset(*TRI, Reg, Offset));
529 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
530 .addCFIIndex(CFIIndex)
531 .setMIFlags(MachineInstr::FrameSetup);
532 }
533 }
534
535 // Find a scratch register that we can use at the start of the prologue to
536 // re-align the stack pointer. We avoid using callee-save registers since they
537 // may appear to be free when this is called from canUseAsPrologue (during
538 // shrink wrapping), but then no longer be free when this is called from
539 // emitPrologue.
540 //
541 // FIXME: This is a bit conservative, since in the above case we could use one
542 // of the callee-save registers as a scratch temp to re-align the stack pointer,
543 // but we would then have to make sure that we were in fact saving at least one
544 // callee-save register in the prologue, which is additional complexity that
545 // doesn't seem worth the benefit.
findScratchNonCalleeSaveRegister(MachineBasicBlock * MBB)546 static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
547 MachineFunction *MF = MBB->getParent();
548
549 // If MBB is an entry block, use X9 as the scratch register
550 if (&MF->front() == MBB)
551 return AArch64::X9;
552
553 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
554 const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
555 LivePhysRegs LiveRegs(TRI);
556 LiveRegs.addLiveIns(*MBB);
557
558 // Mark callee saved registers as used so we will not choose them.
559 const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
560 for (unsigned i = 0; CSRegs[i]; ++i)
561 LiveRegs.addReg(CSRegs[i]);
562
563 // Prefer X9 since it was historically used for the prologue scratch reg.
564 const MachineRegisterInfo &MRI = MF->getRegInfo();
565 if (LiveRegs.available(MRI, AArch64::X9))
566 return AArch64::X9;
567
568 for (unsigned Reg : AArch64::GPR64RegClass) {
569 if (LiveRegs.available(MRI, Reg))
570 return Reg;
571 }
572 return AArch64::NoRegister;
573 }
574
canUseAsPrologue(const MachineBasicBlock & MBB) const575 bool AArch64FrameLowering::canUseAsPrologue(
576 const MachineBasicBlock &MBB) const {
577 const MachineFunction *MF = MBB.getParent();
578 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
579 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
580 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
581
582 // Don't need a scratch register if we're not going to re-align the stack.
583 if (!RegInfo->needsStackRealignment(*MF))
584 return true;
585 // Otherwise, we can use any block as long as it has a scratch register
586 // available.
587 return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
588 }
589
windowsRequiresStackProbe(MachineFunction & MF,uint64_t StackSizeInBytes)590 static bool windowsRequiresStackProbe(MachineFunction &MF,
591 uint64_t StackSizeInBytes) {
592 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
593 if (!Subtarget.isTargetWindows())
594 return false;
595 const Function &F = MF.getFunction();
596 // TODO: When implementing stack protectors, take that into account
597 // for the probe threshold.
598 unsigned StackProbeSize = 4096;
599 if (F.hasFnAttribute("stack-probe-size"))
600 F.getFnAttribute("stack-probe-size")
601 .getValueAsString()
602 .getAsInteger(0, StackProbeSize);
603 return (StackSizeInBytes >= StackProbeSize) &&
604 !F.hasFnAttribute("no-stack-arg-probe");
605 }
606
shouldCombineCSRLocalStackBump(MachineFunction & MF,uint64_t StackBumpBytes) const607 bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
608 MachineFunction &MF, uint64_t StackBumpBytes) const {
609 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
610 const MachineFrameInfo &MFI = MF.getFrameInfo();
611 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
612 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
613
614 if (AFI->getLocalStackSize() == 0)
615 return false;
616
617 // 512 is the maximum immediate for stp/ldp that will be used for
618 // callee-save save/restores
619 if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
620 return false;
621
622 if (MFI.hasVarSizedObjects())
623 return false;
624
625 if (RegInfo->needsStackRealignment(MF))
626 return false;
627
628 // This isn't strictly necessary, but it simplifies things a bit since the
629 // current RedZone handling code assumes the SP is adjusted by the
630 // callee-save save/restore code.
631 if (canUseRedZone(MF))
632 return false;
633
634 // When there is an SVE area on the stack, always allocate the
635 // callee-saves and spills/locals separately.
636 if (getSVEStackSize(MF))
637 return false;
638
639 return true;
640 }
641
shouldCombineCSRLocalStackBumpInEpilogue(MachineBasicBlock & MBB,unsigned StackBumpBytes) const642 bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
643 MachineBasicBlock &MBB, unsigned StackBumpBytes) const {
644 if (!shouldCombineCSRLocalStackBump(*MBB.getParent(), StackBumpBytes))
645 return false;
646
647 if (MBB.empty())
648 return true;
649
650 // Disable combined SP bump if the last instruction is an MTE tag store. It
651 // is almost always better to merge SP adjustment into those instructions.
652 MachineBasicBlock::iterator LastI = MBB.getFirstTerminator();
653 MachineBasicBlock::iterator Begin = MBB.begin();
654 while (LastI != Begin) {
655 --LastI;
656 if (LastI->isTransient())
657 continue;
658 if (!LastI->getFlag(MachineInstr::FrameDestroy))
659 break;
660 }
661 switch (LastI->getOpcode()) {
662 case AArch64::STGloop:
663 case AArch64::STZGloop:
664 case AArch64::STGOffset:
665 case AArch64::STZGOffset:
666 case AArch64::ST2GOffset:
667 case AArch64::STZ2GOffset:
668 return false;
669 default:
670 return true;
671 }
672 llvm_unreachable("unreachable");
673 }
674
675 // Given a load or a store instruction, generate an appropriate unwinding SEH
676 // code on Windows.
InsertSEH(MachineBasicBlock::iterator MBBI,const TargetInstrInfo & TII,MachineInstr::MIFlag Flag)677 static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI,
678 const TargetInstrInfo &TII,
679 MachineInstr::MIFlag Flag) {
680 unsigned Opc = MBBI->getOpcode();
681 MachineBasicBlock *MBB = MBBI->getParent();
682 MachineFunction &MF = *MBB->getParent();
683 DebugLoc DL = MBBI->getDebugLoc();
684 unsigned ImmIdx = MBBI->getNumOperands() - 1;
685 int Imm = MBBI->getOperand(ImmIdx).getImm();
686 MachineInstrBuilder MIB;
687 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
688 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
689
690 switch (Opc) {
691 default:
692 llvm_unreachable("No SEH Opcode for this instruction");
693 case AArch64::LDPDpost:
694 Imm = -Imm;
695 LLVM_FALLTHROUGH;
696 case AArch64::STPDpre: {
697 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
698 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
699 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
700 .addImm(Reg0)
701 .addImm(Reg1)
702 .addImm(Imm * 8)
703 .setMIFlag(Flag);
704 break;
705 }
706 case AArch64::LDPXpost:
707 Imm = -Imm;
708 LLVM_FALLTHROUGH;
709 case AArch64::STPXpre: {
710 Register Reg0 = MBBI->getOperand(1).getReg();
711 Register Reg1 = MBBI->getOperand(2).getReg();
712 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
713 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
714 .addImm(Imm * 8)
715 .setMIFlag(Flag);
716 else
717 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
718 .addImm(RegInfo->getSEHRegNum(Reg0))
719 .addImm(RegInfo->getSEHRegNum(Reg1))
720 .addImm(Imm * 8)
721 .setMIFlag(Flag);
722 break;
723 }
724 case AArch64::LDRDpost:
725 Imm = -Imm;
726 LLVM_FALLTHROUGH;
727 case AArch64::STRDpre: {
728 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
729 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
730 .addImm(Reg)
731 .addImm(Imm)
732 .setMIFlag(Flag);
733 break;
734 }
735 case AArch64::LDRXpost:
736 Imm = -Imm;
737 LLVM_FALLTHROUGH;
738 case AArch64::STRXpre: {
739 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
740 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
741 .addImm(Reg)
742 .addImm(Imm)
743 .setMIFlag(Flag);
744 break;
745 }
746 case AArch64::STPDi:
747 case AArch64::LDPDi: {
748 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
749 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
750 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
751 .addImm(Reg0)
752 .addImm(Reg1)
753 .addImm(Imm * 8)
754 .setMIFlag(Flag);
755 break;
756 }
757 case AArch64::STPXi:
758 case AArch64::LDPXi: {
759 Register Reg0 = MBBI->getOperand(0).getReg();
760 Register Reg1 = MBBI->getOperand(1).getReg();
761 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
762 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
763 .addImm(Imm * 8)
764 .setMIFlag(Flag);
765 else
766 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
767 .addImm(RegInfo->getSEHRegNum(Reg0))
768 .addImm(RegInfo->getSEHRegNum(Reg1))
769 .addImm(Imm * 8)
770 .setMIFlag(Flag);
771 break;
772 }
773 case AArch64::STRXui:
774 case AArch64::LDRXui: {
775 int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
776 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
777 .addImm(Reg)
778 .addImm(Imm * 8)
779 .setMIFlag(Flag);
780 break;
781 }
782 case AArch64::STRDui:
783 case AArch64::LDRDui: {
784 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
785 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
786 .addImm(Reg)
787 .addImm(Imm * 8)
788 .setMIFlag(Flag);
789 break;
790 }
791 }
792 auto I = MBB->insertAfter(MBBI, MIB);
793 return I;
794 }
795
796 // Fix up the SEH opcode associated with the save/restore instruction.
fixupSEHOpcode(MachineBasicBlock::iterator MBBI,unsigned LocalStackSize)797 static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI,
798 unsigned LocalStackSize) {
799 MachineOperand *ImmOpnd = nullptr;
800 unsigned ImmIdx = MBBI->getNumOperands() - 1;
801 switch (MBBI->getOpcode()) {
802 default:
803 llvm_unreachable("Fix the offset in the SEH instruction");
804 case AArch64::SEH_SaveFPLR:
805 case AArch64::SEH_SaveRegP:
806 case AArch64::SEH_SaveReg:
807 case AArch64::SEH_SaveFRegP:
808 case AArch64::SEH_SaveFReg:
809 ImmOpnd = &MBBI->getOperand(ImmIdx);
810 break;
811 }
812 if (ImmOpnd)
813 ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
814 }
815
816 // Convert callee-save register save/restore instruction to do stack pointer
817 // decrement/increment to allocate/deallocate the callee-save stack area by
818 // converting store/load to use pre/post increment version.
convertCalleeSaveRestoreToSPPrePostIncDec(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,const TargetInstrInfo * TII,int CSStackSizeInc,bool NeedsWinCFI,bool * HasWinCFI,bool InProlog=true)819 static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
820 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
821 const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
822 bool NeedsWinCFI, bool *HasWinCFI, bool InProlog = true) {
823 // Ignore instructions that do not operate on SP, i.e. shadow call stack
824 // instructions and associated CFI instruction.
825 while (MBBI->getOpcode() == AArch64::STRXpost ||
826 MBBI->getOpcode() == AArch64::LDRXpre ||
827 MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) {
828 if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION)
829 assert(MBBI->getOperand(0).getReg() != AArch64::SP);
830 ++MBBI;
831 }
832 unsigned NewOpc;
833 int Scale = 1;
834 switch (MBBI->getOpcode()) {
835 default:
836 llvm_unreachable("Unexpected callee-save save/restore opcode!");
837 case AArch64::STPXi:
838 NewOpc = AArch64::STPXpre;
839 Scale = 8;
840 break;
841 case AArch64::STPDi:
842 NewOpc = AArch64::STPDpre;
843 Scale = 8;
844 break;
845 case AArch64::STPQi:
846 NewOpc = AArch64::STPQpre;
847 Scale = 16;
848 break;
849 case AArch64::STRXui:
850 NewOpc = AArch64::STRXpre;
851 break;
852 case AArch64::STRDui:
853 NewOpc = AArch64::STRDpre;
854 break;
855 case AArch64::STRQui:
856 NewOpc = AArch64::STRQpre;
857 break;
858 case AArch64::LDPXi:
859 NewOpc = AArch64::LDPXpost;
860 Scale = 8;
861 break;
862 case AArch64::LDPDi:
863 NewOpc = AArch64::LDPDpost;
864 Scale = 8;
865 break;
866 case AArch64::LDPQi:
867 NewOpc = AArch64::LDPQpost;
868 Scale = 16;
869 break;
870 case AArch64::LDRXui:
871 NewOpc = AArch64::LDRXpost;
872 break;
873 case AArch64::LDRDui:
874 NewOpc = AArch64::LDRDpost;
875 break;
876 case AArch64::LDRQui:
877 NewOpc = AArch64::LDRQpost;
878 break;
879 }
880 // Get rid of the SEH code associated with the old instruction.
881 if (NeedsWinCFI) {
882 auto SEH = std::next(MBBI);
883 if (AArch64InstrInfo::isSEHInstruction(*SEH))
884 SEH->eraseFromParent();
885 }
886
887 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
888 MIB.addReg(AArch64::SP, RegState::Define);
889
890 // Copy all operands other than the immediate offset.
891 unsigned OpndIdx = 0;
892 for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
893 ++OpndIdx)
894 MIB.add(MBBI->getOperand(OpndIdx));
895
896 assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
897 "Unexpected immediate offset in first/last callee-save save/restore "
898 "instruction!");
899 assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
900 "Unexpected base register in callee-save save/restore instruction!");
901 assert(CSStackSizeInc % Scale == 0);
902 MIB.addImm(CSStackSizeInc / Scale);
903
904 MIB.setMIFlags(MBBI->getFlags());
905 MIB.setMemRefs(MBBI->memoperands());
906
907 // Generate a new SEH code that corresponds to the new instruction.
908 if (NeedsWinCFI) {
909 *HasWinCFI = true;
910 InsertSEH(*MIB, *TII,
911 InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy);
912 }
913
914 return std::prev(MBB.erase(MBBI));
915 }
916
917 // Fixup callee-save register save/restore instructions to take into account
918 // combined SP bump by adding the local stack size to the stack offsets.
fixupCalleeSaveRestoreStackOffset(MachineInstr & MI,uint64_t LocalStackSize,bool NeedsWinCFI,bool * HasWinCFI)919 static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
920 uint64_t LocalStackSize,
921 bool NeedsWinCFI,
922 bool *HasWinCFI) {
923 if (AArch64InstrInfo::isSEHInstruction(MI))
924 return;
925
926 unsigned Opc = MI.getOpcode();
927
928 // Ignore instructions that do not operate on SP, i.e. shadow call stack
929 // instructions and associated CFI instruction.
930 if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre ||
931 Opc == AArch64::CFI_INSTRUCTION) {
932 if (Opc != AArch64::CFI_INSTRUCTION)
933 assert(MI.getOperand(0).getReg() != AArch64::SP);
934 return;
935 }
936
937 unsigned Scale;
938 switch (Opc) {
939 case AArch64::STPXi:
940 case AArch64::STRXui:
941 case AArch64::STPDi:
942 case AArch64::STRDui:
943 case AArch64::LDPXi:
944 case AArch64::LDRXui:
945 case AArch64::LDPDi:
946 case AArch64::LDRDui:
947 Scale = 8;
948 break;
949 case AArch64::STPQi:
950 case AArch64::STRQui:
951 case AArch64::LDPQi:
952 case AArch64::LDRQui:
953 Scale = 16;
954 break;
955 default:
956 llvm_unreachable("Unexpected callee-save save/restore opcode!");
957 }
958
959 unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
960 assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
961 "Unexpected base register in callee-save save/restore instruction!");
962 // Last operand is immediate offset that needs fixing.
963 MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
964 // All generated opcodes have scaled offsets.
965 assert(LocalStackSize % Scale == 0);
966 OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
967
968 if (NeedsWinCFI) {
969 *HasWinCFI = true;
970 auto MBBI = std::next(MachineBasicBlock::iterator(MI));
971 assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
972 assert(AArch64InstrInfo::isSEHInstruction(*MBBI) &&
973 "Expecting a SEH instruction");
974 fixupSEHOpcode(MBBI, LocalStackSize);
975 }
976 }
977
adaptForLdStOpt(MachineBasicBlock & MBB,MachineBasicBlock::iterator FirstSPPopI,MachineBasicBlock::iterator LastPopI)978 static void adaptForLdStOpt(MachineBasicBlock &MBB,
979 MachineBasicBlock::iterator FirstSPPopI,
980 MachineBasicBlock::iterator LastPopI) {
981 // Sometimes (when we restore in the same order as we save), we can end up
982 // with code like this:
983 //
984 // ldp x26, x25, [sp]
985 // ldp x24, x23, [sp, #16]
986 // ldp x22, x21, [sp, #32]
987 // ldp x20, x19, [sp, #48]
988 // add sp, sp, #64
989 //
990 // In this case, it is always better to put the first ldp at the end, so
991 // that the load-store optimizer can run and merge the ldp and the add into
992 // a post-index ldp.
993 // If we managed to grab the first pop instruction, move it to the end.
994 if (ReverseCSRRestoreSeq)
995 MBB.splice(FirstSPPopI, &MBB, LastPopI);
996 // We should end up with something like this now:
997 //
998 // ldp x24, x23, [sp, #16]
999 // ldp x22, x21, [sp, #32]
1000 // ldp x20, x19, [sp, #48]
1001 // ldp x26, x25, [sp]
1002 // add sp, sp, #64
1003 //
1004 // and the load-store optimizer can merge the last two instructions into:
1005 //
1006 // ldp x26, x25, [sp], #64
1007 //
1008 }
1009
ShouldSignWithAKey(MachineFunction & MF)1010 static bool ShouldSignWithAKey(MachineFunction &MF) {
1011 const Function &F = MF.getFunction();
1012 if (!F.hasFnAttribute("sign-return-address-key"))
1013 return true;
1014
1015 const StringRef Key =
1016 F.getFnAttribute("sign-return-address-key").getValueAsString();
1017 assert(Key.equals_lower("a_key") || Key.equals_lower("b_key"));
1018 return Key.equals_lower("a_key");
1019 }
1020
needsWinCFI(const MachineFunction & MF)1021 static bool needsWinCFI(const MachineFunction &MF) {
1022 const Function &F = MF.getFunction();
1023 return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
1024 F.needsUnwindTableEntry();
1025 }
1026
isTargetDarwin(const MachineFunction & MF)1027 static bool isTargetDarwin(const MachineFunction &MF) {
1028 return MF.getSubtarget<AArch64Subtarget>().isTargetDarwin();
1029 }
1030
isTargetWindows(const MachineFunction & MF)1031 static bool isTargetWindows(const MachineFunction &MF) {
1032 return MF.getSubtarget<AArch64Subtarget>().isTargetWindows();
1033 }
1034
1035 // Convenience function to determine whether I is an SVE callee save.
IsSVECalleeSave(MachineBasicBlock::iterator I)1036 static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
1037 switch (I->getOpcode()) {
1038 default:
1039 return false;
1040 case AArch64::STR_ZXI:
1041 case AArch64::STR_PXI:
1042 case AArch64::LDR_ZXI:
1043 case AArch64::LDR_PXI:
1044 return I->getFlag(MachineInstr::FrameSetup) ||
1045 I->getFlag(MachineInstr::FrameDestroy);
1046 }
1047 }
1048
emitPrologue(MachineFunction & MF,MachineBasicBlock & MBB) const1049 void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
1050 MachineBasicBlock &MBB) const {
1051 MachineBasicBlock::iterator MBBI = MBB.begin();
1052 const MachineFrameInfo &MFI = MF.getFrameInfo();
1053 const Function &F = MF.getFunction();
1054 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1055 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1056 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1057 MachineModuleInfo &MMI = MF.getMMI();
1058 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1059 bool needsFrameMoves =
1060 MF.needsFrameMoves() && !MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
1061 bool HasFP = hasFP(MF);
1062 bool NeedsWinCFI = needsWinCFI(MF);
1063 bool HasWinCFI = false;
1064 auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); });
1065
1066 bool IsFunclet = MBB.isEHFuncletEntry();
1067
1068 // At this point, we're going to decide whether or not the function uses a
1069 // redzone. In most cases, the function doesn't have a redzone so let's
1070 // assume that's false and set it to true in the case that there's a redzone.
1071 AFI->setHasRedZone(false);
1072
1073 // Debug location must be unknown since the first debug location is used
1074 // to determine the end of the prologue.
1075 DebugLoc DL;
1076
1077 if (ShouldSignReturnAddress(MF)) {
1078 if (ShouldSignWithAKey(MF))
1079 BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
1080 .setMIFlag(MachineInstr::FrameSetup);
1081 else {
1082 BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
1083 .setMIFlag(MachineInstr::FrameSetup);
1084 BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIBSP))
1085 .setMIFlag(MachineInstr::FrameSetup);
1086 }
1087
1088 unsigned CFIIndex =
1089 MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
1090 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1091 .addCFIIndex(CFIIndex)
1092 .setMIFlags(MachineInstr::FrameSetup);
1093 }
1094
1095 // All calls are tail calls in GHC calling conv, and functions have no
1096 // prologue/epilogue.
1097 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1098 return;
1099
1100 // Set tagged base pointer to the bottom of the stack frame.
1101 // Ideally it should match SP value after prologue.
1102 AFI->setTaggedBasePointerOffset(MFI.getStackSize());
1103
1104 const StackOffset &SVEStackSize = getSVEStackSize(MF);
1105
1106 // getStackSize() includes all the locals in its size calculation. We don't
1107 // include these locals when computing the stack size of a funclet, as they
1108 // are allocated in the parent's stack frame and accessed via the frame
1109 // pointer from the funclet. We only save the callee saved registers in the
1110 // funclet, which are really the callee saved registers of the parent
1111 // function, including the funclet.
1112 int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1113 : MFI.getStackSize();
1114 if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
1115 assert(!HasFP && "unexpected function without stack frame but with FP");
1116 assert(!SVEStackSize &&
1117 "unexpected function without stack frame but with SVE objects");
1118 // All of the stack allocation is for locals.
1119 AFI->setLocalStackSize(NumBytes);
1120 if (!NumBytes)
1121 return;
1122 // REDZONE: If the stack size is less than 128 bytes, we don't need
1123 // to actually allocate.
1124 if (canUseRedZone(MF)) {
1125 AFI->setHasRedZone(true);
1126 ++NumRedZoneFunctions;
1127 } else {
1128 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1129 {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup,
1130 false, NeedsWinCFI, &HasWinCFI);
1131 if (!NeedsWinCFI && needsFrameMoves) {
1132 // Label used to tie together the PROLOG_LABEL and the MachineMoves.
1133 MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
1134 // Encode the stack size of the leaf function.
1135 unsigned CFIIndex = MF.addFrameInst(
1136 MCCFIInstruction::cfiDefCfaOffset(FrameLabel, NumBytes));
1137 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1138 .addCFIIndex(CFIIndex)
1139 .setMIFlags(MachineInstr::FrameSetup);
1140 }
1141 }
1142
1143 if (NeedsWinCFI) {
1144 HasWinCFI = true;
1145 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1146 .setMIFlag(MachineInstr::FrameSetup);
1147 }
1148
1149 return;
1150 }
1151
1152 bool IsWin64 =
1153 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1154 unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1155
1156 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1157 // All of the remaining stack allocations are for locals.
1158 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1159 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1160 if (CombineSPBump) {
1161 assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1162 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1163 {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup, false,
1164 NeedsWinCFI, &HasWinCFI);
1165 NumBytes = 0;
1166 } else if (PrologueSaveSize != 0) {
1167 MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(
1168 MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI);
1169 NumBytes -= PrologueSaveSize;
1170 }
1171 assert(NumBytes >= 0 && "Negative stack allocation size!?");
1172
1173 // Move past the saves of the callee-saved registers, fixing up the offsets
1174 // and pre-inc if we decided to combine the callee-save and local stack
1175 // pointer bump above.
1176 MachineBasicBlock::iterator End = MBB.end();
1177 while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) &&
1178 !IsSVECalleeSave(MBBI)) {
1179 if (CombineSPBump)
1180 fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(),
1181 NeedsWinCFI, &HasWinCFI);
1182 ++MBBI;
1183 }
1184
1185 // For funclets the FP belongs to the containing function.
1186 if (!IsFunclet && HasFP) {
1187 // Only set up FP if we actually need to.
1188 int64_t FPOffset = isTargetDarwin(MF) ? (AFI->getCalleeSavedStackSize() - 16) : 0;
1189
1190 if (CombineSPBump)
1191 FPOffset += AFI->getLocalStackSize();
1192
1193 // Issue sub fp, sp, FPOffset or
1194 // mov fp,sp when FPOffset is zero.
1195 // Note: All stores of callee-saved registers are marked as "FrameSetup".
1196 // This code marks the instruction(s) that set the FP also.
1197 emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP,
1198 {FPOffset, MVT::i8}, TII, MachineInstr::FrameSetup, false,
1199 NeedsWinCFI, &HasWinCFI);
1200 }
1201
1202 if (windowsRequiresStackProbe(MF, NumBytes)) {
1203 uint64_t NumWords = NumBytes >> 4;
1204 if (NeedsWinCFI) {
1205 HasWinCFI = true;
1206 // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
1207 // exceed this amount. We need to move at most 2^24 - 1 into x15.
1208 // This is at most two instructions, MOVZ follwed by MOVK.
1209 // TODO: Fix to use multiple stack alloc unwind codes for stacks
1210 // exceeding 256MB in size.
1211 if (NumBytes >= (1 << 28))
1212 report_fatal_error("Stack size cannot exceed 256MB for stack "
1213 "unwinding purposes");
1214
1215 uint32_t LowNumWords = NumWords & 0xFFFF;
1216 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
1217 .addImm(LowNumWords)
1218 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
1219 .setMIFlag(MachineInstr::FrameSetup);
1220 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1221 .setMIFlag(MachineInstr::FrameSetup);
1222 if ((NumWords & 0xFFFF0000) != 0) {
1223 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
1224 .addReg(AArch64::X15)
1225 .addImm((NumWords & 0xFFFF0000) >> 16) // High half
1226 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16))
1227 .setMIFlag(MachineInstr::FrameSetup);
1228 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1229 .setMIFlag(MachineInstr::FrameSetup);
1230 }
1231 } else {
1232 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
1233 .addImm(NumWords)
1234 .setMIFlags(MachineInstr::FrameSetup);
1235 }
1236
1237 switch (MF.getTarget().getCodeModel()) {
1238 case CodeModel::Tiny:
1239 case CodeModel::Small:
1240 case CodeModel::Medium:
1241 case CodeModel::Kernel:
1242 BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
1243 .addExternalSymbol("__chkstk")
1244 .addReg(AArch64::X15, RegState::Implicit)
1245 .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
1246 .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
1247 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
1248 .setMIFlags(MachineInstr::FrameSetup);
1249 if (NeedsWinCFI) {
1250 HasWinCFI = true;
1251 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1252 .setMIFlag(MachineInstr::FrameSetup);
1253 }
1254 break;
1255 case CodeModel::Large:
1256 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
1257 .addReg(AArch64::X16, RegState::Define)
1258 .addExternalSymbol("__chkstk")
1259 .addExternalSymbol("__chkstk")
1260 .setMIFlags(MachineInstr::FrameSetup);
1261 if (NeedsWinCFI) {
1262 HasWinCFI = true;
1263 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1264 .setMIFlag(MachineInstr::FrameSetup);
1265 }
1266
1267 BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF)))
1268 .addReg(AArch64::X16, RegState::Kill)
1269 .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
1270 .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
1271 .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
1272 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
1273 .setMIFlags(MachineInstr::FrameSetup);
1274 if (NeedsWinCFI) {
1275 HasWinCFI = true;
1276 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1277 .setMIFlag(MachineInstr::FrameSetup);
1278 }
1279 break;
1280 }
1281
1282 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
1283 .addReg(AArch64::SP, RegState::Kill)
1284 .addReg(AArch64::X15, RegState::Kill)
1285 .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4))
1286 .setMIFlags(MachineInstr::FrameSetup);
1287 if (NeedsWinCFI) {
1288 HasWinCFI = true;
1289 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1290 .addImm(NumBytes)
1291 .setMIFlag(MachineInstr::FrameSetup);
1292 }
1293 NumBytes = 0;
1294 }
1295
1296 StackOffset AllocateBefore = SVEStackSize, AllocateAfter = {};
1297 MachineBasicBlock::iterator CalleeSavesBegin = MBBI, CalleeSavesEnd = MBBI;
1298
1299 // Process the SVE callee-saves to determine what space needs to be
1300 // allocated.
1301 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
1302 // Find callee save instructions in frame.
1303 CalleeSavesBegin = MBBI;
1304 assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
1305 while (IsSVECalleeSave(MBBI) && MBBI != MBB.getFirstTerminator())
1306 ++MBBI;
1307 CalleeSavesEnd = MBBI;
1308
1309 AllocateBefore = {CalleeSavedSize, MVT::nxv1i8};
1310 AllocateAfter = SVEStackSize - AllocateBefore;
1311 }
1312
1313 // Allocate space for the callee saves (if any).
1314 emitFrameOffset(MBB, CalleeSavesBegin, DL, AArch64::SP, AArch64::SP,
1315 -AllocateBefore, TII,
1316 MachineInstr::FrameSetup);
1317
1318 // Finally allocate remaining SVE stack space.
1319 emitFrameOffset(MBB, CalleeSavesEnd, DL, AArch64::SP, AArch64::SP,
1320 -AllocateAfter, TII,
1321 MachineInstr::FrameSetup);
1322
1323 // Allocate space for the rest of the frame.
1324 if (NumBytes) {
1325 // Alignment is required for the parent frame, not the funclet
1326 const bool NeedsRealignment =
1327 !IsFunclet && RegInfo->needsStackRealignment(MF);
1328 unsigned scratchSPReg = AArch64::SP;
1329
1330 if (NeedsRealignment) {
1331 scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
1332 assert(scratchSPReg != AArch64::NoRegister);
1333 }
1334
1335 // If we're a leaf function, try using the red zone.
1336 if (!canUseRedZone(MF))
1337 // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
1338 // the correct value here, as NumBytes also includes padding bytes,
1339 // which shouldn't be counted here.
1340 emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP,
1341 {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup,
1342 false, NeedsWinCFI, &HasWinCFI);
1343
1344 if (NeedsRealignment) {
1345 const unsigned NrBitsToZero = Log2(MFI.getMaxAlign());
1346 assert(NrBitsToZero > 1);
1347 assert(scratchSPReg != AArch64::SP);
1348
1349 // SUB X9, SP, NumBytes
1350 // -- X9 is temporary register, so shouldn't contain any live data here,
1351 // -- free to use. This is already produced by emitFrameOffset above.
1352 // AND SP, X9, 0b11111...0000
1353 // The logical immediates have a non-trivial encoding. The following
1354 // formula computes the encoded immediate with all ones but
1355 // NrBitsToZero zero bits as least significant bits.
1356 uint32_t andMaskEncoded = (1 << 12) // = N
1357 | ((64 - NrBitsToZero) << 6) // immr
1358 | ((64 - NrBitsToZero - 1) << 0); // imms
1359
1360 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
1361 .addReg(scratchSPReg, RegState::Kill)
1362 .addImm(andMaskEncoded);
1363 AFI->setStackRealigned(true);
1364 if (NeedsWinCFI) {
1365 HasWinCFI = true;
1366 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1367 .addImm(NumBytes & andMaskEncoded)
1368 .setMIFlag(MachineInstr::FrameSetup);
1369 }
1370 }
1371 }
1372
1373 // If we need a base pointer, set it up here. It's whatever the value of the
1374 // stack pointer is at this point. Any variable size objects will be allocated
1375 // after this, so we can still use the base pointer to reference locals.
1376 //
1377 // FIXME: Clarify FrameSetup flags here.
1378 // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
1379 // needed.
1380 // For funclets the BP belongs to the containing function.
1381 if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
1382 TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
1383 false);
1384 if (NeedsWinCFI) {
1385 HasWinCFI = true;
1386 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1387 .setMIFlag(MachineInstr::FrameSetup);
1388 }
1389 }
1390
1391 // The very last FrameSetup instruction indicates the end of prologue. Emit a
1392 // SEH opcode indicating the prologue end.
1393 if (NeedsWinCFI && HasWinCFI) {
1394 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1395 .setMIFlag(MachineInstr::FrameSetup);
1396 }
1397
1398 // SEH funclets are passed the frame pointer in X1. If the parent
1399 // function uses the base register, then the base register is used
1400 // directly, and is not retrieved from X1.
1401 if (IsFunclet && F.hasPersonalityFn()) {
1402 EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
1403 if (isAsynchronousEHPersonality(Per)) {
1404 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
1405 .addReg(AArch64::X1)
1406 .setMIFlag(MachineInstr::FrameSetup);
1407 MBB.addLiveIn(AArch64::X1);
1408 }
1409 }
1410
1411 if (needsFrameMoves) {
1412 const DataLayout &TD = MF.getDataLayout();
1413 const int StackGrowth = isTargetDarwin(MF)
1414 ? (2 * -TD.getPointerSize(0))
1415 : -AFI->getCalleeSavedStackSize();
1416 Register FramePtr = RegInfo->getFrameRegister(MF);
1417 // An example of the prologue:
1418 //
1419 // .globl __foo
1420 // .align 2
1421 // __foo:
1422 // Ltmp0:
1423 // .cfi_startproc
1424 // .cfi_personality 155, ___gxx_personality_v0
1425 // Leh_func_begin:
1426 // .cfi_lsda 16, Lexception33
1427 //
1428 // stp xa,bx, [sp, -#offset]!
1429 // ...
1430 // stp x28, x27, [sp, #offset-32]
1431 // stp fp, lr, [sp, #offset-16]
1432 // add fp, sp, #offset - 16
1433 // sub sp, sp, #1360
1434 //
1435 // The Stack:
1436 // +-------------------------------------------+
1437 // 10000 | ........ | ........ | ........ | ........ |
1438 // 10004 | ........ | ........ | ........ | ........ |
1439 // +-------------------------------------------+
1440 // 10008 | ........ | ........ | ........ | ........ |
1441 // 1000c | ........ | ........ | ........ | ........ |
1442 // +===========================================+
1443 // 10010 | X28 Register |
1444 // 10014 | X28 Register |
1445 // +-------------------------------------------+
1446 // 10018 | X27 Register |
1447 // 1001c | X27 Register |
1448 // +===========================================+
1449 // 10020 | Frame Pointer |
1450 // 10024 | Frame Pointer |
1451 // +-------------------------------------------+
1452 // 10028 | Link Register |
1453 // 1002c | Link Register |
1454 // +===========================================+
1455 // 10030 | ........ | ........ | ........ | ........ |
1456 // 10034 | ........ | ........ | ........ | ........ |
1457 // +-------------------------------------------+
1458 // 10038 | ........ | ........ | ........ | ........ |
1459 // 1003c | ........ | ........ | ........ | ........ |
1460 // +-------------------------------------------+
1461 //
1462 // [sp] = 10030 :: >>initial value<<
1463 // sp = 10020 :: stp fp, lr, [sp, #-16]!
1464 // fp = sp == 10020 :: mov fp, sp
1465 // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
1466 // sp == 10010 :: >>final value<<
1467 //
1468 // The frame pointer (w29) points to address 10020. If we use an offset of
1469 // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
1470 // for w27, and -32 for w28:
1471 //
1472 // Ltmp1:
1473 // .cfi_def_cfa w29, 16
1474 // Ltmp2:
1475 // .cfi_offset w30, -8
1476 // Ltmp3:
1477 // .cfi_offset w29, -16
1478 // Ltmp4:
1479 // .cfi_offset w27, -24
1480 // Ltmp5:
1481 // .cfi_offset w28, -32
1482
1483 if (HasFP) {
1484 // Define the current CFA rule to use the provided FP.
1485 unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
1486 unsigned CFIIndex = MF.addFrameInst(
1487 MCCFIInstruction::cfiDefCfa(nullptr, Reg, FixedObject - StackGrowth));
1488 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1489 .addCFIIndex(CFIIndex)
1490 .setMIFlags(MachineInstr::FrameSetup);
1491 } else {
1492 unsigned CFIIndex;
1493 if (SVEStackSize) {
1494 const TargetSubtargetInfo &STI = MF.getSubtarget();
1495 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
1496 StackOffset TotalSize =
1497 SVEStackSize + StackOffset((int64_t)MFI.getStackSize(), MVT::i8);
1498 CFIIndex = MF.addFrameInst(createDefCFAExpressionFromSP(TRI, TotalSize));
1499 } else {
1500 // Encode the stack size of the leaf function.
1501 CFIIndex = MF.addFrameInst(
1502 MCCFIInstruction::cfiDefCfaOffset(nullptr, MFI.getStackSize()));
1503 }
1504 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1505 .addCFIIndex(CFIIndex)
1506 .setMIFlags(MachineInstr::FrameSetup);
1507 }
1508
1509 // Now emit the moves for whatever callee saved regs we have (including FP,
1510 // LR if those are saved).
1511 emitCalleeSavedFrameMoves(MBB, MBBI);
1512 }
1513 }
1514
InsertReturnAddressAuth(MachineFunction & MF,MachineBasicBlock & MBB)1515 static void InsertReturnAddressAuth(MachineFunction &MF,
1516 MachineBasicBlock &MBB) {
1517 if (!ShouldSignReturnAddress(MF))
1518 return;
1519 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1520 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1521
1522 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
1523 DebugLoc DL;
1524 if (MBBI != MBB.end())
1525 DL = MBBI->getDebugLoc();
1526
1527 // The AUTIASP instruction assembles to a hint instruction before v8.3a so
1528 // this instruction can safely used for any v8a architecture.
1529 // From v8.3a onwards there are optimised authenticate LR and return
1530 // instructions, namely RETA{A,B}, that can be used instead.
1531 if (Subtarget.hasV8_3aOps() && MBBI != MBB.end() &&
1532 MBBI->getOpcode() == AArch64::RET_ReallyLR) {
1533 BuildMI(MBB, MBBI, DL,
1534 TII->get(ShouldSignWithAKey(MF) ? AArch64::RETAA : AArch64::RETAB))
1535 .copyImplicitOps(*MBBI);
1536 MBB.erase(MBBI);
1537 } else {
1538 BuildMI(
1539 MBB, MBBI, DL,
1540 TII->get(ShouldSignWithAKey(MF) ? AArch64::AUTIASP : AArch64::AUTIBSP))
1541 .setMIFlag(MachineInstr::FrameDestroy);
1542 }
1543 }
1544
isFuncletReturnInstr(const MachineInstr & MI)1545 static bool isFuncletReturnInstr(const MachineInstr &MI) {
1546 switch (MI.getOpcode()) {
1547 default:
1548 return false;
1549 case AArch64::CATCHRET:
1550 case AArch64::CLEANUPRET:
1551 return true;
1552 }
1553 }
1554
emitEpilogue(MachineFunction & MF,MachineBasicBlock & MBB) const1555 void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
1556 MachineBasicBlock &MBB) const {
1557 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
1558 MachineFrameInfo &MFI = MF.getFrameInfo();
1559 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1560 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1561 DebugLoc DL;
1562 bool NeedsWinCFI = needsWinCFI(MF);
1563 bool HasWinCFI = false;
1564 bool IsFunclet = false;
1565 auto WinCFI = make_scope_exit([&]() {
1566 if (!MF.hasWinCFI())
1567 MF.setHasWinCFI(HasWinCFI);
1568 });
1569
1570 if (MBB.end() != MBBI) {
1571 DL = MBBI->getDebugLoc();
1572 IsFunclet = isFuncletReturnInstr(*MBBI);
1573 }
1574
1575 int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1576 : MFI.getStackSize();
1577 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1578
1579 // All calls are tail calls in GHC calling conv, and functions have no
1580 // prologue/epilogue.
1581 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1582 return;
1583
1584 // Initial and residual are named for consistency with the prologue. Note that
1585 // in the epilogue, the residual adjustment is executed first.
1586 uint64_t ArgumentPopSize = getArgumentPopSize(MF, MBB);
1587
1588 // The stack frame should be like below,
1589 //
1590 // ---------------------- ---
1591 // | | |
1592 // | BytesInStackArgArea| CalleeArgStackSize
1593 // | (NumReusableBytes) | (of tail call)
1594 // | | ---
1595 // | | |
1596 // ---------------------| --- |
1597 // | | | |
1598 // | CalleeSavedReg | | |
1599 // | (CalleeSavedStackSize)| | |
1600 // | | | |
1601 // ---------------------| | NumBytes
1602 // | | StackSize (StackAdjustUp)
1603 // | LocalStackSize | | |
1604 // | (covering callee | | |
1605 // | args) | | |
1606 // | | | |
1607 // ---------------------- --- ---
1608 //
1609 // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
1610 // = StackSize + ArgumentPopSize
1611 //
1612 // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
1613 // it as the 2nd argument of AArch64ISD::TC_RETURN.
1614
1615 auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
1616
1617 bool IsWin64 =
1618 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1619 unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1620
1621 uint64_t AfterCSRPopSize = ArgumentPopSize;
1622 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1623 // We cannot rely on the local stack size set in emitPrologue if the function
1624 // has funclets, as funclets have different local stack size requirements, and
1625 // the current value set in emitPrologue may be that of the containing
1626 // function.
1627 if (MF.hasEHFunclets())
1628 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1629 bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
1630 // Assume we can't combine the last pop with the sp restore.
1631
1632 if (!CombineSPBump && PrologueSaveSize != 0) {
1633 MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
1634 while (AArch64InstrInfo::isSEHInstruction(*Pop))
1635 Pop = std::prev(Pop);
1636 // Converting the last ldp to a post-index ldp is valid only if the last
1637 // ldp's offset is 0.
1638 const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
1639 // If the offset is 0, convert it to a post-index ldp.
1640 if (OffsetOp.getImm() == 0)
1641 convertCalleeSaveRestoreToSPPrePostIncDec(
1642 MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, false);
1643 else {
1644 // If not, make sure to emit an add after the last ldp.
1645 // We're doing this by transfering the size to be restored from the
1646 // adjustment *before* the CSR pops to the adjustment *after* the CSR
1647 // pops.
1648 AfterCSRPopSize += PrologueSaveSize;
1649 }
1650 }
1651
1652 // Move past the restores of the callee-saved registers.
1653 // If we plan on combining the sp bump of the local stack size and the callee
1654 // save stack size, we might need to adjust the CSR save and restore offsets.
1655 MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
1656 MachineBasicBlock::iterator Begin = MBB.begin();
1657 while (LastPopI != Begin) {
1658 --LastPopI;
1659 if (!LastPopI->getFlag(MachineInstr::FrameDestroy) ||
1660 IsSVECalleeSave(LastPopI)) {
1661 ++LastPopI;
1662 break;
1663 } else if (CombineSPBump)
1664 fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(),
1665 NeedsWinCFI, &HasWinCFI);
1666 }
1667
1668 if (NeedsWinCFI) {
1669 HasWinCFI = true;
1670 BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
1671 .setMIFlag(MachineInstr::FrameDestroy);
1672 }
1673
1674 const StackOffset &SVEStackSize = getSVEStackSize(MF);
1675
1676 // If there is a single SP update, insert it before the ret and we're done.
1677 if (CombineSPBump) {
1678 assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1679 emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
1680 {NumBytes + (int64_t)AfterCSRPopSize, MVT::i8}, TII,
1681 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1682 if (NeedsWinCFI && HasWinCFI)
1683 BuildMI(MBB, MBB.getFirstTerminator(), DL,
1684 TII->get(AArch64::SEH_EpilogEnd))
1685 .setMIFlag(MachineInstr::FrameDestroy);
1686 return;
1687 }
1688
1689 NumBytes -= PrologueSaveSize;
1690 assert(NumBytes >= 0 && "Negative stack allocation size!?");
1691
1692 // Process the SVE callee-saves to determine what space needs to be
1693 // deallocated.
1694 StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
1695 MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
1696 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
1697 RestoreBegin = std::prev(RestoreEnd);
1698 while (RestoreBegin != MBB.begin() &&
1699 IsSVECalleeSave(std::prev(RestoreBegin)))
1700 --RestoreBegin;
1701
1702 assert(IsSVECalleeSave(RestoreBegin) &&
1703 IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
1704
1705 StackOffset CalleeSavedSizeAsOffset = {CalleeSavedSize, MVT::nxv1i8};
1706 DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
1707 DeallocateAfter = CalleeSavedSizeAsOffset;
1708 }
1709
1710 // Deallocate the SVE area.
1711 if (SVEStackSize) {
1712 if (AFI->isStackRealigned()) {
1713 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize())
1714 // Set SP to start of SVE callee-save area from which they can
1715 // be reloaded. The code below will deallocate the stack space
1716 // space by moving FP -> SP.
1717 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
1718 {-CalleeSavedSize, MVT::nxv1i8}, TII,
1719 MachineInstr::FrameDestroy);
1720 } else {
1721 if (AFI->getSVECalleeSavedStackSize()) {
1722 // Deallocate the non-SVE locals first before we can deallocate (and
1723 // restore callee saves) from the SVE area.
1724 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
1725 {NumBytes, MVT::i8}, TII, MachineInstr::FrameDestroy);
1726 NumBytes = 0;
1727 }
1728
1729 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
1730 DeallocateBefore, TII, MachineInstr::FrameDestroy);
1731
1732 emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
1733 DeallocateAfter, TII, MachineInstr::FrameDestroy);
1734 }
1735 }
1736
1737 if (!hasFP(MF)) {
1738 bool RedZone = canUseRedZone(MF);
1739 // If this was a redzone leaf function, we don't need to restore the
1740 // stack pointer (but we may need to pop stack args for fastcc).
1741 if (RedZone && AfterCSRPopSize == 0)
1742 return;
1743
1744 bool NoCalleeSaveRestore = PrologueSaveSize == 0;
1745 int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
1746 if (NoCalleeSaveRestore)
1747 StackRestoreBytes += AfterCSRPopSize;
1748
1749 // If we were able to combine the local stack pop with the argument pop,
1750 // then we're done.
1751 bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
1752
1753 // If we're done after this, make sure to help the load store optimizer.
1754 if (Done)
1755 adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
1756
1757 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1758 {StackRestoreBytes, MVT::i8}, TII,
1759 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1760 if (Done) {
1761 if (NeedsWinCFI) {
1762 HasWinCFI = true;
1763 BuildMI(MBB, MBB.getFirstTerminator(), DL,
1764 TII->get(AArch64::SEH_EpilogEnd))
1765 .setMIFlag(MachineInstr::FrameDestroy);
1766 }
1767 return;
1768 }
1769
1770 NumBytes = 0;
1771 }
1772
1773 // Restore the original stack pointer.
1774 // FIXME: Rather than doing the math here, we should instead just use
1775 // non-post-indexed loads for the restores if we aren't actually going to
1776 // be able to save any instructions.
1777 if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
1778 int64_t OffsetToFrameRecord =
1779 isTargetDarwin(MF) ? (-(int64_t)AFI->getCalleeSavedStackSize() + 16) : 0;
1780 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
1781 {OffsetToFrameRecord, MVT::i8},
1782 TII, MachineInstr::FrameDestroy, false, NeedsWinCFI);
1783 } else if (NumBytes)
1784 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1785 {NumBytes, MVT::i8}, TII, MachineInstr::FrameDestroy, false,
1786 NeedsWinCFI);
1787
1788 // This must be placed after the callee-save restore code because that code
1789 // assumes the SP is at the same location as it was after the callee-save save
1790 // code in the prologue.
1791 if (AfterCSRPopSize) {
1792 // Find an insertion point for the first ldp so that it goes before the
1793 // shadow call stack epilog instruction. This ensures that the restore of
1794 // lr from x18 is placed after the restore from sp.
1795 auto FirstSPPopI = MBB.getFirstTerminator();
1796 while (FirstSPPopI != Begin) {
1797 auto Prev = std::prev(FirstSPPopI);
1798 if (Prev->getOpcode() != AArch64::LDRXpre ||
1799 Prev->getOperand(0).getReg() == AArch64::SP)
1800 break;
1801 FirstSPPopI = Prev;
1802 }
1803
1804 adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
1805
1806 emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
1807 {(int64_t)AfterCSRPopSize, MVT::i8}, TII,
1808 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1809 }
1810 if (NeedsWinCFI && HasWinCFI)
1811 BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
1812 .setMIFlag(MachineInstr::FrameDestroy);
1813
1814 MF.setHasWinCFI(HasWinCFI);
1815 }
1816
1817 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1818 /// debug info. It's the same as what we use for resolving the code-gen
1819 /// references for now. FIXME: This can go wrong when references are
1820 /// SP-relative and simple call frames aren't used.
getFrameIndexReference(const MachineFunction & MF,int FI,Register & FrameReg) const1821 int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
1822 int FI,
1823 Register &FrameReg) const {
1824 return resolveFrameIndexReference(
1825 MF, FI, FrameReg,
1826 /*PreferFP=*/
1827 MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress),
1828 /*ForSimm=*/false)
1829 .getBytes();
1830 }
1831
getNonLocalFrameIndexReference(const MachineFunction & MF,int FI) const1832 int AArch64FrameLowering::getNonLocalFrameIndexReference(
1833 const MachineFunction &MF, int FI) const {
1834 return getSEHFrameIndexOffset(MF, FI);
1835 }
1836
getFPOffset(const MachineFunction & MF,int64_t ObjectOffset)1837 static StackOffset getFPOffset(const MachineFunction &MF, int64_t ObjectOffset) {
1838 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1839 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1840 bool IsWin64 =
1841 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1842
1843 unsigned FixedObject =
1844 getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
1845 unsigned FPAdjust = isTargetDarwin(MF)
1846 ? 16 : AFI->getCalleeSavedStackSize(MF.getFrameInfo());
1847 return {ObjectOffset + FixedObject + FPAdjust, MVT::i8};
1848 }
1849
getStackOffset(const MachineFunction & MF,int64_t ObjectOffset)1850 static StackOffset getStackOffset(const MachineFunction &MF, int64_t ObjectOffset) {
1851 const auto &MFI = MF.getFrameInfo();
1852 return {ObjectOffset + (int64_t)MFI.getStackSize(), MVT::i8};
1853 }
1854
getSEHFrameIndexOffset(const MachineFunction & MF,int FI) const1855 int AArch64FrameLowering::getSEHFrameIndexOffset(const MachineFunction &MF,
1856 int FI) const {
1857 const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
1858 MF.getSubtarget().getRegisterInfo());
1859 int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI);
1860 return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
1861 ? getFPOffset(MF, ObjectOffset).getBytes()
1862 : getStackOffset(MF, ObjectOffset).getBytes();
1863 }
1864
resolveFrameIndexReference(const MachineFunction & MF,int FI,Register & FrameReg,bool PreferFP,bool ForSimm) const1865 StackOffset AArch64FrameLowering::resolveFrameIndexReference(
1866 const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP,
1867 bool ForSimm) const {
1868 const auto &MFI = MF.getFrameInfo();
1869 int64_t ObjectOffset = MFI.getObjectOffset(FI);
1870 bool isFixed = MFI.isFixedObjectIndex(FI);
1871 bool isSVE = MFI.getStackID(FI) == TargetStackID::SVEVector;
1872 return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
1873 PreferFP, ForSimm);
1874 }
1875
resolveFrameOffsetReference(const MachineFunction & MF,int64_t ObjectOffset,bool isFixed,bool isSVE,Register & FrameReg,bool PreferFP,bool ForSimm) const1876 StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
1877 const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
1878 Register &FrameReg, bool PreferFP, bool ForSimm) const {
1879 const auto &MFI = MF.getFrameInfo();
1880 const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
1881 MF.getSubtarget().getRegisterInfo());
1882 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1883 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1884
1885 int64_t FPOffset = getFPOffset(MF, ObjectOffset).getBytes();
1886 int64_t Offset = getStackOffset(MF, ObjectOffset).getBytes();
1887 bool isCSR =
1888 !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
1889
1890 const StackOffset &SVEStackSize = getSVEStackSize(MF);
1891
1892 // Use frame pointer to reference fixed objects. Use it for locals if
1893 // there are VLAs or a dynamically realigned SP (and thus the SP isn't
1894 // reliable as a base). Make sure useFPForScavengingIndex() does the
1895 // right thing for the emergency spill slot.
1896 bool UseFP = false;
1897 if (AFI->hasStackFrame() && !isSVE) {
1898 // We shouldn't prefer using the FP when there is an SVE area
1899 // in between the FP and the non-SVE locals/spills.
1900 PreferFP &= !SVEStackSize;
1901
1902 // Note: Keeping the following as multiple 'if' statements rather than
1903 // merging to a single expression for readability.
1904 //
1905 // Argument access should always use the FP.
1906 if (isFixed) {
1907 UseFP = hasFP(MF);
1908 } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
1909 // References to the CSR area must use FP if we're re-aligning the stack
1910 // since the dynamically-sized alignment padding is between the SP/BP and
1911 // the CSR area.
1912 assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1913 UseFP = true;
1914 } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
1915 // If the FPOffset is negative and we're producing a signed immediate, we
1916 // have to keep in mind that the available offset range for negative
1917 // offsets is smaller than for positive ones. If an offset is available
1918 // via the FP and the SP, use whichever is closest.
1919 bool FPOffsetFits = !ForSimm || FPOffset >= -256;
1920 PreferFP |= Offset > -FPOffset;
1921
1922 if (MFI.hasVarSizedObjects()) {
1923 // If we have variable sized objects, we can use either FP or BP, as the
1924 // SP offset is unknown. We can use the base pointer if we have one and
1925 // FP is not preferred. If not, we're stuck with using FP.
1926 bool CanUseBP = RegInfo->hasBasePointer(MF);
1927 if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
1928 UseFP = PreferFP;
1929 else if (!CanUseBP) // Can't use BP. Forced to use FP.
1930 UseFP = true;
1931 // else we can use BP and FP, but the offset from FP won't fit.
1932 // That will make us scavenge registers which we can probably avoid by
1933 // using BP. If it won't fit for BP either, we'll scavenge anyway.
1934 } else if (FPOffset >= 0) {
1935 // Use SP or FP, whichever gives us the best chance of the offset
1936 // being in range for direct access. If the FPOffset is positive,
1937 // that'll always be best, as the SP will be even further away.
1938 UseFP = true;
1939 } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
1940 // Funclets access the locals contained in the parent's stack frame
1941 // via the frame pointer, so we have to use the FP in the parent
1942 // function.
1943 (void) Subtarget;
1944 assert(
1945 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
1946 "Funclets should only be present on Win64");
1947 UseFP = true;
1948 } else {
1949 // We have the choice between FP and (SP or BP).
1950 if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
1951 UseFP = true;
1952 }
1953 }
1954 }
1955
1956 assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
1957 "In the presence of dynamic stack pointer realignment, "
1958 "non-argument/CSR objects cannot be accessed through the frame pointer");
1959
1960 if (isSVE) {
1961 int64_t OffsetToSVEArea =
1962 MFI.getStackSize() - AFI->getCalleeSavedStackSize();
1963 StackOffset FPOffset = {ObjectOffset, MVT::nxv1i8};
1964 StackOffset SPOffset = SVEStackSize +
1965 StackOffset(ObjectOffset, MVT::nxv1i8) +
1966 StackOffset(OffsetToSVEArea, MVT::i8);
1967 // Always use the FP for SVE spills if available and beneficial.
1968 if (hasFP(MF) &&
1969 (SPOffset.getBytes() ||
1970 FPOffset.getScalableBytes() < SPOffset.getScalableBytes() ||
1971 RegInfo->needsStackRealignment(MF))) {
1972 FrameReg = RegInfo->getFrameRegister(MF);
1973 return FPOffset;
1974 }
1975
1976 FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
1977 : (unsigned)AArch64::SP;
1978 return SPOffset;
1979 }
1980
1981 StackOffset ScalableOffset = {};
1982 if (UseFP && !(isFixed || isCSR))
1983 ScalableOffset = -SVEStackSize;
1984 if (!UseFP && (isFixed || isCSR))
1985 ScalableOffset = SVEStackSize;
1986
1987 if (UseFP) {
1988 FrameReg = RegInfo->getFrameRegister(MF);
1989 return StackOffset(FPOffset, MVT::i8) + ScalableOffset;
1990 }
1991
1992 // Use the base pointer if we have one.
1993 if (RegInfo->hasBasePointer(MF))
1994 FrameReg = RegInfo->getBaseRegister();
1995 else {
1996 assert(!MFI.hasVarSizedObjects() &&
1997 "Can't use SP when we have var sized objects.");
1998 FrameReg = AArch64::SP;
1999 // If we're using the red zone for this function, the SP won't actually
2000 // be adjusted, so the offsets will be negative. They're also all
2001 // within range of the signed 9-bit immediate instructions.
2002 if (canUseRedZone(MF))
2003 Offset -= AFI->getLocalStackSize();
2004 }
2005
2006 return StackOffset(Offset, MVT::i8) + ScalableOffset;
2007 }
2008
getPrologueDeath(MachineFunction & MF,unsigned Reg)2009 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
2010 // Do not set a kill flag on values that are also marked as live-in. This
2011 // happens with the @llvm-returnaddress intrinsic and with arguments passed in
2012 // callee saved registers.
2013 // Omitting the kill flags is conservatively correct even if the live-in
2014 // is not used after all.
2015 bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
2016 return getKillRegState(!IsLiveIn);
2017 }
2018
produceCompactUnwindFrame(MachineFunction & MF)2019 static bool produceCompactUnwindFrame(MachineFunction &MF) {
2020 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2021 AttributeList Attrs = MF.getFunction().getAttributes();
2022 return Subtarget.isTargetMachO() &&
2023 !(Subtarget.getTargetLowering()->supportSwiftError() &&
2024 Attrs.hasAttrSomewhere(Attribute::SwiftError));
2025 }
2026
invalidateWindowsRegisterPairing(unsigned Reg1,unsigned Reg2,bool NeedsWinCFI)2027 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
2028 bool NeedsWinCFI) {
2029 // If we are generating register pairs for a Windows function that requires
2030 // EH support, then pair consecutive registers only. There are no unwind
2031 // opcodes for saves/restores of non-consectuve register pairs.
2032 // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x.
2033 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
2034
2035 // TODO: LR can be paired with any register. We don't support this yet in
2036 // the MCLayer. We need to add support for the save_lrpair unwind code.
2037 if (Reg2 == AArch64::FP)
2038 return true;
2039 if (!NeedsWinCFI)
2040 return false;
2041 if (Reg2 == Reg1 + 1)
2042 return false;
2043 return true;
2044 }
2045
2046 /// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
2047 /// WindowsCFI requires that only consecutive registers can be paired.
2048 /// LR and FP need to be allocated together when the frame needs to save
2049 /// the frame-record. This means any other register pairing with LR is invalid.
invalidateRegisterPairing(unsigned Reg1,unsigned Reg2,bool UsesWinAAPCS,bool NeedsWinCFI,bool NeedsFrameRecord)2050 static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2,
2051 bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord) {
2052 if (UsesWinAAPCS)
2053 return invalidateWindowsRegisterPairing(Reg1, Reg2, NeedsWinCFI);
2054
2055 // If we need to store the frame record, don't pair any register
2056 // with LR other than FP.
2057 if (NeedsFrameRecord)
2058 return Reg2 == AArch64::LR;
2059
2060 return false;
2061 }
2062
2063 namespace {
2064
2065 struct RegPairInfo {
2066 unsigned Reg1 = AArch64::NoRegister;
2067 unsigned Reg2 = AArch64::NoRegister;
2068 int FrameIdx;
2069 int Offset;
2070 enum RegType { GPR, FPR64, FPR128, PPR, ZPR } Type;
2071
2072 RegPairInfo() = default;
2073
isPaired__anon3b2f32c10411::RegPairInfo2074 bool isPaired() const { return Reg2 != AArch64::NoRegister; }
2075
getScale__anon3b2f32c10411::RegPairInfo2076 unsigned getScale() const {
2077 switch (Type) {
2078 case PPR:
2079 return 2;
2080 case GPR:
2081 case FPR64:
2082 return 8;
2083 case ZPR:
2084 case FPR128:
2085 return 16;
2086 }
2087 llvm_unreachable("Unsupported type");
2088 }
2089
isScalable__anon3b2f32c10411::RegPairInfo2090 bool isScalable() const { return Type == PPR || Type == ZPR; }
2091 };
2092
2093 } // end anonymous namespace
2094
computeCalleeSaveRegisterPairs(MachineFunction & MF,ArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI,SmallVectorImpl<RegPairInfo> & RegPairs,bool & NeedShadowCallStackProlog,bool NeedsFrameRecord)2095 static void computeCalleeSaveRegisterPairs(
2096 MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI,
2097 const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
2098 bool &NeedShadowCallStackProlog, bool NeedsFrameRecord) {
2099
2100 if (CSI.empty())
2101 return;
2102
2103 bool IsWindows = isTargetWindows(MF);
2104 bool NeedsWinCFI = needsWinCFI(MF);
2105 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2106 MachineFrameInfo &MFI = MF.getFrameInfo();
2107 CallingConv::ID CC = MF.getFunction().getCallingConv();
2108 unsigned Count = CSI.size();
2109 (void)CC;
2110 // MachO's compact unwind format relies on all registers being stored in
2111 // pairs.
2112 assert((!produceCompactUnwindFrame(MF) ||
2113 CC == CallingConv::PreserveMost ||
2114 (Count & 1) == 0) &&
2115 "Odd number of callee-saved regs to spill!");
2116 int ByteOffset = AFI->getCalleeSavedStackSize();
2117 int ScalableByteOffset = AFI->getSVECalleeSavedStackSize();
2118 // On Linux, we will have either one or zero non-paired register. On Windows
2119 // with CFI, we can have multiple unpaired registers in order to utilize the
2120 // available unwind codes. This flag assures that the alignment fixup is done
2121 // only once, as intened.
2122 bool FixupDone = false;
2123
2124 for (unsigned i = 0; i < Count; ++i) {
2125 RegPairInfo RPI;
2126 RPI.Reg1 = CSI[i].getReg();
2127
2128 if (AArch64::GPR64RegClass.contains(RPI.Reg1))
2129 RPI.Type = RegPairInfo::GPR;
2130 else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
2131 RPI.Type = RegPairInfo::FPR64;
2132 else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
2133 RPI.Type = RegPairInfo::FPR128;
2134 else if (AArch64::ZPRRegClass.contains(RPI.Reg1))
2135 RPI.Type = RegPairInfo::ZPR;
2136 else if (AArch64::PPRRegClass.contains(RPI.Reg1))
2137 RPI.Type = RegPairInfo::PPR;
2138 else
2139 llvm_unreachable("Unsupported register class.");
2140
2141 // Add the next reg to the pair if it is in the same register class.
2142 if (i + 1 < Count) {
2143 unsigned NextReg = CSI[i + 1].getReg();
2144 switch (RPI.Type) {
2145 case RegPairInfo::GPR:
2146 if (AArch64::GPR64RegClass.contains(NextReg) &&
2147 !invalidateRegisterPairing(RPI.Reg1, NextReg, IsWindows, NeedsWinCFI,
2148 NeedsFrameRecord))
2149 RPI.Reg2 = NextReg;
2150 break;
2151 case RegPairInfo::FPR64:
2152 if (AArch64::FPR64RegClass.contains(NextReg) &&
2153 !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI))
2154 RPI.Reg2 = NextReg;
2155 break;
2156 case RegPairInfo::FPR128:
2157 if (AArch64::FPR128RegClass.contains(NextReg))
2158 RPI.Reg2 = NextReg;
2159 break;
2160 case RegPairInfo::PPR:
2161 case RegPairInfo::ZPR:
2162 break;
2163 }
2164 }
2165
2166 // If either of the registers to be saved is the lr register, it means that
2167 // we also need to save lr in the shadow call stack.
2168 if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
2169 MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
2170 if (!MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(18))
2171 report_fatal_error("Must reserve x18 to use shadow call stack");
2172 NeedShadowCallStackProlog = true;
2173 }
2174
2175 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
2176 // list to come in sorted by frame index so that we can issue the store
2177 // pair instructions directly. Assert if we see anything otherwise.
2178 //
2179 // The order of the registers in the list is controlled by
2180 // getCalleeSavedRegs(), so they will always be in-order, as well.
2181 assert((!RPI.isPaired() ||
2182 (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
2183 "Out of order callee saved regs!");
2184
2185 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
2186 RPI.Reg1 == AArch64::LR) &&
2187 "FrameRecord must be allocated together with LR");
2188
2189 // Windows AAPCS has FP and LR reversed.
2190 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
2191 RPI.Reg2 == AArch64::LR) &&
2192 "FrameRecord must be allocated together with LR");
2193
2194 // MachO's compact unwind format relies on all registers being stored in
2195 // adjacent register pairs.
2196 assert((!produceCompactUnwindFrame(MF) ||
2197 CC == CallingConv::PreserveMost ||
2198 (RPI.isPaired() &&
2199 ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
2200 RPI.Reg1 + 1 == RPI.Reg2))) &&
2201 "Callee-save registers not saved as adjacent register pair!");
2202
2203 RPI.FrameIdx = CSI[i].getFrameIdx();
2204
2205 int Scale = RPI.getScale();
2206 if (RPI.isScalable())
2207 ScalableByteOffset -= Scale;
2208 else
2209 ByteOffset -= RPI.isPaired() ? 2 * Scale : Scale;
2210
2211 assert(!(RPI.isScalable() && RPI.isPaired()) &&
2212 "Paired spill/fill instructions don't exist for SVE vectors");
2213
2214 // Round up size of non-pair to pair size if we need to pad the
2215 // callee-save area to ensure 16-byte alignment.
2216 if (AFI->hasCalleeSaveStackFreeSpace() && !FixupDone &&
2217 !RPI.isScalable() && RPI.Type != RegPairInfo::FPR128 &&
2218 !RPI.isPaired()) {
2219 FixupDone = true;
2220 ByteOffset -= 8;
2221 assert(ByteOffset % 16 == 0);
2222 assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
2223 MFI.setObjectAlignment(RPI.FrameIdx, Align(16));
2224 }
2225
2226 int Offset = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2227 assert(Offset % Scale == 0);
2228 RPI.Offset = Offset / Scale;
2229
2230 assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
2231 (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
2232 "Offset out of bounds for LDP/STP immediate");
2233
2234 RegPairs.push_back(RPI);
2235 if (RPI.isPaired())
2236 ++i;
2237 }
2238 }
2239
spillCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,ArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const2240 bool AArch64FrameLowering::spillCalleeSavedRegisters(
2241 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2242 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2243 MachineFunction &MF = *MBB.getParent();
2244 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2245 bool NeedsWinCFI = needsWinCFI(MF);
2246 DebugLoc DL;
2247 SmallVector<RegPairInfo, 8> RegPairs;
2248
2249 bool NeedShadowCallStackProlog = false;
2250 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
2251 NeedShadowCallStackProlog, hasFP(MF));
2252 const MachineRegisterInfo &MRI = MF.getRegInfo();
2253
2254 if (NeedShadowCallStackProlog) {
2255 // Shadow call stack prolog: str x30, [x18], #8
2256 BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
2257 .addReg(AArch64::X18, RegState::Define)
2258 .addReg(AArch64::LR)
2259 .addReg(AArch64::X18)
2260 .addImm(8)
2261 .setMIFlag(MachineInstr::FrameSetup);
2262
2263 if (NeedsWinCFI)
2264 BuildMI(MBB, MI, DL, TII.get(AArch64::SEH_Nop))
2265 .setMIFlag(MachineInstr::FrameSetup);
2266
2267 if (!MF.getFunction().hasFnAttribute(Attribute::NoUnwind)) {
2268 // Emit a CFI instruction that causes 8 to be subtracted from the value of
2269 // x18 when unwinding past this frame.
2270 static const char CFIInst[] = {
2271 dwarf::DW_CFA_val_expression,
2272 18, // register
2273 2, // length
2274 static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
2275 static_cast<char>(-8) & 0x7f, // addend (sleb128)
2276 };
2277 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
2278 nullptr, StringRef(CFIInst, sizeof(CFIInst))));
2279 BuildMI(MBB, MI, DL, TII.get(AArch64::CFI_INSTRUCTION))
2280 .addCFIIndex(CFIIndex)
2281 .setMIFlag(MachineInstr::FrameSetup);
2282 }
2283
2284 // This instruction also makes x18 live-in to the entry block.
2285 MBB.addLiveIn(AArch64::X18);
2286 }
2287
2288 for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
2289 ++RPII) {
2290 RegPairInfo RPI = *RPII;
2291 unsigned Reg1 = RPI.Reg1;
2292 unsigned Reg2 = RPI.Reg2;
2293 unsigned StrOpc;
2294
2295 // Issue sequence of spills for cs regs. The first spill may be converted
2296 // to a pre-decrement store later by emitPrologue if the callee-save stack
2297 // area allocation can't be combined with the local stack area allocation.
2298 // For example:
2299 // stp x22, x21, [sp, #0] // addImm(+0)
2300 // stp x20, x19, [sp, #16] // addImm(+2)
2301 // stp fp, lr, [sp, #32] // addImm(+4)
2302 // Rationale: This sequence saves uop updates compared to a sequence of
2303 // pre-increment spills like stp xi,xj,[sp,#-16]!
2304 // Note: Similar rationale and sequence for restores in epilog.
2305 unsigned Size;
2306 Align Alignment;
2307 switch (RPI.Type) {
2308 case RegPairInfo::GPR:
2309 StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
2310 Size = 8;
2311 Alignment = Align(8);
2312 break;
2313 case RegPairInfo::FPR64:
2314 StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
2315 Size = 8;
2316 Alignment = Align(8);
2317 break;
2318 case RegPairInfo::FPR128:
2319 StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
2320 Size = 16;
2321 Alignment = Align(16);
2322 break;
2323 case RegPairInfo::ZPR:
2324 StrOpc = AArch64::STR_ZXI;
2325 Size = 16;
2326 Alignment = Align(16);
2327 break;
2328 case RegPairInfo::PPR:
2329 StrOpc = AArch64::STR_PXI;
2330 Size = 2;
2331 Alignment = Align(2);
2332 break;
2333 }
2334 LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
2335 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2336 dbgs() << ") -> fi#(" << RPI.FrameIdx;
2337 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2338 dbgs() << ")\n");
2339
2340 assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
2341 "Windows unwdinding requires a consecutive (FP,LR) pair");
2342 // Windows unwind codes require consecutive registers if registers are
2343 // paired. Make the switch here, so that the code below will save (x,x+1)
2344 // and not (x+1,x).
2345 unsigned FrameIdxReg1 = RPI.FrameIdx;
2346 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2347 if (NeedsWinCFI && RPI.isPaired()) {
2348 std::swap(Reg1, Reg2);
2349 std::swap(FrameIdxReg1, FrameIdxReg2);
2350 }
2351 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
2352 if (!MRI.isReserved(Reg1))
2353 MBB.addLiveIn(Reg1);
2354 if (RPI.isPaired()) {
2355 if (!MRI.isReserved(Reg2))
2356 MBB.addLiveIn(Reg2);
2357 MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
2358 MIB.addMemOperand(MF.getMachineMemOperand(
2359 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2360 MachineMemOperand::MOStore, Size, Alignment));
2361 }
2362 MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
2363 .addReg(AArch64::SP)
2364 .addImm(RPI.Offset) // [sp, #offset*scale],
2365 // where factor*scale is implicit
2366 .setMIFlag(MachineInstr::FrameSetup);
2367 MIB.addMemOperand(MF.getMachineMemOperand(
2368 MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2369 MachineMemOperand::MOStore, Size, Alignment));
2370 if (NeedsWinCFI)
2371 InsertSEH(MIB, TII, MachineInstr::FrameSetup);
2372
2373 // Update the StackIDs of the SVE stack slots.
2374 MachineFrameInfo &MFI = MF.getFrameInfo();
2375 if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR)
2376 MFI.setStackID(RPI.FrameIdx, TargetStackID::SVEVector);
2377
2378 }
2379 return true;
2380 }
2381
restoreCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,MutableArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const2382 bool AArch64FrameLowering::restoreCalleeSavedRegisters(
2383 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2384 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2385 MachineFunction &MF = *MBB.getParent();
2386 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2387 DebugLoc DL;
2388 SmallVector<RegPairInfo, 8> RegPairs;
2389 bool NeedsWinCFI = needsWinCFI(MF);
2390
2391 if (MI != MBB.end())
2392 DL = MI->getDebugLoc();
2393
2394 bool NeedShadowCallStackProlog = false;
2395 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
2396 NeedShadowCallStackProlog, hasFP(MF));
2397
2398 auto EmitMI = [&](const RegPairInfo &RPI) {
2399 unsigned Reg1 = RPI.Reg1;
2400 unsigned Reg2 = RPI.Reg2;
2401
2402 // Issue sequence of restores for cs regs. The last restore may be converted
2403 // to a post-increment load later by emitEpilogue if the callee-save stack
2404 // area allocation can't be combined with the local stack area allocation.
2405 // For example:
2406 // ldp fp, lr, [sp, #32] // addImm(+4)
2407 // ldp x20, x19, [sp, #16] // addImm(+2)
2408 // ldp x22, x21, [sp, #0] // addImm(+0)
2409 // Note: see comment in spillCalleeSavedRegisters()
2410 unsigned LdrOpc;
2411 unsigned Size;
2412 Align Alignment;
2413 switch (RPI.Type) {
2414 case RegPairInfo::GPR:
2415 LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
2416 Size = 8;
2417 Alignment = Align(8);
2418 break;
2419 case RegPairInfo::FPR64:
2420 LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
2421 Size = 8;
2422 Alignment = Align(8);
2423 break;
2424 case RegPairInfo::FPR128:
2425 LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
2426 Size = 16;
2427 Alignment = Align(16);
2428 break;
2429 case RegPairInfo::ZPR:
2430 LdrOpc = AArch64::LDR_ZXI;
2431 Size = 16;
2432 Alignment = Align(16);
2433 break;
2434 case RegPairInfo::PPR:
2435 LdrOpc = AArch64::LDR_PXI;
2436 Size = 2;
2437 Alignment = Align(2);
2438 break;
2439 }
2440 LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
2441 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2442 dbgs() << ") -> fi#(" << RPI.FrameIdx;
2443 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2444 dbgs() << ")\n");
2445
2446 // Windows unwind codes require consecutive registers if registers are
2447 // paired. Make the switch here, so that the code below will save (x,x+1)
2448 // and not (x+1,x).
2449 unsigned FrameIdxReg1 = RPI.FrameIdx;
2450 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2451 if (NeedsWinCFI && RPI.isPaired()) {
2452 std::swap(Reg1, Reg2);
2453 std::swap(FrameIdxReg1, FrameIdxReg2);
2454 }
2455 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
2456 if (RPI.isPaired()) {
2457 MIB.addReg(Reg2, getDefRegState(true));
2458 MIB.addMemOperand(MF.getMachineMemOperand(
2459 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2460 MachineMemOperand::MOLoad, Size, Alignment));
2461 }
2462 MIB.addReg(Reg1, getDefRegState(true))
2463 .addReg(AArch64::SP)
2464 .addImm(RPI.Offset) // [sp, #offset*scale]
2465 // where factor*scale is implicit
2466 .setMIFlag(MachineInstr::FrameDestroy);
2467 MIB.addMemOperand(MF.getMachineMemOperand(
2468 MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2469 MachineMemOperand::MOLoad, Size, Alignment));
2470 if (NeedsWinCFI)
2471 InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
2472 };
2473
2474 // SVE objects are always restored in reverse order.
2475 for (const RegPairInfo &RPI : reverse(RegPairs))
2476 if (RPI.isScalable())
2477 EmitMI(RPI);
2478
2479 if (ReverseCSRRestoreSeq) {
2480 for (const RegPairInfo &RPI : reverse(RegPairs))
2481 if (!RPI.isScalable())
2482 EmitMI(RPI);
2483 } else
2484 for (const RegPairInfo &RPI : RegPairs)
2485 if (!RPI.isScalable())
2486 EmitMI(RPI);
2487
2488 if (NeedShadowCallStackProlog) {
2489 // Shadow call stack epilog: ldr x30, [x18, #-8]!
2490 BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
2491 .addReg(AArch64::X18, RegState::Define)
2492 .addReg(AArch64::LR, RegState::Define)
2493 .addReg(AArch64::X18)
2494 .addImm(-8)
2495 .setMIFlag(MachineInstr::FrameDestroy);
2496 }
2497
2498 return true;
2499 }
2500
determineCalleeSaves(MachineFunction & MF,BitVector & SavedRegs,RegScavenger * RS) const2501 void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
2502 BitVector &SavedRegs,
2503 RegScavenger *RS) const {
2504 // All calls are tail calls in GHC calling conv, and functions have no
2505 // prologue/epilogue.
2506 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
2507 return;
2508
2509 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2510 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
2511 MF.getSubtarget().getRegisterInfo());
2512 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2513 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2514 unsigned UnspilledCSGPR = AArch64::NoRegister;
2515 unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
2516
2517 MachineFrameInfo &MFI = MF.getFrameInfo();
2518 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
2519
2520 unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
2521 ? RegInfo->getBaseRegister()
2522 : (unsigned)AArch64::NoRegister;
2523
2524 unsigned ExtraCSSpill = 0;
2525 // Figure out which callee-saved registers to save/restore.
2526 for (unsigned i = 0; CSRegs[i]; ++i) {
2527 const unsigned Reg = CSRegs[i];
2528
2529 // Add the base pointer register to SavedRegs if it is callee-save.
2530 if (Reg == BasePointerReg)
2531 SavedRegs.set(Reg);
2532
2533 bool RegUsed = SavedRegs.test(Reg);
2534 unsigned PairedReg = AArch64::NoRegister;
2535 if (AArch64::GPR64RegClass.contains(Reg) ||
2536 AArch64::FPR64RegClass.contains(Reg) ||
2537 AArch64::FPR128RegClass.contains(Reg))
2538 PairedReg = CSRegs[i ^ 1];
2539
2540 if (!RegUsed) {
2541 if (AArch64::GPR64RegClass.contains(Reg) &&
2542 !RegInfo->isReservedReg(MF, Reg)) {
2543 UnspilledCSGPR = Reg;
2544 UnspilledCSGPRPaired = PairedReg;
2545 }
2546 continue;
2547 }
2548
2549 // MachO's compact unwind format relies on all registers being stored in
2550 // pairs.
2551 // FIXME: the usual format is actually better if unwinding isn't needed.
2552 if (produceCompactUnwindFrame(MF) && PairedReg != AArch64::NoRegister &&
2553 !SavedRegs.test(PairedReg)) {
2554 SavedRegs.set(PairedReg);
2555 if (AArch64::GPR64RegClass.contains(PairedReg) &&
2556 !RegInfo->isReservedReg(MF, PairedReg))
2557 ExtraCSSpill = PairedReg;
2558 }
2559 }
2560
2561 if (MF.getFunction().getCallingConv() == CallingConv::Win64 &&
2562 !Subtarget.isTargetWindows()) {
2563 // For Windows calling convention on a non-windows OS, where X18 is treated
2564 // as reserved, back up X18 when entering non-windows code (marked with the
2565 // Windows calling convention) and restore when returning regardless of
2566 // whether the individual function uses it - it might call other functions
2567 // that clobber it.
2568 SavedRegs.set(AArch64::X18);
2569 }
2570
2571 // Calculates the callee saved stack size.
2572 unsigned CSStackSize = 0;
2573 unsigned SVECSStackSize = 0;
2574 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2575 const MachineRegisterInfo &MRI = MF.getRegInfo();
2576 for (unsigned Reg : SavedRegs.set_bits()) {
2577 auto RegSize = TRI->getRegSizeInBits(Reg, MRI) / 8;
2578 if (AArch64::PPRRegClass.contains(Reg) ||
2579 AArch64::ZPRRegClass.contains(Reg))
2580 SVECSStackSize += RegSize;
2581 else
2582 CSStackSize += RegSize;
2583 }
2584
2585 // Save number of saved regs, so we can easily update CSStackSize later.
2586 unsigned NumSavedRegs = SavedRegs.count();
2587
2588 // The frame record needs to be created by saving the appropriate registers
2589 uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
2590 if (hasFP(MF) ||
2591 windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
2592 SavedRegs.set(AArch64::FP);
2593 SavedRegs.set(AArch64::LR);
2594 }
2595
2596 LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
2597 for (unsigned Reg
2598 : SavedRegs.set_bits()) dbgs()
2599 << ' ' << printReg(Reg, RegInfo);
2600 dbgs() << "\n";);
2601
2602 // If any callee-saved registers are used, the frame cannot be eliminated.
2603 int64_t SVEStackSize =
2604 alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
2605 bool CanEliminateFrame = (SavedRegs.count() == 0) && !SVEStackSize;
2606
2607 // The CSR spill slots have not been allocated yet, so estimateStackSize
2608 // won't include them.
2609 unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
2610
2611 // Conservatively always assume BigStack when there are SVE spills.
2612 bool BigStack = SVEStackSize ||
2613 (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
2614 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
2615 AFI->setHasStackFrame(true);
2616
2617 // Estimate if we might need to scavenge a register at some point in order
2618 // to materialize a stack offset. If so, either spill one additional
2619 // callee-saved register or reserve a special spill slot to facilitate
2620 // register scavenging. If we already spilled an extra callee-saved register
2621 // above to keep the number of spills even, we don't need to do anything else
2622 // here.
2623 if (BigStack) {
2624 if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
2625 LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
2626 << " to get a scratch register.\n");
2627 SavedRegs.set(UnspilledCSGPR);
2628 // MachO's compact unwind format relies on all registers being stored in
2629 // pairs, so if we need to spill one extra for BigStack, then we need to
2630 // store the pair.
2631 if (produceCompactUnwindFrame(MF))
2632 SavedRegs.set(UnspilledCSGPRPaired);
2633 ExtraCSSpill = UnspilledCSGPR;
2634 }
2635
2636 // If we didn't find an extra callee-saved register to spill, create
2637 // an emergency spill slot.
2638 if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
2639 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2640 const TargetRegisterClass &RC = AArch64::GPR64RegClass;
2641 unsigned Size = TRI->getSpillSize(RC);
2642 Align Alignment = TRI->getSpillAlign(RC);
2643 int FI = MFI.CreateStackObject(Size, Alignment, false);
2644 RS->addScavengingFrameIndex(FI);
2645 LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
2646 << " as the emergency spill slot.\n");
2647 }
2648 }
2649
2650 // Adding the size of additional 64bit GPR saves.
2651 CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
2652 uint64_t AlignedCSStackSize = alignTo(CSStackSize, 16);
2653 LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
2654 << EstimatedStackSize + AlignedCSStackSize
2655 << " bytes.\n");
2656
2657 assert((!MFI.isCalleeSavedInfoValid() ||
2658 AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
2659 "Should not invalidate callee saved info");
2660
2661 // Round up to register pair alignment to avoid additional SP adjustment
2662 // instructions.
2663 AFI->setCalleeSavedStackSize(AlignedCSStackSize);
2664 AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
2665 AFI->setSVECalleeSavedStackSize(alignTo(SVECSStackSize, 16));
2666 }
2667
enableStackSlotScavenging(const MachineFunction & MF) const2668 bool AArch64FrameLowering::enableStackSlotScavenging(
2669 const MachineFunction &MF) const {
2670 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2671 return AFI->hasCalleeSaveStackFreeSpace();
2672 }
2673
2674 /// returns true if there are any SVE callee saves.
getSVECalleeSaveSlotRange(const MachineFrameInfo & MFI,int & Min,int & Max)2675 static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI,
2676 int &Min, int &Max) {
2677 Min = std::numeric_limits<int>::max();
2678 Max = std::numeric_limits<int>::min();
2679
2680 if (!MFI.isCalleeSavedInfoValid())
2681 return false;
2682
2683 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
2684 for (auto &CS : CSI) {
2685 if (AArch64::ZPRRegClass.contains(CS.getReg()) ||
2686 AArch64::PPRRegClass.contains(CS.getReg())) {
2687 assert((Max == std::numeric_limits<int>::min() ||
2688 Max + 1 == CS.getFrameIdx()) &&
2689 "SVE CalleeSaves are not consecutive");
2690
2691 Min = std::min(Min, CS.getFrameIdx());
2692 Max = std::max(Max, CS.getFrameIdx());
2693 }
2694 }
2695 return Min != std::numeric_limits<int>::max();
2696 }
2697
2698 // Process all the SVE stack objects and determine offsets for each
2699 // object. If AssignOffsets is true, the offsets get assigned.
2700 // Fills in the first and last callee-saved frame indices into
2701 // Min/MaxCSFrameIndex, respectively.
2702 // Returns the size of the stack.
determineSVEStackObjectOffsets(MachineFrameInfo & MFI,int & MinCSFrameIndex,int & MaxCSFrameIndex,bool AssignOffsets)2703 static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
2704 int &MinCSFrameIndex,
2705 int &MaxCSFrameIndex,
2706 bool AssignOffsets) {
2707 #ifndef NDEBUG
2708 // First process all fixed stack objects.
2709 for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
2710 assert(MFI.getStackID(I) != TargetStackID::SVEVector &&
2711 "SVE vectors should never be passed on the stack by value, only by "
2712 "reference.");
2713 #endif
2714
2715 auto Assign = [&MFI](int FI, int64_t Offset) {
2716 LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
2717 MFI.setObjectOffset(FI, Offset);
2718 };
2719
2720 int64_t Offset = 0;
2721
2722 // Then process all callee saved slots.
2723 if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
2724 // Assign offsets to the callee save slots.
2725 for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
2726 Offset += MFI.getObjectSize(I);
2727 Offset = alignTo(Offset, MFI.getObjectAlign(I));
2728 if (AssignOffsets)
2729 Assign(I, -Offset);
2730 }
2731 }
2732
2733 // Ensure that the Callee-save area is aligned to 16bytes.
2734 Offset = alignTo(Offset, Align(16U));
2735
2736 // Create a buffer of SVE objects to allocate and sort it.
2737 SmallVector<int, 8> ObjectsToAllocate;
2738 for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
2739 unsigned StackID = MFI.getStackID(I);
2740 if (StackID != TargetStackID::SVEVector)
2741 continue;
2742 if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex)
2743 continue;
2744 if (MFI.isDeadObjectIndex(I))
2745 continue;
2746
2747 ObjectsToAllocate.push_back(I);
2748 }
2749
2750 // Allocate all SVE locals and spills
2751 for (unsigned FI : ObjectsToAllocate) {
2752 Align Alignment = MFI.getObjectAlign(FI);
2753 // FIXME: Given that the length of SVE vectors is not necessarily a power of
2754 // two, we'd need to align every object dynamically at runtime if the
2755 // alignment is larger than 16. This is not yet supported.
2756 if (Alignment > Align(16))
2757 report_fatal_error(
2758 "Alignment of scalable vectors > 16 bytes is not yet supported");
2759
2760 Offset = alignTo(Offset + MFI.getObjectSize(FI), Alignment);
2761 if (AssignOffsets)
2762 Assign(FI, -Offset);
2763 }
2764
2765 return Offset;
2766 }
2767
estimateSVEStackObjectOffsets(MachineFrameInfo & MFI) const2768 int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
2769 MachineFrameInfo &MFI) const {
2770 int MinCSFrameIndex, MaxCSFrameIndex;
2771 return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex, false);
2772 }
2773
assignSVEStackObjectOffsets(MachineFrameInfo & MFI,int & MinCSFrameIndex,int & MaxCSFrameIndex) const2774 int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
2775 MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex) const {
2776 return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex,
2777 true);
2778 }
2779
processFunctionBeforeFrameFinalized(MachineFunction & MF,RegScavenger * RS) const2780 void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
2781 MachineFunction &MF, RegScavenger *RS) const {
2782 MachineFrameInfo &MFI = MF.getFrameInfo();
2783
2784 assert(getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown &&
2785 "Upwards growing stack unsupported");
2786
2787 int MinCSFrameIndex, MaxCSFrameIndex;
2788 int64_t SVEStackSize =
2789 assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
2790
2791 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2792 AFI->setStackSizeSVE(alignTo(SVEStackSize, 16U));
2793 AFI->setMinMaxSVECSFrameIndex(MinCSFrameIndex, MaxCSFrameIndex);
2794
2795 // If this function isn't doing Win64-style C++ EH, we don't need to do
2796 // anything.
2797 if (!MF.hasEHFunclets())
2798 return;
2799 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2800 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
2801
2802 MachineBasicBlock &MBB = MF.front();
2803 auto MBBI = MBB.begin();
2804 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
2805 ++MBBI;
2806
2807 // Create an UnwindHelp object.
2808 // The UnwindHelp object is allocated at the start of the fixed object area
2809 int64_t FixedObject =
2810 getFixedObjectSize(MF, AFI, /*IsWin64*/ true, /*IsFunclet*/ false);
2811 int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8,
2812 /*SPOffset*/ -FixedObject,
2813 /*IsImmutable=*/false);
2814 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
2815
2816 // We need to store -2 into the UnwindHelp object at the start of the
2817 // function.
2818 DebugLoc DL;
2819 RS->enterBasicBlockEnd(MBB);
2820 RS->backward(std::prev(MBBI));
2821 unsigned DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
2822 assert(DstReg && "There must be a free register after frame setup");
2823 BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
2824 BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
2825 .addReg(DstReg, getKillRegState(true))
2826 .addFrameIndex(UnwindHelpFI)
2827 .addImm(0);
2828 }
2829
2830 namespace {
2831 struct TagStoreInstr {
2832 MachineInstr *MI;
2833 int64_t Offset, Size;
TagStoreInstr__anon3b2f32c10711::TagStoreInstr2834 explicit TagStoreInstr(MachineInstr *MI, int64_t Offset, int64_t Size)
2835 : MI(MI), Offset(Offset), Size(Size) {}
2836 };
2837
2838 class TagStoreEdit {
2839 MachineFunction *MF;
2840 MachineBasicBlock *MBB;
2841 MachineRegisterInfo *MRI;
2842 // Tag store instructions that are being replaced.
2843 SmallVector<TagStoreInstr, 8> TagStores;
2844 // Combined memref arguments of the above instructions.
2845 SmallVector<MachineMemOperand *, 8> CombinedMemRefs;
2846
2847 // Replace allocation tags in [FrameReg + FrameRegOffset, FrameReg +
2848 // FrameRegOffset + Size) with the address tag of SP.
2849 Register FrameReg;
2850 StackOffset FrameRegOffset;
2851 int64_t Size;
2852 // If not None, move FrameReg to (FrameReg + FrameRegUpdate) at the end.
2853 Optional<int64_t> FrameRegUpdate;
2854 // MIFlags for any FrameReg updating instructions.
2855 unsigned FrameRegUpdateFlags;
2856
2857 // Use zeroing instruction variants.
2858 bool ZeroData;
2859 DebugLoc DL;
2860
2861 void emitUnrolled(MachineBasicBlock::iterator InsertI);
2862 void emitLoop(MachineBasicBlock::iterator InsertI);
2863
2864 public:
TagStoreEdit(MachineBasicBlock * MBB,bool ZeroData)2865 TagStoreEdit(MachineBasicBlock *MBB, bool ZeroData)
2866 : MBB(MBB), ZeroData(ZeroData) {
2867 MF = MBB->getParent();
2868 MRI = &MF->getRegInfo();
2869 }
2870 // Add an instruction to be replaced. Instructions must be added in the
2871 // ascending order of Offset, and have to be adjacent.
addInstruction(TagStoreInstr I)2872 void addInstruction(TagStoreInstr I) {
2873 assert((TagStores.empty() ||
2874 TagStores.back().Offset + TagStores.back().Size == I.Offset) &&
2875 "Non-adjacent tag store instructions.");
2876 TagStores.push_back(I);
2877 }
clear()2878 void clear() { TagStores.clear(); }
2879 // Emit equivalent code at the given location, and erase the current set of
2880 // instructions. May skip if the replacement is not profitable. May invalidate
2881 // the input iterator and replace it with a valid one.
2882 void emitCode(MachineBasicBlock::iterator &InsertI,
2883 const AArch64FrameLowering *TFI, bool IsLast);
2884 };
2885
emitUnrolled(MachineBasicBlock::iterator InsertI)2886 void TagStoreEdit::emitUnrolled(MachineBasicBlock::iterator InsertI) {
2887 const AArch64InstrInfo *TII =
2888 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
2889
2890 const int64_t kMinOffset = -256 * 16;
2891 const int64_t kMaxOffset = 255 * 16;
2892
2893 Register BaseReg = FrameReg;
2894 int64_t BaseRegOffsetBytes = FrameRegOffset.getBytes();
2895 if (BaseRegOffsetBytes < kMinOffset ||
2896 BaseRegOffsetBytes + (Size - Size % 32) > kMaxOffset) {
2897 Register ScratchReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
2898 emitFrameOffset(*MBB, InsertI, DL, ScratchReg, BaseReg,
2899 {BaseRegOffsetBytes, MVT::i8}, TII);
2900 BaseReg = ScratchReg;
2901 BaseRegOffsetBytes = 0;
2902 }
2903
2904 MachineInstr *LastI = nullptr;
2905 while (Size) {
2906 int64_t InstrSize = (Size > 16) ? 32 : 16;
2907 unsigned Opcode =
2908 InstrSize == 16
2909 ? (ZeroData ? AArch64::STZGOffset : AArch64::STGOffset)
2910 : (ZeroData ? AArch64::STZ2GOffset : AArch64::ST2GOffset);
2911 MachineInstr *I = BuildMI(*MBB, InsertI, DL, TII->get(Opcode))
2912 .addReg(AArch64::SP)
2913 .addReg(BaseReg)
2914 .addImm(BaseRegOffsetBytes / 16)
2915 .setMemRefs(CombinedMemRefs);
2916 // A store to [BaseReg, #0] should go last for an opportunity to fold the
2917 // final SP adjustment in the epilogue.
2918 if (BaseRegOffsetBytes == 0)
2919 LastI = I;
2920 BaseRegOffsetBytes += InstrSize;
2921 Size -= InstrSize;
2922 }
2923
2924 if (LastI)
2925 MBB->splice(InsertI, MBB, LastI);
2926 }
2927
emitLoop(MachineBasicBlock::iterator InsertI)2928 void TagStoreEdit::emitLoop(MachineBasicBlock::iterator InsertI) {
2929 const AArch64InstrInfo *TII =
2930 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
2931
2932 Register BaseReg = FrameRegUpdate
2933 ? FrameReg
2934 : MRI->createVirtualRegister(&AArch64::GPR64RegClass);
2935 Register SizeReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
2936
2937 emitFrameOffset(*MBB, InsertI, DL, BaseReg, FrameReg, FrameRegOffset, TII);
2938
2939 int64_t LoopSize = Size;
2940 // If the loop size is not a multiple of 32, split off one 16-byte store at
2941 // the end to fold BaseReg update into.
2942 if (FrameRegUpdate && *FrameRegUpdate)
2943 LoopSize -= LoopSize % 32;
2944 MachineInstr *LoopI = BuildMI(*MBB, InsertI, DL,
2945 TII->get(ZeroData ? AArch64::STZGloop_wback
2946 : AArch64::STGloop_wback))
2947 .addDef(SizeReg)
2948 .addDef(BaseReg)
2949 .addImm(LoopSize)
2950 .addReg(BaseReg)
2951 .setMemRefs(CombinedMemRefs);
2952 if (FrameRegUpdate)
2953 LoopI->setFlags(FrameRegUpdateFlags);
2954
2955 int64_t ExtraBaseRegUpdate =
2956 FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.getBytes() - Size) : 0;
2957 if (LoopSize < Size) {
2958 assert(FrameRegUpdate);
2959 assert(Size - LoopSize == 16);
2960 // Tag 16 more bytes at BaseReg and update BaseReg.
2961 BuildMI(*MBB, InsertI, DL,
2962 TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
2963 .addDef(BaseReg)
2964 .addReg(BaseReg)
2965 .addReg(BaseReg)
2966 .addImm(1 + ExtraBaseRegUpdate / 16)
2967 .setMemRefs(CombinedMemRefs)
2968 .setMIFlags(FrameRegUpdateFlags);
2969 } else if (ExtraBaseRegUpdate) {
2970 // Update BaseReg.
2971 BuildMI(
2972 *MBB, InsertI, DL,
2973 TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
2974 .addDef(BaseReg)
2975 .addReg(BaseReg)
2976 .addImm(std::abs(ExtraBaseRegUpdate))
2977 .addImm(0)
2978 .setMIFlags(FrameRegUpdateFlags);
2979 }
2980 }
2981
2982 // Check if *II is a register update that can be merged into STGloop that ends
2983 // at (Reg + Size). RemainingOffset is the required adjustment to Reg after the
2984 // end of the loop.
canMergeRegUpdate(MachineBasicBlock::iterator II,unsigned Reg,int64_t Size,int64_t * TotalOffset)2985 bool canMergeRegUpdate(MachineBasicBlock::iterator II, unsigned Reg,
2986 int64_t Size, int64_t *TotalOffset) {
2987 MachineInstr &MI = *II;
2988 if ((MI.getOpcode() == AArch64::ADDXri ||
2989 MI.getOpcode() == AArch64::SUBXri) &&
2990 MI.getOperand(0).getReg() == Reg && MI.getOperand(1).getReg() == Reg) {
2991 unsigned Shift = AArch64_AM::getShiftValue(MI.getOperand(3).getImm());
2992 int64_t Offset = MI.getOperand(2).getImm() << Shift;
2993 if (MI.getOpcode() == AArch64::SUBXri)
2994 Offset = -Offset;
2995 int64_t AbsPostOffset = std::abs(Offset - Size);
2996 const int64_t kMaxOffset =
2997 0xFFF; // Max encoding for unshifted ADDXri / SUBXri
2998 if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
2999 *TotalOffset = Offset;
3000 return true;
3001 }
3002 }
3003 return false;
3004 }
3005
mergeMemRefs(const SmallVectorImpl<TagStoreInstr> & TSE,SmallVectorImpl<MachineMemOperand * > & MemRefs)3006 void mergeMemRefs(const SmallVectorImpl<TagStoreInstr> &TSE,
3007 SmallVectorImpl<MachineMemOperand *> &MemRefs) {
3008 MemRefs.clear();
3009 for (auto &TS : TSE) {
3010 MachineInstr *MI = TS.MI;
3011 // An instruction without memory operands may access anything. Be
3012 // conservative and return an empty list.
3013 if (MI->memoperands_empty()) {
3014 MemRefs.clear();
3015 return;
3016 }
3017 MemRefs.append(MI->memoperands_begin(), MI->memoperands_end());
3018 }
3019 }
3020
emitCode(MachineBasicBlock::iterator & InsertI,const AArch64FrameLowering * TFI,bool IsLast)3021 void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
3022 const AArch64FrameLowering *TFI, bool IsLast) {
3023 if (TagStores.empty())
3024 return;
3025 TagStoreInstr &FirstTagStore = TagStores[0];
3026 TagStoreInstr &LastTagStore = TagStores[TagStores.size() - 1];
3027 Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
3028 DL = TagStores[0].MI->getDebugLoc();
3029
3030 Register Reg;
3031 FrameRegOffset = TFI->resolveFrameOffsetReference(
3032 *MF, FirstTagStore.Offset, false /*isFixed*/, false /*isSVE*/, Reg,
3033 /*PreferFP=*/false, /*ForSimm=*/true);
3034 FrameReg = Reg;
3035 FrameRegUpdate = None;
3036
3037 mergeMemRefs(TagStores, CombinedMemRefs);
3038
3039 LLVM_DEBUG(dbgs() << "Replacing adjacent STG instructions:\n";
3040 for (const auto &Instr
3041 : TagStores) { dbgs() << " " << *Instr.MI; });
3042
3043 // Size threshold where a loop becomes shorter than a linear sequence of
3044 // tagging instructions.
3045 const int kSetTagLoopThreshold = 176;
3046 if (Size < kSetTagLoopThreshold) {
3047 if (TagStores.size() < 2)
3048 return;
3049 emitUnrolled(InsertI);
3050 } else {
3051 MachineInstr *UpdateInstr = nullptr;
3052 int64_t TotalOffset;
3053 if (IsLast) {
3054 // See if we can merge base register update into the STGloop.
3055 // This is done in AArch64LoadStoreOptimizer for "normal" stores,
3056 // but STGloop is way too unusual for that, and also it only
3057 // realistically happens in function epilogue. Also, STGloop is expanded
3058 // before that pass.
3059 if (InsertI != MBB->end() &&
3060 canMergeRegUpdate(InsertI, FrameReg, FrameRegOffset.getBytes() + Size,
3061 &TotalOffset)) {
3062 UpdateInstr = &*InsertI++;
3063 LLVM_DEBUG(dbgs() << "Folding SP update into loop:\n "
3064 << *UpdateInstr);
3065 }
3066 }
3067
3068 if (!UpdateInstr && TagStores.size() < 2)
3069 return;
3070
3071 if (UpdateInstr) {
3072 FrameRegUpdate = TotalOffset;
3073 FrameRegUpdateFlags = UpdateInstr->getFlags();
3074 }
3075 emitLoop(InsertI);
3076 if (UpdateInstr)
3077 UpdateInstr->eraseFromParent();
3078 }
3079
3080 for (auto &TS : TagStores)
3081 TS.MI->eraseFromParent();
3082 }
3083
isMergeableStackTaggingInstruction(MachineInstr & MI,int64_t & Offset,int64_t & Size,bool & ZeroData)3084 bool isMergeableStackTaggingInstruction(MachineInstr &MI, int64_t &Offset,
3085 int64_t &Size, bool &ZeroData) {
3086 MachineFunction &MF = *MI.getParent()->getParent();
3087 const MachineFrameInfo &MFI = MF.getFrameInfo();
3088
3089 unsigned Opcode = MI.getOpcode();
3090 ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGOffset ||
3091 Opcode == AArch64::STZ2GOffset);
3092
3093 if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
3094 if (!MI.getOperand(0).isDead() || !MI.getOperand(1).isDead())
3095 return false;
3096 if (!MI.getOperand(2).isImm() || !MI.getOperand(3).isFI())
3097 return false;
3098 Offset = MFI.getObjectOffset(MI.getOperand(3).getIndex());
3099 Size = MI.getOperand(2).getImm();
3100 return true;
3101 }
3102
3103 if (Opcode == AArch64::STGOffset || Opcode == AArch64::STZGOffset)
3104 Size = 16;
3105 else if (Opcode == AArch64::ST2GOffset || Opcode == AArch64::STZ2GOffset)
3106 Size = 32;
3107 else
3108 return false;
3109
3110 if (MI.getOperand(0).getReg() != AArch64::SP || !MI.getOperand(1).isFI())
3111 return false;
3112
3113 Offset = MFI.getObjectOffset(MI.getOperand(1).getIndex()) +
3114 16 * MI.getOperand(2).getImm();
3115 return true;
3116 }
3117
3118 // Detect a run of memory tagging instructions for adjacent stack frame slots,
3119 // and replace them with a shorter instruction sequence:
3120 // * replace STG + STG with ST2G
3121 // * replace STGloop + STGloop with STGloop
3122 // This code needs to run when stack slot offsets are already known, but before
3123 // FrameIndex operands in STG instructions are eliminated.
tryMergeAdjacentSTG(MachineBasicBlock::iterator II,const AArch64FrameLowering * TFI,RegScavenger * RS)3124 MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II,
3125 const AArch64FrameLowering *TFI,
3126 RegScavenger *RS) {
3127 bool FirstZeroData;
3128 int64_t Size, Offset;
3129 MachineInstr &MI = *II;
3130 MachineBasicBlock *MBB = MI.getParent();
3131 MachineBasicBlock::iterator NextI = ++II;
3132 if (&MI == &MBB->instr_back())
3133 return II;
3134 if (!isMergeableStackTaggingInstruction(MI, Offset, Size, FirstZeroData))
3135 return II;
3136
3137 SmallVector<TagStoreInstr, 4> Instrs;
3138 Instrs.emplace_back(&MI, Offset, Size);
3139
3140 constexpr int kScanLimit = 10;
3141 int Count = 0;
3142 for (MachineBasicBlock::iterator E = MBB->end();
3143 NextI != E && Count < kScanLimit; ++NextI) {
3144 MachineInstr &MI = *NextI;
3145 bool ZeroData;
3146 int64_t Size, Offset;
3147 // Collect instructions that update memory tags with a FrameIndex operand
3148 // and (when applicable) constant size, and whose output registers are dead
3149 // (the latter is almost always the case in practice). Since these
3150 // instructions effectively have no inputs or outputs, we are free to skip
3151 // any non-aliasing instructions in between without tracking used registers.
3152 if (isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData)) {
3153 if (ZeroData != FirstZeroData)
3154 break;
3155 Instrs.emplace_back(&MI, Offset, Size);
3156 continue;
3157 }
3158
3159 // Only count non-transient, non-tagging instructions toward the scan
3160 // limit.
3161 if (!MI.isTransient())
3162 ++Count;
3163
3164 // Just in case, stop before the epilogue code starts.
3165 if (MI.getFlag(MachineInstr::FrameSetup) ||
3166 MI.getFlag(MachineInstr::FrameDestroy))
3167 break;
3168
3169 // Reject anything that may alias the collected instructions.
3170 if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects())
3171 break;
3172 }
3173
3174 // New code will be inserted after the last tagging instruction we've found.
3175 MachineBasicBlock::iterator InsertI = Instrs.back().MI;
3176 InsertI++;
3177
3178 llvm::stable_sort(Instrs,
3179 [](const TagStoreInstr &Left, const TagStoreInstr &Right) {
3180 return Left.Offset < Right.Offset;
3181 });
3182
3183 // Make sure that we don't have any overlapping stores.
3184 int64_t CurOffset = Instrs[0].Offset;
3185 for (auto &Instr : Instrs) {
3186 if (CurOffset > Instr.Offset)
3187 return NextI;
3188 CurOffset = Instr.Offset + Instr.Size;
3189 }
3190
3191 // Find contiguous runs of tagged memory and emit shorter instruction
3192 // sequencies for them when possible.
3193 TagStoreEdit TSE(MBB, FirstZeroData);
3194 Optional<int64_t> EndOffset;
3195 for (auto &Instr : Instrs) {
3196 if (EndOffset && *EndOffset != Instr.Offset) {
3197 // Found a gap.
3198 TSE.emitCode(InsertI, TFI, /*IsLast = */ false);
3199 TSE.clear();
3200 }
3201
3202 TSE.addInstruction(Instr);
3203 EndOffset = Instr.Offset + Instr.Size;
3204 }
3205
3206 TSE.emitCode(InsertI, TFI, /*IsLast = */ true);
3207
3208 return InsertI;
3209 }
3210 } // namespace
3211
processFunctionBeforeFrameIndicesReplaced(MachineFunction & MF,RegScavenger * RS=nullptr) const3212 void AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced(
3213 MachineFunction &MF, RegScavenger *RS = nullptr) const {
3214 if (StackTaggingMergeSetTag)
3215 for (auto &BB : MF)
3216 for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();)
3217 II = tryMergeAdjacentSTG(II, this, RS);
3218 }
3219
3220 /// For Win64 AArch64 EH, the offset to the Unwind object is from the SP
3221 /// before the update. This is easily retrieved as it is exactly the offset
3222 /// that is set in processFunctionBeforeFrameFinalized.
getFrameIndexReferencePreferSP(const MachineFunction & MF,int FI,Register & FrameReg,bool IgnoreSPUpdates) const3223 int AArch64FrameLowering::getFrameIndexReferencePreferSP(
3224 const MachineFunction &MF, int FI, Register &FrameReg,
3225 bool IgnoreSPUpdates) const {
3226 const MachineFrameInfo &MFI = MF.getFrameInfo();
3227 if (IgnoreSPUpdates) {
3228 LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
3229 << MFI.getObjectOffset(FI) << "\n");
3230 FrameReg = AArch64::SP;
3231 return MFI.getObjectOffset(FI);
3232 }
3233
3234 return getFrameIndexReference(MF, FI, FrameReg);
3235 }
3236
3237 /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
3238 /// the parent's frame pointer
getWinEHParentFrameOffset(const MachineFunction & MF) const3239 unsigned AArch64FrameLowering::getWinEHParentFrameOffset(
3240 const MachineFunction &MF) const {
3241 return 0;
3242 }
3243
3244 /// Funclets only need to account for space for the callee saved registers,
3245 /// as the locals are accounted for in the parent's stack frame.
getWinEHFuncletFrameSize(const MachineFunction & MF) const3246 unsigned AArch64FrameLowering::getWinEHFuncletFrameSize(
3247 const MachineFunction &MF) const {
3248 // This is the size of the pushed CSRs.
3249 unsigned CSSize =
3250 MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
3251 // This is the amount of stack a funclet needs to allocate.
3252 return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
3253 getStackAlign());
3254 }
3255