10b57cec5SDimitry Andric //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // This file contains the AArch64 implementation of TargetFrameLowering class.
100b57cec5SDimitry Andric //
110b57cec5SDimitry Andric // On AArch64, stack frames are structured as follows:
120b57cec5SDimitry Andric //
130b57cec5SDimitry Andric // The stack grows downward.
140b57cec5SDimitry Andric //
150b57cec5SDimitry Andric // All of the individual frame areas on the frame below are optional, i.e. it's
160b57cec5SDimitry Andric // possible to create a function so that the particular area isn't present
170b57cec5SDimitry Andric // in the frame.
180b57cec5SDimitry Andric //
190b57cec5SDimitry Andric // At function entry, the "frame" looks as follows:
200b57cec5SDimitry Andric //
210b57cec5SDimitry Andric // | | Higher address
220b57cec5SDimitry Andric // |-----------------------------------|
230b57cec5SDimitry Andric // | |
240b57cec5SDimitry Andric // | arguments passed on the stack |
250b57cec5SDimitry Andric // | |
260b57cec5SDimitry Andric // |-----------------------------------| <- sp
270b57cec5SDimitry Andric // | | Lower address
280b57cec5SDimitry Andric //
290b57cec5SDimitry Andric //
300b57cec5SDimitry Andric // After the prologue has run, the frame has the following general structure.
310b57cec5SDimitry Andric // Note that this doesn't depict the case where a red-zone is used. Also,
320b57cec5SDimitry Andric // technically the last frame area (VLAs) doesn't get created until in the
330b57cec5SDimitry Andric // main function body, after the prologue is run. However, it's depicted here
340b57cec5SDimitry Andric // for completeness.
350b57cec5SDimitry Andric //
360b57cec5SDimitry Andric // | | Higher address
370b57cec5SDimitry Andric // |-----------------------------------|
380b57cec5SDimitry Andric // | |
390b57cec5SDimitry Andric // | arguments passed on the stack |
400b57cec5SDimitry Andric // | |
410b57cec5SDimitry Andric // |-----------------------------------|
420b57cec5SDimitry Andric // | |
430b57cec5SDimitry Andric // | (Win64 only) varargs from reg |
440b57cec5SDimitry Andric // | |
450b57cec5SDimitry Andric // |-----------------------------------|
460b57cec5SDimitry Andric // | |
478bcb0991SDimitry Andric // | callee-saved gpr registers | <--.
488bcb0991SDimitry Andric // | | | On Darwin platforms these
498bcb0991SDimitry Andric // |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
50fe6060f1SDimitry Andric // | prev_lr | | (frame record first)
51fe6060f1SDimitry Andric // | prev_fp | <--'
52fe6060f1SDimitry Andric // | async context if needed |
530b57cec5SDimitry Andric // | (a.k.a. "frame record") |
540b57cec5SDimitry Andric // |-----------------------------------| <- fp(=x29)
550b57cec5SDimitry Andric // | |
568bcb0991SDimitry Andric // | callee-saved fp/simd/SVE regs |
578bcb0991SDimitry Andric // | |
588bcb0991SDimitry Andric // |-----------------------------------|
598bcb0991SDimitry Andric // | |
608bcb0991SDimitry Andric // | SVE stack objects |
610b57cec5SDimitry Andric // | |
620b57cec5SDimitry Andric // |-----------------------------------|
630b57cec5SDimitry Andric // |.empty.space.to.make.part.below....|
640b57cec5SDimitry Andric // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
650b57cec5SDimitry Andric // |.the.standard.16-byte.alignment....| compile time; if present)
660b57cec5SDimitry Andric // |-----------------------------------|
670b57cec5SDimitry Andric // | |
680b57cec5SDimitry Andric // | local variables of fixed size |
690b57cec5SDimitry Andric // | including spill slots |
700b57cec5SDimitry Andric // |-----------------------------------| <- bp(not defined by ABI,
710b57cec5SDimitry Andric // |.variable-sized.local.variables....| LLVM chooses X19)
720b57cec5SDimitry Andric // |.(VLAs)............................| (size of this area is unknown at
730b57cec5SDimitry Andric // |...................................| compile time)
740b57cec5SDimitry Andric // |-----------------------------------| <- sp
750b57cec5SDimitry Andric // | | Lower address
760b57cec5SDimitry Andric //
770b57cec5SDimitry Andric //
780b57cec5SDimitry Andric // To access the data in a frame, at-compile time, a constant offset must be
790b57cec5SDimitry Andric // computable from one of the pointers (fp, bp, sp) to access it. The size
800b57cec5SDimitry Andric // of the areas with a dotted background cannot be computed at compile-time
810b57cec5SDimitry Andric // if they are present, making it required to have all three of fp, bp and
820b57cec5SDimitry Andric // sp to be set up to be able to access all contents in the frame areas,
830b57cec5SDimitry Andric // assuming all of the frame areas are non-empty.
840b57cec5SDimitry Andric //
850b57cec5SDimitry Andric // For most functions, some of the frame areas are empty. For those functions,
860b57cec5SDimitry Andric // it may not be necessary to set up fp or bp:
870b57cec5SDimitry Andric // * A base pointer is definitely needed when there are both VLAs and local
880b57cec5SDimitry Andric // variables with more-than-default alignment requirements.
890b57cec5SDimitry Andric // * A frame pointer is definitely needed when there are local variables with
900b57cec5SDimitry Andric // more-than-default alignment requirements.
910b57cec5SDimitry Andric //
928bcb0991SDimitry Andric // For Darwin platforms the frame-record (fp, lr) is stored at the top of the
938bcb0991SDimitry Andric // callee-saved area, since the unwind encoding does not allow for encoding
948bcb0991SDimitry Andric // this dynamically and existing tools depend on this layout. For other
958bcb0991SDimitry Andric // platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
968bcb0991SDimitry Andric // area to allow SVE stack objects (allocated directly below the callee-saves,
978bcb0991SDimitry Andric // if available) to be accessed directly from the framepointer.
988bcb0991SDimitry Andric // The SVE spill/fill instructions have VL-scaled addressing modes such
998bcb0991SDimitry Andric // as:
1008bcb0991SDimitry Andric // ldr z8, [fp, #-7 mul vl]
1018bcb0991SDimitry Andric // For SVE the size of the vector length (VL) is not known at compile-time, so
1028bcb0991SDimitry Andric // '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
1038bcb0991SDimitry Andric // layout, we don't need to add an unscaled offset to the framepointer before
1048bcb0991SDimitry Andric // accessing the SVE object in the frame.
1058bcb0991SDimitry Andric //
1060b57cec5SDimitry Andric // In some cases when a base pointer is not strictly needed, it is generated
1070b57cec5SDimitry Andric // anyway when offsets from the frame pointer to access local variables become
1080b57cec5SDimitry Andric // so large that the offset can't be encoded in the immediate fields of loads
1090b57cec5SDimitry Andric // or stores.
1100b57cec5SDimitry Andric //
111fe6060f1SDimitry Andric // Outgoing function arguments must be at the bottom of the stack frame when
112fe6060f1SDimitry Andric // calling another function. If we do not have variable-sized stack objects, we
113fe6060f1SDimitry Andric // can allocate a "reserved call frame" area at the bottom of the local
114fe6060f1SDimitry Andric // variable area, large enough for all outgoing calls. If we do have VLAs, then
115fe6060f1SDimitry Andric // the stack pointer must be decremented and incremented around each call to
116fe6060f1SDimitry Andric // make space for the arguments below the VLAs.
117fe6060f1SDimitry Andric //
1180b57cec5SDimitry Andric // FIXME: also explain the redzone concept.
1190b57cec5SDimitry Andric //
12081ad6265SDimitry Andric // An example of the prologue:
12181ad6265SDimitry Andric //
12281ad6265SDimitry Andric // .globl __foo
12381ad6265SDimitry Andric // .align 2
12481ad6265SDimitry Andric // __foo:
12581ad6265SDimitry Andric // Ltmp0:
12681ad6265SDimitry Andric // .cfi_startproc
12781ad6265SDimitry Andric // .cfi_personality 155, ___gxx_personality_v0
12881ad6265SDimitry Andric // Leh_func_begin:
12981ad6265SDimitry Andric // .cfi_lsda 16, Lexception33
13081ad6265SDimitry Andric //
13181ad6265SDimitry Andric // stp xa,bx, [sp, -#offset]!
13281ad6265SDimitry Andric // ...
13381ad6265SDimitry Andric // stp x28, x27, [sp, #offset-32]
13481ad6265SDimitry Andric // stp fp, lr, [sp, #offset-16]
13581ad6265SDimitry Andric // add fp, sp, #offset - 16
13681ad6265SDimitry Andric // sub sp, sp, #1360
13781ad6265SDimitry Andric //
13881ad6265SDimitry Andric // The Stack:
13981ad6265SDimitry Andric // +-------------------------------------------+
14081ad6265SDimitry Andric // 10000 | ........ | ........ | ........ | ........ |
14181ad6265SDimitry Andric // 10004 | ........ | ........ | ........ | ........ |
14281ad6265SDimitry Andric // +-------------------------------------------+
14381ad6265SDimitry Andric // 10008 | ........ | ........ | ........ | ........ |
14481ad6265SDimitry Andric // 1000c | ........ | ........ | ........ | ........ |
14581ad6265SDimitry Andric // +===========================================+
14681ad6265SDimitry Andric // 10010 | X28 Register |
14781ad6265SDimitry Andric // 10014 | X28 Register |
14881ad6265SDimitry Andric // +-------------------------------------------+
14981ad6265SDimitry Andric // 10018 | X27 Register |
15081ad6265SDimitry Andric // 1001c | X27 Register |
15181ad6265SDimitry Andric // +===========================================+
15281ad6265SDimitry Andric // 10020 | Frame Pointer |
15381ad6265SDimitry Andric // 10024 | Frame Pointer |
15481ad6265SDimitry Andric // +-------------------------------------------+
15581ad6265SDimitry Andric // 10028 | Link Register |
15681ad6265SDimitry Andric // 1002c | Link Register |
15781ad6265SDimitry Andric // +===========================================+
15881ad6265SDimitry Andric // 10030 | ........ | ........ | ........ | ........ |
15981ad6265SDimitry Andric // 10034 | ........ | ........ | ........ | ........ |
16081ad6265SDimitry Andric // +-------------------------------------------+
16181ad6265SDimitry Andric // 10038 | ........ | ........ | ........ | ........ |
16281ad6265SDimitry Andric // 1003c | ........ | ........ | ........ | ........ |
16381ad6265SDimitry Andric // +-------------------------------------------+
16481ad6265SDimitry Andric //
16581ad6265SDimitry Andric // [sp] = 10030 :: >>initial value<<
16681ad6265SDimitry Andric // sp = 10020 :: stp fp, lr, [sp, #-16]!
16781ad6265SDimitry Andric // fp = sp == 10020 :: mov fp, sp
16881ad6265SDimitry Andric // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
16981ad6265SDimitry Andric // sp == 10010 :: >>final value<<
17081ad6265SDimitry Andric //
17181ad6265SDimitry Andric // The frame pointer (w29) points to address 10020. If we use an offset of
17281ad6265SDimitry Andric // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
17381ad6265SDimitry Andric // for w27, and -32 for w28:
17481ad6265SDimitry Andric //
17581ad6265SDimitry Andric // Ltmp1:
17681ad6265SDimitry Andric // .cfi_def_cfa w29, 16
17781ad6265SDimitry Andric // Ltmp2:
17881ad6265SDimitry Andric // .cfi_offset w30, -8
17981ad6265SDimitry Andric // Ltmp3:
18081ad6265SDimitry Andric // .cfi_offset w29, -16
18181ad6265SDimitry Andric // Ltmp4:
18281ad6265SDimitry Andric // .cfi_offset w27, -24
18381ad6265SDimitry Andric // Ltmp5:
18481ad6265SDimitry Andric // .cfi_offset w28, -32
18581ad6265SDimitry Andric //
1860b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
1870b57cec5SDimitry Andric
1880b57cec5SDimitry Andric #include "AArch64FrameLowering.h"
1890b57cec5SDimitry Andric #include "AArch64InstrInfo.h"
1900b57cec5SDimitry Andric #include "AArch64MachineFunctionInfo.h"
1910b57cec5SDimitry Andric #include "AArch64RegisterInfo.h"
1920b57cec5SDimitry Andric #include "AArch64Subtarget.h"
1930b57cec5SDimitry Andric #include "AArch64TargetMachine.h"
1940b57cec5SDimitry Andric #include "MCTargetDesc/AArch64AddressingModes.h"
19581ad6265SDimitry Andric #include "MCTargetDesc/AArch64MCTargetDesc.h"
1960b57cec5SDimitry Andric #include "llvm/ADT/ScopeExit.h"
1970b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h"
1980b57cec5SDimitry Andric #include "llvm/ADT/Statistic.h"
1990b57cec5SDimitry Andric #include "llvm/CodeGen/LivePhysRegs.h"
2000b57cec5SDimitry Andric #include "llvm/CodeGen/MachineBasicBlock.h"
2010b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFrameInfo.h"
2020b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunction.h"
2030b57cec5SDimitry Andric #include "llvm/CodeGen/MachineInstr.h"
2040b57cec5SDimitry Andric #include "llvm/CodeGen/MachineInstrBuilder.h"
2050b57cec5SDimitry Andric #include "llvm/CodeGen/MachineMemOperand.h"
2060b57cec5SDimitry Andric #include "llvm/CodeGen/MachineModuleInfo.h"
2070b57cec5SDimitry Andric #include "llvm/CodeGen/MachineOperand.h"
2080b57cec5SDimitry Andric #include "llvm/CodeGen/MachineRegisterInfo.h"
2090b57cec5SDimitry Andric #include "llvm/CodeGen/RegisterScavenging.h"
2100b57cec5SDimitry Andric #include "llvm/CodeGen/TargetInstrInfo.h"
2110b57cec5SDimitry Andric #include "llvm/CodeGen/TargetRegisterInfo.h"
2120b57cec5SDimitry Andric #include "llvm/CodeGen/TargetSubtargetInfo.h"
2130b57cec5SDimitry Andric #include "llvm/CodeGen/WinEHFuncInfo.h"
2140b57cec5SDimitry Andric #include "llvm/IR/Attributes.h"
2150b57cec5SDimitry Andric #include "llvm/IR/CallingConv.h"
2160b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h"
2170b57cec5SDimitry Andric #include "llvm/IR/DebugLoc.h"
2180b57cec5SDimitry Andric #include "llvm/IR/Function.h"
2190b57cec5SDimitry Andric #include "llvm/MC/MCAsmInfo.h"
2200b57cec5SDimitry Andric #include "llvm/MC/MCDwarf.h"
2210b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h"
2220b57cec5SDimitry Andric #include "llvm/Support/Debug.h"
2230b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h"
2240b57cec5SDimitry Andric #include "llvm/Support/MathExtras.h"
2250b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h"
2260b57cec5SDimitry Andric #include "llvm/Target/TargetMachine.h"
2270b57cec5SDimitry Andric #include "llvm/Target/TargetOptions.h"
2280b57cec5SDimitry Andric #include <cassert>
2290b57cec5SDimitry Andric #include <cstdint>
2300b57cec5SDimitry Andric #include <iterator>
231bdd1243dSDimitry Andric #include <optional>
2320b57cec5SDimitry Andric #include <vector>
2330b57cec5SDimitry Andric
2340b57cec5SDimitry Andric using namespace llvm;
2350b57cec5SDimitry Andric
2360b57cec5SDimitry Andric #define DEBUG_TYPE "frame-info"
2370b57cec5SDimitry Andric
2380b57cec5SDimitry Andric static cl::opt<bool> EnableRedZone("aarch64-redzone",
2390b57cec5SDimitry Andric cl::desc("enable use of redzone on AArch64"),
2400b57cec5SDimitry Andric cl::init(false), cl::Hidden);
2410b57cec5SDimitry Andric
2420b57cec5SDimitry Andric static cl::opt<bool>
2430b57cec5SDimitry Andric ReverseCSRRestoreSeq("reverse-csr-restore-seq",
2440b57cec5SDimitry Andric cl::desc("reverse the CSR restore sequence"),
2450b57cec5SDimitry Andric cl::init(false), cl::Hidden);
2460b57cec5SDimitry Andric
2475ffd83dbSDimitry Andric static cl::opt<bool> StackTaggingMergeSetTag(
2485ffd83dbSDimitry Andric "stack-tagging-merge-settag",
2495ffd83dbSDimitry Andric cl::desc("merge settag instruction in function epilog"), cl::init(true),
2505ffd83dbSDimitry Andric cl::Hidden);
2515ffd83dbSDimitry Andric
252e8d8bef9SDimitry Andric static cl::opt<bool> OrderFrameObjects("aarch64-order-frame-objects",
253e8d8bef9SDimitry Andric cl::desc("sort stack allocations"),
254e8d8bef9SDimitry Andric cl::init(true), cl::Hidden);
255e8d8bef9SDimitry Andric
256fe6060f1SDimitry Andric cl::opt<bool> EnableHomogeneousPrologEpilog(
25781ad6265SDimitry Andric "homogeneous-prolog-epilog", cl::Hidden,
258fe6060f1SDimitry Andric cl::desc("Emit homogeneous prologue and epilogue for the size "
259fe6060f1SDimitry Andric "optimization (default = off)"));
260fe6060f1SDimitry Andric
2610b57cec5SDimitry Andric STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
2620b57cec5SDimitry Andric
263fe6060f1SDimitry Andric /// Returns how much of the incoming argument stack area (in bytes) we should
264fe6060f1SDimitry Andric /// clean up in an epilogue. For the C calling convention this will be 0, for
265fe6060f1SDimitry Andric /// guaranteed tail call conventions it can be positive (a normal return or a
266fe6060f1SDimitry Andric /// tail call to a function that uses less stack space for arguments) or
267fe6060f1SDimitry Andric /// negative (for a tail call to a function that needs more stack space than us
268fe6060f1SDimitry Andric /// for arguments).
getArgumentStackToRestore(MachineFunction & MF,MachineBasicBlock & MBB)269fe6060f1SDimitry Andric static int64_t getArgumentStackToRestore(MachineFunction &MF,
2705ffd83dbSDimitry Andric MachineBasicBlock &MBB) {
2715ffd83dbSDimitry Andric MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
2725ffd83dbSDimitry Andric AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2735f757f3fSDimitry Andric bool IsTailCallReturn = (MBB.end() != MBBI)
2745f757f3fSDimitry Andric ? AArch64InstrInfo::isTailCallReturnInst(*MBBI)
2755f757f3fSDimitry Andric : false;
2765ffd83dbSDimitry Andric
277fe6060f1SDimitry Andric int64_t ArgumentPopSize = 0;
2785ffd83dbSDimitry Andric if (IsTailCallReturn) {
2795ffd83dbSDimitry Andric MachineOperand &StackAdjust = MBBI->getOperand(1);
2805ffd83dbSDimitry Andric
2815ffd83dbSDimitry Andric // For a tail-call in a callee-pops-arguments environment, some or all of
2825ffd83dbSDimitry Andric // the stack may actually be in use for the call's arguments, this is
2835ffd83dbSDimitry Andric // calculated during LowerCall and consumed here...
2845ffd83dbSDimitry Andric ArgumentPopSize = StackAdjust.getImm();
2855ffd83dbSDimitry Andric } else {
2865ffd83dbSDimitry Andric // ... otherwise the amount to pop is *all* of the argument space,
2875ffd83dbSDimitry Andric // conveniently stored in the MachineFunctionInfo by
2885ffd83dbSDimitry Andric // LowerFormalArguments. This will, of course, be zero for the C calling
2895ffd83dbSDimitry Andric // convention.
2905ffd83dbSDimitry Andric ArgumentPopSize = AFI->getArgumentStackToRestore();
2915ffd83dbSDimitry Andric }
2925ffd83dbSDimitry Andric
2935ffd83dbSDimitry Andric return ArgumentPopSize;
2945ffd83dbSDimitry Andric }
2955ffd83dbSDimitry Andric
296fe6060f1SDimitry Andric static bool produceCompactUnwindFrame(MachineFunction &MF);
297fe6060f1SDimitry Andric static bool needsWinCFI(const MachineFunction &MF);
298fe6060f1SDimitry Andric static StackOffset getSVEStackSize(const MachineFunction &MF);
2995f757f3fSDimitry Andric static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB);
300fe6060f1SDimitry Andric
301fe6060f1SDimitry Andric /// Returns true if a homogeneous prolog or epilog code can be emitted
302fe6060f1SDimitry Andric /// for the size optimization. If possible, a frame helper call is injected.
303fe6060f1SDimitry Andric /// When Exit block is given, this check is for epilog.
homogeneousPrologEpilog(MachineFunction & MF,MachineBasicBlock * Exit) const304fe6060f1SDimitry Andric bool AArch64FrameLowering::homogeneousPrologEpilog(
305fe6060f1SDimitry Andric MachineFunction &MF, MachineBasicBlock *Exit) const {
306fe6060f1SDimitry Andric if (!MF.getFunction().hasMinSize())
307fe6060f1SDimitry Andric return false;
308fe6060f1SDimitry Andric if (!EnableHomogeneousPrologEpilog)
309fe6060f1SDimitry Andric return false;
310fe6060f1SDimitry Andric if (ReverseCSRRestoreSeq)
311fe6060f1SDimitry Andric return false;
312fe6060f1SDimitry Andric if (EnableRedZone)
313fe6060f1SDimitry Andric return false;
314fe6060f1SDimitry Andric
315fe6060f1SDimitry Andric // TODO: Window is supported yet.
316fe6060f1SDimitry Andric if (needsWinCFI(MF))
317fe6060f1SDimitry Andric return false;
318fe6060f1SDimitry Andric // TODO: SVE is not supported yet.
319fe6060f1SDimitry Andric if (getSVEStackSize(MF))
320fe6060f1SDimitry Andric return false;
321fe6060f1SDimitry Andric
322fe6060f1SDimitry Andric // Bail on stack adjustment needed on return for simplicity.
323fe6060f1SDimitry Andric const MachineFrameInfo &MFI = MF.getFrameInfo();
324fe6060f1SDimitry Andric const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
325fe6060f1SDimitry Andric if (MFI.hasVarSizedObjects() || RegInfo->hasStackRealignment(MF))
326fe6060f1SDimitry Andric return false;
327fe6060f1SDimitry Andric if (Exit && getArgumentStackToRestore(MF, *Exit))
328fe6060f1SDimitry Andric return false;
329fe6060f1SDimitry Andric
3305f757f3fSDimitry Andric auto *AFI = MF.getInfo<AArch64FunctionInfo>();
3315f757f3fSDimitry Andric if (AFI->hasSwiftAsyncContext())
3325f757f3fSDimitry Andric return false;
3335f757f3fSDimitry Andric
3345f757f3fSDimitry Andric // If there are an odd number of GPRs before LR and FP in the CSRs list,
3355f757f3fSDimitry Andric // they will not be paired into one RegPairInfo, which is incompatible with
3365f757f3fSDimitry Andric // the assumption made by the homogeneous prolog epilog pass.
3375f757f3fSDimitry Andric const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
3385f757f3fSDimitry Andric unsigned NumGPRs = 0;
3395f757f3fSDimitry Andric for (unsigned I = 0; CSRegs[I]; ++I) {
3405f757f3fSDimitry Andric Register Reg = CSRegs[I];
3415f757f3fSDimitry Andric if (Reg == AArch64::LR) {
3425f757f3fSDimitry Andric assert(CSRegs[I + 1] == AArch64::FP);
3435f757f3fSDimitry Andric if (NumGPRs % 2 != 0)
3445f757f3fSDimitry Andric return false;
3455f757f3fSDimitry Andric break;
3465f757f3fSDimitry Andric }
3475f757f3fSDimitry Andric if (AArch64::GPR64RegClass.contains(Reg))
3485f757f3fSDimitry Andric ++NumGPRs;
3495f757f3fSDimitry Andric }
3505f757f3fSDimitry Andric
351fe6060f1SDimitry Andric return true;
352fe6060f1SDimitry Andric }
353fe6060f1SDimitry Andric
354fe6060f1SDimitry Andric /// Returns true if CSRs should be paired.
producePairRegisters(MachineFunction & MF) const355fe6060f1SDimitry Andric bool AArch64FrameLowering::producePairRegisters(MachineFunction &MF) const {
356fe6060f1SDimitry Andric return produceCompactUnwindFrame(MF) || homogeneousPrologEpilog(MF);
357fe6060f1SDimitry Andric }
358fe6060f1SDimitry Andric
3590b57cec5SDimitry Andric /// This is the biggest offset to the stack pointer we can encode in aarch64
3600b57cec5SDimitry Andric /// instructions (without using a separate calculation and a temp register).
3610b57cec5SDimitry Andric /// Note that the exception here are vector stores/loads which cannot encode any
3620b57cec5SDimitry Andric /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
3630b57cec5SDimitry Andric static const unsigned DefaultSafeSPDisplacement = 255;
3640b57cec5SDimitry Andric
3650b57cec5SDimitry Andric /// Look at each instruction that references stack frames and return the stack
3660b57cec5SDimitry Andric /// size limit beyond which some of these instructions will require a scratch
3670b57cec5SDimitry Andric /// register during their expansion later.
estimateRSStackSizeLimit(MachineFunction & MF)3680b57cec5SDimitry Andric static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
3690b57cec5SDimitry Andric // FIXME: For now, just conservatively guestimate based on unscaled indexing
3700b57cec5SDimitry Andric // range. We'll end up allocating an unnecessary spill slot a lot, but
3710b57cec5SDimitry Andric // realistically that's not a big deal at this stage of the game.
3720b57cec5SDimitry Andric for (MachineBasicBlock &MBB : MF) {
3730b57cec5SDimitry Andric for (MachineInstr &MI : MBB) {
3740b57cec5SDimitry Andric if (MI.isDebugInstr() || MI.isPseudo() ||
3750b57cec5SDimitry Andric MI.getOpcode() == AArch64::ADDXri ||
3760b57cec5SDimitry Andric MI.getOpcode() == AArch64::ADDSXri)
3770b57cec5SDimitry Andric continue;
3780b57cec5SDimitry Andric
3790b57cec5SDimitry Andric for (const MachineOperand &MO : MI.operands()) {
3800b57cec5SDimitry Andric if (!MO.isFI())
3810b57cec5SDimitry Andric continue;
3820b57cec5SDimitry Andric
3838bcb0991SDimitry Andric StackOffset Offset;
3840b57cec5SDimitry Andric if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
3850b57cec5SDimitry Andric AArch64FrameOffsetCannotUpdate)
3860b57cec5SDimitry Andric return 0;
3870b57cec5SDimitry Andric }
3880b57cec5SDimitry Andric }
3890b57cec5SDimitry Andric }
3900b57cec5SDimitry Andric return DefaultSafeSPDisplacement;
3910b57cec5SDimitry Andric }
3920b57cec5SDimitry Andric
393480093f4SDimitry Andric TargetStackID::Value
getStackIDForScalableVectors() const394480093f4SDimitry Andric AArch64FrameLowering::getStackIDForScalableVectors() const {
395e8d8bef9SDimitry Andric return TargetStackID::ScalableVector;
396480093f4SDimitry Andric }
397480093f4SDimitry Andric
39862cfcf62SDimitry Andric /// Returns the size of the fixed object area (allocated next to sp on entry)
39962cfcf62SDimitry Andric /// On Win64 this may include a var args area and an UnwindHelp object for EH.
getFixedObjectSize(const MachineFunction & MF,const AArch64FunctionInfo * AFI,bool IsWin64,bool IsFunclet)40062cfcf62SDimitry Andric static unsigned getFixedObjectSize(const MachineFunction &MF,
40162cfcf62SDimitry Andric const AArch64FunctionInfo *AFI, bool IsWin64,
40262cfcf62SDimitry Andric bool IsFunclet) {
40362cfcf62SDimitry Andric if (!IsWin64 || IsFunclet) {
404fe6060f1SDimitry Andric return AFI->getTailCallReservedStack();
40562cfcf62SDimitry Andric } else {
406fe6060f1SDimitry Andric if (AFI->getTailCallReservedStack() != 0)
407fe6060f1SDimitry Andric report_fatal_error("cannot generate ABI-changing tail call for Win64");
40862cfcf62SDimitry Andric // Var args are stored here in the primary function.
40962cfcf62SDimitry Andric const unsigned VarArgsArea = AFI->getVarArgsGPRSize();
41062cfcf62SDimitry Andric // To support EH funclets we allocate an UnwindHelp object
41162cfcf62SDimitry Andric const unsigned UnwindHelpObject = (MF.hasEHFunclets() ? 8 : 0);
41262cfcf62SDimitry Andric return alignTo(VarArgsArea + UnwindHelpObject, 16);
41362cfcf62SDimitry Andric }
41462cfcf62SDimitry Andric }
41562cfcf62SDimitry Andric
4168bcb0991SDimitry Andric /// Returns the size of the entire SVE stackframe (calleesaves + spills).
getSVEStackSize(const MachineFunction & MF)4178bcb0991SDimitry Andric static StackOffset getSVEStackSize(const MachineFunction &MF) {
4188bcb0991SDimitry Andric const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
419e8d8bef9SDimitry Andric return StackOffset::getScalable((int64_t)AFI->getStackSizeSVE());
4208bcb0991SDimitry Andric }
4218bcb0991SDimitry Andric
canUseRedZone(const MachineFunction & MF) const4220b57cec5SDimitry Andric bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
4230b57cec5SDimitry Andric if (!EnableRedZone)
4240b57cec5SDimitry Andric return false;
425fe6060f1SDimitry Andric
4260b57cec5SDimitry Andric // Don't use the red zone if the function explicitly asks us not to.
4270b57cec5SDimitry Andric // This is typically used for kernel code.
428fe6060f1SDimitry Andric const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
429fe6060f1SDimitry Andric const unsigned RedZoneSize =
430fe6060f1SDimitry Andric Subtarget.getTargetLowering()->getRedZoneSize(MF.getFunction());
431fe6060f1SDimitry Andric if (!RedZoneSize)
4320b57cec5SDimitry Andric return false;
4330b57cec5SDimitry Andric
4340b57cec5SDimitry Andric const MachineFrameInfo &MFI = MF.getFrameInfo();
4350b57cec5SDimitry Andric const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
436480093f4SDimitry Andric uint64_t NumBytes = AFI->getLocalStackSize();
4370b57cec5SDimitry Andric
438fe6060f1SDimitry Andric return !(MFI.hasCalls() || hasFP(MF) || NumBytes > RedZoneSize ||
4398bcb0991SDimitry Andric getSVEStackSize(MF));
4400b57cec5SDimitry Andric }
4410b57cec5SDimitry Andric
4420b57cec5SDimitry Andric /// hasFP - Return true if the specified function should have a dedicated frame
4430b57cec5SDimitry Andric /// pointer register.
hasFP(const MachineFunction & MF) const4440b57cec5SDimitry Andric bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
4450b57cec5SDimitry Andric const MachineFrameInfo &MFI = MF.getFrameInfo();
4460b57cec5SDimitry Andric const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
4475f757f3fSDimitry Andric
4480b57cec5SDimitry Andric // Win64 EH requires a frame pointer if funclets are present, as the locals
4490b57cec5SDimitry Andric // are accessed off the frame pointer in both the parent function and the
4500b57cec5SDimitry Andric // funclets.
4510b57cec5SDimitry Andric if (MF.hasEHFunclets())
4520b57cec5SDimitry Andric return true;
4530b57cec5SDimitry Andric // Retain behavior of always omitting the FP for leaf functions when possible.
454480093f4SDimitry Andric if (MF.getTarget().Options.DisableFramePointerElim(MF))
4550b57cec5SDimitry Andric return true;
4560b57cec5SDimitry Andric if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
4570b57cec5SDimitry Andric MFI.hasStackMap() || MFI.hasPatchPoint() ||
458fe6060f1SDimitry Andric RegInfo->hasStackRealignment(MF))
4590b57cec5SDimitry Andric return true;
4600b57cec5SDimitry Andric // With large callframes around we may need to use FP to access the scavenging
4610b57cec5SDimitry Andric // emergency spillslot.
4620b57cec5SDimitry Andric //
4630b57cec5SDimitry Andric // Unfortunately some calls to hasFP() like machine verifier ->
4640b57cec5SDimitry Andric // getReservedReg() -> hasFP in the middle of global isel are too early
4650b57cec5SDimitry Andric // to know the max call frame size. Hopefully conservatively returning "true"
4660b57cec5SDimitry Andric // in those cases is fine.
4670b57cec5SDimitry Andric // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
4680b57cec5SDimitry Andric if (!MFI.isMaxCallFrameSizeComputed() ||
4690b57cec5SDimitry Andric MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
4700b57cec5SDimitry Andric return true;
4710b57cec5SDimitry Andric
4720b57cec5SDimitry Andric return false;
4730b57cec5SDimitry Andric }
4740b57cec5SDimitry Andric
4750b57cec5SDimitry Andric /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
4760b57cec5SDimitry Andric /// not required, we reserve argument space for call sites in the function
4770b57cec5SDimitry Andric /// immediately on entry to the current function. This eliminates the need for
4780b57cec5SDimitry Andric /// add/sub sp brackets around call sites. Returns true if the call frame is
4790b57cec5SDimitry Andric /// included as part of the stack frame.
4800b57cec5SDimitry Andric bool
hasReservedCallFrame(const MachineFunction & MF) const4810b57cec5SDimitry Andric AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
4825f757f3fSDimitry Andric // The stack probing code for the dynamically allocated outgoing arguments
4835f757f3fSDimitry Andric // area assumes that the stack is probed at the top - either by the prologue
4845f757f3fSDimitry Andric // code, which issues a probe if `hasVarSizedObjects` return true, or by the
4855f757f3fSDimitry Andric // most recent variable-sized object allocation. Changing the condition here
4865f757f3fSDimitry Andric // may need to be followed up by changes to the probe issuing logic.
4870b57cec5SDimitry Andric return !MF.getFrameInfo().hasVarSizedObjects();
4880b57cec5SDimitry Andric }
4890b57cec5SDimitry Andric
eliminateCallFramePseudoInstr(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator I) const4900b57cec5SDimitry Andric MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
4910b57cec5SDimitry Andric MachineFunction &MF, MachineBasicBlock &MBB,
4920b57cec5SDimitry Andric MachineBasicBlock::iterator I) const {
4930b57cec5SDimitry Andric const AArch64InstrInfo *TII =
4940b57cec5SDimitry Andric static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
4955f757f3fSDimitry Andric const AArch64TargetLowering *TLI =
4965f757f3fSDimitry Andric MF.getSubtarget<AArch64Subtarget>().getTargetLowering();
4975f757f3fSDimitry Andric [[maybe_unused]] MachineFrameInfo &MFI = MF.getFrameInfo();
4980b57cec5SDimitry Andric DebugLoc DL = I->getDebugLoc();
4990b57cec5SDimitry Andric unsigned Opc = I->getOpcode();
5000b57cec5SDimitry Andric bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
5010b57cec5SDimitry Andric uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
5020b57cec5SDimitry Andric
5030b57cec5SDimitry Andric if (!hasReservedCallFrame(MF)) {
5040b57cec5SDimitry Andric int64_t Amount = I->getOperand(0).getImm();
5055ffd83dbSDimitry Andric Amount = alignTo(Amount, getStackAlign());
5060b57cec5SDimitry Andric if (!IsDestroy)
5070b57cec5SDimitry Andric Amount = -Amount;
5080b57cec5SDimitry Andric
5090b57cec5SDimitry Andric // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
5100b57cec5SDimitry Andric // doesn't have to pop anything), then the first operand will be zero too so
5110b57cec5SDimitry Andric // this adjustment is a no-op.
5120b57cec5SDimitry Andric if (CalleePopAmount == 0) {
5130b57cec5SDimitry Andric // FIXME: in-function stack adjustment for calls is limited to 24-bits
5140b57cec5SDimitry Andric // because there's no guaranteed temporary register available.
5150b57cec5SDimitry Andric //
5160b57cec5SDimitry Andric // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
5170b57cec5SDimitry Andric // 1) For offset <= 12-bit, we use LSL #0
5180b57cec5SDimitry Andric // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
5190b57cec5SDimitry Andric // LSL #0, and the other uses LSL #12.
5200b57cec5SDimitry Andric //
5210b57cec5SDimitry Andric // Most call frames will be allocated at the start of a function so
5220b57cec5SDimitry Andric // this is OK, but it is a limitation that needs dealing with.
5230b57cec5SDimitry Andric assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
5245f757f3fSDimitry Andric
5255f757f3fSDimitry Andric if (TLI->hasInlineStackProbe(MF) &&
5265f757f3fSDimitry Andric -Amount >= AArch64::StackProbeMaxUnprobedStack) {
5275f757f3fSDimitry Andric // When stack probing is enabled, the decrement of SP may need to be
5285f757f3fSDimitry Andric // probed. We only need to do this if the call site needs 1024 bytes of
5295f757f3fSDimitry Andric // space or more, because a region smaller than that is allowed to be
5305f757f3fSDimitry Andric // unprobed at an ABI boundary. We rely on the fact that SP has been
5315f757f3fSDimitry Andric // probed exactly at this point, either by the prologue or most recent
5325f757f3fSDimitry Andric // dynamic allocation.
5335f757f3fSDimitry Andric assert(MFI.hasVarSizedObjects() &&
5345f757f3fSDimitry Andric "non-reserved call frame without var sized objects?");
5355f757f3fSDimitry Andric Register ScratchReg =
5365f757f3fSDimitry Andric MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
5375f757f3fSDimitry Andric inlineStackProbeFixed(I, ScratchReg, -Amount, StackOffset::get(0, 0));
5385f757f3fSDimitry Andric } else {
539e8d8bef9SDimitry Andric emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
540e8d8bef9SDimitry Andric StackOffset::getFixed(Amount), TII);
5410b57cec5SDimitry Andric }
5425f757f3fSDimitry Andric }
5430b57cec5SDimitry Andric } else if (CalleePopAmount != 0) {
5440b57cec5SDimitry Andric // If the calling convention demands that the callee pops arguments from the
5450b57cec5SDimitry Andric // stack, we want to add it back if we have a reserved call frame.
5460b57cec5SDimitry Andric assert(CalleePopAmount < 0xffffff && "call frame too large");
5478bcb0991SDimitry Andric emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
548e8d8bef9SDimitry Andric StackOffset::getFixed(-(int64_t)CalleePopAmount), TII);
5490b57cec5SDimitry Andric }
5500b57cec5SDimitry Andric return MBB.erase(I);
5510b57cec5SDimitry Andric }
5520b57cec5SDimitry Andric
emitCalleeSavedGPRLocations(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI) const55381ad6265SDimitry Andric void AArch64FrameLowering::emitCalleeSavedGPRLocations(
5540b57cec5SDimitry Andric MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
5550b57cec5SDimitry Andric MachineFunction &MF = *MBB.getParent();
5560b57cec5SDimitry Andric MachineFrameInfo &MFI = MF.getFrameInfo();
55781ad6265SDimitry Andric
55881ad6265SDimitry Andric const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
55981ad6265SDimitry Andric if (CSI.empty())
56081ad6265SDimitry Andric return;
56181ad6265SDimitry Andric
5620b57cec5SDimitry Andric const TargetSubtargetInfo &STI = MF.getSubtarget();
56381ad6265SDimitry Andric const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
56481ad6265SDimitry Andric const TargetInstrInfo &TII = *STI.getInstrInfo();
5650b57cec5SDimitry Andric DebugLoc DL = MBB.findDebugLoc(MBBI);
5660b57cec5SDimitry Andric
56781ad6265SDimitry Andric for (const auto &Info : CSI) {
56881ad6265SDimitry Andric if (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector)
56981ad6265SDimitry Andric continue;
57081ad6265SDimitry Andric
57181ad6265SDimitry Andric assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
57281ad6265SDimitry Andric unsigned DwarfReg = TRI.getDwarfRegNum(Info.getReg(), true);
57381ad6265SDimitry Andric
57481ad6265SDimitry Andric int64_t Offset =
57581ad6265SDimitry Andric MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
57681ad6265SDimitry Andric unsigned CFIIndex = MF.addFrameInst(
57781ad6265SDimitry Andric MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
57881ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
57981ad6265SDimitry Andric .addCFIIndex(CFIIndex)
58081ad6265SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
58181ad6265SDimitry Andric }
58281ad6265SDimitry Andric }
58381ad6265SDimitry Andric
emitCalleeSavedSVELocations(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI) const58481ad6265SDimitry Andric void AArch64FrameLowering::emitCalleeSavedSVELocations(
58581ad6265SDimitry Andric MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
58681ad6265SDimitry Andric MachineFunction &MF = *MBB.getParent();
58781ad6265SDimitry Andric MachineFrameInfo &MFI = MF.getFrameInfo();
58881ad6265SDimitry Andric
5890b57cec5SDimitry Andric // Add callee saved registers to move list.
5900b57cec5SDimitry Andric const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
5910b57cec5SDimitry Andric if (CSI.empty())
5920b57cec5SDimitry Andric return;
5930b57cec5SDimitry Andric
59481ad6265SDimitry Andric const TargetSubtargetInfo &STI = MF.getSubtarget();
59581ad6265SDimitry Andric const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
59681ad6265SDimitry Andric const TargetInstrInfo &TII = *STI.getInstrInfo();
59781ad6265SDimitry Andric DebugLoc DL = MBB.findDebugLoc(MBBI);
59881ad6265SDimitry Andric AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
59981ad6265SDimitry Andric
6000b57cec5SDimitry Andric for (const auto &Info : CSI) {
60181ad6265SDimitry Andric if (!(MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
60281ad6265SDimitry Andric continue;
60375b4d546SDimitry Andric
60475b4d546SDimitry Andric // Not all unwinders may know about SVE registers, so assume the lowest
60575b4d546SDimitry Andric // common demoninator.
60681ad6265SDimitry Andric assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
60781ad6265SDimitry Andric unsigned Reg = Info.getReg();
60881ad6265SDimitry Andric if (!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
60975b4d546SDimitry Andric continue;
61075b4d546SDimitry Andric
61181ad6265SDimitry Andric StackOffset Offset =
612e8d8bef9SDimitry Andric StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) -
61381ad6265SDimitry Andric StackOffset::getFixed(AFI.getCalleeSavedStackSize(MFI));
61481ad6265SDimitry Andric
61581ad6265SDimitry Andric unsigned CFIIndex = MF.addFrameInst(createCFAOffset(TRI, Reg, Offset));
61681ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
6170b57cec5SDimitry Andric .addCFIIndex(CFIIndex)
6180b57cec5SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
6190b57cec5SDimitry Andric }
6200b57cec5SDimitry Andric }
6210b57cec5SDimitry Andric
insertCFISameValue(const MCInstrDesc & Desc,MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator InsertPt,unsigned DwarfReg)62281ad6265SDimitry Andric static void insertCFISameValue(const MCInstrDesc &Desc, MachineFunction &MF,
62381ad6265SDimitry Andric MachineBasicBlock &MBB,
62481ad6265SDimitry Andric MachineBasicBlock::iterator InsertPt,
62581ad6265SDimitry Andric unsigned DwarfReg) {
62681ad6265SDimitry Andric unsigned CFIIndex =
62781ad6265SDimitry Andric MF.addFrameInst(MCCFIInstruction::createSameValue(nullptr, DwarfReg));
62881ad6265SDimitry Andric BuildMI(MBB, InsertPt, DebugLoc(), Desc).addCFIIndex(CFIIndex);
62981ad6265SDimitry Andric }
63081ad6265SDimitry Andric
resetCFIToInitialState(MachineBasicBlock & MBB) const63181ad6265SDimitry Andric void AArch64FrameLowering::resetCFIToInitialState(
63281ad6265SDimitry Andric MachineBasicBlock &MBB) const {
63381ad6265SDimitry Andric
63481ad6265SDimitry Andric MachineFunction &MF = *MBB.getParent();
63581ad6265SDimitry Andric const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
63681ad6265SDimitry Andric const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
63781ad6265SDimitry Andric const auto &TRI =
63881ad6265SDimitry Andric static_cast<const AArch64RegisterInfo &>(*Subtarget.getRegisterInfo());
63981ad6265SDimitry Andric const auto &MFI = *MF.getInfo<AArch64FunctionInfo>();
64081ad6265SDimitry Andric
64181ad6265SDimitry Andric const MCInstrDesc &CFIDesc = TII.get(TargetOpcode::CFI_INSTRUCTION);
64281ad6265SDimitry Andric DebugLoc DL;
64381ad6265SDimitry Andric
64481ad6265SDimitry Andric // Reset the CFA to `SP + 0`.
64581ad6265SDimitry Andric MachineBasicBlock::iterator InsertPt = MBB.begin();
64681ad6265SDimitry Andric unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
64781ad6265SDimitry Andric nullptr, TRI.getDwarfRegNum(AArch64::SP, true), 0));
64881ad6265SDimitry Andric BuildMI(MBB, InsertPt, DL, CFIDesc).addCFIIndex(CFIIndex);
64981ad6265SDimitry Andric
65081ad6265SDimitry Andric // Flip the RA sign state.
651bdd1243dSDimitry Andric if (MFI.shouldSignReturnAddress(MF)) {
65281ad6265SDimitry Andric CFIIndex = MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
65381ad6265SDimitry Andric BuildMI(MBB, InsertPt, DL, CFIDesc).addCFIIndex(CFIIndex);
65481ad6265SDimitry Andric }
65581ad6265SDimitry Andric
65681ad6265SDimitry Andric // Shadow call stack uses X18, reset it.
6575f757f3fSDimitry Andric if (MFI.needsShadowCallStackPrologueEpilogue(MF))
65881ad6265SDimitry Andric insertCFISameValue(CFIDesc, MF, MBB, InsertPt,
65981ad6265SDimitry Andric TRI.getDwarfRegNum(AArch64::X18, true));
66081ad6265SDimitry Andric
66181ad6265SDimitry Andric // Emit .cfi_same_value for callee-saved registers.
66281ad6265SDimitry Andric const std::vector<CalleeSavedInfo> &CSI =
66381ad6265SDimitry Andric MF.getFrameInfo().getCalleeSavedInfo();
66481ad6265SDimitry Andric for (const auto &Info : CSI) {
66581ad6265SDimitry Andric unsigned Reg = Info.getReg();
66681ad6265SDimitry Andric if (!TRI.regNeedsCFI(Reg, Reg))
66781ad6265SDimitry Andric continue;
66881ad6265SDimitry Andric insertCFISameValue(CFIDesc, MF, MBB, InsertPt,
66981ad6265SDimitry Andric TRI.getDwarfRegNum(Reg, true));
67081ad6265SDimitry Andric }
67181ad6265SDimitry Andric }
67281ad6265SDimitry Andric
emitCalleeSavedRestores(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,bool SVE)67381ad6265SDimitry Andric static void emitCalleeSavedRestores(MachineBasicBlock &MBB,
67481ad6265SDimitry Andric MachineBasicBlock::iterator MBBI,
67581ad6265SDimitry Andric bool SVE) {
67681ad6265SDimitry Andric MachineFunction &MF = *MBB.getParent();
67781ad6265SDimitry Andric MachineFrameInfo &MFI = MF.getFrameInfo();
67881ad6265SDimitry Andric
67981ad6265SDimitry Andric const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
68081ad6265SDimitry Andric if (CSI.empty())
68181ad6265SDimitry Andric return;
68281ad6265SDimitry Andric
68381ad6265SDimitry Andric const TargetSubtargetInfo &STI = MF.getSubtarget();
68481ad6265SDimitry Andric const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
68581ad6265SDimitry Andric const TargetInstrInfo &TII = *STI.getInstrInfo();
68681ad6265SDimitry Andric DebugLoc DL = MBB.findDebugLoc(MBBI);
68781ad6265SDimitry Andric
68881ad6265SDimitry Andric for (const auto &Info : CSI) {
68981ad6265SDimitry Andric if (SVE !=
69081ad6265SDimitry Andric (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
69181ad6265SDimitry Andric continue;
69281ad6265SDimitry Andric
69381ad6265SDimitry Andric unsigned Reg = Info.getReg();
69481ad6265SDimitry Andric if (SVE &&
69581ad6265SDimitry Andric !static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
69681ad6265SDimitry Andric continue;
69781ad6265SDimitry Andric
69881ad6265SDimitry Andric unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore(
69981ad6265SDimitry Andric nullptr, TRI.getDwarfRegNum(Info.getReg(), true)));
70081ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
70181ad6265SDimitry Andric .addCFIIndex(CFIIndex)
70281ad6265SDimitry Andric .setMIFlags(MachineInstr::FrameDestroy);
70381ad6265SDimitry Andric }
70481ad6265SDimitry Andric }
70581ad6265SDimitry Andric
emitCalleeSavedGPRRestores(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI) const70681ad6265SDimitry Andric void AArch64FrameLowering::emitCalleeSavedGPRRestores(
70781ad6265SDimitry Andric MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
70881ad6265SDimitry Andric emitCalleeSavedRestores(MBB, MBBI, false);
70981ad6265SDimitry Andric }
71081ad6265SDimitry Andric
emitCalleeSavedSVERestores(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI) const71181ad6265SDimitry Andric void AArch64FrameLowering::emitCalleeSavedSVERestores(
71281ad6265SDimitry Andric MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
71381ad6265SDimitry Andric emitCalleeSavedRestores(MBB, MBBI, true);
71481ad6265SDimitry Andric }
71581ad6265SDimitry Andric
7165f757f3fSDimitry Andric // Return the maximum possible number of bytes for `Size` due to the
7175f757f3fSDimitry Andric // architectural limit on the size of a SVE register.
upperBound(StackOffset Size)7185f757f3fSDimitry Andric static int64_t upperBound(StackOffset Size) {
7195f757f3fSDimitry Andric static const int64_t MAX_BYTES_PER_SCALABLE_BYTE = 16;
7205f757f3fSDimitry Andric return Size.getScalable() * MAX_BYTES_PER_SCALABLE_BYTE + Size.getFixed();
7215f757f3fSDimitry Andric }
7225f757f3fSDimitry Andric
allocateStackSpace(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,int64_t RealignmentPadding,StackOffset AllocSize,bool NeedsWinCFI,bool * HasWinCFI,bool EmitCFI,StackOffset InitialOffset,bool FollowupAllocs) const7235f757f3fSDimitry Andric void AArch64FrameLowering::allocateStackSpace(
7245f757f3fSDimitry Andric MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
7255f757f3fSDimitry Andric int64_t RealignmentPadding, StackOffset AllocSize, bool NeedsWinCFI,
7265f757f3fSDimitry Andric bool *HasWinCFI, bool EmitCFI, StackOffset InitialOffset,
7275f757f3fSDimitry Andric bool FollowupAllocs) const {
7285f757f3fSDimitry Andric
7295f757f3fSDimitry Andric if (!AllocSize)
7305f757f3fSDimitry Andric return;
7315f757f3fSDimitry Andric
7325f757f3fSDimitry Andric DebugLoc DL;
7335f757f3fSDimitry Andric MachineFunction &MF = *MBB.getParent();
7345f757f3fSDimitry Andric const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
7355f757f3fSDimitry Andric const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
7365f757f3fSDimitry Andric AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
7375f757f3fSDimitry Andric const MachineFrameInfo &MFI = MF.getFrameInfo();
7385f757f3fSDimitry Andric
7395f757f3fSDimitry Andric const int64_t MaxAlign = MFI.getMaxAlign().value();
7405f757f3fSDimitry Andric const uint64_t AndMask = ~(MaxAlign - 1);
7415f757f3fSDimitry Andric
7425f757f3fSDimitry Andric if (!Subtarget.getTargetLowering()->hasInlineStackProbe(MF)) {
7435f757f3fSDimitry Andric Register TargetReg = RealignmentPadding
7445f757f3fSDimitry Andric ? findScratchNonCalleeSaveRegister(&MBB)
7455f757f3fSDimitry Andric : AArch64::SP;
7465f757f3fSDimitry Andric // SUB Xd/SP, SP, AllocSize
7475f757f3fSDimitry Andric emitFrameOffset(MBB, MBBI, DL, TargetReg, AArch64::SP, -AllocSize, &TII,
7485f757f3fSDimitry Andric MachineInstr::FrameSetup, false, NeedsWinCFI, HasWinCFI,
7495f757f3fSDimitry Andric EmitCFI, InitialOffset);
7505f757f3fSDimitry Andric
7515f757f3fSDimitry Andric if (RealignmentPadding) {
7525f757f3fSDimitry Andric // AND SP, X9, 0b11111...0000
7535f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::ANDXri), AArch64::SP)
7545f757f3fSDimitry Andric .addReg(TargetReg, RegState::Kill)
7555f757f3fSDimitry Andric .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64))
7565f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
7575f757f3fSDimitry Andric AFI.setStackRealigned(true);
7585f757f3fSDimitry Andric
7595f757f3fSDimitry Andric // No need for SEH instructions here; if we're realigning the stack,
7605f757f3fSDimitry Andric // we've set a frame pointer and already finished the SEH prologue.
7615f757f3fSDimitry Andric assert(!NeedsWinCFI);
7625f757f3fSDimitry Andric }
7635f757f3fSDimitry Andric return;
7645f757f3fSDimitry Andric }
7655f757f3fSDimitry Andric
7665f757f3fSDimitry Andric //
7675f757f3fSDimitry Andric // Stack probing allocation.
7685f757f3fSDimitry Andric //
7695f757f3fSDimitry Andric
7705f757f3fSDimitry Andric // Fixed length allocation. If we don't need to re-align the stack and don't
7715f757f3fSDimitry Andric // have SVE objects, we can use a more efficient sequence for stack probing.
7725f757f3fSDimitry Andric if (AllocSize.getScalable() == 0 && RealignmentPadding == 0) {
7735f757f3fSDimitry Andric Register ScratchReg = findScratchNonCalleeSaveRegister(&MBB);
7745f757f3fSDimitry Andric assert(ScratchReg != AArch64::NoRegister);
7755f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::PROBED_STACKALLOC))
7765f757f3fSDimitry Andric .addDef(ScratchReg)
7775f757f3fSDimitry Andric .addImm(AllocSize.getFixed())
7785f757f3fSDimitry Andric .addImm(InitialOffset.getFixed())
7795f757f3fSDimitry Andric .addImm(InitialOffset.getScalable());
7805f757f3fSDimitry Andric // The fixed allocation may leave unprobed bytes at the top of the
7815f757f3fSDimitry Andric // stack. If we have subsequent alocation (e.g. if we have variable-sized
7825f757f3fSDimitry Andric // objects), we need to issue an extra probe, so these allocations start in
7835f757f3fSDimitry Andric // a known state.
7845f757f3fSDimitry Andric if (FollowupAllocs) {
7855f757f3fSDimitry Andric // STR XZR, [SP]
7865f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXui))
7875f757f3fSDimitry Andric .addReg(AArch64::XZR)
7885f757f3fSDimitry Andric .addReg(AArch64::SP)
7895f757f3fSDimitry Andric .addImm(0)
7905f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
7915f757f3fSDimitry Andric }
7925f757f3fSDimitry Andric
7935f757f3fSDimitry Andric return;
7945f757f3fSDimitry Andric }
7955f757f3fSDimitry Andric
7965f757f3fSDimitry Andric // Variable length allocation.
7975f757f3fSDimitry Andric
7985f757f3fSDimitry Andric // If the (unknown) allocation size cannot exceed the probe size, decrement
7995f757f3fSDimitry Andric // the stack pointer right away.
8005f757f3fSDimitry Andric int64_t ProbeSize = AFI.getStackProbeSize();
8015f757f3fSDimitry Andric if (upperBound(AllocSize) + RealignmentPadding <= ProbeSize) {
8025f757f3fSDimitry Andric Register ScratchReg = RealignmentPadding
8035f757f3fSDimitry Andric ? findScratchNonCalleeSaveRegister(&MBB)
8045f757f3fSDimitry Andric : AArch64::SP;
8055f757f3fSDimitry Andric assert(ScratchReg != AArch64::NoRegister);
8065f757f3fSDimitry Andric // SUB Xd, SP, AllocSize
8075f757f3fSDimitry Andric emitFrameOffset(MBB, MBBI, DL, ScratchReg, AArch64::SP, -AllocSize, &TII,
8085f757f3fSDimitry Andric MachineInstr::FrameSetup, false, NeedsWinCFI, HasWinCFI,
8095f757f3fSDimitry Andric EmitCFI, InitialOffset);
8105f757f3fSDimitry Andric if (RealignmentPadding) {
8115f757f3fSDimitry Andric // AND SP, Xn, 0b11111...0000
8125f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::ANDXri), AArch64::SP)
8135f757f3fSDimitry Andric .addReg(ScratchReg, RegState::Kill)
8145f757f3fSDimitry Andric .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64))
8155f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
8165f757f3fSDimitry Andric AFI.setStackRealigned(true);
8175f757f3fSDimitry Andric }
8185f757f3fSDimitry Andric if (FollowupAllocs || upperBound(AllocSize) + RealignmentPadding >
8195f757f3fSDimitry Andric AArch64::StackProbeMaxUnprobedStack) {
8205f757f3fSDimitry Andric // STR XZR, [SP]
8215f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXui))
8225f757f3fSDimitry Andric .addReg(AArch64::XZR)
8235f757f3fSDimitry Andric .addReg(AArch64::SP)
8245f757f3fSDimitry Andric .addImm(0)
8255f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
8265f757f3fSDimitry Andric }
8275f757f3fSDimitry Andric return;
8285f757f3fSDimitry Andric }
8295f757f3fSDimitry Andric
8305f757f3fSDimitry Andric // Emit a variable-length allocation probing loop.
8315f757f3fSDimitry Andric // TODO: As an optimisation, the loop can be "unrolled" into a few parts,
8325f757f3fSDimitry Andric // each of them guaranteed to adjust the stack by less than the probe size.
8335f757f3fSDimitry Andric Register TargetReg = findScratchNonCalleeSaveRegister(&MBB);
8345f757f3fSDimitry Andric assert(TargetReg != AArch64::NoRegister);
8355f757f3fSDimitry Andric // SUB Xd, SP, AllocSize
8365f757f3fSDimitry Andric emitFrameOffset(MBB, MBBI, DL, TargetReg, AArch64::SP, -AllocSize, &TII,
8375f757f3fSDimitry Andric MachineInstr::FrameSetup, false, NeedsWinCFI, HasWinCFI,
8385f757f3fSDimitry Andric EmitCFI, InitialOffset);
8395f757f3fSDimitry Andric if (RealignmentPadding) {
8405f757f3fSDimitry Andric // AND Xn, Xn, 0b11111...0000
8415f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::ANDXri), TargetReg)
8425f757f3fSDimitry Andric .addReg(TargetReg, RegState::Kill)
8435f757f3fSDimitry Andric .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64))
8445f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
8455f757f3fSDimitry Andric }
8465f757f3fSDimitry Andric
8475f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::PROBED_STACKALLOC_VAR))
8485f757f3fSDimitry Andric .addReg(TargetReg);
8495f757f3fSDimitry Andric if (EmitCFI) {
8505f757f3fSDimitry Andric // Set the CFA register back to SP.
8515f757f3fSDimitry Andric unsigned Reg =
8525f757f3fSDimitry Andric Subtarget.getRegisterInfo()->getDwarfRegNum(AArch64::SP, true);
8535f757f3fSDimitry Andric unsigned CFIIndex =
8545f757f3fSDimitry Andric MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, Reg));
8555f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
8565f757f3fSDimitry Andric .addCFIIndex(CFIIndex)
8575f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
8585f757f3fSDimitry Andric }
8595f757f3fSDimitry Andric if (RealignmentPadding)
8605f757f3fSDimitry Andric AFI.setStackRealigned(true);
8615f757f3fSDimitry Andric }
8625f757f3fSDimitry Andric
getRegisterOrZero(MCRegister Reg,bool HasSVE)86381ad6265SDimitry Andric static MCRegister getRegisterOrZero(MCRegister Reg, bool HasSVE) {
86481ad6265SDimitry Andric switch (Reg.id()) {
86581ad6265SDimitry Andric default:
86681ad6265SDimitry Andric // The called routine is expected to preserve r19-r28
86781ad6265SDimitry Andric // r29 and r30 are used as frame pointer and link register resp.
86881ad6265SDimitry Andric return 0;
86981ad6265SDimitry Andric
87081ad6265SDimitry Andric // GPRs
87181ad6265SDimitry Andric #define CASE(n) \
87281ad6265SDimitry Andric case AArch64::W##n: \
87381ad6265SDimitry Andric case AArch64::X##n: \
87481ad6265SDimitry Andric return AArch64::X##n
87581ad6265SDimitry Andric CASE(0);
87681ad6265SDimitry Andric CASE(1);
87781ad6265SDimitry Andric CASE(2);
87881ad6265SDimitry Andric CASE(3);
87981ad6265SDimitry Andric CASE(4);
88081ad6265SDimitry Andric CASE(5);
88181ad6265SDimitry Andric CASE(6);
88281ad6265SDimitry Andric CASE(7);
88381ad6265SDimitry Andric CASE(8);
88481ad6265SDimitry Andric CASE(9);
88581ad6265SDimitry Andric CASE(10);
88681ad6265SDimitry Andric CASE(11);
88781ad6265SDimitry Andric CASE(12);
88881ad6265SDimitry Andric CASE(13);
88981ad6265SDimitry Andric CASE(14);
89081ad6265SDimitry Andric CASE(15);
89181ad6265SDimitry Andric CASE(16);
89281ad6265SDimitry Andric CASE(17);
89381ad6265SDimitry Andric CASE(18);
89481ad6265SDimitry Andric #undef CASE
89581ad6265SDimitry Andric
89681ad6265SDimitry Andric // FPRs
89781ad6265SDimitry Andric #define CASE(n) \
89881ad6265SDimitry Andric case AArch64::B##n: \
89981ad6265SDimitry Andric case AArch64::H##n: \
90081ad6265SDimitry Andric case AArch64::S##n: \
90181ad6265SDimitry Andric case AArch64::D##n: \
90281ad6265SDimitry Andric case AArch64::Q##n: \
90381ad6265SDimitry Andric return HasSVE ? AArch64::Z##n : AArch64::Q##n
90481ad6265SDimitry Andric CASE(0);
90581ad6265SDimitry Andric CASE(1);
90681ad6265SDimitry Andric CASE(2);
90781ad6265SDimitry Andric CASE(3);
90881ad6265SDimitry Andric CASE(4);
90981ad6265SDimitry Andric CASE(5);
91081ad6265SDimitry Andric CASE(6);
91181ad6265SDimitry Andric CASE(7);
91281ad6265SDimitry Andric CASE(8);
91381ad6265SDimitry Andric CASE(9);
91481ad6265SDimitry Andric CASE(10);
91581ad6265SDimitry Andric CASE(11);
91681ad6265SDimitry Andric CASE(12);
91781ad6265SDimitry Andric CASE(13);
91881ad6265SDimitry Andric CASE(14);
91981ad6265SDimitry Andric CASE(15);
92081ad6265SDimitry Andric CASE(16);
92181ad6265SDimitry Andric CASE(17);
92281ad6265SDimitry Andric CASE(18);
92381ad6265SDimitry Andric CASE(19);
92481ad6265SDimitry Andric CASE(20);
92581ad6265SDimitry Andric CASE(21);
92681ad6265SDimitry Andric CASE(22);
92781ad6265SDimitry Andric CASE(23);
92881ad6265SDimitry Andric CASE(24);
92981ad6265SDimitry Andric CASE(25);
93081ad6265SDimitry Andric CASE(26);
93181ad6265SDimitry Andric CASE(27);
93281ad6265SDimitry Andric CASE(28);
93381ad6265SDimitry Andric CASE(29);
93481ad6265SDimitry Andric CASE(30);
93581ad6265SDimitry Andric CASE(31);
93681ad6265SDimitry Andric #undef CASE
93781ad6265SDimitry Andric }
93881ad6265SDimitry Andric }
93981ad6265SDimitry Andric
emitZeroCallUsedRegs(BitVector RegsToZero,MachineBasicBlock & MBB) const94081ad6265SDimitry Andric void AArch64FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
94181ad6265SDimitry Andric MachineBasicBlock &MBB) const {
94281ad6265SDimitry Andric // Insertion point.
94381ad6265SDimitry Andric MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
94481ad6265SDimitry Andric
94581ad6265SDimitry Andric // Fake a debug loc.
94681ad6265SDimitry Andric DebugLoc DL;
94781ad6265SDimitry Andric if (MBBI != MBB.end())
94881ad6265SDimitry Andric DL = MBBI->getDebugLoc();
94981ad6265SDimitry Andric
95081ad6265SDimitry Andric const MachineFunction &MF = *MBB.getParent();
95181ad6265SDimitry Andric const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
95281ad6265SDimitry Andric const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
95381ad6265SDimitry Andric
95481ad6265SDimitry Andric BitVector GPRsToZero(TRI.getNumRegs());
95581ad6265SDimitry Andric BitVector FPRsToZero(TRI.getNumRegs());
95681ad6265SDimitry Andric bool HasSVE = STI.hasSVE();
95781ad6265SDimitry Andric for (MCRegister Reg : RegsToZero.set_bits()) {
95881ad6265SDimitry Andric if (TRI.isGeneralPurposeRegister(MF, Reg)) {
95981ad6265SDimitry Andric // For GPRs, we only care to clear out the 64-bit register.
96081ad6265SDimitry Andric if (MCRegister XReg = getRegisterOrZero(Reg, HasSVE))
96181ad6265SDimitry Andric GPRsToZero.set(XReg);
96281ad6265SDimitry Andric } else if (AArch64::FPR128RegClass.contains(Reg) ||
96381ad6265SDimitry Andric AArch64::FPR64RegClass.contains(Reg) ||
96481ad6265SDimitry Andric AArch64::FPR32RegClass.contains(Reg) ||
96581ad6265SDimitry Andric AArch64::FPR16RegClass.contains(Reg) ||
96681ad6265SDimitry Andric AArch64::FPR8RegClass.contains(Reg)) {
96781ad6265SDimitry Andric // For FPRs,
96881ad6265SDimitry Andric if (MCRegister XReg = getRegisterOrZero(Reg, HasSVE))
96981ad6265SDimitry Andric FPRsToZero.set(XReg);
97081ad6265SDimitry Andric }
97181ad6265SDimitry Andric }
97281ad6265SDimitry Andric
97381ad6265SDimitry Andric const AArch64InstrInfo &TII = *STI.getInstrInfo();
97481ad6265SDimitry Andric
97581ad6265SDimitry Andric // Zero out GPRs.
97681ad6265SDimitry Andric for (MCRegister Reg : GPRsToZero.set_bits())
9775f757f3fSDimitry Andric TII.buildClearRegister(Reg, MBB, MBBI, DL);
97881ad6265SDimitry Andric
97981ad6265SDimitry Andric // Zero out FP/vector registers.
98081ad6265SDimitry Andric for (MCRegister Reg : FPRsToZero.set_bits())
9815f757f3fSDimitry Andric TII.buildClearRegister(Reg, MBB, MBBI, DL);
98281ad6265SDimitry Andric
98381ad6265SDimitry Andric if (HasSVE) {
98481ad6265SDimitry Andric for (MCRegister PReg :
98581ad6265SDimitry Andric {AArch64::P0, AArch64::P1, AArch64::P2, AArch64::P3, AArch64::P4,
98681ad6265SDimitry Andric AArch64::P5, AArch64::P6, AArch64::P7, AArch64::P8, AArch64::P9,
98781ad6265SDimitry Andric AArch64::P10, AArch64::P11, AArch64::P12, AArch64::P13, AArch64::P14,
98881ad6265SDimitry Andric AArch64::P15}) {
98981ad6265SDimitry Andric if (RegsToZero[PReg])
99081ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::PFALSE), PReg);
99181ad6265SDimitry Andric }
99281ad6265SDimitry Andric }
99381ad6265SDimitry Andric }
99481ad6265SDimitry Andric
getLiveRegsForEntryMBB(LivePhysRegs & LiveRegs,const MachineBasicBlock & MBB)9955f757f3fSDimitry Andric static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs,
9965f757f3fSDimitry Andric const MachineBasicBlock &MBB) {
9975f757f3fSDimitry Andric const MachineFunction *MF = MBB.getParent();
9985f757f3fSDimitry Andric LiveRegs.addLiveIns(MBB);
9995f757f3fSDimitry Andric // Mark callee saved registers as used so we will not choose them.
10005f757f3fSDimitry Andric const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
10015f757f3fSDimitry Andric for (unsigned i = 0; CSRegs[i]; ++i)
10025f757f3fSDimitry Andric LiveRegs.addReg(CSRegs[i]);
10035f757f3fSDimitry Andric }
10045f757f3fSDimitry Andric
10050b57cec5SDimitry Andric // Find a scratch register that we can use at the start of the prologue to
10060b57cec5SDimitry Andric // re-align the stack pointer. We avoid using callee-save registers since they
10070b57cec5SDimitry Andric // may appear to be free when this is called from canUseAsPrologue (during
10080b57cec5SDimitry Andric // shrink wrapping), but then no longer be free when this is called from
10090b57cec5SDimitry Andric // emitPrologue.
10100b57cec5SDimitry Andric //
10110b57cec5SDimitry Andric // FIXME: This is a bit conservative, since in the above case we could use one
10120b57cec5SDimitry Andric // of the callee-save registers as a scratch temp to re-align the stack pointer,
10130b57cec5SDimitry Andric // but we would then have to make sure that we were in fact saving at least one
10140b57cec5SDimitry Andric // callee-save register in the prologue, which is additional complexity that
10150b57cec5SDimitry Andric // doesn't seem worth the benefit.
findScratchNonCalleeSaveRegister(MachineBasicBlock * MBB)10160b57cec5SDimitry Andric static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
10170b57cec5SDimitry Andric MachineFunction *MF = MBB->getParent();
10180b57cec5SDimitry Andric
10190b57cec5SDimitry Andric // If MBB is an entry block, use X9 as the scratch register
10200b57cec5SDimitry Andric if (&MF->front() == MBB)
10210b57cec5SDimitry Andric return AArch64::X9;
10220b57cec5SDimitry Andric
10230b57cec5SDimitry Andric const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
10240b57cec5SDimitry Andric const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
10250b57cec5SDimitry Andric LivePhysRegs LiveRegs(TRI);
10265f757f3fSDimitry Andric getLiveRegsForEntryMBB(LiveRegs, *MBB);
10270b57cec5SDimitry Andric
10280b57cec5SDimitry Andric // Prefer X9 since it was historically used for the prologue scratch reg.
10290b57cec5SDimitry Andric const MachineRegisterInfo &MRI = MF->getRegInfo();
10300b57cec5SDimitry Andric if (LiveRegs.available(MRI, AArch64::X9))
10310b57cec5SDimitry Andric return AArch64::X9;
10320b57cec5SDimitry Andric
10330b57cec5SDimitry Andric for (unsigned Reg : AArch64::GPR64RegClass) {
10340b57cec5SDimitry Andric if (LiveRegs.available(MRI, Reg))
10350b57cec5SDimitry Andric return Reg;
10360b57cec5SDimitry Andric }
10370b57cec5SDimitry Andric return AArch64::NoRegister;
10380b57cec5SDimitry Andric }
10390b57cec5SDimitry Andric
canUseAsPrologue(const MachineBasicBlock & MBB) const10400b57cec5SDimitry Andric bool AArch64FrameLowering::canUseAsPrologue(
10410b57cec5SDimitry Andric const MachineBasicBlock &MBB) const {
10420b57cec5SDimitry Andric const MachineFunction *MF = MBB.getParent();
10430b57cec5SDimitry Andric MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
10440b57cec5SDimitry Andric const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
10450b57cec5SDimitry Andric const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
10465f757f3fSDimitry Andric const AArch64TargetLowering *TLI = Subtarget.getTargetLowering();
10475f757f3fSDimitry Andric const AArch64FunctionInfo *AFI = MF->getInfo<AArch64FunctionInfo>();
10480b57cec5SDimitry Andric
10495f757f3fSDimitry Andric if (AFI->hasSwiftAsyncContext()) {
10505f757f3fSDimitry Andric const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
10515f757f3fSDimitry Andric const MachineRegisterInfo &MRI = MF->getRegInfo();
10525f757f3fSDimitry Andric LivePhysRegs LiveRegs(TRI);
10535f757f3fSDimitry Andric getLiveRegsForEntryMBB(LiveRegs, MBB);
10545f757f3fSDimitry Andric // The StoreSwiftAsyncContext clobbers X16 and X17. Make sure they are
10555f757f3fSDimitry Andric // available.
10565f757f3fSDimitry Andric if (!LiveRegs.available(MRI, AArch64::X16) ||
10575f757f3fSDimitry Andric !LiveRegs.available(MRI, AArch64::X17))
10585f757f3fSDimitry Andric return false;
10595f757f3fSDimitry Andric }
10605f757f3fSDimitry Andric
10615f757f3fSDimitry Andric // Don't need a scratch register if we're not going to re-align the stack or
10625f757f3fSDimitry Andric // emit stack probes.
10635f757f3fSDimitry Andric if (!RegInfo->hasStackRealignment(*MF) && TLI->hasInlineStackProbe(*MF))
10640b57cec5SDimitry Andric return true;
10650b57cec5SDimitry Andric // Otherwise, we can use any block as long as it has a scratch register
10660b57cec5SDimitry Andric // available.
10670b57cec5SDimitry Andric return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
10680b57cec5SDimitry Andric }
10690b57cec5SDimitry Andric
windowsRequiresStackProbe(MachineFunction & MF,uint64_t StackSizeInBytes)10700b57cec5SDimitry Andric static bool windowsRequiresStackProbe(MachineFunction &MF,
1071480093f4SDimitry Andric uint64_t StackSizeInBytes) {
10720b57cec5SDimitry Andric const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
10735f757f3fSDimitry Andric const AArch64FunctionInfo &MFI = *MF.getInfo<AArch64FunctionInfo>();
10740b57cec5SDimitry Andric // TODO: When implementing stack protectors, take that into account
10750b57cec5SDimitry Andric // for the probe threshold.
10765f757f3fSDimitry Andric return Subtarget.isTargetWindows() && MFI.hasStackProbing() &&
10775f757f3fSDimitry Andric StackSizeInBytes >= uint64_t(MFI.getStackProbeSize());
10780b57cec5SDimitry Andric }
10790b57cec5SDimitry Andric
needsWinCFI(const MachineFunction & MF)1080e8d8bef9SDimitry Andric static bool needsWinCFI(const MachineFunction &MF) {
1081e8d8bef9SDimitry Andric const Function &F = MF.getFunction();
1082e8d8bef9SDimitry Andric return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
1083e8d8bef9SDimitry Andric F.needsUnwindTableEntry();
1084e8d8bef9SDimitry Andric }
1085e8d8bef9SDimitry Andric
shouldCombineCSRLocalStackBump(MachineFunction & MF,uint64_t StackBumpBytes) const10860b57cec5SDimitry Andric bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
1087480093f4SDimitry Andric MachineFunction &MF, uint64_t StackBumpBytes) const {
10880b57cec5SDimitry Andric AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
10890b57cec5SDimitry Andric const MachineFrameInfo &MFI = MF.getFrameInfo();
10900b57cec5SDimitry Andric const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
10910b57cec5SDimitry Andric const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1092fe6060f1SDimitry Andric if (homogeneousPrologEpilog(MF))
1093fe6060f1SDimitry Andric return false;
10940b57cec5SDimitry Andric
10950b57cec5SDimitry Andric if (AFI->getLocalStackSize() == 0)
10960b57cec5SDimitry Andric return false;
10970b57cec5SDimitry Andric
1098e8d8bef9SDimitry Andric // For WinCFI, if optimizing for size, prefer to not combine the stack bump
1099e8d8bef9SDimitry Andric // (to force a stp with predecrement) to match the packed unwind format,
1100e8d8bef9SDimitry Andric // provided that there actually are any callee saved registers to merge the
1101e8d8bef9SDimitry Andric // decrement with.
1102e8d8bef9SDimitry Andric // This is potentially marginally slower, but allows using the packed
1103e8d8bef9SDimitry Andric // unwind format for functions that both have a local area and callee saved
1104e8d8bef9SDimitry Andric // registers. Using the packed unwind format notably reduces the size of
1105e8d8bef9SDimitry Andric // the unwind info.
1106e8d8bef9SDimitry Andric if (needsWinCFI(MF) && AFI->getCalleeSavedStackSize() > 0 &&
1107e8d8bef9SDimitry Andric MF.getFunction().hasOptSize())
1108e8d8bef9SDimitry Andric return false;
1109e8d8bef9SDimitry Andric
11100b57cec5SDimitry Andric // 512 is the maximum immediate for stp/ldp that will be used for
11110b57cec5SDimitry Andric // callee-save save/restores
11120b57cec5SDimitry Andric if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
11130b57cec5SDimitry Andric return false;
11140b57cec5SDimitry Andric
11150b57cec5SDimitry Andric if (MFI.hasVarSizedObjects())
11160b57cec5SDimitry Andric return false;
11170b57cec5SDimitry Andric
1118fe6060f1SDimitry Andric if (RegInfo->hasStackRealignment(MF))
11190b57cec5SDimitry Andric return false;
11200b57cec5SDimitry Andric
11210b57cec5SDimitry Andric // This isn't strictly necessary, but it simplifies things a bit since the
11220b57cec5SDimitry Andric // current RedZone handling code assumes the SP is adjusted by the
11230b57cec5SDimitry Andric // callee-save save/restore code.
11240b57cec5SDimitry Andric if (canUseRedZone(MF))
11250b57cec5SDimitry Andric return false;
11260b57cec5SDimitry Andric
11278bcb0991SDimitry Andric // When there is an SVE area on the stack, always allocate the
11288bcb0991SDimitry Andric // callee-saves and spills/locals separately.
11298bcb0991SDimitry Andric if (getSVEStackSize(MF))
11308bcb0991SDimitry Andric return false;
11318bcb0991SDimitry Andric
11320b57cec5SDimitry Andric return true;
11330b57cec5SDimitry Andric }
11340b57cec5SDimitry Andric
shouldCombineCSRLocalStackBumpInEpilogue(MachineBasicBlock & MBB,unsigned StackBumpBytes) const11355ffd83dbSDimitry Andric bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
11365ffd83dbSDimitry Andric MachineBasicBlock &MBB, unsigned StackBumpBytes) const {
11375ffd83dbSDimitry Andric if (!shouldCombineCSRLocalStackBump(*MBB.getParent(), StackBumpBytes))
11385ffd83dbSDimitry Andric return false;
11395ffd83dbSDimitry Andric
11405ffd83dbSDimitry Andric if (MBB.empty())
11415ffd83dbSDimitry Andric return true;
11425ffd83dbSDimitry Andric
11435ffd83dbSDimitry Andric // Disable combined SP bump if the last instruction is an MTE tag store. It
11445ffd83dbSDimitry Andric // is almost always better to merge SP adjustment into those instructions.
11455ffd83dbSDimitry Andric MachineBasicBlock::iterator LastI = MBB.getFirstTerminator();
11465ffd83dbSDimitry Andric MachineBasicBlock::iterator Begin = MBB.begin();
11475ffd83dbSDimitry Andric while (LastI != Begin) {
11485ffd83dbSDimitry Andric --LastI;
11495ffd83dbSDimitry Andric if (LastI->isTransient())
11505ffd83dbSDimitry Andric continue;
11515ffd83dbSDimitry Andric if (!LastI->getFlag(MachineInstr::FrameDestroy))
11525ffd83dbSDimitry Andric break;
11535ffd83dbSDimitry Andric }
11545ffd83dbSDimitry Andric switch (LastI->getOpcode()) {
11555ffd83dbSDimitry Andric case AArch64::STGloop:
11565ffd83dbSDimitry Andric case AArch64::STZGloop:
115706c3fb27SDimitry Andric case AArch64::STGi:
115806c3fb27SDimitry Andric case AArch64::STZGi:
115906c3fb27SDimitry Andric case AArch64::ST2Gi:
116006c3fb27SDimitry Andric case AArch64::STZ2Gi:
11615ffd83dbSDimitry Andric return false;
11625ffd83dbSDimitry Andric default:
11635ffd83dbSDimitry Andric return true;
11645ffd83dbSDimitry Andric }
11655ffd83dbSDimitry Andric llvm_unreachable("unreachable");
11665ffd83dbSDimitry Andric }
11675ffd83dbSDimitry Andric
11680b57cec5SDimitry Andric // Given a load or a store instruction, generate an appropriate unwinding SEH
11690b57cec5SDimitry Andric // code on Windows.
InsertSEH(MachineBasicBlock::iterator MBBI,const TargetInstrInfo & TII,MachineInstr::MIFlag Flag)11700b57cec5SDimitry Andric static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI,
11710b57cec5SDimitry Andric const TargetInstrInfo &TII,
11720b57cec5SDimitry Andric MachineInstr::MIFlag Flag) {
11730b57cec5SDimitry Andric unsigned Opc = MBBI->getOpcode();
11740b57cec5SDimitry Andric MachineBasicBlock *MBB = MBBI->getParent();
11750b57cec5SDimitry Andric MachineFunction &MF = *MBB->getParent();
11760b57cec5SDimitry Andric DebugLoc DL = MBBI->getDebugLoc();
11770b57cec5SDimitry Andric unsigned ImmIdx = MBBI->getNumOperands() - 1;
11780b57cec5SDimitry Andric int Imm = MBBI->getOperand(ImmIdx).getImm();
11790b57cec5SDimitry Andric MachineInstrBuilder MIB;
11800b57cec5SDimitry Andric const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
11810b57cec5SDimitry Andric const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
11820b57cec5SDimitry Andric
11830b57cec5SDimitry Andric switch (Opc) {
11840b57cec5SDimitry Andric default:
11850b57cec5SDimitry Andric llvm_unreachable("No SEH Opcode for this instruction");
11860b57cec5SDimitry Andric case AArch64::LDPDpost:
11870b57cec5SDimitry Andric Imm = -Imm;
1188bdd1243dSDimitry Andric [[fallthrough]];
11890b57cec5SDimitry Andric case AArch64::STPDpre: {
11900b57cec5SDimitry Andric unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
11910b57cec5SDimitry Andric unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
11920b57cec5SDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
11930b57cec5SDimitry Andric .addImm(Reg0)
11940b57cec5SDimitry Andric .addImm(Reg1)
11950b57cec5SDimitry Andric .addImm(Imm * 8)
11960b57cec5SDimitry Andric .setMIFlag(Flag);
11970b57cec5SDimitry Andric break;
11980b57cec5SDimitry Andric }
11990b57cec5SDimitry Andric case AArch64::LDPXpost:
12000b57cec5SDimitry Andric Imm = -Imm;
1201bdd1243dSDimitry Andric [[fallthrough]];
12020b57cec5SDimitry Andric case AArch64::STPXpre: {
12038bcb0991SDimitry Andric Register Reg0 = MBBI->getOperand(1).getReg();
12048bcb0991SDimitry Andric Register Reg1 = MBBI->getOperand(2).getReg();
12050b57cec5SDimitry Andric if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
12060b57cec5SDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
12070b57cec5SDimitry Andric .addImm(Imm * 8)
12080b57cec5SDimitry Andric .setMIFlag(Flag);
12090b57cec5SDimitry Andric else
12100b57cec5SDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
12110b57cec5SDimitry Andric .addImm(RegInfo->getSEHRegNum(Reg0))
12120b57cec5SDimitry Andric .addImm(RegInfo->getSEHRegNum(Reg1))
12130b57cec5SDimitry Andric .addImm(Imm * 8)
12140b57cec5SDimitry Andric .setMIFlag(Flag);
12150b57cec5SDimitry Andric break;
12160b57cec5SDimitry Andric }
12170b57cec5SDimitry Andric case AArch64::LDRDpost:
12180b57cec5SDimitry Andric Imm = -Imm;
1219bdd1243dSDimitry Andric [[fallthrough]];
12200b57cec5SDimitry Andric case AArch64::STRDpre: {
12210b57cec5SDimitry Andric unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
12220b57cec5SDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
12230b57cec5SDimitry Andric .addImm(Reg)
12240b57cec5SDimitry Andric .addImm(Imm)
12250b57cec5SDimitry Andric .setMIFlag(Flag);
12260b57cec5SDimitry Andric break;
12270b57cec5SDimitry Andric }
12280b57cec5SDimitry Andric case AArch64::LDRXpost:
12290b57cec5SDimitry Andric Imm = -Imm;
1230bdd1243dSDimitry Andric [[fallthrough]];
12310b57cec5SDimitry Andric case AArch64::STRXpre: {
12320b57cec5SDimitry Andric unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
12330b57cec5SDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
12340b57cec5SDimitry Andric .addImm(Reg)
12350b57cec5SDimitry Andric .addImm(Imm)
12360b57cec5SDimitry Andric .setMIFlag(Flag);
12370b57cec5SDimitry Andric break;
12380b57cec5SDimitry Andric }
12390b57cec5SDimitry Andric case AArch64::STPDi:
12400b57cec5SDimitry Andric case AArch64::LDPDi: {
12410b57cec5SDimitry Andric unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
12420b57cec5SDimitry Andric unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
12430b57cec5SDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
12440b57cec5SDimitry Andric .addImm(Reg0)
12450b57cec5SDimitry Andric .addImm(Reg1)
12460b57cec5SDimitry Andric .addImm(Imm * 8)
12470b57cec5SDimitry Andric .setMIFlag(Flag);
12480b57cec5SDimitry Andric break;
12490b57cec5SDimitry Andric }
12500b57cec5SDimitry Andric case AArch64::STPXi:
12510b57cec5SDimitry Andric case AArch64::LDPXi: {
12528bcb0991SDimitry Andric Register Reg0 = MBBI->getOperand(0).getReg();
12538bcb0991SDimitry Andric Register Reg1 = MBBI->getOperand(1).getReg();
12540b57cec5SDimitry Andric if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
12550b57cec5SDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
12560b57cec5SDimitry Andric .addImm(Imm * 8)
12570b57cec5SDimitry Andric .setMIFlag(Flag);
12580b57cec5SDimitry Andric else
12590b57cec5SDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
12600b57cec5SDimitry Andric .addImm(RegInfo->getSEHRegNum(Reg0))
12610b57cec5SDimitry Andric .addImm(RegInfo->getSEHRegNum(Reg1))
12620b57cec5SDimitry Andric .addImm(Imm * 8)
12630b57cec5SDimitry Andric .setMIFlag(Flag);
12640b57cec5SDimitry Andric break;
12650b57cec5SDimitry Andric }
12660b57cec5SDimitry Andric case AArch64::STRXui:
12670b57cec5SDimitry Andric case AArch64::LDRXui: {
12680b57cec5SDimitry Andric int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
12690b57cec5SDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
12700b57cec5SDimitry Andric .addImm(Reg)
12710b57cec5SDimitry Andric .addImm(Imm * 8)
12720b57cec5SDimitry Andric .setMIFlag(Flag);
12730b57cec5SDimitry Andric break;
12740b57cec5SDimitry Andric }
12750b57cec5SDimitry Andric case AArch64::STRDui:
12760b57cec5SDimitry Andric case AArch64::LDRDui: {
12770b57cec5SDimitry Andric unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
12780b57cec5SDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
12790b57cec5SDimitry Andric .addImm(Reg)
12800b57cec5SDimitry Andric .addImm(Imm * 8)
12810b57cec5SDimitry Andric .setMIFlag(Flag);
12820b57cec5SDimitry Andric break;
12830b57cec5SDimitry Andric }
12847a6dacacSDimitry Andric case AArch64::STPQi:
12857a6dacacSDimitry Andric case AArch64::LDPQi: {
12867a6dacacSDimitry Andric unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
12877a6dacacSDimitry Andric unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
12887a6dacacSDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveAnyRegQP))
12897a6dacacSDimitry Andric .addImm(Reg0)
12907a6dacacSDimitry Andric .addImm(Reg1)
12917a6dacacSDimitry Andric .addImm(Imm * 16)
12927a6dacacSDimitry Andric .setMIFlag(Flag);
12937a6dacacSDimitry Andric break;
12947a6dacacSDimitry Andric }
12957a6dacacSDimitry Andric case AArch64::LDPQpost:
12967a6dacacSDimitry Andric Imm = -Imm;
12977a6dacacSDimitry Andric LLVM_FALLTHROUGH;
12987a6dacacSDimitry Andric case AArch64::STPQpre: {
12997a6dacacSDimitry Andric unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
13007a6dacacSDimitry Andric unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
13017a6dacacSDimitry Andric MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveAnyRegQPX))
13027a6dacacSDimitry Andric .addImm(Reg0)
13037a6dacacSDimitry Andric .addImm(Reg1)
13047a6dacacSDimitry Andric .addImm(Imm * 16)
13057a6dacacSDimitry Andric .setMIFlag(Flag);
13067a6dacacSDimitry Andric break;
13077a6dacacSDimitry Andric }
13080b57cec5SDimitry Andric }
13090b57cec5SDimitry Andric auto I = MBB->insertAfter(MBBI, MIB);
13100b57cec5SDimitry Andric return I;
13110b57cec5SDimitry Andric }
13120b57cec5SDimitry Andric
13130b57cec5SDimitry Andric // Fix up the SEH opcode associated with the save/restore instruction.
fixupSEHOpcode(MachineBasicBlock::iterator MBBI,unsigned LocalStackSize)13140b57cec5SDimitry Andric static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI,
13150b57cec5SDimitry Andric unsigned LocalStackSize) {
13160b57cec5SDimitry Andric MachineOperand *ImmOpnd = nullptr;
13170b57cec5SDimitry Andric unsigned ImmIdx = MBBI->getNumOperands() - 1;
13180b57cec5SDimitry Andric switch (MBBI->getOpcode()) {
13190b57cec5SDimitry Andric default:
13200b57cec5SDimitry Andric llvm_unreachable("Fix the offset in the SEH instruction");
13210b57cec5SDimitry Andric case AArch64::SEH_SaveFPLR:
13220b57cec5SDimitry Andric case AArch64::SEH_SaveRegP:
13230b57cec5SDimitry Andric case AArch64::SEH_SaveReg:
13240b57cec5SDimitry Andric case AArch64::SEH_SaveFRegP:
13250b57cec5SDimitry Andric case AArch64::SEH_SaveFReg:
13267a6dacacSDimitry Andric case AArch64::SEH_SaveAnyRegQP:
13277a6dacacSDimitry Andric case AArch64::SEH_SaveAnyRegQPX:
13280b57cec5SDimitry Andric ImmOpnd = &MBBI->getOperand(ImmIdx);
13290b57cec5SDimitry Andric break;
13300b57cec5SDimitry Andric }
13310b57cec5SDimitry Andric if (ImmOpnd)
13320b57cec5SDimitry Andric ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
13330b57cec5SDimitry Andric }
13340b57cec5SDimitry Andric
13350b57cec5SDimitry Andric // Convert callee-save register save/restore instruction to do stack pointer
13360b57cec5SDimitry Andric // decrement/increment to allocate/deallocate the callee-save stack area by
13370b57cec5SDimitry Andric // converting store/load to use pre/post increment version.
convertCalleeSaveRestoreToSPPrePostIncDec(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,const TargetInstrInfo * TII,int CSStackSizeInc,bool NeedsWinCFI,bool * HasWinCFI,bool EmitCFI,MachineInstr::MIFlag FrameFlag=MachineInstr::FrameSetup,int CFAOffset=0)13380b57cec5SDimitry Andric static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
13390b57cec5SDimitry Andric MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
13400b57cec5SDimitry Andric const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
134181ad6265SDimitry Andric bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI,
134281ad6265SDimitry Andric MachineInstr::MIFlag FrameFlag = MachineInstr::FrameSetup,
134381ad6265SDimitry Andric int CFAOffset = 0) {
13440b57cec5SDimitry Andric unsigned NewOpc;
13450b57cec5SDimitry Andric switch (MBBI->getOpcode()) {
13460b57cec5SDimitry Andric default:
13470b57cec5SDimitry Andric llvm_unreachable("Unexpected callee-save save/restore opcode!");
13480b57cec5SDimitry Andric case AArch64::STPXi:
13490b57cec5SDimitry Andric NewOpc = AArch64::STPXpre;
13500b57cec5SDimitry Andric break;
13510b57cec5SDimitry Andric case AArch64::STPDi:
13520b57cec5SDimitry Andric NewOpc = AArch64::STPDpre;
13530b57cec5SDimitry Andric break;
13540b57cec5SDimitry Andric case AArch64::STPQi:
13550b57cec5SDimitry Andric NewOpc = AArch64::STPQpre;
13560b57cec5SDimitry Andric break;
13570b57cec5SDimitry Andric case AArch64::STRXui:
13580b57cec5SDimitry Andric NewOpc = AArch64::STRXpre;
13590b57cec5SDimitry Andric break;
13600b57cec5SDimitry Andric case AArch64::STRDui:
13610b57cec5SDimitry Andric NewOpc = AArch64::STRDpre;
13620b57cec5SDimitry Andric break;
13630b57cec5SDimitry Andric case AArch64::STRQui:
13640b57cec5SDimitry Andric NewOpc = AArch64::STRQpre;
13650b57cec5SDimitry Andric break;
13660b57cec5SDimitry Andric case AArch64::LDPXi:
13670b57cec5SDimitry Andric NewOpc = AArch64::LDPXpost;
13680b57cec5SDimitry Andric break;
13690b57cec5SDimitry Andric case AArch64::LDPDi:
13700b57cec5SDimitry Andric NewOpc = AArch64::LDPDpost;
13710b57cec5SDimitry Andric break;
13720b57cec5SDimitry Andric case AArch64::LDPQi:
13730b57cec5SDimitry Andric NewOpc = AArch64::LDPQpost;
13740b57cec5SDimitry Andric break;
13750b57cec5SDimitry Andric case AArch64::LDRXui:
13760b57cec5SDimitry Andric NewOpc = AArch64::LDRXpost;
13770b57cec5SDimitry Andric break;
13780b57cec5SDimitry Andric case AArch64::LDRDui:
13790b57cec5SDimitry Andric NewOpc = AArch64::LDRDpost;
13800b57cec5SDimitry Andric break;
13810b57cec5SDimitry Andric case AArch64::LDRQui:
13820b57cec5SDimitry Andric NewOpc = AArch64::LDRQpost;
13830b57cec5SDimitry Andric break;
13840b57cec5SDimitry Andric }
13850b57cec5SDimitry Andric // Get rid of the SEH code associated with the old instruction.
13860b57cec5SDimitry Andric if (NeedsWinCFI) {
13870b57cec5SDimitry Andric auto SEH = std::next(MBBI);
13880b57cec5SDimitry Andric if (AArch64InstrInfo::isSEHInstruction(*SEH))
13890b57cec5SDimitry Andric SEH->eraseFromParent();
13900b57cec5SDimitry Andric }
13910b57cec5SDimitry Andric
13925f757f3fSDimitry Andric TypeSize Scale = TypeSize::getFixed(1), Width = TypeSize::getFixed(0);
1393fe6060f1SDimitry Andric int64_t MinOffset, MaxOffset;
1394fe6060f1SDimitry Andric bool Success = static_cast<const AArch64InstrInfo *>(TII)->getMemOpInfo(
1395fe6060f1SDimitry Andric NewOpc, Scale, Width, MinOffset, MaxOffset);
1396fe6060f1SDimitry Andric (void)Success;
1397fe6060f1SDimitry Andric assert(Success && "unknown load/store opcode");
1398fe6060f1SDimitry Andric
1399fe6060f1SDimitry Andric // If the first store isn't right where we want SP then we can't fold the
1400fe6060f1SDimitry Andric // update in so create a normal arithmetic instruction instead.
140181ad6265SDimitry Andric MachineFunction &MF = *MBB.getParent();
1402fe6060f1SDimitry Andric if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0 ||
1403fe6060f1SDimitry Andric CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) {
1404fe6060f1SDimitry Andric emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
140581ad6265SDimitry Andric StackOffset::getFixed(CSStackSizeInc), TII, FrameFlag,
140681ad6265SDimitry Andric false, false, nullptr, EmitCFI,
140781ad6265SDimitry Andric StackOffset::getFixed(CFAOffset));
140881ad6265SDimitry Andric
1409fe6060f1SDimitry Andric return std::prev(MBBI);
1410fe6060f1SDimitry Andric }
1411fe6060f1SDimitry Andric
14120b57cec5SDimitry Andric MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
14130b57cec5SDimitry Andric MIB.addReg(AArch64::SP, RegState::Define);
14140b57cec5SDimitry Andric
14150b57cec5SDimitry Andric // Copy all operands other than the immediate offset.
14160b57cec5SDimitry Andric unsigned OpndIdx = 0;
14170b57cec5SDimitry Andric for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
14180b57cec5SDimitry Andric ++OpndIdx)
14190b57cec5SDimitry Andric MIB.add(MBBI->getOperand(OpndIdx));
14200b57cec5SDimitry Andric
14210b57cec5SDimitry Andric assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
14220b57cec5SDimitry Andric "Unexpected immediate offset in first/last callee-save save/restore "
14230b57cec5SDimitry Andric "instruction!");
14240b57cec5SDimitry Andric assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
14250b57cec5SDimitry Andric "Unexpected base register in callee-save save/restore instruction!");
14260b57cec5SDimitry Andric assert(CSStackSizeInc % Scale == 0);
1427fe6060f1SDimitry Andric MIB.addImm(CSStackSizeInc / (int)Scale);
14280b57cec5SDimitry Andric
14290b57cec5SDimitry Andric MIB.setMIFlags(MBBI->getFlags());
14300b57cec5SDimitry Andric MIB.setMemRefs(MBBI->memoperands());
14310b57cec5SDimitry Andric
14320b57cec5SDimitry Andric // Generate a new SEH code that corresponds to the new instruction.
14330b57cec5SDimitry Andric if (NeedsWinCFI) {
14340b57cec5SDimitry Andric *HasWinCFI = true;
143581ad6265SDimitry Andric InsertSEH(*MIB, *TII, FrameFlag);
143681ad6265SDimitry Andric }
143781ad6265SDimitry Andric
143881ad6265SDimitry Andric if (EmitCFI) {
143981ad6265SDimitry Andric unsigned CFIIndex = MF.addFrameInst(
144081ad6265SDimitry Andric MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset - CSStackSizeInc));
144181ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
144281ad6265SDimitry Andric .addCFIIndex(CFIIndex)
144381ad6265SDimitry Andric .setMIFlags(FrameFlag);
14440b57cec5SDimitry Andric }
14450b57cec5SDimitry Andric
14460b57cec5SDimitry Andric return std::prev(MBB.erase(MBBI));
14470b57cec5SDimitry Andric }
14480b57cec5SDimitry Andric
14490b57cec5SDimitry Andric // Fixup callee-save register save/restore instructions to take into account
14500b57cec5SDimitry Andric // combined SP bump by adding the local stack size to the stack offsets.
fixupCalleeSaveRestoreStackOffset(MachineInstr & MI,uint64_t LocalStackSize,bool NeedsWinCFI,bool * HasWinCFI)14510b57cec5SDimitry Andric static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
1452480093f4SDimitry Andric uint64_t LocalStackSize,
14530b57cec5SDimitry Andric bool NeedsWinCFI,
14540b57cec5SDimitry Andric bool *HasWinCFI) {
14550b57cec5SDimitry Andric if (AArch64InstrInfo::isSEHInstruction(MI))
14560b57cec5SDimitry Andric return;
14570b57cec5SDimitry Andric
14580b57cec5SDimitry Andric unsigned Opc = MI.getOpcode();
14590b57cec5SDimitry Andric unsigned Scale;
14600b57cec5SDimitry Andric switch (Opc) {
14610b57cec5SDimitry Andric case AArch64::STPXi:
14620b57cec5SDimitry Andric case AArch64::STRXui:
14630b57cec5SDimitry Andric case AArch64::STPDi:
14640b57cec5SDimitry Andric case AArch64::STRDui:
14650b57cec5SDimitry Andric case AArch64::LDPXi:
14660b57cec5SDimitry Andric case AArch64::LDRXui:
14670b57cec5SDimitry Andric case AArch64::LDPDi:
14680b57cec5SDimitry Andric case AArch64::LDRDui:
14690b57cec5SDimitry Andric Scale = 8;
14700b57cec5SDimitry Andric break;
14710b57cec5SDimitry Andric case AArch64::STPQi:
14720b57cec5SDimitry Andric case AArch64::STRQui:
14730b57cec5SDimitry Andric case AArch64::LDPQi:
14740b57cec5SDimitry Andric case AArch64::LDRQui:
14750b57cec5SDimitry Andric Scale = 16;
14760b57cec5SDimitry Andric break;
14770b57cec5SDimitry Andric default:
14780b57cec5SDimitry Andric llvm_unreachable("Unexpected callee-save save/restore opcode!");
14790b57cec5SDimitry Andric }
14800b57cec5SDimitry Andric
14810b57cec5SDimitry Andric unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
14820b57cec5SDimitry Andric assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
14830b57cec5SDimitry Andric "Unexpected base register in callee-save save/restore instruction!");
14840b57cec5SDimitry Andric // Last operand is immediate offset that needs fixing.
14850b57cec5SDimitry Andric MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
14860b57cec5SDimitry Andric // All generated opcodes have scaled offsets.
14870b57cec5SDimitry Andric assert(LocalStackSize % Scale == 0);
14880b57cec5SDimitry Andric OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
14890b57cec5SDimitry Andric
14900b57cec5SDimitry Andric if (NeedsWinCFI) {
14910b57cec5SDimitry Andric *HasWinCFI = true;
14920b57cec5SDimitry Andric auto MBBI = std::next(MachineBasicBlock::iterator(MI));
14930b57cec5SDimitry Andric assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
14940b57cec5SDimitry Andric assert(AArch64InstrInfo::isSEHInstruction(*MBBI) &&
14950b57cec5SDimitry Andric "Expecting a SEH instruction");
14960b57cec5SDimitry Andric fixupSEHOpcode(MBBI, LocalStackSize);
14970b57cec5SDimitry Andric }
14980b57cec5SDimitry Andric }
14990b57cec5SDimitry Andric
isTargetWindows(const MachineFunction & MF)1500480093f4SDimitry Andric static bool isTargetWindows(const MachineFunction &MF) {
1501480093f4SDimitry Andric return MF.getSubtarget<AArch64Subtarget>().isTargetWindows();
1502480093f4SDimitry Andric }
1503480093f4SDimitry Andric
1504480093f4SDimitry Andric // Convenience function to determine whether I is an SVE callee save.
IsSVECalleeSave(MachineBasicBlock::iterator I)1505480093f4SDimitry Andric static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
1506480093f4SDimitry Andric switch (I->getOpcode()) {
1507480093f4SDimitry Andric default:
1508480093f4SDimitry Andric return false;
1509480093f4SDimitry Andric case AArch64::STR_ZXI:
1510480093f4SDimitry Andric case AArch64::STR_PXI:
1511480093f4SDimitry Andric case AArch64::LDR_ZXI:
1512480093f4SDimitry Andric case AArch64::LDR_PXI:
1513480093f4SDimitry Andric return I->getFlag(MachineInstr::FrameSetup) ||
1514480093f4SDimitry Andric I->getFlag(MachineInstr::FrameDestroy);
1515480093f4SDimitry Andric }
1516480093f4SDimitry Andric }
1517480093f4SDimitry Andric
emitShadowCallStackPrologue(const TargetInstrInfo & TII,MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool NeedsWinCFI,bool NeedsUnwindInfo)151881ad6265SDimitry Andric static void emitShadowCallStackPrologue(const TargetInstrInfo &TII,
151981ad6265SDimitry Andric MachineFunction &MF,
152081ad6265SDimitry Andric MachineBasicBlock &MBB,
152181ad6265SDimitry Andric MachineBasicBlock::iterator MBBI,
152281ad6265SDimitry Andric const DebugLoc &DL, bool NeedsWinCFI,
152381ad6265SDimitry Andric bool NeedsUnwindInfo) {
152481ad6265SDimitry Andric // Shadow call stack prolog: str x30, [x18], #8
152581ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXpost))
152681ad6265SDimitry Andric .addReg(AArch64::X18, RegState::Define)
152781ad6265SDimitry Andric .addReg(AArch64::LR)
152881ad6265SDimitry Andric .addReg(AArch64::X18)
152981ad6265SDimitry Andric .addImm(8)
153081ad6265SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
153181ad6265SDimitry Andric
153281ad6265SDimitry Andric // This instruction also makes x18 live-in to the entry block.
153381ad6265SDimitry Andric MBB.addLiveIn(AArch64::X18);
153481ad6265SDimitry Andric
153581ad6265SDimitry Andric if (NeedsWinCFI)
153681ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::SEH_Nop))
153781ad6265SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
153881ad6265SDimitry Andric
153981ad6265SDimitry Andric if (NeedsUnwindInfo) {
154081ad6265SDimitry Andric // Emit a CFI instruction that causes 8 to be subtracted from the value of
154181ad6265SDimitry Andric // x18 when unwinding past this frame.
154281ad6265SDimitry Andric static const char CFIInst[] = {
154381ad6265SDimitry Andric dwarf::DW_CFA_val_expression,
154481ad6265SDimitry Andric 18, // register
154581ad6265SDimitry Andric 2, // length
154681ad6265SDimitry Andric static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
154781ad6265SDimitry Andric static_cast<char>(-8) & 0x7f, // addend (sleb128)
154881ad6265SDimitry Andric };
154981ad6265SDimitry Andric unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
155081ad6265SDimitry Andric nullptr, StringRef(CFIInst, sizeof(CFIInst))));
155181ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::CFI_INSTRUCTION))
155281ad6265SDimitry Andric .addCFIIndex(CFIIndex)
155381ad6265SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
155481ad6265SDimitry Andric }
155581ad6265SDimitry Andric }
155681ad6265SDimitry Andric
emitShadowCallStackEpilogue(const TargetInstrInfo & TII,MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL)155781ad6265SDimitry Andric static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII,
155881ad6265SDimitry Andric MachineFunction &MF,
155981ad6265SDimitry Andric MachineBasicBlock &MBB,
156081ad6265SDimitry Andric MachineBasicBlock::iterator MBBI,
156181ad6265SDimitry Andric const DebugLoc &DL) {
156281ad6265SDimitry Andric // Shadow call stack epilog: ldr x30, [x18, #-8]!
156381ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::LDRXpre))
156481ad6265SDimitry Andric .addReg(AArch64::X18, RegState::Define)
156581ad6265SDimitry Andric .addReg(AArch64::LR, RegState::Define)
156681ad6265SDimitry Andric .addReg(AArch64::X18)
156781ad6265SDimitry Andric .addImm(-8)
156881ad6265SDimitry Andric .setMIFlag(MachineInstr::FrameDestroy);
156981ad6265SDimitry Andric
1570bdd1243dSDimitry Andric if (MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF)) {
157181ad6265SDimitry Andric unsigned CFIIndex =
157281ad6265SDimitry Andric MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, 18));
157381ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
157481ad6265SDimitry Andric .addCFIIndex(CFIIndex)
157581ad6265SDimitry Andric .setMIFlags(MachineInstr::FrameDestroy);
157681ad6265SDimitry Andric }
157781ad6265SDimitry Andric }
157881ad6265SDimitry Andric
157906c3fb27SDimitry Andric // Define the current CFA rule to use the provided FP.
emitDefineCFAWithFP(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,unsigned FixedObject)158006c3fb27SDimitry Andric static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB,
158106c3fb27SDimitry Andric MachineBasicBlock::iterator MBBI,
158206c3fb27SDimitry Andric const DebugLoc &DL, unsigned FixedObject) {
158306c3fb27SDimitry Andric const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
158406c3fb27SDimitry Andric const AArch64RegisterInfo *TRI = STI.getRegisterInfo();
158506c3fb27SDimitry Andric const TargetInstrInfo *TII = STI.getInstrInfo();
158606c3fb27SDimitry Andric AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
158706c3fb27SDimitry Andric
158806c3fb27SDimitry Andric const int OffsetToFirstCalleeSaveFromFP =
158906c3fb27SDimitry Andric AFI->getCalleeSaveBaseToFrameRecordOffset() -
159006c3fb27SDimitry Andric AFI->getCalleeSavedStackSize();
159106c3fb27SDimitry Andric Register FramePtr = TRI->getFrameRegister(MF);
159206c3fb27SDimitry Andric unsigned Reg = TRI->getDwarfRegNum(FramePtr, true);
159306c3fb27SDimitry Andric unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
159406c3fb27SDimitry Andric nullptr, Reg, FixedObject - OffsetToFirstCalleeSaveFromFP));
159506c3fb27SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
159606c3fb27SDimitry Andric .addCFIIndex(CFIIndex)
159706c3fb27SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
159806c3fb27SDimitry Andric }
159906c3fb27SDimitry Andric
16005f757f3fSDimitry Andric #ifndef NDEBUG
16015f757f3fSDimitry Andric /// Collect live registers from the end of \p MI's parent up to (including) \p
16025f757f3fSDimitry Andric /// MI in \p LiveRegs.
getLivePhysRegsUpTo(MachineInstr & MI,const TargetRegisterInfo & TRI,LivePhysRegs & LiveRegs)16035f757f3fSDimitry Andric static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI,
16045f757f3fSDimitry Andric LivePhysRegs &LiveRegs) {
16055f757f3fSDimitry Andric
16065f757f3fSDimitry Andric MachineBasicBlock &MBB = *MI.getParent();
16075f757f3fSDimitry Andric LiveRegs.addLiveOuts(MBB);
16085f757f3fSDimitry Andric for (const MachineInstr &MI :
16095f757f3fSDimitry Andric reverse(make_range(MI.getIterator(), MBB.instr_end())))
16105f757f3fSDimitry Andric LiveRegs.stepBackward(MI);
16115f757f3fSDimitry Andric }
16125f757f3fSDimitry Andric #endif
16135f757f3fSDimitry Andric
emitPrologue(MachineFunction & MF,MachineBasicBlock & MBB) const16140b57cec5SDimitry Andric void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
16150b57cec5SDimitry Andric MachineBasicBlock &MBB) const {
16160b57cec5SDimitry Andric MachineBasicBlock::iterator MBBI = MBB.begin();
16170b57cec5SDimitry Andric const MachineFrameInfo &MFI = MF.getFrameInfo();
16180b57cec5SDimitry Andric const Function &F = MF.getFunction();
16190b57cec5SDimitry Andric const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
16200b57cec5SDimitry Andric const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
16210b57cec5SDimitry Andric const TargetInstrInfo *TII = Subtarget.getInstrInfo();
16225f757f3fSDimitry Andric
16230b57cec5SDimitry Andric MachineModuleInfo &MMI = MF.getMMI();
16240b57cec5SDimitry Andric AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1625bdd1243dSDimitry Andric bool EmitCFI = AFI->needsDwarfUnwindInfo(MF);
162606c3fb27SDimitry Andric bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
16270b57cec5SDimitry Andric bool HasFP = hasFP(MF);
16280b57cec5SDimitry Andric bool NeedsWinCFI = needsWinCFI(MF);
16290b57cec5SDimitry Andric bool HasWinCFI = false;
16300b57cec5SDimitry Andric auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); });
16310b57cec5SDimitry Andric
16325f757f3fSDimitry Andric MachineBasicBlock::iterator End = MBB.end();
16335f757f3fSDimitry Andric #ifndef NDEBUG
16345f757f3fSDimitry Andric const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
16355f757f3fSDimitry Andric // Collect live register from the end of MBB up to the start of the existing
16365f757f3fSDimitry Andric // frame setup instructions.
16375f757f3fSDimitry Andric MachineBasicBlock::iterator NonFrameStart = MBB.begin();
16385f757f3fSDimitry Andric while (NonFrameStart != End &&
16395f757f3fSDimitry Andric NonFrameStart->getFlag(MachineInstr::FrameSetup))
16405f757f3fSDimitry Andric ++NonFrameStart;
16415f757f3fSDimitry Andric
16425f757f3fSDimitry Andric LivePhysRegs LiveRegs(*TRI);
16435f757f3fSDimitry Andric if (NonFrameStart != MBB.end()) {
16445f757f3fSDimitry Andric getLivePhysRegsUpTo(*NonFrameStart, *TRI, LiveRegs);
16455f757f3fSDimitry Andric // Ignore registers used for stack management for now.
16465f757f3fSDimitry Andric LiveRegs.removeReg(AArch64::SP);
16475f757f3fSDimitry Andric LiveRegs.removeReg(AArch64::X19);
16485f757f3fSDimitry Andric LiveRegs.removeReg(AArch64::FP);
16495f757f3fSDimitry Andric LiveRegs.removeReg(AArch64::LR);
16505f757f3fSDimitry Andric }
16515f757f3fSDimitry Andric
16525f757f3fSDimitry Andric auto VerifyClobberOnExit = make_scope_exit([&]() {
16535f757f3fSDimitry Andric if (NonFrameStart == MBB.end())
16545f757f3fSDimitry Andric return;
16555f757f3fSDimitry Andric // Check if any of the newly instructions clobber any of the live registers.
16565f757f3fSDimitry Andric for (MachineInstr &MI :
16575f757f3fSDimitry Andric make_range(MBB.instr_begin(), NonFrameStart->getIterator())) {
16585f757f3fSDimitry Andric for (auto &Op : MI.operands())
16595f757f3fSDimitry Andric if (Op.isReg() && Op.isDef())
16605f757f3fSDimitry Andric assert(!LiveRegs.contains(Op.getReg()) &&
16615f757f3fSDimitry Andric "live register clobbered by inserted prologue instructions");
16625f757f3fSDimitry Andric }
16635f757f3fSDimitry Andric });
16645f757f3fSDimitry Andric #endif
16655f757f3fSDimitry Andric
16660b57cec5SDimitry Andric bool IsFunclet = MBB.isEHFuncletEntry();
16670b57cec5SDimitry Andric
16680b57cec5SDimitry Andric // At this point, we're going to decide whether or not the function uses a
16690b57cec5SDimitry Andric // redzone. In most cases, the function doesn't have a redzone so let's
16700b57cec5SDimitry Andric // assume that's false and set it to true in the case that there's a redzone.
16710b57cec5SDimitry Andric AFI->setHasRedZone(false);
16720b57cec5SDimitry Andric
16730b57cec5SDimitry Andric // Debug location must be unknown since the first debug location is used
16740b57cec5SDimitry Andric // to determine the end of the prologue.
16750b57cec5SDimitry Andric DebugLoc DL;
16760b57cec5SDimitry Andric
1677e8d8bef9SDimitry Andric const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
16785f757f3fSDimitry Andric if (MFnI.needsShadowCallStackPrologueEpilogue(MF))
167981ad6265SDimitry Andric emitShadowCallStackPrologue(*TII, MF, MBB, MBBI, DL, NeedsWinCFI,
1680bdd1243dSDimitry Andric MFnI.needsDwarfUnwindInfo(MF));
1681fe6060f1SDimitry Andric
1682bdd1243dSDimitry Andric if (MFnI.shouldSignReturnAddress(MF)) {
16835f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::PAUTH_PROLOGUE))
16840b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
16855f757f3fSDimitry Andric if (NeedsWinCFI)
16865f757f3fSDimitry Andric HasWinCFI = true; // AArch64PointerAuth pass will insert SEH_PACSignLR
16870b57cec5SDimitry Andric }
16880b57cec5SDimitry Andric
168981ad6265SDimitry Andric if (EmitCFI && MFnI.isMTETagged()) {
169081ad6265SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITMTETAGGED))
169181ad6265SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
169281ad6265SDimitry Andric }
16930b57cec5SDimitry Andric
1694fe6060f1SDimitry Andric // We signal the presence of a Swift extended frame to external tools by
1695fe6060f1SDimitry Andric // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple
1696fe6060f1SDimitry Andric // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI
1697fe6060f1SDimitry Andric // bits so that is still true.
1698fe6060f1SDimitry Andric if (HasFP && AFI->hasSwiftAsyncContext()) {
1699349cc55cSDimitry Andric switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
1700349cc55cSDimitry Andric case SwiftAsyncFramePointerMode::DeploymentBased:
1701349cc55cSDimitry Andric if (Subtarget.swiftAsyncContextIsDynamicallySet()) {
1702349cc55cSDimitry Andric // The special symbol below is absolute and has a *value* that can be
1703349cc55cSDimitry Andric // combined with the frame pointer to signal an extended frame.
1704349cc55cSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::LOADgot), AArch64::X16)
1705349cc55cSDimitry Andric .addExternalSymbol("swift_async_extendedFramePointerFlags",
1706349cc55cSDimitry Andric AArch64II::MO_GOT);
17075f757f3fSDimitry Andric if (NeedsWinCFI) {
17085f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
17095f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
17105f757f3fSDimitry Andric HasWinCFI = true;
17115f757f3fSDimitry Andric }
1712349cc55cSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::FP)
1713349cc55cSDimitry Andric .addUse(AArch64::FP)
1714349cc55cSDimitry Andric .addUse(AArch64::X16)
1715349cc55cSDimitry Andric .addImm(Subtarget.isTargetILP32() ? 32 : 0);
17165f757f3fSDimitry Andric if (NeedsWinCFI) {
17175f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
17185f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
17195f757f3fSDimitry Andric HasWinCFI = true;
17205f757f3fSDimitry Andric }
1721349cc55cSDimitry Andric break;
1722349cc55cSDimitry Andric }
1723bdd1243dSDimitry Andric [[fallthrough]];
1724349cc55cSDimitry Andric
1725349cc55cSDimitry Andric case SwiftAsyncFramePointerMode::Always:
1726fe6060f1SDimitry Andric // ORR x29, x29, #0x1000_0000_0000_0000
1727fe6060f1SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP)
1728fe6060f1SDimitry Andric .addUse(AArch64::FP)
1729fe6060f1SDimitry Andric .addImm(0x1100)
1730fe6060f1SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
17315f757f3fSDimitry Andric if (NeedsWinCFI) {
17325f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
17335f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
17345f757f3fSDimitry Andric HasWinCFI = true;
17355f757f3fSDimitry Andric }
1736349cc55cSDimitry Andric break;
1737349cc55cSDimitry Andric
1738349cc55cSDimitry Andric case SwiftAsyncFramePointerMode::Never:
1739349cc55cSDimitry Andric break;
1740349cc55cSDimitry Andric }
1741fe6060f1SDimitry Andric }
1742fe6060f1SDimitry Andric
17430b57cec5SDimitry Andric // All calls are tail calls in GHC calling conv, and functions have no
17440b57cec5SDimitry Andric // prologue/epilogue.
17450b57cec5SDimitry Andric if (MF.getFunction().getCallingConv() == CallingConv::GHC)
17460b57cec5SDimitry Andric return;
17470b57cec5SDimitry Andric
1748e8d8bef9SDimitry Andric // Set tagged base pointer to the requested stack slot.
17490b57cec5SDimitry Andric // Ideally it should match SP value after prologue.
1750bdd1243dSDimitry Andric std::optional<int> TBPI = AFI->getTaggedBasePointerIndex();
1751e8d8bef9SDimitry Andric if (TBPI)
1752e8d8bef9SDimitry Andric AFI->setTaggedBasePointerOffset(-MFI.getObjectOffset(*TBPI));
1753e8d8bef9SDimitry Andric else
17540b57cec5SDimitry Andric AFI->setTaggedBasePointerOffset(MFI.getStackSize());
17550b57cec5SDimitry Andric
17568bcb0991SDimitry Andric const StackOffset &SVEStackSize = getSVEStackSize(MF);
17578bcb0991SDimitry Andric
17580b57cec5SDimitry Andric // getStackSize() includes all the locals in its size calculation. We don't
17590b57cec5SDimitry Andric // include these locals when computing the stack size of a funclet, as they
17600b57cec5SDimitry Andric // are allocated in the parent's stack frame and accessed via the frame
17610b57cec5SDimitry Andric // pointer from the funclet. We only save the callee saved registers in the
17620b57cec5SDimitry Andric // funclet, which are really the callee saved registers of the parent
17630b57cec5SDimitry Andric // function, including the funclet.
1764480093f4SDimitry Andric int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1765480093f4SDimitry Andric : MFI.getStackSize();
17660b57cec5SDimitry Andric if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
17670b57cec5SDimitry Andric assert(!HasFP && "unexpected function without stack frame but with FP");
17688bcb0991SDimitry Andric assert(!SVEStackSize &&
17698bcb0991SDimitry Andric "unexpected function without stack frame but with SVE objects");
17700b57cec5SDimitry Andric // All of the stack allocation is for locals.
17710b57cec5SDimitry Andric AFI->setLocalStackSize(NumBytes);
17720b57cec5SDimitry Andric if (!NumBytes)
17730b57cec5SDimitry Andric return;
17740b57cec5SDimitry Andric // REDZONE: If the stack size is less than 128 bytes, we don't need
17750b57cec5SDimitry Andric // to actually allocate.
17760b57cec5SDimitry Andric if (canUseRedZone(MF)) {
17770b57cec5SDimitry Andric AFI->setHasRedZone(true);
17780b57cec5SDimitry Andric ++NumRedZoneFunctions;
17790b57cec5SDimitry Andric } else {
17808bcb0991SDimitry Andric emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1781e8d8bef9SDimitry Andric StackOffset::getFixed(-NumBytes), TII,
1782e8d8bef9SDimitry Andric MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
178381ad6265SDimitry Andric if (EmitCFI) {
17840b57cec5SDimitry Andric // Label used to tie together the PROLOG_LABEL and the MachineMoves.
17850b57cec5SDimitry Andric MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
17860b57cec5SDimitry Andric // Encode the stack size of the leaf function.
17870b57cec5SDimitry Andric unsigned CFIIndex = MF.addFrameInst(
17885ffd83dbSDimitry Andric MCCFIInstruction::cfiDefCfaOffset(FrameLabel, NumBytes));
17890b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
17900b57cec5SDimitry Andric .addCFIIndex(CFIIndex)
17910b57cec5SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
17920b57cec5SDimitry Andric }
17930b57cec5SDimitry Andric }
17940b57cec5SDimitry Andric
17950b57cec5SDimitry Andric if (NeedsWinCFI) {
17960b57cec5SDimitry Andric HasWinCFI = true;
17970b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
17980b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
17990b57cec5SDimitry Andric }
18000b57cec5SDimitry Andric
18010b57cec5SDimitry Andric return;
18020b57cec5SDimitry Andric }
18030b57cec5SDimitry Andric
18040b57cec5SDimitry Andric bool IsWin64 =
18050b57cec5SDimitry Andric Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
180662cfcf62SDimitry Andric unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
18070b57cec5SDimitry Andric
18080b57cec5SDimitry Andric auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
18090b57cec5SDimitry Andric // All of the remaining stack allocations are for locals.
18100b57cec5SDimitry Andric AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
18110b57cec5SDimitry Andric bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1812fe6060f1SDimitry Andric bool HomPrologEpilog = homogeneousPrologEpilog(MF);
18130b57cec5SDimitry Andric if (CombineSPBump) {
18148bcb0991SDimitry Andric assert(!SVEStackSize && "Cannot combine SP bump with SVE");
18158bcb0991SDimitry Andric emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1816e8d8bef9SDimitry Andric StackOffset::getFixed(-NumBytes), TII,
181781ad6265SDimitry Andric MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI,
181806c3fb27SDimitry Andric EmitAsyncCFI);
18190b57cec5SDimitry Andric NumBytes = 0;
1820fe6060f1SDimitry Andric } else if (HomPrologEpilog) {
1821fe6060f1SDimitry Andric // Stack has been already adjusted.
1822fe6060f1SDimitry Andric NumBytes -= PrologueSaveSize;
18230b57cec5SDimitry Andric } else if (PrologueSaveSize != 0) {
18240b57cec5SDimitry Andric MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(
182581ad6265SDimitry Andric MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI,
182606c3fb27SDimitry Andric EmitAsyncCFI);
18270b57cec5SDimitry Andric NumBytes -= PrologueSaveSize;
18280b57cec5SDimitry Andric }
18290b57cec5SDimitry Andric assert(NumBytes >= 0 && "Negative stack allocation size!?");
18300b57cec5SDimitry Andric
18310b57cec5SDimitry Andric // Move past the saves of the callee-saved registers, fixing up the offsets
18320b57cec5SDimitry Andric // and pre-inc if we decided to combine the callee-save and local stack
18330b57cec5SDimitry Andric // pointer bump above.
1834480093f4SDimitry Andric while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) &&
1835480093f4SDimitry Andric !IsSVECalleeSave(MBBI)) {
18360b57cec5SDimitry Andric if (CombineSPBump)
18370b57cec5SDimitry Andric fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(),
18380b57cec5SDimitry Andric NeedsWinCFI, &HasWinCFI);
18390b57cec5SDimitry Andric ++MBBI;
18400b57cec5SDimitry Andric }
18410b57cec5SDimitry Andric
184262cfcf62SDimitry Andric // For funclets the FP belongs to the containing function.
184362cfcf62SDimitry Andric if (!IsFunclet && HasFP) {
18448bcb0991SDimitry Andric // Only set up FP if we actually need to.
1845e8d8bef9SDimitry Andric int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset();
18468bcb0991SDimitry Andric
18470b57cec5SDimitry Andric if (CombineSPBump)
18480b57cec5SDimitry Andric FPOffset += AFI->getLocalStackSize();
18490b57cec5SDimitry Andric
1850fe6060f1SDimitry Andric if (AFI->hasSwiftAsyncContext()) {
1851fe6060f1SDimitry Andric // Before we update the live FP we have to ensure there's a valid (or
1852fe6060f1SDimitry Andric // null) asynchronous context in its slot just before FP in the frame
1853fe6060f1SDimitry Andric // record, so store it now.
1854fe6060f1SDimitry Andric const auto &Attrs = MF.getFunction().getAttributes();
1855fe6060f1SDimitry Andric bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync);
1856fe6060f1SDimitry Andric if (HaveInitialContext)
1857fe6060f1SDimitry Andric MBB.addLiveIn(AArch64::X22);
18585f757f3fSDimitry Andric Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR;
1859fe6060f1SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext))
18605f757f3fSDimitry Andric .addUse(Reg)
1861fe6060f1SDimitry Andric .addUse(AArch64::SP)
1862fe6060f1SDimitry Andric .addImm(FPOffset - 8)
1863fe6060f1SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
18645f757f3fSDimitry Andric if (NeedsWinCFI) {
18655f757f3fSDimitry Andric // WinCFI and arm64e, where StoreSwiftAsyncContext is expanded
18665f757f3fSDimitry Andric // to multiple instructions, should be mutually-exclusive.
18675f757f3fSDimitry Andric assert(Subtarget.getTargetTriple().getArchName() != "arm64e");
18685f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
18695f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
18705f757f3fSDimitry Andric HasWinCFI = true;
18715f757f3fSDimitry Andric }
1872fe6060f1SDimitry Andric }
1873fe6060f1SDimitry Andric
1874fe6060f1SDimitry Andric if (HomPrologEpilog) {
1875fe6060f1SDimitry Andric auto Prolog = MBBI;
1876fe6060f1SDimitry Andric --Prolog;
1877fe6060f1SDimitry Andric assert(Prolog->getOpcode() == AArch64::HOM_Prolog);
1878fe6060f1SDimitry Andric Prolog->addOperand(MachineOperand::CreateImm(FPOffset));
1879fe6060f1SDimitry Andric } else {
18800b57cec5SDimitry Andric // Issue sub fp, sp, FPOffset or
18810b57cec5SDimitry Andric // mov fp,sp when FPOffset is zero.
18820b57cec5SDimitry Andric // Note: All stores of callee-saved registers are marked as "FrameSetup".
18830b57cec5SDimitry Andric // This code marks the instruction(s) that set the FP also.
18848bcb0991SDimitry Andric emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP,
1885e8d8bef9SDimitry Andric StackOffset::getFixed(FPOffset), TII,
1886e8d8bef9SDimitry Andric MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
1887bdd1243dSDimitry Andric if (NeedsWinCFI && HasWinCFI) {
1888bdd1243dSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1889bdd1243dSDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
1890bdd1243dSDimitry Andric // After setting up the FP, the rest of the prolog doesn't need to be
1891bdd1243dSDimitry Andric // included in the SEH unwind info.
1892bdd1243dSDimitry Andric NeedsWinCFI = false;
1893bdd1243dSDimitry Andric }
18940b57cec5SDimitry Andric }
189506c3fb27SDimitry Andric if (EmitAsyncCFI)
189606c3fb27SDimitry Andric emitDefineCFAWithFP(MF, MBB, MBBI, DL, FixedObject);
189781ad6265SDimitry Andric }
189881ad6265SDimitry Andric
189981ad6265SDimitry Andric // Now emit the moves for whatever callee saved regs we have (including FP,
190081ad6265SDimitry Andric // LR if those are saved). Frame instructions for SVE register are emitted
190181ad6265SDimitry Andric // later, after the instruction which actually save SVE regs.
190206c3fb27SDimitry Andric if (EmitAsyncCFI)
190381ad6265SDimitry Andric emitCalleeSavedGPRLocations(MBB, MBBI);
19040b57cec5SDimitry Andric
1905bdd1243dSDimitry Andric // Alignment is required for the parent frame, not the funclet
1906bdd1243dSDimitry Andric const bool NeedsRealignment =
1907bdd1243dSDimitry Andric NumBytes && !IsFunclet && RegInfo->hasStackRealignment(MF);
19085f757f3fSDimitry Andric const int64_t RealignmentPadding =
1909bdd1243dSDimitry Andric (NeedsRealignment && MFI.getMaxAlign() > Align(16))
1910bdd1243dSDimitry Andric ? MFI.getMaxAlign().value() - 16
1911bdd1243dSDimitry Andric : 0;
1912bdd1243dSDimitry Andric
1913bdd1243dSDimitry Andric if (windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding)) {
1914bdd1243dSDimitry Andric uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4;
19150b57cec5SDimitry Andric if (NeedsWinCFI) {
19160b57cec5SDimitry Andric HasWinCFI = true;
19170b57cec5SDimitry Andric // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
19180b57cec5SDimitry Andric // exceed this amount. We need to move at most 2^24 - 1 into x15.
19190b57cec5SDimitry Andric // This is at most two instructions, MOVZ follwed by MOVK.
19200b57cec5SDimitry Andric // TODO: Fix to use multiple stack alloc unwind codes for stacks
19210b57cec5SDimitry Andric // exceeding 256MB in size.
19220b57cec5SDimitry Andric if (NumBytes >= (1 << 28))
19230b57cec5SDimitry Andric report_fatal_error("Stack size cannot exceed 256MB for stack "
19240b57cec5SDimitry Andric "unwinding purposes");
19250b57cec5SDimitry Andric
19260b57cec5SDimitry Andric uint32_t LowNumWords = NumWords & 0xFFFF;
19270b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
19280b57cec5SDimitry Andric .addImm(LowNumWords)
19290b57cec5SDimitry Andric .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
19300b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
19310b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
19320b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
19330b57cec5SDimitry Andric if ((NumWords & 0xFFFF0000) != 0) {
19340b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
19350b57cec5SDimitry Andric .addReg(AArch64::X15)
19360b57cec5SDimitry Andric .addImm((NumWords & 0xFFFF0000) >> 16) // High half
19370b57cec5SDimitry Andric .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16))
19380b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
19390b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
19400b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
19410b57cec5SDimitry Andric }
19420b57cec5SDimitry Andric } else {
19430b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
19440b57cec5SDimitry Andric .addImm(NumWords)
19450b57cec5SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
19460b57cec5SDimitry Andric }
19470b57cec5SDimitry Andric
1948bdd1243dSDimitry Andric const char* ChkStk = Subtarget.getChkStkName();
19490b57cec5SDimitry Andric switch (MF.getTarget().getCodeModel()) {
19500b57cec5SDimitry Andric case CodeModel::Tiny:
19510b57cec5SDimitry Andric case CodeModel::Small:
19520b57cec5SDimitry Andric case CodeModel::Medium:
19530b57cec5SDimitry Andric case CodeModel::Kernel:
19540b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
1955bdd1243dSDimitry Andric .addExternalSymbol(ChkStk)
19560b57cec5SDimitry Andric .addReg(AArch64::X15, RegState::Implicit)
19570b57cec5SDimitry Andric .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
19580b57cec5SDimitry Andric .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
19590b57cec5SDimitry Andric .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
19600b57cec5SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
19610b57cec5SDimitry Andric if (NeedsWinCFI) {
19620b57cec5SDimitry Andric HasWinCFI = true;
19630b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
19640b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
19650b57cec5SDimitry Andric }
19660b57cec5SDimitry Andric break;
19670b57cec5SDimitry Andric case CodeModel::Large:
19680b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
19690b57cec5SDimitry Andric .addReg(AArch64::X16, RegState::Define)
1970bdd1243dSDimitry Andric .addExternalSymbol(ChkStk)
1971bdd1243dSDimitry Andric .addExternalSymbol(ChkStk)
19720b57cec5SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
19730b57cec5SDimitry Andric if (NeedsWinCFI) {
19740b57cec5SDimitry Andric HasWinCFI = true;
19750b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
19760b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
19770b57cec5SDimitry Andric }
19780b57cec5SDimitry Andric
19795ffd83dbSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF)))
19800b57cec5SDimitry Andric .addReg(AArch64::X16, RegState::Kill)
19810b57cec5SDimitry Andric .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
19820b57cec5SDimitry Andric .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
19830b57cec5SDimitry Andric .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
19840b57cec5SDimitry Andric .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
19850b57cec5SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
19860b57cec5SDimitry Andric if (NeedsWinCFI) {
19870b57cec5SDimitry Andric HasWinCFI = true;
19880b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
19890b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
19900b57cec5SDimitry Andric }
19910b57cec5SDimitry Andric break;
19920b57cec5SDimitry Andric }
19930b57cec5SDimitry Andric
19940b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
19950b57cec5SDimitry Andric .addReg(AArch64::SP, RegState::Kill)
19960b57cec5SDimitry Andric .addReg(AArch64::X15, RegState::Kill)
19970b57cec5SDimitry Andric .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4))
19980b57cec5SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
19990b57cec5SDimitry Andric if (NeedsWinCFI) {
20000b57cec5SDimitry Andric HasWinCFI = true;
20010b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
20020b57cec5SDimitry Andric .addImm(NumBytes)
20030b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
20040b57cec5SDimitry Andric }
20050b57cec5SDimitry Andric NumBytes = 0;
2006bdd1243dSDimitry Andric
2007bdd1243dSDimitry Andric if (RealignmentPadding > 0) {
200806c3fb27SDimitry Andric if (RealignmentPadding >= 4096) {
200906c3fb27SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm))
201006c3fb27SDimitry Andric .addReg(AArch64::X16, RegState::Define)
201106c3fb27SDimitry Andric .addImm(RealignmentPadding)
201206c3fb27SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
201306c3fb27SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXrx64), AArch64::X15)
201406c3fb27SDimitry Andric .addReg(AArch64::SP)
201506c3fb27SDimitry Andric .addReg(AArch64::X16, RegState::Kill)
201606c3fb27SDimitry Andric .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0))
201706c3fb27SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
201806c3fb27SDimitry Andric } else {
2019bdd1243dSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), AArch64::X15)
2020bdd1243dSDimitry Andric .addReg(AArch64::SP)
2021bdd1243dSDimitry Andric .addImm(RealignmentPadding)
202206c3fb27SDimitry Andric .addImm(0)
202306c3fb27SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
202406c3fb27SDimitry Andric }
2025bdd1243dSDimitry Andric
2026bdd1243dSDimitry Andric uint64_t AndMask = ~(MFI.getMaxAlign().value() - 1);
2027bdd1243dSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
2028bdd1243dSDimitry Andric .addReg(AArch64::X15, RegState::Kill)
2029bdd1243dSDimitry Andric .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64));
2030bdd1243dSDimitry Andric AFI->setStackRealigned(true);
2031bdd1243dSDimitry Andric
2032bdd1243dSDimitry Andric // No need for SEH instructions here; if we're realigning the stack,
2033bdd1243dSDimitry Andric // we've set a frame pointer and already finished the SEH prologue.
2034bdd1243dSDimitry Andric assert(!NeedsWinCFI);
2035bdd1243dSDimitry Andric }
20360b57cec5SDimitry Andric }
20370b57cec5SDimitry Andric
20385f757f3fSDimitry Andric StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize;
2039480093f4SDimitry Andric MachineBasicBlock::iterator CalleeSavesBegin = MBBI, CalleeSavesEnd = MBBI;
2040480093f4SDimitry Andric
2041480093f4SDimitry Andric // Process the SVE callee-saves to determine what space needs to be
2042480093f4SDimitry Andric // allocated.
2043979e22ffSDimitry Andric if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
20445f757f3fSDimitry Andric LLVM_DEBUG(dbgs() << "SVECalleeSavedStackSize = " << CalleeSavedSize
20455f757f3fSDimitry Andric << "\n");
2046480093f4SDimitry Andric // Find callee save instructions in frame.
2047480093f4SDimitry Andric CalleeSavesBegin = MBBI;
2048480093f4SDimitry Andric assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
2049480093f4SDimitry Andric while (IsSVECalleeSave(MBBI) && MBBI != MBB.getFirstTerminator())
2050480093f4SDimitry Andric ++MBBI;
2051480093f4SDimitry Andric CalleeSavesEnd = MBBI;
2052480093f4SDimitry Andric
20535f757f3fSDimitry Andric SVECalleeSavesSize = StackOffset::getScalable(CalleeSavedSize);
20545f757f3fSDimitry Andric SVELocalsSize = SVEStackSize - SVECalleeSavesSize;
2055480093f4SDimitry Andric }
2056480093f4SDimitry Andric
2057480093f4SDimitry Andric // Allocate space for the callee saves (if any).
20585f757f3fSDimitry Andric StackOffset CFAOffset =
20595f757f3fSDimitry Andric StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes);
20605f757f3fSDimitry Andric StackOffset LocalsSize = SVELocalsSize + StackOffset::getFixed(NumBytes);
20615f757f3fSDimitry Andric allocateStackSpace(MBB, CalleeSavesBegin, 0, SVECalleeSavesSize, false,
20625f757f3fSDimitry Andric nullptr, EmitAsyncCFI && !HasFP, CFAOffset,
20635f757f3fSDimitry Andric MFI.hasVarSizedObjects() || LocalsSize);
20645f757f3fSDimitry Andric CFAOffset += SVECalleeSavesSize;
206581ad6265SDimitry Andric
206606c3fb27SDimitry Andric if (EmitAsyncCFI)
206781ad6265SDimitry Andric emitCalleeSavedSVELocations(MBB, CalleeSavesEnd);
2068480093f4SDimitry Andric
20695f757f3fSDimitry Andric // Allocate space for the rest of the frame including SVE locals. Align the
20705f757f3fSDimitry Andric // stack as necessary.
20715f757f3fSDimitry Andric assert(!(canUseRedZone(MF) && NeedsRealignment) &&
20725f757f3fSDimitry Andric "Cannot use redzone with stack realignment");
207381ad6265SDimitry Andric if (!canUseRedZone(MF)) {
20740b57cec5SDimitry Andric // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
20750b57cec5SDimitry Andric // the correct value here, as NumBytes also includes padding bytes,
20760b57cec5SDimitry Andric // which shouldn't be counted here.
20775f757f3fSDimitry Andric allocateStackSpace(MBB, CalleeSavesEnd, RealignmentPadding,
20785f757f3fSDimitry Andric SVELocalsSize + StackOffset::getFixed(NumBytes),
20795f757f3fSDimitry Andric NeedsWinCFI, &HasWinCFI, EmitAsyncCFI && !HasFP,
20805f757f3fSDimitry Andric CFAOffset, MFI.hasVarSizedObjects());
20810b57cec5SDimitry Andric }
20820b57cec5SDimitry Andric
20830b57cec5SDimitry Andric // If we need a base pointer, set it up here. It's whatever the value of the
20840b57cec5SDimitry Andric // stack pointer is at this point. Any variable size objects will be allocated
20850b57cec5SDimitry Andric // after this, so we can still use the base pointer to reference locals.
20860b57cec5SDimitry Andric //
20870b57cec5SDimitry Andric // FIXME: Clarify FrameSetup flags here.
20880b57cec5SDimitry Andric // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
20890b57cec5SDimitry Andric // needed.
209062cfcf62SDimitry Andric // For funclets the BP belongs to the containing function.
209162cfcf62SDimitry Andric if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
20920b57cec5SDimitry Andric TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
20930b57cec5SDimitry Andric false);
20940b57cec5SDimitry Andric if (NeedsWinCFI) {
20950b57cec5SDimitry Andric HasWinCFI = true;
20960b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
20970b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
20980b57cec5SDimitry Andric }
20990b57cec5SDimitry Andric }
21000b57cec5SDimitry Andric
21010b57cec5SDimitry Andric // The very last FrameSetup instruction indicates the end of prologue. Emit a
21020b57cec5SDimitry Andric // SEH opcode indicating the prologue end.
21030b57cec5SDimitry Andric if (NeedsWinCFI && HasWinCFI) {
21040b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
21050b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
21060b57cec5SDimitry Andric }
21070b57cec5SDimitry Andric
210862cfcf62SDimitry Andric // SEH funclets are passed the frame pointer in X1. If the parent
210962cfcf62SDimitry Andric // function uses the base register, then the base register is used
211062cfcf62SDimitry Andric // directly, and is not retrieved from X1.
211162cfcf62SDimitry Andric if (IsFunclet && F.hasPersonalityFn()) {
211262cfcf62SDimitry Andric EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
211362cfcf62SDimitry Andric if (isAsynchronousEHPersonality(Per)) {
211462cfcf62SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
211562cfcf62SDimitry Andric .addReg(AArch64::X1)
211662cfcf62SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
211762cfcf62SDimitry Andric MBB.addLiveIn(AArch64::X1);
211862cfcf62SDimitry Andric }
211962cfcf62SDimitry Andric }
212006c3fb27SDimitry Andric
212106c3fb27SDimitry Andric if (EmitCFI && !EmitAsyncCFI) {
212206c3fb27SDimitry Andric if (HasFP) {
212306c3fb27SDimitry Andric emitDefineCFAWithFP(MF, MBB, MBBI, DL, FixedObject);
212406c3fb27SDimitry Andric } else {
212506c3fb27SDimitry Andric StackOffset TotalSize =
212606c3fb27SDimitry Andric SVEStackSize + StackOffset::getFixed((int64_t)MFI.getStackSize());
212706c3fb27SDimitry Andric unsigned CFIIndex = MF.addFrameInst(createDefCFA(
212806c3fb27SDimitry Andric *RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP, TotalSize,
212906c3fb27SDimitry Andric /*LastAdjustmentWasScalable=*/false));
213006c3fb27SDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
213106c3fb27SDimitry Andric .addCFIIndex(CFIIndex)
213206c3fb27SDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
213306c3fb27SDimitry Andric }
213406c3fb27SDimitry Andric emitCalleeSavedGPRLocations(MBB, MBBI);
213506c3fb27SDimitry Andric emitCalleeSavedSVELocations(MBB, MBBI);
213606c3fb27SDimitry Andric }
21370b57cec5SDimitry Andric }
21380b57cec5SDimitry Andric
isFuncletReturnInstr(const MachineInstr & MI)21390b57cec5SDimitry Andric static bool isFuncletReturnInstr(const MachineInstr &MI) {
21400b57cec5SDimitry Andric switch (MI.getOpcode()) {
21410b57cec5SDimitry Andric default:
21420b57cec5SDimitry Andric return false;
21430b57cec5SDimitry Andric case AArch64::CATCHRET:
21440b57cec5SDimitry Andric case AArch64::CLEANUPRET:
21450b57cec5SDimitry Andric return true;
21460b57cec5SDimitry Andric }
21470b57cec5SDimitry Andric }
21480b57cec5SDimitry Andric
emitEpilogue(MachineFunction & MF,MachineBasicBlock & MBB) const21490b57cec5SDimitry Andric void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
21500b57cec5SDimitry Andric MachineBasicBlock &MBB) const {
21510b57cec5SDimitry Andric MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
21520b57cec5SDimitry Andric MachineFrameInfo &MFI = MF.getFrameInfo();
21535f757f3fSDimitry Andric AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
21540b57cec5SDimitry Andric const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
21550b57cec5SDimitry Andric const TargetInstrInfo *TII = Subtarget.getInstrInfo();
21560b57cec5SDimitry Andric DebugLoc DL;
21570b57cec5SDimitry Andric bool NeedsWinCFI = needsWinCFI(MF);
21585f757f3fSDimitry Andric bool EmitCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
21590b57cec5SDimitry Andric bool HasWinCFI = false;
21600b57cec5SDimitry Andric bool IsFunclet = false;
21610b57cec5SDimitry Andric
21620b57cec5SDimitry Andric if (MBB.end() != MBBI) {
21630b57cec5SDimitry Andric DL = MBBI->getDebugLoc();
21640b57cec5SDimitry Andric IsFunclet = isFuncletReturnInstr(*MBBI);
21650b57cec5SDimitry Andric }
21660b57cec5SDimitry Andric
21675f757f3fSDimitry Andric MachineBasicBlock::iterator EpilogStartI = MBB.end();
21685f757f3fSDimitry Andric
216981ad6265SDimitry Andric auto FinishingTouches = make_scope_exit([&]() {
21705f757f3fSDimitry Andric if (AFI->shouldSignReturnAddress(MF)) {
21715f757f3fSDimitry Andric BuildMI(MBB, MBB.getFirstTerminator(), DL,
21725f757f3fSDimitry Andric TII->get(AArch64::PAUTH_EPILOGUE))
21735f757f3fSDimitry Andric .setMIFlag(MachineInstr::FrameDestroy);
21745f757f3fSDimitry Andric if (NeedsWinCFI)
21755f757f3fSDimitry Andric HasWinCFI = true; // AArch64PointerAuth pass will insert SEH_PACSignLR
21765f757f3fSDimitry Andric }
21775f757f3fSDimitry Andric if (AFI->needsShadowCallStackPrologueEpilogue(MF))
217881ad6265SDimitry Andric emitShadowCallStackEpilogue(*TII, MF, MBB, MBB.getFirstTerminator(), DL);
217981ad6265SDimitry Andric if (EmitCFI)
218081ad6265SDimitry Andric emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
21815f757f3fSDimitry Andric if (HasWinCFI) {
2182bdd1243dSDimitry Andric BuildMI(MBB, MBB.getFirstTerminator(), DL,
2183bdd1243dSDimitry Andric TII->get(AArch64::SEH_EpilogEnd))
2184bdd1243dSDimitry Andric .setMIFlag(MachineInstr::FrameDestroy);
21855f757f3fSDimitry Andric if (!MF.hasWinCFI())
21865f757f3fSDimitry Andric MF.setHasWinCFI(true);
21875f757f3fSDimitry Andric }
21885f757f3fSDimitry Andric if (NeedsWinCFI) {
21895f757f3fSDimitry Andric assert(EpilogStartI != MBB.end());
21905f757f3fSDimitry Andric if (!HasWinCFI)
21915f757f3fSDimitry Andric MBB.erase(EpilogStartI);
21925f757f3fSDimitry Andric }
219381ad6265SDimitry Andric });
219481ad6265SDimitry Andric
2195480093f4SDimitry Andric int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
21960b57cec5SDimitry Andric : MFI.getStackSize();
21970b57cec5SDimitry Andric
21980b57cec5SDimitry Andric // All calls are tail calls in GHC calling conv, and functions have no
21990b57cec5SDimitry Andric // prologue/epilogue.
22000b57cec5SDimitry Andric if (MF.getFunction().getCallingConv() == CallingConv::GHC)
22010b57cec5SDimitry Andric return;
22020b57cec5SDimitry Andric
2203fe6060f1SDimitry Andric // How much of the stack used by incoming arguments this function is expected
2204fe6060f1SDimitry Andric // to restore in this particular epilogue.
2205fe6060f1SDimitry Andric int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB);
22060b57cec5SDimitry Andric bool IsWin64 =
22070b57cec5SDimitry Andric Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
220862cfcf62SDimitry Andric unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
22090b57cec5SDimitry Andric
2210fe6060f1SDimitry Andric int64_t AfterCSRPopSize = ArgumentStackToRestore;
22110b57cec5SDimitry Andric auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
22120b57cec5SDimitry Andric // We cannot rely on the local stack size set in emitPrologue if the function
22130b57cec5SDimitry Andric // has funclets, as funclets have different local stack size requirements, and
22140b57cec5SDimitry Andric // the current value set in emitPrologue may be that of the containing
22150b57cec5SDimitry Andric // function.
22160b57cec5SDimitry Andric if (MF.hasEHFunclets())
22170b57cec5SDimitry Andric AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
2218fe6060f1SDimitry Andric if (homogeneousPrologEpilog(MF, &MBB)) {
2219fe6060f1SDimitry Andric assert(!NeedsWinCFI);
2220fe6060f1SDimitry Andric auto LastPopI = MBB.getFirstTerminator();
2221fe6060f1SDimitry Andric if (LastPopI != MBB.begin()) {
2222fe6060f1SDimitry Andric auto HomogeneousEpilog = std::prev(LastPopI);
2223fe6060f1SDimitry Andric if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog)
2224fe6060f1SDimitry Andric LastPopI = HomogeneousEpilog;
2225fe6060f1SDimitry Andric }
2226fe6060f1SDimitry Andric
2227fe6060f1SDimitry Andric // Adjust local stack
2228fe6060f1SDimitry Andric emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
2229fe6060f1SDimitry Andric StackOffset::getFixed(AFI->getLocalStackSize()), TII,
22305f757f3fSDimitry Andric MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
2231fe6060f1SDimitry Andric
2232fe6060f1SDimitry Andric // SP has been already adjusted while restoring callee save regs.
2233fe6060f1SDimitry Andric // We've bailed-out the case with adjusting SP for arguments.
2234fe6060f1SDimitry Andric assert(AfterCSRPopSize == 0);
2235fe6060f1SDimitry Andric return;
2236fe6060f1SDimitry Andric }
22375ffd83dbSDimitry Andric bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
22380b57cec5SDimitry Andric // Assume we can't combine the last pop with the sp restore.
22390b57cec5SDimitry Andric
224081ad6265SDimitry Andric bool CombineAfterCSRBump = false;
22410b57cec5SDimitry Andric if (!CombineSPBump && PrologueSaveSize != 0) {
22420b57cec5SDimitry Andric MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
224381ad6265SDimitry Andric while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
224481ad6265SDimitry Andric AArch64InstrInfo::isSEHInstruction(*Pop))
22450b57cec5SDimitry Andric Pop = std::prev(Pop);
22460b57cec5SDimitry Andric // Converting the last ldp to a post-index ldp is valid only if the last
22470b57cec5SDimitry Andric // ldp's offset is 0.
22480b57cec5SDimitry Andric const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
2249fe6060f1SDimitry Andric // If the offset is 0 and the AfterCSR pop is not actually trying to
2250fe6060f1SDimitry Andric // allocate more stack for arguments (in space that an untimely interrupt
2251fe6060f1SDimitry Andric // may clobber), convert it to a post-index ldp.
225281ad6265SDimitry Andric if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) {
22530b57cec5SDimitry Andric convertCalleeSaveRestoreToSPPrePostIncDec(
225481ad6265SDimitry Andric MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, EmitCFI,
225581ad6265SDimitry Andric MachineInstr::FrameDestroy, PrologueSaveSize);
225681ad6265SDimitry Andric } else {
22570b57cec5SDimitry Andric // If not, make sure to emit an add after the last ldp.
22580b57cec5SDimitry Andric // We're doing this by transfering the size to be restored from the
22590b57cec5SDimitry Andric // adjustment *before* the CSR pops to the adjustment *after* the CSR
22600b57cec5SDimitry Andric // pops.
22610b57cec5SDimitry Andric AfterCSRPopSize += PrologueSaveSize;
226281ad6265SDimitry Andric CombineAfterCSRBump = true;
22630b57cec5SDimitry Andric }
22640b57cec5SDimitry Andric }
22650b57cec5SDimitry Andric
22660b57cec5SDimitry Andric // Move past the restores of the callee-saved registers.
22670b57cec5SDimitry Andric // If we plan on combining the sp bump of the local stack size and the callee
22680b57cec5SDimitry Andric // save stack size, we might need to adjust the CSR save and restore offsets.
22690b57cec5SDimitry Andric MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
22700b57cec5SDimitry Andric MachineBasicBlock::iterator Begin = MBB.begin();
22710b57cec5SDimitry Andric while (LastPopI != Begin) {
22720b57cec5SDimitry Andric --LastPopI;
2273480093f4SDimitry Andric if (!LastPopI->getFlag(MachineInstr::FrameDestroy) ||
2274480093f4SDimitry Andric IsSVECalleeSave(LastPopI)) {
22750b57cec5SDimitry Andric ++LastPopI;
22760b57cec5SDimitry Andric break;
22770b57cec5SDimitry Andric } else if (CombineSPBump)
22780b57cec5SDimitry Andric fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(),
22790b57cec5SDimitry Andric NeedsWinCFI, &HasWinCFI);
22800b57cec5SDimitry Andric }
22810b57cec5SDimitry Andric
22825f757f3fSDimitry Andric if (NeedsWinCFI) {
22835f757f3fSDimitry Andric // Note that there are cases where we insert SEH opcodes in the
22845f757f3fSDimitry Andric // epilogue when we had no SEH opcodes in the prologue. For
22855f757f3fSDimitry Andric // example, when there is no stack frame but there are stack
22865f757f3fSDimitry Andric // arguments. Insert the SEH_EpilogStart and remove it later if it
22875f757f3fSDimitry Andric // we didn't emit any SEH opcodes to avoid generating WinCFI for
22885f757f3fSDimitry Andric // functions that don't need it.
22890b57cec5SDimitry Andric BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
22900b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameDestroy);
22915f757f3fSDimitry Andric EpilogStartI = LastPopI;
22925f757f3fSDimitry Andric --EpilogStartI;
22930b57cec5SDimitry Andric }
22940b57cec5SDimitry Andric
2295fe6060f1SDimitry Andric if (hasFP(MF) && AFI->hasSwiftAsyncContext()) {
229681ad6265SDimitry Andric switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
229781ad6265SDimitry Andric case SwiftAsyncFramePointerMode::DeploymentBased:
229881ad6265SDimitry Andric // Avoid the reload as it is GOT relative, and instead fall back to the
229981ad6265SDimitry Andric // hardcoded value below. This allows a mismatch between the OS and
230081ad6265SDimitry Andric // application without immediately terminating on the difference.
2301bdd1243dSDimitry Andric [[fallthrough]];
230281ad6265SDimitry Andric case SwiftAsyncFramePointerMode::Always:
230381ad6265SDimitry Andric // We need to reset FP to its untagged state on return. Bit 60 is
230481ad6265SDimitry Andric // currently used to show the presence of an extended frame.
2305fe6060f1SDimitry Andric
2306fe6060f1SDimitry Andric // BIC x29, x29, #0x1000_0000_0000_0000
2307fe6060f1SDimitry Andric BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::ANDXri),
2308fe6060f1SDimitry Andric AArch64::FP)
2309fe6060f1SDimitry Andric .addUse(AArch64::FP)
2310fe6060f1SDimitry Andric .addImm(0x10fe)
2311fe6060f1SDimitry Andric .setMIFlag(MachineInstr::FrameDestroy);
23125f757f3fSDimitry Andric if (NeedsWinCFI) {
23135f757f3fSDimitry Andric BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
23145f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameDestroy);
23155f757f3fSDimitry Andric HasWinCFI = true;
23165f757f3fSDimitry Andric }
231781ad6265SDimitry Andric break;
231881ad6265SDimitry Andric
231981ad6265SDimitry Andric case SwiftAsyncFramePointerMode::Never:
232081ad6265SDimitry Andric break;
232181ad6265SDimitry Andric }
2322fe6060f1SDimitry Andric }
2323fe6060f1SDimitry Andric
23248bcb0991SDimitry Andric const StackOffset &SVEStackSize = getSVEStackSize(MF);
23258bcb0991SDimitry Andric
23260b57cec5SDimitry Andric // If there is a single SP update, insert it before the ret and we're done.
23270b57cec5SDimitry Andric if (CombineSPBump) {
23288bcb0991SDimitry Andric assert(!SVEStackSize && "Cannot combine SP bump with SVE");
232981ad6265SDimitry Andric
233081ad6265SDimitry Andric // When we are about to restore the CSRs, the CFA register is SP again.
233181ad6265SDimitry Andric if (EmitCFI && hasFP(MF)) {
233281ad6265SDimitry Andric const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
233381ad6265SDimitry Andric unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
233481ad6265SDimitry Andric unsigned CFIIndex =
233581ad6265SDimitry Andric MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, NumBytes));
233681ad6265SDimitry Andric BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
233781ad6265SDimitry Andric .addCFIIndex(CFIIndex)
233881ad6265SDimitry Andric .setMIFlags(MachineInstr::FrameDestroy);
233981ad6265SDimitry Andric }
234081ad6265SDimitry Andric
23410b57cec5SDimitry Andric emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
2342e8d8bef9SDimitry Andric StackOffset::getFixed(NumBytes + (int64_t)AfterCSRPopSize),
2343e8d8bef9SDimitry Andric TII, MachineInstr::FrameDestroy, false, NeedsWinCFI,
234481ad6265SDimitry Andric &HasWinCFI, EmitCFI, StackOffset::getFixed(NumBytes));
23450b57cec5SDimitry Andric return;
23460b57cec5SDimitry Andric }
23470b57cec5SDimitry Andric
23480b57cec5SDimitry Andric NumBytes -= PrologueSaveSize;
23490b57cec5SDimitry Andric assert(NumBytes >= 0 && "Negative stack allocation size!?");
23500b57cec5SDimitry Andric
2351480093f4SDimitry Andric // Process the SVE callee-saves to determine what space needs to be
2352480093f4SDimitry Andric // deallocated.
2353480093f4SDimitry Andric StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
2354480093f4SDimitry Andric MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
2355979e22ffSDimitry Andric if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
235616d6b3b3SDimitry Andric RestoreBegin = std::prev(RestoreEnd);
235716d6b3b3SDimitry Andric while (RestoreBegin != MBB.begin() &&
235816d6b3b3SDimitry Andric IsSVECalleeSave(std::prev(RestoreBegin)))
2359480093f4SDimitry Andric --RestoreBegin;
2360480093f4SDimitry Andric
2361480093f4SDimitry Andric assert(IsSVECalleeSave(RestoreBegin) &&
2362480093f4SDimitry Andric IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
2363480093f4SDimitry Andric
2364e8d8bef9SDimitry Andric StackOffset CalleeSavedSizeAsOffset =
2365e8d8bef9SDimitry Andric StackOffset::getScalable(CalleeSavedSize);
2366979e22ffSDimitry Andric DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
2367979e22ffSDimitry Andric DeallocateAfter = CalleeSavedSizeAsOffset;
2368480093f4SDimitry Andric }
2369480093f4SDimitry Andric
23708bcb0991SDimitry Andric // Deallocate the SVE area.
2371480093f4SDimitry Andric if (SVEStackSize) {
237281ad6265SDimitry Andric // If we have stack realignment or variable sized objects on the stack,
237381ad6265SDimitry Andric // restore the stack pointer from the frame pointer prior to SVE CSR
237481ad6265SDimitry Andric // restoration.
237581ad6265SDimitry Andric if (AFI->isStackRealigned() || MFI.hasVarSizedObjects()) {
237681ad6265SDimitry Andric if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
2377979e22ffSDimitry Andric // Set SP to start of SVE callee-save area from which they can
2378979e22ffSDimitry Andric // be reloaded. The code below will deallocate the stack space
2379480093f4SDimitry Andric // space by moving FP -> SP.
2380480093f4SDimitry Andric emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
2381e8d8bef9SDimitry Andric StackOffset::getScalable(-CalleeSavedSize), TII,
2382979e22ffSDimitry Andric MachineInstr::FrameDestroy);
238381ad6265SDimitry Andric }
2384480093f4SDimitry Andric } else {
2385480093f4SDimitry Andric if (AFI->getSVECalleeSavedStackSize()) {
2386480093f4SDimitry Andric // Deallocate the non-SVE locals first before we can deallocate (and
2387480093f4SDimitry Andric // restore callee saves) from the SVE area.
238881ad6265SDimitry Andric emitFrameOffset(
238981ad6265SDimitry Andric MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
239081ad6265SDimitry Andric StackOffset::getFixed(NumBytes), TII, MachineInstr::FrameDestroy,
239181ad6265SDimitry Andric false, false, nullptr, EmitCFI && !hasFP(MF),
239281ad6265SDimitry Andric SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize));
2393480093f4SDimitry Andric NumBytes = 0;
2394480093f4SDimitry Andric }
2395480093f4SDimitry Andric
2396480093f4SDimitry Andric emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
239781ad6265SDimitry Andric DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
239881ad6265SDimitry Andric false, nullptr, EmitCFI && !hasFP(MF),
239981ad6265SDimitry Andric SVEStackSize +
240081ad6265SDimitry Andric StackOffset::getFixed(NumBytes + PrologueSaveSize));
2401480093f4SDimitry Andric
2402480093f4SDimitry Andric emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
240381ad6265SDimitry Andric DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
240481ad6265SDimitry Andric false, nullptr, EmitCFI && !hasFP(MF),
240581ad6265SDimitry Andric DeallocateAfter +
240681ad6265SDimitry Andric StackOffset::getFixed(NumBytes + PrologueSaveSize));
2407480093f4SDimitry Andric }
240881ad6265SDimitry Andric if (EmitCFI)
240981ad6265SDimitry Andric emitCalleeSavedSVERestores(MBB, RestoreEnd);
2410480093f4SDimitry Andric }
24118bcb0991SDimitry Andric
24120b57cec5SDimitry Andric if (!hasFP(MF)) {
24130b57cec5SDimitry Andric bool RedZone = canUseRedZone(MF);
24140b57cec5SDimitry Andric // If this was a redzone leaf function, we don't need to restore the
24150b57cec5SDimitry Andric // stack pointer (but we may need to pop stack args for fastcc).
24160b57cec5SDimitry Andric if (RedZone && AfterCSRPopSize == 0)
24170b57cec5SDimitry Andric return;
24180b57cec5SDimitry Andric
241981ad6265SDimitry Andric // Pop the local variables off the stack. If there are no callee-saved
242081ad6265SDimitry Andric // registers, it means we are actually positioned at the terminator and can
242181ad6265SDimitry Andric // combine stack increment for the locals and the stack increment for
242281ad6265SDimitry Andric // callee-popped arguments into (possibly) a single instruction and be done.
24230b57cec5SDimitry Andric bool NoCalleeSaveRestore = PrologueSaveSize == 0;
2424480093f4SDimitry Andric int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
24250b57cec5SDimitry Andric if (NoCalleeSaveRestore)
24260b57cec5SDimitry Andric StackRestoreBytes += AfterCSRPopSize;
24270b57cec5SDimitry Andric
242881ad6265SDimitry Andric emitFrameOffset(
242981ad6265SDimitry Andric MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
243081ad6265SDimitry Andric StackOffset::getFixed(StackRestoreBytes), TII,
243181ad6265SDimitry Andric MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI,
243281ad6265SDimitry Andric StackOffset::getFixed((RedZone ? 0 : NumBytes) + PrologueSaveSize));
243381ad6265SDimitry Andric
24340b57cec5SDimitry Andric // If we were able to combine the local stack pop with the argument pop,
24350b57cec5SDimitry Andric // then we're done.
243681ad6265SDimitry Andric if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
24370b57cec5SDimitry Andric return;
24380b57cec5SDimitry Andric }
24390b57cec5SDimitry Andric
24400b57cec5SDimitry Andric NumBytes = 0;
24410b57cec5SDimitry Andric }
24420b57cec5SDimitry Andric
24430b57cec5SDimitry Andric // Restore the original stack pointer.
24440b57cec5SDimitry Andric // FIXME: Rather than doing the math here, we should instead just use
24450b57cec5SDimitry Andric // non-post-indexed loads for the restores if we aren't actually going to
24460b57cec5SDimitry Andric // be able to save any instructions.
24478bcb0991SDimitry Andric if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
2448e8d8bef9SDimitry Andric emitFrameOffset(
2449e8d8bef9SDimitry Andric MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
2450e8d8bef9SDimitry Andric StackOffset::getFixed(-AFI->getCalleeSaveBaseToFrameRecordOffset()),
24515f757f3fSDimitry Andric TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
24528bcb0991SDimitry Andric } else if (NumBytes)
24538bcb0991SDimitry Andric emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
2454e8d8bef9SDimitry Andric StackOffset::getFixed(NumBytes), TII,
24555f757f3fSDimitry Andric MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
24560b57cec5SDimitry Andric
245781ad6265SDimitry Andric // When we are about to restore the CSRs, the CFA register is SP again.
245881ad6265SDimitry Andric if (EmitCFI && hasFP(MF)) {
245981ad6265SDimitry Andric const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
246081ad6265SDimitry Andric unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
246181ad6265SDimitry Andric unsigned CFIIndex = MF.addFrameInst(
246281ad6265SDimitry Andric MCCFIInstruction::cfiDefCfa(nullptr, Reg, PrologueSaveSize));
246381ad6265SDimitry Andric BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
246481ad6265SDimitry Andric .addCFIIndex(CFIIndex)
246581ad6265SDimitry Andric .setMIFlags(MachineInstr::FrameDestroy);
246681ad6265SDimitry Andric }
246781ad6265SDimitry Andric
24680b57cec5SDimitry Andric // This must be placed after the callee-save restore code because that code
24690b57cec5SDimitry Andric // assumes the SP is at the same location as it was after the callee-save save
24700b57cec5SDimitry Andric // code in the prologue.
24710b57cec5SDimitry Andric if (AfterCSRPopSize) {
2472fe6060f1SDimitry Andric assert(AfterCSRPopSize > 0 && "attempting to reallocate arg stack that an "
2473fe6060f1SDimitry Andric "interrupt may have clobbered");
24740b57cec5SDimitry Andric
247581ad6265SDimitry Andric emitFrameOffset(
247681ad6265SDimitry Andric MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
247781ad6265SDimitry Andric StackOffset::getFixed(AfterCSRPopSize), TII, MachineInstr::FrameDestroy,
247881ad6265SDimitry Andric false, NeedsWinCFI, &HasWinCFI, EmitCFI,
247981ad6265SDimitry Andric StackOffset::getFixed(CombineAfterCSRBump ? PrologueSaveSize : 0));
24800b57cec5SDimitry Andric }
24810b57cec5SDimitry Andric }
24820b57cec5SDimitry Andric
enableCFIFixup(MachineFunction & MF) const248306c3fb27SDimitry Andric bool AArch64FrameLowering::enableCFIFixup(MachineFunction &MF) const {
248406c3fb27SDimitry Andric return TargetFrameLowering::enableCFIFixup(MF) &&
248506c3fb27SDimitry Andric MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF);
248606c3fb27SDimitry Andric }
248706c3fb27SDimitry Andric
24880b57cec5SDimitry Andric /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
24890b57cec5SDimitry Andric /// debug info. It's the same as what we use for resolving the code-gen
24900b57cec5SDimitry Andric /// references for now. FIXME: This can go wrong when references are
24910b57cec5SDimitry Andric /// SP-relative and simple call frames aren't used.
2492e8d8bef9SDimitry Andric StackOffset
getFrameIndexReference(const MachineFunction & MF,int FI,Register & FrameReg) const2493e8d8bef9SDimitry Andric AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
24945ffd83dbSDimitry Andric Register &FrameReg) const {
24950b57cec5SDimitry Andric return resolveFrameIndexReference(
24960b57cec5SDimitry Andric MF, FI, FrameReg,
24970b57cec5SDimitry Andric /*PreferFP=*/
24980b57cec5SDimitry Andric MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress),
2499e8d8bef9SDimitry Andric /*ForSimm=*/false);
25000b57cec5SDimitry Andric }
25010b57cec5SDimitry Andric
2502e8d8bef9SDimitry Andric StackOffset
getNonLocalFrameIndexReference(const MachineFunction & MF,int FI) const2503e8d8bef9SDimitry Andric AArch64FrameLowering::getNonLocalFrameIndexReference(const MachineFunction &MF,
2504e8d8bef9SDimitry Andric int FI) const {
2505e8d8bef9SDimitry Andric return StackOffset::getFixed(getSEHFrameIndexOffset(MF, FI));
25060b57cec5SDimitry Andric }
25070b57cec5SDimitry Andric
getFPOffset(const MachineFunction & MF,int64_t ObjectOffset)2508e8d8bef9SDimitry Andric static StackOffset getFPOffset(const MachineFunction &MF,
2509e8d8bef9SDimitry Andric int64_t ObjectOffset) {
25100b57cec5SDimitry Andric const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
25110b57cec5SDimitry Andric const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
25120b57cec5SDimitry Andric bool IsWin64 =
25130b57cec5SDimitry Andric Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
251462cfcf62SDimitry Andric unsigned FixedObject =
251562cfcf62SDimitry Andric getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
2516e8d8bef9SDimitry Andric int64_t CalleeSaveSize = AFI->getCalleeSavedStackSize(MF.getFrameInfo());
2517e8d8bef9SDimitry Andric int64_t FPAdjust =
2518e8d8bef9SDimitry Andric CalleeSaveSize - AFI->getCalleeSaveBaseToFrameRecordOffset();
2519e8d8bef9SDimitry Andric return StackOffset::getFixed(ObjectOffset + FixedObject + FPAdjust);
25200b57cec5SDimitry Andric }
25210b57cec5SDimitry Andric
getStackOffset(const MachineFunction & MF,int64_t ObjectOffset)2522e8d8bef9SDimitry Andric static StackOffset getStackOffset(const MachineFunction &MF,
2523e8d8bef9SDimitry Andric int64_t ObjectOffset) {
25240b57cec5SDimitry Andric const auto &MFI = MF.getFrameInfo();
2525e8d8bef9SDimitry Andric return StackOffset::getFixed(ObjectOffset + (int64_t)MFI.getStackSize());
25260b57cec5SDimitry Andric }
25270b57cec5SDimitry Andric
2528e8d8bef9SDimitry Andric // TODO: This function currently does not work for scalable vectors.
getSEHFrameIndexOffset(const MachineFunction & MF,int FI) const25290b57cec5SDimitry Andric int AArch64FrameLowering::getSEHFrameIndexOffset(const MachineFunction &MF,
25300b57cec5SDimitry Andric int FI) const {
25310b57cec5SDimitry Andric const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
25320b57cec5SDimitry Andric MF.getSubtarget().getRegisterInfo());
25330b57cec5SDimitry Andric int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI);
25340b57cec5SDimitry Andric return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
2535e8d8bef9SDimitry Andric ? getFPOffset(MF, ObjectOffset).getFixed()
2536e8d8bef9SDimitry Andric : getStackOffset(MF, ObjectOffset).getFixed();
25370b57cec5SDimitry Andric }
25380b57cec5SDimitry Andric
resolveFrameIndexReference(const MachineFunction & MF,int FI,Register & FrameReg,bool PreferFP,bool ForSimm) const25398bcb0991SDimitry Andric StackOffset AArch64FrameLowering::resolveFrameIndexReference(
25405ffd83dbSDimitry Andric const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP,
25410b57cec5SDimitry Andric bool ForSimm) const {
25420b57cec5SDimitry Andric const auto &MFI = MF.getFrameInfo();
2543480093f4SDimitry Andric int64_t ObjectOffset = MFI.getObjectOffset(FI);
25440b57cec5SDimitry Andric bool isFixed = MFI.isFixedObjectIndex(FI);
2545e8d8bef9SDimitry Andric bool isSVE = MFI.getStackID(FI) == TargetStackID::ScalableVector;
25468bcb0991SDimitry Andric return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
25470b57cec5SDimitry Andric PreferFP, ForSimm);
25480b57cec5SDimitry Andric }
25490b57cec5SDimitry Andric
resolveFrameOffsetReference(const MachineFunction & MF,int64_t ObjectOffset,bool isFixed,bool isSVE,Register & FrameReg,bool PreferFP,bool ForSimm) const25508bcb0991SDimitry Andric StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
2551480093f4SDimitry Andric const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
25525ffd83dbSDimitry Andric Register &FrameReg, bool PreferFP, bool ForSimm) const {
25530b57cec5SDimitry Andric const auto &MFI = MF.getFrameInfo();
25540b57cec5SDimitry Andric const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
25550b57cec5SDimitry Andric MF.getSubtarget().getRegisterInfo());
25560b57cec5SDimitry Andric const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
25570b57cec5SDimitry Andric const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
25580b57cec5SDimitry Andric
2559e8d8bef9SDimitry Andric int64_t FPOffset = getFPOffset(MF, ObjectOffset).getFixed();
2560e8d8bef9SDimitry Andric int64_t Offset = getStackOffset(MF, ObjectOffset).getFixed();
25610b57cec5SDimitry Andric bool isCSR =
2562480093f4SDimitry Andric !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
25630b57cec5SDimitry Andric
25648bcb0991SDimitry Andric const StackOffset &SVEStackSize = getSVEStackSize(MF);
25658bcb0991SDimitry Andric
25660b57cec5SDimitry Andric // Use frame pointer to reference fixed objects. Use it for locals if
25670b57cec5SDimitry Andric // there are VLAs or a dynamically realigned SP (and thus the SP isn't
25680b57cec5SDimitry Andric // reliable as a base). Make sure useFPForScavengingIndex() does the
25690b57cec5SDimitry Andric // right thing for the emergency spill slot.
25700b57cec5SDimitry Andric bool UseFP = false;
25718bcb0991SDimitry Andric if (AFI->hasStackFrame() && !isSVE) {
257281ad6265SDimitry Andric // We shouldn't prefer using the FP to access fixed-sized stack objects when
257381ad6265SDimitry Andric // there are scalable (SVE) objects in between the FP and the fixed-sized
257481ad6265SDimitry Andric // objects.
25758bcb0991SDimitry Andric PreferFP &= !SVEStackSize;
25768bcb0991SDimitry Andric
25770b57cec5SDimitry Andric // Note: Keeping the following as multiple 'if' statements rather than
25780b57cec5SDimitry Andric // merging to a single expression for readability.
25790b57cec5SDimitry Andric //
25800b57cec5SDimitry Andric // Argument access should always use the FP.
25810b57cec5SDimitry Andric if (isFixed) {
25820b57cec5SDimitry Andric UseFP = hasFP(MF);
2583fe6060f1SDimitry Andric } else if (isCSR && RegInfo->hasStackRealignment(MF)) {
25840b57cec5SDimitry Andric // References to the CSR area must use FP if we're re-aligning the stack
25850b57cec5SDimitry Andric // since the dynamically-sized alignment padding is between the SP/BP and
25860b57cec5SDimitry Andric // the CSR area.
25870b57cec5SDimitry Andric assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
25880b57cec5SDimitry Andric UseFP = true;
2589fe6060f1SDimitry Andric } else if (hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
25900b57cec5SDimitry Andric // If the FPOffset is negative and we're producing a signed immediate, we
25910b57cec5SDimitry Andric // have to keep in mind that the available offset range for negative
25920b57cec5SDimitry Andric // offsets is smaller than for positive ones. If an offset is available
25930b57cec5SDimitry Andric // via the FP and the SP, use whichever is closest.
25940b57cec5SDimitry Andric bool FPOffsetFits = !ForSimm || FPOffset >= -256;
259581ad6265SDimitry Andric PreferFP |= Offset > -FPOffset && !SVEStackSize;
25960b57cec5SDimitry Andric
25970b57cec5SDimitry Andric if (MFI.hasVarSizedObjects()) {
25980b57cec5SDimitry Andric // If we have variable sized objects, we can use either FP or BP, as the
25990b57cec5SDimitry Andric // SP offset is unknown. We can use the base pointer if we have one and
26000b57cec5SDimitry Andric // FP is not preferred. If not, we're stuck with using FP.
26010b57cec5SDimitry Andric bool CanUseBP = RegInfo->hasBasePointer(MF);
26020b57cec5SDimitry Andric if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
26030b57cec5SDimitry Andric UseFP = PreferFP;
26045ffd83dbSDimitry Andric else if (!CanUseBP) // Can't use BP. Forced to use FP.
26050b57cec5SDimitry Andric UseFP = true;
26060b57cec5SDimitry Andric // else we can use BP and FP, but the offset from FP won't fit.
26070b57cec5SDimitry Andric // That will make us scavenge registers which we can probably avoid by
26080b57cec5SDimitry Andric // using BP. If it won't fit for BP either, we'll scavenge anyway.
26090b57cec5SDimitry Andric } else if (FPOffset >= 0) {
26100b57cec5SDimitry Andric // Use SP or FP, whichever gives us the best chance of the offset
26110b57cec5SDimitry Andric // being in range for direct access. If the FPOffset is positive,
26120b57cec5SDimitry Andric // that'll always be best, as the SP will be even further away.
26130b57cec5SDimitry Andric UseFP = true;
26140b57cec5SDimitry Andric } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
26150b57cec5SDimitry Andric // Funclets access the locals contained in the parent's stack frame
26160b57cec5SDimitry Andric // via the frame pointer, so we have to use the FP in the parent
26170b57cec5SDimitry Andric // function.
26180b57cec5SDimitry Andric (void) Subtarget;
26190b57cec5SDimitry Andric assert(
26200b57cec5SDimitry Andric Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
26210b57cec5SDimitry Andric "Funclets should only be present on Win64");
26220b57cec5SDimitry Andric UseFP = true;
26230b57cec5SDimitry Andric } else {
26240b57cec5SDimitry Andric // We have the choice between FP and (SP or BP).
26250b57cec5SDimitry Andric if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
26260b57cec5SDimitry Andric UseFP = true;
26270b57cec5SDimitry Andric }
26280b57cec5SDimitry Andric }
26290b57cec5SDimitry Andric }
26300b57cec5SDimitry Andric
2631fe6060f1SDimitry Andric assert(
2632fe6060f1SDimitry Andric ((isFixed || isCSR) || !RegInfo->hasStackRealignment(MF) || !UseFP) &&
26330b57cec5SDimitry Andric "In the presence of dynamic stack pointer realignment, "
26340b57cec5SDimitry Andric "non-argument/CSR objects cannot be accessed through the frame pointer");
26350b57cec5SDimitry Andric
26368bcb0991SDimitry Andric if (isSVE) {
2637e8d8bef9SDimitry Andric StackOffset FPOffset =
2638e8d8bef9SDimitry Andric StackOffset::get(-AFI->getCalleeSaveBaseToFrameRecordOffset(), ObjectOffset);
2639e8d8bef9SDimitry Andric StackOffset SPOffset =
2640e8d8bef9SDimitry Andric SVEStackSize +
2641e8d8bef9SDimitry Andric StackOffset::get(MFI.getStackSize() - AFI->getCalleeSavedStackSize(),
2642e8d8bef9SDimitry Andric ObjectOffset);
26438bcb0991SDimitry Andric // Always use the FP for SVE spills if available and beneficial.
2644fe6060f1SDimitry Andric if (hasFP(MF) && (SPOffset.getFixed() ||
2645e8d8bef9SDimitry Andric FPOffset.getScalable() < SPOffset.getScalable() ||
2646fe6060f1SDimitry Andric RegInfo->hasStackRealignment(MF))) {
26470b57cec5SDimitry Andric FrameReg = RegInfo->getFrameRegister(MF);
26480b57cec5SDimitry Andric return FPOffset;
26490b57cec5SDimitry Andric }
26500b57cec5SDimitry Andric
26518bcb0991SDimitry Andric FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
26528bcb0991SDimitry Andric : (unsigned)AArch64::SP;
26538bcb0991SDimitry Andric return SPOffset;
26548bcb0991SDimitry Andric }
26558bcb0991SDimitry Andric
26568bcb0991SDimitry Andric StackOffset ScalableOffset = {};
26578bcb0991SDimitry Andric if (UseFP && !(isFixed || isCSR))
26588bcb0991SDimitry Andric ScalableOffset = -SVEStackSize;
26598bcb0991SDimitry Andric if (!UseFP && (isFixed || isCSR))
26608bcb0991SDimitry Andric ScalableOffset = SVEStackSize;
26618bcb0991SDimitry Andric
26628bcb0991SDimitry Andric if (UseFP) {
26638bcb0991SDimitry Andric FrameReg = RegInfo->getFrameRegister(MF);
2664e8d8bef9SDimitry Andric return StackOffset::getFixed(FPOffset) + ScalableOffset;
26658bcb0991SDimitry Andric }
26668bcb0991SDimitry Andric
26670b57cec5SDimitry Andric // Use the base pointer if we have one.
26680b57cec5SDimitry Andric if (RegInfo->hasBasePointer(MF))
26690b57cec5SDimitry Andric FrameReg = RegInfo->getBaseRegister();
26700b57cec5SDimitry Andric else {
26710b57cec5SDimitry Andric assert(!MFI.hasVarSizedObjects() &&
26720b57cec5SDimitry Andric "Can't use SP when we have var sized objects.");
26730b57cec5SDimitry Andric FrameReg = AArch64::SP;
26740b57cec5SDimitry Andric // If we're using the red zone for this function, the SP won't actually
26750b57cec5SDimitry Andric // be adjusted, so the offsets will be negative. They're also all
26760b57cec5SDimitry Andric // within range of the signed 9-bit immediate instructions.
26770b57cec5SDimitry Andric if (canUseRedZone(MF))
26780b57cec5SDimitry Andric Offset -= AFI->getLocalStackSize();
26790b57cec5SDimitry Andric }
26800b57cec5SDimitry Andric
2681e8d8bef9SDimitry Andric return StackOffset::getFixed(Offset) + ScalableOffset;
26820b57cec5SDimitry Andric }
26830b57cec5SDimitry Andric
getPrologueDeath(MachineFunction & MF,unsigned Reg)26840b57cec5SDimitry Andric static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
26850b57cec5SDimitry Andric // Do not set a kill flag on values that are also marked as live-in. This
26860b57cec5SDimitry Andric // happens with the @llvm-returnaddress intrinsic and with arguments passed in
26870b57cec5SDimitry Andric // callee saved registers.
26880b57cec5SDimitry Andric // Omitting the kill flags is conservatively correct even if the live-in
26890b57cec5SDimitry Andric // is not used after all.
26900b57cec5SDimitry Andric bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
26910b57cec5SDimitry Andric return getKillRegState(!IsLiveIn);
26920b57cec5SDimitry Andric }
26930b57cec5SDimitry Andric
produceCompactUnwindFrame(MachineFunction & MF)26940b57cec5SDimitry Andric static bool produceCompactUnwindFrame(MachineFunction &MF) {
26950b57cec5SDimitry Andric const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
26960b57cec5SDimitry Andric AttributeList Attrs = MF.getFunction().getAttributes();
26970b57cec5SDimitry Andric return Subtarget.isTargetMachO() &&
26980b57cec5SDimitry Andric !(Subtarget.getTargetLowering()->supportSwiftError() &&
2699fe6060f1SDimitry Andric Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
2700fe6060f1SDimitry Andric MF.getFunction().getCallingConv() != CallingConv::SwiftTail;
27010b57cec5SDimitry Andric }
27020b57cec5SDimitry Andric
invalidateWindowsRegisterPairing(unsigned Reg1,unsigned Reg2,bool NeedsWinCFI,bool IsFirst,const TargetRegisterInfo * TRI)27030b57cec5SDimitry Andric static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
2704bdd1243dSDimitry Andric bool NeedsWinCFI, bool IsFirst,
2705bdd1243dSDimitry Andric const TargetRegisterInfo *TRI) {
27060b57cec5SDimitry Andric // If we are generating register pairs for a Windows function that requires
27070b57cec5SDimitry Andric // EH support, then pair consecutive registers only. There are no unwind
27080b57cec5SDimitry Andric // opcodes for saves/restores of non-consectuve register pairs.
2709e8d8bef9SDimitry Andric // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x,
2710e8d8bef9SDimitry Andric // save_lrpair.
27110b57cec5SDimitry Andric // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
27120b57cec5SDimitry Andric
2713480093f4SDimitry Andric if (Reg2 == AArch64::FP)
2714480093f4SDimitry Andric return true;
27150b57cec5SDimitry Andric if (!NeedsWinCFI)
27160b57cec5SDimitry Andric return false;
2717bdd1243dSDimitry Andric if (TRI->getEncodingValue(Reg2) == TRI->getEncodingValue(Reg1) + 1)
27180b57cec5SDimitry Andric return false;
2719e8d8bef9SDimitry Andric // If pairing a GPR with LR, the pair can be described by the save_lrpair
2720e8d8bef9SDimitry Andric // opcode. If this is the first register pair, it would end up with a
2721e8d8bef9SDimitry Andric // predecrement, but there's no save_lrpair_x opcode, so we can only do this
2722e8d8bef9SDimitry Andric // if LR is paired with something else than the first register.
2723e8d8bef9SDimitry Andric // The save_lrpair opcode requires the first register to be an odd one.
2724e8d8bef9SDimitry Andric if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
2725e8d8bef9SDimitry Andric (Reg1 - AArch64::X19) % 2 == 0 && Reg2 == AArch64::LR && !IsFirst)
2726e8d8bef9SDimitry Andric return false;
27270b57cec5SDimitry Andric return true;
27280b57cec5SDimitry Andric }
27290b57cec5SDimitry Andric
27308bcb0991SDimitry Andric /// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
27318bcb0991SDimitry Andric /// WindowsCFI requires that only consecutive registers can be paired.
27328bcb0991SDimitry Andric /// LR and FP need to be allocated together when the frame needs to save
27338bcb0991SDimitry Andric /// the frame-record. This means any other register pairing with LR is invalid.
invalidateRegisterPairing(unsigned Reg1,unsigned Reg2,bool UsesWinAAPCS,bool NeedsWinCFI,bool NeedsFrameRecord,bool IsFirst,const TargetRegisterInfo * TRI)27348bcb0991SDimitry Andric static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2,
2735e8d8bef9SDimitry Andric bool UsesWinAAPCS, bool NeedsWinCFI,
2736bdd1243dSDimitry Andric bool NeedsFrameRecord, bool IsFirst,
2737bdd1243dSDimitry Andric const TargetRegisterInfo *TRI) {
2738480093f4SDimitry Andric if (UsesWinAAPCS)
2739bdd1243dSDimitry Andric return invalidateWindowsRegisterPairing(Reg1, Reg2, NeedsWinCFI, IsFirst,
2740bdd1243dSDimitry Andric TRI);
27418bcb0991SDimitry Andric
27428bcb0991SDimitry Andric // If we need to store the frame record, don't pair any register
27438bcb0991SDimitry Andric // with LR other than FP.
27448bcb0991SDimitry Andric if (NeedsFrameRecord)
27458bcb0991SDimitry Andric return Reg2 == AArch64::LR;
27468bcb0991SDimitry Andric
27478bcb0991SDimitry Andric return false;
27488bcb0991SDimitry Andric }
27498bcb0991SDimitry Andric
27500b57cec5SDimitry Andric namespace {
27510b57cec5SDimitry Andric
27520b57cec5SDimitry Andric struct RegPairInfo {
27530b57cec5SDimitry Andric unsigned Reg1 = AArch64::NoRegister;
27540b57cec5SDimitry Andric unsigned Reg2 = AArch64::NoRegister;
27550b57cec5SDimitry Andric int FrameIdx;
27560b57cec5SDimitry Andric int Offset;
2757480093f4SDimitry Andric enum RegType { GPR, FPR64, FPR128, PPR, ZPR } Type;
27580b57cec5SDimitry Andric
27590b57cec5SDimitry Andric RegPairInfo() = default;
27600b57cec5SDimitry Andric
isPaired__anon79f8f0610411::RegPairInfo27610b57cec5SDimitry Andric bool isPaired() const { return Reg2 != AArch64::NoRegister; }
2762480093f4SDimitry Andric
getScale__anon79f8f0610411::RegPairInfo2763480093f4SDimitry Andric unsigned getScale() const {
2764480093f4SDimitry Andric switch (Type) {
2765480093f4SDimitry Andric case PPR:
2766480093f4SDimitry Andric return 2;
2767480093f4SDimitry Andric case GPR:
2768480093f4SDimitry Andric case FPR64:
2769480093f4SDimitry Andric return 8;
2770480093f4SDimitry Andric case ZPR:
2771480093f4SDimitry Andric case FPR128:
2772480093f4SDimitry Andric return 16;
2773480093f4SDimitry Andric }
2774480093f4SDimitry Andric llvm_unreachable("Unsupported type");
2775480093f4SDimitry Andric }
2776480093f4SDimitry Andric
isScalable__anon79f8f0610411::RegPairInfo2777480093f4SDimitry Andric bool isScalable() const { return Type == PPR || Type == ZPR; }
27780b57cec5SDimitry Andric };
27790b57cec5SDimitry Andric
27800b57cec5SDimitry Andric } // end anonymous namespace
27810b57cec5SDimitry Andric
computeCalleeSaveRegisterPairs(MachineFunction & MF,ArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI,SmallVectorImpl<RegPairInfo> & RegPairs,bool NeedsFrameRecord)27820b57cec5SDimitry Andric static void computeCalleeSaveRegisterPairs(
27835ffd83dbSDimitry Andric MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI,
27840b57cec5SDimitry Andric const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
278581ad6265SDimitry Andric bool NeedsFrameRecord) {
27860b57cec5SDimitry Andric
27870b57cec5SDimitry Andric if (CSI.empty())
27880b57cec5SDimitry Andric return;
27890b57cec5SDimitry Andric
2790480093f4SDimitry Andric bool IsWindows = isTargetWindows(MF);
27910b57cec5SDimitry Andric bool NeedsWinCFI = needsWinCFI(MF);
27920b57cec5SDimitry Andric AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
27930b57cec5SDimitry Andric MachineFrameInfo &MFI = MF.getFrameInfo();
27940b57cec5SDimitry Andric CallingConv::ID CC = MF.getFunction().getCallingConv();
27950b57cec5SDimitry Andric unsigned Count = CSI.size();
27960b57cec5SDimitry Andric (void)CC;
27970b57cec5SDimitry Andric // MachO's compact unwind format relies on all registers being stored in
27980b57cec5SDimitry Andric // pairs.
2799bdd1243dSDimitry Andric assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost ||
280006c3fb27SDimitry Andric CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS ||
280106c3fb27SDimitry Andric CC == CallingConv::Win64 || (Count & 1) == 0) &&
28020b57cec5SDimitry Andric "Odd number of callee-saved regs to spill!");
2803480093f4SDimitry Andric int ByteOffset = AFI->getCalleeSavedStackSize();
2804e8d8bef9SDimitry Andric int StackFillDir = -1;
2805e8d8bef9SDimitry Andric int RegInc = 1;
2806e8d8bef9SDimitry Andric unsigned FirstReg = 0;
2807e8d8bef9SDimitry Andric if (NeedsWinCFI) {
2808e8d8bef9SDimitry Andric // For WinCFI, fill the stack from the bottom up.
2809e8d8bef9SDimitry Andric ByteOffset = 0;
2810e8d8bef9SDimitry Andric StackFillDir = 1;
2811e8d8bef9SDimitry Andric // As the CSI array is reversed to match PrologEpilogInserter, iterate
2812e8d8bef9SDimitry Andric // backwards, to pair up registers starting from lower numbered registers.
2813e8d8bef9SDimitry Andric RegInc = -1;
2814e8d8bef9SDimitry Andric FirstReg = Count - 1;
2815e8d8bef9SDimitry Andric }
2816480093f4SDimitry Andric int ScalableByteOffset = AFI->getSVECalleeSavedStackSize();
2817fe6060f1SDimitry Andric bool NeedGapToAlignStack = AFI->hasCalleeSaveStackFreeSpace();
281875b4d546SDimitry Andric
2819e8d8bef9SDimitry Andric // When iterating backwards, the loop condition relies on unsigned wraparound.
2820e8d8bef9SDimitry Andric for (unsigned i = FirstReg; i < Count; i += RegInc) {
28210b57cec5SDimitry Andric RegPairInfo RPI;
28220b57cec5SDimitry Andric RPI.Reg1 = CSI[i].getReg();
28230b57cec5SDimitry Andric
28240b57cec5SDimitry Andric if (AArch64::GPR64RegClass.contains(RPI.Reg1))
28250b57cec5SDimitry Andric RPI.Type = RegPairInfo::GPR;
28260b57cec5SDimitry Andric else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
28270b57cec5SDimitry Andric RPI.Type = RegPairInfo::FPR64;
28280b57cec5SDimitry Andric else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
28290b57cec5SDimitry Andric RPI.Type = RegPairInfo::FPR128;
2830480093f4SDimitry Andric else if (AArch64::ZPRRegClass.contains(RPI.Reg1))
2831480093f4SDimitry Andric RPI.Type = RegPairInfo::ZPR;
2832480093f4SDimitry Andric else if (AArch64::PPRRegClass.contains(RPI.Reg1))
2833480093f4SDimitry Andric RPI.Type = RegPairInfo::PPR;
28340b57cec5SDimitry Andric else
28350b57cec5SDimitry Andric llvm_unreachable("Unsupported register class.");
28360b57cec5SDimitry Andric
28370b57cec5SDimitry Andric // Add the next reg to the pair if it is in the same register class.
2838e8d8bef9SDimitry Andric if (unsigned(i + RegInc) < Count) {
283904eeddc0SDimitry Andric Register NextReg = CSI[i + RegInc].getReg();
2840e8d8bef9SDimitry Andric bool IsFirst = i == FirstReg;
28410b57cec5SDimitry Andric switch (RPI.Type) {
28420b57cec5SDimitry Andric case RegPairInfo::GPR:
28430b57cec5SDimitry Andric if (AArch64::GPR64RegClass.contains(NextReg) &&
2844e8d8bef9SDimitry Andric !invalidateRegisterPairing(RPI.Reg1, NextReg, IsWindows,
2845bdd1243dSDimitry Andric NeedsWinCFI, NeedsFrameRecord, IsFirst,
2846bdd1243dSDimitry Andric TRI))
28470b57cec5SDimitry Andric RPI.Reg2 = NextReg;
28480b57cec5SDimitry Andric break;
28490b57cec5SDimitry Andric case RegPairInfo::FPR64:
28500b57cec5SDimitry Andric if (AArch64::FPR64RegClass.contains(NextReg) &&
2851e8d8bef9SDimitry Andric !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI,
2852bdd1243dSDimitry Andric IsFirst, TRI))
28530b57cec5SDimitry Andric RPI.Reg2 = NextReg;
28540b57cec5SDimitry Andric break;
28550b57cec5SDimitry Andric case RegPairInfo::FPR128:
28560b57cec5SDimitry Andric if (AArch64::FPR128RegClass.contains(NextReg))
28570b57cec5SDimitry Andric RPI.Reg2 = NextReg;
28580b57cec5SDimitry Andric break;
2859480093f4SDimitry Andric case RegPairInfo::PPR:
2860480093f4SDimitry Andric case RegPairInfo::ZPR:
2861480093f4SDimitry Andric break;
28620b57cec5SDimitry Andric }
28630b57cec5SDimitry Andric }
28640b57cec5SDimitry Andric
28650b57cec5SDimitry Andric // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
28660b57cec5SDimitry Andric // list to come in sorted by frame index so that we can issue the store
28670b57cec5SDimitry Andric // pair instructions directly. Assert if we see anything otherwise.
28680b57cec5SDimitry Andric //
28690b57cec5SDimitry Andric // The order of the registers in the list is controlled by
28700b57cec5SDimitry Andric // getCalleeSavedRegs(), so they will always be in-order, as well.
28710b57cec5SDimitry Andric assert((!RPI.isPaired() ||
2872e8d8bef9SDimitry Andric (CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
28730b57cec5SDimitry Andric "Out of order callee saved regs!");
28740b57cec5SDimitry Andric
28758bcb0991SDimitry Andric assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
28768bcb0991SDimitry Andric RPI.Reg1 == AArch64::LR) &&
28778bcb0991SDimitry Andric "FrameRecord must be allocated together with LR");
28788bcb0991SDimitry Andric
2879480093f4SDimitry Andric // Windows AAPCS has FP and LR reversed.
2880480093f4SDimitry Andric assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
2881480093f4SDimitry Andric RPI.Reg2 == AArch64::LR) &&
2882480093f4SDimitry Andric "FrameRecord must be allocated together with LR");
2883480093f4SDimitry Andric
28840b57cec5SDimitry Andric // MachO's compact unwind format relies on all registers being stored in
28850b57cec5SDimitry Andric // adjacent register pairs.
2886bdd1243dSDimitry Andric assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost ||
288706c3fb27SDimitry Andric CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS ||
288806c3fb27SDimitry Andric CC == CallingConv::Win64 ||
28890b57cec5SDimitry Andric (RPI.isPaired() &&
28900b57cec5SDimitry Andric ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
28910b57cec5SDimitry Andric RPI.Reg1 + 1 == RPI.Reg2))) &&
28920b57cec5SDimitry Andric "Callee-save registers not saved as adjacent register pair!");
28930b57cec5SDimitry Andric
28940b57cec5SDimitry Andric RPI.FrameIdx = CSI[i].getFrameIdx();
2895e8d8bef9SDimitry Andric if (NeedsWinCFI &&
2896e8d8bef9SDimitry Andric RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
2897e8d8bef9SDimitry Andric RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
28980b57cec5SDimitry Andric
2899480093f4SDimitry Andric int Scale = RPI.getScale();
2900e8d8bef9SDimitry Andric
2901e8d8bef9SDimitry Andric int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2902e8d8bef9SDimitry Andric assert(OffsetPre % Scale == 0);
2903e8d8bef9SDimitry Andric
2904480093f4SDimitry Andric if (RPI.isScalable())
2905e8d8bef9SDimitry Andric ScalableByteOffset += StackFillDir * Scale;
2906480093f4SDimitry Andric else
2907e8d8bef9SDimitry Andric ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
2908480093f4SDimitry Andric
2909fe6060f1SDimitry Andric // Swift's async context is directly before FP, so allocate an extra
2910fe6060f1SDimitry Andric // 8 bytes for it.
2911fe6060f1SDimitry Andric if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
29125f757f3fSDimitry Andric ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
29135f757f3fSDimitry Andric (IsWindows && RPI.Reg2 == AArch64::LR)))
2914fe6060f1SDimitry Andric ByteOffset += StackFillDir * 8;
2915fe6060f1SDimitry Andric
2916480093f4SDimitry Andric assert(!(RPI.isScalable() && RPI.isPaired()) &&
2917480093f4SDimitry Andric "Paired spill/fill instructions don't exist for SVE vectors");
29180b57cec5SDimitry Andric
29190b57cec5SDimitry Andric // Round up size of non-pair to pair size if we need to pad the
29200b57cec5SDimitry Andric // callee-save area to ensure 16-byte alignment.
2921fe6060f1SDimitry Andric if (NeedGapToAlignStack && !NeedsWinCFI &&
2922480093f4SDimitry Andric !RPI.isScalable() && RPI.Type != RegPairInfo::FPR128 &&
2923fe6060f1SDimitry Andric !RPI.isPaired() && ByteOffset % 16 != 0) {
2924e8d8bef9SDimitry Andric ByteOffset += 8 * StackFillDir;
29255ffd83dbSDimitry Andric assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
2926e8d8bef9SDimitry Andric // A stack frame with a gap looks like this, bottom up:
2927e8d8bef9SDimitry Andric // d9, d8. x21, gap, x20, x19.
2928fe6060f1SDimitry Andric // Set extra alignment on the x21 object to create the gap above it.
29295ffd83dbSDimitry Andric MFI.setObjectAlignment(RPI.FrameIdx, Align(16));
2930fe6060f1SDimitry Andric NeedGapToAlignStack = false;
29310b57cec5SDimitry Andric }
29320b57cec5SDimitry Andric
2933e8d8bef9SDimitry Andric int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2934e8d8bef9SDimitry Andric assert(OffsetPost % Scale == 0);
2935e8d8bef9SDimitry Andric // If filling top down (default), we want the offset after incrementing it.
29365f757f3fSDimitry Andric // If filling bottom up (WinCFI) we need the original offset.
2937e8d8bef9SDimitry Andric int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
2938fe6060f1SDimitry Andric
2939fe6060f1SDimitry Andric // The FP, LR pair goes 8 bytes into our expanded 24-byte slot so that the
2940fe6060f1SDimitry Andric // Swift context can directly precede FP.
2941fe6060f1SDimitry Andric if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
29425f757f3fSDimitry Andric ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
29435f757f3fSDimitry Andric (IsWindows && RPI.Reg2 == AArch64::LR)))
2944fe6060f1SDimitry Andric Offset += 8;
29450b57cec5SDimitry Andric RPI.Offset = Offset / Scale;
2946480093f4SDimitry Andric
2947480093f4SDimitry Andric assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
2948480093f4SDimitry Andric (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
29490b57cec5SDimitry Andric "Offset out of bounds for LDP/STP immediate");
29500b57cec5SDimitry Andric
2951e8d8bef9SDimitry Andric // Save the offset to frame record so that the FP register can point to the
2952e8d8bef9SDimitry Andric // innermost frame record (spilled FP and LR registers).
2953e8d8bef9SDimitry Andric if (NeedsFrameRecord && ((!IsWindows && RPI.Reg1 == AArch64::LR &&
2954e8d8bef9SDimitry Andric RPI.Reg2 == AArch64::FP) ||
2955e8d8bef9SDimitry Andric (IsWindows && RPI.Reg1 == AArch64::FP &&
2956e8d8bef9SDimitry Andric RPI.Reg2 == AArch64::LR)))
2957e8d8bef9SDimitry Andric AFI->setCalleeSaveBaseToFrameRecordOffset(Offset);
2958e8d8bef9SDimitry Andric
29590b57cec5SDimitry Andric RegPairs.push_back(RPI);
29600b57cec5SDimitry Andric if (RPI.isPaired())
2961e8d8bef9SDimitry Andric i += RegInc;
2962e8d8bef9SDimitry Andric }
2963e8d8bef9SDimitry Andric if (NeedsWinCFI) {
2964e8d8bef9SDimitry Andric // If we need an alignment gap in the stack, align the topmost stack
2965e8d8bef9SDimitry Andric // object. A stack frame with a gap looks like this, bottom up:
2966e8d8bef9SDimitry Andric // x19, d8. d9, gap.
2967e8d8bef9SDimitry Andric // Set extra alignment on the topmost stack object (the first element in
2968e8d8bef9SDimitry Andric // CSI, which goes top down), to create the gap above it.
2969e8d8bef9SDimitry Andric if (AFI->hasCalleeSaveStackFreeSpace())
2970e8d8bef9SDimitry Andric MFI.setObjectAlignment(CSI[0].getFrameIdx(), Align(16));
2971e8d8bef9SDimitry Andric // We iterated bottom up over the registers; flip RegPairs back to top
2972e8d8bef9SDimitry Andric // down order.
2973e8d8bef9SDimitry Andric std::reverse(RegPairs.begin(), RegPairs.end());
29740b57cec5SDimitry Andric }
29750b57cec5SDimitry Andric }
29760b57cec5SDimitry Andric
spillCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,ArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const29770b57cec5SDimitry Andric bool AArch64FrameLowering::spillCalleeSavedRegisters(
29780b57cec5SDimitry Andric MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
29795ffd83dbSDimitry Andric ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
29800b57cec5SDimitry Andric MachineFunction &MF = *MBB.getParent();
29810b57cec5SDimitry Andric const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
29820b57cec5SDimitry Andric bool NeedsWinCFI = needsWinCFI(MF);
29830b57cec5SDimitry Andric DebugLoc DL;
29840b57cec5SDimitry Andric SmallVector<RegPairInfo, 8> RegPairs;
29850b57cec5SDimitry Andric
298681ad6265SDimitry Andric computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF));
298781ad6265SDimitry Andric
29880b57cec5SDimitry Andric const MachineRegisterInfo &MRI = MF.getRegInfo();
2989fe6060f1SDimitry Andric if (homogeneousPrologEpilog(MF)) {
2990fe6060f1SDimitry Andric auto MIB = BuildMI(MBB, MI, DL, TII.get(AArch64::HOM_Prolog))
2991fe6060f1SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
2992fe6060f1SDimitry Andric
2993fe6060f1SDimitry Andric for (auto &RPI : RegPairs) {
2994fe6060f1SDimitry Andric MIB.addReg(RPI.Reg1);
2995fe6060f1SDimitry Andric MIB.addReg(RPI.Reg2);
2996fe6060f1SDimitry Andric
2997fe6060f1SDimitry Andric // Update register live in.
2998fe6060f1SDimitry Andric if (!MRI.isReserved(RPI.Reg1))
2999fe6060f1SDimitry Andric MBB.addLiveIn(RPI.Reg1);
30005f757f3fSDimitry Andric if (RPI.isPaired() && !MRI.isReserved(RPI.Reg2))
3001fe6060f1SDimitry Andric MBB.addLiveIn(RPI.Reg2);
3002fe6060f1SDimitry Andric }
3003fe6060f1SDimitry Andric return true;
3004fe6060f1SDimitry Andric }
3005349cc55cSDimitry Andric for (const RegPairInfo &RPI : llvm::reverse(RegPairs)) {
30060b57cec5SDimitry Andric unsigned Reg1 = RPI.Reg1;
30070b57cec5SDimitry Andric unsigned Reg2 = RPI.Reg2;
30080b57cec5SDimitry Andric unsigned StrOpc;
30090b57cec5SDimitry Andric
30100b57cec5SDimitry Andric // Issue sequence of spills for cs regs. The first spill may be converted
30110b57cec5SDimitry Andric // to a pre-decrement store later by emitPrologue if the callee-save stack
30120b57cec5SDimitry Andric // area allocation can't be combined with the local stack area allocation.
30130b57cec5SDimitry Andric // For example:
30140b57cec5SDimitry Andric // stp x22, x21, [sp, #0] // addImm(+0)
30150b57cec5SDimitry Andric // stp x20, x19, [sp, #16] // addImm(+2)
30160b57cec5SDimitry Andric // stp fp, lr, [sp, #32] // addImm(+4)
30170b57cec5SDimitry Andric // Rationale: This sequence saves uop updates compared to a sequence of
30180b57cec5SDimitry Andric // pre-increment spills like stp xi,xj,[sp,#-16]!
30190b57cec5SDimitry Andric // Note: Similar rationale and sequence for restores in epilog.
30205ffd83dbSDimitry Andric unsigned Size;
30215ffd83dbSDimitry Andric Align Alignment;
30220b57cec5SDimitry Andric switch (RPI.Type) {
30230b57cec5SDimitry Andric case RegPairInfo::GPR:
30240b57cec5SDimitry Andric StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
30250b57cec5SDimitry Andric Size = 8;
30265ffd83dbSDimitry Andric Alignment = Align(8);
30270b57cec5SDimitry Andric break;
30280b57cec5SDimitry Andric case RegPairInfo::FPR64:
30290b57cec5SDimitry Andric StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
30300b57cec5SDimitry Andric Size = 8;
30315ffd83dbSDimitry Andric Alignment = Align(8);
30320b57cec5SDimitry Andric break;
30330b57cec5SDimitry Andric case RegPairInfo::FPR128:
30340b57cec5SDimitry Andric StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
30350b57cec5SDimitry Andric Size = 16;
30365ffd83dbSDimitry Andric Alignment = Align(16);
30370b57cec5SDimitry Andric break;
3038480093f4SDimitry Andric case RegPairInfo::ZPR:
3039480093f4SDimitry Andric StrOpc = AArch64::STR_ZXI;
3040480093f4SDimitry Andric Size = 16;
30415ffd83dbSDimitry Andric Alignment = Align(16);
3042480093f4SDimitry Andric break;
3043480093f4SDimitry Andric case RegPairInfo::PPR:
3044480093f4SDimitry Andric StrOpc = AArch64::STR_PXI;
3045480093f4SDimitry Andric Size = 2;
30465ffd83dbSDimitry Andric Alignment = Align(2);
3047480093f4SDimitry Andric break;
30480b57cec5SDimitry Andric }
30490b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
30500b57cec5SDimitry Andric if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
30510b57cec5SDimitry Andric dbgs() << ") -> fi#(" << RPI.FrameIdx;
30520b57cec5SDimitry Andric if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
30530b57cec5SDimitry Andric dbgs() << ")\n");
30540b57cec5SDimitry Andric
30550b57cec5SDimitry Andric assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
30560b57cec5SDimitry Andric "Windows unwdinding requires a consecutive (FP,LR) pair");
30570b57cec5SDimitry Andric // Windows unwind codes require consecutive registers if registers are
30580b57cec5SDimitry Andric // paired. Make the switch here, so that the code below will save (x,x+1)
30590b57cec5SDimitry Andric // and not (x+1,x).
30600b57cec5SDimitry Andric unsigned FrameIdxReg1 = RPI.FrameIdx;
30610b57cec5SDimitry Andric unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
30620b57cec5SDimitry Andric if (NeedsWinCFI && RPI.isPaired()) {
30630b57cec5SDimitry Andric std::swap(Reg1, Reg2);
30640b57cec5SDimitry Andric std::swap(FrameIdxReg1, FrameIdxReg2);
30650b57cec5SDimitry Andric }
30660b57cec5SDimitry Andric MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
30670b57cec5SDimitry Andric if (!MRI.isReserved(Reg1))
30680b57cec5SDimitry Andric MBB.addLiveIn(Reg1);
30690b57cec5SDimitry Andric if (RPI.isPaired()) {
30700b57cec5SDimitry Andric if (!MRI.isReserved(Reg2))
30710b57cec5SDimitry Andric MBB.addLiveIn(Reg2);
30720b57cec5SDimitry Andric MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
30730b57cec5SDimitry Andric MIB.addMemOperand(MF.getMachineMemOperand(
30740b57cec5SDimitry Andric MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
30755ffd83dbSDimitry Andric MachineMemOperand::MOStore, Size, Alignment));
30760b57cec5SDimitry Andric }
30770b57cec5SDimitry Andric MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
30780b57cec5SDimitry Andric .addReg(AArch64::SP)
30790b57cec5SDimitry Andric .addImm(RPI.Offset) // [sp, #offset*scale],
30800b57cec5SDimitry Andric // where factor*scale is implicit
30810b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameSetup);
30820b57cec5SDimitry Andric MIB.addMemOperand(MF.getMachineMemOperand(
30830b57cec5SDimitry Andric MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
30845ffd83dbSDimitry Andric MachineMemOperand::MOStore, Size, Alignment));
30850b57cec5SDimitry Andric if (NeedsWinCFI)
30860b57cec5SDimitry Andric InsertSEH(MIB, TII, MachineInstr::FrameSetup);
30870b57cec5SDimitry Andric
3088480093f4SDimitry Andric // Update the StackIDs of the SVE stack slots.
3089480093f4SDimitry Andric MachineFrameInfo &MFI = MF.getFrameInfo();
3090480093f4SDimitry Andric if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR)
3091e8d8bef9SDimitry Andric MFI.setStackID(RPI.FrameIdx, TargetStackID::ScalableVector);
3092480093f4SDimitry Andric
30930b57cec5SDimitry Andric }
30940b57cec5SDimitry Andric return true;
30950b57cec5SDimitry Andric }
30960b57cec5SDimitry Andric
restoreCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MutableArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const30970b57cec5SDimitry Andric bool AArch64FrameLowering::restoreCalleeSavedRegisters(
309881ad6265SDimitry Andric MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
30995ffd83dbSDimitry Andric MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
31000b57cec5SDimitry Andric MachineFunction &MF = *MBB.getParent();
31010b57cec5SDimitry Andric const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
31020b57cec5SDimitry Andric DebugLoc DL;
31030b57cec5SDimitry Andric SmallVector<RegPairInfo, 8> RegPairs;
31040b57cec5SDimitry Andric bool NeedsWinCFI = needsWinCFI(MF);
31050b57cec5SDimitry Andric
310681ad6265SDimitry Andric if (MBBI != MBB.end())
310781ad6265SDimitry Andric DL = MBBI->getDebugLoc();
31080b57cec5SDimitry Andric
310981ad6265SDimitry Andric computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF));
31100b57cec5SDimitry Andric
311181ad6265SDimitry Andric auto EmitMI = [&](const RegPairInfo &RPI) -> MachineBasicBlock::iterator {
31120b57cec5SDimitry Andric unsigned Reg1 = RPI.Reg1;
31130b57cec5SDimitry Andric unsigned Reg2 = RPI.Reg2;
31140b57cec5SDimitry Andric
31150b57cec5SDimitry Andric // Issue sequence of restores for cs regs. The last restore may be converted
31160b57cec5SDimitry Andric // to a post-increment load later by emitEpilogue if the callee-save stack
31170b57cec5SDimitry Andric // area allocation can't be combined with the local stack area allocation.
31180b57cec5SDimitry Andric // For example:
31190b57cec5SDimitry Andric // ldp fp, lr, [sp, #32] // addImm(+4)
31200b57cec5SDimitry Andric // ldp x20, x19, [sp, #16] // addImm(+2)
31210b57cec5SDimitry Andric // ldp x22, x21, [sp, #0] // addImm(+0)
31220b57cec5SDimitry Andric // Note: see comment in spillCalleeSavedRegisters()
31230b57cec5SDimitry Andric unsigned LdrOpc;
31245ffd83dbSDimitry Andric unsigned Size;
31255ffd83dbSDimitry Andric Align Alignment;
31260b57cec5SDimitry Andric switch (RPI.Type) {
31270b57cec5SDimitry Andric case RegPairInfo::GPR:
31280b57cec5SDimitry Andric LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
31290b57cec5SDimitry Andric Size = 8;
31305ffd83dbSDimitry Andric Alignment = Align(8);
31310b57cec5SDimitry Andric break;
31320b57cec5SDimitry Andric case RegPairInfo::FPR64:
31330b57cec5SDimitry Andric LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
31340b57cec5SDimitry Andric Size = 8;
31355ffd83dbSDimitry Andric Alignment = Align(8);
31360b57cec5SDimitry Andric break;
31370b57cec5SDimitry Andric case RegPairInfo::FPR128:
31380b57cec5SDimitry Andric LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
31390b57cec5SDimitry Andric Size = 16;
31405ffd83dbSDimitry Andric Alignment = Align(16);
31410b57cec5SDimitry Andric break;
3142480093f4SDimitry Andric case RegPairInfo::ZPR:
3143480093f4SDimitry Andric LdrOpc = AArch64::LDR_ZXI;
3144480093f4SDimitry Andric Size = 16;
31455ffd83dbSDimitry Andric Alignment = Align(16);
3146480093f4SDimitry Andric break;
3147480093f4SDimitry Andric case RegPairInfo::PPR:
3148480093f4SDimitry Andric LdrOpc = AArch64::LDR_PXI;
3149480093f4SDimitry Andric Size = 2;
31505ffd83dbSDimitry Andric Alignment = Align(2);
3151480093f4SDimitry Andric break;
31520b57cec5SDimitry Andric }
31530b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
31540b57cec5SDimitry Andric if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
31550b57cec5SDimitry Andric dbgs() << ") -> fi#(" << RPI.FrameIdx;
31560b57cec5SDimitry Andric if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
31570b57cec5SDimitry Andric dbgs() << ")\n");
31580b57cec5SDimitry Andric
31590b57cec5SDimitry Andric // Windows unwind codes require consecutive registers if registers are
31600b57cec5SDimitry Andric // paired. Make the switch here, so that the code below will save (x,x+1)
31610b57cec5SDimitry Andric // and not (x+1,x).
31620b57cec5SDimitry Andric unsigned FrameIdxReg1 = RPI.FrameIdx;
31630b57cec5SDimitry Andric unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
31640b57cec5SDimitry Andric if (NeedsWinCFI && RPI.isPaired()) {
31650b57cec5SDimitry Andric std::swap(Reg1, Reg2);
31660b57cec5SDimitry Andric std::swap(FrameIdxReg1, FrameIdxReg2);
31670b57cec5SDimitry Andric }
316881ad6265SDimitry Andric MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
31690b57cec5SDimitry Andric if (RPI.isPaired()) {
31700b57cec5SDimitry Andric MIB.addReg(Reg2, getDefRegState(true));
31710b57cec5SDimitry Andric MIB.addMemOperand(MF.getMachineMemOperand(
31720b57cec5SDimitry Andric MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
31735ffd83dbSDimitry Andric MachineMemOperand::MOLoad, Size, Alignment));
31740b57cec5SDimitry Andric }
31750b57cec5SDimitry Andric MIB.addReg(Reg1, getDefRegState(true))
31760b57cec5SDimitry Andric .addReg(AArch64::SP)
31770b57cec5SDimitry Andric .addImm(RPI.Offset) // [sp, #offset*scale]
31780b57cec5SDimitry Andric // where factor*scale is implicit
31790b57cec5SDimitry Andric .setMIFlag(MachineInstr::FrameDestroy);
31800b57cec5SDimitry Andric MIB.addMemOperand(MF.getMachineMemOperand(
31810b57cec5SDimitry Andric MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
31825ffd83dbSDimitry Andric MachineMemOperand::MOLoad, Size, Alignment));
31830b57cec5SDimitry Andric if (NeedsWinCFI)
31840b57cec5SDimitry Andric InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
318581ad6265SDimitry Andric
318681ad6265SDimitry Andric return MIB->getIterator();
31870b57cec5SDimitry Andric };
3188480093f4SDimitry Andric
3189480093f4SDimitry Andric // SVE objects are always restored in reverse order.
31900b57cec5SDimitry Andric for (const RegPairInfo &RPI : reverse(RegPairs))
3191480093f4SDimitry Andric if (RPI.isScalable())
31920b57cec5SDimitry Andric EmitMI(RPI);
3193480093f4SDimitry Andric
319481ad6265SDimitry Andric if (homogeneousPrologEpilog(MF, &MBB)) {
319581ad6265SDimitry Andric auto MIB = BuildMI(MBB, MBBI, DL, TII.get(AArch64::HOM_Epilog))
3196fe6060f1SDimitry Andric .setMIFlag(MachineInstr::FrameDestroy);
3197fe6060f1SDimitry Andric for (auto &RPI : RegPairs) {
3198fe6060f1SDimitry Andric MIB.addReg(RPI.Reg1, RegState::Define);
3199fe6060f1SDimitry Andric MIB.addReg(RPI.Reg2, RegState::Define);
3200fe6060f1SDimitry Andric }
3201fe6060f1SDimitry Andric return true;
320281ad6265SDimitry Andric }
32030b57cec5SDimitry Andric
320481ad6265SDimitry Andric if (ReverseCSRRestoreSeq) {
320581ad6265SDimitry Andric MachineBasicBlock::iterator First = MBB.end();
320681ad6265SDimitry Andric for (const RegPairInfo &RPI : reverse(RegPairs)) {
320781ad6265SDimitry Andric if (RPI.isScalable())
320881ad6265SDimitry Andric continue;
320981ad6265SDimitry Andric MachineBasicBlock::iterator It = EmitMI(RPI);
321081ad6265SDimitry Andric if (First == MBB.end())
321181ad6265SDimitry Andric First = It;
321281ad6265SDimitry Andric }
321381ad6265SDimitry Andric if (First != MBB.end())
321481ad6265SDimitry Andric MBB.splice(MBBI, &MBB, First);
321581ad6265SDimitry Andric } else {
321681ad6265SDimitry Andric for (const RegPairInfo &RPI : RegPairs) {
321781ad6265SDimitry Andric if (RPI.isScalable())
321881ad6265SDimitry Andric continue;
321981ad6265SDimitry Andric (void)EmitMI(RPI);
322081ad6265SDimitry Andric }
32210b57cec5SDimitry Andric }
32220b57cec5SDimitry Andric
32230b57cec5SDimitry Andric return true;
32240b57cec5SDimitry Andric }
32250b57cec5SDimitry Andric
determineCalleeSaves(MachineFunction & MF,BitVector & SavedRegs,RegScavenger * RS) const32260b57cec5SDimitry Andric void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
32270b57cec5SDimitry Andric BitVector &SavedRegs,
32280b57cec5SDimitry Andric RegScavenger *RS) const {
32290b57cec5SDimitry Andric // All calls are tail calls in GHC calling conv, and functions have no
32300b57cec5SDimitry Andric // prologue/epilogue.
32310b57cec5SDimitry Andric if (MF.getFunction().getCallingConv() == CallingConv::GHC)
32320b57cec5SDimitry Andric return;
32330b57cec5SDimitry Andric
32340b57cec5SDimitry Andric TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
32350b57cec5SDimitry Andric const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
32360b57cec5SDimitry Andric MF.getSubtarget().getRegisterInfo());
32375ffd83dbSDimitry Andric const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
32380b57cec5SDimitry Andric AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
32390b57cec5SDimitry Andric unsigned UnspilledCSGPR = AArch64::NoRegister;
32400b57cec5SDimitry Andric unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
32410b57cec5SDimitry Andric
32420b57cec5SDimitry Andric MachineFrameInfo &MFI = MF.getFrameInfo();
32430b57cec5SDimitry Andric const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
32440b57cec5SDimitry Andric
32450b57cec5SDimitry Andric unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
32460b57cec5SDimitry Andric ? RegInfo->getBaseRegister()
32470b57cec5SDimitry Andric : (unsigned)AArch64::NoRegister;
32480b57cec5SDimitry Andric
32490b57cec5SDimitry Andric unsigned ExtraCSSpill = 0;
32505f757f3fSDimitry Andric bool HasUnpairedGPR64 = false;
32510b57cec5SDimitry Andric // Figure out which callee-saved registers to save/restore.
32520b57cec5SDimitry Andric for (unsigned i = 0; CSRegs[i]; ++i) {
32530b57cec5SDimitry Andric const unsigned Reg = CSRegs[i];
32540b57cec5SDimitry Andric
32550b57cec5SDimitry Andric // Add the base pointer register to SavedRegs if it is callee-save.
32560b57cec5SDimitry Andric if (Reg == BasePointerReg)
32570b57cec5SDimitry Andric SavedRegs.set(Reg);
32580b57cec5SDimitry Andric
32590b57cec5SDimitry Andric bool RegUsed = SavedRegs.test(Reg);
3260480093f4SDimitry Andric unsigned PairedReg = AArch64::NoRegister;
32615f757f3fSDimitry Andric const bool RegIsGPR64 = AArch64::GPR64RegClass.contains(Reg);
32625f757f3fSDimitry Andric if (RegIsGPR64 || AArch64::FPR64RegClass.contains(Reg) ||
32635f757f3fSDimitry Andric AArch64::FPR128RegClass.contains(Reg)) {
32645f757f3fSDimitry Andric // Compensate for odd numbers of GP CSRs.
32655f757f3fSDimitry Andric // For now, all the known cases of odd number of CSRs are of GPRs.
32665f757f3fSDimitry Andric if (HasUnpairedGPR64)
32675f757f3fSDimitry Andric PairedReg = CSRegs[i % 2 == 0 ? i - 1 : i + 1];
32685f757f3fSDimitry Andric else
3269480093f4SDimitry Andric PairedReg = CSRegs[i ^ 1];
32705f757f3fSDimitry Andric }
32715f757f3fSDimitry Andric
32725f757f3fSDimitry Andric // If the function requires all the GP registers to save (SavedRegs),
32735f757f3fSDimitry Andric // and there are an odd number of GP CSRs at the same time (CSRegs),
32745f757f3fSDimitry Andric // PairedReg could be in a different register class from Reg, which would
32755f757f3fSDimitry Andric // lead to a FPR (usually D8) accidentally being marked saved.
32765f757f3fSDimitry Andric if (RegIsGPR64 && !AArch64::GPR64RegClass.contains(PairedReg)) {
32775f757f3fSDimitry Andric PairedReg = AArch64::NoRegister;
32785f757f3fSDimitry Andric HasUnpairedGPR64 = true;
32795f757f3fSDimitry Andric }
32805f757f3fSDimitry Andric assert(PairedReg == AArch64::NoRegister ||
32815f757f3fSDimitry Andric AArch64::GPR64RegClass.contains(Reg, PairedReg) ||
32825f757f3fSDimitry Andric AArch64::FPR64RegClass.contains(Reg, PairedReg) ||
32835f757f3fSDimitry Andric AArch64::FPR128RegClass.contains(Reg, PairedReg));
3284480093f4SDimitry Andric
32850b57cec5SDimitry Andric if (!RegUsed) {
32860b57cec5SDimitry Andric if (AArch64::GPR64RegClass.contains(Reg) &&
32870b57cec5SDimitry Andric !RegInfo->isReservedReg(MF, Reg)) {
32880b57cec5SDimitry Andric UnspilledCSGPR = Reg;
32890b57cec5SDimitry Andric UnspilledCSGPRPaired = PairedReg;
32900b57cec5SDimitry Andric }
32910b57cec5SDimitry Andric continue;
32920b57cec5SDimitry Andric }
32930b57cec5SDimitry Andric
32940b57cec5SDimitry Andric // MachO's compact unwind format relies on all registers being stored in
32950b57cec5SDimitry Andric // pairs.
32960b57cec5SDimitry Andric // FIXME: the usual format is actually better if unwinding isn't needed.
3297fe6060f1SDimitry Andric if (producePairRegisters(MF) && PairedReg != AArch64::NoRegister &&
32980b57cec5SDimitry Andric !SavedRegs.test(PairedReg)) {
32990b57cec5SDimitry Andric SavedRegs.set(PairedReg);
33000b57cec5SDimitry Andric if (AArch64::GPR64RegClass.contains(PairedReg) &&
33010b57cec5SDimitry Andric !RegInfo->isReservedReg(MF, PairedReg))
33020b57cec5SDimitry Andric ExtraCSSpill = PairedReg;
33030b57cec5SDimitry Andric }
33040b57cec5SDimitry Andric }
33050b57cec5SDimitry Andric
33065ffd83dbSDimitry Andric if (MF.getFunction().getCallingConv() == CallingConv::Win64 &&
33075ffd83dbSDimitry Andric !Subtarget.isTargetWindows()) {
33085ffd83dbSDimitry Andric // For Windows calling convention on a non-windows OS, where X18 is treated
33095ffd83dbSDimitry Andric // as reserved, back up X18 when entering non-windows code (marked with the
33105ffd83dbSDimitry Andric // Windows calling convention) and restore when returning regardless of
33115ffd83dbSDimitry Andric // whether the individual function uses it - it might call other functions
33125ffd83dbSDimitry Andric // that clobber it.
33135ffd83dbSDimitry Andric SavedRegs.set(AArch64::X18);
33145ffd83dbSDimitry Andric }
33155ffd83dbSDimitry Andric
33160b57cec5SDimitry Andric // Calculates the callee saved stack size.
33170b57cec5SDimitry Andric unsigned CSStackSize = 0;
3318480093f4SDimitry Andric unsigned SVECSStackSize = 0;
33190b57cec5SDimitry Andric const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
33200b57cec5SDimitry Andric const MachineRegisterInfo &MRI = MF.getRegInfo();
3321480093f4SDimitry Andric for (unsigned Reg : SavedRegs.set_bits()) {
3322480093f4SDimitry Andric auto RegSize = TRI->getRegSizeInBits(Reg, MRI) / 8;
3323480093f4SDimitry Andric if (AArch64::PPRRegClass.contains(Reg) ||
3324480093f4SDimitry Andric AArch64::ZPRRegClass.contains(Reg))
3325480093f4SDimitry Andric SVECSStackSize += RegSize;
3326480093f4SDimitry Andric else
3327480093f4SDimitry Andric CSStackSize += RegSize;
3328480093f4SDimitry Andric }
33290b57cec5SDimitry Andric
33300b57cec5SDimitry Andric // Save number of saved regs, so we can easily update CSStackSize later.
33310b57cec5SDimitry Andric unsigned NumSavedRegs = SavedRegs.count();
33320b57cec5SDimitry Andric
33330b57cec5SDimitry Andric // The frame record needs to be created by saving the appropriate registers
3334480093f4SDimitry Andric uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
33350b57cec5SDimitry Andric if (hasFP(MF) ||
33360b57cec5SDimitry Andric windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
33370b57cec5SDimitry Andric SavedRegs.set(AArch64::FP);
33380b57cec5SDimitry Andric SavedRegs.set(AArch64::LR);
33390b57cec5SDimitry Andric }
33400b57cec5SDimitry Andric
33418bcb0991SDimitry Andric LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
33420b57cec5SDimitry Andric for (unsigned Reg
33430b57cec5SDimitry Andric : SavedRegs.set_bits()) dbgs()
33440b57cec5SDimitry Andric << ' ' << printReg(Reg, RegInfo);
33450b57cec5SDimitry Andric dbgs() << "\n";);
33460b57cec5SDimitry Andric
33470b57cec5SDimitry Andric // If any callee-saved registers are used, the frame cannot be eliminated.
33488bcb0991SDimitry Andric int64_t SVEStackSize =
3349480093f4SDimitry Andric alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
33508bcb0991SDimitry Andric bool CanEliminateFrame = (SavedRegs.count() == 0) && !SVEStackSize;
33510b57cec5SDimitry Andric
33520b57cec5SDimitry Andric // The CSR spill slots have not been allocated yet, so estimateStackSize
33530b57cec5SDimitry Andric // won't include them.
33540b57cec5SDimitry Andric unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
33558bcb0991SDimitry Andric
335606c3fb27SDimitry Andric // We may address some of the stack above the canonical frame address, either
335706c3fb27SDimitry Andric // for our own arguments or during a call. Include that in calculating whether
335806c3fb27SDimitry Andric // we have complicated addressing concerns.
335906c3fb27SDimitry Andric int64_t CalleeStackUsed = 0;
336006c3fb27SDimitry Andric for (int I = MFI.getObjectIndexBegin(); I != 0; ++I) {
336106c3fb27SDimitry Andric int64_t FixedOff = MFI.getObjectOffset(I);
336206c3fb27SDimitry Andric if (FixedOff > CalleeStackUsed) CalleeStackUsed = FixedOff;
336306c3fb27SDimitry Andric }
336406c3fb27SDimitry Andric
33658bcb0991SDimitry Andric // Conservatively always assume BigStack when there are SVE spills.
336606c3fb27SDimitry Andric bool BigStack = SVEStackSize || (EstimatedStackSize + CSStackSize +
336706c3fb27SDimitry Andric CalleeStackUsed) > EstimatedStackSizeLimit;
33680b57cec5SDimitry Andric if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
33690b57cec5SDimitry Andric AFI->setHasStackFrame(true);
33700b57cec5SDimitry Andric
33710b57cec5SDimitry Andric // Estimate if we might need to scavenge a register at some point in order
33720b57cec5SDimitry Andric // to materialize a stack offset. If so, either spill one additional
33730b57cec5SDimitry Andric // callee-saved register or reserve a special spill slot to facilitate
33740b57cec5SDimitry Andric // register scavenging. If we already spilled an extra callee-saved register
33750b57cec5SDimitry Andric // above to keep the number of spills even, we don't need to do anything else
33760b57cec5SDimitry Andric // here.
33770b57cec5SDimitry Andric if (BigStack) {
33780b57cec5SDimitry Andric if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
33790b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
33800b57cec5SDimitry Andric << " to get a scratch register.\n");
33810b57cec5SDimitry Andric SavedRegs.set(UnspilledCSGPR);
33825f757f3fSDimitry Andric ExtraCSSpill = UnspilledCSGPR;
33835f757f3fSDimitry Andric
33840b57cec5SDimitry Andric // MachO's compact unwind format relies on all registers being stored in
33850b57cec5SDimitry Andric // pairs, so if we need to spill one extra for BigStack, then we need to
33860b57cec5SDimitry Andric // store the pair.
33875f757f3fSDimitry Andric if (producePairRegisters(MF)) {
33885f757f3fSDimitry Andric if (UnspilledCSGPRPaired == AArch64::NoRegister) {
33895f757f3fSDimitry Andric // Failed to make a pair for compact unwind format, revert spilling.
33905f757f3fSDimitry Andric if (produceCompactUnwindFrame(MF)) {
33915f757f3fSDimitry Andric SavedRegs.reset(UnspilledCSGPR);
33925f757f3fSDimitry Andric ExtraCSSpill = AArch64::NoRegister;
33935f757f3fSDimitry Andric }
33945f757f3fSDimitry Andric } else
33950b57cec5SDimitry Andric SavedRegs.set(UnspilledCSGPRPaired);
33965f757f3fSDimitry Andric }
33970b57cec5SDimitry Andric }
33980b57cec5SDimitry Andric
33990b57cec5SDimitry Andric // If we didn't find an extra callee-saved register to spill, create
34000b57cec5SDimitry Andric // an emergency spill slot.
34010b57cec5SDimitry Andric if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
34020b57cec5SDimitry Andric const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
34030b57cec5SDimitry Andric const TargetRegisterClass &RC = AArch64::GPR64RegClass;
34040b57cec5SDimitry Andric unsigned Size = TRI->getSpillSize(RC);
34055ffd83dbSDimitry Andric Align Alignment = TRI->getSpillAlign(RC);
34065ffd83dbSDimitry Andric int FI = MFI.CreateStackObject(Size, Alignment, false);
34070b57cec5SDimitry Andric RS->addScavengingFrameIndex(FI);
34080b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
34090b57cec5SDimitry Andric << " as the emergency spill slot.\n");
34100b57cec5SDimitry Andric }
34110b57cec5SDimitry Andric }
34120b57cec5SDimitry Andric
34130b57cec5SDimitry Andric // Adding the size of additional 64bit GPR saves.
34140b57cec5SDimitry Andric CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
3415fe6060f1SDimitry Andric
3416fe6060f1SDimitry Andric // A Swift asynchronous context extends the frame record with a pointer
3417fe6060f1SDimitry Andric // directly before FP.
3418fe6060f1SDimitry Andric if (hasFP(MF) && AFI->hasSwiftAsyncContext())
3419fe6060f1SDimitry Andric CSStackSize += 8;
3420fe6060f1SDimitry Andric
3421480093f4SDimitry Andric uint64_t AlignedCSStackSize = alignTo(CSStackSize, 16);
34220b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
34230b57cec5SDimitry Andric << EstimatedStackSize + AlignedCSStackSize
34240b57cec5SDimitry Andric << " bytes.\n");
34250b57cec5SDimitry Andric
3426480093f4SDimitry Andric assert((!MFI.isCalleeSavedInfoValid() ||
3427480093f4SDimitry Andric AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
3428480093f4SDimitry Andric "Should not invalidate callee saved info");
3429480093f4SDimitry Andric
34300b57cec5SDimitry Andric // Round up to register pair alignment to avoid additional SP adjustment
34310b57cec5SDimitry Andric // instructions.
34320b57cec5SDimitry Andric AFI->setCalleeSavedStackSize(AlignedCSStackSize);
34330b57cec5SDimitry Andric AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
3434480093f4SDimitry Andric AFI->setSVECalleeSavedStackSize(alignTo(SVECSStackSize, 16));
34350b57cec5SDimitry Andric }
34360b57cec5SDimitry Andric
assignCalleeSavedSpillSlots(MachineFunction & MF,const TargetRegisterInfo * RegInfo,std::vector<CalleeSavedInfo> & CSI,unsigned & MinCSFrameIndex,unsigned & MaxCSFrameIndex) const3437e8d8bef9SDimitry Andric bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
3438fe6060f1SDimitry Andric MachineFunction &MF, const TargetRegisterInfo *RegInfo,
3439fe6060f1SDimitry Andric std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
3440fe6060f1SDimitry Andric unsigned &MaxCSFrameIndex) const {
3441e8d8bef9SDimitry Andric bool NeedsWinCFI = needsWinCFI(MF);
3442e8d8bef9SDimitry Andric // To match the canonical windows frame layout, reverse the list of
3443e8d8bef9SDimitry Andric // callee saved registers to get them laid out by PrologEpilogInserter
3444e8d8bef9SDimitry Andric // in the right order. (PrologEpilogInserter allocates stack objects top
3445e8d8bef9SDimitry Andric // down. Windows canonical prologs store higher numbered registers at
3446e8d8bef9SDimitry Andric // the top, thus have the CSI array start from the highest registers.)
3447e8d8bef9SDimitry Andric if (NeedsWinCFI)
3448e8d8bef9SDimitry Andric std::reverse(CSI.begin(), CSI.end());
3449fe6060f1SDimitry Andric
3450fe6060f1SDimitry Andric if (CSI.empty())
3451fe6060f1SDimitry Andric return true; // Early exit if no callee saved registers are modified!
3452fe6060f1SDimitry Andric
3453fe6060f1SDimitry Andric // Now that we know which registers need to be saved and restored, allocate
3454fe6060f1SDimitry Andric // stack slots for them.
3455fe6060f1SDimitry Andric MachineFrameInfo &MFI = MF.getFrameInfo();
3456fe6060f1SDimitry Andric auto *AFI = MF.getInfo<AArch64FunctionInfo>();
345781ad6265SDimitry Andric
345881ad6265SDimitry Andric bool UsesWinAAPCS = isTargetWindows(MF);
345981ad6265SDimitry Andric if (UsesWinAAPCS && hasFP(MF) && AFI->hasSwiftAsyncContext()) {
346081ad6265SDimitry Andric int FrameIdx = MFI.CreateStackObject(8, Align(16), true);
346181ad6265SDimitry Andric AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
346281ad6265SDimitry Andric if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
346381ad6265SDimitry Andric if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
346481ad6265SDimitry Andric }
346581ad6265SDimitry Andric
3466fe6060f1SDimitry Andric for (auto &CS : CSI) {
3467fe6060f1SDimitry Andric Register Reg = CS.getReg();
3468fe6060f1SDimitry Andric const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
3469fe6060f1SDimitry Andric
3470fe6060f1SDimitry Andric unsigned Size = RegInfo->getSpillSize(*RC);
3471fe6060f1SDimitry Andric Align Alignment(RegInfo->getSpillAlign(*RC));
3472fe6060f1SDimitry Andric int FrameIdx = MFI.CreateStackObject(Size, Alignment, true);
3473fe6060f1SDimitry Andric CS.setFrameIdx(FrameIdx);
3474fe6060f1SDimitry Andric
3475fe6060f1SDimitry Andric if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3476fe6060f1SDimitry Andric if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3477fe6060f1SDimitry Andric
3478fe6060f1SDimitry Andric // Grab 8 bytes below FP for the extended asynchronous frame info.
347981ad6265SDimitry Andric if (hasFP(MF) && AFI->hasSwiftAsyncContext() && !UsesWinAAPCS &&
348081ad6265SDimitry Andric Reg == AArch64::FP) {
3481fe6060f1SDimitry Andric FrameIdx = MFI.CreateStackObject(8, Alignment, true);
3482fe6060f1SDimitry Andric AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
3483fe6060f1SDimitry Andric if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3484fe6060f1SDimitry Andric if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3485fe6060f1SDimitry Andric }
3486fe6060f1SDimitry Andric }
3487fe6060f1SDimitry Andric return true;
3488e8d8bef9SDimitry Andric }
3489e8d8bef9SDimitry Andric
enableStackSlotScavenging(const MachineFunction & MF) const34900b57cec5SDimitry Andric bool AArch64FrameLowering::enableStackSlotScavenging(
34910b57cec5SDimitry Andric const MachineFunction &MF) const {
34920b57cec5SDimitry Andric const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
34935f757f3fSDimitry Andric // If the function has streaming-mode changes, don't scavenge a
34945f757f3fSDimitry Andric // spillslot in the callee-save area, as that might require an
34955f757f3fSDimitry Andric // 'addvl' in the streaming-mode-changing call-sequence when the
34965f757f3fSDimitry Andric // function doesn't use a FP.
34975f757f3fSDimitry Andric if (AFI->hasStreamingModeChanges() && !hasFP(MF))
34985f757f3fSDimitry Andric return false;
34990b57cec5SDimitry Andric return AFI->hasCalleeSaveStackFreeSpace();
35000b57cec5SDimitry Andric }
35010b57cec5SDimitry Andric
3502480093f4SDimitry Andric /// returns true if there are any SVE callee saves.
getSVECalleeSaveSlotRange(const MachineFrameInfo & MFI,int & Min,int & Max)3503480093f4SDimitry Andric static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI,
3504480093f4SDimitry Andric int &Min, int &Max) {
3505480093f4SDimitry Andric Min = std::numeric_limits<int>::max();
3506480093f4SDimitry Andric Max = std::numeric_limits<int>::min();
3507480093f4SDimitry Andric
3508480093f4SDimitry Andric if (!MFI.isCalleeSavedInfoValid())
3509480093f4SDimitry Andric return false;
3510480093f4SDimitry Andric
3511480093f4SDimitry Andric const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
3512480093f4SDimitry Andric for (auto &CS : CSI) {
3513480093f4SDimitry Andric if (AArch64::ZPRRegClass.contains(CS.getReg()) ||
3514480093f4SDimitry Andric AArch64::PPRRegClass.contains(CS.getReg())) {
3515480093f4SDimitry Andric assert((Max == std::numeric_limits<int>::min() ||
3516480093f4SDimitry Andric Max + 1 == CS.getFrameIdx()) &&
3517480093f4SDimitry Andric "SVE CalleeSaves are not consecutive");
3518480093f4SDimitry Andric
3519480093f4SDimitry Andric Min = std::min(Min, CS.getFrameIdx());
3520480093f4SDimitry Andric Max = std::max(Max, CS.getFrameIdx());
3521480093f4SDimitry Andric }
3522480093f4SDimitry Andric }
3523480093f4SDimitry Andric return Min != std::numeric_limits<int>::max();
3524480093f4SDimitry Andric }
3525480093f4SDimitry Andric
3526480093f4SDimitry Andric // Process all the SVE stack objects and determine offsets for each
3527480093f4SDimitry Andric // object. If AssignOffsets is true, the offsets get assigned.
3528480093f4SDimitry Andric // Fills in the first and last callee-saved frame indices into
3529480093f4SDimitry Andric // Min/MaxCSFrameIndex, respectively.
3530480093f4SDimitry Andric // Returns the size of the stack.
determineSVEStackObjectOffsets(MachineFrameInfo & MFI,int & MinCSFrameIndex,int & MaxCSFrameIndex,bool AssignOffsets)3531480093f4SDimitry Andric static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
3532480093f4SDimitry Andric int &MinCSFrameIndex,
3533480093f4SDimitry Andric int &MaxCSFrameIndex,
3534480093f4SDimitry Andric bool AssignOffsets) {
3535979e22ffSDimitry Andric #ifndef NDEBUG
3536480093f4SDimitry Andric // First process all fixed stack objects.
35378bcb0991SDimitry Andric for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
3538e8d8bef9SDimitry Andric assert(MFI.getStackID(I) != TargetStackID::ScalableVector &&
3539979e22ffSDimitry Andric "SVE vectors should never be passed on the stack by value, only by "
3540979e22ffSDimitry Andric "reference.");
3541979e22ffSDimitry Andric #endif
35428bcb0991SDimitry Andric
3543480093f4SDimitry Andric auto Assign = [&MFI](int FI, int64_t Offset) {
3544480093f4SDimitry Andric LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
3545480093f4SDimitry Andric MFI.setObjectOffset(FI, Offset);
3546480093f4SDimitry Andric };
3547480093f4SDimitry Andric
3548979e22ffSDimitry Andric int64_t Offset = 0;
3549979e22ffSDimitry Andric
3550480093f4SDimitry Andric // Then process all callee saved slots.
3551480093f4SDimitry Andric if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
3552480093f4SDimitry Andric // Assign offsets to the callee save slots.
3553480093f4SDimitry Andric for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
3554480093f4SDimitry Andric Offset += MFI.getObjectSize(I);
35555ffd83dbSDimitry Andric Offset = alignTo(Offset, MFI.getObjectAlign(I));
3556480093f4SDimitry Andric if (AssignOffsets)
3557480093f4SDimitry Andric Assign(I, -Offset);
3558480093f4SDimitry Andric }
3559480093f4SDimitry Andric }
3560480093f4SDimitry Andric
3561979e22ffSDimitry Andric // Ensure that the Callee-save area is aligned to 16bytes.
3562979e22ffSDimitry Andric Offset = alignTo(Offset, Align(16U));
3563979e22ffSDimitry Andric
3564480093f4SDimitry Andric // Create a buffer of SVE objects to allocate and sort it.
3565480093f4SDimitry Andric SmallVector<int, 8> ObjectsToAllocate;
35660eae32dcSDimitry Andric // If we have a stack protector, and we've previously decided that we have SVE
35670eae32dcSDimitry Andric // objects on the stack and thus need it to go in the SVE stack area, then it
35680eae32dcSDimitry Andric // needs to go first.
35690eae32dcSDimitry Andric int StackProtectorFI = -1;
35700eae32dcSDimitry Andric if (MFI.hasStackProtectorIndex()) {
35710eae32dcSDimitry Andric StackProtectorFI = MFI.getStackProtectorIndex();
35720eae32dcSDimitry Andric if (MFI.getStackID(StackProtectorFI) == TargetStackID::ScalableVector)
35730eae32dcSDimitry Andric ObjectsToAllocate.push_back(StackProtectorFI);
35740eae32dcSDimitry Andric }
3575480093f4SDimitry Andric for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
3576480093f4SDimitry Andric unsigned StackID = MFI.getStackID(I);
3577e8d8bef9SDimitry Andric if (StackID != TargetStackID::ScalableVector)
3578480093f4SDimitry Andric continue;
35790eae32dcSDimitry Andric if (I == StackProtectorFI)
35800eae32dcSDimitry Andric continue;
3581480093f4SDimitry Andric if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex)
3582480093f4SDimitry Andric continue;
3583480093f4SDimitry Andric if (MFI.isDeadObjectIndex(I))
3584480093f4SDimitry Andric continue;
3585480093f4SDimitry Andric
3586480093f4SDimitry Andric ObjectsToAllocate.push_back(I);
3587480093f4SDimitry Andric }
3588480093f4SDimitry Andric
3589480093f4SDimitry Andric // Allocate all SVE locals and spills
3590480093f4SDimitry Andric for (unsigned FI : ObjectsToAllocate) {
35915ffd83dbSDimitry Andric Align Alignment = MFI.getObjectAlign(FI);
3592480093f4SDimitry Andric // FIXME: Given that the length of SVE vectors is not necessarily a power of
3593480093f4SDimitry Andric // two, we'd need to align every object dynamically at runtime if the
3594480093f4SDimitry Andric // alignment is larger than 16. This is not yet supported.
35955ffd83dbSDimitry Andric if (Alignment > Align(16))
3596480093f4SDimitry Andric report_fatal_error(
3597480093f4SDimitry Andric "Alignment of scalable vectors > 16 bytes is not yet supported");
3598480093f4SDimitry Andric
35995ffd83dbSDimitry Andric Offset = alignTo(Offset + MFI.getObjectSize(FI), Alignment);
3600480093f4SDimitry Andric if (AssignOffsets)
3601480093f4SDimitry Andric Assign(FI, -Offset);
3602480093f4SDimitry Andric }
3603480093f4SDimitry Andric
36048bcb0991SDimitry Andric return Offset;
36058bcb0991SDimitry Andric }
36068bcb0991SDimitry Andric
estimateSVEStackObjectOffsets(MachineFrameInfo & MFI) const3607480093f4SDimitry Andric int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
3608480093f4SDimitry Andric MachineFrameInfo &MFI) const {
3609480093f4SDimitry Andric int MinCSFrameIndex, MaxCSFrameIndex;
3610480093f4SDimitry Andric return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex, false);
3611480093f4SDimitry Andric }
3612480093f4SDimitry Andric
assignSVEStackObjectOffsets(MachineFrameInfo & MFI,int & MinCSFrameIndex,int & MaxCSFrameIndex) const3613480093f4SDimitry Andric int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
3614480093f4SDimitry Andric MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex) const {
3615480093f4SDimitry Andric return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex,
3616480093f4SDimitry Andric true);
3617480093f4SDimitry Andric }
3618480093f4SDimitry Andric
processFunctionBeforeFrameFinalized(MachineFunction & MF,RegScavenger * RS) const36190b57cec5SDimitry Andric void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
36200b57cec5SDimitry Andric MachineFunction &MF, RegScavenger *RS) const {
36218bcb0991SDimitry Andric MachineFrameInfo &MFI = MF.getFrameInfo();
36228bcb0991SDimitry Andric
36238bcb0991SDimitry Andric assert(getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown &&
36248bcb0991SDimitry Andric "Upwards growing stack unsupported");
36258bcb0991SDimitry Andric
3626480093f4SDimitry Andric int MinCSFrameIndex, MaxCSFrameIndex;
3627480093f4SDimitry Andric int64_t SVEStackSize =
3628480093f4SDimitry Andric assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
36298bcb0991SDimitry Andric
36308bcb0991SDimitry Andric AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
3631480093f4SDimitry Andric AFI->setStackSizeSVE(alignTo(SVEStackSize, 16U));
3632480093f4SDimitry Andric AFI->setMinMaxSVECSFrameIndex(MinCSFrameIndex, MaxCSFrameIndex);
36338bcb0991SDimitry Andric
36340b57cec5SDimitry Andric // If this function isn't doing Win64-style C++ EH, we don't need to do
36350b57cec5SDimitry Andric // anything.
36360b57cec5SDimitry Andric if (!MF.hasEHFunclets())
36370b57cec5SDimitry Andric return;
36380b57cec5SDimitry Andric const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
36390b57cec5SDimitry Andric WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
36400b57cec5SDimitry Andric
36410b57cec5SDimitry Andric MachineBasicBlock &MBB = MF.front();
36420b57cec5SDimitry Andric auto MBBI = MBB.begin();
36430b57cec5SDimitry Andric while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
36440b57cec5SDimitry Andric ++MBBI;
36450b57cec5SDimitry Andric
36460b57cec5SDimitry Andric // Create an UnwindHelp object.
364762cfcf62SDimitry Andric // The UnwindHelp object is allocated at the start of the fixed object area
364862cfcf62SDimitry Andric int64_t FixedObject =
364962cfcf62SDimitry Andric getFixedObjectSize(MF, AFI, /*IsWin64*/ true, /*IsFunclet*/ false);
365062cfcf62SDimitry Andric int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8,
365162cfcf62SDimitry Andric /*SPOffset*/ -FixedObject,
365262cfcf62SDimitry Andric /*IsImmutable=*/false);
36530b57cec5SDimitry Andric EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
365462cfcf62SDimitry Andric
36550b57cec5SDimitry Andric // We need to store -2 into the UnwindHelp object at the start of the
36560b57cec5SDimitry Andric // function.
36570b57cec5SDimitry Andric DebugLoc DL;
36580b57cec5SDimitry Andric RS->enterBasicBlockEnd(MBB);
36595f757f3fSDimitry Andric RS->backward(MBBI);
366004eeddc0SDimitry Andric Register DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
36610b57cec5SDimitry Andric assert(DstReg && "There must be a free register after frame setup");
36620b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
36630b57cec5SDimitry Andric BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
36640b57cec5SDimitry Andric .addReg(DstReg, getKillRegState(true))
36650b57cec5SDimitry Andric .addFrameIndex(UnwindHelpFI)
36660b57cec5SDimitry Andric .addImm(0);
36670b57cec5SDimitry Andric }
36680b57cec5SDimitry Andric
36695ffd83dbSDimitry Andric namespace {
36705ffd83dbSDimitry Andric struct TagStoreInstr {
36715ffd83dbSDimitry Andric MachineInstr *MI;
36725ffd83dbSDimitry Andric int64_t Offset, Size;
TagStoreInstr__anon79f8f0610711::TagStoreInstr36735ffd83dbSDimitry Andric explicit TagStoreInstr(MachineInstr *MI, int64_t Offset, int64_t Size)
36745ffd83dbSDimitry Andric : MI(MI), Offset(Offset), Size(Size) {}
36755ffd83dbSDimitry Andric };
36765ffd83dbSDimitry Andric
36775ffd83dbSDimitry Andric class TagStoreEdit {
36785ffd83dbSDimitry Andric MachineFunction *MF;
36795ffd83dbSDimitry Andric MachineBasicBlock *MBB;
36805ffd83dbSDimitry Andric MachineRegisterInfo *MRI;
36815ffd83dbSDimitry Andric // Tag store instructions that are being replaced.
36825ffd83dbSDimitry Andric SmallVector<TagStoreInstr, 8> TagStores;
36835ffd83dbSDimitry Andric // Combined memref arguments of the above instructions.
36845ffd83dbSDimitry Andric SmallVector<MachineMemOperand *, 8> CombinedMemRefs;
36855ffd83dbSDimitry Andric
36865ffd83dbSDimitry Andric // Replace allocation tags in [FrameReg + FrameRegOffset, FrameReg +
36875ffd83dbSDimitry Andric // FrameRegOffset + Size) with the address tag of SP.
36885ffd83dbSDimitry Andric Register FrameReg;
36895ffd83dbSDimitry Andric StackOffset FrameRegOffset;
36905ffd83dbSDimitry Andric int64_t Size;
369106c3fb27SDimitry Andric // If not std::nullopt, move FrameReg to (FrameReg + FrameRegUpdate) at the
369206c3fb27SDimitry Andric // end.
3693bdd1243dSDimitry Andric std::optional<int64_t> FrameRegUpdate;
36945ffd83dbSDimitry Andric // MIFlags for any FrameReg updating instructions.
36955ffd83dbSDimitry Andric unsigned FrameRegUpdateFlags;
36965ffd83dbSDimitry Andric
36975ffd83dbSDimitry Andric // Use zeroing instruction variants.
36985ffd83dbSDimitry Andric bool ZeroData;
36995ffd83dbSDimitry Andric DebugLoc DL;
37005ffd83dbSDimitry Andric
37015ffd83dbSDimitry Andric void emitUnrolled(MachineBasicBlock::iterator InsertI);
37025ffd83dbSDimitry Andric void emitLoop(MachineBasicBlock::iterator InsertI);
37035ffd83dbSDimitry Andric
37045ffd83dbSDimitry Andric public:
TagStoreEdit(MachineBasicBlock * MBB,bool ZeroData)37055ffd83dbSDimitry Andric TagStoreEdit(MachineBasicBlock *MBB, bool ZeroData)
37065ffd83dbSDimitry Andric : MBB(MBB), ZeroData(ZeroData) {
37075ffd83dbSDimitry Andric MF = MBB->getParent();
37085ffd83dbSDimitry Andric MRI = &MF->getRegInfo();
37095ffd83dbSDimitry Andric }
37105ffd83dbSDimitry Andric // Add an instruction to be replaced. Instructions must be added in the
37115ffd83dbSDimitry Andric // ascending order of Offset, and have to be adjacent.
addInstruction(TagStoreInstr I)37125ffd83dbSDimitry Andric void addInstruction(TagStoreInstr I) {
37135ffd83dbSDimitry Andric assert((TagStores.empty() ||
37145ffd83dbSDimitry Andric TagStores.back().Offset + TagStores.back().Size == I.Offset) &&
37155ffd83dbSDimitry Andric "Non-adjacent tag store instructions.");
37165ffd83dbSDimitry Andric TagStores.push_back(I);
37175ffd83dbSDimitry Andric }
clear()37185ffd83dbSDimitry Andric void clear() { TagStores.clear(); }
37195ffd83dbSDimitry Andric // Emit equivalent code at the given location, and erase the current set of
37205ffd83dbSDimitry Andric // instructions. May skip if the replacement is not profitable. May invalidate
37215ffd83dbSDimitry Andric // the input iterator and replace it with a valid one.
37225ffd83dbSDimitry Andric void emitCode(MachineBasicBlock::iterator &InsertI,
372381ad6265SDimitry Andric const AArch64FrameLowering *TFI, bool TryMergeSPUpdate);
37245ffd83dbSDimitry Andric };
37255ffd83dbSDimitry Andric
emitUnrolled(MachineBasicBlock::iterator InsertI)37265ffd83dbSDimitry Andric void TagStoreEdit::emitUnrolled(MachineBasicBlock::iterator InsertI) {
37275ffd83dbSDimitry Andric const AArch64InstrInfo *TII =
37285ffd83dbSDimitry Andric MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
37295ffd83dbSDimitry Andric
37305ffd83dbSDimitry Andric const int64_t kMinOffset = -256 * 16;
37315ffd83dbSDimitry Andric const int64_t kMaxOffset = 255 * 16;
37325ffd83dbSDimitry Andric
37335ffd83dbSDimitry Andric Register BaseReg = FrameReg;
3734e8d8bef9SDimitry Andric int64_t BaseRegOffsetBytes = FrameRegOffset.getFixed();
37355ffd83dbSDimitry Andric if (BaseRegOffsetBytes < kMinOffset ||
373606c3fb27SDimitry Andric BaseRegOffsetBytes + (Size - Size % 32) > kMaxOffset ||
373706c3fb27SDimitry Andric // BaseReg can be FP, which is not necessarily aligned to 16-bytes. In
373806c3fb27SDimitry Andric // that case, BaseRegOffsetBytes will not be aligned to 16 bytes, which
373906c3fb27SDimitry Andric // is required for the offset of ST2G.
374006c3fb27SDimitry Andric BaseRegOffsetBytes % 16 != 0) {
37415ffd83dbSDimitry Andric Register ScratchReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
37425ffd83dbSDimitry Andric emitFrameOffset(*MBB, InsertI, DL, ScratchReg, BaseReg,
3743e8d8bef9SDimitry Andric StackOffset::getFixed(BaseRegOffsetBytes), TII);
37445ffd83dbSDimitry Andric BaseReg = ScratchReg;
37455ffd83dbSDimitry Andric BaseRegOffsetBytes = 0;
37465ffd83dbSDimitry Andric }
37475ffd83dbSDimitry Andric
37485ffd83dbSDimitry Andric MachineInstr *LastI = nullptr;
37495ffd83dbSDimitry Andric while (Size) {
37505ffd83dbSDimitry Andric int64_t InstrSize = (Size > 16) ? 32 : 16;
37515ffd83dbSDimitry Andric unsigned Opcode =
37525ffd83dbSDimitry Andric InstrSize == 16
375306c3fb27SDimitry Andric ? (ZeroData ? AArch64::STZGi : AArch64::STGi)
375406c3fb27SDimitry Andric : (ZeroData ? AArch64::STZ2Gi : AArch64::ST2Gi);
375506c3fb27SDimitry Andric assert(BaseRegOffsetBytes % 16 == 0);
37565ffd83dbSDimitry Andric MachineInstr *I = BuildMI(*MBB, InsertI, DL, TII->get(Opcode))
37575ffd83dbSDimitry Andric .addReg(AArch64::SP)
37585ffd83dbSDimitry Andric .addReg(BaseReg)
37595ffd83dbSDimitry Andric .addImm(BaseRegOffsetBytes / 16)
37605ffd83dbSDimitry Andric .setMemRefs(CombinedMemRefs);
37615ffd83dbSDimitry Andric // A store to [BaseReg, #0] should go last for an opportunity to fold the
37625ffd83dbSDimitry Andric // final SP adjustment in the epilogue.
37635ffd83dbSDimitry Andric if (BaseRegOffsetBytes == 0)
37645ffd83dbSDimitry Andric LastI = I;
37655ffd83dbSDimitry Andric BaseRegOffsetBytes += InstrSize;
37665ffd83dbSDimitry Andric Size -= InstrSize;
37675ffd83dbSDimitry Andric }
37685ffd83dbSDimitry Andric
37695ffd83dbSDimitry Andric if (LastI)
37705ffd83dbSDimitry Andric MBB->splice(InsertI, MBB, LastI);
37715ffd83dbSDimitry Andric }
37725ffd83dbSDimitry Andric
emitLoop(MachineBasicBlock::iterator InsertI)37735ffd83dbSDimitry Andric void TagStoreEdit::emitLoop(MachineBasicBlock::iterator InsertI) {
37745ffd83dbSDimitry Andric const AArch64InstrInfo *TII =
37755ffd83dbSDimitry Andric MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
37765ffd83dbSDimitry Andric
37775ffd83dbSDimitry Andric Register BaseReg = FrameRegUpdate
37785ffd83dbSDimitry Andric ? FrameReg
37795ffd83dbSDimitry Andric : MRI->createVirtualRegister(&AArch64::GPR64RegClass);
37805ffd83dbSDimitry Andric Register SizeReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
37815ffd83dbSDimitry Andric
37825ffd83dbSDimitry Andric emitFrameOffset(*MBB, InsertI, DL, BaseReg, FrameReg, FrameRegOffset, TII);
37835ffd83dbSDimitry Andric
37845ffd83dbSDimitry Andric int64_t LoopSize = Size;
37855ffd83dbSDimitry Andric // If the loop size is not a multiple of 32, split off one 16-byte store at
37865ffd83dbSDimitry Andric // the end to fold BaseReg update into.
37875ffd83dbSDimitry Andric if (FrameRegUpdate && *FrameRegUpdate)
37885ffd83dbSDimitry Andric LoopSize -= LoopSize % 32;
37895ffd83dbSDimitry Andric MachineInstr *LoopI = BuildMI(*MBB, InsertI, DL,
37905ffd83dbSDimitry Andric TII->get(ZeroData ? AArch64::STZGloop_wback
37915ffd83dbSDimitry Andric : AArch64::STGloop_wback))
37925ffd83dbSDimitry Andric .addDef(SizeReg)
37935ffd83dbSDimitry Andric .addDef(BaseReg)
37945ffd83dbSDimitry Andric .addImm(LoopSize)
37955ffd83dbSDimitry Andric .addReg(BaseReg)
37965ffd83dbSDimitry Andric .setMemRefs(CombinedMemRefs);
37975ffd83dbSDimitry Andric if (FrameRegUpdate)
37985ffd83dbSDimitry Andric LoopI->setFlags(FrameRegUpdateFlags);
37995ffd83dbSDimitry Andric
38005ffd83dbSDimitry Andric int64_t ExtraBaseRegUpdate =
3801e8d8bef9SDimitry Andric FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.getFixed() - Size) : 0;
38025ffd83dbSDimitry Andric if (LoopSize < Size) {
38035ffd83dbSDimitry Andric assert(FrameRegUpdate);
38045ffd83dbSDimitry Andric assert(Size - LoopSize == 16);
38055ffd83dbSDimitry Andric // Tag 16 more bytes at BaseReg and update BaseReg.
38065ffd83dbSDimitry Andric BuildMI(*MBB, InsertI, DL,
38075ffd83dbSDimitry Andric TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
38085ffd83dbSDimitry Andric .addDef(BaseReg)
38095ffd83dbSDimitry Andric .addReg(BaseReg)
38105ffd83dbSDimitry Andric .addReg(BaseReg)
38115ffd83dbSDimitry Andric .addImm(1 + ExtraBaseRegUpdate / 16)
38125ffd83dbSDimitry Andric .setMemRefs(CombinedMemRefs)
38135ffd83dbSDimitry Andric .setMIFlags(FrameRegUpdateFlags);
38145ffd83dbSDimitry Andric } else if (ExtraBaseRegUpdate) {
38155ffd83dbSDimitry Andric // Update BaseReg.
38165ffd83dbSDimitry Andric BuildMI(
38175ffd83dbSDimitry Andric *MBB, InsertI, DL,
38185ffd83dbSDimitry Andric TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
38195ffd83dbSDimitry Andric .addDef(BaseReg)
38205ffd83dbSDimitry Andric .addReg(BaseReg)
38215ffd83dbSDimitry Andric .addImm(std::abs(ExtraBaseRegUpdate))
38225ffd83dbSDimitry Andric .addImm(0)
38235ffd83dbSDimitry Andric .setMIFlags(FrameRegUpdateFlags);
38245ffd83dbSDimitry Andric }
38255ffd83dbSDimitry Andric }
38265ffd83dbSDimitry Andric
38275ffd83dbSDimitry Andric // Check if *II is a register update that can be merged into STGloop that ends
38285ffd83dbSDimitry Andric // at (Reg + Size). RemainingOffset is the required adjustment to Reg after the
38295ffd83dbSDimitry Andric // end of the loop.
canMergeRegUpdate(MachineBasicBlock::iterator II,unsigned Reg,int64_t Size,int64_t * TotalOffset)38305ffd83dbSDimitry Andric bool canMergeRegUpdate(MachineBasicBlock::iterator II, unsigned Reg,
38315ffd83dbSDimitry Andric int64_t Size, int64_t *TotalOffset) {
38325ffd83dbSDimitry Andric MachineInstr &MI = *II;
38335ffd83dbSDimitry Andric if ((MI.getOpcode() == AArch64::ADDXri ||
38345ffd83dbSDimitry Andric MI.getOpcode() == AArch64::SUBXri) &&
38355ffd83dbSDimitry Andric MI.getOperand(0).getReg() == Reg && MI.getOperand(1).getReg() == Reg) {
38365ffd83dbSDimitry Andric unsigned Shift = AArch64_AM::getShiftValue(MI.getOperand(3).getImm());
38375ffd83dbSDimitry Andric int64_t Offset = MI.getOperand(2).getImm() << Shift;
38385ffd83dbSDimitry Andric if (MI.getOpcode() == AArch64::SUBXri)
38395ffd83dbSDimitry Andric Offset = -Offset;
38405ffd83dbSDimitry Andric int64_t AbsPostOffset = std::abs(Offset - Size);
38415ffd83dbSDimitry Andric const int64_t kMaxOffset =
38425ffd83dbSDimitry Andric 0xFFF; // Max encoding for unshifted ADDXri / SUBXri
38435ffd83dbSDimitry Andric if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
38445ffd83dbSDimitry Andric *TotalOffset = Offset;
38455ffd83dbSDimitry Andric return true;
38465ffd83dbSDimitry Andric }
38475ffd83dbSDimitry Andric }
38485ffd83dbSDimitry Andric return false;
38495ffd83dbSDimitry Andric }
38505ffd83dbSDimitry Andric
mergeMemRefs(const SmallVectorImpl<TagStoreInstr> & TSE,SmallVectorImpl<MachineMemOperand * > & MemRefs)38515ffd83dbSDimitry Andric void mergeMemRefs(const SmallVectorImpl<TagStoreInstr> &TSE,
38525ffd83dbSDimitry Andric SmallVectorImpl<MachineMemOperand *> &MemRefs) {
38535ffd83dbSDimitry Andric MemRefs.clear();
38545ffd83dbSDimitry Andric for (auto &TS : TSE) {
38555ffd83dbSDimitry Andric MachineInstr *MI = TS.MI;
38565ffd83dbSDimitry Andric // An instruction without memory operands may access anything. Be
38575ffd83dbSDimitry Andric // conservative and return an empty list.
38585ffd83dbSDimitry Andric if (MI->memoperands_empty()) {
38595ffd83dbSDimitry Andric MemRefs.clear();
38605ffd83dbSDimitry Andric return;
38615ffd83dbSDimitry Andric }
38625ffd83dbSDimitry Andric MemRefs.append(MI->memoperands_begin(), MI->memoperands_end());
38635ffd83dbSDimitry Andric }
38645ffd83dbSDimitry Andric }
38655ffd83dbSDimitry Andric
emitCode(MachineBasicBlock::iterator & InsertI,const AArch64FrameLowering * TFI,bool TryMergeSPUpdate)38665ffd83dbSDimitry Andric void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
386781ad6265SDimitry Andric const AArch64FrameLowering *TFI,
386881ad6265SDimitry Andric bool TryMergeSPUpdate) {
38695ffd83dbSDimitry Andric if (TagStores.empty())
38705ffd83dbSDimitry Andric return;
38715ffd83dbSDimitry Andric TagStoreInstr &FirstTagStore = TagStores[0];
38725ffd83dbSDimitry Andric TagStoreInstr &LastTagStore = TagStores[TagStores.size() - 1];
38735ffd83dbSDimitry Andric Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
38745ffd83dbSDimitry Andric DL = TagStores[0].MI->getDebugLoc();
38755ffd83dbSDimitry Andric
38765ffd83dbSDimitry Andric Register Reg;
38775ffd83dbSDimitry Andric FrameRegOffset = TFI->resolveFrameOffsetReference(
38785ffd83dbSDimitry Andric *MF, FirstTagStore.Offset, false /*isFixed*/, false /*isSVE*/, Reg,
38795ffd83dbSDimitry Andric /*PreferFP=*/false, /*ForSimm=*/true);
38805ffd83dbSDimitry Andric FrameReg = Reg;
3881bdd1243dSDimitry Andric FrameRegUpdate = std::nullopt;
38825ffd83dbSDimitry Andric
38835ffd83dbSDimitry Andric mergeMemRefs(TagStores, CombinedMemRefs);
38845ffd83dbSDimitry Andric
38855ffd83dbSDimitry Andric LLVM_DEBUG(dbgs() << "Replacing adjacent STG instructions:\n";
38865ffd83dbSDimitry Andric for (const auto &Instr
38875ffd83dbSDimitry Andric : TagStores) { dbgs() << " " << *Instr.MI; });
38885ffd83dbSDimitry Andric
38895ffd83dbSDimitry Andric // Size threshold where a loop becomes shorter than a linear sequence of
38905ffd83dbSDimitry Andric // tagging instructions.
38915ffd83dbSDimitry Andric const int kSetTagLoopThreshold = 176;
38925ffd83dbSDimitry Andric if (Size < kSetTagLoopThreshold) {
38935ffd83dbSDimitry Andric if (TagStores.size() < 2)
38945ffd83dbSDimitry Andric return;
38955ffd83dbSDimitry Andric emitUnrolled(InsertI);
38965ffd83dbSDimitry Andric } else {
38975ffd83dbSDimitry Andric MachineInstr *UpdateInstr = nullptr;
389881ad6265SDimitry Andric int64_t TotalOffset = 0;
389981ad6265SDimitry Andric if (TryMergeSPUpdate) {
39005ffd83dbSDimitry Andric // See if we can merge base register update into the STGloop.
39015ffd83dbSDimitry Andric // This is done in AArch64LoadStoreOptimizer for "normal" stores,
39025ffd83dbSDimitry Andric // but STGloop is way too unusual for that, and also it only
39035ffd83dbSDimitry Andric // realistically happens in function epilogue. Also, STGloop is expanded
39045ffd83dbSDimitry Andric // before that pass.
39055ffd83dbSDimitry Andric if (InsertI != MBB->end() &&
3906e8d8bef9SDimitry Andric canMergeRegUpdate(InsertI, FrameReg, FrameRegOffset.getFixed() + Size,
39075ffd83dbSDimitry Andric &TotalOffset)) {
39085ffd83dbSDimitry Andric UpdateInstr = &*InsertI++;
39095ffd83dbSDimitry Andric LLVM_DEBUG(dbgs() << "Folding SP update into loop:\n "
39105ffd83dbSDimitry Andric << *UpdateInstr);
39115ffd83dbSDimitry Andric }
39125ffd83dbSDimitry Andric }
39135ffd83dbSDimitry Andric
39145ffd83dbSDimitry Andric if (!UpdateInstr && TagStores.size() < 2)
39155ffd83dbSDimitry Andric return;
39165ffd83dbSDimitry Andric
39175ffd83dbSDimitry Andric if (UpdateInstr) {
39185ffd83dbSDimitry Andric FrameRegUpdate = TotalOffset;
39195ffd83dbSDimitry Andric FrameRegUpdateFlags = UpdateInstr->getFlags();
39205ffd83dbSDimitry Andric }
39215ffd83dbSDimitry Andric emitLoop(InsertI);
39225ffd83dbSDimitry Andric if (UpdateInstr)
39235ffd83dbSDimitry Andric UpdateInstr->eraseFromParent();
39245ffd83dbSDimitry Andric }
39255ffd83dbSDimitry Andric
39265ffd83dbSDimitry Andric for (auto &TS : TagStores)
39275ffd83dbSDimitry Andric TS.MI->eraseFromParent();
39285ffd83dbSDimitry Andric }
39295ffd83dbSDimitry Andric
isMergeableStackTaggingInstruction(MachineInstr & MI,int64_t & Offset,int64_t & Size,bool & ZeroData)39305ffd83dbSDimitry Andric bool isMergeableStackTaggingInstruction(MachineInstr &MI, int64_t &Offset,
39315ffd83dbSDimitry Andric int64_t &Size, bool &ZeroData) {
39325ffd83dbSDimitry Andric MachineFunction &MF = *MI.getParent()->getParent();
39335ffd83dbSDimitry Andric const MachineFrameInfo &MFI = MF.getFrameInfo();
39345ffd83dbSDimitry Andric
39355ffd83dbSDimitry Andric unsigned Opcode = MI.getOpcode();
393606c3fb27SDimitry Andric ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGi ||
393706c3fb27SDimitry Andric Opcode == AArch64::STZ2Gi);
39385ffd83dbSDimitry Andric
39395ffd83dbSDimitry Andric if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
39405ffd83dbSDimitry Andric if (!MI.getOperand(0).isDead() || !MI.getOperand(1).isDead())
39415ffd83dbSDimitry Andric return false;
39425ffd83dbSDimitry Andric if (!MI.getOperand(2).isImm() || !MI.getOperand(3).isFI())
39435ffd83dbSDimitry Andric return false;
39445ffd83dbSDimitry Andric Offset = MFI.getObjectOffset(MI.getOperand(3).getIndex());
39455ffd83dbSDimitry Andric Size = MI.getOperand(2).getImm();
39465ffd83dbSDimitry Andric return true;
39475ffd83dbSDimitry Andric }
39485ffd83dbSDimitry Andric
394906c3fb27SDimitry Andric if (Opcode == AArch64::STGi || Opcode == AArch64::STZGi)
39505ffd83dbSDimitry Andric Size = 16;
395106c3fb27SDimitry Andric else if (Opcode == AArch64::ST2Gi || Opcode == AArch64::STZ2Gi)
39525ffd83dbSDimitry Andric Size = 32;
39535ffd83dbSDimitry Andric else
39545ffd83dbSDimitry Andric return false;
39555ffd83dbSDimitry Andric
39565ffd83dbSDimitry Andric if (MI.getOperand(0).getReg() != AArch64::SP || !MI.getOperand(1).isFI())
39575ffd83dbSDimitry Andric return false;
39585ffd83dbSDimitry Andric
39595ffd83dbSDimitry Andric Offset = MFI.getObjectOffset(MI.getOperand(1).getIndex()) +
39605ffd83dbSDimitry Andric 16 * MI.getOperand(2).getImm();
39615ffd83dbSDimitry Andric return true;
39625ffd83dbSDimitry Andric }
39635ffd83dbSDimitry Andric
39645ffd83dbSDimitry Andric // Detect a run of memory tagging instructions for adjacent stack frame slots,
39655ffd83dbSDimitry Andric // and replace them with a shorter instruction sequence:
39665ffd83dbSDimitry Andric // * replace STG + STG with ST2G
39675ffd83dbSDimitry Andric // * replace STGloop + STGloop with STGloop
39685ffd83dbSDimitry Andric // This code needs to run when stack slot offsets are already known, but before
39695ffd83dbSDimitry Andric // FrameIndex operands in STG instructions are eliminated.
tryMergeAdjacentSTG(MachineBasicBlock::iterator II,const AArch64FrameLowering * TFI,RegScavenger * RS)39705ffd83dbSDimitry Andric MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II,
39715ffd83dbSDimitry Andric const AArch64FrameLowering *TFI,
39725ffd83dbSDimitry Andric RegScavenger *RS) {
39735ffd83dbSDimitry Andric bool FirstZeroData;
39745ffd83dbSDimitry Andric int64_t Size, Offset;
39755ffd83dbSDimitry Andric MachineInstr &MI = *II;
39765ffd83dbSDimitry Andric MachineBasicBlock *MBB = MI.getParent();
39775ffd83dbSDimitry Andric MachineBasicBlock::iterator NextI = ++II;
39785ffd83dbSDimitry Andric if (&MI == &MBB->instr_back())
39795ffd83dbSDimitry Andric return II;
39805ffd83dbSDimitry Andric if (!isMergeableStackTaggingInstruction(MI, Offset, Size, FirstZeroData))
39815ffd83dbSDimitry Andric return II;
39825ffd83dbSDimitry Andric
39835ffd83dbSDimitry Andric SmallVector<TagStoreInstr, 4> Instrs;
39845ffd83dbSDimitry Andric Instrs.emplace_back(&MI, Offset, Size);
39855ffd83dbSDimitry Andric
39865ffd83dbSDimitry Andric constexpr int kScanLimit = 10;
39875ffd83dbSDimitry Andric int Count = 0;
39885ffd83dbSDimitry Andric for (MachineBasicBlock::iterator E = MBB->end();
39895ffd83dbSDimitry Andric NextI != E && Count < kScanLimit; ++NextI) {
39905ffd83dbSDimitry Andric MachineInstr &MI = *NextI;
39915ffd83dbSDimitry Andric bool ZeroData;
39925ffd83dbSDimitry Andric int64_t Size, Offset;
39935ffd83dbSDimitry Andric // Collect instructions that update memory tags with a FrameIndex operand
39945ffd83dbSDimitry Andric // and (when applicable) constant size, and whose output registers are dead
39955ffd83dbSDimitry Andric // (the latter is almost always the case in practice). Since these
39965ffd83dbSDimitry Andric // instructions effectively have no inputs or outputs, we are free to skip
39975ffd83dbSDimitry Andric // any non-aliasing instructions in between without tracking used registers.
39985ffd83dbSDimitry Andric if (isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData)) {
39995ffd83dbSDimitry Andric if (ZeroData != FirstZeroData)
40005ffd83dbSDimitry Andric break;
40015ffd83dbSDimitry Andric Instrs.emplace_back(&MI, Offset, Size);
40025ffd83dbSDimitry Andric continue;
40035ffd83dbSDimitry Andric }
40045ffd83dbSDimitry Andric
40055ffd83dbSDimitry Andric // Only count non-transient, non-tagging instructions toward the scan
40065ffd83dbSDimitry Andric // limit.
40075ffd83dbSDimitry Andric if (!MI.isTransient())
40085ffd83dbSDimitry Andric ++Count;
40095ffd83dbSDimitry Andric
40105ffd83dbSDimitry Andric // Just in case, stop before the epilogue code starts.
40115ffd83dbSDimitry Andric if (MI.getFlag(MachineInstr::FrameSetup) ||
40125ffd83dbSDimitry Andric MI.getFlag(MachineInstr::FrameDestroy))
40135ffd83dbSDimitry Andric break;
40145ffd83dbSDimitry Andric
40155ffd83dbSDimitry Andric // Reject anything that may alias the collected instructions.
40165ffd83dbSDimitry Andric if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects())
40175ffd83dbSDimitry Andric break;
40185ffd83dbSDimitry Andric }
40195ffd83dbSDimitry Andric
40205ffd83dbSDimitry Andric // New code will be inserted after the last tagging instruction we've found.
40215ffd83dbSDimitry Andric MachineBasicBlock::iterator InsertI = Instrs.back().MI;
40225f757f3fSDimitry Andric
40235f757f3fSDimitry Andric // All the gathered stack tag instructions are merged and placed after
40245f757f3fSDimitry Andric // last tag store in the list. The check should be made if the nzcv
40255f757f3fSDimitry Andric // flag is live at the point where we are trying to insert. Otherwise
40265f757f3fSDimitry Andric // the nzcv flag might get clobbered if any stg loops are present.
40275f757f3fSDimitry Andric
40285f757f3fSDimitry Andric // FIXME : This approach of bailing out from merge is conservative in
40295f757f3fSDimitry Andric // some ways like even if stg loops are not present after merge the
40305f757f3fSDimitry Andric // insert list, this liveness check is done (which is not needed).
40315f757f3fSDimitry Andric LivePhysRegs LiveRegs(*(MBB->getParent()->getSubtarget().getRegisterInfo()));
40325f757f3fSDimitry Andric LiveRegs.addLiveOuts(*MBB);
40335f757f3fSDimitry Andric for (auto I = MBB->rbegin();; ++I) {
40345f757f3fSDimitry Andric MachineInstr &MI = *I;
40355f757f3fSDimitry Andric if (MI == InsertI)
40365f757f3fSDimitry Andric break;
40375f757f3fSDimitry Andric LiveRegs.stepBackward(*I);
40385f757f3fSDimitry Andric }
40395ffd83dbSDimitry Andric InsertI++;
40405f757f3fSDimitry Andric if (LiveRegs.contains(AArch64::NZCV))
40415f757f3fSDimitry Andric return InsertI;
40425ffd83dbSDimitry Andric
40435ffd83dbSDimitry Andric llvm::stable_sort(Instrs,
40445ffd83dbSDimitry Andric [](const TagStoreInstr &Left, const TagStoreInstr &Right) {
40455ffd83dbSDimitry Andric return Left.Offset < Right.Offset;
40465ffd83dbSDimitry Andric });
40475ffd83dbSDimitry Andric
40485ffd83dbSDimitry Andric // Make sure that we don't have any overlapping stores.
40495ffd83dbSDimitry Andric int64_t CurOffset = Instrs[0].Offset;
40505ffd83dbSDimitry Andric for (auto &Instr : Instrs) {
40515ffd83dbSDimitry Andric if (CurOffset > Instr.Offset)
40525ffd83dbSDimitry Andric return NextI;
40535ffd83dbSDimitry Andric CurOffset = Instr.Offset + Instr.Size;
40545ffd83dbSDimitry Andric }
40555ffd83dbSDimitry Andric
40565ffd83dbSDimitry Andric // Find contiguous runs of tagged memory and emit shorter instruction
40575ffd83dbSDimitry Andric // sequencies for them when possible.
40585ffd83dbSDimitry Andric TagStoreEdit TSE(MBB, FirstZeroData);
4059bdd1243dSDimitry Andric std::optional<int64_t> EndOffset;
40605ffd83dbSDimitry Andric for (auto &Instr : Instrs) {
40615ffd83dbSDimitry Andric if (EndOffset && *EndOffset != Instr.Offset) {
40625ffd83dbSDimitry Andric // Found a gap.
406381ad6265SDimitry Andric TSE.emitCode(InsertI, TFI, /*TryMergeSPUpdate = */ false);
40645ffd83dbSDimitry Andric TSE.clear();
40655ffd83dbSDimitry Andric }
40665ffd83dbSDimitry Andric
40675ffd83dbSDimitry Andric TSE.addInstruction(Instr);
40685ffd83dbSDimitry Andric EndOffset = Instr.Offset + Instr.Size;
40695ffd83dbSDimitry Andric }
40705ffd83dbSDimitry Andric
4071bdd1243dSDimitry Andric const MachineFunction *MF = MBB->getParent();
407281ad6265SDimitry Andric // Multiple FP/SP updates in a loop cannot be described by CFI instructions.
4073bdd1243dSDimitry Andric TSE.emitCode(
4074bdd1243dSDimitry Andric InsertI, TFI, /*TryMergeSPUpdate = */
4075bdd1243dSDimitry Andric !MF->getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(*MF));
40765ffd83dbSDimitry Andric
40775ffd83dbSDimitry Andric return InsertI;
40785ffd83dbSDimitry Andric }
40795ffd83dbSDimitry Andric } // namespace
40805ffd83dbSDimitry Andric
processFunctionBeforeFrameIndicesReplaced(MachineFunction & MF,RegScavenger * RS=nullptr) const40815ffd83dbSDimitry Andric void AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced(
40825ffd83dbSDimitry Andric MachineFunction &MF, RegScavenger *RS = nullptr) const {
40835ffd83dbSDimitry Andric if (StackTaggingMergeSetTag)
40845ffd83dbSDimitry Andric for (auto &BB : MF)
40855ffd83dbSDimitry Andric for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();)
40865ffd83dbSDimitry Andric II = tryMergeAdjacentSTG(II, this, RS);
40875ffd83dbSDimitry Andric }
40885ffd83dbSDimitry Andric
40895ffd83dbSDimitry Andric /// For Win64 AArch64 EH, the offset to the Unwind object is from the SP
40905ffd83dbSDimitry Andric /// before the update. This is easily retrieved as it is exactly the offset
40915ffd83dbSDimitry Andric /// that is set in processFunctionBeforeFrameFinalized.
getFrameIndexReferencePreferSP(const MachineFunction & MF,int FI,Register & FrameReg,bool IgnoreSPUpdates) const4092e8d8bef9SDimitry Andric StackOffset AArch64FrameLowering::getFrameIndexReferencePreferSP(
40935ffd83dbSDimitry Andric const MachineFunction &MF, int FI, Register &FrameReg,
40940b57cec5SDimitry Andric bool IgnoreSPUpdates) const {
40950b57cec5SDimitry Andric const MachineFrameInfo &MFI = MF.getFrameInfo();
409662cfcf62SDimitry Andric if (IgnoreSPUpdates) {
40970b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
40980b57cec5SDimitry Andric << MFI.getObjectOffset(FI) << "\n");
40990b57cec5SDimitry Andric FrameReg = AArch64::SP;
4100e8d8bef9SDimitry Andric return StackOffset::getFixed(MFI.getObjectOffset(FI));
41010b57cec5SDimitry Andric }
41020b57cec5SDimitry Andric
4103349cc55cSDimitry Andric // Go to common code if we cannot provide sp + offset.
4104349cc55cSDimitry Andric if (MFI.hasVarSizedObjects() ||
4105349cc55cSDimitry Andric MF.getInfo<AArch64FunctionInfo>()->getStackSizeSVE() ||
4106349cc55cSDimitry Andric MF.getSubtarget().getRegisterInfo()->hasStackRealignment(MF))
410762cfcf62SDimitry Andric return getFrameIndexReference(MF, FI, FrameReg);
4108349cc55cSDimitry Andric
4109349cc55cSDimitry Andric FrameReg = AArch64::SP;
4110349cc55cSDimitry Andric return getStackOffset(MF, MFI.getObjectOffset(FI));
411162cfcf62SDimitry Andric }
411262cfcf62SDimitry Andric
41130b57cec5SDimitry Andric /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
41140b57cec5SDimitry Andric /// the parent's frame pointer
getWinEHParentFrameOffset(const MachineFunction & MF) const41150b57cec5SDimitry Andric unsigned AArch64FrameLowering::getWinEHParentFrameOffset(
41160b57cec5SDimitry Andric const MachineFunction &MF) const {
41170b57cec5SDimitry Andric return 0;
41180b57cec5SDimitry Andric }
41190b57cec5SDimitry Andric
41200b57cec5SDimitry Andric /// Funclets only need to account for space for the callee saved registers,
41210b57cec5SDimitry Andric /// as the locals are accounted for in the parent's stack frame.
getWinEHFuncletFrameSize(const MachineFunction & MF) const41220b57cec5SDimitry Andric unsigned AArch64FrameLowering::getWinEHFuncletFrameSize(
41230b57cec5SDimitry Andric const MachineFunction &MF) const {
41240b57cec5SDimitry Andric // This is the size of the pushed CSRs.
41250b57cec5SDimitry Andric unsigned CSSize =
41260b57cec5SDimitry Andric MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
41270b57cec5SDimitry Andric // This is the amount of stack a funclet needs to allocate.
41280b57cec5SDimitry Andric return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
41295ffd83dbSDimitry Andric getStackAlign());
41300b57cec5SDimitry Andric }
4131e8d8bef9SDimitry Andric
4132e8d8bef9SDimitry Andric namespace {
4133e8d8bef9SDimitry Andric struct FrameObject {
4134e8d8bef9SDimitry Andric bool IsValid = false;
4135e8d8bef9SDimitry Andric // Index of the object in MFI.
4136e8d8bef9SDimitry Andric int ObjectIndex = 0;
4137e8d8bef9SDimitry Andric // Group ID this object belongs to.
4138e8d8bef9SDimitry Andric int GroupIndex = -1;
4139e8d8bef9SDimitry Andric // This object should be placed first (closest to SP).
4140e8d8bef9SDimitry Andric bool ObjectFirst = false;
4141e8d8bef9SDimitry Andric // This object's group (which always contains the object with
4142e8d8bef9SDimitry Andric // ObjectFirst==true) should be placed first.
4143e8d8bef9SDimitry Andric bool GroupFirst = false;
4144e8d8bef9SDimitry Andric };
4145e8d8bef9SDimitry Andric
4146e8d8bef9SDimitry Andric class GroupBuilder {
4147e8d8bef9SDimitry Andric SmallVector<int, 8> CurrentMembers;
4148e8d8bef9SDimitry Andric int NextGroupIndex = 0;
4149e8d8bef9SDimitry Andric std::vector<FrameObject> &Objects;
4150e8d8bef9SDimitry Andric
4151e8d8bef9SDimitry Andric public:
GroupBuilder(std::vector<FrameObject> & Objects)4152e8d8bef9SDimitry Andric GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
AddMember(int Index)4153e8d8bef9SDimitry Andric void AddMember(int Index) { CurrentMembers.push_back(Index); }
EndCurrentGroup()4154e8d8bef9SDimitry Andric void EndCurrentGroup() {
4155e8d8bef9SDimitry Andric if (CurrentMembers.size() > 1) {
4156e8d8bef9SDimitry Andric // Create a new group with the current member list. This might remove them
4157e8d8bef9SDimitry Andric // from their pre-existing groups. That's OK, dealing with overlapping
4158e8d8bef9SDimitry Andric // groups is too hard and unlikely to make a difference.
4159e8d8bef9SDimitry Andric LLVM_DEBUG(dbgs() << "group:");
4160e8d8bef9SDimitry Andric for (int Index : CurrentMembers) {
4161e8d8bef9SDimitry Andric Objects[Index].GroupIndex = NextGroupIndex;
4162e8d8bef9SDimitry Andric LLVM_DEBUG(dbgs() << " " << Index);
4163e8d8bef9SDimitry Andric }
4164e8d8bef9SDimitry Andric LLVM_DEBUG(dbgs() << "\n");
4165e8d8bef9SDimitry Andric NextGroupIndex++;
4166e8d8bef9SDimitry Andric }
4167e8d8bef9SDimitry Andric CurrentMembers.clear();
4168e8d8bef9SDimitry Andric }
4169e8d8bef9SDimitry Andric };
4170e8d8bef9SDimitry Andric
FrameObjectCompare(const FrameObject & A,const FrameObject & B)4171e8d8bef9SDimitry Andric bool FrameObjectCompare(const FrameObject &A, const FrameObject &B) {
4172e8d8bef9SDimitry Andric // Objects at a lower index are closer to FP; objects at a higher index are
4173e8d8bef9SDimitry Andric // closer to SP.
4174e8d8bef9SDimitry Andric //
4175e8d8bef9SDimitry Andric // For consistency in our comparison, all invalid objects are placed
4176e8d8bef9SDimitry Andric // at the end. This also allows us to stop walking when we hit the
4177e8d8bef9SDimitry Andric // first invalid item after it's all sorted.
4178e8d8bef9SDimitry Andric //
4179e8d8bef9SDimitry Andric // The "first" object goes first (closest to SP), followed by the members of
4180e8d8bef9SDimitry Andric // the "first" group.
4181e8d8bef9SDimitry Andric //
4182e8d8bef9SDimitry Andric // The rest are sorted by the group index to keep the groups together.
4183e8d8bef9SDimitry Andric // Higher numbered groups are more likely to be around longer (i.e. untagged
4184e8d8bef9SDimitry Andric // in the function epilogue and not at some earlier point). Place them closer
4185e8d8bef9SDimitry Andric // to SP.
4186e8d8bef9SDimitry Andric //
4187e8d8bef9SDimitry Andric // If all else equal, sort by the object index to keep the objects in the
4188e8d8bef9SDimitry Andric // original order.
4189e8d8bef9SDimitry Andric return std::make_tuple(!A.IsValid, A.ObjectFirst, A.GroupFirst, A.GroupIndex,
4190e8d8bef9SDimitry Andric A.ObjectIndex) <
4191e8d8bef9SDimitry Andric std::make_tuple(!B.IsValid, B.ObjectFirst, B.GroupFirst, B.GroupIndex,
4192e8d8bef9SDimitry Andric B.ObjectIndex);
4193e8d8bef9SDimitry Andric }
4194e8d8bef9SDimitry Andric } // namespace
4195e8d8bef9SDimitry Andric
orderFrameObjects(const MachineFunction & MF,SmallVectorImpl<int> & ObjectsToAllocate) const4196e8d8bef9SDimitry Andric void AArch64FrameLowering::orderFrameObjects(
4197e8d8bef9SDimitry Andric const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
4198e8d8bef9SDimitry Andric if (!OrderFrameObjects || ObjectsToAllocate.empty())
4199e8d8bef9SDimitry Andric return;
4200e8d8bef9SDimitry Andric
4201e8d8bef9SDimitry Andric const MachineFrameInfo &MFI = MF.getFrameInfo();
4202e8d8bef9SDimitry Andric std::vector<FrameObject> FrameObjects(MFI.getObjectIndexEnd());
4203e8d8bef9SDimitry Andric for (auto &Obj : ObjectsToAllocate) {
4204e8d8bef9SDimitry Andric FrameObjects[Obj].IsValid = true;
4205e8d8bef9SDimitry Andric FrameObjects[Obj].ObjectIndex = Obj;
4206e8d8bef9SDimitry Andric }
4207e8d8bef9SDimitry Andric
4208e8d8bef9SDimitry Andric // Identify stack slots that are tagged at the same time.
4209e8d8bef9SDimitry Andric GroupBuilder GB(FrameObjects);
4210e8d8bef9SDimitry Andric for (auto &MBB : MF) {
4211e8d8bef9SDimitry Andric for (auto &MI : MBB) {
4212e8d8bef9SDimitry Andric if (MI.isDebugInstr())
4213e8d8bef9SDimitry Andric continue;
4214e8d8bef9SDimitry Andric int OpIndex;
4215e8d8bef9SDimitry Andric switch (MI.getOpcode()) {
4216e8d8bef9SDimitry Andric case AArch64::STGloop:
4217e8d8bef9SDimitry Andric case AArch64::STZGloop:
4218e8d8bef9SDimitry Andric OpIndex = 3;
4219e8d8bef9SDimitry Andric break;
422006c3fb27SDimitry Andric case AArch64::STGi:
422106c3fb27SDimitry Andric case AArch64::STZGi:
422206c3fb27SDimitry Andric case AArch64::ST2Gi:
422306c3fb27SDimitry Andric case AArch64::STZ2Gi:
4224e8d8bef9SDimitry Andric OpIndex = 1;
4225e8d8bef9SDimitry Andric break;
4226e8d8bef9SDimitry Andric default:
4227e8d8bef9SDimitry Andric OpIndex = -1;
4228e8d8bef9SDimitry Andric }
4229e8d8bef9SDimitry Andric
4230e8d8bef9SDimitry Andric int TaggedFI = -1;
4231e8d8bef9SDimitry Andric if (OpIndex >= 0) {
4232e8d8bef9SDimitry Andric const MachineOperand &MO = MI.getOperand(OpIndex);
4233e8d8bef9SDimitry Andric if (MO.isFI()) {
4234e8d8bef9SDimitry Andric int FI = MO.getIndex();
4235e8d8bef9SDimitry Andric if (FI >= 0 && FI < MFI.getObjectIndexEnd() &&
4236e8d8bef9SDimitry Andric FrameObjects[FI].IsValid)
4237e8d8bef9SDimitry Andric TaggedFI = FI;
4238e8d8bef9SDimitry Andric }
4239e8d8bef9SDimitry Andric }
4240e8d8bef9SDimitry Andric
4241e8d8bef9SDimitry Andric // If this is a stack tagging instruction for a slot that is not part of a
4242e8d8bef9SDimitry Andric // group yet, either start a new group or add it to the current one.
4243e8d8bef9SDimitry Andric if (TaggedFI >= 0)
4244e8d8bef9SDimitry Andric GB.AddMember(TaggedFI);
4245e8d8bef9SDimitry Andric else
4246e8d8bef9SDimitry Andric GB.EndCurrentGroup();
4247e8d8bef9SDimitry Andric }
4248e8d8bef9SDimitry Andric // Groups should never span multiple basic blocks.
4249e8d8bef9SDimitry Andric GB.EndCurrentGroup();
4250e8d8bef9SDimitry Andric }
4251e8d8bef9SDimitry Andric
4252e8d8bef9SDimitry Andric // If the function's tagged base pointer is pinned to a stack slot, we want to
4253e8d8bef9SDimitry Andric // put that slot first when possible. This will likely place it at SP + 0,
4254e8d8bef9SDimitry Andric // and save one instruction when generating the base pointer because IRG does
4255e8d8bef9SDimitry Andric // not allow an immediate offset.
4256e8d8bef9SDimitry Andric const AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
4257bdd1243dSDimitry Andric std::optional<int> TBPI = AFI.getTaggedBasePointerIndex();
4258e8d8bef9SDimitry Andric if (TBPI) {
4259e8d8bef9SDimitry Andric FrameObjects[*TBPI].ObjectFirst = true;
4260e8d8bef9SDimitry Andric FrameObjects[*TBPI].GroupFirst = true;
4261e8d8bef9SDimitry Andric int FirstGroupIndex = FrameObjects[*TBPI].GroupIndex;
4262e8d8bef9SDimitry Andric if (FirstGroupIndex >= 0)
4263e8d8bef9SDimitry Andric for (FrameObject &Object : FrameObjects)
4264e8d8bef9SDimitry Andric if (Object.GroupIndex == FirstGroupIndex)
4265e8d8bef9SDimitry Andric Object.GroupFirst = true;
4266e8d8bef9SDimitry Andric }
4267e8d8bef9SDimitry Andric
4268e8d8bef9SDimitry Andric llvm::stable_sort(FrameObjects, FrameObjectCompare);
4269e8d8bef9SDimitry Andric
4270e8d8bef9SDimitry Andric int i = 0;
4271e8d8bef9SDimitry Andric for (auto &Obj : FrameObjects) {
4272e8d8bef9SDimitry Andric // All invalid items are sorted at the end, so it's safe to stop.
4273e8d8bef9SDimitry Andric if (!Obj.IsValid)
4274e8d8bef9SDimitry Andric break;
4275e8d8bef9SDimitry Andric ObjectsToAllocate[i++] = Obj.ObjectIndex;
4276e8d8bef9SDimitry Andric }
4277e8d8bef9SDimitry Andric
4278e8d8bef9SDimitry Andric LLVM_DEBUG(dbgs() << "Final frame order:\n"; for (auto &Obj
4279e8d8bef9SDimitry Andric : FrameObjects) {
4280e8d8bef9SDimitry Andric if (!Obj.IsValid)
4281e8d8bef9SDimitry Andric break;
4282e8d8bef9SDimitry Andric dbgs() << " " << Obj.ObjectIndex << ": group " << Obj.GroupIndex;
4283e8d8bef9SDimitry Andric if (Obj.ObjectFirst)
4284e8d8bef9SDimitry Andric dbgs() << ", first";
4285e8d8bef9SDimitry Andric if (Obj.GroupFirst)
4286e8d8bef9SDimitry Andric dbgs() << ", group-first";
4287e8d8bef9SDimitry Andric dbgs() << "\n";
4288e8d8bef9SDimitry Andric });
4289e8d8bef9SDimitry Andric }
42905f757f3fSDimitry Andric
42915f757f3fSDimitry Andric /// Emit a loop to decrement SP until it is equal to TargetReg, with probes at
42925f757f3fSDimitry Andric /// least every ProbeSize bytes. Returns an iterator of the first instruction
42935f757f3fSDimitry Andric /// after the loop. The difference between SP and TargetReg must be an exact
42945f757f3fSDimitry Andric /// multiple of ProbeSize.
42955f757f3fSDimitry Andric MachineBasicBlock::iterator
inlineStackProbeLoopExactMultiple(MachineBasicBlock::iterator MBBI,int64_t ProbeSize,Register TargetReg) const42965f757f3fSDimitry Andric AArch64FrameLowering::inlineStackProbeLoopExactMultiple(
42975f757f3fSDimitry Andric MachineBasicBlock::iterator MBBI, int64_t ProbeSize,
42985f757f3fSDimitry Andric Register TargetReg) const {
42995f757f3fSDimitry Andric MachineBasicBlock &MBB = *MBBI->getParent();
43005f757f3fSDimitry Andric MachineFunction &MF = *MBB.getParent();
43015f757f3fSDimitry Andric const AArch64InstrInfo *TII =
43025f757f3fSDimitry Andric MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
43035f757f3fSDimitry Andric DebugLoc DL = MBB.findDebugLoc(MBBI);
43045f757f3fSDimitry Andric
43055f757f3fSDimitry Andric MachineFunction::iterator MBBInsertPoint = std::next(MBB.getIterator());
43065f757f3fSDimitry Andric MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(MBB.getBasicBlock());
43075f757f3fSDimitry Andric MF.insert(MBBInsertPoint, LoopMBB);
43085f757f3fSDimitry Andric MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(MBB.getBasicBlock());
43095f757f3fSDimitry Andric MF.insert(MBBInsertPoint, ExitMBB);
43105f757f3fSDimitry Andric
43115f757f3fSDimitry Andric // SUB SP, SP, #ProbeSize (or equivalent if ProbeSize is not encodable
43125f757f3fSDimitry Andric // in SUB).
43135f757f3fSDimitry Andric emitFrameOffset(*LoopMBB, LoopMBB->end(), DL, AArch64::SP, AArch64::SP,
43145f757f3fSDimitry Andric StackOffset::getFixed(-ProbeSize), TII,
43155f757f3fSDimitry Andric MachineInstr::FrameSetup);
43165f757f3fSDimitry Andric // STR XZR, [SP]
43175f757f3fSDimitry Andric BuildMI(*LoopMBB, LoopMBB->end(), DL, TII->get(AArch64::STRXui))
43185f757f3fSDimitry Andric .addReg(AArch64::XZR)
43195f757f3fSDimitry Andric .addReg(AArch64::SP)
43205f757f3fSDimitry Andric .addImm(0)
43215f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
43225f757f3fSDimitry Andric // CMP SP, TargetReg
43235f757f3fSDimitry Andric BuildMI(*LoopMBB, LoopMBB->end(), DL, TII->get(AArch64::SUBSXrx64),
43245f757f3fSDimitry Andric AArch64::XZR)
43255f757f3fSDimitry Andric .addReg(AArch64::SP)
43265f757f3fSDimitry Andric .addReg(TargetReg)
43275f757f3fSDimitry Andric .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0))
43285f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
43295f757f3fSDimitry Andric // B.CC Loop
43305f757f3fSDimitry Andric BuildMI(*LoopMBB, LoopMBB->end(), DL, TII->get(AArch64::Bcc))
43315f757f3fSDimitry Andric .addImm(AArch64CC::NE)
43325f757f3fSDimitry Andric .addMBB(LoopMBB)
43335f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
43345f757f3fSDimitry Andric
43355f757f3fSDimitry Andric LoopMBB->addSuccessor(ExitMBB);
43365f757f3fSDimitry Andric LoopMBB->addSuccessor(LoopMBB);
43375f757f3fSDimitry Andric // Synthesize the exit MBB.
43385f757f3fSDimitry Andric ExitMBB->splice(ExitMBB->end(), &MBB, MBBI, MBB.end());
43395f757f3fSDimitry Andric ExitMBB->transferSuccessorsAndUpdatePHIs(&MBB);
43405f757f3fSDimitry Andric MBB.addSuccessor(LoopMBB);
43415f757f3fSDimitry Andric // Update liveins.
4342b3edf446SDimitry Andric bool anyChange = false;
4343b3edf446SDimitry Andric do {
4344b3edf446SDimitry Andric anyChange = recomputeLiveIns(*ExitMBB) || recomputeLiveIns(*LoopMBB);
4345b3edf446SDimitry Andric } while (anyChange);
43465f757f3fSDimitry Andric
43475f757f3fSDimitry Andric return ExitMBB->begin();
43485f757f3fSDimitry Andric }
43495f757f3fSDimitry Andric
inlineStackProbeFixed(MachineBasicBlock::iterator MBBI,Register ScratchReg,int64_t FrameSize,StackOffset CFAOffset) const43505f757f3fSDimitry Andric void AArch64FrameLowering::inlineStackProbeFixed(
43515f757f3fSDimitry Andric MachineBasicBlock::iterator MBBI, Register ScratchReg, int64_t FrameSize,
43525f757f3fSDimitry Andric StackOffset CFAOffset) const {
43535f757f3fSDimitry Andric MachineBasicBlock *MBB = MBBI->getParent();
43545f757f3fSDimitry Andric MachineFunction &MF = *MBB->getParent();
43555f757f3fSDimitry Andric const AArch64InstrInfo *TII =
43565f757f3fSDimitry Andric MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
43575f757f3fSDimitry Andric AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
43585f757f3fSDimitry Andric bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
43595f757f3fSDimitry Andric bool HasFP = hasFP(MF);
43605f757f3fSDimitry Andric
43615f757f3fSDimitry Andric DebugLoc DL;
43625f757f3fSDimitry Andric int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize();
43635f757f3fSDimitry Andric int64_t NumBlocks = FrameSize / ProbeSize;
43645f757f3fSDimitry Andric int64_t ResidualSize = FrameSize % ProbeSize;
43655f757f3fSDimitry Andric
43665f757f3fSDimitry Andric LLVM_DEBUG(dbgs() << "Stack probing: total " << FrameSize << " bytes, "
43675f757f3fSDimitry Andric << NumBlocks << " blocks of " << ProbeSize
43685f757f3fSDimitry Andric << " bytes, plus " << ResidualSize << " bytes\n");
43695f757f3fSDimitry Andric
43705f757f3fSDimitry Andric // Decrement SP by NumBlock * ProbeSize bytes, with either unrolled or
43715f757f3fSDimitry Andric // ordinary loop.
43725f757f3fSDimitry Andric if (NumBlocks <= AArch64::StackProbeMaxLoopUnroll) {
43735f757f3fSDimitry Andric for (int i = 0; i < NumBlocks; ++i) {
43745f757f3fSDimitry Andric // SUB SP, SP, #ProbeSize (or equivalent if ProbeSize is not
43755f757f3fSDimitry Andric // encodable in a SUB).
43765f757f3fSDimitry Andric emitFrameOffset(*MBB, MBBI, DL, AArch64::SP, AArch64::SP,
43775f757f3fSDimitry Andric StackOffset::getFixed(-ProbeSize), TII,
43785f757f3fSDimitry Andric MachineInstr::FrameSetup, false, false, nullptr,
43795f757f3fSDimitry Andric EmitAsyncCFI && !HasFP, CFAOffset);
43805f757f3fSDimitry Andric CFAOffset += StackOffset::getFixed(ProbeSize);
43815f757f3fSDimitry Andric // STR XZR, [SP]
43825f757f3fSDimitry Andric BuildMI(*MBB, MBBI, DL, TII->get(AArch64::STRXui))
43835f757f3fSDimitry Andric .addReg(AArch64::XZR)
43845f757f3fSDimitry Andric .addReg(AArch64::SP)
43855f757f3fSDimitry Andric .addImm(0)
43865f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
43875f757f3fSDimitry Andric }
43885f757f3fSDimitry Andric } else if (NumBlocks != 0) {
43895f757f3fSDimitry Andric // SUB ScratchReg, SP, #FrameSize (or equivalent if FrameSize is not
43905f757f3fSDimitry Andric // encodable in ADD). ScrathReg may temporarily become the CFA register.
43915f757f3fSDimitry Andric emitFrameOffset(*MBB, MBBI, DL, ScratchReg, AArch64::SP,
43925f757f3fSDimitry Andric StackOffset::getFixed(-ProbeSize * NumBlocks), TII,
43935f757f3fSDimitry Andric MachineInstr::FrameSetup, false, false, nullptr,
43945f757f3fSDimitry Andric EmitAsyncCFI && !HasFP, CFAOffset);
43955f757f3fSDimitry Andric CFAOffset += StackOffset::getFixed(ProbeSize * NumBlocks);
43965f757f3fSDimitry Andric MBBI = inlineStackProbeLoopExactMultiple(MBBI, ProbeSize, ScratchReg);
43975f757f3fSDimitry Andric MBB = MBBI->getParent();
43985f757f3fSDimitry Andric if (EmitAsyncCFI && !HasFP) {
43995f757f3fSDimitry Andric // Set the CFA register back to SP.
44005f757f3fSDimitry Andric const AArch64RegisterInfo &RegInfo =
44015f757f3fSDimitry Andric *MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
44025f757f3fSDimitry Andric unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
44035f757f3fSDimitry Andric unsigned CFIIndex =
44045f757f3fSDimitry Andric MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, Reg));
44055f757f3fSDimitry Andric BuildMI(*MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
44065f757f3fSDimitry Andric .addCFIIndex(CFIIndex)
44075f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
44085f757f3fSDimitry Andric }
44095f757f3fSDimitry Andric }
44105f757f3fSDimitry Andric
44115f757f3fSDimitry Andric if (ResidualSize != 0) {
44125f757f3fSDimitry Andric // SUB SP, SP, #ResidualSize (or equivalent if ResidualSize is not encodable
44135f757f3fSDimitry Andric // in SUB).
44145f757f3fSDimitry Andric emitFrameOffset(*MBB, MBBI, DL, AArch64::SP, AArch64::SP,
44155f757f3fSDimitry Andric StackOffset::getFixed(-ResidualSize), TII,
44165f757f3fSDimitry Andric MachineInstr::FrameSetup, false, false, nullptr,
44175f757f3fSDimitry Andric EmitAsyncCFI && !HasFP, CFAOffset);
44185f757f3fSDimitry Andric if (ResidualSize > AArch64::StackProbeMaxUnprobedStack) {
44195f757f3fSDimitry Andric // STR XZR, [SP]
44205f757f3fSDimitry Andric BuildMI(*MBB, MBBI, DL, TII->get(AArch64::STRXui))
44215f757f3fSDimitry Andric .addReg(AArch64::XZR)
44225f757f3fSDimitry Andric .addReg(AArch64::SP)
44235f757f3fSDimitry Andric .addImm(0)
44245f757f3fSDimitry Andric .setMIFlags(MachineInstr::FrameSetup);
44255f757f3fSDimitry Andric }
44265f757f3fSDimitry Andric }
44275f757f3fSDimitry Andric }
44285f757f3fSDimitry Andric
inlineStackProbe(MachineFunction & MF,MachineBasicBlock & MBB) const44295f757f3fSDimitry Andric void AArch64FrameLowering::inlineStackProbe(MachineFunction &MF,
44305f757f3fSDimitry Andric MachineBasicBlock &MBB) const {
44315f757f3fSDimitry Andric // Get the instructions that need to be replaced. We emit at most two of
44325f757f3fSDimitry Andric // these. Remember them in order to avoid complications coming from the need
44335f757f3fSDimitry Andric // to traverse the block while potentially creating more blocks.
44345f757f3fSDimitry Andric SmallVector<MachineInstr *, 4> ToReplace;
44355f757f3fSDimitry Andric for (MachineInstr &MI : MBB)
44365f757f3fSDimitry Andric if (MI.getOpcode() == AArch64::PROBED_STACKALLOC ||
44375f757f3fSDimitry Andric MI.getOpcode() == AArch64::PROBED_STACKALLOC_VAR)
44385f757f3fSDimitry Andric ToReplace.push_back(&MI);
44395f757f3fSDimitry Andric
44405f757f3fSDimitry Andric for (MachineInstr *MI : ToReplace) {
44415f757f3fSDimitry Andric if (MI->getOpcode() == AArch64::PROBED_STACKALLOC) {
44425f757f3fSDimitry Andric Register ScratchReg = MI->getOperand(0).getReg();
44435f757f3fSDimitry Andric int64_t FrameSize = MI->getOperand(1).getImm();
44445f757f3fSDimitry Andric StackOffset CFAOffset = StackOffset::get(MI->getOperand(2).getImm(),
44455f757f3fSDimitry Andric MI->getOperand(3).getImm());
44465f757f3fSDimitry Andric inlineStackProbeFixed(MI->getIterator(), ScratchReg, FrameSize,
44475f757f3fSDimitry Andric CFAOffset);
44485f757f3fSDimitry Andric } else {
44495f757f3fSDimitry Andric assert(MI->getOpcode() == AArch64::PROBED_STACKALLOC_VAR &&
44505f757f3fSDimitry Andric "Stack probe pseudo-instruction expected");
44515f757f3fSDimitry Andric const AArch64InstrInfo *TII =
44525f757f3fSDimitry Andric MI->getMF()->getSubtarget<AArch64Subtarget>().getInstrInfo();
44535f757f3fSDimitry Andric Register TargetReg = MI->getOperand(0).getReg();
44545f757f3fSDimitry Andric (void)TII->probedStackAlloc(MI->getIterator(), TargetReg, true);
44555f757f3fSDimitry Andric }
44565f757f3fSDimitry Andric MI->eraseFromParent();
44575f757f3fSDimitry Andric }
44585f757f3fSDimitry Andric }
4459