1 //===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of the TargetRegisterInfo
10 // class.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64RegisterInfo.h"
15 #include "AArch64FrameLowering.h"
16 #include "AArch64InstrInfo.h"
17 #include "AArch64MachineFunctionInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "MCTargetDesc/AArch64AddressingModes.h"
20 #include "llvm/ADT/BitVector.h"
21 #include "llvm/ADT/Triple.h"
22 #include "llvm/BinaryFormat/Dwarf.h"
23 #include "llvm/CodeGen/MachineFrameInfo.h"
24 #include "llvm/CodeGen/MachineInstrBuilder.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/RegisterScavenging.h"
27 #include "llvm/CodeGen/TargetFrameLowering.h"
28 #include "llvm/IR/DebugInfoMetadata.h"
29 #include "llvm/IR/DiagnosticInfo.h"
30 #include "llvm/IR/Function.h"
31 #include "llvm/Support/raw_ostream.h"
32 #include "llvm/Target/TargetOptions.h"
33 
34 using namespace llvm;
35 
36 #define GET_CC_REGISTER_LISTS
37 #include "AArch64GenCallingConv.inc"
38 #define GET_REGINFO_TARGET_DESC
39 #include "AArch64GenRegisterInfo.inc"
40 
41 AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT)
42     : AArch64GenRegisterInfo(AArch64::LR), TT(TT) {
43   AArch64_MC::initLLVMToCVRegMapping(this);
44 }
45 
46 /// Return whether the register needs a CFI entry. Not all unwinders may know
47 /// about SVE registers, so we assume the lowest common denominator, i.e. the
48 /// callee-saves required by the base ABI. For the SVE registers z8-z15 only the
49 /// lower 64-bits (d8-d15) need to be saved. The lower 64-bits subreg is
50 /// returned in \p RegToUseForCFI.
51 bool AArch64RegisterInfo::regNeedsCFI(unsigned Reg,
52                                       unsigned &RegToUseForCFI) const {
53   if (AArch64::PPRRegClass.contains(Reg))
54     return false;
55 
56   if (AArch64::ZPRRegClass.contains(Reg)) {
57     RegToUseForCFI = getSubReg(Reg, AArch64::dsub);
58     for (int I = 0; CSR_AArch64_AAPCS_SaveList[I]; ++I) {
59       if (CSR_AArch64_AAPCS_SaveList[I] == RegToUseForCFI)
60         return true;
61     }
62     return false;
63   }
64 
65   RegToUseForCFI = Reg;
66   return true;
67 }
68 
69 const MCPhysReg *
70 AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
71   assert(MF && "Invalid MachineFunction pointer.");
72 
73   if (MF->getFunction().getCallingConv() == CallingConv::GHC)
74     // GHC set of callee saved regs is empty as all those regs are
75     // used for passing STG regs around
76     return CSR_AArch64_NoRegs_SaveList;
77   if (MF->getFunction().getCallingConv() == CallingConv::AnyReg)
78     return CSR_AArch64_AllRegs_SaveList;
79 
80   // Darwin has its own CSR_AArch64_AAPCS_SaveList, which means most CSR save
81   // lists depending on that will need to have their Darwin variant as well.
82   if (MF->getSubtarget<AArch64Subtarget>().isTargetDarwin())
83     return getDarwinCalleeSavedRegs(MF);
84 
85   if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check)
86     return CSR_Win_AArch64_CFGuard_Check_SaveList;
87   if (MF->getSubtarget<AArch64Subtarget>().isTargetWindows())
88     return CSR_Win_AArch64_AAPCS_SaveList;
89   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall)
90     return CSR_AArch64_AAVPCS_SaveList;
91   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall)
92     return CSR_AArch64_SVE_AAPCS_SaveList;
93   if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering()
94           ->supportSwiftError() &&
95       MF->getFunction().getAttributes().hasAttrSomewhere(
96           Attribute::SwiftError))
97     return CSR_AArch64_AAPCS_SwiftError_SaveList;
98   if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail)
99     return CSR_AArch64_AAPCS_SwiftTail_SaveList;
100   if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost)
101     return CSR_AArch64_RT_MostRegs_SaveList;
102   if (MF->getFunction().getCallingConv() == CallingConv::Win64)
103     // This is for OSes other than Windows; Windows is a separate case further
104     // above.
105     return CSR_AArch64_AAPCS_X18_SaveList;
106   if (MF->getInfo<AArch64FunctionInfo>()->isSVECC())
107     return CSR_AArch64_SVE_AAPCS_SaveList;
108   return CSR_AArch64_AAPCS_SaveList;
109 }
110 
111 const MCPhysReg *
112 AArch64RegisterInfo::getDarwinCalleeSavedRegs(const MachineFunction *MF) const {
113   assert(MF && "Invalid MachineFunction pointer.");
114   assert(MF->getSubtarget<AArch64Subtarget>().isTargetDarwin() &&
115          "Invalid subtarget for getDarwinCalleeSavedRegs");
116 
117   if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check)
118     report_fatal_error(
119         "Calling convention CFGuard_Check is unsupported on Darwin.");
120   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall)
121     return CSR_Darwin_AArch64_AAVPCS_SaveList;
122   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall)
123     report_fatal_error(
124         "Calling convention SVE_VectorCall is unsupported on Darwin.");
125   if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS)
126     return MF->getInfo<AArch64FunctionInfo>()->isSplitCSR()
127                ? CSR_Darwin_AArch64_CXX_TLS_PE_SaveList
128                : CSR_Darwin_AArch64_CXX_TLS_SaveList;
129   if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering()
130           ->supportSwiftError() &&
131       MF->getFunction().getAttributes().hasAttrSomewhere(
132           Attribute::SwiftError))
133     return CSR_Darwin_AArch64_AAPCS_SwiftError_SaveList;
134   if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail)
135     return CSR_Darwin_AArch64_AAPCS_SwiftTail_SaveList;
136   if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost)
137     return CSR_Darwin_AArch64_RT_MostRegs_SaveList;
138   return CSR_Darwin_AArch64_AAPCS_SaveList;
139 }
140 
141 const MCPhysReg *AArch64RegisterInfo::getCalleeSavedRegsViaCopy(
142     const MachineFunction *MF) const {
143   assert(MF && "Invalid MachineFunction pointer.");
144   if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
145       MF->getInfo<AArch64FunctionInfo>()->isSplitCSR())
146     return CSR_Darwin_AArch64_CXX_TLS_ViaCopy_SaveList;
147   return nullptr;
148 }
149 
150 void AArch64RegisterInfo::UpdateCustomCalleeSavedRegs(
151     MachineFunction &MF) const {
152   const MCPhysReg *CSRs = getCalleeSavedRegs(&MF);
153   SmallVector<MCPhysReg, 32> UpdatedCSRs;
154   for (const MCPhysReg *I = CSRs; *I; ++I)
155     UpdatedCSRs.push_back(*I);
156 
157   for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {
158     if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {
159       UpdatedCSRs.push_back(AArch64::GPR64commonRegClass.getRegister(i));
160     }
161   }
162   // Register lists are zero-terminated.
163   UpdatedCSRs.push_back(0);
164   MF.getRegInfo().setCalleeSavedRegs(UpdatedCSRs);
165 }
166 
167 const TargetRegisterClass *
168 AArch64RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC,
169                                        unsigned Idx) const {
170   // edge case for GPR/FPR register classes
171   if (RC == &AArch64::GPR32allRegClass && Idx == AArch64::hsub)
172     return &AArch64::FPR32RegClass;
173   else if (RC == &AArch64::GPR64allRegClass && Idx == AArch64::hsub)
174     return &AArch64::FPR64RegClass;
175 
176   // Forward to TableGen's default version.
177   return AArch64GenRegisterInfo::getSubClassWithSubReg(RC, Idx);
178 }
179 
180 const uint32_t *
181 AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF,
182                                                 CallingConv::ID CC) const {
183   assert(MF.getSubtarget<AArch64Subtarget>().isTargetDarwin() &&
184          "Invalid subtarget for getDarwinCallPreservedMask");
185 
186   if (CC == CallingConv::CXX_FAST_TLS)
187     return CSR_Darwin_AArch64_CXX_TLS_RegMask;
188   if (CC == CallingConv::AArch64_VectorCall)
189     return CSR_Darwin_AArch64_AAVPCS_RegMask;
190   if (CC == CallingConv::AArch64_SVE_VectorCall)
191     report_fatal_error(
192         "Calling convention SVE_VectorCall is unsupported on Darwin.");
193   if (CC == CallingConv::CFGuard_Check)
194     report_fatal_error(
195         "Calling convention CFGuard_Check is unsupported on Darwin.");
196   if (MF.getSubtarget<AArch64Subtarget>()
197           .getTargetLowering()
198           ->supportSwiftError() &&
199       MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError))
200     return CSR_Darwin_AArch64_AAPCS_SwiftError_RegMask;
201   if (CC == CallingConv::SwiftTail)
202     return CSR_Darwin_AArch64_AAPCS_SwiftTail_RegMask;
203   if (CC == CallingConv::PreserveMost)
204     return CSR_Darwin_AArch64_RT_MostRegs_RegMask;
205   return CSR_Darwin_AArch64_AAPCS_RegMask;
206 }
207 
208 const uint32_t *
209 AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
210                                           CallingConv::ID CC) const {
211   bool SCS = MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack);
212   if (CC == CallingConv::GHC)
213     // This is academic because all GHC calls are (supposed to be) tail calls
214     return SCS ? CSR_AArch64_NoRegs_SCS_RegMask : CSR_AArch64_NoRegs_RegMask;
215   if (CC == CallingConv::AnyReg)
216     return SCS ? CSR_AArch64_AllRegs_SCS_RegMask : CSR_AArch64_AllRegs_RegMask;
217 
218   // All the following calling conventions are handled differently on Darwin.
219   if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) {
220     if (SCS)
221       report_fatal_error("ShadowCallStack attribute not supported on Darwin.");
222     return getDarwinCallPreservedMask(MF, CC);
223   }
224 
225   if (CC == CallingConv::AArch64_VectorCall)
226     return SCS ? CSR_AArch64_AAVPCS_SCS_RegMask : CSR_AArch64_AAVPCS_RegMask;
227   if (CC == CallingConv::AArch64_SVE_VectorCall)
228     return SCS ? CSR_AArch64_SVE_AAPCS_SCS_RegMask
229                : CSR_AArch64_SVE_AAPCS_RegMask;
230   if (CC == CallingConv::CFGuard_Check)
231     return CSR_Win_AArch64_CFGuard_Check_RegMask;
232   if (MF.getSubtarget<AArch64Subtarget>().getTargetLowering()
233           ->supportSwiftError() &&
234       MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError))
235     return SCS ? CSR_AArch64_AAPCS_SwiftError_SCS_RegMask
236                : CSR_AArch64_AAPCS_SwiftError_RegMask;
237   if (CC == CallingConv::SwiftTail) {
238     if (SCS)
239       report_fatal_error("ShadowCallStack attribute not supported with swifttail");
240     return CSR_AArch64_AAPCS_SwiftTail_RegMask;
241   }
242   if (CC == CallingConv::PreserveMost)
243     return SCS ? CSR_AArch64_RT_MostRegs_SCS_RegMask
244                : CSR_AArch64_RT_MostRegs_RegMask;
245   else
246     return SCS ? CSR_AArch64_AAPCS_SCS_RegMask : CSR_AArch64_AAPCS_RegMask;
247 }
248 
249 const uint32_t *AArch64RegisterInfo::getCustomEHPadPreservedMask(
250     const MachineFunction &MF) const {
251   if (MF.getSubtarget<AArch64Subtarget>().isTargetLinux())
252     return CSR_AArch64_AAPCS_RegMask;
253 
254   return nullptr;
255 }
256 
257 const uint32_t *AArch64RegisterInfo::getTLSCallPreservedMask() const {
258   if (TT.isOSDarwin())
259     return CSR_Darwin_AArch64_TLS_RegMask;
260 
261   assert(TT.isOSBinFormatELF() && "Invalid target");
262   return CSR_AArch64_TLS_ELF_RegMask;
263 }
264 
265 void AArch64RegisterInfo::UpdateCustomCallPreservedMask(MachineFunction &MF,
266                                                  const uint32_t **Mask) const {
267   uint32_t *UpdatedMask = MF.allocateRegMask();
268   unsigned RegMaskSize = MachineOperand::getRegMaskSize(getNumRegs());
269   memcpy(UpdatedMask, *Mask, sizeof(UpdatedMask[0]) * RegMaskSize);
270 
271   for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {
272     if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {
273       for (MCSubRegIterator SubReg(AArch64::GPR64commonRegClass.getRegister(i),
274                                    this, true);
275            SubReg.isValid(); ++SubReg) {
276         // See TargetRegisterInfo::getCallPreservedMask for how to interpret the
277         // register mask.
278         UpdatedMask[*SubReg / 32] |= 1u << (*SubReg % 32);
279       }
280     }
281   }
282   *Mask = UpdatedMask;
283 }
284 
285 const uint32_t *AArch64RegisterInfo::getNoPreservedMask() const {
286   return CSR_AArch64_NoRegs_RegMask;
287 }
288 
289 const uint32_t *
290 AArch64RegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF,
291                                                 CallingConv::ID CC) const {
292   // This should return a register mask that is the same as that returned by
293   // getCallPreservedMask but that additionally preserves the register used for
294   // the first i64 argument (which must also be the register used to return a
295   // single i64 return value)
296   //
297   // In case that the calling convention does not use the same register for
298   // both, the function should return NULL (does not currently apply)
299   assert(CC != CallingConv::GHC && "should not be GHC calling convention.");
300   if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin())
301     return CSR_Darwin_AArch64_AAPCS_ThisReturn_RegMask;
302   return CSR_AArch64_AAPCS_ThisReturn_RegMask;
303 }
304 
305 const uint32_t *AArch64RegisterInfo::getWindowsStackProbePreservedMask() const {
306   return CSR_AArch64_StackProbe_Windows_RegMask;
307 }
308 
309 BitVector
310 AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
311   const AArch64FrameLowering *TFI = getFrameLowering(MF);
312 
313   // FIXME: avoid re-calculating this every time.
314   BitVector Reserved(getNumRegs());
315   markSuperRegs(Reserved, AArch64::WSP);
316   markSuperRegs(Reserved, AArch64::WZR);
317 
318   if (TFI->hasFP(MF) || TT.isOSDarwin())
319     markSuperRegs(Reserved, AArch64::W29);
320 
321   for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
322     if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
323       markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));
324   }
325 
326   if (hasBasePointer(MF))
327     markSuperRegs(Reserved, AArch64::W19);
328 
329   // SLH uses register W16/X16 as the taint register.
330   if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
331     markSuperRegs(Reserved, AArch64::W16);
332 
333   // SME tiles are not allocatable.
334   if (MF.getSubtarget<AArch64Subtarget>().hasSME()) {
335     for (MCSubRegIterator SubReg(AArch64::ZA, this, /*self=*/true);
336          SubReg.isValid(); ++SubReg)
337       Reserved.set(*SubReg);
338   }
339 
340   assert(checkAllSuperRegsMarked(Reserved));
341   return Reserved;
342 }
343 
344 bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,
345                                         MCRegister Reg) const {
346   return getReservedRegs(MF)[Reg];
347 }
348 
349 bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const {
350   return llvm::any_of(*AArch64::GPR64argRegClass.MC, [this, &MF](MCPhysReg r) {
351     return isReservedReg(MF, r);
352   });
353 }
354 
355 void AArch64RegisterInfo::emitReservedArgRegCallError(
356     const MachineFunction &MF) const {
357   const Function &F = MF.getFunction();
358   F.getContext().diagnose(DiagnosticInfoUnsupported{F, ("AArch64 doesn't support"
359     " function calls if any of the argument registers is reserved.")});
360 }
361 
362 bool AArch64RegisterInfo::isAsmClobberable(const MachineFunction &MF,
363                                           MCRegister PhysReg) const {
364   return !isReservedReg(MF, PhysReg);
365 }
366 
367 bool AArch64RegisterInfo::isConstantPhysReg(MCRegister PhysReg) const {
368   return PhysReg == AArch64::WZR || PhysReg == AArch64::XZR;
369 }
370 
371 const TargetRegisterClass *
372 AArch64RegisterInfo::getPointerRegClass(const MachineFunction &MF,
373                                       unsigned Kind) const {
374   return &AArch64::GPR64spRegClass;
375 }
376 
377 const TargetRegisterClass *
378 AArch64RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {
379   if (RC == &AArch64::CCRRegClass)
380     return &AArch64::GPR64RegClass; // Only MSR & MRS copy NZCV.
381   return RC;
382 }
383 
384 unsigned AArch64RegisterInfo::getBaseRegister() const { return AArch64::X19; }
385 
386 bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
387   const MachineFrameInfo &MFI = MF.getFrameInfo();
388 
389   // In the presence of variable sized objects or funclets, if the fixed stack
390   // size is large enough that referencing from the FP won't result in things
391   // being in range relatively often, we can use a base pointer to allow access
392   // from the other direction like the SP normally works.
393   //
394   // Furthermore, if both variable sized objects are present, and the
395   // stack needs to be dynamically re-aligned, the base pointer is the only
396   // reliable way to reference the locals.
397   if (MFI.hasVarSizedObjects() || MF.hasEHFunclets()) {
398     if (hasStackRealignment(MF))
399       return true;
400 
401     if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
402       const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
403       // Frames that have variable sized objects and scalable SVE objects,
404       // should always use a basepointer.
405       if (!AFI->hasCalculatedStackSizeSVE() || AFI->getStackSizeSVE())
406         return true;
407     }
408 
409     // Conservatively estimate whether the negative offset from the frame
410     // pointer will be sufficient to reach. If a function has a smallish
411     // frame, it's less likely to have lots of spills and callee saved
412     // space, so it's all more likely to be within range of the frame pointer.
413     // If it's wrong, we'll materialize the constant and still get to the
414     // object; it's just suboptimal. Negative offsets use the unscaled
415     // load/store instructions, which have a 9-bit signed immediate.
416     return MFI.getLocalFrameSize() >= 256;
417   }
418 
419   return false;
420 }
421 
422 bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF,
423                                              MCRegister Reg) const {
424   CallingConv::ID CC = MF.getFunction().getCallingConv();
425   const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
426   bool IsVarArg = STI.isCallingConvWin64(MF.getFunction().getCallingConv());
427 
428   auto HasReg = [](ArrayRef<MCRegister> RegList, MCRegister Reg) {
429     return llvm::any_of(RegList,
430                         [Reg](const MCRegister R) { return R == Reg; });
431   };
432 
433   switch (CC) {
434   default:
435     report_fatal_error("Unsupported calling convention.");
436   case CallingConv::WebKit_JS:
437     return HasReg(CC_AArch64_WebKit_JS_ArgRegs, Reg);
438   case CallingConv::GHC:
439     return HasReg(CC_AArch64_GHC_ArgRegs, Reg);
440   case CallingConv::C:
441   case CallingConv::Fast:
442   case CallingConv::PreserveMost:
443   case CallingConv::CXX_FAST_TLS:
444   case CallingConv::Swift:
445   case CallingConv::SwiftTail:
446   case CallingConv::Tail:
447     if (STI.isTargetWindows() && IsVarArg)
448       return HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);
449     if (!STI.isTargetDarwin()) {
450       switch (CC) {
451       default:
452         return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
453       case CallingConv::Swift:
454       case CallingConv::SwiftTail:
455         return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg) ||
456                HasReg(CC_AArch64_AAPCS_Swift_ArgRegs, Reg);
457       }
458     }
459     if (!IsVarArg) {
460       switch (CC) {
461       default:
462         return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg);
463       case CallingConv::Swift:
464       case CallingConv::SwiftTail:
465         return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg) ||
466                HasReg(CC_AArch64_DarwinPCS_Swift_ArgRegs, Reg);
467       }
468     }
469     if (STI.isTargetILP32())
470       return HasReg(CC_AArch64_DarwinPCS_ILP32_VarArg_ArgRegs, Reg);
471     return HasReg(CC_AArch64_DarwinPCS_VarArg_ArgRegs, Reg);
472   case CallingConv::Win64:
473     if (IsVarArg)
474       HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);
475     return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
476   case CallingConv::CFGuard_Check:
477     return HasReg(CC_AArch64_Win64_CFGuard_Check_ArgRegs, Reg);
478   case CallingConv::AArch64_VectorCall:
479   case CallingConv::AArch64_SVE_VectorCall:
480     return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
481   }
482 }
483 
484 Register
485 AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
486   const AArch64FrameLowering *TFI = getFrameLowering(MF);
487   return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP;
488 }
489 
490 bool AArch64RegisterInfo::requiresRegisterScavenging(
491     const MachineFunction &MF) const {
492   return true;
493 }
494 
495 bool AArch64RegisterInfo::requiresVirtualBaseRegisters(
496     const MachineFunction &MF) const {
497   return true;
498 }
499 
500 bool
501 AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const {
502   // This function indicates whether the emergency spillslot should be placed
503   // close to the beginning of the stackframe (closer to FP) or the end
504   // (closer to SP).
505   //
506   // The beginning works most reliably if we have a frame pointer.
507   // In the presence of any non-constant space between FP and locals,
508   // (e.g. in case of stack realignment or a scalable SVE area), it is
509   // better to use SP or BP.
510   const AArch64FrameLowering &TFI = *getFrameLowering(MF);
511   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
512   assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() ||
513           AFI->hasCalculatedStackSizeSVE()) &&
514          "Expected SVE area to be calculated by this point");
515   return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->getStackSizeSVE();
516 }
517 
518 bool AArch64RegisterInfo::requiresFrameIndexScavenging(
519     const MachineFunction &MF) const {
520   return true;
521 }
522 
523 bool
524 AArch64RegisterInfo::cannotEliminateFrame(const MachineFunction &MF) const {
525   const MachineFrameInfo &MFI = MF.getFrameInfo();
526   if (MF.getTarget().Options.DisableFramePointerElim(MF) && MFI.adjustsStack())
527     return true;
528   return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken();
529 }
530 
531 /// needsFrameBaseReg - Returns true if the instruction's frame index
532 /// reference would be better served by a base register other than FP
533 /// or SP. Used by LocalStackFrameAllocation to determine which frame index
534 /// references it should create new base registers for.
535 bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI,
536                                             int64_t Offset) const {
537   for (unsigned i = 0; !MI->getOperand(i).isFI(); ++i)
538     assert(i < MI->getNumOperands() &&
539            "Instr doesn't have FrameIndex operand!");
540 
541   // It's the load/store FI references that cause issues, as it can be difficult
542   // to materialize the offset if it won't fit in the literal field. Estimate
543   // based on the size of the local frame and some conservative assumptions
544   // about the rest of the stack frame (note, this is pre-regalloc, so
545   // we don't know everything for certain yet) whether this offset is likely
546   // to be out of range of the immediate. Return true if so.
547 
548   // We only generate virtual base registers for loads and stores, so
549   // return false for everything else.
550   if (!MI->mayLoad() && !MI->mayStore())
551     return false;
552 
553   // Without a virtual base register, if the function has variable sized
554   // objects, all fixed-size local references will be via the frame pointer,
555   // Approximate the offset and see if it's legal for the instruction.
556   // Note that the incoming offset is based on the SP value at function entry,
557   // so it'll be negative.
558   MachineFunction &MF = *MI->getParent()->getParent();
559   const AArch64FrameLowering *TFI = getFrameLowering(MF);
560   MachineFrameInfo &MFI = MF.getFrameInfo();
561 
562   // Estimate an offset from the frame pointer.
563   // Conservatively assume all GPR callee-saved registers get pushed.
564   // FP, LR, X19-X28, D8-D15. 64-bits each.
565   int64_t FPOffset = Offset - 16 * 20;
566   // Estimate an offset from the stack pointer.
567   // The incoming offset is relating to the SP at the start of the function,
568   // but when we access the local it'll be relative to the SP after local
569   // allocation, so adjust our SP-relative offset by that allocation size.
570   Offset += MFI.getLocalFrameSize();
571   // Assume that we'll have at least some spill slots allocated.
572   // FIXME: This is a total SWAG number. We should run some statistics
573   //        and pick a real one.
574   Offset += 128; // 128 bytes of spill slots
575 
576   // If there is a frame pointer, try using it.
577   // The FP is only available if there is no dynamic realignment. We
578   // don't know for sure yet whether we'll need that, so we guess based
579   // on whether there are any local variables that would trigger it.
580   if (TFI->hasFP(MF) && isFrameOffsetLegal(MI, AArch64::FP, FPOffset))
581     return false;
582 
583   // If we can reference via the stack pointer or base pointer, try that.
584   // FIXME: This (and the code that resolves the references) can be improved
585   //        to only disallow SP relative references in the live range of
586   //        the VLA(s). In practice, it's unclear how much difference that
587   //        would make, but it may be worth doing.
588   if (isFrameOffsetLegal(MI, AArch64::SP, Offset))
589     return false;
590 
591   // If even offset 0 is illegal, we don't want a virtual base register.
592   if (!isFrameOffsetLegal(MI, AArch64::SP, 0))
593     return false;
594 
595   // The offset likely isn't legal; we want to allocate a virtual base register.
596   return true;
597 }
598 
599 bool AArch64RegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
600                                              Register BaseReg,
601                                              int64_t Offset) const {
602   assert(MI && "Unable to get the legal offset for nil instruction.");
603   StackOffset SaveOffset = StackOffset::getFixed(Offset);
604   return isAArch64FrameOffsetLegal(*MI, SaveOffset) & AArch64FrameOffsetIsLegal;
605 }
606 
607 /// Insert defining instruction(s) for BaseReg to be a pointer to FrameIdx
608 /// at the beginning of the basic block.
609 Register
610 AArch64RegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,
611                                                   int FrameIdx,
612                                                   int64_t Offset) const {
613   MachineBasicBlock::iterator Ins = MBB->begin();
614   DebugLoc DL; // Defaults to "unknown"
615   if (Ins != MBB->end())
616     DL = Ins->getDebugLoc();
617   const MachineFunction &MF = *MBB->getParent();
618   const AArch64InstrInfo *TII =
619       MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
620   const MCInstrDesc &MCID = TII->get(AArch64::ADDXri);
621   MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
622   Register BaseReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
623   MRI.constrainRegClass(BaseReg, TII->getRegClass(MCID, 0, this, MF));
624   unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0);
625 
626   BuildMI(*MBB, Ins, DL, MCID, BaseReg)
627       .addFrameIndex(FrameIdx)
628       .addImm(Offset)
629       .addImm(Shifter);
630 
631   return BaseReg;
632 }
633 
634 void AArch64RegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg,
635                                             int64_t Offset) const {
636   // ARM doesn't need the general 64-bit offsets
637   StackOffset Off = StackOffset::getFixed(Offset);
638 
639   unsigned i = 0;
640   while (!MI.getOperand(i).isFI()) {
641     ++i;
642     assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!");
643   }
644 
645   const MachineFunction *MF = MI.getParent()->getParent();
646   const AArch64InstrInfo *TII =
647       MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
648   bool Done = rewriteAArch64FrameIndex(MI, i, BaseReg, Off, TII);
649   assert(Done && "Unable to resolve frame index!");
650   (void)Done;
651 }
652 
653 // Create a scratch register for the frame index elimination in an instruction.
654 // This function has special handling of stack tagging loop pseudos, in which
655 // case it can also change the instruction opcode.
656 static Register
657 createScratchRegisterForInstruction(MachineInstr &MI, unsigned FIOperandNum,
658                                     const AArch64InstrInfo *TII) {
659   // ST*Gloop have a reserved scratch register in operand 1. Use it, and also
660   // replace the instruction with the writeback variant because it will now
661   // satisfy the operand constraints for it.
662   Register ScratchReg;
663   if (MI.getOpcode() == AArch64::STGloop ||
664       MI.getOpcode() == AArch64::STZGloop) {
665     assert(FIOperandNum == 3 &&
666            "Wrong frame index operand for STGloop/STZGloop");
667     unsigned Op = MI.getOpcode() == AArch64::STGloop ? AArch64::STGloop_wback
668                                                      : AArch64::STZGloop_wback;
669     ScratchReg = MI.getOperand(1).getReg();
670     MI.getOperand(3).ChangeToRegister(ScratchReg, false, false, true);
671     MI.setDesc(TII->get(Op));
672     MI.tieOperands(1, 3);
673   } else {
674     ScratchReg =
675         MI.getMF()->getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
676     MI.getOperand(FIOperandNum)
677         .ChangeToRegister(ScratchReg, false, false, true);
678   }
679   return ScratchReg;
680 }
681 
682 void AArch64RegisterInfo::getOffsetOpcodes(
683     const StackOffset &Offset, SmallVectorImpl<uint64_t> &Ops) const {
684   // The smallest scalable element supported by scaled SVE addressing
685   // modes are predicates, which are 2 scalable bytes in size. So the scalable
686   // byte offset must always be a multiple of 2.
687   assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
688 
689   // Add fixed-sized offset using existing DIExpression interface.
690   DIExpression::appendOffset(Ops, Offset.getFixed());
691 
692   unsigned VG = getDwarfRegNum(AArch64::VG, true);
693   int64_t VGSized = Offset.getScalable() / 2;
694   if (VGSized > 0) {
695     Ops.push_back(dwarf::DW_OP_constu);
696     Ops.push_back(VGSized);
697     Ops.append({dwarf::DW_OP_bregx, VG, 0ULL});
698     Ops.push_back(dwarf::DW_OP_mul);
699     Ops.push_back(dwarf::DW_OP_plus);
700   } else if (VGSized < 0) {
701     Ops.push_back(dwarf::DW_OP_constu);
702     Ops.push_back(-VGSized);
703     Ops.append({dwarf::DW_OP_bregx, VG, 0ULL});
704     Ops.push_back(dwarf::DW_OP_mul);
705     Ops.push_back(dwarf::DW_OP_minus);
706   }
707 }
708 
709 void AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
710                                               int SPAdj, unsigned FIOperandNum,
711                                               RegScavenger *RS) const {
712   assert(SPAdj == 0 && "Unexpected");
713 
714   MachineInstr &MI = *II;
715   MachineBasicBlock &MBB = *MI.getParent();
716   MachineFunction &MF = *MBB.getParent();
717   const MachineFrameInfo &MFI = MF.getFrameInfo();
718   const AArch64InstrInfo *TII =
719       MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
720   const AArch64FrameLowering *TFI = getFrameLowering(MF);
721   int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
722   bool Tagged =
723       MI.getOperand(FIOperandNum).getTargetFlags() & AArch64II::MO_TAGGED;
724   Register FrameReg;
725 
726   // Special handling of dbg_value, stackmap patchpoint statepoint instructions.
727   if (MI.getOpcode() == TargetOpcode::STACKMAP ||
728       MI.getOpcode() == TargetOpcode::PATCHPOINT ||
729       MI.getOpcode() == TargetOpcode::STATEPOINT) {
730     StackOffset Offset =
731         TFI->resolveFrameIndexReference(MF, FrameIndex, FrameReg,
732                                         /*PreferFP=*/true,
733                                         /*ForSimm=*/false);
734     Offset += StackOffset::getFixed(MI.getOperand(FIOperandNum + 1).getImm());
735     MI.getOperand(FIOperandNum).ChangeToRegister(FrameReg, false /*isDef*/);
736     MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Offset.getFixed());
737     return;
738   }
739 
740   if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) {
741     MachineOperand &FI = MI.getOperand(FIOperandNum);
742     StackOffset Offset = TFI->getNonLocalFrameIndexReference(MF, FrameIndex);
743     assert(!Offset.getScalable() &&
744            "Frame offsets with a scalable component are not supported");
745     FI.ChangeToImmediate(Offset.getFixed());
746     return;
747   }
748 
749   StackOffset Offset;
750   if (MI.getOpcode() == AArch64::TAGPstack) {
751     // TAGPstack must use the virtual frame register in its 3rd operand.
752     const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
753     FrameReg = MI.getOperand(3).getReg();
754     Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) +
755                                       AFI->getTaggedBasePointerOffset());
756   } else if (Tagged) {
757     StackOffset SPOffset = StackOffset::getFixed(
758         MFI.getObjectOffset(FrameIndex) + (int64_t)MFI.getStackSize());
759     if (MFI.hasVarSizedObjects() ||
760         isAArch64FrameOffsetLegal(MI, SPOffset, nullptr, nullptr, nullptr) !=
761             (AArch64FrameOffsetCanUpdate | AArch64FrameOffsetIsLegal)) {
762       // Can't update to SP + offset in place. Precalculate the tagged pointer
763       // in a scratch register.
764       Offset = TFI->resolveFrameIndexReference(
765           MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);
766       Register ScratchReg =
767           MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
768       emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset,
769                       TII);
770       BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AArch64::LDG), ScratchReg)
771           .addReg(ScratchReg)
772           .addReg(ScratchReg)
773           .addImm(0);
774       MI.getOperand(FIOperandNum)
775           .ChangeToRegister(ScratchReg, false, false, true);
776       return;
777     }
778     FrameReg = AArch64::SP;
779     Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) +
780                                    (int64_t)MFI.getStackSize());
781   } else {
782     Offset = TFI->resolveFrameIndexReference(
783         MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);
784   }
785 
786   // Modify MI as necessary to handle as much of 'Offset' as possible
787   if (rewriteAArch64FrameIndex(MI, FIOperandNum, FrameReg, Offset, TII))
788     return;
789 
790   assert((!RS || !RS->isScavengingFrameIndex(FrameIndex)) &&
791          "Emergency spill slot is out of reach");
792 
793   // If we get here, the immediate doesn't fit into the instruction.  We folded
794   // as much as possible above.  Handle the rest, providing a register that is
795   // SP+LargeImm.
796   Register ScratchReg =
797       createScratchRegisterForInstruction(MI, FIOperandNum, TII);
798   emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset, TII);
799 }
800 
801 unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
802                                                   MachineFunction &MF) const {
803   const AArch64FrameLowering *TFI = getFrameLowering(MF);
804 
805   switch (RC->getID()) {
806   default:
807     return 0;
808   case AArch64::GPR32RegClassID:
809   case AArch64::GPR32spRegClassID:
810   case AArch64::GPR32allRegClassID:
811   case AArch64::GPR64spRegClassID:
812   case AArch64::GPR64allRegClassID:
813   case AArch64::GPR64RegClassID:
814   case AArch64::GPR32commonRegClassID:
815   case AArch64::GPR64commonRegClassID:
816     return 32 - 1                                   // XZR/SP
817               - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP
818               - MF.getSubtarget<AArch64Subtarget>().getNumXRegisterReserved()
819               - hasBasePointer(MF);  // X19
820   case AArch64::FPR8RegClassID:
821   case AArch64::FPR16RegClassID:
822   case AArch64::FPR32RegClassID:
823   case AArch64::FPR64RegClassID:
824   case AArch64::FPR128RegClassID:
825     return 32;
826 
827   case AArch64::MatrixIndexGPR32_12_15RegClassID:
828     return 4;
829 
830   case AArch64::DDRegClassID:
831   case AArch64::DDDRegClassID:
832   case AArch64::DDDDRegClassID:
833   case AArch64::QQRegClassID:
834   case AArch64::QQQRegClassID:
835   case AArch64::QQQQRegClassID:
836     return 32;
837 
838   case AArch64::FPR128_loRegClassID:
839   case AArch64::FPR64_loRegClassID:
840   case AArch64::FPR16_loRegClassID:
841     return 16;
842   }
843 }
844 
845 unsigned AArch64RegisterInfo::getLocalAddressRegister(
846   const MachineFunction &MF) const {
847   const auto &MFI = MF.getFrameInfo();
848   if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects())
849     return AArch64::SP;
850   else if (hasStackRealignment(MF))
851     return getBaseRegister();
852   return getFrameRegister(MF);
853 }
854 
855 /// SrcRC and DstRC will be morphed into NewRC if this returns true
856 bool AArch64RegisterInfo::shouldCoalesce(
857     MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg,
858     const TargetRegisterClass *DstRC, unsigned DstSubReg,
859     const TargetRegisterClass *NewRC, LiveIntervals &LIS) const {
860   if (MI->isCopy() &&
861       ((DstRC->getID() == AArch64::GPR64RegClassID) ||
862        (DstRC->getID() == AArch64::GPR64commonRegClassID)) &&
863       MI->getOperand(0).getSubReg() && MI->getOperand(1).getSubReg())
864     // Do not coalesce in the case of a 32-bit subregister copy
865     // which implements a 32 to 64 bit zero extension
866     // which relies on the upper 32 bits being zeroed.
867     return false;
868   return true;
869 }
870