1 //===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of the TargetRegisterInfo
10 // class.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "AArch64RegisterInfo.h"
15 #include "AArch64FrameLowering.h"
16 #include "AArch64InstrInfo.h"
17 #include "AArch64MachineFunctionInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "MCTargetDesc/AArch64AddressingModes.h"
20 #include "MCTargetDesc/AArch64InstPrinter.h"
21 #include "llvm/ADT/BitVector.h"
22 #include "llvm/BinaryFormat/Dwarf.h"
23 #include "llvm/CodeGen/MachineFrameInfo.h"
24 #include "llvm/CodeGen/MachineInstrBuilder.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/RegisterScavenging.h"
27 #include "llvm/CodeGen/TargetFrameLowering.h"
28 #include "llvm/IR/DebugInfoMetadata.h"
29 #include "llvm/IR/DiagnosticInfo.h"
30 #include "llvm/IR/Function.h"
31 #include "llvm/Support/raw_ostream.h"
32 #include "llvm/Target/TargetOptions.h"
33 #include "llvm/TargetParser/Triple.h"
34
35 using namespace llvm;
36
37 #define GET_CC_REGISTER_LISTS
38 #include "AArch64GenCallingConv.inc"
39 #define GET_REGINFO_TARGET_DESC
40 #include "AArch64GenRegisterInfo.inc"
41
AArch64RegisterInfo(const Triple & TT)42 AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT)
43 : AArch64GenRegisterInfo(AArch64::LR), TT(TT) {
44 AArch64_MC::initLLVMToCVRegMapping(this);
45 }
46
47 /// Return whether the register needs a CFI entry. Not all unwinders may know
48 /// about SVE registers, so we assume the lowest common denominator, i.e. the
49 /// callee-saves required by the base ABI. For the SVE registers z8-z15 only the
50 /// lower 64-bits (d8-d15) need to be saved. The lower 64-bits subreg is
51 /// returned in \p RegToUseForCFI.
regNeedsCFI(unsigned Reg,unsigned & RegToUseForCFI) const52 bool AArch64RegisterInfo::regNeedsCFI(unsigned Reg,
53 unsigned &RegToUseForCFI) const {
54 if (AArch64::PPRRegClass.contains(Reg))
55 return false;
56
57 if (AArch64::ZPRRegClass.contains(Reg)) {
58 RegToUseForCFI = getSubReg(Reg, AArch64::dsub);
59 for (int I = 0; CSR_AArch64_AAPCS_SaveList[I]; ++I) {
60 if (CSR_AArch64_AAPCS_SaveList[I] == RegToUseForCFI)
61 return true;
62 }
63 return false;
64 }
65
66 RegToUseForCFI = Reg;
67 return true;
68 }
69
70 const MCPhysReg *
getCalleeSavedRegs(const MachineFunction * MF) const71 AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
72 assert(MF && "Invalid MachineFunction pointer.");
73
74 if (MF->getFunction().getCallingConv() == CallingConv::GHC)
75 // GHC set of callee saved regs is empty as all those regs are
76 // used for passing STG regs around
77 return CSR_AArch64_NoRegs_SaveList;
78 if (MF->getFunction().getCallingConv() == CallingConv::AnyReg)
79 return CSR_AArch64_AllRegs_SaveList;
80
81 if (MF->getFunction().getCallingConv() == CallingConv::ARM64EC_Thunk_X64)
82 return CSR_Win_AArch64_Arm64EC_Thunk_SaveList;
83
84 // Darwin has its own CSR_AArch64_AAPCS_SaveList, which means most CSR save
85 // lists depending on that will need to have their Darwin variant as well.
86 if (MF->getSubtarget<AArch64Subtarget>().isTargetDarwin())
87 return getDarwinCalleeSavedRegs(MF);
88
89 if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check)
90 return CSR_Win_AArch64_CFGuard_Check_SaveList;
91 if (MF->getSubtarget<AArch64Subtarget>().isTargetWindows()) {
92 if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering()
93 ->supportSwiftError() &&
94 MF->getFunction().getAttributes().hasAttrSomewhere(
95 Attribute::SwiftError))
96 return CSR_Win_AArch64_AAPCS_SwiftError_SaveList;
97 if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail)
98 return CSR_Win_AArch64_AAPCS_SwiftTail_SaveList;
99 return CSR_Win_AArch64_AAPCS_SaveList;
100 }
101 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall)
102 return CSR_AArch64_AAVPCS_SaveList;
103 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall)
104 return CSR_AArch64_SVE_AAPCS_SaveList;
105 if (MF->getFunction().getCallingConv() ==
106 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
107 report_fatal_error(
108 "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is "
109 "only supported to improve calls to SME ACLE save/restore/disable-za "
110 "functions, and is not intended to be used beyond that scope.");
111 if (MF->getFunction().getCallingConv() ==
112 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
113 report_fatal_error(
114 "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
115 "only supported to improve calls to SME ACLE __arm_sme_state "
116 "and is not intended to be used beyond that scope.");
117 if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering()
118 ->supportSwiftError() &&
119 MF->getFunction().getAttributes().hasAttrSomewhere(
120 Attribute::SwiftError))
121 return CSR_AArch64_AAPCS_SwiftError_SaveList;
122 if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail)
123 return CSR_AArch64_AAPCS_SwiftTail_SaveList;
124 if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost)
125 return CSR_AArch64_RT_MostRegs_SaveList;
126 if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll)
127 return CSR_AArch64_RT_AllRegs_SaveList;
128 if (MF->getFunction().getCallingConv() == CallingConv::Win64)
129 // This is for OSes other than Windows; Windows is a separate case further
130 // above.
131 return CSR_AArch64_AAPCS_X18_SaveList;
132 if (MF->getInfo<AArch64FunctionInfo>()->isSVECC())
133 return CSR_AArch64_SVE_AAPCS_SaveList;
134 return CSR_AArch64_AAPCS_SaveList;
135 }
136
137 const MCPhysReg *
getDarwinCalleeSavedRegs(const MachineFunction * MF) const138 AArch64RegisterInfo::getDarwinCalleeSavedRegs(const MachineFunction *MF) const {
139 assert(MF && "Invalid MachineFunction pointer.");
140 assert(MF->getSubtarget<AArch64Subtarget>().isTargetDarwin() &&
141 "Invalid subtarget for getDarwinCalleeSavedRegs");
142
143 if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check)
144 report_fatal_error(
145 "Calling convention CFGuard_Check is unsupported on Darwin.");
146 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall)
147 return CSR_Darwin_AArch64_AAVPCS_SaveList;
148 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall)
149 report_fatal_error(
150 "Calling convention SVE_VectorCall is unsupported on Darwin.");
151 if (MF->getFunction().getCallingConv() ==
152 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
153 report_fatal_error(
154 "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is "
155 "only supported to improve calls to SME ACLE save/restore/disable-za "
156 "functions, and is not intended to be used beyond that scope.");
157 if (MF->getFunction().getCallingConv() ==
158 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
159 report_fatal_error(
160 "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
161 "only supported to improve calls to SME ACLE __arm_sme_state "
162 "and is not intended to be used beyond that scope.");
163 if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS)
164 return MF->getInfo<AArch64FunctionInfo>()->isSplitCSR()
165 ? CSR_Darwin_AArch64_CXX_TLS_PE_SaveList
166 : CSR_Darwin_AArch64_CXX_TLS_SaveList;
167 if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering()
168 ->supportSwiftError() &&
169 MF->getFunction().getAttributes().hasAttrSomewhere(
170 Attribute::SwiftError))
171 return CSR_Darwin_AArch64_AAPCS_SwiftError_SaveList;
172 if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail)
173 return CSR_Darwin_AArch64_AAPCS_SwiftTail_SaveList;
174 if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost)
175 return CSR_Darwin_AArch64_RT_MostRegs_SaveList;
176 if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll)
177 return CSR_Darwin_AArch64_RT_AllRegs_SaveList;
178 if (MF->getFunction().getCallingConv() == CallingConv::Win64)
179 return CSR_Darwin_AArch64_AAPCS_Win64_SaveList;
180 return CSR_Darwin_AArch64_AAPCS_SaveList;
181 }
182
getCalleeSavedRegsViaCopy(const MachineFunction * MF) const183 const MCPhysReg *AArch64RegisterInfo::getCalleeSavedRegsViaCopy(
184 const MachineFunction *MF) const {
185 assert(MF && "Invalid MachineFunction pointer.");
186 if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
187 MF->getInfo<AArch64FunctionInfo>()->isSplitCSR())
188 return CSR_Darwin_AArch64_CXX_TLS_ViaCopy_SaveList;
189 return nullptr;
190 }
191
UpdateCustomCalleeSavedRegs(MachineFunction & MF) const192 void AArch64RegisterInfo::UpdateCustomCalleeSavedRegs(
193 MachineFunction &MF) const {
194 const MCPhysReg *CSRs = getCalleeSavedRegs(&MF);
195 SmallVector<MCPhysReg, 32> UpdatedCSRs;
196 for (const MCPhysReg *I = CSRs; *I; ++I)
197 UpdatedCSRs.push_back(*I);
198
199 for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {
200 if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {
201 UpdatedCSRs.push_back(AArch64::GPR64commonRegClass.getRegister(i));
202 }
203 }
204 // Register lists are zero-terminated.
205 UpdatedCSRs.push_back(0);
206 MF.getRegInfo().setCalleeSavedRegs(UpdatedCSRs);
207 }
208
209 const TargetRegisterClass *
getSubClassWithSubReg(const TargetRegisterClass * RC,unsigned Idx) const210 AArch64RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC,
211 unsigned Idx) const {
212 // edge case for GPR/FPR register classes
213 if (RC == &AArch64::GPR32allRegClass && Idx == AArch64::hsub)
214 return &AArch64::FPR32RegClass;
215 else if (RC == &AArch64::GPR64allRegClass && Idx == AArch64::hsub)
216 return &AArch64::FPR64RegClass;
217
218 // Forward to TableGen's default version.
219 return AArch64GenRegisterInfo::getSubClassWithSubReg(RC, Idx);
220 }
221
222 const uint32_t *
getDarwinCallPreservedMask(const MachineFunction & MF,CallingConv::ID CC) const223 AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF,
224 CallingConv::ID CC) const {
225 assert(MF.getSubtarget<AArch64Subtarget>().isTargetDarwin() &&
226 "Invalid subtarget for getDarwinCallPreservedMask");
227
228 if (CC == CallingConv::CXX_FAST_TLS)
229 return CSR_Darwin_AArch64_CXX_TLS_RegMask;
230 if (CC == CallingConv::AArch64_VectorCall)
231 return CSR_Darwin_AArch64_AAVPCS_RegMask;
232 if (CC == CallingConv::AArch64_SVE_VectorCall)
233 report_fatal_error(
234 "Calling convention SVE_VectorCall is unsupported on Darwin.");
235 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
236 report_fatal_error(
237 "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is "
238 "unsupported on Darwin.");
239 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
240 report_fatal_error(
241 "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
242 "unsupported on Darwin.");
243 if (CC == CallingConv::CFGuard_Check)
244 report_fatal_error(
245 "Calling convention CFGuard_Check is unsupported on Darwin.");
246 if (MF.getSubtarget<AArch64Subtarget>()
247 .getTargetLowering()
248 ->supportSwiftError() &&
249 MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError))
250 return CSR_Darwin_AArch64_AAPCS_SwiftError_RegMask;
251 if (CC == CallingConv::SwiftTail)
252 return CSR_Darwin_AArch64_AAPCS_SwiftTail_RegMask;
253 if (CC == CallingConv::PreserveMost)
254 return CSR_Darwin_AArch64_RT_MostRegs_RegMask;
255 if (CC == CallingConv::PreserveAll)
256 return CSR_Darwin_AArch64_RT_AllRegs_RegMask;
257 return CSR_Darwin_AArch64_AAPCS_RegMask;
258 }
259
260 const uint32_t *
getCallPreservedMask(const MachineFunction & MF,CallingConv::ID CC) const261 AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
262 CallingConv::ID CC) const {
263 bool SCS = MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack);
264 if (CC == CallingConv::GHC)
265 // This is academic because all GHC calls are (supposed to be) tail calls
266 return SCS ? CSR_AArch64_NoRegs_SCS_RegMask : CSR_AArch64_NoRegs_RegMask;
267 if (CC == CallingConv::AnyReg)
268 return SCS ? CSR_AArch64_AllRegs_SCS_RegMask : CSR_AArch64_AllRegs_RegMask;
269
270 // All the following calling conventions are handled differently on Darwin.
271 if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) {
272 if (SCS)
273 report_fatal_error("ShadowCallStack attribute not supported on Darwin.");
274 return getDarwinCallPreservedMask(MF, CC);
275 }
276
277 if (CC == CallingConv::AArch64_VectorCall)
278 return SCS ? CSR_AArch64_AAVPCS_SCS_RegMask : CSR_AArch64_AAVPCS_RegMask;
279 if (CC == CallingConv::AArch64_SVE_VectorCall)
280 return SCS ? CSR_AArch64_SVE_AAPCS_SCS_RegMask
281 : CSR_AArch64_SVE_AAPCS_RegMask;
282 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
283 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
284 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
285 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;
286 if (CC == CallingConv::CFGuard_Check)
287 return CSR_Win_AArch64_CFGuard_Check_RegMask;
288 if (MF.getSubtarget<AArch64Subtarget>().getTargetLowering()
289 ->supportSwiftError() &&
290 MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError))
291 return SCS ? CSR_AArch64_AAPCS_SwiftError_SCS_RegMask
292 : CSR_AArch64_AAPCS_SwiftError_RegMask;
293 if (CC == CallingConv::SwiftTail) {
294 if (SCS)
295 report_fatal_error("ShadowCallStack attribute not supported with swifttail");
296 return CSR_AArch64_AAPCS_SwiftTail_RegMask;
297 }
298 if (CC == CallingConv::PreserveMost)
299 return SCS ? CSR_AArch64_RT_MostRegs_SCS_RegMask
300 : CSR_AArch64_RT_MostRegs_RegMask;
301 else if (CC == CallingConv::PreserveAll)
302 return SCS ? CSR_AArch64_RT_AllRegs_SCS_RegMask
303 : CSR_AArch64_RT_AllRegs_RegMask;
304
305 else
306 return SCS ? CSR_AArch64_AAPCS_SCS_RegMask : CSR_AArch64_AAPCS_RegMask;
307 }
308
getCustomEHPadPreservedMask(const MachineFunction & MF) const309 const uint32_t *AArch64RegisterInfo::getCustomEHPadPreservedMask(
310 const MachineFunction &MF) const {
311 if (MF.getSubtarget<AArch64Subtarget>().isTargetLinux())
312 return CSR_AArch64_AAPCS_RegMask;
313
314 return nullptr;
315 }
316
getTLSCallPreservedMask() const317 const uint32_t *AArch64RegisterInfo::getTLSCallPreservedMask() const {
318 if (TT.isOSDarwin())
319 return CSR_Darwin_AArch64_TLS_RegMask;
320
321 assert(TT.isOSBinFormatELF() && "Invalid target");
322 return CSR_AArch64_TLS_ELF_RegMask;
323 }
324
UpdateCustomCallPreservedMask(MachineFunction & MF,const uint32_t ** Mask) const325 void AArch64RegisterInfo::UpdateCustomCallPreservedMask(MachineFunction &MF,
326 const uint32_t **Mask) const {
327 uint32_t *UpdatedMask = MF.allocateRegMask();
328 unsigned RegMaskSize = MachineOperand::getRegMaskSize(getNumRegs());
329 memcpy(UpdatedMask, *Mask, sizeof(UpdatedMask[0]) * RegMaskSize);
330
331 for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {
332 if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {
333 for (MCPhysReg SubReg :
334 subregs_inclusive(AArch64::GPR64commonRegClass.getRegister(i))) {
335 // See TargetRegisterInfo::getCallPreservedMask for how to interpret the
336 // register mask.
337 UpdatedMask[SubReg / 32] |= 1u << (SubReg % 32);
338 }
339 }
340 }
341 *Mask = UpdatedMask;
342 }
343
getSMStartStopCallPreservedMask() const344 const uint32_t *AArch64RegisterInfo::getSMStartStopCallPreservedMask() const {
345 return CSR_AArch64_SMStartStop_RegMask;
346 }
347
348 const uint32_t *
SMEABISupportRoutinesCallPreservedMaskFromX0() const349 AArch64RegisterInfo::SMEABISupportRoutinesCallPreservedMaskFromX0() const {
350 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
351 }
352
getNoPreservedMask() const353 const uint32_t *AArch64RegisterInfo::getNoPreservedMask() const {
354 return CSR_AArch64_NoRegs_RegMask;
355 }
356
357 const uint32_t *
getThisReturnPreservedMask(const MachineFunction & MF,CallingConv::ID CC) const358 AArch64RegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF,
359 CallingConv::ID CC) const {
360 // This should return a register mask that is the same as that returned by
361 // getCallPreservedMask but that additionally preserves the register used for
362 // the first i64 argument (which must also be the register used to return a
363 // single i64 return value)
364 //
365 // In case that the calling convention does not use the same register for
366 // both, the function should return NULL (does not currently apply)
367 assert(CC != CallingConv::GHC && "should not be GHC calling convention.");
368 if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin())
369 return CSR_Darwin_AArch64_AAPCS_ThisReturn_RegMask;
370 return CSR_AArch64_AAPCS_ThisReturn_RegMask;
371 }
372
getWindowsStackProbePreservedMask() const373 const uint32_t *AArch64RegisterInfo::getWindowsStackProbePreservedMask() const {
374 return CSR_AArch64_StackProbe_Windows_RegMask;
375 }
376
377 std::optional<std::string>
explainReservedReg(const MachineFunction & MF,MCRegister PhysReg) const378 AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF,
379 MCRegister PhysReg) const {
380 if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(PhysReg, AArch64::X19))
381 return std::string("X19 is used as the frame base pointer register.");
382
383 if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
384 bool warn = false;
385 if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X13) ||
386 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X14) ||
387 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X23) ||
388 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X24) ||
389 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X28))
390 warn = true;
391
392 for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i)
393 if (MCRegisterInfo::regsOverlap(PhysReg, i))
394 warn = true;
395
396 if (warn)
397 return std::string(AArch64InstPrinter::getRegisterName(PhysReg)) +
398 " is clobbered by asynchronous signals when using Arm64EC.";
399 }
400
401 return {};
402 }
403
404 BitVector
getStrictlyReservedRegs(const MachineFunction & MF) const405 AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
406 const AArch64FrameLowering *TFI = getFrameLowering(MF);
407
408 // FIXME: avoid re-calculating this every time.
409 BitVector Reserved(getNumRegs());
410 markSuperRegs(Reserved, AArch64::WSP);
411 markSuperRegs(Reserved, AArch64::WZR);
412
413 if (TFI->hasFP(MF) || TT.isOSDarwin())
414 markSuperRegs(Reserved, AArch64::W29);
415
416 if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
417 // x13, x14, x23, x24, x28, and v16-v31 are clobbered by asynchronous
418 // signals, so we can't ever use them.
419 markSuperRegs(Reserved, AArch64::W13);
420 markSuperRegs(Reserved, AArch64::W14);
421 markSuperRegs(Reserved, AArch64::W23);
422 markSuperRegs(Reserved, AArch64::W24);
423 markSuperRegs(Reserved, AArch64::W28);
424 for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i)
425 markSuperRegs(Reserved, i);
426 }
427
428 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
429 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
430 markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));
431 }
432
433 if (hasBasePointer(MF))
434 markSuperRegs(Reserved, AArch64::W19);
435
436 // SLH uses register W16/X16 as the taint register.
437 if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
438 markSuperRegs(Reserved, AArch64::W16);
439
440 // SME tiles are not allocatable.
441 if (MF.getSubtarget<AArch64Subtarget>().hasSME()) {
442 for (MCPhysReg SubReg : subregs_inclusive(AArch64::ZA))
443 Reserved.set(SubReg);
444 }
445
446 if (MF.getSubtarget<AArch64Subtarget>().hasSME2()) {
447 for (MCSubRegIterator SubReg(AArch64::ZT0, this, /*self=*/true);
448 SubReg.isValid(); ++SubReg)
449 Reserved.set(*SubReg);
450 }
451
452 markSuperRegs(Reserved, AArch64::FPCR);
453
454 if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
455 markSuperRegs(Reserved, AArch64::X27);
456 markSuperRegs(Reserved, AArch64::X28);
457 markSuperRegs(Reserved, AArch64::W27);
458 markSuperRegs(Reserved, AArch64::W28);
459 }
460
461 assert(checkAllSuperRegsMarked(Reserved));
462 return Reserved;
463 }
464
465 BitVector
getReservedRegs(const MachineFunction & MF) const466 AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
467 BitVector Reserved = getStrictlyReservedRegs(MF);
468
469 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
470 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReservedForRA(i))
471 markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));
472 }
473
474 assert(checkAllSuperRegsMarked(Reserved));
475 return Reserved;
476 }
477
isReservedReg(const MachineFunction & MF,MCRegister Reg) const478 bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,
479 MCRegister Reg) const {
480 return getReservedRegs(MF)[Reg];
481 }
482
isStrictlyReservedReg(const MachineFunction & MF,MCRegister Reg) const483 bool AArch64RegisterInfo::isStrictlyReservedReg(const MachineFunction &MF,
484 MCRegister Reg) const {
485 return getStrictlyReservedRegs(MF)[Reg];
486 }
487
isAnyArgRegReserved(const MachineFunction & MF) const488 bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const {
489 return llvm::any_of(*AArch64::GPR64argRegClass.MC, [this, &MF](MCPhysReg r) {
490 return isStrictlyReservedReg(MF, r);
491 });
492 }
493
emitReservedArgRegCallError(const MachineFunction & MF) const494 void AArch64RegisterInfo::emitReservedArgRegCallError(
495 const MachineFunction &MF) const {
496 const Function &F = MF.getFunction();
497 F.getContext().diagnose(DiagnosticInfoUnsupported{F, ("AArch64 doesn't support"
498 " function calls if any of the argument registers is reserved.")});
499 }
500
isAsmClobberable(const MachineFunction & MF,MCRegister PhysReg) const501 bool AArch64RegisterInfo::isAsmClobberable(const MachineFunction &MF,
502 MCRegister PhysReg) const {
503 // SLH uses register X16 as the taint register but it will fallback to a different
504 // method if the user clobbers it. So X16 is not reserved for inline asm but is
505 // for normal codegen.
506 if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening) &&
507 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X16))
508 return true;
509
510 // ZA/ZT0 registers are reserved but may be permitted in the clobber list.
511 if (PhysReg == AArch64::ZA || PhysReg == AArch64::ZT0)
512 return true;
513
514 return !isReservedReg(MF, PhysReg);
515 }
516
517 const TargetRegisterClass *
getPointerRegClass(const MachineFunction & MF,unsigned Kind) const518 AArch64RegisterInfo::getPointerRegClass(const MachineFunction &MF,
519 unsigned Kind) const {
520 return &AArch64::GPR64spRegClass;
521 }
522
523 const TargetRegisterClass *
getCrossCopyRegClass(const TargetRegisterClass * RC) const524 AArch64RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {
525 if (RC == &AArch64::CCRRegClass)
526 return &AArch64::GPR64RegClass; // Only MSR & MRS copy NZCV.
527 return RC;
528 }
529
getBaseRegister() const530 unsigned AArch64RegisterInfo::getBaseRegister() const { return AArch64::X19; }
531
hasBasePointer(const MachineFunction & MF) const532 bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
533 const MachineFrameInfo &MFI = MF.getFrameInfo();
534
535 // In the presence of variable sized objects or funclets, if the fixed stack
536 // size is large enough that referencing from the FP won't result in things
537 // being in range relatively often, we can use a base pointer to allow access
538 // from the other direction like the SP normally works.
539 //
540 // Furthermore, if both variable sized objects are present, and the
541 // stack needs to be dynamically re-aligned, the base pointer is the only
542 // reliable way to reference the locals.
543 if (MFI.hasVarSizedObjects() || MF.hasEHFunclets()) {
544 if (hasStackRealignment(MF))
545 return true;
546
547 if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
548 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
549 // Frames that have variable sized objects and scalable SVE objects,
550 // should always use a basepointer.
551 if (!AFI->hasCalculatedStackSizeSVE() || AFI->getStackSizeSVE())
552 return true;
553 }
554
555 // Conservatively estimate whether the negative offset from the frame
556 // pointer will be sufficient to reach. If a function has a smallish
557 // frame, it's less likely to have lots of spills and callee saved
558 // space, so it's all more likely to be within range of the frame pointer.
559 // If it's wrong, we'll materialize the constant and still get to the
560 // object; it's just suboptimal. Negative offsets use the unscaled
561 // load/store instructions, which have a 9-bit signed immediate.
562 return MFI.getLocalFrameSize() >= 256;
563 }
564
565 return false;
566 }
567
isArgumentRegister(const MachineFunction & MF,MCRegister Reg) const568 bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF,
569 MCRegister Reg) const {
570 CallingConv::ID CC = MF.getFunction().getCallingConv();
571 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
572 bool IsVarArg = STI.isCallingConvWin64(MF.getFunction().getCallingConv());
573
574 auto HasReg = [](ArrayRef<MCRegister> RegList, MCRegister Reg) {
575 return llvm::is_contained(RegList, Reg);
576 };
577
578 switch (CC) {
579 default:
580 report_fatal_error("Unsupported calling convention.");
581 case CallingConv::GHC:
582 return HasReg(CC_AArch64_GHC_ArgRegs, Reg);
583 case CallingConv::C:
584 case CallingConv::Fast:
585 case CallingConv::PreserveMost:
586 case CallingConv::PreserveAll:
587 case CallingConv::CXX_FAST_TLS:
588 case CallingConv::Swift:
589 case CallingConv::SwiftTail:
590 case CallingConv::Tail:
591 if (STI.isTargetWindows()) {
592 if (IsVarArg)
593 return HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);
594 switch (CC) {
595 default:
596 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
597 case CallingConv::Swift:
598 case CallingConv::SwiftTail:
599 return HasReg(CC_AArch64_Win64PCS_Swift_ArgRegs, Reg) ||
600 HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
601 }
602 }
603 if (!STI.isTargetDarwin()) {
604 switch (CC) {
605 default:
606 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
607 case CallingConv::Swift:
608 case CallingConv::SwiftTail:
609 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg) ||
610 HasReg(CC_AArch64_AAPCS_Swift_ArgRegs, Reg);
611 }
612 }
613 if (!IsVarArg) {
614 switch (CC) {
615 default:
616 return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg);
617 case CallingConv::Swift:
618 case CallingConv::SwiftTail:
619 return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg) ||
620 HasReg(CC_AArch64_DarwinPCS_Swift_ArgRegs, Reg);
621 }
622 }
623 if (STI.isTargetILP32())
624 return HasReg(CC_AArch64_DarwinPCS_ILP32_VarArg_ArgRegs, Reg);
625 return HasReg(CC_AArch64_DarwinPCS_VarArg_ArgRegs, Reg);
626 case CallingConv::Win64:
627 if (IsVarArg)
628 HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);
629 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
630 case CallingConv::CFGuard_Check:
631 return HasReg(CC_AArch64_Win64_CFGuard_Check_ArgRegs, Reg);
632 case CallingConv::AArch64_VectorCall:
633 case CallingConv::AArch64_SVE_VectorCall:
634 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0:
635 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2:
636 if (STI.isTargetWindows())
637 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
638 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
639 }
640 }
641
642 Register
getFrameRegister(const MachineFunction & MF) const643 AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
644 const AArch64FrameLowering *TFI = getFrameLowering(MF);
645 return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP;
646 }
647
requiresRegisterScavenging(const MachineFunction & MF) const648 bool AArch64RegisterInfo::requiresRegisterScavenging(
649 const MachineFunction &MF) const {
650 return true;
651 }
652
requiresVirtualBaseRegisters(const MachineFunction & MF) const653 bool AArch64RegisterInfo::requiresVirtualBaseRegisters(
654 const MachineFunction &MF) const {
655 return true;
656 }
657
658 bool
useFPForScavengingIndex(const MachineFunction & MF) const659 AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const {
660 // This function indicates whether the emergency spillslot should be placed
661 // close to the beginning of the stackframe (closer to FP) or the end
662 // (closer to SP).
663 //
664 // The beginning works most reliably if we have a frame pointer.
665 // In the presence of any non-constant space between FP and locals,
666 // (e.g. in case of stack realignment or a scalable SVE area), it is
667 // better to use SP or BP.
668 const AArch64FrameLowering &TFI = *getFrameLowering(MF);
669 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
670 assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() ||
671 AFI->hasCalculatedStackSizeSVE()) &&
672 "Expected SVE area to be calculated by this point");
673 return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->getStackSizeSVE();
674 }
675
requiresFrameIndexScavenging(const MachineFunction & MF) const676 bool AArch64RegisterInfo::requiresFrameIndexScavenging(
677 const MachineFunction &MF) const {
678 return true;
679 }
680
681 bool
cannotEliminateFrame(const MachineFunction & MF) const682 AArch64RegisterInfo::cannotEliminateFrame(const MachineFunction &MF) const {
683 const MachineFrameInfo &MFI = MF.getFrameInfo();
684 if (MF.getTarget().Options.DisableFramePointerElim(MF) && MFI.adjustsStack())
685 return true;
686 return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken();
687 }
688
689 /// needsFrameBaseReg - Returns true if the instruction's frame index
690 /// reference would be better served by a base register other than FP
691 /// or SP. Used by LocalStackFrameAllocation to determine which frame index
692 /// references it should create new base registers for.
needsFrameBaseReg(MachineInstr * MI,int64_t Offset) const693 bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI,
694 int64_t Offset) const {
695 for (unsigned i = 0; !MI->getOperand(i).isFI(); ++i)
696 assert(i < MI->getNumOperands() &&
697 "Instr doesn't have FrameIndex operand!");
698
699 // It's the load/store FI references that cause issues, as it can be difficult
700 // to materialize the offset if it won't fit in the literal field. Estimate
701 // based on the size of the local frame and some conservative assumptions
702 // about the rest of the stack frame (note, this is pre-regalloc, so
703 // we don't know everything for certain yet) whether this offset is likely
704 // to be out of range of the immediate. Return true if so.
705
706 // We only generate virtual base registers for loads and stores, so
707 // return false for everything else.
708 if (!MI->mayLoad() && !MI->mayStore())
709 return false;
710
711 // Without a virtual base register, if the function has variable sized
712 // objects, all fixed-size local references will be via the frame pointer,
713 // Approximate the offset and see if it's legal for the instruction.
714 // Note that the incoming offset is based on the SP value at function entry,
715 // so it'll be negative.
716 MachineFunction &MF = *MI->getParent()->getParent();
717 const AArch64FrameLowering *TFI = getFrameLowering(MF);
718 MachineFrameInfo &MFI = MF.getFrameInfo();
719
720 // Estimate an offset from the frame pointer.
721 // Conservatively assume all GPR callee-saved registers get pushed.
722 // FP, LR, X19-X28, D8-D15. 64-bits each.
723 int64_t FPOffset = Offset - 16 * 20;
724 // Estimate an offset from the stack pointer.
725 // The incoming offset is relating to the SP at the start of the function,
726 // but when we access the local it'll be relative to the SP after local
727 // allocation, so adjust our SP-relative offset by that allocation size.
728 Offset += MFI.getLocalFrameSize();
729 // Assume that we'll have at least some spill slots allocated.
730 // FIXME: This is a total SWAG number. We should run some statistics
731 // and pick a real one.
732 Offset += 128; // 128 bytes of spill slots
733
734 // If there is a frame pointer, try using it.
735 // The FP is only available if there is no dynamic realignment. We
736 // don't know for sure yet whether we'll need that, so we guess based
737 // on whether there are any local variables that would trigger it.
738 if (TFI->hasFP(MF) && isFrameOffsetLegal(MI, AArch64::FP, FPOffset))
739 return false;
740
741 // If we can reference via the stack pointer or base pointer, try that.
742 // FIXME: This (and the code that resolves the references) can be improved
743 // to only disallow SP relative references in the live range of
744 // the VLA(s). In practice, it's unclear how much difference that
745 // would make, but it may be worth doing.
746 if (isFrameOffsetLegal(MI, AArch64::SP, Offset))
747 return false;
748
749 // If even offset 0 is illegal, we don't want a virtual base register.
750 if (!isFrameOffsetLegal(MI, AArch64::SP, 0))
751 return false;
752
753 // The offset likely isn't legal; we want to allocate a virtual base register.
754 return true;
755 }
756
isFrameOffsetLegal(const MachineInstr * MI,Register BaseReg,int64_t Offset) const757 bool AArch64RegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
758 Register BaseReg,
759 int64_t Offset) const {
760 assert(MI && "Unable to get the legal offset for nil instruction.");
761 StackOffset SaveOffset = StackOffset::getFixed(Offset);
762 return isAArch64FrameOffsetLegal(*MI, SaveOffset) & AArch64FrameOffsetIsLegal;
763 }
764
765 /// Insert defining instruction(s) for BaseReg to be a pointer to FrameIdx
766 /// at the beginning of the basic block.
767 Register
materializeFrameBaseRegister(MachineBasicBlock * MBB,int FrameIdx,int64_t Offset) const768 AArch64RegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,
769 int FrameIdx,
770 int64_t Offset) const {
771 MachineBasicBlock::iterator Ins = MBB->begin();
772 DebugLoc DL; // Defaults to "unknown"
773 if (Ins != MBB->end())
774 DL = Ins->getDebugLoc();
775 const MachineFunction &MF = *MBB->getParent();
776 const AArch64InstrInfo *TII =
777 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
778 const MCInstrDesc &MCID = TII->get(AArch64::ADDXri);
779 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
780 Register BaseReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
781 MRI.constrainRegClass(BaseReg, TII->getRegClass(MCID, 0, this, MF));
782 unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0);
783
784 BuildMI(*MBB, Ins, DL, MCID, BaseReg)
785 .addFrameIndex(FrameIdx)
786 .addImm(Offset)
787 .addImm(Shifter);
788
789 return BaseReg;
790 }
791
resolveFrameIndex(MachineInstr & MI,Register BaseReg,int64_t Offset) const792 void AArch64RegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg,
793 int64_t Offset) const {
794 // ARM doesn't need the general 64-bit offsets
795 StackOffset Off = StackOffset::getFixed(Offset);
796
797 unsigned i = 0;
798 while (!MI.getOperand(i).isFI()) {
799 ++i;
800 assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!");
801 }
802
803 const MachineFunction *MF = MI.getParent()->getParent();
804 const AArch64InstrInfo *TII =
805 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
806 bool Done = rewriteAArch64FrameIndex(MI, i, BaseReg, Off, TII);
807 assert(Done && "Unable to resolve frame index!");
808 (void)Done;
809 }
810
811 // Create a scratch register for the frame index elimination in an instruction.
812 // This function has special handling of stack tagging loop pseudos, in which
813 // case it can also change the instruction opcode.
814 static Register
createScratchRegisterForInstruction(MachineInstr & MI,unsigned FIOperandNum,const AArch64InstrInfo * TII)815 createScratchRegisterForInstruction(MachineInstr &MI, unsigned FIOperandNum,
816 const AArch64InstrInfo *TII) {
817 // ST*Gloop have a reserved scratch register in operand 1. Use it, and also
818 // replace the instruction with the writeback variant because it will now
819 // satisfy the operand constraints for it.
820 Register ScratchReg;
821 if (MI.getOpcode() == AArch64::STGloop ||
822 MI.getOpcode() == AArch64::STZGloop) {
823 assert(FIOperandNum == 3 &&
824 "Wrong frame index operand for STGloop/STZGloop");
825 unsigned Op = MI.getOpcode() == AArch64::STGloop ? AArch64::STGloop_wback
826 : AArch64::STZGloop_wback;
827 ScratchReg = MI.getOperand(1).getReg();
828 MI.getOperand(3).ChangeToRegister(ScratchReg, false, false, true);
829 MI.setDesc(TII->get(Op));
830 MI.tieOperands(1, 3);
831 } else {
832 ScratchReg =
833 MI.getMF()->getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
834 MI.getOperand(FIOperandNum)
835 .ChangeToRegister(ScratchReg, false, false, true);
836 }
837 return ScratchReg;
838 }
839
getOffsetOpcodes(const StackOffset & Offset,SmallVectorImpl<uint64_t> & Ops) const840 void AArch64RegisterInfo::getOffsetOpcodes(
841 const StackOffset &Offset, SmallVectorImpl<uint64_t> &Ops) const {
842 // The smallest scalable element supported by scaled SVE addressing
843 // modes are predicates, which are 2 scalable bytes in size. So the scalable
844 // byte offset must always be a multiple of 2.
845 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
846
847 // Add fixed-sized offset using existing DIExpression interface.
848 DIExpression::appendOffset(Ops, Offset.getFixed());
849
850 unsigned VG = getDwarfRegNum(AArch64::VG, true);
851 int64_t VGSized = Offset.getScalable() / 2;
852 if (VGSized > 0) {
853 Ops.push_back(dwarf::DW_OP_constu);
854 Ops.push_back(VGSized);
855 Ops.append({dwarf::DW_OP_bregx, VG, 0ULL});
856 Ops.push_back(dwarf::DW_OP_mul);
857 Ops.push_back(dwarf::DW_OP_plus);
858 } else if (VGSized < 0) {
859 Ops.push_back(dwarf::DW_OP_constu);
860 Ops.push_back(-VGSized);
861 Ops.append({dwarf::DW_OP_bregx, VG, 0ULL});
862 Ops.push_back(dwarf::DW_OP_mul);
863 Ops.push_back(dwarf::DW_OP_minus);
864 }
865 }
866
eliminateFrameIndex(MachineBasicBlock::iterator II,int SPAdj,unsigned FIOperandNum,RegScavenger * RS) const867 bool AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
868 int SPAdj, unsigned FIOperandNum,
869 RegScavenger *RS) const {
870 assert(SPAdj == 0 && "Unexpected");
871
872 MachineInstr &MI = *II;
873 MachineBasicBlock &MBB = *MI.getParent();
874 MachineFunction &MF = *MBB.getParent();
875 const MachineFrameInfo &MFI = MF.getFrameInfo();
876 const AArch64InstrInfo *TII =
877 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
878 const AArch64FrameLowering *TFI = getFrameLowering(MF);
879 int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
880 bool Tagged =
881 MI.getOperand(FIOperandNum).getTargetFlags() & AArch64II::MO_TAGGED;
882 Register FrameReg;
883
884 // Special handling of dbg_value, stackmap patchpoint statepoint instructions.
885 if (MI.getOpcode() == TargetOpcode::STACKMAP ||
886 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
887 MI.getOpcode() == TargetOpcode::STATEPOINT) {
888 StackOffset Offset =
889 TFI->resolveFrameIndexReference(MF, FrameIndex, FrameReg,
890 /*PreferFP=*/true,
891 /*ForSimm=*/false);
892 Offset += StackOffset::getFixed(MI.getOperand(FIOperandNum + 1).getImm());
893 MI.getOperand(FIOperandNum).ChangeToRegister(FrameReg, false /*isDef*/);
894 MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Offset.getFixed());
895 return false;
896 }
897
898 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) {
899 MachineOperand &FI = MI.getOperand(FIOperandNum);
900 StackOffset Offset = TFI->getNonLocalFrameIndexReference(MF, FrameIndex);
901 assert(!Offset.getScalable() &&
902 "Frame offsets with a scalable component are not supported");
903 FI.ChangeToImmediate(Offset.getFixed());
904 return false;
905 }
906
907 StackOffset Offset;
908 if (MI.getOpcode() == AArch64::TAGPstack) {
909 // TAGPstack must use the virtual frame register in its 3rd operand.
910 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
911 FrameReg = MI.getOperand(3).getReg();
912 Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) +
913 AFI->getTaggedBasePointerOffset());
914 } else if (Tagged) {
915 StackOffset SPOffset = StackOffset::getFixed(
916 MFI.getObjectOffset(FrameIndex) + (int64_t)MFI.getStackSize());
917 if (MFI.hasVarSizedObjects() ||
918 isAArch64FrameOffsetLegal(MI, SPOffset, nullptr, nullptr, nullptr) !=
919 (AArch64FrameOffsetCanUpdate | AArch64FrameOffsetIsLegal)) {
920 // Can't update to SP + offset in place. Precalculate the tagged pointer
921 // in a scratch register.
922 Offset = TFI->resolveFrameIndexReference(
923 MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);
924 Register ScratchReg =
925 MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
926 emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset,
927 TII);
928 BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AArch64::LDG), ScratchReg)
929 .addReg(ScratchReg)
930 .addReg(ScratchReg)
931 .addImm(0);
932 MI.getOperand(FIOperandNum)
933 .ChangeToRegister(ScratchReg, false, false, true);
934 return false;
935 }
936 FrameReg = AArch64::SP;
937 Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) +
938 (int64_t)MFI.getStackSize());
939 } else {
940 Offset = TFI->resolveFrameIndexReference(
941 MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);
942 }
943
944 // Modify MI as necessary to handle as much of 'Offset' as possible
945 if (rewriteAArch64FrameIndex(MI, FIOperandNum, FrameReg, Offset, TII))
946 return true;
947
948 assert((!RS || !RS->isScavengingFrameIndex(FrameIndex)) &&
949 "Emergency spill slot is out of reach");
950
951 // If we get here, the immediate doesn't fit into the instruction. We folded
952 // as much as possible above. Handle the rest, providing a register that is
953 // SP+LargeImm.
954 Register ScratchReg =
955 createScratchRegisterForInstruction(MI, FIOperandNum, TII);
956 emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset, TII);
957 return false;
958 }
959
getRegPressureLimit(const TargetRegisterClass * RC,MachineFunction & MF) const960 unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
961 MachineFunction &MF) const {
962 const AArch64FrameLowering *TFI = getFrameLowering(MF);
963
964 switch (RC->getID()) {
965 default:
966 return 0;
967 case AArch64::GPR32RegClassID:
968 case AArch64::GPR32spRegClassID:
969 case AArch64::GPR32allRegClassID:
970 case AArch64::GPR64spRegClassID:
971 case AArch64::GPR64allRegClassID:
972 case AArch64::GPR64RegClassID:
973 case AArch64::GPR32commonRegClassID:
974 case AArch64::GPR64commonRegClassID:
975 return 32 - 1 // XZR/SP
976 - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP
977 - MF.getSubtarget<AArch64Subtarget>().getNumXRegisterReserved()
978 - hasBasePointer(MF); // X19
979 case AArch64::FPR8RegClassID:
980 case AArch64::FPR16RegClassID:
981 case AArch64::FPR32RegClassID:
982 case AArch64::FPR64RegClassID:
983 case AArch64::FPR128RegClassID:
984 return 32;
985
986 case AArch64::MatrixIndexGPR32_8_11RegClassID:
987 case AArch64::MatrixIndexGPR32_12_15RegClassID:
988 return 4;
989
990 case AArch64::DDRegClassID:
991 case AArch64::DDDRegClassID:
992 case AArch64::DDDDRegClassID:
993 case AArch64::QQRegClassID:
994 case AArch64::QQQRegClassID:
995 case AArch64::QQQQRegClassID:
996 return 32;
997
998 case AArch64::FPR128_loRegClassID:
999 case AArch64::FPR64_loRegClassID:
1000 case AArch64::FPR16_loRegClassID:
1001 return 16;
1002 case AArch64::FPR128_0to7RegClassID:
1003 return 8;
1004 }
1005 }
1006
getLocalAddressRegister(const MachineFunction & MF) const1007 unsigned AArch64RegisterInfo::getLocalAddressRegister(
1008 const MachineFunction &MF) const {
1009 const auto &MFI = MF.getFrameInfo();
1010 if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects())
1011 return AArch64::SP;
1012 else if (hasStackRealignment(MF))
1013 return getBaseRegister();
1014 return getFrameRegister(MF);
1015 }
1016
1017 /// SrcRC and DstRC will be morphed into NewRC if this returns true
shouldCoalesce(MachineInstr * MI,const TargetRegisterClass * SrcRC,unsigned SubReg,const TargetRegisterClass * DstRC,unsigned DstSubReg,const TargetRegisterClass * NewRC,LiveIntervals & LIS) const1018 bool AArch64RegisterInfo::shouldCoalesce(
1019 MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg,
1020 const TargetRegisterClass *DstRC, unsigned DstSubReg,
1021 const TargetRegisterClass *NewRC, LiveIntervals &LIS) const {
1022 MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
1023
1024 if (MI->isCopy() &&
1025 ((DstRC->getID() == AArch64::GPR64RegClassID) ||
1026 (DstRC->getID() == AArch64::GPR64commonRegClassID)) &&
1027 MI->getOperand(0).getSubReg() && MI->getOperand(1).getSubReg())
1028 // Do not coalesce in the case of a 32-bit subregister copy
1029 // which implements a 32 to 64 bit zero extension
1030 // which relies on the upper 32 bits being zeroed.
1031 return false;
1032
1033 auto IsCoalescerBarrier = [](const MachineInstr &MI) {
1034 switch (MI.getOpcode()) {
1035 case AArch64::COALESCER_BARRIER_FPR16:
1036 case AArch64::COALESCER_BARRIER_FPR32:
1037 case AArch64::COALESCER_BARRIER_FPR64:
1038 case AArch64::COALESCER_BARRIER_FPR128:
1039 return true;
1040 default:
1041 return false;
1042 }
1043 };
1044
1045 // For calls that temporarily have to toggle streaming mode as part of the
1046 // call-sequence, we need to be more careful when coalescing copy instructions
1047 // so that we don't end up coalescing the NEON/FP result or argument register
1048 // with a whole Z-register, such that after coalescing the register allocator
1049 // will try to spill/reload the entire Z register.
1050 //
1051 // We do this by checking if the node has any defs/uses that are
1052 // COALESCER_BARRIER pseudos. These are 'nops' in practice, but they exist to
1053 // instruct the coalescer to avoid coalescing the copy.
1054 if (MI->isCopy() && SubReg != DstSubReg &&
1055 (AArch64::ZPRRegClass.hasSubClassEq(DstRC) ||
1056 AArch64::ZPRRegClass.hasSubClassEq(SrcRC))) {
1057 unsigned SrcReg = MI->getOperand(1).getReg();
1058 if (any_of(MRI.def_instructions(SrcReg), IsCoalescerBarrier))
1059 return false;
1060 unsigned DstReg = MI->getOperand(0).getReg();
1061 if (any_of(MRI.use_nodbg_instructions(DstReg), IsCoalescerBarrier))
1062 return false;
1063 }
1064
1065 return true;
1066 }
1067