109467b48Spatrick //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
209467b48Spatrick //
309467b48Spatrick // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
409467b48Spatrick // See https://llvm.org/LICENSE.txt for license information.
509467b48Spatrick // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
609467b48Spatrick //
709467b48Spatrick //===----------------------------------------------------------------------===//
809467b48Spatrick //
909467b48Spatrick // This file contains a printer that converts from our internal representation
1009467b48Spatrick // of machine-dependent LLVM code to the AArch64 assembly language.
1109467b48Spatrick //
1209467b48Spatrick //===----------------------------------------------------------------------===//
1309467b48Spatrick
1409467b48Spatrick #include "AArch64.h"
1509467b48Spatrick #include "AArch64MCInstLower.h"
1609467b48Spatrick #include "AArch64MachineFunctionInfo.h"
1709467b48Spatrick #include "AArch64RegisterInfo.h"
1809467b48Spatrick #include "AArch64Subtarget.h"
1909467b48Spatrick #include "AArch64TargetObjectFile.h"
2009467b48Spatrick #include "MCTargetDesc/AArch64AddressingModes.h"
2109467b48Spatrick #include "MCTargetDesc/AArch64InstPrinter.h"
2209467b48Spatrick #include "MCTargetDesc/AArch64MCExpr.h"
2309467b48Spatrick #include "MCTargetDesc/AArch64MCTargetDesc.h"
2409467b48Spatrick #include "MCTargetDesc/AArch64TargetStreamer.h"
2509467b48Spatrick #include "TargetInfo/AArch64TargetInfo.h"
2609467b48Spatrick #include "Utils/AArch64BaseInfo.h"
2709467b48Spatrick #include "llvm/ADT/SmallString.h"
2809467b48Spatrick #include "llvm/ADT/SmallVector.h"
2909467b48Spatrick #include "llvm/ADT/StringRef.h"
3009467b48Spatrick #include "llvm/ADT/Triple.h"
3109467b48Spatrick #include "llvm/ADT/Twine.h"
3209467b48Spatrick #include "llvm/BinaryFormat/COFF.h"
3309467b48Spatrick #include "llvm/BinaryFormat/ELF.h"
3409467b48Spatrick #include "llvm/CodeGen/AsmPrinter.h"
35a0747c9fSpatrick #include "llvm/CodeGen/FaultMaps.h"
3609467b48Spatrick #include "llvm/CodeGen/MachineBasicBlock.h"
3709467b48Spatrick #include "llvm/CodeGen/MachineFunction.h"
3809467b48Spatrick #include "llvm/CodeGen/MachineInstr.h"
3909467b48Spatrick #include "llvm/CodeGen/MachineJumpTableInfo.h"
4009467b48Spatrick #include "llvm/CodeGen/MachineModuleInfoImpls.h"
4109467b48Spatrick #include "llvm/CodeGen/MachineOperand.h"
4209467b48Spatrick #include "llvm/CodeGen/StackMaps.h"
4309467b48Spatrick #include "llvm/CodeGen/TargetRegisterInfo.h"
4409467b48Spatrick #include "llvm/IR/DataLayout.h"
4509467b48Spatrick #include "llvm/IR/DebugInfoMetadata.h"
4609467b48Spatrick #include "llvm/MC/MCAsmInfo.h"
4709467b48Spatrick #include "llvm/MC/MCContext.h"
4809467b48Spatrick #include "llvm/MC/MCInst.h"
4909467b48Spatrick #include "llvm/MC/MCInstBuilder.h"
5009467b48Spatrick #include "llvm/MC/MCSectionELF.h"
5109467b48Spatrick #include "llvm/MC/MCStreamer.h"
5209467b48Spatrick #include "llvm/MC/MCSymbol.h"
53*a96b3639Srobert #include "llvm/MC/TargetRegistry.h"
5409467b48Spatrick #include "llvm/Support/Casting.h"
5509467b48Spatrick #include "llvm/Support/ErrorHandling.h"
5609467b48Spatrick #include "llvm/Support/raw_ostream.h"
5709467b48Spatrick #include "llvm/Target/TargetMachine.h"
58a0747c9fSpatrick #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
5909467b48Spatrick #include <algorithm>
6009467b48Spatrick #include <cassert>
6109467b48Spatrick #include <cstdint>
6209467b48Spatrick #include <map>
6309467b48Spatrick #include <memory>
6409467b48Spatrick
6509467b48Spatrick using namespace llvm;
6609467b48Spatrick
6709467b48Spatrick #define DEBUG_TYPE "asm-printer"
6809467b48Spatrick
6909467b48Spatrick namespace {
7009467b48Spatrick
7109467b48Spatrick class AArch64AsmPrinter : public AsmPrinter {
7209467b48Spatrick AArch64MCInstLower MCInstLowering;
73a0747c9fSpatrick FaultMaps FM;
7409467b48Spatrick const AArch64Subtarget *STI;
75*a96b3639Srobert bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
7609467b48Spatrick
7709467b48Spatrick public:
AArch64AsmPrinter(TargetMachine & TM,std::unique_ptr<MCStreamer> Streamer)7809467b48Spatrick AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
7909467b48Spatrick : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
80*a96b3639Srobert FM(*this) {}
8109467b48Spatrick
getPassName() const8209467b48Spatrick StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
8309467b48Spatrick
8409467b48Spatrick /// Wrapper for MCInstLowering.lowerOperand() for the
8509467b48Spatrick /// tblgen'erated pseudo lowering.
lowerOperand(const MachineOperand & MO,MCOperand & MCOp) const8609467b48Spatrick bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
8709467b48Spatrick return MCInstLowering.lowerOperand(MO, MCOp);
8809467b48Spatrick }
8909467b48Spatrick
90097a140dSpatrick void emitStartOfAsmFile(Module &M) override;
91097a140dSpatrick void emitJumpTableInfo() override;
9209467b48Spatrick
93097a140dSpatrick void emitFunctionEntryLabel() override;
94097a140dSpatrick
95a0747c9fSpatrick void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
9609467b48Spatrick
97*a96b3639Srobert void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
98*a96b3639Srobert
9909467b48Spatrick void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
10009467b48Spatrick const MachineInstr &MI);
10109467b48Spatrick void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
10209467b48Spatrick const MachineInstr &MI);
103a0747c9fSpatrick void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
104a0747c9fSpatrick const MachineInstr &MI);
105a0747c9fSpatrick void LowerFAULTING_OP(const MachineInstr &MI);
10609467b48Spatrick
10709467b48Spatrick void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
10809467b48Spatrick void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
10909467b48Spatrick void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
11009467b48Spatrick
11109467b48Spatrick typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
11209467b48Spatrick std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
113*a96b3639Srobert void LowerKCFI_CHECK(const MachineInstr &MI);
11409467b48Spatrick void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
115a0747c9fSpatrick void emitHwasanMemaccessSymbols(Module &M);
11609467b48Spatrick
117a0747c9fSpatrick void emitSled(const MachineInstr &MI, SledKind Kind);
11809467b48Spatrick
11909467b48Spatrick /// tblgen'erated driver function for lowering simple MI->MC
12009467b48Spatrick /// pseudo instructions.
12109467b48Spatrick bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
12209467b48Spatrick const MachineInstr *MI);
12309467b48Spatrick
124097a140dSpatrick void emitInstruction(const MachineInstr *MI) override;
125097a140dSpatrick
126097a140dSpatrick void emitFunctionHeaderComment() override;
12709467b48Spatrick
getAnalysisUsage(AnalysisUsage & AU) const12809467b48Spatrick void getAnalysisUsage(AnalysisUsage &AU) const override {
12909467b48Spatrick AsmPrinter::getAnalysisUsage(AU);
13009467b48Spatrick AU.setPreservesAll();
13109467b48Spatrick }
13209467b48Spatrick
runOnMachineFunction(MachineFunction & MF)13309467b48Spatrick bool runOnMachineFunction(MachineFunction &MF) override {
13409467b48Spatrick AArch64FI = MF.getInfo<AArch64FunctionInfo>();
135*a96b3639Srobert STI = &MF.getSubtarget<AArch64Subtarget>();
13609467b48Spatrick
13709467b48Spatrick SetupMachineFunction(MF);
13809467b48Spatrick
13909467b48Spatrick if (STI->isTargetCOFF()) {
14009467b48Spatrick bool Internal = MF.getFunction().hasInternalLinkage();
14109467b48Spatrick COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
14209467b48Spatrick : COFF::IMAGE_SYM_CLASS_EXTERNAL;
14309467b48Spatrick int Type =
14409467b48Spatrick COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
14509467b48Spatrick
146*a96b3639Srobert OutStreamer->beginCOFFSymbolDef(CurrentFnSym);
147*a96b3639Srobert OutStreamer->emitCOFFSymbolStorageClass(Scl);
148*a96b3639Srobert OutStreamer->emitCOFFSymbolType(Type);
149*a96b3639Srobert OutStreamer->endCOFFSymbolDef();
15009467b48Spatrick }
15109467b48Spatrick
15209467b48Spatrick // Emit the rest of the function body.
153097a140dSpatrick emitFunctionBody();
15409467b48Spatrick
15509467b48Spatrick // Emit the XRay table for this function.
15609467b48Spatrick emitXRayTable();
15709467b48Spatrick
15809467b48Spatrick // We didn't modify anything.
15909467b48Spatrick return false;
16009467b48Spatrick }
16109467b48Spatrick
16209467b48Spatrick private:
16309467b48Spatrick void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
16409467b48Spatrick bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
16509467b48Spatrick bool printAsmRegInClass(const MachineOperand &MO,
16609467b48Spatrick const TargetRegisterClass *RC, unsigned AltName,
16709467b48Spatrick raw_ostream &O);
16809467b48Spatrick
16909467b48Spatrick bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
17009467b48Spatrick const char *ExtraCode, raw_ostream &O) override;
17109467b48Spatrick bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
17209467b48Spatrick const char *ExtraCode, raw_ostream &O) override;
17309467b48Spatrick
17409467b48Spatrick void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
17509467b48Spatrick
176097a140dSpatrick void emitFunctionBodyEnd() override;
17709467b48Spatrick
17809467b48Spatrick MCSymbol *GetCPISymbol(unsigned CPID) const override;
179097a140dSpatrick void emitEndOfAsmFile(Module &M) override;
18009467b48Spatrick
18109467b48Spatrick AArch64FunctionInfo *AArch64FI = nullptr;
18209467b48Spatrick
18309467b48Spatrick /// Emit the LOHs contained in AArch64FI.
184a0747c9fSpatrick void emitLOHs();
18509467b48Spatrick
18609467b48Spatrick /// Emit instruction to set float register to zero.
187a0747c9fSpatrick void emitFMov0(const MachineInstr &MI);
18809467b48Spatrick
18909467b48Spatrick using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
19009467b48Spatrick
19109467b48Spatrick MInstToMCSymbol LOHInstToLabel;
192*a96b3639Srobert
shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const193*a96b3639Srobert bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
194*a96b3639Srobert return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
195*a96b3639Srobert }
19609467b48Spatrick };
19709467b48Spatrick
19809467b48Spatrick } // end anonymous namespace
19909467b48Spatrick
emitStartOfAsmFile(Module & M)200097a140dSpatrick void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
201a0747c9fSpatrick const Triple &TT = TM.getTargetTriple();
202a0747c9fSpatrick
203a0747c9fSpatrick if (TT.isOSBinFormatCOFF()) {
204*a96b3639Srobert // Emit an absolute @feat.00 symbol
205a0747c9fSpatrick MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00"));
206*a96b3639Srobert OutStreamer->beginCOFFSymbolDef(S);
207*a96b3639Srobert OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC);
208*a96b3639Srobert OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL);
209*a96b3639Srobert OutStreamer->endCOFFSymbolDef();
210*a96b3639Srobert int64_t Feat00Value = 0;
211a0747c9fSpatrick
212a0747c9fSpatrick if (M.getModuleFlag("cfguard")) {
213*a96b3639Srobert // Object is CFG-aware.
214*a96b3639Srobert Feat00Value |= COFF::Feat00Flags::GuardCF;
215a0747c9fSpatrick }
216a0747c9fSpatrick
217a0747c9fSpatrick if (M.getModuleFlag("ehcontguard")) {
218*a96b3639Srobert // Object also has EHCont.
219*a96b3639Srobert Feat00Value |= COFF::Feat00Flags::GuardEHCont;
220*a96b3639Srobert }
221*a96b3639Srobert
222*a96b3639Srobert if (M.getModuleFlag("ms-kernel")) {
223*a96b3639Srobert // Object is compiled with /kernel.
224*a96b3639Srobert Feat00Value |= COFF::Feat00Flags::Kernel;
225a0747c9fSpatrick }
226a0747c9fSpatrick
227a0747c9fSpatrick OutStreamer->emitSymbolAttribute(S, MCSA_Global);
228a0747c9fSpatrick OutStreamer->emitAssignment(
229*a96b3639Srobert S, MCConstantExpr::create(Feat00Value, MMI->getContext()));
230a0747c9fSpatrick }
231a0747c9fSpatrick
232a0747c9fSpatrick if (!TT.isOSBinFormatELF())
23309467b48Spatrick return;
23409467b48Spatrick
23509467b48Spatrick // Assemble feature flags that may require creation of a note section.
236a0747c9fSpatrick unsigned Flags = 0;
237a0747c9fSpatrick if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
238a0747c9fSpatrick M.getModuleFlag("branch-target-enforcement")))
239a0747c9fSpatrick if (BTE->getZExtValue())
240a0747c9fSpatrick Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
24109467b48Spatrick
242a0747c9fSpatrick if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
243a0747c9fSpatrick M.getModuleFlag("sign-return-address")))
244a0747c9fSpatrick if (Sign->getZExtValue())
245a0747c9fSpatrick Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
24609467b48Spatrick
24709467b48Spatrick if (Flags == 0)
24809467b48Spatrick return;
24909467b48Spatrick
25009467b48Spatrick // Emit a .note.gnu.property section with the flags.
251*a96b3639Srobert auto *TS =
252*a96b3639Srobert static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
253a0747c9fSpatrick TS->emitNoteSection(Flags);
25409467b48Spatrick }
25509467b48Spatrick
emitFunctionHeaderComment()256097a140dSpatrick void AArch64AsmPrinter::emitFunctionHeaderComment() {
257097a140dSpatrick const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
258*a96b3639Srobert std::optional<std::string> OutlinerString = FI->getOutliningStyle();
259*a96b3639Srobert if (OutlinerString != std::nullopt)
260*a96b3639Srobert OutStreamer->getCommentOS() << ' ' << OutlinerString;
261097a140dSpatrick }
262097a140dSpatrick
LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr & MI)26309467b48Spatrick void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
26409467b48Spatrick {
26509467b48Spatrick const Function &F = MF->getFunction();
26609467b48Spatrick if (F.hasFnAttribute("patchable-function-entry")) {
26709467b48Spatrick unsigned Num;
26809467b48Spatrick if (F.getFnAttribute("patchable-function-entry")
26909467b48Spatrick .getValueAsString()
27009467b48Spatrick .getAsInteger(10, Num))
27109467b48Spatrick return;
27209467b48Spatrick emitNops(Num);
27309467b48Spatrick return;
27409467b48Spatrick }
27509467b48Spatrick
276a0747c9fSpatrick emitSled(MI, SledKind::FUNCTION_ENTER);
27709467b48Spatrick }
27809467b48Spatrick
LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr & MI)279a0747c9fSpatrick void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
280a0747c9fSpatrick emitSled(MI, SledKind::FUNCTION_EXIT);
28109467b48Spatrick }
28209467b48Spatrick
LowerPATCHABLE_TAIL_CALL(const MachineInstr & MI)283a0747c9fSpatrick void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
284a0747c9fSpatrick emitSled(MI, SledKind::TAIL_CALL);
28509467b48Spatrick }
28609467b48Spatrick
emitSled(const MachineInstr & MI,SledKind Kind)287a0747c9fSpatrick void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
28809467b48Spatrick static const int8_t NoopsInSledCount = 7;
28909467b48Spatrick // We want to emit the following pattern:
29009467b48Spatrick //
29109467b48Spatrick // .Lxray_sled_N:
29209467b48Spatrick // ALIGN
29309467b48Spatrick // B #32
29409467b48Spatrick // ; 7 NOP instructions (28 bytes)
29509467b48Spatrick // .tmpN
29609467b48Spatrick //
29709467b48Spatrick // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
29809467b48Spatrick // over the full 32 bytes (8 instructions) with the following pattern:
29909467b48Spatrick //
30009467b48Spatrick // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
30109467b48Spatrick // LDR W0, #12 ; W0 := function ID
30209467b48Spatrick // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
30309467b48Spatrick // BLR X16 ; call the tracing trampoline
30409467b48Spatrick // ;DATA: 32 bits of function ID
30509467b48Spatrick // ;DATA: lower 32 bits of the address of the trampoline
30609467b48Spatrick // ;DATA: higher 32 bits of the address of the trampoline
30709467b48Spatrick // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
30809467b48Spatrick //
309*a96b3639Srobert OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo());
31009467b48Spatrick auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
311097a140dSpatrick OutStreamer->emitLabel(CurSled);
31209467b48Spatrick auto Target = OutContext.createTempSymbol();
31309467b48Spatrick
31409467b48Spatrick // Emit "B #32" instruction, which jumps over the next 28 bytes.
31509467b48Spatrick // The operand has to be the number of 4-byte instructions to jump over,
31609467b48Spatrick // including the current instruction.
31709467b48Spatrick EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
31809467b48Spatrick
31909467b48Spatrick for (int8_t I = 0; I < NoopsInSledCount; I++)
32009467b48Spatrick EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
32109467b48Spatrick
322097a140dSpatrick OutStreamer->emitLabel(Target);
323097a140dSpatrick recordSled(CurSled, MI, Kind, 2);
32409467b48Spatrick }
32509467b48Spatrick
LowerKCFI_CHECK(const MachineInstr & MI)326*a96b3639Srobert void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
327*a96b3639Srobert Register AddrReg = MI.getOperand(0).getReg();
328*a96b3639Srobert assert(std::next(MI.getIterator())->isCall() &&
329*a96b3639Srobert "KCFI_CHECK not followed by a call instruction");
330*a96b3639Srobert assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
331*a96b3639Srobert "KCFI_CHECK call target doesn't match call operand");
332*a96b3639Srobert
333*a96b3639Srobert // Default to using the intra-procedure-call temporary registers for
334*a96b3639Srobert // comparing the hashes.
335*a96b3639Srobert unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
336*a96b3639Srobert if (AddrReg == AArch64::XZR) {
337*a96b3639Srobert // Checking XZR makes no sense. Instead of emitting a load, zero
338*a96b3639Srobert // ScratchRegs[0] and use it for the ESR AddrIndex below.
339*a96b3639Srobert AddrReg = getXRegFromWReg(ScratchRegs[0]);
340*a96b3639Srobert EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
341*a96b3639Srobert .addReg(AddrReg)
342*a96b3639Srobert .addReg(AArch64::XZR)
343*a96b3639Srobert .addReg(AArch64::XZR)
344*a96b3639Srobert .addImm(0));
345*a96b3639Srobert } else {
346*a96b3639Srobert // If one of the scratch registers is used for the call target (e.g.
347*a96b3639Srobert // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
348*a96b3639Srobert // temporary register instead (in this case, AArch64::W9) as the check
349*a96b3639Srobert // is immediately followed by the call instruction.
350*a96b3639Srobert for (auto &Reg : ScratchRegs) {
351*a96b3639Srobert if (Reg == getWRegFromXReg(AddrReg)) {
352*a96b3639Srobert Reg = AArch64::W9;
353*a96b3639Srobert break;
354*a96b3639Srobert }
355*a96b3639Srobert }
356*a96b3639Srobert assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
357*a96b3639Srobert "Invalid scratch registers for KCFI_CHECK");
358*a96b3639Srobert
359*a96b3639Srobert // Adjust the offset for patchable-function-prefix. This assumes that
360*a96b3639Srobert // patchable-function-prefix is the same for all functions.
361*a96b3639Srobert int64_t PrefixNops = 0;
362*a96b3639Srobert (void)MI.getMF()
363*a96b3639Srobert ->getFunction()
364*a96b3639Srobert .getFnAttribute("patchable-function-prefix")
365*a96b3639Srobert .getValueAsString()
366*a96b3639Srobert .getAsInteger(10, PrefixNops);
367*a96b3639Srobert
368*a96b3639Srobert // Load the target function type hash.
369*a96b3639Srobert EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi)
370*a96b3639Srobert .addReg(ScratchRegs[0])
371*a96b3639Srobert .addReg(AddrReg)
372*a96b3639Srobert .addImm(-(PrefixNops * 4 + 4)));
373*a96b3639Srobert }
374*a96b3639Srobert
375*a96b3639Srobert // Load the expected type hash.
376*a96b3639Srobert const int64_t Type = MI.getOperand(1).getImm();
377*a96b3639Srobert EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
378*a96b3639Srobert .addReg(ScratchRegs[1])
379*a96b3639Srobert .addReg(ScratchRegs[1])
380*a96b3639Srobert .addImm(Type & 0xFFFF)
381*a96b3639Srobert .addImm(0));
382*a96b3639Srobert EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
383*a96b3639Srobert .addReg(ScratchRegs[1])
384*a96b3639Srobert .addReg(ScratchRegs[1])
385*a96b3639Srobert .addImm((Type >> 16) & 0xFFFF)
386*a96b3639Srobert .addImm(16));
387*a96b3639Srobert
388*a96b3639Srobert // Compare the hashes and trap if there's a mismatch.
389*a96b3639Srobert EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs)
390*a96b3639Srobert .addReg(AArch64::WZR)
391*a96b3639Srobert .addReg(ScratchRegs[0])
392*a96b3639Srobert .addReg(ScratchRegs[1])
393*a96b3639Srobert .addImm(0));
394*a96b3639Srobert
395*a96b3639Srobert MCSymbol *Pass = OutContext.createTempSymbol();
396*a96b3639Srobert EmitToStreamer(*OutStreamer,
397*a96b3639Srobert MCInstBuilder(AArch64::Bcc)
398*a96b3639Srobert .addImm(AArch64CC::EQ)
399*a96b3639Srobert .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
400*a96b3639Srobert
401*a96b3639Srobert // The base ESR is 0x8000 and the register information is encoded in bits
402*a96b3639Srobert // 0-9 as follows:
403*a96b3639Srobert // - 0-4: n, where the register Xn contains the target address
404*a96b3639Srobert // - 5-9: m, where the register Wm contains the expected type hash
405*a96b3639Srobert // Where n, m are in [0, 30].
406*a96b3639Srobert unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
407*a96b3639Srobert unsigned AddrIndex;
408*a96b3639Srobert switch (AddrReg) {
409*a96b3639Srobert default:
410*a96b3639Srobert AddrIndex = AddrReg - AArch64::X0;
411*a96b3639Srobert break;
412*a96b3639Srobert case AArch64::FP:
413*a96b3639Srobert AddrIndex = 29;
414*a96b3639Srobert break;
415*a96b3639Srobert case AArch64::LR:
416*a96b3639Srobert AddrIndex = 30;
417*a96b3639Srobert break;
418*a96b3639Srobert }
419*a96b3639Srobert
420*a96b3639Srobert assert(AddrIndex < 31 && TypeIndex < 31);
421*a96b3639Srobert
422*a96b3639Srobert unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
423*a96b3639Srobert EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR));
424*a96b3639Srobert OutStreamer->emitLabel(Pass);
425*a96b3639Srobert }
426*a96b3639Srobert
LowerHWASAN_CHECK_MEMACCESS(const MachineInstr & MI)42709467b48Spatrick void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
42809467b48Spatrick Register Reg = MI.getOperand(0).getReg();
42909467b48Spatrick bool IsShort =
43009467b48Spatrick MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
43109467b48Spatrick uint32_t AccessInfo = MI.getOperand(1).getImm();
43209467b48Spatrick MCSymbol *&Sym =
43309467b48Spatrick HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
43409467b48Spatrick if (!Sym) {
43509467b48Spatrick // FIXME: Make this work on non-ELF.
43609467b48Spatrick if (!TM.getTargetTriple().isOSBinFormatELF())
43709467b48Spatrick report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
43809467b48Spatrick
43909467b48Spatrick std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
44009467b48Spatrick utostr(AccessInfo);
44109467b48Spatrick if (IsShort)
442a0747c9fSpatrick SymName += "_short_v2";
44309467b48Spatrick Sym = OutContext.getOrCreateSymbol(SymName);
44409467b48Spatrick }
44509467b48Spatrick
44609467b48Spatrick EmitToStreamer(*OutStreamer,
44709467b48Spatrick MCInstBuilder(AArch64::BL)
44809467b48Spatrick .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
44909467b48Spatrick }
45009467b48Spatrick
emitHwasanMemaccessSymbols(Module & M)451a0747c9fSpatrick void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
45209467b48Spatrick if (HwasanMemaccessSymbols.empty())
45309467b48Spatrick return;
45409467b48Spatrick
45509467b48Spatrick const Triple &TT = TM.getTargetTriple();
45609467b48Spatrick assert(TT.isOSBinFormatELF());
45709467b48Spatrick std::unique_ptr<MCSubtargetInfo> STI(
45809467b48Spatrick TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
459a0747c9fSpatrick assert(STI && "Unable to create subtarget info");
46009467b48Spatrick
46109467b48Spatrick MCSymbol *HwasanTagMismatchV1Sym =
46209467b48Spatrick OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
46309467b48Spatrick MCSymbol *HwasanTagMismatchV2Sym =
46409467b48Spatrick OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
46509467b48Spatrick
46609467b48Spatrick const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
46709467b48Spatrick MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
46809467b48Spatrick const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
46909467b48Spatrick MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
47009467b48Spatrick
47109467b48Spatrick for (auto &P : HwasanMemaccessSymbols) {
47209467b48Spatrick unsigned Reg = std::get<0>(P.first);
47309467b48Spatrick bool IsShort = std::get<1>(P.first);
47409467b48Spatrick uint32_t AccessInfo = std::get<2>(P.first);
47509467b48Spatrick const MCSymbolRefExpr *HwasanTagMismatchRef =
47609467b48Spatrick IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
47709467b48Spatrick MCSymbol *Sym = P.second;
47809467b48Spatrick
479a0747c9fSpatrick bool HasMatchAllTag =
480a0747c9fSpatrick (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
481a0747c9fSpatrick uint8_t MatchAllTag =
482a0747c9fSpatrick (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
483a0747c9fSpatrick unsigned Size =
484a0747c9fSpatrick 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
485a0747c9fSpatrick bool CompileKernel =
486a0747c9fSpatrick (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
487a0747c9fSpatrick
488*a96b3639Srobert OutStreamer->switchSection(OutContext.getELFSection(
48909467b48Spatrick ".text.hot", ELF::SHT_PROGBITS,
490*a96b3639Srobert ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(),
491*a96b3639Srobert /*IsComdat=*/true));
49209467b48Spatrick
493097a140dSpatrick OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
494097a140dSpatrick OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
495097a140dSpatrick OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
496097a140dSpatrick OutStreamer->emitLabel(Sym);
49709467b48Spatrick
498a0747c9fSpatrick OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri)
49909467b48Spatrick .addReg(AArch64::X16)
50009467b48Spatrick .addReg(Reg)
50109467b48Spatrick .addImm(4)
50209467b48Spatrick .addImm(55),
50309467b48Spatrick *STI);
504a0747c9fSpatrick OutStreamer->emitInstruction(
505a0747c9fSpatrick MCInstBuilder(AArch64::LDRBBroX)
50609467b48Spatrick .addReg(AArch64::W16)
507a0747c9fSpatrick .addReg(IsShort ? AArch64::X20 : AArch64::X9)
50809467b48Spatrick .addReg(AArch64::X16)
50909467b48Spatrick .addImm(0)
51009467b48Spatrick .addImm(0),
51109467b48Spatrick *STI);
512097a140dSpatrick OutStreamer->emitInstruction(
51309467b48Spatrick MCInstBuilder(AArch64::SUBSXrs)
51409467b48Spatrick .addReg(AArch64::XZR)
51509467b48Spatrick .addReg(AArch64::X16)
51609467b48Spatrick .addReg(Reg)
51709467b48Spatrick .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
51809467b48Spatrick *STI);
51909467b48Spatrick MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
520097a140dSpatrick OutStreamer->emitInstruction(
52109467b48Spatrick MCInstBuilder(AArch64::Bcc)
52209467b48Spatrick .addImm(AArch64CC::NE)
52309467b48Spatrick .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
52409467b48Spatrick OutContext)),
52509467b48Spatrick *STI);
52609467b48Spatrick MCSymbol *ReturnSym = OutContext.createTempSymbol();
527097a140dSpatrick OutStreamer->emitLabel(ReturnSym);
528097a140dSpatrick OutStreamer->emitInstruction(
52909467b48Spatrick MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
530097a140dSpatrick OutStreamer->emitLabel(HandleMismatchOrPartialSym);
53109467b48Spatrick
532a0747c9fSpatrick if (HasMatchAllTag) {
533a0747c9fSpatrick OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
534a0747c9fSpatrick .addReg(AArch64::X16)
535a0747c9fSpatrick .addReg(Reg)
536a0747c9fSpatrick .addImm(56)
537a0747c9fSpatrick .addImm(63),
538a0747c9fSpatrick *STI);
539a0747c9fSpatrick OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
540a0747c9fSpatrick .addReg(AArch64::XZR)
541a0747c9fSpatrick .addReg(AArch64::X16)
542a0747c9fSpatrick .addImm(MatchAllTag)
543a0747c9fSpatrick .addImm(0),
544a0747c9fSpatrick *STI);
545a0747c9fSpatrick OutStreamer->emitInstruction(
546a0747c9fSpatrick MCInstBuilder(AArch64::Bcc)
547a0747c9fSpatrick .addImm(AArch64CC::EQ)
548a0747c9fSpatrick .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
549a0747c9fSpatrick *STI);
550a0747c9fSpatrick }
551a0747c9fSpatrick
55209467b48Spatrick if (IsShort) {
553097a140dSpatrick OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
55409467b48Spatrick .addReg(AArch64::WZR)
55509467b48Spatrick .addReg(AArch64::W16)
55609467b48Spatrick .addImm(15)
55709467b48Spatrick .addImm(0),
55809467b48Spatrick *STI);
55909467b48Spatrick MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
560097a140dSpatrick OutStreamer->emitInstruction(
56109467b48Spatrick MCInstBuilder(AArch64::Bcc)
56209467b48Spatrick .addImm(AArch64CC::HI)
56309467b48Spatrick .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
56409467b48Spatrick *STI);
56509467b48Spatrick
566097a140dSpatrick OutStreamer->emitInstruction(
56709467b48Spatrick MCInstBuilder(AArch64::ANDXri)
56809467b48Spatrick .addReg(AArch64::X17)
56909467b48Spatrick .addReg(Reg)
57009467b48Spatrick .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
57109467b48Spatrick *STI);
57209467b48Spatrick if (Size != 1)
573097a140dSpatrick OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
57409467b48Spatrick .addReg(AArch64::X17)
57509467b48Spatrick .addReg(AArch64::X17)
57609467b48Spatrick .addImm(Size - 1)
57709467b48Spatrick .addImm(0),
57809467b48Spatrick *STI);
579097a140dSpatrick OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
58009467b48Spatrick .addReg(AArch64::WZR)
58109467b48Spatrick .addReg(AArch64::W16)
58209467b48Spatrick .addReg(AArch64::W17)
58309467b48Spatrick .addImm(0),
58409467b48Spatrick *STI);
585097a140dSpatrick OutStreamer->emitInstruction(
58609467b48Spatrick MCInstBuilder(AArch64::Bcc)
58709467b48Spatrick .addImm(AArch64CC::LS)
58809467b48Spatrick .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
58909467b48Spatrick *STI);
59009467b48Spatrick
591097a140dSpatrick OutStreamer->emitInstruction(
59209467b48Spatrick MCInstBuilder(AArch64::ORRXri)
59309467b48Spatrick .addReg(AArch64::X16)
59409467b48Spatrick .addReg(Reg)
59509467b48Spatrick .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
59609467b48Spatrick *STI);
597097a140dSpatrick OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
59809467b48Spatrick .addReg(AArch64::W16)
59909467b48Spatrick .addReg(AArch64::X16)
60009467b48Spatrick .addImm(0),
60109467b48Spatrick *STI);
602097a140dSpatrick OutStreamer->emitInstruction(
60309467b48Spatrick MCInstBuilder(AArch64::SUBSXrs)
60409467b48Spatrick .addReg(AArch64::XZR)
60509467b48Spatrick .addReg(AArch64::X16)
60609467b48Spatrick .addReg(Reg)
60709467b48Spatrick .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
60809467b48Spatrick *STI);
609097a140dSpatrick OutStreamer->emitInstruction(
61009467b48Spatrick MCInstBuilder(AArch64::Bcc)
61109467b48Spatrick .addImm(AArch64CC::EQ)
61209467b48Spatrick .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
61309467b48Spatrick *STI);
61409467b48Spatrick
615097a140dSpatrick OutStreamer->emitLabel(HandleMismatchSym);
61609467b48Spatrick }
61709467b48Spatrick
618097a140dSpatrick OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
61909467b48Spatrick .addReg(AArch64::SP)
62009467b48Spatrick .addReg(AArch64::X0)
62109467b48Spatrick .addReg(AArch64::X1)
62209467b48Spatrick .addReg(AArch64::SP)
62309467b48Spatrick .addImm(-32),
62409467b48Spatrick *STI);
625097a140dSpatrick OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
62609467b48Spatrick .addReg(AArch64::FP)
62709467b48Spatrick .addReg(AArch64::LR)
62809467b48Spatrick .addReg(AArch64::SP)
62909467b48Spatrick .addImm(29),
63009467b48Spatrick *STI);
63109467b48Spatrick
63209467b48Spatrick if (Reg != AArch64::X0)
633097a140dSpatrick OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
63409467b48Spatrick .addReg(AArch64::X0)
63509467b48Spatrick .addReg(AArch64::XZR)
63609467b48Spatrick .addReg(Reg)
63709467b48Spatrick .addImm(0),
63809467b48Spatrick *STI);
639a0747c9fSpatrick OutStreamer->emitInstruction(
640a0747c9fSpatrick MCInstBuilder(AArch64::MOVZXi)
64109467b48Spatrick .addReg(AArch64::X1)
642a0747c9fSpatrick .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask)
64309467b48Spatrick .addImm(0),
64409467b48Spatrick *STI);
64509467b48Spatrick
646a0747c9fSpatrick if (CompileKernel) {
647a0747c9fSpatrick // The Linux kernel's dynamic loader doesn't support GOT relative
648a0747c9fSpatrick // relocations, but it doesn't support late binding either, so just call
649a0747c9fSpatrick // the function directly.
650a0747c9fSpatrick OutStreamer->emitInstruction(
651a0747c9fSpatrick MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
652a0747c9fSpatrick } else {
65309467b48Spatrick // Intentionally load the GOT entry and branch to it, rather than possibly
654a0747c9fSpatrick // late binding the function, which may clobber the registers before we
655a0747c9fSpatrick // have a chance to save them.
656097a140dSpatrick OutStreamer->emitInstruction(
65709467b48Spatrick MCInstBuilder(AArch64::ADRP)
65809467b48Spatrick .addReg(AArch64::X16)
65909467b48Spatrick .addExpr(AArch64MCExpr::create(
66009467b48Spatrick HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
66109467b48Spatrick OutContext)),
66209467b48Spatrick *STI);
663097a140dSpatrick OutStreamer->emitInstruction(
66409467b48Spatrick MCInstBuilder(AArch64::LDRXui)
66509467b48Spatrick .addReg(AArch64::X16)
66609467b48Spatrick .addReg(AArch64::X16)
66709467b48Spatrick .addExpr(AArch64MCExpr::create(
66809467b48Spatrick HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
66909467b48Spatrick OutContext)),
67009467b48Spatrick *STI);
671097a140dSpatrick OutStreamer->emitInstruction(
67209467b48Spatrick MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
67309467b48Spatrick }
67409467b48Spatrick }
675a0747c9fSpatrick }
67609467b48Spatrick
emitEndOfAsmFile(Module & M)677097a140dSpatrick void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
678a0747c9fSpatrick emitHwasanMemaccessSymbols(M);
67909467b48Spatrick
68009467b48Spatrick const Triple &TT = TM.getTargetTriple();
68109467b48Spatrick if (TT.isOSBinFormatMachO()) {
68209467b48Spatrick // Funny Darwin hack: This flag tells the linker that no global symbols
68309467b48Spatrick // contain code that falls through to other global symbols (e.g. the obvious
68409467b48Spatrick // implementation of multiple entry points). If this doesn't occur, the
68509467b48Spatrick // linker can safely perform dead code stripping. Since LLVM never
68609467b48Spatrick // generates code that does this, it is always safe to set.
687097a140dSpatrick OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
68809467b48Spatrick }
689a0747c9fSpatrick
690a0747c9fSpatrick // Emit stack and fault map information.
691a0747c9fSpatrick FM.serializeToFaultMapSection();
692a0747c9fSpatrick
69309467b48Spatrick }
69409467b48Spatrick
emitLOHs()695a0747c9fSpatrick void AArch64AsmPrinter::emitLOHs() {
69609467b48Spatrick SmallVector<MCSymbol *, 3> MCArgs;
69709467b48Spatrick
69809467b48Spatrick for (const auto &D : AArch64FI->getLOHContainer()) {
69909467b48Spatrick for (const MachineInstr *MI : D.getArgs()) {
70009467b48Spatrick MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
70109467b48Spatrick assert(LabelIt != LOHInstToLabel.end() &&
70209467b48Spatrick "Label hasn't been inserted for LOH related instruction");
70309467b48Spatrick MCArgs.push_back(LabelIt->second);
70409467b48Spatrick }
705097a140dSpatrick OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
70609467b48Spatrick MCArgs.clear();
70709467b48Spatrick }
70809467b48Spatrick }
70909467b48Spatrick
emitFunctionBodyEnd()710097a140dSpatrick void AArch64AsmPrinter::emitFunctionBodyEnd() {
71109467b48Spatrick if (!AArch64FI->getLOHRelated().empty())
712a0747c9fSpatrick emitLOHs();
71309467b48Spatrick }
71409467b48Spatrick
71509467b48Spatrick /// GetCPISymbol - Return the symbol for the specified constant pool entry.
GetCPISymbol(unsigned CPID) const71609467b48Spatrick MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
71709467b48Spatrick // Darwin uses a linker-private symbol name for constant-pools (to
71809467b48Spatrick // avoid addends on the relocation?), ELF has no such concept and
71909467b48Spatrick // uses a normal private symbol.
72009467b48Spatrick if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
72109467b48Spatrick return OutContext.getOrCreateSymbol(
72209467b48Spatrick Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
72309467b48Spatrick Twine(getFunctionNumber()) + "_" + Twine(CPID));
72409467b48Spatrick
72509467b48Spatrick return AsmPrinter::GetCPISymbol(CPID);
72609467b48Spatrick }
72709467b48Spatrick
printOperand(const MachineInstr * MI,unsigned OpNum,raw_ostream & O)72809467b48Spatrick void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
72909467b48Spatrick raw_ostream &O) {
73009467b48Spatrick const MachineOperand &MO = MI->getOperand(OpNum);
73109467b48Spatrick switch (MO.getType()) {
73209467b48Spatrick default:
73309467b48Spatrick llvm_unreachable("<unknown operand type>");
73409467b48Spatrick case MachineOperand::MO_Register: {
73509467b48Spatrick Register Reg = MO.getReg();
736*a96b3639Srobert assert(Reg.isPhysical());
73709467b48Spatrick assert(!MO.getSubReg() && "Subregs should be eliminated!");
73809467b48Spatrick O << AArch64InstPrinter::getRegisterName(Reg);
73909467b48Spatrick break;
74009467b48Spatrick }
74109467b48Spatrick case MachineOperand::MO_Immediate: {
74209467b48Spatrick O << MO.getImm();
74309467b48Spatrick break;
74409467b48Spatrick }
74509467b48Spatrick case MachineOperand::MO_GlobalAddress: {
74609467b48Spatrick PrintSymbolOperand(MO, O);
74709467b48Spatrick break;
74809467b48Spatrick }
74909467b48Spatrick case MachineOperand::MO_BlockAddress: {
75009467b48Spatrick MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
75109467b48Spatrick Sym->print(O, MAI);
75209467b48Spatrick break;
75309467b48Spatrick }
75409467b48Spatrick }
75509467b48Spatrick }
75609467b48Spatrick
printAsmMRegister(const MachineOperand & MO,char Mode,raw_ostream & O)75709467b48Spatrick bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
75809467b48Spatrick raw_ostream &O) {
75909467b48Spatrick Register Reg = MO.getReg();
76009467b48Spatrick switch (Mode) {
76109467b48Spatrick default:
76209467b48Spatrick return true; // Unknown mode.
76309467b48Spatrick case 'w':
76409467b48Spatrick Reg = getWRegFromXReg(Reg);
76509467b48Spatrick break;
76609467b48Spatrick case 'x':
76709467b48Spatrick Reg = getXRegFromWReg(Reg);
76809467b48Spatrick break;
769a0747c9fSpatrick case 't':
770a0747c9fSpatrick Reg = getXRegFromXRegTuple(Reg);
771a0747c9fSpatrick break;
77209467b48Spatrick }
77309467b48Spatrick
77409467b48Spatrick O << AArch64InstPrinter::getRegisterName(Reg);
77509467b48Spatrick return false;
77609467b48Spatrick }
77709467b48Spatrick
77809467b48Spatrick // Prints the register in MO using class RC using the offset in the
77909467b48Spatrick // new register class. This should not be used for cross class
78009467b48Spatrick // printing.
printAsmRegInClass(const MachineOperand & MO,const TargetRegisterClass * RC,unsigned AltName,raw_ostream & O)78109467b48Spatrick bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
78209467b48Spatrick const TargetRegisterClass *RC,
78309467b48Spatrick unsigned AltName, raw_ostream &O) {
78409467b48Spatrick assert(MO.isReg() && "Should only get here with a register!");
78509467b48Spatrick const TargetRegisterInfo *RI = STI->getRegisterInfo();
78609467b48Spatrick Register Reg = MO.getReg();
78709467b48Spatrick unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
788a0747c9fSpatrick if (!RI->regsOverlap(RegToPrint, Reg))
789a0747c9fSpatrick return true;
79009467b48Spatrick O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
79109467b48Spatrick return false;
79209467b48Spatrick }
79309467b48Spatrick
PrintAsmOperand(const MachineInstr * MI,unsigned OpNum,const char * ExtraCode,raw_ostream & O)79409467b48Spatrick bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
79509467b48Spatrick const char *ExtraCode, raw_ostream &O) {
79609467b48Spatrick const MachineOperand &MO = MI->getOperand(OpNum);
79709467b48Spatrick
79809467b48Spatrick // First try the generic code, which knows about modifiers like 'c' and 'n'.
79909467b48Spatrick if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
80009467b48Spatrick return false;
80109467b48Spatrick
80209467b48Spatrick // Does this asm operand have a single letter operand modifier?
80309467b48Spatrick if (ExtraCode && ExtraCode[0]) {
80409467b48Spatrick if (ExtraCode[1] != 0)
80509467b48Spatrick return true; // Unknown modifier.
80609467b48Spatrick
80709467b48Spatrick switch (ExtraCode[0]) {
80809467b48Spatrick default:
80909467b48Spatrick return true; // Unknown modifier.
81009467b48Spatrick case 'w': // Print W register
81109467b48Spatrick case 'x': // Print X register
81209467b48Spatrick if (MO.isReg())
81309467b48Spatrick return printAsmMRegister(MO, ExtraCode[0], O);
81409467b48Spatrick if (MO.isImm() && MO.getImm() == 0) {
81509467b48Spatrick unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
81609467b48Spatrick O << AArch64InstPrinter::getRegisterName(Reg);
81709467b48Spatrick return false;
81809467b48Spatrick }
81909467b48Spatrick printOperand(MI, OpNum, O);
82009467b48Spatrick return false;
82109467b48Spatrick case 'b': // Print B register.
82209467b48Spatrick case 'h': // Print H register.
82309467b48Spatrick case 's': // Print S register.
82409467b48Spatrick case 'd': // Print D register.
82509467b48Spatrick case 'q': // Print Q register.
82609467b48Spatrick case 'z': // Print Z register.
82709467b48Spatrick if (MO.isReg()) {
82809467b48Spatrick const TargetRegisterClass *RC;
82909467b48Spatrick switch (ExtraCode[0]) {
83009467b48Spatrick case 'b':
83109467b48Spatrick RC = &AArch64::FPR8RegClass;
83209467b48Spatrick break;
83309467b48Spatrick case 'h':
83409467b48Spatrick RC = &AArch64::FPR16RegClass;
83509467b48Spatrick break;
83609467b48Spatrick case 's':
83709467b48Spatrick RC = &AArch64::FPR32RegClass;
83809467b48Spatrick break;
83909467b48Spatrick case 'd':
84009467b48Spatrick RC = &AArch64::FPR64RegClass;
84109467b48Spatrick break;
84209467b48Spatrick case 'q':
84309467b48Spatrick RC = &AArch64::FPR128RegClass;
84409467b48Spatrick break;
84509467b48Spatrick case 'z':
84609467b48Spatrick RC = &AArch64::ZPRRegClass;
84709467b48Spatrick break;
84809467b48Spatrick default:
84909467b48Spatrick return true;
85009467b48Spatrick }
85109467b48Spatrick return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
85209467b48Spatrick }
85309467b48Spatrick printOperand(MI, OpNum, O);
85409467b48Spatrick return false;
85509467b48Spatrick }
85609467b48Spatrick }
85709467b48Spatrick
85809467b48Spatrick // According to ARM, we should emit x and v registers unless we have a
85909467b48Spatrick // modifier.
86009467b48Spatrick if (MO.isReg()) {
86109467b48Spatrick Register Reg = MO.getReg();
86209467b48Spatrick
86309467b48Spatrick // If this is a w or x register, print an x register.
86409467b48Spatrick if (AArch64::GPR32allRegClass.contains(Reg) ||
86509467b48Spatrick AArch64::GPR64allRegClass.contains(Reg))
86609467b48Spatrick return printAsmMRegister(MO, 'x', O);
86709467b48Spatrick
868a0747c9fSpatrick // If this is an x register tuple, print an x register.
869a0747c9fSpatrick if (AArch64::GPR64x8ClassRegClass.contains(Reg))
870a0747c9fSpatrick return printAsmMRegister(MO, 't', O);
871a0747c9fSpatrick
87209467b48Spatrick unsigned AltName = AArch64::NoRegAltName;
87309467b48Spatrick const TargetRegisterClass *RegClass;
87409467b48Spatrick if (AArch64::ZPRRegClass.contains(Reg)) {
87509467b48Spatrick RegClass = &AArch64::ZPRRegClass;
87609467b48Spatrick } else if (AArch64::PPRRegClass.contains(Reg)) {
87709467b48Spatrick RegClass = &AArch64::PPRRegClass;
87809467b48Spatrick } else {
87909467b48Spatrick RegClass = &AArch64::FPR128RegClass;
88009467b48Spatrick AltName = AArch64::vreg;
88109467b48Spatrick }
88209467b48Spatrick
88309467b48Spatrick // If this is a b, h, s, d, or q register, print it as a v register.
88409467b48Spatrick return printAsmRegInClass(MO, RegClass, AltName, O);
88509467b48Spatrick }
88609467b48Spatrick
88709467b48Spatrick printOperand(MI, OpNum, O);
88809467b48Spatrick return false;
88909467b48Spatrick }
89009467b48Spatrick
PrintAsmMemoryOperand(const MachineInstr * MI,unsigned OpNum,const char * ExtraCode,raw_ostream & O)89109467b48Spatrick bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
89209467b48Spatrick unsigned OpNum,
89309467b48Spatrick const char *ExtraCode,
89409467b48Spatrick raw_ostream &O) {
89509467b48Spatrick if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
89609467b48Spatrick return true; // Unknown modifier.
89709467b48Spatrick
89809467b48Spatrick const MachineOperand &MO = MI->getOperand(OpNum);
89909467b48Spatrick assert(MO.isReg() && "unexpected inline asm memory operand");
90009467b48Spatrick O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
90109467b48Spatrick return false;
90209467b48Spatrick }
90309467b48Spatrick
PrintDebugValueComment(const MachineInstr * MI,raw_ostream & OS)90409467b48Spatrick void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
90509467b48Spatrick raw_ostream &OS) {
90609467b48Spatrick unsigned NOps = MI->getNumOperands();
90709467b48Spatrick assert(NOps == 4);
90809467b48Spatrick OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
90909467b48Spatrick // cast away const; DIetc do not take const operands for some reason.
910097a140dSpatrick OS << MI->getDebugVariable()->getName();
91109467b48Spatrick OS << " <- ";
91209467b48Spatrick // Frame address. Currently handles register +- offset only.
913a0747c9fSpatrick assert(MI->isIndirectDebugValue());
91409467b48Spatrick OS << '[';
915a0747c9fSpatrick for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(),
916a0747c9fSpatrick MI->debug_operands().end());
917a0747c9fSpatrick I < E; ++I) {
918a0747c9fSpatrick if (I != 0)
919a0747c9fSpatrick OS << ", ";
920a0747c9fSpatrick printOperand(MI, I, OS);
921a0747c9fSpatrick }
92209467b48Spatrick OS << ']';
92309467b48Spatrick OS << "+";
92409467b48Spatrick printOperand(MI, NOps - 2, OS);
92509467b48Spatrick }
92609467b48Spatrick
emitJumpTableInfo()927097a140dSpatrick void AArch64AsmPrinter::emitJumpTableInfo() {
92809467b48Spatrick const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
92909467b48Spatrick if (!MJTI) return;
93009467b48Spatrick
93109467b48Spatrick const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
93209467b48Spatrick if (JT.empty()) return;
93309467b48Spatrick
93409467b48Spatrick const TargetLoweringObjectFile &TLOF = getObjFileLowering();
935*a96b3639Srobert MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM);
936*a96b3639Srobert OutStreamer->switchSection(ReadOnlySec);
93709467b48Spatrick
93809467b48Spatrick auto AFI = MF->getInfo<AArch64FunctionInfo>();
93909467b48Spatrick for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
94009467b48Spatrick const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
94109467b48Spatrick
94209467b48Spatrick // If this jump table was deleted, ignore it.
94309467b48Spatrick if (JTBBs.empty()) continue;
94409467b48Spatrick
94509467b48Spatrick unsigned Size = AFI->getJumpTableEntrySize(JTI);
946097a140dSpatrick emitAlignment(Align(Size));
947097a140dSpatrick OutStreamer->emitLabel(GetJTISymbol(JTI));
94809467b48Spatrick
949a0747c9fSpatrick const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
95009467b48Spatrick const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
951a0747c9fSpatrick
952a0747c9fSpatrick for (auto *JTBB : JTBBs) {
953a0747c9fSpatrick const MCExpr *Value =
954a0747c9fSpatrick MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
955a0747c9fSpatrick
956a0747c9fSpatrick // Each entry is:
957a0747c9fSpatrick // .byte/.hword (LBB - Lbase)>>2
958a0747c9fSpatrick // or plain:
959a0747c9fSpatrick // .word LBB - Lbase
96009467b48Spatrick Value = MCBinaryExpr::createSub(Value, Base, OutContext);
961a0747c9fSpatrick if (Size != 4)
96209467b48Spatrick Value = MCBinaryExpr::createLShr(
96309467b48Spatrick Value, MCConstantExpr::create(2, OutContext), OutContext);
96409467b48Spatrick
965097a140dSpatrick OutStreamer->emitValue(Value, Size);
966097a140dSpatrick }
967a0747c9fSpatrick }
968a0747c9fSpatrick }
969097a140dSpatrick
emitFunctionEntryLabel()970097a140dSpatrick void AArch64AsmPrinter::emitFunctionEntryLabel() {
971097a140dSpatrick if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
972097a140dSpatrick MF->getFunction().getCallingConv() ==
973097a140dSpatrick CallingConv::AArch64_SVE_VectorCall ||
974*a96b3639Srobert MF->getInfo<AArch64FunctionInfo>()->isSVECC()) {
975097a140dSpatrick auto *TS =
976097a140dSpatrick static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
977097a140dSpatrick TS->emitDirectiveVariantPCS(CurrentFnSym);
978097a140dSpatrick }
979097a140dSpatrick
980097a140dSpatrick return AsmPrinter::emitFunctionEntryLabel();
98109467b48Spatrick }
98209467b48Spatrick
98309467b48Spatrick /// Small jump tables contain an unsigned byte or half, representing the offset
98409467b48Spatrick /// from the lowest-addressed possible destination to the desired basic
98509467b48Spatrick /// block. Since all instructions are 4-byte aligned, this is further compressed
98609467b48Spatrick /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
98709467b48Spatrick /// materialize the correct destination we need:
98809467b48Spatrick ///
98909467b48Spatrick /// adr xDest, .LBB0_0
99009467b48Spatrick /// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
991a0747c9fSpatrick /// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
LowerJumpTableDest(llvm::MCStreamer & OutStreamer,const llvm::MachineInstr & MI)992a0747c9fSpatrick void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
99309467b48Spatrick const llvm::MachineInstr &MI) {
99409467b48Spatrick Register DestReg = MI.getOperand(0).getReg();
99509467b48Spatrick Register ScratchReg = MI.getOperand(1).getReg();
99609467b48Spatrick Register ScratchRegW =
99709467b48Spatrick STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
99809467b48Spatrick Register TableReg = MI.getOperand(2).getReg();
99909467b48Spatrick Register EntryReg = MI.getOperand(3).getReg();
100009467b48Spatrick int JTIdx = MI.getOperand(4).getIndex();
1001a0747c9fSpatrick int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
100209467b48Spatrick
100309467b48Spatrick // This has to be first because the compression pass based its reachability
100409467b48Spatrick // calculations on the start of the JumpTableDest instruction.
100509467b48Spatrick auto Label =
100609467b48Spatrick MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
1007a0747c9fSpatrick
1008a0747c9fSpatrick // If we don't already have a symbol to use as the base, use the ADR
1009a0747c9fSpatrick // instruction itself.
1010a0747c9fSpatrick if (!Label) {
1011a0747c9fSpatrick Label = MF->getContext().createTempSymbol();
1012a0747c9fSpatrick AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
1013a0747c9fSpatrick OutStreamer.emitLabel(Label);
1014a0747c9fSpatrick }
1015a0747c9fSpatrick
1016a0747c9fSpatrick auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
101709467b48Spatrick EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
101809467b48Spatrick .addReg(DestReg)
1019a0747c9fSpatrick .addExpr(LabelExpr));
102009467b48Spatrick
102109467b48Spatrick // Load the number of instruction-steps to offset from the label.
1022a0747c9fSpatrick unsigned LdrOpcode;
1023a0747c9fSpatrick switch (Size) {
1024a0747c9fSpatrick case 1: LdrOpcode = AArch64::LDRBBroX; break;
1025a0747c9fSpatrick case 2: LdrOpcode = AArch64::LDRHHroX; break;
1026a0747c9fSpatrick case 4: LdrOpcode = AArch64::LDRSWroX; break;
1027a0747c9fSpatrick default:
1028a0747c9fSpatrick llvm_unreachable("Unknown jump table size");
1029a0747c9fSpatrick }
1030a0747c9fSpatrick
103109467b48Spatrick EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
1032a0747c9fSpatrick .addReg(Size == 4 ? ScratchReg : ScratchRegW)
103309467b48Spatrick .addReg(TableReg)
103409467b48Spatrick .addReg(EntryReg)
103509467b48Spatrick .addImm(0)
1036a0747c9fSpatrick .addImm(Size == 1 ? 0 : 1));
103709467b48Spatrick
1038a0747c9fSpatrick // Add to the already materialized base label address, multiplying by 4 if
1039a0747c9fSpatrick // compressed.
104009467b48Spatrick EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
104109467b48Spatrick .addReg(DestReg)
104209467b48Spatrick .addReg(DestReg)
104309467b48Spatrick .addReg(ScratchReg)
1044a0747c9fSpatrick .addImm(Size == 4 ? 0 : 2));
104509467b48Spatrick }
104609467b48Spatrick
LowerMOPS(llvm::MCStreamer & OutStreamer,const llvm::MachineInstr & MI)1047*a96b3639Srobert void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1048*a96b3639Srobert const llvm::MachineInstr &MI) {
1049*a96b3639Srobert unsigned Opcode = MI.getOpcode();
1050*a96b3639Srobert assert(STI->hasMOPS());
1051*a96b3639Srobert assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1052*a96b3639Srobert
1053*a96b3639Srobert const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1054*a96b3639Srobert if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1055*a96b3639Srobert return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1056*a96b3639Srobert if (Opcode == AArch64::MOPSMemoryMovePseudo)
1057*a96b3639Srobert return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1058*a96b3639Srobert if (Opcode == AArch64::MOPSMemorySetPseudo)
1059*a96b3639Srobert return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1060*a96b3639Srobert if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1061*a96b3639Srobert return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1062*a96b3639Srobert llvm_unreachable("Unhandled memory operation pseudo");
1063*a96b3639Srobert }();
1064*a96b3639Srobert const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1065*a96b3639Srobert Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1066*a96b3639Srobert
1067*a96b3639Srobert for (auto Op : Ops) {
1068*a96b3639Srobert int i = 0;
1069*a96b3639Srobert auto MCIB = MCInstBuilder(Op);
1070*a96b3639Srobert // Destination registers
1071*a96b3639Srobert MCIB.addReg(MI.getOperand(i++).getReg());
1072*a96b3639Srobert MCIB.addReg(MI.getOperand(i++).getReg());
1073*a96b3639Srobert if (!IsSet)
1074*a96b3639Srobert MCIB.addReg(MI.getOperand(i++).getReg());
1075*a96b3639Srobert // Input registers
1076*a96b3639Srobert MCIB.addReg(MI.getOperand(i++).getReg());
1077*a96b3639Srobert MCIB.addReg(MI.getOperand(i++).getReg());
1078*a96b3639Srobert MCIB.addReg(MI.getOperand(i++).getReg());
1079*a96b3639Srobert
1080*a96b3639Srobert EmitToStreamer(OutStreamer, MCIB);
1081*a96b3639Srobert }
1082*a96b3639Srobert }
1083*a96b3639Srobert
LowerSTACKMAP(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)108409467b48Spatrick void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
108509467b48Spatrick const MachineInstr &MI) {
108609467b48Spatrick unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
108709467b48Spatrick
108809467b48Spatrick auto &Ctx = OutStreamer.getContext();
108909467b48Spatrick MCSymbol *MILabel = Ctx.createTempSymbol();
1090097a140dSpatrick OutStreamer.emitLabel(MILabel);
109109467b48Spatrick
109209467b48Spatrick SM.recordStackMap(*MILabel, MI);
109309467b48Spatrick assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
109409467b48Spatrick
109509467b48Spatrick // Scan ahead to trim the shadow.
109609467b48Spatrick const MachineBasicBlock &MBB = *MI.getParent();
109709467b48Spatrick MachineBasicBlock::const_iterator MII(MI);
109809467b48Spatrick ++MII;
109909467b48Spatrick while (NumNOPBytes > 0) {
110009467b48Spatrick if (MII == MBB.end() || MII->isCall() ||
110109467b48Spatrick MII->getOpcode() == AArch64::DBG_VALUE ||
110209467b48Spatrick MII->getOpcode() == TargetOpcode::PATCHPOINT ||
110309467b48Spatrick MII->getOpcode() == TargetOpcode::STACKMAP)
110409467b48Spatrick break;
110509467b48Spatrick ++MII;
110609467b48Spatrick NumNOPBytes -= 4;
110709467b48Spatrick }
110809467b48Spatrick
110909467b48Spatrick // Emit nops.
111009467b48Spatrick for (unsigned i = 0; i < NumNOPBytes; i += 4)
111109467b48Spatrick EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
111209467b48Spatrick }
111309467b48Spatrick
111409467b48Spatrick // Lower a patchpoint of the form:
111509467b48Spatrick // [<def>], <id>, <numBytes>, <target>, <numArgs>
LowerPATCHPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)111609467b48Spatrick void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
111709467b48Spatrick const MachineInstr &MI) {
111809467b48Spatrick auto &Ctx = OutStreamer.getContext();
111909467b48Spatrick MCSymbol *MILabel = Ctx.createTempSymbol();
1120097a140dSpatrick OutStreamer.emitLabel(MILabel);
112109467b48Spatrick SM.recordPatchPoint(*MILabel, MI);
112209467b48Spatrick
112309467b48Spatrick PatchPointOpers Opers(&MI);
112409467b48Spatrick
112509467b48Spatrick int64_t CallTarget = Opers.getCallTarget().getImm();
112609467b48Spatrick unsigned EncodedBytes = 0;
112709467b48Spatrick if (CallTarget) {
112809467b48Spatrick assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
112909467b48Spatrick "High 16 bits of call target should be zero.");
113009467b48Spatrick Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
113109467b48Spatrick EncodedBytes = 16;
113209467b48Spatrick // Materialize the jump address:
113309467b48Spatrick EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
113409467b48Spatrick .addReg(ScratchReg)
113509467b48Spatrick .addImm((CallTarget >> 32) & 0xFFFF)
113609467b48Spatrick .addImm(32));
113709467b48Spatrick EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
113809467b48Spatrick .addReg(ScratchReg)
113909467b48Spatrick .addReg(ScratchReg)
114009467b48Spatrick .addImm((CallTarget >> 16) & 0xFFFF)
114109467b48Spatrick .addImm(16));
114209467b48Spatrick EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
114309467b48Spatrick .addReg(ScratchReg)
114409467b48Spatrick .addReg(ScratchReg)
114509467b48Spatrick .addImm(CallTarget & 0xFFFF)
114609467b48Spatrick .addImm(0));
114709467b48Spatrick EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
114809467b48Spatrick }
114909467b48Spatrick // Emit padding.
115009467b48Spatrick unsigned NumBytes = Opers.getNumPatchBytes();
115109467b48Spatrick assert(NumBytes >= EncodedBytes &&
115209467b48Spatrick "Patchpoint can't request size less than the length of a call.");
115309467b48Spatrick assert((NumBytes - EncodedBytes) % 4 == 0 &&
115409467b48Spatrick "Invalid number of NOP bytes requested!");
115509467b48Spatrick for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
115609467b48Spatrick EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
115709467b48Spatrick }
115809467b48Spatrick
LowerSTATEPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)1159a0747c9fSpatrick void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1160a0747c9fSpatrick const MachineInstr &MI) {
1161a0747c9fSpatrick StatepointOpers SOpers(&MI);
1162a0747c9fSpatrick if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1163a0747c9fSpatrick assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1164a0747c9fSpatrick for (unsigned i = 0; i < PatchBytes; i += 4)
1165a0747c9fSpatrick EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1166a0747c9fSpatrick } else {
1167a0747c9fSpatrick // Lower call target and choose correct opcode
1168a0747c9fSpatrick const MachineOperand &CallTarget = SOpers.getCallTarget();
1169a0747c9fSpatrick MCOperand CallTargetMCOp;
1170a0747c9fSpatrick unsigned CallOpcode;
1171a0747c9fSpatrick switch (CallTarget.getType()) {
1172a0747c9fSpatrick case MachineOperand::MO_GlobalAddress:
1173a0747c9fSpatrick case MachineOperand::MO_ExternalSymbol:
1174a0747c9fSpatrick MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1175a0747c9fSpatrick CallOpcode = AArch64::BL;
1176a0747c9fSpatrick break;
1177a0747c9fSpatrick case MachineOperand::MO_Immediate:
1178a0747c9fSpatrick CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1179a0747c9fSpatrick CallOpcode = AArch64::BL;
1180a0747c9fSpatrick break;
1181a0747c9fSpatrick case MachineOperand::MO_Register:
1182a0747c9fSpatrick CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1183a0747c9fSpatrick CallOpcode = AArch64::BLR;
1184a0747c9fSpatrick break;
1185a0747c9fSpatrick default:
1186a0747c9fSpatrick llvm_unreachable("Unsupported operand type in statepoint call target");
1187a0747c9fSpatrick break;
1188a0747c9fSpatrick }
1189a0747c9fSpatrick
1190a0747c9fSpatrick EmitToStreamer(OutStreamer,
1191a0747c9fSpatrick MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1192a0747c9fSpatrick }
1193a0747c9fSpatrick
1194a0747c9fSpatrick auto &Ctx = OutStreamer.getContext();
1195a0747c9fSpatrick MCSymbol *MILabel = Ctx.createTempSymbol();
1196a0747c9fSpatrick OutStreamer.emitLabel(MILabel);
1197a0747c9fSpatrick SM.recordStatepoint(*MILabel, MI);
1198a0747c9fSpatrick }
1199a0747c9fSpatrick
LowerFAULTING_OP(const MachineInstr & FaultingMI)1200a0747c9fSpatrick void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1201a0747c9fSpatrick // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1202a0747c9fSpatrick // <opcode>, <operands>
1203a0747c9fSpatrick
1204a0747c9fSpatrick Register DefRegister = FaultingMI.getOperand(0).getReg();
1205a0747c9fSpatrick FaultMaps::FaultKind FK =
1206a0747c9fSpatrick static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1207a0747c9fSpatrick MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1208a0747c9fSpatrick unsigned Opcode = FaultingMI.getOperand(3).getImm();
1209a0747c9fSpatrick unsigned OperandsBeginIdx = 4;
1210a0747c9fSpatrick
1211a0747c9fSpatrick auto &Ctx = OutStreamer->getContext();
1212a0747c9fSpatrick MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1213a0747c9fSpatrick OutStreamer->emitLabel(FaultingLabel);
1214a0747c9fSpatrick
1215a0747c9fSpatrick assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1216a0747c9fSpatrick FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1217a0747c9fSpatrick
1218a0747c9fSpatrick MCInst MI;
1219a0747c9fSpatrick MI.setOpcode(Opcode);
1220a0747c9fSpatrick
1221a0747c9fSpatrick if (DefRegister != (Register)0)
1222a0747c9fSpatrick MI.addOperand(MCOperand::createReg(DefRegister));
1223a0747c9fSpatrick
1224*a96b3639Srobert for (const MachineOperand &MO :
1225*a96b3639Srobert llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx)) {
1226a0747c9fSpatrick MCOperand Dest;
1227*a96b3639Srobert lowerOperand(MO, Dest);
1228a0747c9fSpatrick MI.addOperand(Dest);
1229a0747c9fSpatrick }
1230a0747c9fSpatrick
1231a0747c9fSpatrick OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1232a0747c9fSpatrick OutStreamer->emitInstruction(MI, getSubtargetInfo());
1233a0747c9fSpatrick }
1234a0747c9fSpatrick
emitFMov0(const MachineInstr & MI)1235a0747c9fSpatrick void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
123609467b48Spatrick Register DestReg = MI.getOperand(0).getReg();
1237*a96b3639Srobert if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() &&
1238*a96b3639Srobert STI->hasNEON()) {
1239a0747c9fSpatrick // Convert H/S register to corresponding D register
124009467b48Spatrick if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1241a0747c9fSpatrick DestReg = AArch64::D0 + (DestReg - AArch64::H0);
124209467b48Spatrick else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1243a0747c9fSpatrick DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1244a0747c9fSpatrick else
124509467b48Spatrick assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1246a0747c9fSpatrick
124709467b48Spatrick MCInst MOVI;
1248a0747c9fSpatrick MOVI.setOpcode(AArch64::MOVID);
124909467b48Spatrick MOVI.addOperand(MCOperand::createReg(DestReg));
125009467b48Spatrick MOVI.addOperand(MCOperand::createImm(0));
125109467b48Spatrick EmitToStreamer(*OutStreamer, MOVI);
125209467b48Spatrick } else {
125309467b48Spatrick MCInst FMov;
125409467b48Spatrick switch (MI.getOpcode()) {
125509467b48Spatrick default: llvm_unreachable("Unexpected opcode");
125609467b48Spatrick case AArch64::FMOVH0:
125709467b48Spatrick FMov.setOpcode(AArch64::FMOVWHr);
125809467b48Spatrick FMov.addOperand(MCOperand::createReg(DestReg));
125909467b48Spatrick FMov.addOperand(MCOperand::createReg(AArch64::WZR));
126009467b48Spatrick break;
126109467b48Spatrick case AArch64::FMOVS0:
126209467b48Spatrick FMov.setOpcode(AArch64::FMOVWSr);
126309467b48Spatrick FMov.addOperand(MCOperand::createReg(DestReg));
126409467b48Spatrick FMov.addOperand(MCOperand::createReg(AArch64::WZR));
126509467b48Spatrick break;
126609467b48Spatrick case AArch64::FMOVD0:
126709467b48Spatrick FMov.setOpcode(AArch64::FMOVXDr);
126809467b48Spatrick FMov.addOperand(MCOperand::createReg(DestReg));
126909467b48Spatrick FMov.addOperand(MCOperand::createReg(AArch64::XZR));
127009467b48Spatrick break;
127109467b48Spatrick }
127209467b48Spatrick EmitToStreamer(*OutStreamer, FMov);
127309467b48Spatrick }
127409467b48Spatrick }
127509467b48Spatrick
127609467b48Spatrick // Simple pseudo-instructions have their lowering (with expansion to real
127709467b48Spatrick // instructions) auto-generated.
127809467b48Spatrick #include "AArch64GenMCPseudoLowering.inc"
127909467b48Spatrick
emitInstruction(const MachineInstr * MI)1280097a140dSpatrick void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1281*a96b3639Srobert AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
1282*a96b3639Srobert
128309467b48Spatrick // Do any auto-generated pseudo lowerings.
128409467b48Spatrick if (emitPseudoExpansionLowering(*OutStreamer, MI))
128509467b48Spatrick return;
128609467b48Spatrick
1287*a96b3639Srobert if (MI->getOpcode() == AArch64::ADRP) {
1288*a96b3639Srobert for (auto &Opd : MI->operands()) {
1289*a96b3639Srobert if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
1290*a96b3639Srobert "swift_async_extendedFramePointerFlags") {
1291*a96b3639Srobert ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
1292*a96b3639Srobert }
1293*a96b3639Srobert }
1294*a96b3639Srobert }
1295*a96b3639Srobert
129609467b48Spatrick if (AArch64FI->getLOHRelated().count(MI)) {
129709467b48Spatrick // Generate a label for LOH related instruction
129809467b48Spatrick MCSymbol *LOHLabel = createTempSymbol("loh");
129909467b48Spatrick // Associate the instruction with the label
130009467b48Spatrick LOHInstToLabel[MI] = LOHLabel;
1301097a140dSpatrick OutStreamer->emitLabel(LOHLabel);
130209467b48Spatrick }
130309467b48Spatrick
130409467b48Spatrick AArch64TargetStreamer *TS =
130509467b48Spatrick static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
130609467b48Spatrick // Do any manual lowerings.
130709467b48Spatrick switch (MI->getOpcode()) {
130809467b48Spatrick default:
130909467b48Spatrick break;
131009467b48Spatrick case AArch64::HINT: {
131109467b48Spatrick // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
131209467b48Spatrick // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
131309467b48Spatrick // non-empty. If MI is the initial BTI, place the
131409467b48Spatrick // __patchable_function_entries label after BTI.
131509467b48Spatrick if (CurrentPatchableFunctionEntrySym &&
131609467b48Spatrick CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
131709467b48Spatrick MI == &MF->front().front()) {
131809467b48Spatrick int64_t Imm = MI->getOperand(0).getImm();
131909467b48Spatrick if ((Imm & 32) && (Imm & 6)) {
132009467b48Spatrick MCInst Inst;
132109467b48Spatrick MCInstLowering.Lower(MI, Inst);
132209467b48Spatrick EmitToStreamer(*OutStreamer, Inst);
132309467b48Spatrick CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1324097a140dSpatrick OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
132509467b48Spatrick return;
132609467b48Spatrick }
132709467b48Spatrick }
132809467b48Spatrick break;
132909467b48Spatrick }
133009467b48Spatrick case AArch64::MOVMCSym: {
133109467b48Spatrick Register DestReg = MI->getOperand(0).getReg();
133209467b48Spatrick const MachineOperand &MO_Sym = MI->getOperand(1);
133309467b48Spatrick MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
133409467b48Spatrick MCOperand Hi_MCSym, Lo_MCSym;
133509467b48Spatrick
133609467b48Spatrick Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
133709467b48Spatrick Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
133809467b48Spatrick
133909467b48Spatrick MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
134009467b48Spatrick MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
134109467b48Spatrick
134209467b48Spatrick MCInst MovZ;
134309467b48Spatrick MovZ.setOpcode(AArch64::MOVZXi);
134409467b48Spatrick MovZ.addOperand(MCOperand::createReg(DestReg));
134509467b48Spatrick MovZ.addOperand(Hi_MCSym);
134609467b48Spatrick MovZ.addOperand(MCOperand::createImm(16));
134709467b48Spatrick EmitToStreamer(*OutStreamer, MovZ);
134809467b48Spatrick
134909467b48Spatrick MCInst MovK;
135009467b48Spatrick MovK.setOpcode(AArch64::MOVKXi);
135109467b48Spatrick MovK.addOperand(MCOperand::createReg(DestReg));
135209467b48Spatrick MovK.addOperand(MCOperand::createReg(DestReg));
135309467b48Spatrick MovK.addOperand(Lo_MCSym);
135409467b48Spatrick MovK.addOperand(MCOperand::createImm(0));
135509467b48Spatrick EmitToStreamer(*OutStreamer, MovK);
135609467b48Spatrick return;
135709467b48Spatrick }
135809467b48Spatrick case AArch64::MOVIv2d_ns:
135909467b48Spatrick // If the target has <rdar://problem/16473581>, lower this
136009467b48Spatrick // instruction to movi.16b instead.
136109467b48Spatrick if (STI->hasZeroCycleZeroingFPWorkaround() &&
136209467b48Spatrick MI->getOperand(1).getImm() == 0) {
136309467b48Spatrick MCInst TmpInst;
136409467b48Spatrick TmpInst.setOpcode(AArch64::MOVIv16b_ns);
136509467b48Spatrick TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
136609467b48Spatrick TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
136709467b48Spatrick EmitToStreamer(*OutStreamer, TmpInst);
136809467b48Spatrick return;
136909467b48Spatrick }
137009467b48Spatrick break;
137109467b48Spatrick
1372a0747c9fSpatrick case AArch64::DBG_VALUE:
1373*a96b3639Srobert case AArch64::DBG_VALUE_LIST:
137409467b48Spatrick if (isVerbose() && OutStreamer->hasRawTextSupport()) {
137509467b48Spatrick SmallString<128> TmpStr;
137609467b48Spatrick raw_svector_ostream OS(TmpStr);
137709467b48Spatrick PrintDebugValueComment(MI, OS);
1378097a140dSpatrick OutStreamer->emitRawText(StringRef(OS.str()));
137909467b48Spatrick }
138009467b48Spatrick return;
138109467b48Spatrick
138209467b48Spatrick case AArch64::EMITBKEY: {
138309467b48Spatrick ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
138409467b48Spatrick if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
138509467b48Spatrick ExceptionHandlingType != ExceptionHandling::ARM)
138609467b48Spatrick return;
138709467b48Spatrick
1388a0747c9fSpatrick if (getFunctionCFISectionType(*MF) == CFISection::None)
138909467b48Spatrick return;
139009467b48Spatrick
1391097a140dSpatrick OutStreamer->emitCFIBKeyFrame();
139209467b48Spatrick return;
139309467b48Spatrick }
1394*a96b3639Srobert
1395*a96b3639Srobert case AArch64::EMITMTETAGGED: {
1396*a96b3639Srobert ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1397*a96b3639Srobert if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1398*a96b3639Srobert ExceptionHandlingType != ExceptionHandling::ARM)
1399*a96b3639Srobert return;
1400*a96b3639Srobert
1401*a96b3639Srobert if (getFunctionCFISectionType(*MF) != CFISection::None)
1402*a96b3639Srobert OutStreamer->emitCFIMTETaggedFrame();
1403*a96b3639Srobert return;
140409467b48Spatrick }
140509467b48Spatrick
140609467b48Spatrick // Tail calls use pseudo instructions so they have the proper code-gen
140709467b48Spatrick // attributes (isCall, isReturn, etc.). We lower them to the real
140809467b48Spatrick // instruction here.
140909467b48Spatrick case AArch64::TCRETURNri:
141009467b48Spatrick case AArch64::TCRETURNriBTI:
141109467b48Spatrick case AArch64::TCRETURNriALL: {
141209467b48Spatrick MCInst TmpInst;
141309467b48Spatrick TmpInst.setOpcode(AArch64::BR);
141409467b48Spatrick TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
141509467b48Spatrick EmitToStreamer(*OutStreamer, TmpInst);
141609467b48Spatrick return;
141709467b48Spatrick }
141809467b48Spatrick case AArch64::TCRETURNdi: {
141909467b48Spatrick MCOperand Dest;
142009467b48Spatrick MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
142109467b48Spatrick MCInst TmpInst;
142209467b48Spatrick TmpInst.setOpcode(AArch64::B);
142309467b48Spatrick TmpInst.addOperand(Dest);
142409467b48Spatrick EmitToStreamer(*OutStreamer, TmpInst);
142509467b48Spatrick return;
142609467b48Spatrick }
1427097a140dSpatrick case AArch64::SpeculationBarrierISBDSBEndBB: {
1428097a140dSpatrick // Print DSB SYS + ISB
1429097a140dSpatrick MCInst TmpInstDSB;
1430097a140dSpatrick TmpInstDSB.setOpcode(AArch64::DSB);
1431097a140dSpatrick TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1432097a140dSpatrick EmitToStreamer(*OutStreamer, TmpInstDSB);
1433097a140dSpatrick MCInst TmpInstISB;
1434097a140dSpatrick TmpInstISB.setOpcode(AArch64::ISB);
1435097a140dSpatrick TmpInstISB.addOperand(MCOperand::createImm(0xf));
1436097a140dSpatrick EmitToStreamer(*OutStreamer, TmpInstISB);
1437097a140dSpatrick return;
1438097a140dSpatrick }
1439097a140dSpatrick case AArch64::SpeculationBarrierSBEndBB: {
1440097a140dSpatrick // Print SB
1441097a140dSpatrick MCInst TmpInstSB;
1442097a140dSpatrick TmpInstSB.setOpcode(AArch64::SB);
1443097a140dSpatrick EmitToStreamer(*OutStreamer, TmpInstSB);
1444097a140dSpatrick return;
1445097a140dSpatrick }
144609467b48Spatrick case AArch64::TLSDESC_CALLSEQ: {
144709467b48Spatrick /// lower this to:
144809467b48Spatrick /// adrp x0, :tlsdesc:var
144909467b48Spatrick /// ldr x1, [x0, #:tlsdesc_lo12:var]
145009467b48Spatrick /// add x0, x0, #:tlsdesc_lo12:var
145109467b48Spatrick /// .tlsdesccall var
145209467b48Spatrick /// blr x1
145309467b48Spatrick /// (TPIDR_EL0 offset now in x0)
145409467b48Spatrick const MachineOperand &MO_Sym = MI->getOperand(0);
145509467b48Spatrick MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
145609467b48Spatrick MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
145709467b48Spatrick MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
145809467b48Spatrick MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
145909467b48Spatrick MCInstLowering.lowerOperand(MO_Sym, Sym);
146009467b48Spatrick MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
146109467b48Spatrick MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
146209467b48Spatrick
146309467b48Spatrick MCInst Adrp;
146409467b48Spatrick Adrp.setOpcode(AArch64::ADRP);
146509467b48Spatrick Adrp.addOperand(MCOperand::createReg(AArch64::X0));
146609467b48Spatrick Adrp.addOperand(SymTLSDesc);
146709467b48Spatrick EmitToStreamer(*OutStreamer, Adrp);
146809467b48Spatrick
146909467b48Spatrick MCInst Ldr;
1470a0747c9fSpatrick if (STI->isTargetILP32()) {
1471a0747c9fSpatrick Ldr.setOpcode(AArch64::LDRWui);
1472a0747c9fSpatrick Ldr.addOperand(MCOperand::createReg(AArch64::W1));
1473a0747c9fSpatrick } else {
147409467b48Spatrick Ldr.setOpcode(AArch64::LDRXui);
147509467b48Spatrick Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1476a0747c9fSpatrick }
147709467b48Spatrick Ldr.addOperand(MCOperand::createReg(AArch64::X0));
147809467b48Spatrick Ldr.addOperand(SymTLSDescLo12);
147909467b48Spatrick Ldr.addOperand(MCOperand::createImm(0));
148009467b48Spatrick EmitToStreamer(*OutStreamer, Ldr);
148109467b48Spatrick
148209467b48Spatrick MCInst Add;
1483a0747c9fSpatrick if (STI->isTargetILP32()) {
1484a0747c9fSpatrick Add.setOpcode(AArch64::ADDWri);
1485a0747c9fSpatrick Add.addOperand(MCOperand::createReg(AArch64::W0));
1486a0747c9fSpatrick Add.addOperand(MCOperand::createReg(AArch64::W0));
1487a0747c9fSpatrick } else {
148809467b48Spatrick Add.setOpcode(AArch64::ADDXri);
148909467b48Spatrick Add.addOperand(MCOperand::createReg(AArch64::X0));
149009467b48Spatrick Add.addOperand(MCOperand::createReg(AArch64::X0));
1491a0747c9fSpatrick }
149209467b48Spatrick Add.addOperand(SymTLSDescLo12);
149309467b48Spatrick Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
149409467b48Spatrick EmitToStreamer(*OutStreamer, Add);
149509467b48Spatrick
149609467b48Spatrick // Emit a relocation-annotation. This expands to no code, but requests
149709467b48Spatrick // the following instruction gets an R_AARCH64_TLSDESC_CALL.
149809467b48Spatrick MCInst TLSDescCall;
149909467b48Spatrick TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
150009467b48Spatrick TLSDescCall.addOperand(Sym);
150109467b48Spatrick EmitToStreamer(*OutStreamer, TLSDescCall);
150209467b48Spatrick
150309467b48Spatrick MCInst Blr;
150409467b48Spatrick Blr.setOpcode(AArch64::BLR);
150509467b48Spatrick Blr.addOperand(MCOperand::createReg(AArch64::X1));
150609467b48Spatrick EmitToStreamer(*OutStreamer, Blr);
150709467b48Spatrick
150809467b48Spatrick return;
150909467b48Spatrick }
151009467b48Spatrick
1511a0747c9fSpatrick case AArch64::JumpTableDest32:
151209467b48Spatrick case AArch64::JumpTableDest16:
151309467b48Spatrick case AArch64::JumpTableDest8:
1514a0747c9fSpatrick LowerJumpTableDest(*OutStreamer, *MI);
151509467b48Spatrick return;
151609467b48Spatrick
151709467b48Spatrick case AArch64::FMOVH0:
151809467b48Spatrick case AArch64::FMOVS0:
151909467b48Spatrick case AArch64::FMOVD0:
1520a0747c9fSpatrick emitFMov0(*MI);
152109467b48Spatrick return;
152209467b48Spatrick
1523*a96b3639Srobert case AArch64::MOPSMemoryCopyPseudo:
1524*a96b3639Srobert case AArch64::MOPSMemoryMovePseudo:
1525*a96b3639Srobert case AArch64::MOPSMemorySetPseudo:
1526*a96b3639Srobert case AArch64::MOPSMemorySetTaggingPseudo:
1527*a96b3639Srobert LowerMOPS(*OutStreamer, *MI);
1528*a96b3639Srobert return;
1529*a96b3639Srobert
153009467b48Spatrick case TargetOpcode::STACKMAP:
153109467b48Spatrick return LowerSTACKMAP(*OutStreamer, SM, *MI);
153209467b48Spatrick
153309467b48Spatrick case TargetOpcode::PATCHPOINT:
153409467b48Spatrick return LowerPATCHPOINT(*OutStreamer, SM, *MI);
153509467b48Spatrick
1536a0747c9fSpatrick case TargetOpcode::STATEPOINT:
1537a0747c9fSpatrick return LowerSTATEPOINT(*OutStreamer, SM, *MI);
1538a0747c9fSpatrick
1539a0747c9fSpatrick case TargetOpcode::FAULTING_OP:
1540a0747c9fSpatrick return LowerFAULTING_OP(*MI);
1541a0747c9fSpatrick
154209467b48Spatrick case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
154309467b48Spatrick LowerPATCHABLE_FUNCTION_ENTER(*MI);
154409467b48Spatrick return;
154509467b48Spatrick
154609467b48Spatrick case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
154709467b48Spatrick LowerPATCHABLE_FUNCTION_EXIT(*MI);
154809467b48Spatrick return;
154909467b48Spatrick
155009467b48Spatrick case TargetOpcode::PATCHABLE_TAIL_CALL:
155109467b48Spatrick LowerPATCHABLE_TAIL_CALL(*MI);
155209467b48Spatrick return;
155309467b48Spatrick
1554*a96b3639Srobert case AArch64::KCFI_CHECK:
1555*a96b3639Srobert LowerKCFI_CHECK(*MI);
1556*a96b3639Srobert return;
1557*a96b3639Srobert
155809467b48Spatrick case AArch64::HWASAN_CHECK_MEMACCESS:
155909467b48Spatrick case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
156009467b48Spatrick LowerHWASAN_CHECK_MEMACCESS(*MI);
156109467b48Spatrick return;
156209467b48Spatrick
156309467b48Spatrick case AArch64::SEH_StackAlloc:
1564a0747c9fSpatrick TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
156509467b48Spatrick return;
156609467b48Spatrick
156709467b48Spatrick case AArch64::SEH_SaveFPLR:
1568a0747c9fSpatrick TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
156909467b48Spatrick return;
157009467b48Spatrick
157109467b48Spatrick case AArch64::SEH_SaveFPLR_X:
157209467b48Spatrick assert(MI->getOperand(0).getImm() < 0 &&
157309467b48Spatrick "Pre increment SEH opcode must have a negative offset");
1574a0747c9fSpatrick TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
157509467b48Spatrick return;
157609467b48Spatrick
157709467b48Spatrick case AArch64::SEH_SaveReg:
1578a0747c9fSpatrick TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
157909467b48Spatrick MI->getOperand(1).getImm());
158009467b48Spatrick return;
158109467b48Spatrick
158209467b48Spatrick case AArch64::SEH_SaveReg_X:
158309467b48Spatrick assert(MI->getOperand(1).getImm() < 0 &&
158409467b48Spatrick "Pre increment SEH opcode must have a negative offset");
1585a0747c9fSpatrick TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
158609467b48Spatrick -MI->getOperand(1).getImm());
158709467b48Spatrick return;
158809467b48Spatrick
158909467b48Spatrick case AArch64::SEH_SaveRegP:
1590a0747c9fSpatrick if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
1591a0747c9fSpatrick MI->getOperand(0).getImm() <= 28) {
1592a0747c9fSpatrick assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1593a0747c9fSpatrick "Register paired with LR must be odd");
1594a0747c9fSpatrick TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
1595a0747c9fSpatrick MI->getOperand(2).getImm());
1596a0747c9fSpatrick return;
1597a0747c9fSpatrick }
159809467b48Spatrick assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
159909467b48Spatrick "Non-consecutive registers not allowed for save_regp");
1600a0747c9fSpatrick TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
160109467b48Spatrick MI->getOperand(2).getImm());
160209467b48Spatrick return;
160309467b48Spatrick
160409467b48Spatrick case AArch64::SEH_SaveRegP_X:
160509467b48Spatrick assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
160609467b48Spatrick "Non-consecutive registers not allowed for save_regp_x");
160709467b48Spatrick assert(MI->getOperand(2).getImm() < 0 &&
160809467b48Spatrick "Pre increment SEH opcode must have a negative offset");
1609a0747c9fSpatrick TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
161009467b48Spatrick -MI->getOperand(2).getImm());
161109467b48Spatrick return;
161209467b48Spatrick
161309467b48Spatrick case AArch64::SEH_SaveFReg:
1614a0747c9fSpatrick TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
161509467b48Spatrick MI->getOperand(1).getImm());
161609467b48Spatrick return;
161709467b48Spatrick
161809467b48Spatrick case AArch64::SEH_SaveFReg_X:
161909467b48Spatrick assert(MI->getOperand(1).getImm() < 0 &&
162009467b48Spatrick "Pre increment SEH opcode must have a negative offset");
1621a0747c9fSpatrick TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
162209467b48Spatrick -MI->getOperand(1).getImm());
162309467b48Spatrick return;
162409467b48Spatrick
162509467b48Spatrick case AArch64::SEH_SaveFRegP:
162609467b48Spatrick assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
162709467b48Spatrick "Non-consecutive registers not allowed for save_regp");
1628a0747c9fSpatrick TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
162909467b48Spatrick MI->getOperand(2).getImm());
163009467b48Spatrick return;
163109467b48Spatrick
163209467b48Spatrick case AArch64::SEH_SaveFRegP_X:
163309467b48Spatrick assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
163409467b48Spatrick "Non-consecutive registers not allowed for save_regp_x");
163509467b48Spatrick assert(MI->getOperand(2).getImm() < 0 &&
163609467b48Spatrick "Pre increment SEH opcode must have a negative offset");
1637a0747c9fSpatrick TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
163809467b48Spatrick -MI->getOperand(2).getImm());
163909467b48Spatrick return;
164009467b48Spatrick
164109467b48Spatrick case AArch64::SEH_SetFP:
1642a0747c9fSpatrick TS->emitARM64WinCFISetFP();
164309467b48Spatrick return;
164409467b48Spatrick
164509467b48Spatrick case AArch64::SEH_AddFP:
1646a0747c9fSpatrick TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm());
164709467b48Spatrick return;
164809467b48Spatrick
164909467b48Spatrick case AArch64::SEH_Nop:
1650a0747c9fSpatrick TS->emitARM64WinCFINop();
165109467b48Spatrick return;
165209467b48Spatrick
165309467b48Spatrick case AArch64::SEH_PrologEnd:
1654a0747c9fSpatrick TS->emitARM64WinCFIPrologEnd();
165509467b48Spatrick return;
165609467b48Spatrick
165709467b48Spatrick case AArch64::SEH_EpilogStart:
1658a0747c9fSpatrick TS->emitARM64WinCFIEpilogStart();
165909467b48Spatrick return;
166009467b48Spatrick
166109467b48Spatrick case AArch64::SEH_EpilogEnd:
1662a0747c9fSpatrick TS->emitARM64WinCFIEpilogEnd();
166309467b48Spatrick return;
1664adae0cfdSpatrick
1665adae0cfdSpatrick case AArch64::RETGUARD_JMP_TRAP:
1666adae0cfdSpatrick {
1667adae0cfdSpatrick MCSymbol *RGSuccSym = OutContext.createTempSymbol();
1668adae0cfdSpatrick /* Compare and branch */
1669adae0cfdSpatrick EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::CBZX)
1670adae0cfdSpatrick .addReg(MI->getOperand(0).getReg())
1671adae0cfdSpatrick .addExpr(MCSymbolRefExpr::create(RGSuccSym, OutContext)));
1672adae0cfdSpatrick EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(1));
1673adae0cfdSpatrick OutStreamer->emitLabel(RGSuccSym);
1674adae0cfdSpatrick return;
1675adae0cfdSpatrick }
1676adae0cfdSpatrick
1677*a96b3639Srobert case AArch64::SEH_PACSignLR:
1678*a96b3639Srobert TS->emitARM64WinCFIPACSignLR();
1679*a96b3639Srobert return;
168009467b48Spatrick }
168109467b48Spatrick
168209467b48Spatrick // Finally, do the automated lowerings for everything else.
168309467b48Spatrick MCInst TmpInst;
168409467b48Spatrick MCInstLowering.Lower(MI, TmpInst);
168509467b48Spatrick EmitToStreamer(*OutStreamer, TmpInst);
168609467b48Spatrick }
168709467b48Spatrick
168809467b48Spatrick // Force static initialization.
LLVMInitializeAArch64AsmPrinter()168909467b48Spatrick extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
169009467b48Spatrick RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
169109467b48Spatrick RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
169209467b48Spatrick RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
169309467b48Spatrick RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
169409467b48Spatrick RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
169509467b48Spatrick }
1696