1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the AArch64 assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64.h"
15 #include "AArch64MCInstLower.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "MCTargetDesc/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64MCExpr.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "TargetInfo/AArch64TargetInfo.h"
26 #include "Utils/AArch64BaseInfo.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/BinaryFormat/COFF.h"
33 #include "llvm/BinaryFormat/ELF.h"
34 #include "llvm/CodeGen/AsmPrinter.h"
35 #include "llvm/CodeGen/FaultMaps.h"
36 #include "llvm/CodeGen/MachineBasicBlock.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineJumpTableInfo.h"
40 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
41 #include "llvm/CodeGen/MachineOperand.h"
42 #include "llvm/CodeGen/StackMaps.h"
43 #include "llvm/CodeGen/TargetRegisterInfo.h"
44 #include "llvm/IR/DataLayout.h"
45 #include "llvm/IR/DebugInfoMetadata.h"
46 #include "llvm/MC/MCAsmInfo.h"
47 #include "llvm/MC/MCContext.h"
48 #include "llvm/MC/MCInst.h"
49 #include "llvm/MC/MCInstBuilder.h"
50 #include "llvm/MC/MCSectionELF.h"
51 #include "llvm/MC/MCStreamer.h"
52 #include "llvm/MC/MCSymbol.h"
53 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/ErrorHandling.h"
55 #include "llvm/Support/TargetRegistry.h"
56 #include "llvm/Support/raw_ostream.h"
57 #include "llvm/Target/TargetMachine.h"
58 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
59 #include <algorithm>
60 #include <cassert>
61 #include <cstdint>
62 #include <map>
63 #include <memory>
64 
65 using namespace llvm;
66 
67 #define DEBUG_TYPE "asm-printer"
68 
69 namespace {
70 
71 class AArch64AsmPrinter : public AsmPrinter {
72   AArch64MCInstLower MCInstLowering;
73   StackMaps SM;
74   FaultMaps FM;
75   const AArch64Subtarget *STI;
76 
77 public:
AArch64AsmPrinter(TargetMachine & TM,std::unique_ptr<MCStreamer> Streamer)78   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
79       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
80         SM(*this), FM(*this) {}
81 
getPassName() const82   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
83 
84   /// Wrapper for MCInstLowering.lowerOperand() for the
85   /// tblgen'erated pseudo lowering.
lowerOperand(const MachineOperand & MO,MCOperand & MCOp) const86   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
87     return MCInstLowering.lowerOperand(MO, MCOp);
88   }
89 
90   void emitStartOfAsmFile(Module &M) override;
91   void emitJumpTableInfo() override;
92 
93   void emitFunctionEntryLabel() override;
94 
95   void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
96 
97   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
98                      const MachineInstr &MI);
99   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
100                        const MachineInstr &MI);
101   void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
102                        const MachineInstr &MI);
103   void LowerFAULTING_OP(const MachineInstr &MI);
104 
105   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
106   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
107   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
108 
109   typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
110   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
111   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
112   void emitHwasanMemaccessSymbols(Module &M);
113 
114   void emitSled(const MachineInstr &MI, SledKind Kind);
115 
116   /// tblgen'erated driver function for lowering simple MI->MC
117   /// pseudo instructions.
118   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
119                                    const MachineInstr *MI);
120 
121   void emitInstruction(const MachineInstr *MI) override;
122 
123   void emitFunctionHeaderComment() override;
124 
getAnalysisUsage(AnalysisUsage & AU) const125   void getAnalysisUsage(AnalysisUsage &AU) const override {
126     AsmPrinter::getAnalysisUsage(AU);
127     AU.setPreservesAll();
128   }
129 
runOnMachineFunction(MachineFunction & MF)130   bool runOnMachineFunction(MachineFunction &MF) override {
131     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
132     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
133 
134     SetupMachineFunction(MF);
135 
136     if (STI->isTargetCOFF()) {
137       bool Internal = MF.getFunction().hasInternalLinkage();
138       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
139                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
140       int Type =
141         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
142 
143       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
144       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
145       OutStreamer->EmitCOFFSymbolType(Type);
146       OutStreamer->EndCOFFSymbolDef();
147     }
148 
149     // Emit the rest of the function body.
150     emitFunctionBody();
151 
152     // Emit the XRay table for this function.
153     emitXRayTable();
154 
155     // We didn't modify anything.
156     return false;
157   }
158 
159 private:
160   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
161   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
162   bool printAsmRegInClass(const MachineOperand &MO,
163                           const TargetRegisterClass *RC, unsigned AltName,
164                           raw_ostream &O);
165 
166   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
167                        const char *ExtraCode, raw_ostream &O) override;
168   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
169                              const char *ExtraCode, raw_ostream &O) override;
170 
171   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
172 
173   void emitFunctionBodyEnd() override;
174 
175   MCSymbol *GetCPISymbol(unsigned CPID) const override;
176   void emitEndOfAsmFile(Module &M) override;
177 
178   AArch64FunctionInfo *AArch64FI = nullptr;
179 
180   /// Emit the LOHs contained in AArch64FI.
181   void emitLOHs();
182 
183   /// Emit instruction to set float register to zero.
184   void emitFMov0(const MachineInstr &MI);
185 
186   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
187 
188   MInstToMCSymbol LOHInstToLabel;
189 };
190 
191 } // end anonymous namespace
192 
emitStartOfAsmFile(Module & M)193 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
194   const Triple &TT = TM.getTargetTriple();
195 
196   if (TT.isOSBinFormatCOFF()) {
197     // Emit an absolute @feat.00 symbol.  This appears to be some kind of
198     // compiler features bitfield read by link.exe.
199     MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00"));
200     OutStreamer->BeginCOFFSymbolDef(S);
201     OutStreamer->EmitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC);
202     OutStreamer->EmitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL);
203     OutStreamer->EndCOFFSymbolDef();
204     int64_t Feat00Flags = 0;
205 
206     if (M.getModuleFlag("cfguard")) {
207       Feat00Flags |= 0x800; // Object is CFG-aware.
208     }
209 
210     if (M.getModuleFlag("ehcontguard")) {
211       Feat00Flags |= 0x4000; // Object also has EHCont.
212     }
213 
214     OutStreamer->emitSymbolAttribute(S, MCSA_Global);
215     OutStreamer->emitAssignment(
216         S, MCConstantExpr::create(Feat00Flags, MMI->getContext()));
217   }
218 
219   if (!TT.isOSBinFormatELF())
220     return;
221 
222   // Assemble feature flags that may require creation of a note section.
223   unsigned Flags = 0;
224   if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
225           M.getModuleFlag("branch-target-enforcement")))
226     if (BTE->getZExtValue())
227       Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
228 
229   if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
230           M.getModuleFlag("sign-return-address")))
231     if (Sign->getZExtValue())
232       Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
233 
234   if (Flags == 0)
235     return;
236 
237   // Emit a .note.gnu.property section with the flags.
238   if (auto *TS = static_cast<AArch64TargetStreamer *>(
239           OutStreamer->getTargetStreamer()))
240     TS->emitNoteSection(Flags);
241 }
242 
emitFunctionHeaderComment()243 void AArch64AsmPrinter::emitFunctionHeaderComment() {
244   const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
245   Optional<std::string> OutlinerString = FI->getOutliningStyle();
246   if (OutlinerString != None)
247     OutStreamer->GetCommentOS() << ' ' << OutlinerString;
248 }
249 
LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr & MI)250 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
251 {
252   const Function &F = MF->getFunction();
253   if (F.hasFnAttribute("patchable-function-entry")) {
254     unsigned Num;
255     if (F.getFnAttribute("patchable-function-entry")
256             .getValueAsString()
257             .getAsInteger(10, Num))
258       return;
259     emitNops(Num);
260     return;
261   }
262 
263   emitSled(MI, SledKind::FUNCTION_ENTER);
264 }
265 
LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr & MI)266 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
267   emitSled(MI, SledKind::FUNCTION_EXIT);
268 }
269 
LowerPATCHABLE_TAIL_CALL(const MachineInstr & MI)270 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
271   emitSled(MI, SledKind::TAIL_CALL);
272 }
273 
emitSled(const MachineInstr & MI,SledKind Kind)274 void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
275   static const int8_t NoopsInSledCount = 7;
276   // We want to emit the following pattern:
277   //
278   // .Lxray_sled_N:
279   //   ALIGN
280   //   B #32
281   //   ; 7 NOP instructions (28 bytes)
282   // .tmpN
283   //
284   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
285   // over the full 32 bytes (8 instructions) with the following pattern:
286   //
287   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
288   //   LDR W0, #12 ; W0 := function ID
289   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
290   //   BLR X16 ; call the tracing trampoline
291   //   ;DATA: 32 bits of function ID
292   //   ;DATA: lower 32 bits of the address of the trampoline
293   //   ;DATA: higher 32 bits of the address of the trampoline
294   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
295   //
296   OutStreamer->emitCodeAlignment(4);
297   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
298   OutStreamer->emitLabel(CurSled);
299   auto Target = OutContext.createTempSymbol();
300 
301   // Emit "B #32" instruction, which jumps over the next 28 bytes.
302   // The operand has to be the number of 4-byte instructions to jump over,
303   // including the current instruction.
304   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
305 
306   for (int8_t I = 0; I < NoopsInSledCount; I++)
307     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
308 
309   OutStreamer->emitLabel(Target);
310   recordSled(CurSled, MI, Kind, 2);
311 }
312 
LowerHWASAN_CHECK_MEMACCESS(const MachineInstr & MI)313 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
314   Register Reg = MI.getOperand(0).getReg();
315   bool IsShort =
316       MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
317   uint32_t AccessInfo = MI.getOperand(1).getImm();
318   MCSymbol *&Sym =
319       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
320   if (!Sym) {
321     // FIXME: Make this work on non-ELF.
322     if (!TM.getTargetTriple().isOSBinFormatELF())
323       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
324 
325     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
326                           utostr(AccessInfo);
327     if (IsShort)
328       SymName += "_short_v2";
329     Sym = OutContext.getOrCreateSymbol(SymName);
330   }
331 
332   EmitToStreamer(*OutStreamer,
333                  MCInstBuilder(AArch64::BL)
334                      .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
335 }
336 
emitHwasanMemaccessSymbols(Module & M)337 void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
338   if (HwasanMemaccessSymbols.empty())
339     return;
340 
341   const Triple &TT = TM.getTargetTriple();
342   assert(TT.isOSBinFormatELF());
343   std::unique_ptr<MCSubtargetInfo> STI(
344       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
345   assert(STI && "Unable to create subtarget info");
346 
347   MCSymbol *HwasanTagMismatchV1Sym =
348       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
349   MCSymbol *HwasanTagMismatchV2Sym =
350       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
351 
352   const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
353       MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
354   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
355       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
356 
357   for (auto &P : HwasanMemaccessSymbols) {
358     unsigned Reg = std::get<0>(P.first);
359     bool IsShort = std::get<1>(P.first);
360     uint32_t AccessInfo = std::get<2>(P.first);
361     const MCSymbolRefExpr *HwasanTagMismatchRef =
362         IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
363     MCSymbol *Sym = P.second;
364 
365     bool HasMatchAllTag =
366         (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
367     uint8_t MatchAllTag =
368         (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
369     unsigned Size =
370         1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
371     bool CompileKernel =
372         (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
373 
374     OutStreamer->SwitchSection(OutContext.getELFSection(
375         ".text.hot", ELF::SHT_PROGBITS,
376         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
377         Sym->getName(), /*IsComdat=*/true));
378 
379     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
380     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
381     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
382     OutStreamer->emitLabel(Sym);
383 
384     OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri)
385                                      .addReg(AArch64::X16)
386                                      .addReg(Reg)
387                                      .addImm(4)
388                                      .addImm(55),
389                                  *STI);
390     OutStreamer->emitInstruction(
391         MCInstBuilder(AArch64::LDRBBroX)
392             .addReg(AArch64::W16)
393             .addReg(IsShort ? AArch64::X20 : AArch64::X9)
394             .addReg(AArch64::X16)
395             .addImm(0)
396             .addImm(0),
397         *STI);
398     OutStreamer->emitInstruction(
399         MCInstBuilder(AArch64::SUBSXrs)
400             .addReg(AArch64::XZR)
401             .addReg(AArch64::X16)
402             .addReg(Reg)
403             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
404         *STI);
405     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
406     OutStreamer->emitInstruction(
407         MCInstBuilder(AArch64::Bcc)
408             .addImm(AArch64CC::NE)
409             .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
410                                              OutContext)),
411         *STI);
412     MCSymbol *ReturnSym = OutContext.createTempSymbol();
413     OutStreamer->emitLabel(ReturnSym);
414     OutStreamer->emitInstruction(
415         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
416     OutStreamer->emitLabel(HandleMismatchOrPartialSym);
417 
418     if (HasMatchAllTag) {
419       OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
420                                        .addReg(AArch64::X16)
421                                        .addReg(Reg)
422                                        .addImm(56)
423                                        .addImm(63),
424                                    *STI);
425       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
426                                        .addReg(AArch64::XZR)
427                                        .addReg(AArch64::X16)
428                                        .addImm(MatchAllTag)
429                                        .addImm(0),
430                                    *STI);
431       OutStreamer->emitInstruction(
432           MCInstBuilder(AArch64::Bcc)
433               .addImm(AArch64CC::EQ)
434               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
435           *STI);
436     }
437 
438     if (IsShort) {
439       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
440                                        .addReg(AArch64::WZR)
441                                        .addReg(AArch64::W16)
442                                        .addImm(15)
443                                        .addImm(0),
444                                    *STI);
445       MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
446       OutStreamer->emitInstruction(
447           MCInstBuilder(AArch64::Bcc)
448               .addImm(AArch64CC::HI)
449               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
450           *STI);
451 
452       OutStreamer->emitInstruction(
453           MCInstBuilder(AArch64::ANDXri)
454               .addReg(AArch64::X17)
455               .addReg(Reg)
456               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
457           *STI);
458       if (Size != 1)
459         OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
460                                          .addReg(AArch64::X17)
461                                          .addReg(AArch64::X17)
462                                          .addImm(Size - 1)
463                                          .addImm(0),
464                                      *STI);
465       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
466                                        .addReg(AArch64::WZR)
467                                        .addReg(AArch64::W16)
468                                        .addReg(AArch64::W17)
469                                        .addImm(0),
470                                    *STI);
471       OutStreamer->emitInstruction(
472           MCInstBuilder(AArch64::Bcc)
473               .addImm(AArch64CC::LS)
474               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
475           *STI);
476 
477       OutStreamer->emitInstruction(
478           MCInstBuilder(AArch64::ORRXri)
479               .addReg(AArch64::X16)
480               .addReg(Reg)
481               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
482           *STI);
483       OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
484                                        .addReg(AArch64::W16)
485                                        .addReg(AArch64::X16)
486                                        .addImm(0),
487                                    *STI);
488       OutStreamer->emitInstruction(
489           MCInstBuilder(AArch64::SUBSXrs)
490               .addReg(AArch64::XZR)
491               .addReg(AArch64::X16)
492               .addReg(Reg)
493               .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
494           *STI);
495       OutStreamer->emitInstruction(
496           MCInstBuilder(AArch64::Bcc)
497               .addImm(AArch64CC::EQ)
498               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
499           *STI);
500 
501       OutStreamer->emitLabel(HandleMismatchSym);
502     }
503 
504     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
505                                      .addReg(AArch64::SP)
506                                      .addReg(AArch64::X0)
507                                      .addReg(AArch64::X1)
508                                      .addReg(AArch64::SP)
509                                      .addImm(-32),
510                                  *STI);
511     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
512                                      .addReg(AArch64::FP)
513                                      .addReg(AArch64::LR)
514                                      .addReg(AArch64::SP)
515                                      .addImm(29),
516                                  *STI);
517 
518     if (Reg != AArch64::X0)
519       OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
520                                        .addReg(AArch64::X0)
521                                        .addReg(AArch64::XZR)
522                                        .addReg(Reg)
523                                        .addImm(0),
524                                    *STI);
525     OutStreamer->emitInstruction(
526         MCInstBuilder(AArch64::MOVZXi)
527             .addReg(AArch64::X1)
528             .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask)
529             .addImm(0),
530         *STI);
531 
532     if (CompileKernel) {
533       // The Linux kernel's dynamic loader doesn't support GOT relative
534       // relocations, but it doesn't support late binding either, so just call
535       // the function directly.
536       OutStreamer->emitInstruction(
537           MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
538     } else {
539       // Intentionally load the GOT entry and branch to it, rather than possibly
540       // late binding the function, which may clobber the registers before we
541       // have a chance to save them.
542       OutStreamer->emitInstruction(
543           MCInstBuilder(AArch64::ADRP)
544               .addReg(AArch64::X16)
545               .addExpr(AArch64MCExpr::create(
546                   HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
547                   OutContext)),
548           *STI);
549       OutStreamer->emitInstruction(
550           MCInstBuilder(AArch64::LDRXui)
551               .addReg(AArch64::X16)
552               .addReg(AArch64::X16)
553               .addExpr(AArch64MCExpr::create(
554                   HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
555                   OutContext)),
556           *STI);
557       OutStreamer->emitInstruction(
558           MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
559     }
560   }
561 }
562 
emitEndOfAsmFile(Module & M)563 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
564   emitHwasanMemaccessSymbols(M);
565 
566   const Triple &TT = TM.getTargetTriple();
567   if (TT.isOSBinFormatMachO()) {
568     // Funny Darwin hack: This flag tells the linker that no global symbols
569     // contain code that falls through to other global symbols (e.g. the obvious
570     // implementation of multiple entry points).  If this doesn't occur, the
571     // linker can safely perform dead code stripping.  Since LLVM never
572     // generates code that does this, it is always safe to set.
573     OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
574   }
575 
576   // Emit stack and fault map information.
577   emitStackMaps(SM);
578   FM.serializeToFaultMapSection();
579 
580 }
581 
emitLOHs()582 void AArch64AsmPrinter::emitLOHs() {
583   SmallVector<MCSymbol *, 3> MCArgs;
584 
585   for (const auto &D : AArch64FI->getLOHContainer()) {
586     for (const MachineInstr *MI : D.getArgs()) {
587       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
588       assert(LabelIt != LOHInstToLabel.end() &&
589              "Label hasn't been inserted for LOH related instruction");
590       MCArgs.push_back(LabelIt->second);
591     }
592     OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
593     MCArgs.clear();
594   }
595 }
596 
emitFunctionBodyEnd()597 void AArch64AsmPrinter::emitFunctionBodyEnd() {
598   if (!AArch64FI->getLOHRelated().empty())
599     emitLOHs();
600 }
601 
602 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
GetCPISymbol(unsigned CPID) const603 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
604   // Darwin uses a linker-private symbol name for constant-pools (to
605   // avoid addends on the relocation?), ELF has no such concept and
606   // uses a normal private symbol.
607   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
608     return OutContext.getOrCreateSymbol(
609         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
610         Twine(getFunctionNumber()) + "_" + Twine(CPID));
611 
612   return AsmPrinter::GetCPISymbol(CPID);
613 }
614 
printOperand(const MachineInstr * MI,unsigned OpNum,raw_ostream & O)615 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
616                                      raw_ostream &O) {
617   const MachineOperand &MO = MI->getOperand(OpNum);
618   switch (MO.getType()) {
619   default:
620     llvm_unreachable("<unknown operand type>");
621   case MachineOperand::MO_Register: {
622     Register Reg = MO.getReg();
623     assert(Register::isPhysicalRegister(Reg));
624     assert(!MO.getSubReg() && "Subregs should be eliminated!");
625     O << AArch64InstPrinter::getRegisterName(Reg);
626     break;
627   }
628   case MachineOperand::MO_Immediate: {
629     O << MO.getImm();
630     break;
631   }
632   case MachineOperand::MO_GlobalAddress: {
633     PrintSymbolOperand(MO, O);
634     break;
635   }
636   case MachineOperand::MO_BlockAddress: {
637     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
638     Sym->print(O, MAI);
639     break;
640   }
641   }
642 }
643 
printAsmMRegister(const MachineOperand & MO,char Mode,raw_ostream & O)644 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
645                                           raw_ostream &O) {
646   Register Reg = MO.getReg();
647   switch (Mode) {
648   default:
649     return true; // Unknown mode.
650   case 'w':
651     Reg = getWRegFromXReg(Reg);
652     break;
653   case 'x':
654     Reg = getXRegFromWReg(Reg);
655     break;
656   case 't':
657     Reg = getXRegFromXRegTuple(Reg);
658     break;
659   }
660 
661   O << AArch64InstPrinter::getRegisterName(Reg);
662   return false;
663 }
664 
665 // Prints the register in MO using class RC using the offset in the
666 // new register class. This should not be used for cross class
667 // printing.
printAsmRegInClass(const MachineOperand & MO,const TargetRegisterClass * RC,unsigned AltName,raw_ostream & O)668 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
669                                            const TargetRegisterClass *RC,
670                                            unsigned AltName, raw_ostream &O) {
671   assert(MO.isReg() && "Should only get here with a register!");
672   const TargetRegisterInfo *RI = STI->getRegisterInfo();
673   Register Reg = MO.getReg();
674   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
675   if (!RI->regsOverlap(RegToPrint, Reg))
676     return true;
677   O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
678   return false;
679 }
680 
PrintAsmOperand(const MachineInstr * MI,unsigned OpNum,const char * ExtraCode,raw_ostream & O)681 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
682                                         const char *ExtraCode, raw_ostream &O) {
683   const MachineOperand &MO = MI->getOperand(OpNum);
684 
685   // First try the generic code, which knows about modifiers like 'c' and 'n'.
686   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
687     return false;
688 
689   // Does this asm operand have a single letter operand modifier?
690   if (ExtraCode && ExtraCode[0]) {
691     if (ExtraCode[1] != 0)
692       return true; // Unknown modifier.
693 
694     switch (ExtraCode[0]) {
695     default:
696       return true; // Unknown modifier.
697     case 'w':      // Print W register
698     case 'x':      // Print X register
699       if (MO.isReg())
700         return printAsmMRegister(MO, ExtraCode[0], O);
701       if (MO.isImm() && MO.getImm() == 0) {
702         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
703         O << AArch64InstPrinter::getRegisterName(Reg);
704         return false;
705       }
706       printOperand(MI, OpNum, O);
707       return false;
708     case 'b': // Print B register.
709     case 'h': // Print H register.
710     case 's': // Print S register.
711     case 'd': // Print D register.
712     case 'q': // Print Q register.
713     case 'z': // Print Z register.
714       if (MO.isReg()) {
715         const TargetRegisterClass *RC;
716         switch (ExtraCode[0]) {
717         case 'b':
718           RC = &AArch64::FPR8RegClass;
719           break;
720         case 'h':
721           RC = &AArch64::FPR16RegClass;
722           break;
723         case 's':
724           RC = &AArch64::FPR32RegClass;
725           break;
726         case 'd':
727           RC = &AArch64::FPR64RegClass;
728           break;
729         case 'q':
730           RC = &AArch64::FPR128RegClass;
731           break;
732         case 'z':
733           RC = &AArch64::ZPRRegClass;
734           break;
735         default:
736           return true;
737         }
738         return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
739       }
740       printOperand(MI, OpNum, O);
741       return false;
742     }
743   }
744 
745   // According to ARM, we should emit x and v registers unless we have a
746   // modifier.
747   if (MO.isReg()) {
748     Register Reg = MO.getReg();
749 
750     // If this is a w or x register, print an x register.
751     if (AArch64::GPR32allRegClass.contains(Reg) ||
752         AArch64::GPR64allRegClass.contains(Reg))
753       return printAsmMRegister(MO, 'x', O);
754 
755     // If this is an x register tuple, print an x register.
756     if (AArch64::GPR64x8ClassRegClass.contains(Reg))
757       return printAsmMRegister(MO, 't', O);
758 
759     unsigned AltName = AArch64::NoRegAltName;
760     const TargetRegisterClass *RegClass;
761     if (AArch64::ZPRRegClass.contains(Reg)) {
762       RegClass = &AArch64::ZPRRegClass;
763     } else if (AArch64::PPRRegClass.contains(Reg)) {
764       RegClass = &AArch64::PPRRegClass;
765     } else {
766       RegClass = &AArch64::FPR128RegClass;
767       AltName = AArch64::vreg;
768     }
769 
770     // If this is a b, h, s, d, or q register, print it as a v register.
771     return printAsmRegInClass(MO, RegClass, AltName, O);
772   }
773 
774   printOperand(MI, OpNum, O);
775   return false;
776 }
777 
PrintAsmMemoryOperand(const MachineInstr * MI,unsigned OpNum,const char * ExtraCode,raw_ostream & O)778 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
779                                               unsigned OpNum,
780                                               const char *ExtraCode,
781                                               raw_ostream &O) {
782   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
783     return true; // Unknown modifier.
784 
785   const MachineOperand &MO = MI->getOperand(OpNum);
786   assert(MO.isReg() && "unexpected inline asm memory operand");
787   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
788   return false;
789 }
790 
PrintDebugValueComment(const MachineInstr * MI,raw_ostream & OS)791 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
792                                                raw_ostream &OS) {
793   unsigned NOps = MI->getNumOperands();
794   assert(NOps == 4);
795   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
796   // cast away const; DIetc do not take const operands for some reason.
797   OS << MI->getDebugVariable()->getName();
798   OS << " <- ";
799   // Frame address.  Currently handles register +- offset only.
800   assert(MI->isIndirectDebugValue());
801   OS << '[';
802   for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(),
803                                          MI->debug_operands().end());
804        I < E; ++I) {
805     if (I != 0)
806       OS << ", ";
807     printOperand(MI, I, OS);
808   }
809   OS << ']';
810   OS << "+";
811   printOperand(MI, NOps - 2, OS);
812 }
813 
emitJumpTableInfo()814 void AArch64AsmPrinter::emitJumpTableInfo() {
815   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
816   if (!MJTI) return;
817 
818   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
819   if (JT.empty()) return;
820 
821   const Function &F = MF->getFunction();
822   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
823   bool JTInDiffSection =
824       !STI->isTargetCOFF() ||
825       !TLOF.shouldPutJumpTableInFunctionSection(
826           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
827           F);
828   if (JTInDiffSection) {
829       // Drop it in the readonly section.
830       MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
831       OutStreamer->SwitchSection(ReadOnlySec);
832   }
833 
834   auto AFI = MF->getInfo<AArch64FunctionInfo>();
835   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
836     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
837 
838     // If this jump table was deleted, ignore it.
839     if (JTBBs.empty()) continue;
840 
841     unsigned Size = AFI->getJumpTableEntrySize(JTI);
842     emitAlignment(Align(Size));
843     OutStreamer->emitLabel(GetJTISymbol(JTI));
844 
845     const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
846     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
847 
848     for (auto *JTBB : JTBBs) {
849       const MCExpr *Value =
850           MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
851 
852       // Each entry is:
853       //     .byte/.hword (LBB - Lbase)>>2
854       // or plain:
855       //     .word LBB - Lbase
856       Value = MCBinaryExpr::createSub(Value, Base, OutContext);
857       if (Size != 4)
858         Value = MCBinaryExpr::createLShr(
859             Value, MCConstantExpr::create(2, OutContext), OutContext);
860 
861       OutStreamer->emitValue(Value, Size);
862     }
863   }
864 }
865 
emitFunctionEntryLabel()866 void AArch64AsmPrinter::emitFunctionEntryLabel() {
867   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
868       MF->getFunction().getCallingConv() ==
869           CallingConv::AArch64_SVE_VectorCall ||
870       STI->getRegisterInfo()->hasSVEArgsOrReturn(MF)) {
871     auto *TS =
872         static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
873     TS->emitDirectiveVariantPCS(CurrentFnSym);
874   }
875 
876   return AsmPrinter::emitFunctionEntryLabel();
877 }
878 
879 /// Small jump tables contain an unsigned byte or half, representing the offset
880 /// from the lowest-addressed possible destination to the desired basic
881 /// block. Since all instructions are 4-byte aligned, this is further compressed
882 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
883 /// materialize the correct destination we need:
884 ///
885 ///             adr xDest, .LBB0_0
886 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
887 ///             add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
LowerJumpTableDest(llvm::MCStreamer & OutStreamer,const llvm::MachineInstr & MI)888 void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
889                                            const llvm::MachineInstr &MI) {
890   Register DestReg = MI.getOperand(0).getReg();
891   Register ScratchReg = MI.getOperand(1).getReg();
892   Register ScratchRegW =
893       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
894   Register TableReg = MI.getOperand(2).getReg();
895   Register EntryReg = MI.getOperand(3).getReg();
896   int JTIdx = MI.getOperand(4).getIndex();
897   int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
898 
899   // This has to be first because the compression pass based its reachability
900   // calculations on the start of the JumpTableDest instruction.
901   auto Label =
902       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
903 
904   // If we don't already have a symbol to use as the base, use the ADR
905   // instruction itself.
906   if (!Label) {
907     Label = MF->getContext().createTempSymbol();
908     AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
909     OutStreamer.emitLabel(Label);
910   }
911 
912   auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
913   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
914                                   .addReg(DestReg)
915                                   .addExpr(LabelExpr));
916 
917   // Load the number of instruction-steps to offset from the label.
918   unsigned LdrOpcode;
919   switch (Size) {
920   case 1: LdrOpcode = AArch64::LDRBBroX; break;
921   case 2: LdrOpcode = AArch64::LDRHHroX; break;
922   case 4: LdrOpcode = AArch64::LDRSWroX; break;
923   default:
924     llvm_unreachable("Unknown jump table size");
925   }
926 
927   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
928                                   .addReg(Size == 4 ? ScratchReg : ScratchRegW)
929                                   .addReg(TableReg)
930                                   .addReg(EntryReg)
931                                   .addImm(0)
932                                   .addImm(Size == 1 ? 0 : 1));
933 
934   // Add to the already materialized base label address, multiplying by 4 if
935   // compressed.
936   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
937                                   .addReg(DestReg)
938                                   .addReg(DestReg)
939                                   .addReg(ScratchReg)
940                                   .addImm(Size == 4 ? 0 : 2));
941 }
942 
LowerSTACKMAP(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)943 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
944                                       const MachineInstr &MI) {
945   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
946 
947   auto &Ctx = OutStreamer.getContext();
948   MCSymbol *MILabel = Ctx.createTempSymbol();
949   OutStreamer.emitLabel(MILabel);
950 
951   SM.recordStackMap(*MILabel, MI);
952   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
953 
954   // Scan ahead to trim the shadow.
955   const MachineBasicBlock &MBB = *MI.getParent();
956   MachineBasicBlock::const_iterator MII(MI);
957   ++MII;
958   while (NumNOPBytes > 0) {
959     if (MII == MBB.end() || MII->isCall() ||
960         MII->getOpcode() == AArch64::DBG_VALUE ||
961         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
962         MII->getOpcode() == TargetOpcode::STACKMAP)
963       break;
964     ++MII;
965     NumNOPBytes -= 4;
966   }
967 
968   // Emit nops.
969   for (unsigned i = 0; i < NumNOPBytes; i += 4)
970     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
971 }
972 
973 // Lower a patchpoint of the form:
974 // [<def>], <id>, <numBytes>, <target>, <numArgs>
LowerPATCHPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)975 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
976                                         const MachineInstr &MI) {
977   auto &Ctx = OutStreamer.getContext();
978   MCSymbol *MILabel = Ctx.createTempSymbol();
979   OutStreamer.emitLabel(MILabel);
980   SM.recordPatchPoint(*MILabel, MI);
981 
982   PatchPointOpers Opers(&MI);
983 
984   int64_t CallTarget = Opers.getCallTarget().getImm();
985   unsigned EncodedBytes = 0;
986   if (CallTarget) {
987     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
988            "High 16 bits of call target should be zero.");
989     Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
990     EncodedBytes = 16;
991     // Materialize the jump address:
992     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
993                                     .addReg(ScratchReg)
994                                     .addImm((CallTarget >> 32) & 0xFFFF)
995                                     .addImm(32));
996     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
997                                     .addReg(ScratchReg)
998                                     .addReg(ScratchReg)
999                                     .addImm((CallTarget >> 16) & 0xFFFF)
1000                                     .addImm(16));
1001     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
1002                                     .addReg(ScratchReg)
1003                                     .addReg(ScratchReg)
1004                                     .addImm(CallTarget & 0xFFFF)
1005                                     .addImm(0));
1006     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
1007   }
1008   // Emit padding.
1009   unsigned NumBytes = Opers.getNumPatchBytes();
1010   assert(NumBytes >= EncodedBytes &&
1011          "Patchpoint can't request size less than the length of a call.");
1012   assert((NumBytes - EncodedBytes) % 4 == 0 &&
1013          "Invalid number of NOP bytes requested!");
1014   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1015     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1016 }
1017 
LowerSTATEPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)1018 void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1019                                         const MachineInstr &MI) {
1020   StatepointOpers SOpers(&MI);
1021   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1022     assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1023     for (unsigned i = 0; i < PatchBytes; i += 4)
1024       EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1025   } else {
1026     // Lower call target and choose correct opcode
1027     const MachineOperand &CallTarget = SOpers.getCallTarget();
1028     MCOperand CallTargetMCOp;
1029     unsigned CallOpcode;
1030     switch (CallTarget.getType()) {
1031     case MachineOperand::MO_GlobalAddress:
1032     case MachineOperand::MO_ExternalSymbol:
1033       MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1034       CallOpcode = AArch64::BL;
1035       break;
1036     case MachineOperand::MO_Immediate:
1037       CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1038       CallOpcode = AArch64::BL;
1039       break;
1040     case MachineOperand::MO_Register:
1041       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1042       CallOpcode = AArch64::BLR;
1043       break;
1044     default:
1045       llvm_unreachable("Unsupported operand type in statepoint call target");
1046       break;
1047     }
1048 
1049     EmitToStreamer(OutStreamer,
1050                    MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1051   }
1052 
1053   auto &Ctx = OutStreamer.getContext();
1054   MCSymbol *MILabel = Ctx.createTempSymbol();
1055   OutStreamer.emitLabel(MILabel);
1056   SM.recordStatepoint(*MILabel, MI);
1057 }
1058 
LowerFAULTING_OP(const MachineInstr & FaultingMI)1059 void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1060   // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1061   //                  <opcode>, <operands>
1062 
1063   Register DefRegister = FaultingMI.getOperand(0).getReg();
1064   FaultMaps::FaultKind FK =
1065       static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1066   MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1067   unsigned Opcode = FaultingMI.getOperand(3).getImm();
1068   unsigned OperandsBeginIdx = 4;
1069 
1070   auto &Ctx = OutStreamer->getContext();
1071   MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1072   OutStreamer->emitLabel(FaultingLabel);
1073 
1074   assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1075   FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1076 
1077   MCInst MI;
1078   MI.setOpcode(Opcode);
1079 
1080   if (DefRegister != (Register)0)
1081     MI.addOperand(MCOperand::createReg(DefRegister));
1082 
1083   for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx,
1084             E = FaultingMI.operands_end();
1085        I != E; ++I) {
1086     MCOperand Dest;
1087     lowerOperand(*I, Dest);
1088     MI.addOperand(Dest);
1089   }
1090 
1091   OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1092   OutStreamer->emitInstruction(MI, getSubtargetInfo());
1093 }
1094 
emitFMov0(const MachineInstr & MI)1095 void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1096   Register DestReg = MI.getOperand(0).getReg();
1097   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
1098     // Convert H/S register to corresponding D register
1099     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1100       DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1101     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1102       DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1103     else
1104       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1105 
1106     MCInst MOVI;
1107     MOVI.setOpcode(AArch64::MOVID);
1108     MOVI.addOperand(MCOperand::createReg(DestReg));
1109     MOVI.addOperand(MCOperand::createImm(0));
1110     EmitToStreamer(*OutStreamer, MOVI);
1111   } else {
1112     MCInst FMov;
1113     switch (MI.getOpcode()) {
1114     default: llvm_unreachable("Unexpected opcode");
1115     case AArch64::FMOVH0:
1116       FMov.setOpcode(AArch64::FMOVWHr);
1117       FMov.addOperand(MCOperand::createReg(DestReg));
1118       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1119       break;
1120     case AArch64::FMOVS0:
1121       FMov.setOpcode(AArch64::FMOVWSr);
1122       FMov.addOperand(MCOperand::createReg(DestReg));
1123       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1124       break;
1125     case AArch64::FMOVD0:
1126       FMov.setOpcode(AArch64::FMOVXDr);
1127       FMov.addOperand(MCOperand::createReg(DestReg));
1128       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1129       break;
1130     }
1131     EmitToStreamer(*OutStreamer, FMov);
1132   }
1133 }
1134 
1135 // Simple pseudo-instructions have their lowering (with expansion to real
1136 // instructions) auto-generated.
1137 #include "AArch64GenMCPseudoLowering.inc"
1138 
emitInstruction(const MachineInstr * MI)1139 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1140   // Do any auto-generated pseudo lowerings.
1141   if (emitPseudoExpansionLowering(*OutStreamer, MI))
1142     return;
1143 
1144   if (AArch64FI->getLOHRelated().count(MI)) {
1145     // Generate a label for LOH related instruction
1146     MCSymbol *LOHLabel = createTempSymbol("loh");
1147     // Associate the instruction with the label
1148     LOHInstToLabel[MI] = LOHLabel;
1149     OutStreamer->emitLabel(LOHLabel);
1150   }
1151 
1152   AArch64TargetStreamer *TS =
1153     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1154   // Do any manual lowerings.
1155   switch (MI->getOpcode()) {
1156   default:
1157     break;
1158   case AArch64::HINT: {
1159     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1160     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1161     // non-empty. If MI is the initial BTI, place the
1162     // __patchable_function_entries label after BTI.
1163     if (CurrentPatchableFunctionEntrySym &&
1164         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1165         MI == &MF->front().front()) {
1166       int64_t Imm = MI->getOperand(0).getImm();
1167       if ((Imm & 32) && (Imm & 6)) {
1168         MCInst Inst;
1169         MCInstLowering.Lower(MI, Inst);
1170         EmitToStreamer(*OutStreamer, Inst);
1171         CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1172         OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1173         return;
1174       }
1175     }
1176     break;
1177   }
1178     case AArch64::MOVMCSym: {
1179       Register DestReg = MI->getOperand(0).getReg();
1180       const MachineOperand &MO_Sym = MI->getOperand(1);
1181       MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1182       MCOperand Hi_MCSym, Lo_MCSym;
1183 
1184       Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1185       Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1186 
1187       MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1188       MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1189 
1190       MCInst MovZ;
1191       MovZ.setOpcode(AArch64::MOVZXi);
1192       MovZ.addOperand(MCOperand::createReg(DestReg));
1193       MovZ.addOperand(Hi_MCSym);
1194       MovZ.addOperand(MCOperand::createImm(16));
1195       EmitToStreamer(*OutStreamer, MovZ);
1196 
1197       MCInst MovK;
1198       MovK.setOpcode(AArch64::MOVKXi);
1199       MovK.addOperand(MCOperand::createReg(DestReg));
1200       MovK.addOperand(MCOperand::createReg(DestReg));
1201       MovK.addOperand(Lo_MCSym);
1202       MovK.addOperand(MCOperand::createImm(0));
1203       EmitToStreamer(*OutStreamer, MovK);
1204       return;
1205   }
1206   case AArch64::MOVIv2d_ns:
1207     // If the target has <rdar://problem/16473581>, lower this
1208     // instruction to movi.16b instead.
1209     if (STI->hasZeroCycleZeroingFPWorkaround() &&
1210         MI->getOperand(1).getImm() == 0) {
1211       MCInst TmpInst;
1212       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1213       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1214       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1215       EmitToStreamer(*OutStreamer, TmpInst);
1216       return;
1217     }
1218     break;
1219 
1220   case AArch64::DBG_VALUE:
1221   case AArch64::DBG_VALUE_LIST: {
1222     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1223       SmallString<128> TmpStr;
1224       raw_svector_ostream OS(TmpStr);
1225       PrintDebugValueComment(MI, OS);
1226       OutStreamer->emitRawText(StringRef(OS.str()));
1227     }
1228     return;
1229 
1230   case AArch64::EMITBKEY: {
1231       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1232       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1233           ExceptionHandlingType != ExceptionHandling::ARM)
1234         return;
1235 
1236       if (getFunctionCFISectionType(*MF) == CFISection::None)
1237         return;
1238 
1239       OutStreamer->emitCFIBKeyFrame();
1240       return;
1241     }
1242     }
1243 
1244   // Tail calls use pseudo instructions so they have the proper code-gen
1245   // attributes (isCall, isReturn, etc.). We lower them to the real
1246   // instruction here.
1247   case AArch64::TCRETURNri:
1248   case AArch64::TCRETURNriBTI:
1249   case AArch64::TCRETURNriALL: {
1250     MCInst TmpInst;
1251     TmpInst.setOpcode(AArch64::BR);
1252     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1253     EmitToStreamer(*OutStreamer, TmpInst);
1254     return;
1255   }
1256   case AArch64::TCRETURNdi: {
1257     MCOperand Dest;
1258     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1259     MCInst TmpInst;
1260     TmpInst.setOpcode(AArch64::B);
1261     TmpInst.addOperand(Dest);
1262     EmitToStreamer(*OutStreamer, TmpInst);
1263     return;
1264   }
1265   case AArch64::SpeculationBarrierISBDSBEndBB: {
1266     // Print DSB SYS + ISB
1267     MCInst TmpInstDSB;
1268     TmpInstDSB.setOpcode(AArch64::DSB);
1269     TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1270     EmitToStreamer(*OutStreamer, TmpInstDSB);
1271     MCInst TmpInstISB;
1272     TmpInstISB.setOpcode(AArch64::ISB);
1273     TmpInstISB.addOperand(MCOperand::createImm(0xf));
1274     EmitToStreamer(*OutStreamer, TmpInstISB);
1275     return;
1276   }
1277   case AArch64::SpeculationBarrierSBEndBB: {
1278     // Print SB
1279     MCInst TmpInstSB;
1280     TmpInstSB.setOpcode(AArch64::SB);
1281     EmitToStreamer(*OutStreamer, TmpInstSB);
1282     return;
1283   }
1284   case AArch64::TLSDESC_CALLSEQ: {
1285     /// lower this to:
1286     ///    adrp  x0, :tlsdesc:var
1287     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
1288     ///    add   x0, x0, #:tlsdesc_lo12:var
1289     ///    .tlsdesccall var
1290     ///    blr   x1
1291     ///    (TPIDR_EL0 offset now in x0)
1292     const MachineOperand &MO_Sym = MI->getOperand(0);
1293     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1294     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1295     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1296     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1297     MCInstLowering.lowerOperand(MO_Sym, Sym);
1298     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1299     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1300 
1301     MCInst Adrp;
1302     Adrp.setOpcode(AArch64::ADRP);
1303     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1304     Adrp.addOperand(SymTLSDesc);
1305     EmitToStreamer(*OutStreamer, Adrp);
1306 
1307     MCInst Ldr;
1308     if (STI->isTargetILP32()) {
1309       Ldr.setOpcode(AArch64::LDRWui);
1310       Ldr.addOperand(MCOperand::createReg(AArch64::W1));
1311     } else {
1312       Ldr.setOpcode(AArch64::LDRXui);
1313       Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1314     }
1315     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1316     Ldr.addOperand(SymTLSDescLo12);
1317     Ldr.addOperand(MCOperand::createImm(0));
1318     EmitToStreamer(*OutStreamer, Ldr);
1319 
1320     MCInst Add;
1321     if (STI->isTargetILP32()) {
1322       Add.setOpcode(AArch64::ADDWri);
1323       Add.addOperand(MCOperand::createReg(AArch64::W0));
1324       Add.addOperand(MCOperand::createReg(AArch64::W0));
1325     } else {
1326       Add.setOpcode(AArch64::ADDXri);
1327       Add.addOperand(MCOperand::createReg(AArch64::X0));
1328       Add.addOperand(MCOperand::createReg(AArch64::X0));
1329     }
1330     Add.addOperand(SymTLSDescLo12);
1331     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1332     EmitToStreamer(*OutStreamer, Add);
1333 
1334     // Emit a relocation-annotation. This expands to no code, but requests
1335     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1336     MCInst TLSDescCall;
1337     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1338     TLSDescCall.addOperand(Sym);
1339     EmitToStreamer(*OutStreamer, TLSDescCall);
1340 
1341     MCInst Blr;
1342     Blr.setOpcode(AArch64::BLR);
1343     Blr.addOperand(MCOperand::createReg(AArch64::X1));
1344     EmitToStreamer(*OutStreamer, Blr);
1345 
1346     return;
1347   }
1348 
1349   case AArch64::JumpTableDest32:
1350   case AArch64::JumpTableDest16:
1351   case AArch64::JumpTableDest8:
1352     LowerJumpTableDest(*OutStreamer, *MI);
1353     return;
1354 
1355   case AArch64::FMOVH0:
1356   case AArch64::FMOVS0:
1357   case AArch64::FMOVD0:
1358     emitFMov0(*MI);
1359     return;
1360 
1361   case TargetOpcode::STACKMAP:
1362     return LowerSTACKMAP(*OutStreamer, SM, *MI);
1363 
1364   case TargetOpcode::PATCHPOINT:
1365     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1366 
1367   case TargetOpcode::STATEPOINT:
1368     return LowerSTATEPOINT(*OutStreamer, SM, *MI);
1369 
1370   case TargetOpcode::FAULTING_OP:
1371     return LowerFAULTING_OP(*MI);
1372 
1373   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1374     LowerPATCHABLE_FUNCTION_ENTER(*MI);
1375     return;
1376 
1377   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1378     LowerPATCHABLE_FUNCTION_EXIT(*MI);
1379     return;
1380 
1381   case TargetOpcode::PATCHABLE_TAIL_CALL:
1382     LowerPATCHABLE_TAIL_CALL(*MI);
1383     return;
1384 
1385   case AArch64::HWASAN_CHECK_MEMACCESS:
1386   case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1387     LowerHWASAN_CHECK_MEMACCESS(*MI);
1388     return;
1389 
1390   case AArch64::SEH_StackAlloc:
1391     TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1392     return;
1393 
1394   case AArch64::SEH_SaveFPLR:
1395     TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1396     return;
1397 
1398   case AArch64::SEH_SaveFPLR_X:
1399     assert(MI->getOperand(0).getImm() < 0 &&
1400            "Pre increment SEH opcode must have a negative offset");
1401     TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1402     return;
1403 
1404   case AArch64::SEH_SaveReg:
1405     TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1406                                MI->getOperand(1).getImm());
1407     return;
1408 
1409   case AArch64::SEH_SaveReg_X:
1410     assert(MI->getOperand(1).getImm() < 0 &&
1411            "Pre increment SEH opcode must have a negative offset");
1412     TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1413                                 -MI->getOperand(1).getImm());
1414     return;
1415 
1416   case AArch64::SEH_SaveRegP:
1417     if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
1418         MI->getOperand(0).getImm() <= 28) {
1419       assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1420              "Register paired with LR must be odd");
1421       TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
1422                                     MI->getOperand(2).getImm());
1423       return;
1424     }
1425     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1426             "Non-consecutive registers not allowed for save_regp");
1427     TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1428                                 MI->getOperand(2).getImm());
1429     return;
1430 
1431   case AArch64::SEH_SaveRegP_X:
1432     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1433             "Non-consecutive registers not allowed for save_regp_x");
1434     assert(MI->getOperand(2).getImm() < 0 &&
1435            "Pre increment SEH opcode must have a negative offset");
1436     TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1437                                  -MI->getOperand(2).getImm());
1438     return;
1439 
1440   case AArch64::SEH_SaveFReg:
1441     TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1442                                 MI->getOperand(1).getImm());
1443     return;
1444 
1445   case AArch64::SEH_SaveFReg_X:
1446     assert(MI->getOperand(1).getImm() < 0 &&
1447            "Pre increment SEH opcode must have a negative offset");
1448     TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1449                                  -MI->getOperand(1).getImm());
1450     return;
1451 
1452   case AArch64::SEH_SaveFRegP:
1453     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1454             "Non-consecutive registers not allowed for save_regp");
1455     TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1456                                  MI->getOperand(2).getImm());
1457     return;
1458 
1459   case AArch64::SEH_SaveFRegP_X:
1460     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1461             "Non-consecutive registers not allowed for save_regp_x");
1462     assert(MI->getOperand(2).getImm() < 0 &&
1463            "Pre increment SEH opcode must have a negative offset");
1464     TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1465                                   -MI->getOperand(2).getImm());
1466     return;
1467 
1468   case AArch64::SEH_SetFP:
1469     TS->emitARM64WinCFISetFP();
1470     return;
1471 
1472   case AArch64::SEH_AddFP:
1473     TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1474     return;
1475 
1476   case AArch64::SEH_Nop:
1477     TS->emitARM64WinCFINop();
1478     return;
1479 
1480   case AArch64::SEH_PrologEnd:
1481     TS->emitARM64WinCFIPrologEnd();
1482     return;
1483 
1484   case AArch64::SEH_EpilogStart:
1485     TS->emitARM64WinCFIEpilogStart();
1486     return;
1487 
1488   case AArch64::SEH_EpilogEnd:
1489     TS->emitARM64WinCFIEpilogEnd();
1490     return;
1491   }
1492 
1493   // Finally, do the automated lowerings for everything else.
1494   MCInst TmpInst;
1495   MCInstLowering.Lower(MI, TmpInst);
1496   EmitToStreamer(*OutStreamer, TmpInst);
1497 }
1498 
1499 // Force static initialization.
LLVMInitializeAArch64AsmPrinter()1500 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1501   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1502   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1503   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1504   RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1505   RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1506 }
1507