1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the AArch64 assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64.h"
15 #include "AArch64MCInstLower.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "MCTargetDesc/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64MCExpr.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "TargetInfo/AArch64TargetInfo.h"
26 #include "Utils/AArch64BaseInfo.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/BinaryFormat/COFF.h"
33 #include "llvm/BinaryFormat/ELF.h"
34 #include "llvm/CodeGen/AsmPrinter.h"
35 #include "llvm/CodeGen/MachineBasicBlock.h"
36 #include "llvm/CodeGen/MachineFunction.h"
37 #include "llvm/CodeGen/MachineInstr.h"
38 #include "llvm/CodeGen/MachineJumpTableInfo.h"
39 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
40 #include "llvm/CodeGen/MachineOperand.h"
41 #include "llvm/CodeGen/StackMaps.h"
42 #include "llvm/CodeGen/TargetRegisterInfo.h"
43 #include "llvm/IR/DataLayout.h"
44 #include "llvm/IR/DebugInfoMetadata.h"
45 #include "llvm/MC/MCAsmInfo.h"
46 #include "llvm/MC/MCContext.h"
47 #include "llvm/MC/MCInst.h"
48 #include "llvm/MC/MCInstBuilder.h"
49 #include "llvm/MC/MCSectionELF.h"
50 #include "llvm/MC/MCStreamer.h"
51 #include "llvm/MC/MCSymbol.h"
52 #include "llvm/Support/Casting.h"
53 #include "llvm/Support/ErrorHandling.h"
54 #include "llvm/Support/TargetRegistry.h"
55 #include "llvm/Support/raw_ostream.h"
56 #include "llvm/Target/TargetMachine.h"
57 #include <algorithm>
58 #include <cassert>
59 #include <cstdint>
60 #include <map>
61 #include <memory>
62 
63 using namespace llvm;
64 
65 #define DEBUG_TYPE "asm-printer"
66 
67 namespace {
68 
69 class AArch64AsmPrinter : public AsmPrinter {
70   AArch64MCInstLower MCInstLowering;
71   StackMaps SM;
72   const AArch64Subtarget *STI;
73 
74 public:
75   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
76       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
77         SM(*this) {}
78 
79   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
80 
81   /// Wrapper for MCInstLowering.lowerOperand() for the
82   /// tblgen'erated pseudo lowering.
83   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
84     return MCInstLowering.lowerOperand(MO, MCOp);
85   }
86 
87   void EmitStartOfAsmFile(Module &M) override;
88   void EmitJumpTableInfo() override;
89   void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
90                           const MachineBasicBlock *MBB, unsigned JTI);
91 
92   void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI);
93 
94   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
95                      const MachineInstr &MI);
96   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
97                        const MachineInstr &MI);
98 
99   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
100   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
101   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
102 
103   typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
104   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
105   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
106   void EmitHwasanMemaccessSymbols(Module &M);
107 
108   void EmitSled(const MachineInstr &MI, SledKind Kind);
109 
110   /// tblgen'erated driver function for lowering simple MI->MC
111   /// pseudo instructions.
112   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
113                                    const MachineInstr *MI);
114 
115   void EmitInstruction(const MachineInstr *MI) override;
116 
117   void getAnalysisUsage(AnalysisUsage &AU) const override {
118     AsmPrinter::getAnalysisUsage(AU);
119     AU.setPreservesAll();
120   }
121 
122   bool runOnMachineFunction(MachineFunction &MF) override {
123     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
124     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
125 
126     SetupMachineFunction(MF);
127 
128     if (STI->isTargetCOFF()) {
129       bool Internal = MF.getFunction().hasInternalLinkage();
130       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
131                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
132       int Type =
133         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
134 
135       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
136       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
137       OutStreamer->EmitCOFFSymbolType(Type);
138       OutStreamer->EndCOFFSymbolDef();
139     }
140 
141     // Emit the rest of the function body.
142     EmitFunctionBody();
143 
144     // Emit the XRay table for this function.
145     emitXRayTable();
146 
147     // We didn't modify anything.
148     return false;
149   }
150 
151 private:
152   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
153   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
154   bool printAsmRegInClass(const MachineOperand &MO,
155                           const TargetRegisterClass *RC, unsigned AltName,
156                           raw_ostream &O);
157 
158   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
159                        const char *ExtraCode, raw_ostream &O) override;
160   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
161                              const char *ExtraCode, raw_ostream &O) override;
162 
163   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
164 
165   void EmitFunctionBodyEnd() override;
166 
167   MCSymbol *GetCPISymbol(unsigned CPID) const override;
168   void EmitEndOfAsmFile(Module &M) override;
169 
170   AArch64FunctionInfo *AArch64FI = nullptr;
171 
172   /// Emit the LOHs contained in AArch64FI.
173   void EmitLOHs();
174 
175   /// Emit instruction to set float register to zero.
176   void EmitFMov0(const MachineInstr &MI);
177 
178   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
179 
180   MInstToMCSymbol LOHInstToLabel;
181 };
182 
183 } // end anonymous namespace
184 
185 void AArch64AsmPrinter::EmitStartOfAsmFile(Module &M) {
186   if (!TM.getTargetTriple().isOSBinFormatELF())
187     return;
188 
189   // Assemble feature flags that may require creation of a note section.
190   unsigned Flags = ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI |
191                    ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
192 
193   if (any_of(M, [](const Function &F) {
194         return !F.isDeclaration() &&
195                !F.hasFnAttribute("branch-target-enforcement");
196       })) {
197     Flags &= ~ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
198   }
199 
200   if ((Flags & ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI) == 0 &&
201       any_of(M, [](const Function &F) {
202         return F.hasFnAttribute("branch-target-enforcement");
203       })) {
204     errs() << "warning: some functions compiled with BTI and some compiled "
205               "without BTI\n"
206            << "warning: not setting BTI in feature flags\n";
207   }
208 
209   if (any_of(M, [](const Function &F) {
210         if (F.isDeclaration())
211           return false;
212         Attribute A = F.getFnAttribute("sign-return-address");
213         return !A.isStringAttribute() || A.getValueAsString() == "none";
214       })) {
215     Flags &= ~ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
216   }
217 
218   if (Flags == 0)
219     return;
220 
221   // Emit a .note.gnu.property section with the flags.
222   MCSection *Cur = OutStreamer->getCurrentSectionOnly();
223   MCSection *Nt = MMI->getContext().getELFSection(
224       ".note.gnu.property", ELF::SHT_NOTE, ELF::SHF_ALLOC);
225   OutStreamer->SwitchSection(Nt);
226 
227   // Emit the note header.
228   EmitAlignment(Align(8));
229   OutStreamer->EmitIntValue(4, 4);     // data size for "GNU\0"
230   OutStreamer->EmitIntValue(4 * 4, 4); // Elf_Prop size
231   OutStreamer->EmitIntValue(ELF::NT_GNU_PROPERTY_TYPE_0, 4);
232   OutStreamer->EmitBytes(StringRef("GNU", 4)); // note name
233 
234   // Emit the PAC/BTI properties.
235   OutStreamer->EmitIntValue(ELF::GNU_PROPERTY_AARCH64_FEATURE_1_AND, 4);
236   OutStreamer->EmitIntValue(4, 4);     // data size
237   OutStreamer->EmitIntValue(Flags, 4); // data
238   OutStreamer->EmitIntValue(0, 4);     // pad
239 
240   OutStreamer->endSection(Nt);
241   OutStreamer->SwitchSection(Cur);
242 }
243 
244 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
245 {
246   const Function &F = MF->getFunction();
247   if (F.hasFnAttribute("patchable-function-entry")) {
248     unsigned Num;
249     if (F.getFnAttribute("patchable-function-entry")
250             .getValueAsString()
251             .getAsInteger(10, Num))
252       return;
253     emitNops(Num);
254     return;
255   }
256 
257   EmitSled(MI, SledKind::FUNCTION_ENTER);
258 }
259 
260 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
261 {
262   EmitSled(MI, SledKind::FUNCTION_EXIT);
263 }
264 
265 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
266 {
267   EmitSled(MI, SledKind::TAIL_CALL);
268 }
269 
270 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
271 {
272   static const int8_t NoopsInSledCount = 7;
273   // We want to emit the following pattern:
274   //
275   // .Lxray_sled_N:
276   //   ALIGN
277   //   B #32
278   //   ; 7 NOP instructions (28 bytes)
279   // .tmpN
280   //
281   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
282   // over the full 32 bytes (8 instructions) with the following pattern:
283   //
284   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
285   //   LDR W0, #12 ; W0 := function ID
286   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
287   //   BLR X16 ; call the tracing trampoline
288   //   ;DATA: 32 bits of function ID
289   //   ;DATA: lower 32 bits of the address of the trampoline
290   //   ;DATA: higher 32 bits of the address of the trampoline
291   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
292   //
293   OutStreamer->EmitCodeAlignment(4);
294   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
295   OutStreamer->EmitLabel(CurSled);
296   auto Target = OutContext.createTempSymbol();
297 
298   // Emit "B #32" instruction, which jumps over the next 28 bytes.
299   // The operand has to be the number of 4-byte instructions to jump over,
300   // including the current instruction.
301   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
302 
303   for (int8_t I = 0; I < NoopsInSledCount; I++)
304     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
305 
306   OutStreamer->EmitLabel(Target);
307   recordSled(CurSled, MI, Kind);
308 }
309 
310 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
311   Register Reg = MI.getOperand(0).getReg();
312   bool IsShort =
313       MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
314   uint32_t AccessInfo = MI.getOperand(1).getImm();
315   MCSymbol *&Sym =
316       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
317   if (!Sym) {
318     // FIXME: Make this work on non-ELF.
319     if (!TM.getTargetTriple().isOSBinFormatELF())
320       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
321 
322     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
323                           utostr(AccessInfo);
324     if (IsShort)
325       SymName += "_short";
326     Sym = OutContext.getOrCreateSymbol(SymName);
327   }
328 
329   EmitToStreamer(*OutStreamer,
330                  MCInstBuilder(AArch64::BL)
331                      .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
332 }
333 
334 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
335   if (HwasanMemaccessSymbols.empty())
336     return;
337 
338   const Triple &TT = TM.getTargetTriple();
339   assert(TT.isOSBinFormatELF());
340   std::unique_ptr<MCSubtargetInfo> STI(
341       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
342 
343   MCSymbol *HwasanTagMismatchV1Sym =
344       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
345   MCSymbol *HwasanTagMismatchV2Sym =
346       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
347 
348   const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
349       MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
350   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
351       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
352 
353   for (auto &P : HwasanMemaccessSymbols) {
354     unsigned Reg = std::get<0>(P.first);
355     bool IsShort = std::get<1>(P.first);
356     uint32_t AccessInfo = std::get<2>(P.first);
357     const MCSymbolRefExpr *HwasanTagMismatchRef =
358         IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
359     MCSymbol *Sym = P.second;
360 
361     OutStreamer->SwitchSection(OutContext.getELFSection(
362         ".text.hot", ELF::SHT_PROGBITS,
363         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
364         Sym->getName()));
365 
366     OutStreamer->EmitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
367     OutStreamer->EmitSymbolAttribute(Sym, MCSA_Weak);
368     OutStreamer->EmitSymbolAttribute(Sym, MCSA_Hidden);
369     OutStreamer->EmitLabel(Sym);
370 
371     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::UBFMXri)
372                                      .addReg(AArch64::X16)
373                                      .addReg(Reg)
374                                      .addImm(4)
375                                      .addImm(55),
376                                  *STI);
377     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBroX)
378                                      .addReg(AArch64::W16)
379                                      .addReg(AArch64::X9)
380                                      .addReg(AArch64::X16)
381                                      .addImm(0)
382                                      .addImm(0),
383                                  *STI);
384     OutStreamer->EmitInstruction(
385         MCInstBuilder(AArch64::SUBSXrs)
386             .addReg(AArch64::XZR)
387             .addReg(AArch64::X16)
388             .addReg(Reg)
389             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
390         *STI);
391     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
392     OutStreamer->EmitInstruction(
393         MCInstBuilder(AArch64::Bcc)
394             .addImm(AArch64CC::NE)
395             .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
396                                              OutContext)),
397         *STI);
398     MCSymbol *ReturnSym = OutContext.createTempSymbol();
399     OutStreamer->EmitLabel(ReturnSym);
400     OutStreamer->EmitInstruction(
401         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
402     OutStreamer->EmitLabel(HandleMismatchOrPartialSym);
403 
404     if (IsShort) {
405       OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWri)
406                                        .addReg(AArch64::WZR)
407                                        .addReg(AArch64::W16)
408                                        .addImm(15)
409                                        .addImm(0),
410                                    *STI);
411       MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
412       OutStreamer->EmitInstruction(
413           MCInstBuilder(AArch64::Bcc)
414               .addImm(AArch64CC::HI)
415               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
416           *STI);
417 
418       OutStreamer->EmitInstruction(
419           MCInstBuilder(AArch64::ANDXri)
420               .addReg(AArch64::X17)
421               .addReg(Reg)
422               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
423           *STI);
424       unsigned Size = 1 << (AccessInfo & 0xf);
425       if (Size != 1)
426         OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ADDXri)
427                                          .addReg(AArch64::X17)
428                                          .addReg(AArch64::X17)
429                                          .addImm(Size - 1)
430                                          .addImm(0),
431                                      *STI);
432       OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWrs)
433                                        .addReg(AArch64::WZR)
434                                        .addReg(AArch64::W16)
435                                        .addReg(AArch64::W17)
436                                        .addImm(0),
437                                    *STI);
438       OutStreamer->EmitInstruction(
439           MCInstBuilder(AArch64::Bcc)
440               .addImm(AArch64CC::LS)
441               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
442           *STI);
443 
444       OutStreamer->EmitInstruction(
445           MCInstBuilder(AArch64::ORRXri)
446               .addReg(AArch64::X16)
447               .addReg(Reg)
448               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
449           *STI);
450       OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBui)
451                                        .addReg(AArch64::W16)
452                                        .addReg(AArch64::X16)
453                                        .addImm(0),
454                                    *STI);
455       OutStreamer->EmitInstruction(
456           MCInstBuilder(AArch64::SUBSXrs)
457               .addReg(AArch64::XZR)
458               .addReg(AArch64::X16)
459               .addReg(Reg)
460               .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
461           *STI);
462       OutStreamer->EmitInstruction(
463           MCInstBuilder(AArch64::Bcc)
464               .addImm(AArch64CC::EQ)
465               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
466           *STI);
467 
468       OutStreamer->EmitLabel(HandleMismatchSym);
469     }
470 
471     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXpre)
472                                      .addReg(AArch64::SP)
473                                      .addReg(AArch64::X0)
474                                      .addReg(AArch64::X1)
475                                      .addReg(AArch64::SP)
476                                      .addImm(-32),
477                                  *STI);
478     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXi)
479                                      .addReg(AArch64::FP)
480                                      .addReg(AArch64::LR)
481                                      .addReg(AArch64::SP)
482                                      .addImm(29),
483                                  *STI);
484 
485     if (Reg != AArch64::X0)
486       OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ORRXrs)
487                                        .addReg(AArch64::X0)
488                                        .addReg(AArch64::XZR)
489                                        .addReg(Reg)
490                                        .addImm(0),
491                                    *STI);
492     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::MOVZXi)
493                                      .addReg(AArch64::X1)
494                                      .addImm(AccessInfo)
495                                      .addImm(0),
496                                  *STI);
497 
498     // Intentionally load the GOT entry and branch to it, rather than possibly
499     // late binding the function, which may clobber the registers before we have
500     // a chance to save them.
501     OutStreamer->EmitInstruction(
502         MCInstBuilder(AArch64::ADRP)
503             .addReg(AArch64::X16)
504             .addExpr(AArch64MCExpr::create(
505                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
506                 OutContext)),
507         *STI);
508     OutStreamer->EmitInstruction(
509         MCInstBuilder(AArch64::LDRXui)
510             .addReg(AArch64::X16)
511             .addReg(AArch64::X16)
512             .addExpr(AArch64MCExpr::create(
513                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
514                 OutContext)),
515         *STI);
516     OutStreamer->EmitInstruction(
517         MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
518   }
519 }
520 
521 void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
522   EmitHwasanMemaccessSymbols(M);
523 
524   const Triple &TT = TM.getTargetTriple();
525   if (TT.isOSBinFormatMachO()) {
526     // Funny Darwin hack: This flag tells the linker that no global symbols
527     // contain code that falls through to other global symbols (e.g. the obvious
528     // implementation of multiple entry points).  If this doesn't occur, the
529     // linker can safely perform dead code stripping.  Since LLVM never
530     // generates code that does this, it is always safe to set.
531     OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
532   }
533   emitStackMaps(SM);
534 }
535 
536 void AArch64AsmPrinter::EmitLOHs() {
537   SmallVector<MCSymbol *, 3> MCArgs;
538 
539   for (const auto &D : AArch64FI->getLOHContainer()) {
540     for (const MachineInstr *MI : D.getArgs()) {
541       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
542       assert(LabelIt != LOHInstToLabel.end() &&
543              "Label hasn't been inserted for LOH related instruction");
544       MCArgs.push_back(LabelIt->second);
545     }
546     OutStreamer->EmitLOHDirective(D.getKind(), MCArgs);
547     MCArgs.clear();
548   }
549 }
550 
551 void AArch64AsmPrinter::EmitFunctionBodyEnd() {
552   if (!AArch64FI->getLOHRelated().empty())
553     EmitLOHs();
554 }
555 
556 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
557 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
558   // Darwin uses a linker-private symbol name for constant-pools (to
559   // avoid addends on the relocation?), ELF has no such concept and
560   // uses a normal private symbol.
561   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
562     return OutContext.getOrCreateSymbol(
563         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
564         Twine(getFunctionNumber()) + "_" + Twine(CPID));
565 
566   return AsmPrinter::GetCPISymbol(CPID);
567 }
568 
569 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
570                                      raw_ostream &O) {
571   const MachineOperand &MO = MI->getOperand(OpNum);
572   switch (MO.getType()) {
573   default:
574     llvm_unreachable("<unknown operand type>");
575   case MachineOperand::MO_Register: {
576     Register Reg = MO.getReg();
577     assert(Register::isPhysicalRegister(Reg));
578     assert(!MO.getSubReg() && "Subregs should be eliminated!");
579     O << AArch64InstPrinter::getRegisterName(Reg);
580     break;
581   }
582   case MachineOperand::MO_Immediate: {
583     O << MO.getImm();
584     break;
585   }
586   case MachineOperand::MO_GlobalAddress: {
587     PrintSymbolOperand(MO, O);
588     break;
589   }
590   case MachineOperand::MO_BlockAddress: {
591     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
592     Sym->print(O, MAI);
593     break;
594   }
595   }
596 }
597 
598 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
599                                           raw_ostream &O) {
600   Register Reg = MO.getReg();
601   switch (Mode) {
602   default:
603     return true; // Unknown mode.
604   case 'w':
605     Reg = getWRegFromXReg(Reg);
606     break;
607   case 'x':
608     Reg = getXRegFromWReg(Reg);
609     break;
610   }
611 
612   O << AArch64InstPrinter::getRegisterName(Reg);
613   return false;
614 }
615 
616 // Prints the register in MO using class RC using the offset in the
617 // new register class. This should not be used for cross class
618 // printing.
619 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
620                                            const TargetRegisterClass *RC,
621                                            unsigned AltName, raw_ostream &O) {
622   assert(MO.isReg() && "Should only get here with a register!");
623   const TargetRegisterInfo *RI = STI->getRegisterInfo();
624   Register Reg = MO.getReg();
625   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
626   assert(RI->regsOverlap(RegToPrint, Reg));
627   O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
628   return false;
629 }
630 
631 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
632                                         const char *ExtraCode, raw_ostream &O) {
633   const MachineOperand &MO = MI->getOperand(OpNum);
634 
635   // First try the generic code, which knows about modifiers like 'c' and 'n'.
636   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
637     return false;
638 
639   // Does this asm operand have a single letter operand modifier?
640   if (ExtraCode && ExtraCode[0]) {
641     if (ExtraCode[1] != 0)
642       return true; // Unknown modifier.
643 
644     switch (ExtraCode[0]) {
645     default:
646       return true; // Unknown modifier.
647     case 'w':      // Print W register
648     case 'x':      // Print X register
649       if (MO.isReg())
650         return printAsmMRegister(MO, ExtraCode[0], O);
651       if (MO.isImm() && MO.getImm() == 0) {
652         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
653         O << AArch64InstPrinter::getRegisterName(Reg);
654         return false;
655       }
656       printOperand(MI, OpNum, O);
657       return false;
658     case 'b': // Print B register.
659     case 'h': // Print H register.
660     case 's': // Print S register.
661     case 'd': // Print D register.
662     case 'q': // Print Q register.
663     case 'z': // Print Z register.
664       if (MO.isReg()) {
665         const TargetRegisterClass *RC;
666         switch (ExtraCode[0]) {
667         case 'b':
668           RC = &AArch64::FPR8RegClass;
669           break;
670         case 'h':
671           RC = &AArch64::FPR16RegClass;
672           break;
673         case 's':
674           RC = &AArch64::FPR32RegClass;
675           break;
676         case 'd':
677           RC = &AArch64::FPR64RegClass;
678           break;
679         case 'q':
680           RC = &AArch64::FPR128RegClass;
681           break;
682         case 'z':
683           RC = &AArch64::ZPRRegClass;
684           break;
685         default:
686           return true;
687         }
688         return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
689       }
690       printOperand(MI, OpNum, O);
691       return false;
692     }
693   }
694 
695   // According to ARM, we should emit x and v registers unless we have a
696   // modifier.
697   if (MO.isReg()) {
698     Register Reg = MO.getReg();
699 
700     // If this is a w or x register, print an x register.
701     if (AArch64::GPR32allRegClass.contains(Reg) ||
702         AArch64::GPR64allRegClass.contains(Reg))
703       return printAsmMRegister(MO, 'x', O);
704 
705     unsigned AltName = AArch64::NoRegAltName;
706     const TargetRegisterClass *RegClass;
707     if (AArch64::ZPRRegClass.contains(Reg)) {
708       RegClass = &AArch64::ZPRRegClass;
709     } else if (AArch64::PPRRegClass.contains(Reg)) {
710       RegClass = &AArch64::PPRRegClass;
711     } else {
712       RegClass = &AArch64::FPR128RegClass;
713       AltName = AArch64::vreg;
714     }
715 
716     // If this is a b, h, s, d, or q register, print it as a v register.
717     return printAsmRegInClass(MO, RegClass, AltName, O);
718   }
719 
720   printOperand(MI, OpNum, O);
721   return false;
722 }
723 
724 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
725                                               unsigned OpNum,
726                                               const char *ExtraCode,
727                                               raw_ostream &O) {
728   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
729     return true; // Unknown modifier.
730 
731   const MachineOperand &MO = MI->getOperand(OpNum);
732   assert(MO.isReg() && "unexpected inline asm memory operand");
733   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
734   return false;
735 }
736 
737 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
738                                                raw_ostream &OS) {
739   unsigned NOps = MI->getNumOperands();
740   assert(NOps == 4);
741   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
742   // cast away const; DIetc do not take const operands for some reason.
743   OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
744             ->getName();
745   OS << " <- ";
746   // Frame address.  Currently handles register +- offset only.
747   assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
748   OS << '[';
749   printOperand(MI, 0, OS);
750   OS << '+';
751   printOperand(MI, 1, OS);
752   OS << ']';
753   OS << "+";
754   printOperand(MI, NOps - 2, OS);
755 }
756 
757 void AArch64AsmPrinter::EmitJumpTableInfo() {
758   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
759   if (!MJTI) return;
760 
761   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
762   if (JT.empty()) return;
763 
764   const Function &F = MF->getFunction();
765   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
766   bool JTInDiffSection =
767       !STI->isTargetCOFF() ||
768       !TLOF.shouldPutJumpTableInFunctionSection(
769           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
770           F);
771   if (JTInDiffSection) {
772       // Drop it in the readonly section.
773       MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
774       OutStreamer->SwitchSection(ReadOnlySec);
775   }
776 
777   auto AFI = MF->getInfo<AArch64FunctionInfo>();
778   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
779     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
780 
781     // If this jump table was deleted, ignore it.
782     if (JTBBs.empty()) continue;
783 
784     unsigned Size = AFI->getJumpTableEntrySize(JTI);
785     EmitAlignment(Align(Size));
786     OutStreamer->EmitLabel(GetJTISymbol(JTI));
787 
788     for (auto *JTBB : JTBBs)
789       emitJumpTableEntry(MJTI, JTBB, JTI);
790   }
791 }
792 
793 void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
794                                            const MachineBasicBlock *MBB,
795                                            unsigned JTI) {
796   const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
797   auto AFI = MF->getInfo<AArch64FunctionInfo>();
798   unsigned Size = AFI->getJumpTableEntrySize(JTI);
799 
800   if (Size == 4) {
801     // .word LBB - LJTI
802     const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
803     const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
804     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
805   } else {
806     // .byte (LBB - LBB) >> 2 (or .hword)
807     const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI);
808     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
809     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
810     Value = MCBinaryExpr::createLShr(
811         Value, MCConstantExpr::create(2, OutContext), OutContext);
812   }
813 
814   OutStreamer->EmitValue(Value, Size);
815 }
816 
817 /// Small jump tables contain an unsigned byte or half, representing the offset
818 /// from the lowest-addressed possible destination to the desired basic
819 /// block. Since all instructions are 4-byte aligned, this is further compressed
820 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
821 /// materialize the correct destination we need:
822 ///
823 ///             adr xDest, .LBB0_0
824 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
825 ///             add xDest, xDest, xScratch, lsl #2
826 void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer,
827                                                 const llvm::MachineInstr &MI) {
828   Register DestReg = MI.getOperand(0).getReg();
829   Register ScratchReg = MI.getOperand(1).getReg();
830   Register ScratchRegW =
831       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
832   Register TableReg = MI.getOperand(2).getReg();
833   Register EntryReg = MI.getOperand(3).getReg();
834   int JTIdx = MI.getOperand(4).getIndex();
835   bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8;
836 
837   // This has to be first because the compression pass based its reachability
838   // calculations on the start of the JumpTableDest instruction.
839   auto Label =
840       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
841   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
842                                   .addReg(DestReg)
843                                   .addExpr(MCSymbolRefExpr::create(
844                                       Label, MF->getContext())));
845 
846   // Load the number of instruction-steps to offset from the label.
847   unsigned LdrOpcode = IsByteEntry ? AArch64::LDRBBroX : AArch64::LDRHHroX;
848   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
849                                   .addReg(ScratchRegW)
850                                   .addReg(TableReg)
851                                   .addReg(EntryReg)
852                                   .addImm(0)
853                                   .addImm(IsByteEntry ? 0 : 1));
854 
855   // Multiply the steps by 4 and add to the already materialized base label
856   // address.
857   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
858                                   .addReg(DestReg)
859                                   .addReg(DestReg)
860                                   .addReg(ScratchReg)
861                                   .addImm(2));
862 }
863 
864 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
865                                       const MachineInstr &MI) {
866   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
867 
868   auto &Ctx = OutStreamer.getContext();
869   MCSymbol *MILabel = Ctx.createTempSymbol();
870   OutStreamer.EmitLabel(MILabel);
871 
872   SM.recordStackMap(*MILabel, MI);
873   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
874 
875   // Scan ahead to trim the shadow.
876   const MachineBasicBlock &MBB = *MI.getParent();
877   MachineBasicBlock::const_iterator MII(MI);
878   ++MII;
879   while (NumNOPBytes > 0) {
880     if (MII == MBB.end() || MII->isCall() ||
881         MII->getOpcode() == AArch64::DBG_VALUE ||
882         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
883         MII->getOpcode() == TargetOpcode::STACKMAP)
884       break;
885     ++MII;
886     NumNOPBytes -= 4;
887   }
888 
889   // Emit nops.
890   for (unsigned i = 0; i < NumNOPBytes; i += 4)
891     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
892 }
893 
894 // Lower a patchpoint of the form:
895 // [<def>], <id>, <numBytes>, <target>, <numArgs>
896 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
897                                         const MachineInstr &MI) {
898   auto &Ctx = OutStreamer.getContext();
899   MCSymbol *MILabel = Ctx.createTempSymbol();
900   OutStreamer.EmitLabel(MILabel);
901   SM.recordPatchPoint(*MILabel, MI);
902 
903   PatchPointOpers Opers(&MI);
904 
905   int64_t CallTarget = Opers.getCallTarget().getImm();
906   unsigned EncodedBytes = 0;
907   if (CallTarget) {
908     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
909            "High 16 bits of call target should be zero.");
910     Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
911     EncodedBytes = 16;
912     // Materialize the jump address:
913     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
914                                     .addReg(ScratchReg)
915                                     .addImm((CallTarget >> 32) & 0xFFFF)
916                                     .addImm(32));
917     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
918                                     .addReg(ScratchReg)
919                                     .addReg(ScratchReg)
920                                     .addImm((CallTarget >> 16) & 0xFFFF)
921                                     .addImm(16));
922     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
923                                     .addReg(ScratchReg)
924                                     .addReg(ScratchReg)
925                                     .addImm(CallTarget & 0xFFFF)
926                                     .addImm(0));
927     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
928   }
929   // Emit padding.
930   unsigned NumBytes = Opers.getNumPatchBytes();
931   assert(NumBytes >= EncodedBytes &&
932          "Patchpoint can't request size less than the length of a call.");
933   assert((NumBytes - EncodedBytes) % 4 == 0 &&
934          "Invalid number of NOP bytes requested!");
935   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
936     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
937 }
938 
939 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
940   Register DestReg = MI.getOperand(0).getReg();
941   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
942     // Convert H/S/D register to corresponding Q register
943     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
944       DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
945     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
946       DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
947     else {
948       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
949       DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
950     }
951     MCInst MOVI;
952     MOVI.setOpcode(AArch64::MOVIv2d_ns);
953     MOVI.addOperand(MCOperand::createReg(DestReg));
954     MOVI.addOperand(MCOperand::createImm(0));
955     EmitToStreamer(*OutStreamer, MOVI);
956   } else {
957     MCInst FMov;
958     switch (MI.getOpcode()) {
959     default: llvm_unreachable("Unexpected opcode");
960     case AArch64::FMOVH0:
961       FMov.setOpcode(AArch64::FMOVWHr);
962       FMov.addOperand(MCOperand::createReg(DestReg));
963       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
964       break;
965     case AArch64::FMOVS0:
966       FMov.setOpcode(AArch64::FMOVWSr);
967       FMov.addOperand(MCOperand::createReg(DestReg));
968       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
969       break;
970     case AArch64::FMOVD0:
971       FMov.setOpcode(AArch64::FMOVXDr);
972       FMov.addOperand(MCOperand::createReg(DestReg));
973       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
974       break;
975     }
976     EmitToStreamer(*OutStreamer, FMov);
977   }
978 }
979 
980 // Simple pseudo-instructions have their lowering (with expansion to real
981 // instructions) auto-generated.
982 #include "AArch64GenMCPseudoLowering.inc"
983 
984 void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
985   // Do any auto-generated pseudo lowerings.
986   if (emitPseudoExpansionLowering(*OutStreamer, MI))
987     return;
988 
989   if (AArch64FI->getLOHRelated().count(MI)) {
990     // Generate a label for LOH related instruction
991     MCSymbol *LOHLabel = createTempSymbol("loh");
992     // Associate the instruction with the label
993     LOHInstToLabel[MI] = LOHLabel;
994     OutStreamer->EmitLabel(LOHLabel);
995   }
996 
997   AArch64TargetStreamer *TS =
998     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
999   // Do any manual lowerings.
1000   switch (MI->getOpcode()) {
1001   default:
1002     break;
1003   case AArch64::HINT: {
1004     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1005     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1006     // non-empty. If MI is the initial BTI, place the
1007     // __patchable_function_entries label after BTI.
1008     if (CurrentPatchableFunctionEntrySym &&
1009         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1010         MI == &MF->front().front()) {
1011       int64_t Imm = MI->getOperand(0).getImm();
1012       if ((Imm & 32) && (Imm & 6)) {
1013         MCInst Inst;
1014         MCInstLowering.Lower(MI, Inst);
1015         EmitToStreamer(*OutStreamer, Inst);
1016         CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1017         OutStreamer->EmitLabel(CurrentPatchableFunctionEntrySym);
1018         return;
1019       }
1020     }
1021     break;
1022   }
1023     case AArch64::MOVMCSym: {
1024       Register DestReg = MI->getOperand(0).getReg();
1025       const MachineOperand &MO_Sym = MI->getOperand(1);
1026       MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1027       MCOperand Hi_MCSym, Lo_MCSym;
1028 
1029       Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1030       Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1031 
1032       MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1033       MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1034 
1035       MCInst MovZ;
1036       MovZ.setOpcode(AArch64::MOVZXi);
1037       MovZ.addOperand(MCOperand::createReg(DestReg));
1038       MovZ.addOperand(Hi_MCSym);
1039       MovZ.addOperand(MCOperand::createImm(16));
1040       EmitToStreamer(*OutStreamer, MovZ);
1041 
1042       MCInst MovK;
1043       MovK.setOpcode(AArch64::MOVKXi);
1044       MovK.addOperand(MCOperand::createReg(DestReg));
1045       MovK.addOperand(MCOperand::createReg(DestReg));
1046       MovK.addOperand(Lo_MCSym);
1047       MovK.addOperand(MCOperand::createImm(0));
1048       EmitToStreamer(*OutStreamer, MovK);
1049       return;
1050   }
1051   case AArch64::MOVIv2d_ns:
1052     // If the target has <rdar://problem/16473581>, lower this
1053     // instruction to movi.16b instead.
1054     if (STI->hasZeroCycleZeroingFPWorkaround() &&
1055         MI->getOperand(1).getImm() == 0) {
1056       MCInst TmpInst;
1057       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1058       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1059       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1060       EmitToStreamer(*OutStreamer, TmpInst);
1061       return;
1062     }
1063     break;
1064 
1065   case AArch64::DBG_VALUE: {
1066     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1067       SmallString<128> TmpStr;
1068       raw_svector_ostream OS(TmpStr);
1069       PrintDebugValueComment(MI, OS);
1070       OutStreamer->EmitRawText(StringRef(OS.str()));
1071     }
1072     return;
1073 
1074   case AArch64::EMITBKEY: {
1075       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1076       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1077           ExceptionHandlingType != ExceptionHandling::ARM)
1078         return;
1079 
1080       if (needsCFIMoves() == CFI_M_None)
1081         return;
1082 
1083       OutStreamer->EmitCFIBKeyFrame();
1084       return;
1085     }
1086   }
1087 
1088   // Tail calls use pseudo instructions so they have the proper code-gen
1089   // attributes (isCall, isReturn, etc.). We lower them to the real
1090   // instruction here.
1091   case AArch64::TCRETURNri:
1092   case AArch64::TCRETURNriBTI:
1093   case AArch64::TCRETURNriALL: {
1094     MCInst TmpInst;
1095     TmpInst.setOpcode(AArch64::BR);
1096     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1097     EmitToStreamer(*OutStreamer, TmpInst);
1098     return;
1099   }
1100   case AArch64::TCRETURNdi: {
1101     MCOperand Dest;
1102     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1103     MCInst TmpInst;
1104     TmpInst.setOpcode(AArch64::B);
1105     TmpInst.addOperand(Dest);
1106     EmitToStreamer(*OutStreamer, TmpInst);
1107     return;
1108   }
1109   case AArch64::TLSDESC_CALLSEQ: {
1110     /// lower this to:
1111     ///    adrp  x0, :tlsdesc:var
1112     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
1113     ///    add   x0, x0, #:tlsdesc_lo12:var
1114     ///    .tlsdesccall var
1115     ///    blr   x1
1116     ///    (TPIDR_EL0 offset now in x0)
1117     const MachineOperand &MO_Sym = MI->getOperand(0);
1118     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1119     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1120     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1121     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1122     MCInstLowering.lowerOperand(MO_Sym, Sym);
1123     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1124     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1125 
1126     MCInst Adrp;
1127     Adrp.setOpcode(AArch64::ADRP);
1128     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1129     Adrp.addOperand(SymTLSDesc);
1130     EmitToStreamer(*OutStreamer, Adrp);
1131 
1132     MCInst Ldr;
1133     Ldr.setOpcode(AArch64::LDRXui);
1134     Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1135     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1136     Ldr.addOperand(SymTLSDescLo12);
1137     Ldr.addOperand(MCOperand::createImm(0));
1138     EmitToStreamer(*OutStreamer, Ldr);
1139 
1140     MCInst Add;
1141     Add.setOpcode(AArch64::ADDXri);
1142     Add.addOperand(MCOperand::createReg(AArch64::X0));
1143     Add.addOperand(MCOperand::createReg(AArch64::X0));
1144     Add.addOperand(SymTLSDescLo12);
1145     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1146     EmitToStreamer(*OutStreamer, Add);
1147 
1148     // Emit a relocation-annotation. This expands to no code, but requests
1149     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1150     MCInst TLSDescCall;
1151     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1152     TLSDescCall.addOperand(Sym);
1153     EmitToStreamer(*OutStreamer, TLSDescCall);
1154 
1155     MCInst Blr;
1156     Blr.setOpcode(AArch64::BLR);
1157     Blr.addOperand(MCOperand::createReg(AArch64::X1));
1158     EmitToStreamer(*OutStreamer, Blr);
1159 
1160     return;
1161   }
1162 
1163   case AArch64::JumpTableDest32: {
1164     // We want:
1165     //     ldrsw xScratch, [xTable, xEntry, lsl #2]
1166     //     add xDest, xTable, xScratch
1167     unsigned DestReg = MI->getOperand(0).getReg(),
1168              ScratchReg = MI->getOperand(1).getReg(),
1169              TableReg = MI->getOperand(2).getReg(),
1170              EntryReg = MI->getOperand(3).getReg();
1171     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
1172                                      .addReg(ScratchReg)
1173                                      .addReg(TableReg)
1174                                      .addReg(EntryReg)
1175                                      .addImm(0)
1176                                      .addImm(1));
1177     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1178                                      .addReg(DestReg)
1179                                      .addReg(TableReg)
1180                                      .addReg(ScratchReg)
1181                                      .addImm(0));
1182     return;
1183   }
1184   case AArch64::JumpTableDest16:
1185   case AArch64::JumpTableDest8:
1186     LowerJumpTableDestSmall(*OutStreamer, *MI);
1187     return;
1188 
1189   case AArch64::FMOVH0:
1190   case AArch64::FMOVS0:
1191   case AArch64::FMOVD0:
1192     EmitFMov0(*MI);
1193     return;
1194 
1195   case TargetOpcode::STACKMAP:
1196     return LowerSTACKMAP(*OutStreamer, SM, *MI);
1197 
1198   case TargetOpcode::PATCHPOINT:
1199     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1200 
1201   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1202     LowerPATCHABLE_FUNCTION_ENTER(*MI);
1203     return;
1204 
1205   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1206     LowerPATCHABLE_FUNCTION_EXIT(*MI);
1207     return;
1208 
1209   case TargetOpcode::PATCHABLE_TAIL_CALL:
1210     LowerPATCHABLE_TAIL_CALL(*MI);
1211     return;
1212 
1213   case AArch64::HWASAN_CHECK_MEMACCESS:
1214   case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1215     LowerHWASAN_CHECK_MEMACCESS(*MI);
1216     return;
1217 
1218   case AArch64::SEH_StackAlloc:
1219     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1220     return;
1221 
1222   case AArch64::SEH_SaveFPLR:
1223     TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1224     return;
1225 
1226   case AArch64::SEH_SaveFPLR_X:
1227     assert(MI->getOperand(0).getImm() < 0 &&
1228            "Pre increment SEH opcode must have a negative offset");
1229     TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1230     return;
1231 
1232   case AArch64::SEH_SaveReg:
1233     TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1234                                MI->getOperand(1).getImm());
1235     return;
1236 
1237   case AArch64::SEH_SaveReg_X:
1238     assert(MI->getOperand(1).getImm() < 0 &&
1239            "Pre increment SEH opcode must have a negative offset");
1240     TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1241 		                -MI->getOperand(1).getImm());
1242     return;
1243 
1244   case AArch64::SEH_SaveRegP:
1245     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1246             "Non-consecutive registers not allowed for save_regp");
1247     TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1248                                 MI->getOperand(2).getImm());
1249     return;
1250 
1251   case AArch64::SEH_SaveRegP_X:
1252     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1253             "Non-consecutive registers not allowed for save_regp_x");
1254     assert(MI->getOperand(2).getImm() < 0 &&
1255            "Pre increment SEH opcode must have a negative offset");
1256     TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1257                                  -MI->getOperand(2).getImm());
1258     return;
1259 
1260   case AArch64::SEH_SaveFReg:
1261     TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1262                                 MI->getOperand(1).getImm());
1263     return;
1264 
1265   case AArch64::SEH_SaveFReg_X:
1266     assert(MI->getOperand(1).getImm() < 0 &&
1267            "Pre increment SEH opcode must have a negative offset");
1268     TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1269                                  -MI->getOperand(1).getImm());
1270     return;
1271 
1272   case AArch64::SEH_SaveFRegP:
1273     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1274             "Non-consecutive registers not allowed for save_regp");
1275     TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1276                                  MI->getOperand(2).getImm());
1277     return;
1278 
1279   case AArch64::SEH_SaveFRegP_X:
1280     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1281             "Non-consecutive registers not allowed for save_regp_x");
1282     assert(MI->getOperand(2).getImm() < 0 &&
1283            "Pre increment SEH opcode must have a negative offset");
1284     TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1285                                   -MI->getOperand(2).getImm());
1286     return;
1287 
1288   case AArch64::SEH_SetFP:
1289     TS->EmitARM64WinCFISetFP();
1290     return;
1291 
1292   case AArch64::SEH_AddFP:
1293     TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1294     return;
1295 
1296   case AArch64::SEH_Nop:
1297     TS->EmitARM64WinCFINop();
1298     return;
1299 
1300   case AArch64::SEH_PrologEnd:
1301     TS->EmitARM64WinCFIPrologEnd();
1302     return;
1303 
1304   case AArch64::SEH_EpilogStart:
1305     TS->EmitARM64WinCFIEpilogStart();
1306     return;
1307 
1308   case AArch64::SEH_EpilogEnd:
1309     TS->EmitARM64WinCFIEpilogEnd();
1310     return;
1311   }
1312 
1313   // Finally, do the automated lowerings for everything else.
1314   MCInst TmpInst;
1315   MCInstLowering.Lower(MI, TmpInst);
1316   EmitToStreamer(*OutStreamer, TmpInst);
1317 }
1318 
1319 // Force static initialization.
1320 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1321   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1322   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1323   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1324   RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1325   RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1326 }
1327