1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the AArch64 assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "AArch64.h"
15 #include "AArch64MCInstLower.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "MCTargetDesc/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64MCExpr.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "TargetInfo/AArch64TargetInfo.h"
26 #include "Utils/AArch64BaseInfo.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/BinaryFormat/COFF.h"
33 #include "llvm/BinaryFormat/ELF.h"
34 #include "llvm/CodeGen/AsmPrinter.h"
35 #include "llvm/CodeGen/MachineBasicBlock.h"
36 #include "llvm/CodeGen/MachineFunction.h"
37 #include "llvm/CodeGen/MachineInstr.h"
38 #include "llvm/CodeGen/MachineJumpTableInfo.h"
39 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
40 #include "llvm/CodeGen/MachineOperand.h"
41 #include "llvm/CodeGen/StackMaps.h"
42 #include "llvm/CodeGen/TargetRegisterInfo.h"
43 #include "llvm/IR/DataLayout.h"
44 #include "llvm/IR/DebugInfoMetadata.h"
45 #include "llvm/MC/MCAsmInfo.h"
46 #include "llvm/MC/MCContext.h"
47 #include "llvm/MC/MCInst.h"
48 #include "llvm/MC/MCInstBuilder.h"
49 #include "llvm/MC/MCSectionELF.h"
50 #include "llvm/MC/MCStreamer.h"
51 #include "llvm/MC/MCSymbol.h"
52 #include "llvm/Support/Casting.h"
53 #include "llvm/Support/ErrorHandling.h"
54 #include "llvm/Support/TargetRegistry.h"
55 #include "llvm/Support/raw_ostream.h"
56 #include "llvm/Target/TargetMachine.h"
57 #include <algorithm>
58 #include <cassert>
59 #include <cstdint>
60 #include <map>
61 #include <memory>
62
63 using namespace llvm;
64
65 #define DEBUG_TYPE "asm-printer"
66
67 namespace {
68
69 class AArch64AsmPrinter : public AsmPrinter {
70 AArch64MCInstLower MCInstLowering;
71 StackMaps SM;
72 const AArch64Subtarget *STI;
73
74 public:
AArch64AsmPrinter(TargetMachine & TM,std::unique_ptr<MCStreamer> Streamer)75 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
76 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
77 SM(*this) {}
78
getPassName() const79 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
80
81 /// Wrapper for MCInstLowering.lowerOperand() for the
82 /// tblgen'erated pseudo lowering.
lowerOperand(const MachineOperand & MO,MCOperand & MCOp) const83 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
84 return MCInstLowering.lowerOperand(MO, MCOp);
85 }
86
87 void EmitStartOfAsmFile(Module &M) override;
88 void EmitJumpTableInfo() override;
89 void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
90 const MachineBasicBlock *MBB, unsigned JTI);
91
92 void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI);
93
94 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
95 const MachineInstr &MI);
96 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
97 const MachineInstr &MI);
98
99 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
100 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
101 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
102
103 typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
104 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
105 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
106 void EmitHwasanMemaccessSymbols(Module &M);
107
108 void EmitSled(const MachineInstr &MI, SledKind Kind);
109
110 /// tblgen'erated driver function for lowering simple MI->MC
111 /// pseudo instructions.
112 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
113 const MachineInstr *MI);
114
115 void EmitInstruction(const MachineInstr *MI) override;
116
getAnalysisUsage(AnalysisUsage & AU) const117 void getAnalysisUsage(AnalysisUsage &AU) const override {
118 AsmPrinter::getAnalysisUsage(AU);
119 AU.setPreservesAll();
120 }
121
runOnMachineFunction(MachineFunction & MF)122 bool runOnMachineFunction(MachineFunction &MF) override {
123 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
124 STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
125
126 SetupMachineFunction(MF);
127
128 if (STI->isTargetCOFF()) {
129 bool Internal = MF.getFunction().hasInternalLinkage();
130 COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
131 : COFF::IMAGE_SYM_CLASS_EXTERNAL;
132 int Type =
133 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
134
135 OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
136 OutStreamer->EmitCOFFSymbolStorageClass(Scl);
137 OutStreamer->EmitCOFFSymbolType(Type);
138 OutStreamer->EndCOFFSymbolDef();
139 }
140
141 // Emit the rest of the function body.
142 EmitFunctionBody();
143
144 // Emit the XRay table for this function.
145 emitXRayTable();
146
147 // We didn't modify anything.
148 return false;
149 }
150
151 private:
152 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
153 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
154 bool printAsmRegInClass(const MachineOperand &MO,
155 const TargetRegisterClass *RC, unsigned AltName,
156 raw_ostream &O);
157
158 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
159 const char *ExtraCode, raw_ostream &O) override;
160 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
161 const char *ExtraCode, raw_ostream &O) override;
162
163 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
164
165 void EmitFunctionBodyEnd() override;
166
167 MCSymbol *GetCPISymbol(unsigned CPID) const override;
168 void EmitEndOfAsmFile(Module &M) override;
169
170 AArch64FunctionInfo *AArch64FI = nullptr;
171
172 /// Emit the LOHs contained in AArch64FI.
173 void EmitLOHs();
174
175 /// Emit instruction to set float register to zero.
176 void EmitFMov0(const MachineInstr &MI);
177
178 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
179
180 MInstToMCSymbol LOHInstToLabel;
181 };
182
183 } // end anonymous namespace
184
EmitStartOfAsmFile(Module & M)185 void AArch64AsmPrinter::EmitStartOfAsmFile(Module &M) {
186 if (!TM.getTargetTriple().isOSBinFormatELF())
187 return;
188
189 // Assemble feature flags that may require creation of a note section.
190 unsigned Flags = ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI |
191 ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
192
193 if (any_of(M, [](const Function &F) {
194 return !F.isDeclaration() &&
195 !F.hasFnAttribute("branch-target-enforcement");
196 })) {
197 Flags &= ~ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
198 }
199
200 if ((Flags & ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI) == 0 &&
201 any_of(M, [](const Function &F) {
202 return F.hasFnAttribute("branch-target-enforcement");
203 })) {
204 errs() << "warning: some functions compiled with BTI and some compiled "
205 "without BTI\n"
206 << "warning: not setting BTI in feature flags\n";
207 }
208
209 if (any_of(M, [](const Function &F) {
210 if (F.isDeclaration())
211 return false;
212 Attribute A = F.getFnAttribute("sign-return-address");
213 return !A.isStringAttribute() || A.getValueAsString() == "none";
214 })) {
215 Flags &= ~ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
216 }
217
218 if (Flags == 0)
219 return;
220
221 // Emit a .note.gnu.property section with the flags.
222 MCSection *Cur = OutStreamer->getCurrentSectionOnly();
223 MCSection *Nt = MMI->getContext().getELFSection(
224 ".note.gnu.property", ELF::SHT_NOTE, ELF::SHF_ALLOC);
225 OutStreamer->SwitchSection(Nt);
226
227 // Emit the note header.
228 EmitAlignment(Align(8));
229 OutStreamer->EmitIntValue(4, 4); // data size for "GNU\0"
230 OutStreamer->EmitIntValue(4 * 4, 4); // Elf_Prop size
231 OutStreamer->EmitIntValue(ELF::NT_GNU_PROPERTY_TYPE_0, 4);
232 OutStreamer->EmitBytes(StringRef("GNU", 4)); // note name
233
234 // Emit the PAC/BTI properties.
235 OutStreamer->EmitIntValue(ELF::GNU_PROPERTY_AARCH64_FEATURE_1_AND, 4);
236 OutStreamer->EmitIntValue(4, 4); // data size
237 OutStreamer->EmitIntValue(Flags, 4); // data
238 OutStreamer->EmitIntValue(0, 4); // pad
239
240 OutStreamer->endSection(Nt);
241 OutStreamer->SwitchSection(Cur);
242 }
243
LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr & MI)244 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
245 {
246 const Function &F = MF->getFunction();
247 if (F.hasFnAttribute("patchable-function-entry")) {
248 unsigned Num;
249 if (F.getFnAttribute("patchable-function-entry")
250 .getValueAsString()
251 .getAsInteger(10, Num))
252 return;
253 emitNops(Num);
254 return;
255 }
256
257 EmitSled(MI, SledKind::FUNCTION_ENTER);
258 }
259
LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr & MI)260 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
261 {
262 EmitSled(MI, SledKind::FUNCTION_EXIT);
263 }
264
LowerPATCHABLE_TAIL_CALL(const MachineInstr & MI)265 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
266 {
267 EmitSled(MI, SledKind::TAIL_CALL);
268 }
269
EmitSled(const MachineInstr & MI,SledKind Kind)270 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
271 {
272 static const int8_t NoopsInSledCount = 7;
273 // We want to emit the following pattern:
274 //
275 // .Lxray_sled_N:
276 // ALIGN
277 // B #32
278 // ; 7 NOP instructions (28 bytes)
279 // .tmpN
280 //
281 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
282 // over the full 32 bytes (8 instructions) with the following pattern:
283 //
284 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
285 // LDR W0, #12 ; W0 := function ID
286 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
287 // BLR X16 ; call the tracing trampoline
288 // ;DATA: 32 bits of function ID
289 // ;DATA: lower 32 bits of the address of the trampoline
290 // ;DATA: higher 32 bits of the address of the trampoline
291 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
292 //
293 OutStreamer->EmitCodeAlignment(4);
294 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
295 OutStreamer->EmitLabel(CurSled);
296 auto Target = OutContext.createTempSymbol();
297
298 // Emit "B #32" instruction, which jumps over the next 28 bytes.
299 // The operand has to be the number of 4-byte instructions to jump over,
300 // including the current instruction.
301 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
302
303 for (int8_t I = 0; I < NoopsInSledCount; I++)
304 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
305
306 OutStreamer->EmitLabel(Target);
307 recordSled(CurSled, MI, Kind);
308 }
309
LowerHWASAN_CHECK_MEMACCESS(const MachineInstr & MI)310 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
311 Register Reg = MI.getOperand(0).getReg();
312 bool IsShort =
313 MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
314 uint32_t AccessInfo = MI.getOperand(1).getImm();
315 MCSymbol *&Sym =
316 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
317 if (!Sym) {
318 // FIXME: Make this work on non-ELF.
319 if (!TM.getTargetTriple().isOSBinFormatELF())
320 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
321
322 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
323 utostr(AccessInfo);
324 if (IsShort)
325 SymName += "_short";
326 Sym = OutContext.getOrCreateSymbol(SymName);
327 }
328
329 EmitToStreamer(*OutStreamer,
330 MCInstBuilder(AArch64::BL)
331 .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
332 }
333
EmitHwasanMemaccessSymbols(Module & M)334 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
335 if (HwasanMemaccessSymbols.empty())
336 return;
337
338 const Triple &TT = TM.getTargetTriple();
339 assert(TT.isOSBinFormatELF());
340 std::unique_ptr<MCSubtargetInfo> STI(
341 TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
342
343 MCSymbol *HwasanTagMismatchV1Sym =
344 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
345 MCSymbol *HwasanTagMismatchV2Sym =
346 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
347
348 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
349 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
350 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
351 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
352
353 for (auto &P : HwasanMemaccessSymbols) {
354 unsigned Reg = std::get<0>(P.first);
355 bool IsShort = std::get<1>(P.first);
356 uint32_t AccessInfo = std::get<2>(P.first);
357 const MCSymbolRefExpr *HwasanTagMismatchRef =
358 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
359 MCSymbol *Sym = P.second;
360
361 OutStreamer->SwitchSection(OutContext.getELFSection(
362 ".text.hot", ELF::SHT_PROGBITS,
363 ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
364 Sym->getName()));
365
366 OutStreamer->EmitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
367 OutStreamer->EmitSymbolAttribute(Sym, MCSA_Weak);
368 OutStreamer->EmitSymbolAttribute(Sym, MCSA_Hidden);
369 OutStreamer->EmitLabel(Sym);
370
371 OutStreamer->EmitInstruction(MCInstBuilder(AArch64::UBFMXri)
372 .addReg(AArch64::X16)
373 .addReg(Reg)
374 .addImm(4)
375 .addImm(55),
376 *STI);
377 OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBroX)
378 .addReg(AArch64::W16)
379 .addReg(AArch64::X9)
380 .addReg(AArch64::X16)
381 .addImm(0)
382 .addImm(0),
383 *STI);
384 OutStreamer->EmitInstruction(
385 MCInstBuilder(AArch64::SUBSXrs)
386 .addReg(AArch64::XZR)
387 .addReg(AArch64::X16)
388 .addReg(Reg)
389 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
390 *STI);
391 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
392 OutStreamer->EmitInstruction(
393 MCInstBuilder(AArch64::Bcc)
394 .addImm(AArch64CC::NE)
395 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
396 OutContext)),
397 *STI);
398 MCSymbol *ReturnSym = OutContext.createTempSymbol();
399 OutStreamer->EmitLabel(ReturnSym);
400 OutStreamer->EmitInstruction(
401 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
402 OutStreamer->EmitLabel(HandleMismatchOrPartialSym);
403
404 if (IsShort) {
405 OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWri)
406 .addReg(AArch64::WZR)
407 .addReg(AArch64::W16)
408 .addImm(15)
409 .addImm(0),
410 *STI);
411 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
412 OutStreamer->EmitInstruction(
413 MCInstBuilder(AArch64::Bcc)
414 .addImm(AArch64CC::HI)
415 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
416 *STI);
417
418 OutStreamer->EmitInstruction(
419 MCInstBuilder(AArch64::ANDXri)
420 .addReg(AArch64::X17)
421 .addReg(Reg)
422 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
423 *STI);
424 unsigned Size = 1 << (AccessInfo & 0xf);
425 if (Size != 1)
426 OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ADDXri)
427 .addReg(AArch64::X17)
428 .addReg(AArch64::X17)
429 .addImm(Size - 1)
430 .addImm(0),
431 *STI);
432 OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWrs)
433 .addReg(AArch64::WZR)
434 .addReg(AArch64::W16)
435 .addReg(AArch64::W17)
436 .addImm(0),
437 *STI);
438 OutStreamer->EmitInstruction(
439 MCInstBuilder(AArch64::Bcc)
440 .addImm(AArch64CC::LS)
441 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
442 *STI);
443
444 OutStreamer->EmitInstruction(
445 MCInstBuilder(AArch64::ORRXri)
446 .addReg(AArch64::X16)
447 .addReg(Reg)
448 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
449 *STI);
450 OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBui)
451 .addReg(AArch64::W16)
452 .addReg(AArch64::X16)
453 .addImm(0),
454 *STI);
455 OutStreamer->EmitInstruction(
456 MCInstBuilder(AArch64::SUBSXrs)
457 .addReg(AArch64::XZR)
458 .addReg(AArch64::X16)
459 .addReg(Reg)
460 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
461 *STI);
462 OutStreamer->EmitInstruction(
463 MCInstBuilder(AArch64::Bcc)
464 .addImm(AArch64CC::EQ)
465 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
466 *STI);
467
468 OutStreamer->EmitLabel(HandleMismatchSym);
469 }
470
471 OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXpre)
472 .addReg(AArch64::SP)
473 .addReg(AArch64::X0)
474 .addReg(AArch64::X1)
475 .addReg(AArch64::SP)
476 .addImm(-32),
477 *STI);
478 OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXi)
479 .addReg(AArch64::FP)
480 .addReg(AArch64::LR)
481 .addReg(AArch64::SP)
482 .addImm(29),
483 *STI);
484
485 if (Reg != AArch64::X0)
486 OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ORRXrs)
487 .addReg(AArch64::X0)
488 .addReg(AArch64::XZR)
489 .addReg(Reg)
490 .addImm(0),
491 *STI);
492 OutStreamer->EmitInstruction(MCInstBuilder(AArch64::MOVZXi)
493 .addReg(AArch64::X1)
494 .addImm(AccessInfo)
495 .addImm(0),
496 *STI);
497
498 // Intentionally load the GOT entry and branch to it, rather than possibly
499 // late binding the function, which may clobber the registers before we have
500 // a chance to save them.
501 OutStreamer->EmitInstruction(
502 MCInstBuilder(AArch64::ADRP)
503 .addReg(AArch64::X16)
504 .addExpr(AArch64MCExpr::create(
505 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
506 OutContext)),
507 *STI);
508 OutStreamer->EmitInstruction(
509 MCInstBuilder(AArch64::LDRXui)
510 .addReg(AArch64::X16)
511 .addReg(AArch64::X16)
512 .addExpr(AArch64MCExpr::create(
513 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
514 OutContext)),
515 *STI);
516 OutStreamer->EmitInstruction(
517 MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
518 }
519 }
520
EmitEndOfAsmFile(Module & M)521 void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
522 EmitHwasanMemaccessSymbols(M);
523
524 const Triple &TT = TM.getTargetTriple();
525 if (TT.isOSBinFormatMachO()) {
526 // Funny Darwin hack: This flag tells the linker that no global symbols
527 // contain code that falls through to other global symbols (e.g. the obvious
528 // implementation of multiple entry points). If this doesn't occur, the
529 // linker can safely perform dead code stripping. Since LLVM never
530 // generates code that does this, it is always safe to set.
531 OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
532 }
533 emitStackMaps(SM);
534 }
535
EmitLOHs()536 void AArch64AsmPrinter::EmitLOHs() {
537 SmallVector<MCSymbol *, 3> MCArgs;
538
539 for (const auto &D : AArch64FI->getLOHContainer()) {
540 for (const MachineInstr *MI : D.getArgs()) {
541 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
542 assert(LabelIt != LOHInstToLabel.end() &&
543 "Label hasn't been inserted for LOH related instruction");
544 MCArgs.push_back(LabelIt->second);
545 }
546 OutStreamer->EmitLOHDirective(D.getKind(), MCArgs);
547 MCArgs.clear();
548 }
549 }
550
EmitFunctionBodyEnd()551 void AArch64AsmPrinter::EmitFunctionBodyEnd() {
552 if (!AArch64FI->getLOHRelated().empty())
553 EmitLOHs();
554 }
555
556 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
GetCPISymbol(unsigned CPID) const557 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
558 // Darwin uses a linker-private symbol name for constant-pools (to
559 // avoid addends on the relocation?), ELF has no such concept and
560 // uses a normal private symbol.
561 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
562 return OutContext.getOrCreateSymbol(
563 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
564 Twine(getFunctionNumber()) + "_" + Twine(CPID));
565
566 return AsmPrinter::GetCPISymbol(CPID);
567 }
568
printOperand(const MachineInstr * MI,unsigned OpNum,raw_ostream & O)569 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
570 raw_ostream &O) {
571 const MachineOperand &MO = MI->getOperand(OpNum);
572 switch (MO.getType()) {
573 default:
574 llvm_unreachable("<unknown operand type>");
575 case MachineOperand::MO_Register: {
576 Register Reg = MO.getReg();
577 assert(Register::isPhysicalRegister(Reg));
578 assert(!MO.getSubReg() && "Subregs should be eliminated!");
579 O << AArch64InstPrinter::getRegisterName(Reg);
580 break;
581 }
582 case MachineOperand::MO_Immediate: {
583 O << MO.getImm();
584 break;
585 }
586 case MachineOperand::MO_GlobalAddress: {
587 PrintSymbolOperand(MO, O);
588 break;
589 }
590 case MachineOperand::MO_BlockAddress: {
591 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
592 Sym->print(O, MAI);
593 break;
594 }
595 }
596 }
597
printAsmMRegister(const MachineOperand & MO,char Mode,raw_ostream & O)598 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
599 raw_ostream &O) {
600 Register Reg = MO.getReg();
601 switch (Mode) {
602 default:
603 return true; // Unknown mode.
604 case 'w':
605 Reg = getWRegFromXReg(Reg);
606 break;
607 case 'x':
608 Reg = getXRegFromWReg(Reg);
609 break;
610 }
611
612 O << AArch64InstPrinter::getRegisterName(Reg);
613 return false;
614 }
615
616 // Prints the register in MO using class RC using the offset in the
617 // new register class. This should not be used for cross class
618 // printing.
printAsmRegInClass(const MachineOperand & MO,const TargetRegisterClass * RC,unsigned AltName,raw_ostream & O)619 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
620 const TargetRegisterClass *RC,
621 unsigned AltName, raw_ostream &O) {
622 assert(MO.isReg() && "Should only get here with a register!");
623 const TargetRegisterInfo *RI = STI->getRegisterInfo();
624 Register Reg = MO.getReg();
625 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
626 assert(RI->regsOverlap(RegToPrint, Reg));
627 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
628 return false;
629 }
630
PrintAsmOperand(const MachineInstr * MI,unsigned OpNum,const char * ExtraCode,raw_ostream & O)631 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
632 const char *ExtraCode, raw_ostream &O) {
633 const MachineOperand &MO = MI->getOperand(OpNum);
634
635 // First try the generic code, which knows about modifiers like 'c' and 'n'.
636 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
637 return false;
638
639 // Does this asm operand have a single letter operand modifier?
640 if (ExtraCode && ExtraCode[0]) {
641 if (ExtraCode[1] != 0)
642 return true; // Unknown modifier.
643
644 switch (ExtraCode[0]) {
645 default:
646 return true; // Unknown modifier.
647 case 'w': // Print W register
648 case 'x': // Print X register
649 if (MO.isReg())
650 return printAsmMRegister(MO, ExtraCode[0], O);
651 if (MO.isImm() && MO.getImm() == 0) {
652 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
653 O << AArch64InstPrinter::getRegisterName(Reg);
654 return false;
655 }
656 printOperand(MI, OpNum, O);
657 return false;
658 case 'b': // Print B register.
659 case 'h': // Print H register.
660 case 's': // Print S register.
661 case 'd': // Print D register.
662 case 'q': // Print Q register.
663 case 'z': // Print Z register.
664 if (MO.isReg()) {
665 const TargetRegisterClass *RC;
666 switch (ExtraCode[0]) {
667 case 'b':
668 RC = &AArch64::FPR8RegClass;
669 break;
670 case 'h':
671 RC = &AArch64::FPR16RegClass;
672 break;
673 case 's':
674 RC = &AArch64::FPR32RegClass;
675 break;
676 case 'd':
677 RC = &AArch64::FPR64RegClass;
678 break;
679 case 'q':
680 RC = &AArch64::FPR128RegClass;
681 break;
682 case 'z':
683 RC = &AArch64::ZPRRegClass;
684 break;
685 default:
686 return true;
687 }
688 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
689 }
690 printOperand(MI, OpNum, O);
691 return false;
692 }
693 }
694
695 // According to ARM, we should emit x and v registers unless we have a
696 // modifier.
697 if (MO.isReg()) {
698 Register Reg = MO.getReg();
699
700 // If this is a w or x register, print an x register.
701 if (AArch64::GPR32allRegClass.contains(Reg) ||
702 AArch64::GPR64allRegClass.contains(Reg))
703 return printAsmMRegister(MO, 'x', O);
704
705 unsigned AltName = AArch64::NoRegAltName;
706 const TargetRegisterClass *RegClass;
707 if (AArch64::ZPRRegClass.contains(Reg)) {
708 RegClass = &AArch64::ZPRRegClass;
709 } else if (AArch64::PPRRegClass.contains(Reg)) {
710 RegClass = &AArch64::PPRRegClass;
711 } else {
712 RegClass = &AArch64::FPR128RegClass;
713 AltName = AArch64::vreg;
714 }
715
716 // If this is a b, h, s, d, or q register, print it as a v register.
717 return printAsmRegInClass(MO, RegClass, AltName, O);
718 }
719
720 printOperand(MI, OpNum, O);
721 return false;
722 }
723
PrintAsmMemoryOperand(const MachineInstr * MI,unsigned OpNum,const char * ExtraCode,raw_ostream & O)724 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
725 unsigned OpNum,
726 const char *ExtraCode,
727 raw_ostream &O) {
728 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
729 return true; // Unknown modifier.
730
731 const MachineOperand &MO = MI->getOperand(OpNum);
732 assert(MO.isReg() && "unexpected inline asm memory operand");
733 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
734 return false;
735 }
736
PrintDebugValueComment(const MachineInstr * MI,raw_ostream & OS)737 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
738 raw_ostream &OS) {
739 unsigned NOps = MI->getNumOperands();
740 assert(NOps == 4);
741 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
742 // cast away const; DIetc do not take const operands for some reason.
743 OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
744 ->getName();
745 OS << " <- ";
746 // Frame address. Currently handles register +- offset only.
747 assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
748 OS << '[';
749 printOperand(MI, 0, OS);
750 OS << '+';
751 printOperand(MI, 1, OS);
752 OS << ']';
753 OS << "+";
754 printOperand(MI, NOps - 2, OS);
755 }
756
EmitJumpTableInfo()757 void AArch64AsmPrinter::EmitJumpTableInfo() {
758 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
759 if (!MJTI) return;
760
761 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
762 if (JT.empty()) return;
763
764 const Function &F = MF->getFunction();
765 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
766 bool JTInDiffSection =
767 !STI->isTargetCOFF() ||
768 !TLOF.shouldPutJumpTableInFunctionSection(
769 MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
770 F);
771 if (JTInDiffSection) {
772 // Drop it in the readonly section.
773 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
774 OutStreamer->SwitchSection(ReadOnlySec);
775 }
776
777 auto AFI = MF->getInfo<AArch64FunctionInfo>();
778 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
779 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
780
781 // If this jump table was deleted, ignore it.
782 if (JTBBs.empty()) continue;
783
784 unsigned Size = AFI->getJumpTableEntrySize(JTI);
785 EmitAlignment(Align(Size));
786 OutStreamer->EmitLabel(GetJTISymbol(JTI));
787
788 for (auto *JTBB : JTBBs)
789 emitJumpTableEntry(MJTI, JTBB, JTI);
790 }
791 }
792
emitJumpTableEntry(const MachineJumpTableInfo * MJTI,const MachineBasicBlock * MBB,unsigned JTI)793 void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
794 const MachineBasicBlock *MBB,
795 unsigned JTI) {
796 const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
797 auto AFI = MF->getInfo<AArch64FunctionInfo>();
798 unsigned Size = AFI->getJumpTableEntrySize(JTI);
799
800 if (Size == 4) {
801 // .word LBB - LJTI
802 const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
803 const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
804 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
805 } else {
806 // .byte (LBB - LBB) >> 2 (or .hword)
807 const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI);
808 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
809 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
810 Value = MCBinaryExpr::createLShr(
811 Value, MCConstantExpr::create(2, OutContext), OutContext);
812 }
813
814 OutStreamer->EmitValue(Value, Size);
815 }
816
817 /// Small jump tables contain an unsigned byte or half, representing the offset
818 /// from the lowest-addressed possible destination to the desired basic
819 /// block. Since all instructions are 4-byte aligned, this is further compressed
820 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
821 /// materialize the correct destination we need:
822 ///
823 /// adr xDest, .LBB0_0
824 /// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
825 /// add xDest, xDest, xScratch, lsl #2
LowerJumpTableDestSmall(llvm::MCStreamer & OutStreamer,const llvm::MachineInstr & MI)826 void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer,
827 const llvm::MachineInstr &MI) {
828 Register DestReg = MI.getOperand(0).getReg();
829 Register ScratchReg = MI.getOperand(1).getReg();
830 Register ScratchRegW =
831 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
832 Register TableReg = MI.getOperand(2).getReg();
833 Register EntryReg = MI.getOperand(3).getReg();
834 int JTIdx = MI.getOperand(4).getIndex();
835 bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8;
836
837 // This has to be first because the compression pass based its reachability
838 // calculations on the start of the JumpTableDest instruction.
839 auto Label =
840 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
841 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
842 .addReg(DestReg)
843 .addExpr(MCSymbolRefExpr::create(
844 Label, MF->getContext())));
845
846 // Load the number of instruction-steps to offset from the label.
847 unsigned LdrOpcode = IsByteEntry ? AArch64::LDRBBroX : AArch64::LDRHHroX;
848 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
849 .addReg(ScratchRegW)
850 .addReg(TableReg)
851 .addReg(EntryReg)
852 .addImm(0)
853 .addImm(IsByteEntry ? 0 : 1));
854
855 // Multiply the steps by 4 and add to the already materialized base label
856 // address.
857 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
858 .addReg(DestReg)
859 .addReg(DestReg)
860 .addReg(ScratchReg)
861 .addImm(2));
862 }
863
LowerSTACKMAP(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)864 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
865 const MachineInstr &MI) {
866 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
867
868 auto &Ctx = OutStreamer.getContext();
869 MCSymbol *MILabel = Ctx.createTempSymbol();
870 OutStreamer.EmitLabel(MILabel);
871
872 SM.recordStackMap(*MILabel, MI);
873 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
874
875 // Scan ahead to trim the shadow.
876 const MachineBasicBlock &MBB = *MI.getParent();
877 MachineBasicBlock::const_iterator MII(MI);
878 ++MII;
879 while (NumNOPBytes > 0) {
880 if (MII == MBB.end() || MII->isCall() ||
881 MII->getOpcode() == AArch64::DBG_VALUE ||
882 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
883 MII->getOpcode() == TargetOpcode::STACKMAP)
884 break;
885 ++MII;
886 NumNOPBytes -= 4;
887 }
888
889 // Emit nops.
890 for (unsigned i = 0; i < NumNOPBytes; i += 4)
891 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
892 }
893
894 // Lower a patchpoint of the form:
895 // [<def>], <id>, <numBytes>, <target>, <numArgs>
LowerPATCHPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)896 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
897 const MachineInstr &MI) {
898 auto &Ctx = OutStreamer.getContext();
899 MCSymbol *MILabel = Ctx.createTempSymbol();
900 OutStreamer.EmitLabel(MILabel);
901 SM.recordPatchPoint(*MILabel, MI);
902
903 PatchPointOpers Opers(&MI);
904
905 int64_t CallTarget = Opers.getCallTarget().getImm();
906 unsigned EncodedBytes = 0;
907 if (CallTarget) {
908 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
909 "High 16 bits of call target should be zero.");
910 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
911 EncodedBytes = 16;
912 // Materialize the jump address:
913 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
914 .addReg(ScratchReg)
915 .addImm((CallTarget >> 32) & 0xFFFF)
916 .addImm(32));
917 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
918 .addReg(ScratchReg)
919 .addReg(ScratchReg)
920 .addImm((CallTarget >> 16) & 0xFFFF)
921 .addImm(16));
922 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
923 .addReg(ScratchReg)
924 .addReg(ScratchReg)
925 .addImm(CallTarget & 0xFFFF)
926 .addImm(0));
927 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
928 }
929 // Emit padding.
930 unsigned NumBytes = Opers.getNumPatchBytes();
931 assert(NumBytes >= EncodedBytes &&
932 "Patchpoint can't request size less than the length of a call.");
933 assert((NumBytes - EncodedBytes) % 4 == 0 &&
934 "Invalid number of NOP bytes requested!");
935 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
936 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
937 }
938
EmitFMov0(const MachineInstr & MI)939 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
940 Register DestReg = MI.getOperand(0).getReg();
941 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
942 // Convert H/S/D register to corresponding Q register
943 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
944 DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
945 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
946 DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
947 else {
948 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
949 DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
950 }
951 MCInst MOVI;
952 MOVI.setOpcode(AArch64::MOVIv2d_ns);
953 MOVI.addOperand(MCOperand::createReg(DestReg));
954 MOVI.addOperand(MCOperand::createImm(0));
955 EmitToStreamer(*OutStreamer, MOVI);
956 } else {
957 MCInst FMov;
958 switch (MI.getOpcode()) {
959 default: llvm_unreachable("Unexpected opcode");
960 case AArch64::FMOVH0:
961 FMov.setOpcode(AArch64::FMOVWHr);
962 FMov.addOperand(MCOperand::createReg(DestReg));
963 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
964 break;
965 case AArch64::FMOVS0:
966 FMov.setOpcode(AArch64::FMOVWSr);
967 FMov.addOperand(MCOperand::createReg(DestReg));
968 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
969 break;
970 case AArch64::FMOVD0:
971 FMov.setOpcode(AArch64::FMOVXDr);
972 FMov.addOperand(MCOperand::createReg(DestReg));
973 FMov.addOperand(MCOperand::createReg(AArch64::XZR));
974 break;
975 }
976 EmitToStreamer(*OutStreamer, FMov);
977 }
978 }
979
980 // Simple pseudo-instructions have their lowering (with expansion to real
981 // instructions) auto-generated.
982 #include "AArch64GenMCPseudoLowering.inc"
983
EmitInstruction(const MachineInstr * MI)984 void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
985 // Do any auto-generated pseudo lowerings.
986 if (emitPseudoExpansionLowering(*OutStreamer, MI))
987 return;
988
989 if (AArch64FI->getLOHRelated().count(MI)) {
990 // Generate a label for LOH related instruction
991 MCSymbol *LOHLabel = createTempSymbol("loh");
992 // Associate the instruction with the label
993 LOHInstToLabel[MI] = LOHLabel;
994 OutStreamer->EmitLabel(LOHLabel);
995 }
996
997 AArch64TargetStreamer *TS =
998 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
999 // Do any manual lowerings.
1000 switch (MI->getOpcode()) {
1001 default:
1002 break;
1003 case AArch64::HINT: {
1004 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1005 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1006 // non-empty. If MI is the initial BTI, place the
1007 // __patchable_function_entries label after BTI.
1008 if (CurrentPatchableFunctionEntrySym &&
1009 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1010 MI == &MF->front().front()) {
1011 int64_t Imm = MI->getOperand(0).getImm();
1012 if ((Imm & 32) && (Imm & 6)) {
1013 MCInst Inst;
1014 MCInstLowering.Lower(MI, Inst);
1015 EmitToStreamer(*OutStreamer, Inst);
1016 CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1017 OutStreamer->EmitLabel(CurrentPatchableFunctionEntrySym);
1018 return;
1019 }
1020 }
1021 break;
1022 }
1023 case AArch64::MOVMCSym: {
1024 Register DestReg = MI->getOperand(0).getReg();
1025 const MachineOperand &MO_Sym = MI->getOperand(1);
1026 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1027 MCOperand Hi_MCSym, Lo_MCSym;
1028
1029 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1030 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1031
1032 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1033 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1034
1035 MCInst MovZ;
1036 MovZ.setOpcode(AArch64::MOVZXi);
1037 MovZ.addOperand(MCOperand::createReg(DestReg));
1038 MovZ.addOperand(Hi_MCSym);
1039 MovZ.addOperand(MCOperand::createImm(16));
1040 EmitToStreamer(*OutStreamer, MovZ);
1041
1042 MCInst MovK;
1043 MovK.setOpcode(AArch64::MOVKXi);
1044 MovK.addOperand(MCOperand::createReg(DestReg));
1045 MovK.addOperand(MCOperand::createReg(DestReg));
1046 MovK.addOperand(Lo_MCSym);
1047 MovK.addOperand(MCOperand::createImm(0));
1048 EmitToStreamer(*OutStreamer, MovK);
1049 return;
1050 }
1051 case AArch64::MOVIv2d_ns:
1052 // If the target has <rdar://problem/16473581>, lower this
1053 // instruction to movi.16b instead.
1054 if (STI->hasZeroCycleZeroingFPWorkaround() &&
1055 MI->getOperand(1).getImm() == 0) {
1056 MCInst TmpInst;
1057 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1058 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1059 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1060 EmitToStreamer(*OutStreamer, TmpInst);
1061 return;
1062 }
1063 break;
1064
1065 case AArch64::DBG_VALUE: {
1066 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1067 SmallString<128> TmpStr;
1068 raw_svector_ostream OS(TmpStr);
1069 PrintDebugValueComment(MI, OS);
1070 OutStreamer->EmitRawText(StringRef(OS.str()));
1071 }
1072 return;
1073
1074 case AArch64::EMITBKEY: {
1075 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1076 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1077 ExceptionHandlingType != ExceptionHandling::ARM)
1078 return;
1079
1080 if (needsCFIMoves() == CFI_M_None)
1081 return;
1082
1083 OutStreamer->EmitCFIBKeyFrame();
1084 return;
1085 }
1086 }
1087
1088 // Tail calls use pseudo instructions so they have the proper code-gen
1089 // attributes (isCall, isReturn, etc.). We lower them to the real
1090 // instruction here.
1091 case AArch64::TCRETURNri:
1092 case AArch64::TCRETURNriBTI:
1093 case AArch64::TCRETURNriALL: {
1094 MCInst TmpInst;
1095 TmpInst.setOpcode(AArch64::BR);
1096 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1097 EmitToStreamer(*OutStreamer, TmpInst);
1098 return;
1099 }
1100 case AArch64::TCRETURNdi: {
1101 MCOperand Dest;
1102 MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1103 MCInst TmpInst;
1104 TmpInst.setOpcode(AArch64::B);
1105 TmpInst.addOperand(Dest);
1106 EmitToStreamer(*OutStreamer, TmpInst);
1107 return;
1108 }
1109 case AArch64::TLSDESC_CALLSEQ: {
1110 /// lower this to:
1111 /// adrp x0, :tlsdesc:var
1112 /// ldr x1, [x0, #:tlsdesc_lo12:var]
1113 /// add x0, x0, #:tlsdesc_lo12:var
1114 /// .tlsdesccall var
1115 /// blr x1
1116 /// (TPIDR_EL0 offset now in x0)
1117 const MachineOperand &MO_Sym = MI->getOperand(0);
1118 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1119 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1120 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1121 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1122 MCInstLowering.lowerOperand(MO_Sym, Sym);
1123 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1124 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1125
1126 MCInst Adrp;
1127 Adrp.setOpcode(AArch64::ADRP);
1128 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1129 Adrp.addOperand(SymTLSDesc);
1130 EmitToStreamer(*OutStreamer, Adrp);
1131
1132 MCInst Ldr;
1133 Ldr.setOpcode(AArch64::LDRXui);
1134 Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1135 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1136 Ldr.addOperand(SymTLSDescLo12);
1137 Ldr.addOperand(MCOperand::createImm(0));
1138 EmitToStreamer(*OutStreamer, Ldr);
1139
1140 MCInst Add;
1141 Add.setOpcode(AArch64::ADDXri);
1142 Add.addOperand(MCOperand::createReg(AArch64::X0));
1143 Add.addOperand(MCOperand::createReg(AArch64::X0));
1144 Add.addOperand(SymTLSDescLo12);
1145 Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1146 EmitToStreamer(*OutStreamer, Add);
1147
1148 // Emit a relocation-annotation. This expands to no code, but requests
1149 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1150 MCInst TLSDescCall;
1151 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1152 TLSDescCall.addOperand(Sym);
1153 EmitToStreamer(*OutStreamer, TLSDescCall);
1154
1155 MCInst Blr;
1156 Blr.setOpcode(AArch64::BLR);
1157 Blr.addOperand(MCOperand::createReg(AArch64::X1));
1158 EmitToStreamer(*OutStreamer, Blr);
1159
1160 return;
1161 }
1162
1163 case AArch64::JumpTableDest32: {
1164 // We want:
1165 // ldrsw xScratch, [xTable, xEntry, lsl #2]
1166 // add xDest, xTable, xScratch
1167 unsigned DestReg = MI->getOperand(0).getReg(),
1168 ScratchReg = MI->getOperand(1).getReg(),
1169 TableReg = MI->getOperand(2).getReg(),
1170 EntryReg = MI->getOperand(3).getReg();
1171 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
1172 .addReg(ScratchReg)
1173 .addReg(TableReg)
1174 .addReg(EntryReg)
1175 .addImm(0)
1176 .addImm(1));
1177 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1178 .addReg(DestReg)
1179 .addReg(TableReg)
1180 .addReg(ScratchReg)
1181 .addImm(0));
1182 return;
1183 }
1184 case AArch64::JumpTableDest16:
1185 case AArch64::JumpTableDest8:
1186 LowerJumpTableDestSmall(*OutStreamer, *MI);
1187 return;
1188
1189 case AArch64::FMOVH0:
1190 case AArch64::FMOVS0:
1191 case AArch64::FMOVD0:
1192 EmitFMov0(*MI);
1193 return;
1194
1195 case TargetOpcode::STACKMAP:
1196 return LowerSTACKMAP(*OutStreamer, SM, *MI);
1197
1198 case TargetOpcode::PATCHPOINT:
1199 return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1200
1201 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1202 LowerPATCHABLE_FUNCTION_ENTER(*MI);
1203 return;
1204
1205 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1206 LowerPATCHABLE_FUNCTION_EXIT(*MI);
1207 return;
1208
1209 case TargetOpcode::PATCHABLE_TAIL_CALL:
1210 LowerPATCHABLE_TAIL_CALL(*MI);
1211 return;
1212
1213 case AArch64::HWASAN_CHECK_MEMACCESS:
1214 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1215 LowerHWASAN_CHECK_MEMACCESS(*MI);
1216 return;
1217
1218 case AArch64::SEH_StackAlloc:
1219 TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1220 return;
1221
1222 case AArch64::SEH_SaveFPLR:
1223 TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1224 return;
1225
1226 case AArch64::SEH_SaveFPLR_X:
1227 assert(MI->getOperand(0).getImm() < 0 &&
1228 "Pre increment SEH opcode must have a negative offset");
1229 TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1230 return;
1231
1232 case AArch64::SEH_SaveReg:
1233 TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1234 MI->getOperand(1).getImm());
1235 return;
1236
1237 case AArch64::SEH_SaveReg_X:
1238 assert(MI->getOperand(1).getImm() < 0 &&
1239 "Pre increment SEH opcode must have a negative offset");
1240 TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1241 -MI->getOperand(1).getImm());
1242 return;
1243
1244 case AArch64::SEH_SaveRegP:
1245 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1246 "Non-consecutive registers not allowed for save_regp");
1247 TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1248 MI->getOperand(2).getImm());
1249 return;
1250
1251 case AArch64::SEH_SaveRegP_X:
1252 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1253 "Non-consecutive registers not allowed for save_regp_x");
1254 assert(MI->getOperand(2).getImm() < 0 &&
1255 "Pre increment SEH opcode must have a negative offset");
1256 TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1257 -MI->getOperand(2).getImm());
1258 return;
1259
1260 case AArch64::SEH_SaveFReg:
1261 TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1262 MI->getOperand(1).getImm());
1263 return;
1264
1265 case AArch64::SEH_SaveFReg_X:
1266 assert(MI->getOperand(1).getImm() < 0 &&
1267 "Pre increment SEH opcode must have a negative offset");
1268 TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1269 -MI->getOperand(1).getImm());
1270 return;
1271
1272 case AArch64::SEH_SaveFRegP:
1273 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1274 "Non-consecutive registers not allowed for save_regp");
1275 TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1276 MI->getOperand(2).getImm());
1277 return;
1278
1279 case AArch64::SEH_SaveFRegP_X:
1280 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1281 "Non-consecutive registers not allowed for save_regp_x");
1282 assert(MI->getOperand(2).getImm() < 0 &&
1283 "Pre increment SEH opcode must have a negative offset");
1284 TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1285 -MI->getOperand(2).getImm());
1286 return;
1287
1288 case AArch64::SEH_SetFP:
1289 TS->EmitARM64WinCFISetFP();
1290 return;
1291
1292 case AArch64::SEH_AddFP:
1293 TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1294 return;
1295
1296 case AArch64::SEH_Nop:
1297 TS->EmitARM64WinCFINop();
1298 return;
1299
1300 case AArch64::SEH_PrologEnd:
1301 TS->EmitARM64WinCFIPrologEnd();
1302 return;
1303
1304 case AArch64::SEH_EpilogStart:
1305 TS->EmitARM64WinCFIEpilogStart();
1306 return;
1307
1308 case AArch64::SEH_EpilogEnd:
1309 TS->EmitARM64WinCFIEpilogEnd();
1310 return;
1311 }
1312
1313 // Finally, do the automated lowerings for everything else.
1314 MCInst TmpInst;
1315 MCInstLowering.Lower(MI, TmpInst);
1316 EmitToStreamer(*OutStreamer, TmpInst);
1317 }
1318
1319 // Force static initialization.
LLVMInitializeAArch64AsmPrinter()1320 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1321 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1322 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1323 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1324 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1325 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1326 }
1327