1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the AArch64 assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "AArch64.h"
15 #include "AArch64MCInstLower.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "MCTargetDesc/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64MCExpr.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "TargetInfo/AArch64TargetInfo.h"
26 #include "Utils/AArch64BaseInfo.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/BinaryFormat/COFF.h"
33 #include "llvm/BinaryFormat/ELF.h"
34 #include "llvm/CodeGen/AsmPrinter.h"
35 #include "llvm/CodeGen/FaultMaps.h"
36 #include "llvm/CodeGen/MachineBasicBlock.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineJumpTableInfo.h"
40 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
41 #include "llvm/CodeGen/MachineOperand.h"
42 #include "llvm/CodeGen/StackMaps.h"
43 #include "llvm/CodeGen/TargetRegisterInfo.h"
44 #include "llvm/IR/DataLayout.h"
45 #include "llvm/IR/DebugInfoMetadata.h"
46 #include "llvm/MC/MCAsmInfo.h"
47 #include "llvm/MC/MCContext.h"
48 #include "llvm/MC/MCInst.h"
49 #include "llvm/MC/MCInstBuilder.h"
50 #include "llvm/MC/MCSectionELF.h"
51 #include "llvm/MC/MCStreamer.h"
52 #include "llvm/MC/MCSymbol.h"
53 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/ErrorHandling.h"
55 #include "llvm/Support/TargetRegistry.h"
56 #include "llvm/Support/raw_ostream.h"
57 #include "llvm/Target/TargetMachine.h"
58 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
59 #include <algorithm>
60 #include <cassert>
61 #include <cstdint>
62 #include <map>
63 #include <memory>
64
65 using namespace llvm;
66
67 #define DEBUG_TYPE "asm-printer"
68
69 namespace {
70
71 class AArch64AsmPrinter : public AsmPrinter {
72 AArch64MCInstLower MCInstLowering;
73 StackMaps SM;
74 FaultMaps FM;
75 const AArch64Subtarget *STI;
76
77 public:
AArch64AsmPrinter(TargetMachine & TM,std::unique_ptr<MCStreamer> Streamer)78 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
79 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
80 SM(*this), FM(*this) {}
81
getPassName() const82 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
83
84 /// Wrapper for MCInstLowering.lowerOperand() for the
85 /// tblgen'erated pseudo lowering.
lowerOperand(const MachineOperand & MO,MCOperand & MCOp) const86 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
87 return MCInstLowering.lowerOperand(MO, MCOp);
88 }
89
90 void emitStartOfAsmFile(Module &M) override;
91 void emitJumpTableInfo() override;
92
93 void emitFunctionEntryLabel() override;
94
95 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
96
97 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
98 const MachineInstr &MI);
99 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
100 const MachineInstr &MI);
101 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
102 const MachineInstr &MI);
103 void LowerFAULTING_OP(const MachineInstr &MI);
104
105 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
106 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
107 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
108
109 typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
110 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
111 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
112 void EmitHwasanMemaccessSymbols(Module &M);
113
114 void EmitSled(const MachineInstr &MI, SledKind Kind);
115
116 /// tblgen'erated driver function for lowering simple MI->MC
117 /// pseudo instructions.
118 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
119 const MachineInstr *MI);
120
121 void emitInstruction(const MachineInstr *MI) override;
122
123 void emitFunctionHeaderComment() override;
124
getAnalysisUsage(AnalysisUsage & AU) const125 void getAnalysisUsage(AnalysisUsage &AU) const override {
126 AsmPrinter::getAnalysisUsage(AU);
127 AU.setPreservesAll();
128 }
129
runOnMachineFunction(MachineFunction & MF)130 bool runOnMachineFunction(MachineFunction &MF) override {
131 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
132 STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
133
134 SetupMachineFunction(MF);
135
136 if (STI->isTargetCOFF()) {
137 bool Internal = MF.getFunction().hasInternalLinkage();
138 COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
139 : COFF::IMAGE_SYM_CLASS_EXTERNAL;
140 int Type =
141 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
142
143 OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
144 OutStreamer->EmitCOFFSymbolStorageClass(Scl);
145 OutStreamer->EmitCOFFSymbolType(Type);
146 OutStreamer->EndCOFFSymbolDef();
147 }
148
149 // Emit the rest of the function body.
150 emitFunctionBody();
151
152 // Emit the XRay table for this function.
153 emitXRayTable();
154
155 // We didn't modify anything.
156 return false;
157 }
158
159 private:
160 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
161 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
162 bool printAsmRegInClass(const MachineOperand &MO,
163 const TargetRegisterClass *RC, unsigned AltName,
164 raw_ostream &O);
165
166 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
167 const char *ExtraCode, raw_ostream &O) override;
168 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
169 const char *ExtraCode, raw_ostream &O) override;
170
171 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
172
173 void emitFunctionBodyEnd() override;
174
175 MCSymbol *GetCPISymbol(unsigned CPID) const override;
176 void emitEndOfAsmFile(Module &M) override;
177
178 AArch64FunctionInfo *AArch64FI = nullptr;
179
180 /// Emit the LOHs contained in AArch64FI.
181 void EmitLOHs();
182
183 /// Emit instruction to set float register to zero.
184 void EmitFMov0(const MachineInstr &MI);
185
186 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
187
188 MInstToMCSymbol LOHInstToLabel;
189 };
190
191 } // end anonymous namespace
192
emitStartOfAsmFile(Module & M)193 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
194 if (!TM.getTargetTriple().isOSBinFormatELF())
195 return;
196
197 // Assemble feature flags that may require creation of a note section.
198 unsigned Flags = 0;
199 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
200 M.getModuleFlag("branch-target-enforcement")))
201 if (BTE->getZExtValue())
202 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
203
204 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
205 M.getModuleFlag("sign-return-address")))
206 if (Sign->getZExtValue())
207 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
208
209 if (Flags == 0)
210 return;
211
212 // Emit a .note.gnu.property section with the flags.
213 if (auto *TS = static_cast<AArch64TargetStreamer *>(
214 OutStreamer->getTargetStreamer()))
215 TS->emitNoteSection(Flags);
216 }
217
emitFunctionHeaderComment()218 void AArch64AsmPrinter::emitFunctionHeaderComment() {
219 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
220 Optional<std::string> OutlinerString = FI->getOutliningStyle();
221 if (OutlinerString != None)
222 OutStreamer->GetCommentOS() << ' ' << OutlinerString;
223 }
224
LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr & MI)225 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
226 {
227 const Function &F = MF->getFunction();
228 if (F.hasFnAttribute("patchable-function-entry")) {
229 unsigned Num;
230 if (F.getFnAttribute("patchable-function-entry")
231 .getValueAsString()
232 .getAsInteger(10, Num))
233 return;
234 emitNops(Num);
235 return;
236 }
237
238 EmitSled(MI, SledKind::FUNCTION_ENTER);
239 }
240
LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr & MI)241 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
242 {
243 EmitSled(MI, SledKind::FUNCTION_EXIT);
244 }
245
LowerPATCHABLE_TAIL_CALL(const MachineInstr & MI)246 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
247 {
248 EmitSled(MI, SledKind::TAIL_CALL);
249 }
250
EmitSled(const MachineInstr & MI,SledKind Kind)251 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
252 {
253 static const int8_t NoopsInSledCount = 7;
254 // We want to emit the following pattern:
255 //
256 // .Lxray_sled_N:
257 // ALIGN
258 // B #32
259 // ; 7 NOP instructions (28 bytes)
260 // .tmpN
261 //
262 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
263 // over the full 32 bytes (8 instructions) with the following pattern:
264 //
265 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
266 // LDR W0, #12 ; W0 := function ID
267 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
268 // BLR X16 ; call the tracing trampoline
269 // ;DATA: 32 bits of function ID
270 // ;DATA: lower 32 bits of the address of the trampoline
271 // ;DATA: higher 32 bits of the address of the trampoline
272 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
273 //
274 OutStreamer->emitCodeAlignment(4);
275 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
276 OutStreamer->emitLabel(CurSled);
277 auto Target = OutContext.createTempSymbol();
278
279 // Emit "B #32" instruction, which jumps over the next 28 bytes.
280 // The operand has to be the number of 4-byte instructions to jump over,
281 // including the current instruction.
282 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
283
284 for (int8_t I = 0; I < NoopsInSledCount; I++)
285 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
286
287 OutStreamer->emitLabel(Target);
288 recordSled(CurSled, MI, Kind, 2);
289 }
290
LowerHWASAN_CHECK_MEMACCESS(const MachineInstr & MI)291 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
292 Register Reg = MI.getOperand(0).getReg();
293 bool IsShort =
294 MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
295 uint32_t AccessInfo = MI.getOperand(1).getImm();
296 MCSymbol *&Sym =
297 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
298 if (!Sym) {
299 // FIXME: Make this work on non-ELF.
300 if (!TM.getTargetTriple().isOSBinFormatELF())
301 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
302
303 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
304 utostr(AccessInfo);
305 if (IsShort)
306 SymName += "_short_v2";
307 Sym = OutContext.getOrCreateSymbol(SymName);
308 }
309
310 EmitToStreamer(*OutStreamer,
311 MCInstBuilder(AArch64::BL)
312 .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
313 }
314
EmitHwasanMemaccessSymbols(Module & M)315 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
316 if (HwasanMemaccessSymbols.empty())
317 return;
318
319 const Triple &TT = TM.getTargetTriple();
320 assert(TT.isOSBinFormatELF());
321 std::unique_ptr<MCSubtargetInfo> STI(
322 TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
323 assert(STI && "Unable to create subtarget info");
324
325 MCSymbol *HwasanTagMismatchV1Sym =
326 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
327 MCSymbol *HwasanTagMismatchV2Sym =
328 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
329
330 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
331 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
332 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
333 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
334
335 for (auto &P : HwasanMemaccessSymbols) {
336 unsigned Reg = std::get<0>(P.first);
337 bool IsShort = std::get<1>(P.first);
338 uint32_t AccessInfo = std::get<2>(P.first);
339 const MCSymbolRefExpr *HwasanTagMismatchRef =
340 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
341 MCSymbol *Sym = P.second;
342
343 bool HasMatchAllTag =
344 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
345 uint8_t MatchAllTag =
346 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
347 unsigned Size =
348 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
349 bool CompileKernel =
350 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
351
352 OutStreamer->SwitchSection(OutContext.getELFSection(
353 ".text.hot", ELF::SHT_PROGBITS,
354 ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
355 Sym->getName()));
356
357 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
358 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
359 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
360 OutStreamer->emitLabel(Sym);
361
362 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri)
363 .addReg(AArch64::X16)
364 .addReg(Reg)
365 .addImm(4)
366 .addImm(55),
367 *STI);
368 OutStreamer->emitInstruction(
369 MCInstBuilder(AArch64::LDRBBroX)
370 .addReg(AArch64::W16)
371 .addReg(IsShort ? AArch64::X20 : AArch64::X9)
372 .addReg(AArch64::X16)
373 .addImm(0)
374 .addImm(0),
375 *STI);
376 OutStreamer->emitInstruction(
377 MCInstBuilder(AArch64::SUBSXrs)
378 .addReg(AArch64::XZR)
379 .addReg(AArch64::X16)
380 .addReg(Reg)
381 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
382 *STI);
383 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
384 OutStreamer->emitInstruction(
385 MCInstBuilder(AArch64::Bcc)
386 .addImm(AArch64CC::NE)
387 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
388 OutContext)),
389 *STI);
390 MCSymbol *ReturnSym = OutContext.createTempSymbol();
391 OutStreamer->emitLabel(ReturnSym);
392 OutStreamer->emitInstruction(
393 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
394 OutStreamer->emitLabel(HandleMismatchOrPartialSym);
395
396 if (HasMatchAllTag) {
397 OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
398 .addReg(AArch64::X16)
399 .addReg(Reg)
400 .addImm(56)
401 .addImm(63),
402 *STI);
403 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
404 .addReg(AArch64::XZR)
405 .addReg(AArch64::X16)
406 .addImm(MatchAllTag)
407 .addImm(0),
408 *STI);
409 OutStreamer->emitInstruction(
410 MCInstBuilder(AArch64::Bcc)
411 .addImm(AArch64CC::EQ)
412 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
413 *STI);
414 }
415
416 if (IsShort) {
417 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
418 .addReg(AArch64::WZR)
419 .addReg(AArch64::W16)
420 .addImm(15)
421 .addImm(0),
422 *STI);
423 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
424 OutStreamer->emitInstruction(
425 MCInstBuilder(AArch64::Bcc)
426 .addImm(AArch64CC::HI)
427 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
428 *STI);
429
430 OutStreamer->emitInstruction(
431 MCInstBuilder(AArch64::ANDXri)
432 .addReg(AArch64::X17)
433 .addReg(Reg)
434 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
435 *STI);
436 if (Size != 1)
437 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
438 .addReg(AArch64::X17)
439 .addReg(AArch64::X17)
440 .addImm(Size - 1)
441 .addImm(0),
442 *STI);
443 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
444 .addReg(AArch64::WZR)
445 .addReg(AArch64::W16)
446 .addReg(AArch64::W17)
447 .addImm(0),
448 *STI);
449 OutStreamer->emitInstruction(
450 MCInstBuilder(AArch64::Bcc)
451 .addImm(AArch64CC::LS)
452 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
453 *STI);
454
455 OutStreamer->emitInstruction(
456 MCInstBuilder(AArch64::ORRXri)
457 .addReg(AArch64::X16)
458 .addReg(Reg)
459 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
460 *STI);
461 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
462 .addReg(AArch64::W16)
463 .addReg(AArch64::X16)
464 .addImm(0),
465 *STI);
466 OutStreamer->emitInstruction(
467 MCInstBuilder(AArch64::SUBSXrs)
468 .addReg(AArch64::XZR)
469 .addReg(AArch64::X16)
470 .addReg(Reg)
471 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
472 *STI);
473 OutStreamer->emitInstruction(
474 MCInstBuilder(AArch64::Bcc)
475 .addImm(AArch64CC::EQ)
476 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
477 *STI);
478
479 OutStreamer->emitLabel(HandleMismatchSym);
480 }
481
482 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
483 .addReg(AArch64::SP)
484 .addReg(AArch64::X0)
485 .addReg(AArch64::X1)
486 .addReg(AArch64::SP)
487 .addImm(-32),
488 *STI);
489 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
490 .addReg(AArch64::FP)
491 .addReg(AArch64::LR)
492 .addReg(AArch64::SP)
493 .addImm(29),
494 *STI);
495
496 if (Reg != AArch64::X0)
497 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
498 .addReg(AArch64::X0)
499 .addReg(AArch64::XZR)
500 .addReg(Reg)
501 .addImm(0),
502 *STI);
503 OutStreamer->emitInstruction(
504 MCInstBuilder(AArch64::MOVZXi)
505 .addReg(AArch64::X1)
506 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask)
507 .addImm(0),
508 *STI);
509
510 if (CompileKernel) {
511 // The Linux kernel's dynamic loader doesn't support GOT relative
512 // relocations, but it doesn't support late binding either, so just call
513 // the function directly.
514 OutStreamer->emitInstruction(
515 MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
516 } else {
517 // Intentionally load the GOT entry and branch to it, rather than possibly
518 // late binding the function, which may clobber the registers before we
519 // have a chance to save them.
520 OutStreamer->emitInstruction(
521 MCInstBuilder(AArch64::ADRP)
522 .addReg(AArch64::X16)
523 .addExpr(AArch64MCExpr::create(
524 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
525 OutContext)),
526 *STI);
527 OutStreamer->emitInstruction(
528 MCInstBuilder(AArch64::LDRXui)
529 .addReg(AArch64::X16)
530 .addReg(AArch64::X16)
531 .addExpr(AArch64MCExpr::create(
532 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
533 OutContext)),
534 *STI);
535 OutStreamer->emitInstruction(
536 MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
537 }
538 }
539 }
540
emitEndOfAsmFile(Module & M)541 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
542 EmitHwasanMemaccessSymbols(M);
543
544 const Triple &TT = TM.getTargetTriple();
545 if (TT.isOSBinFormatMachO()) {
546 // Funny Darwin hack: This flag tells the linker that no global symbols
547 // contain code that falls through to other global symbols (e.g. the obvious
548 // implementation of multiple entry points). If this doesn't occur, the
549 // linker can safely perform dead code stripping. Since LLVM never
550 // generates code that does this, it is always safe to set.
551 OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
552 }
553
554 // Emit stack and fault map information.
555 emitStackMaps(SM);
556 FM.serializeToFaultMapSection();
557
558 }
559
EmitLOHs()560 void AArch64AsmPrinter::EmitLOHs() {
561 SmallVector<MCSymbol *, 3> MCArgs;
562
563 for (const auto &D : AArch64FI->getLOHContainer()) {
564 for (const MachineInstr *MI : D.getArgs()) {
565 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
566 assert(LabelIt != LOHInstToLabel.end() &&
567 "Label hasn't been inserted for LOH related instruction");
568 MCArgs.push_back(LabelIt->second);
569 }
570 OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
571 MCArgs.clear();
572 }
573 }
574
emitFunctionBodyEnd()575 void AArch64AsmPrinter::emitFunctionBodyEnd() {
576 if (!AArch64FI->getLOHRelated().empty())
577 EmitLOHs();
578 }
579
580 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
GetCPISymbol(unsigned CPID) const581 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
582 // Darwin uses a linker-private symbol name for constant-pools (to
583 // avoid addends on the relocation?), ELF has no such concept and
584 // uses a normal private symbol.
585 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
586 return OutContext.getOrCreateSymbol(
587 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
588 Twine(getFunctionNumber()) + "_" + Twine(CPID));
589
590 return AsmPrinter::GetCPISymbol(CPID);
591 }
592
printOperand(const MachineInstr * MI,unsigned OpNum,raw_ostream & O)593 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
594 raw_ostream &O) {
595 const MachineOperand &MO = MI->getOperand(OpNum);
596 switch (MO.getType()) {
597 default:
598 llvm_unreachable("<unknown operand type>");
599 case MachineOperand::MO_Register: {
600 Register Reg = MO.getReg();
601 assert(Register::isPhysicalRegister(Reg));
602 assert(!MO.getSubReg() && "Subregs should be eliminated!");
603 O << AArch64InstPrinter::getRegisterName(Reg);
604 break;
605 }
606 case MachineOperand::MO_Immediate: {
607 O << MO.getImm();
608 break;
609 }
610 case MachineOperand::MO_GlobalAddress: {
611 PrintSymbolOperand(MO, O);
612 break;
613 }
614 case MachineOperand::MO_BlockAddress: {
615 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
616 Sym->print(O, MAI);
617 break;
618 }
619 }
620 }
621
printAsmMRegister(const MachineOperand & MO,char Mode,raw_ostream & O)622 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
623 raw_ostream &O) {
624 Register Reg = MO.getReg();
625 switch (Mode) {
626 default:
627 return true; // Unknown mode.
628 case 'w':
629 Reg = getWRegFromXReg(Reg);
630 break;
631 case 'x':
632 Reg = getXRegFromWReg(Reg);
633 break;
634 }
635
636 O << AArch64InstPrinter::getRegisterName(Reg);
637 return false;
638 }
639
640 // Prints the register in MO using class RC using the offset in the
641 // new register class. This should not be used for cross class
642 // printing.
printAsmRegInClass(const MachineOperand & MO,const TargetRegisterClass * RC,unsigned AltName,raw_ostream & O)643 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
644 const TargetRegisterClass *RC,
645 unsigned AltName, raw_ostream &O) {
646 assert(MO.isReg() && "Should only get here with a register!");
647 const TargetRegisterInfo *RI = STI->getRegisterInfo();
648 Register Reg = MO.getReg();
649 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
650 if (!RI->regsOverlap(RegToPrint, Reg))
651 return true;
652 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
653 return false;
654 }
655
PrintAsmOperand(const MachineInstr * MI,unsigned OpNum,const char * ExtraCode,raw_ostream & O)656 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
657 const char *ExtraCode, raw_ostream &O) {
658 const MachineOperand &MO = MI->getOperand(OpNum);
659
660 // First try the generic code, which knows about modifiers like 'c' and 'n'.
661 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
662 return false;
663
664 // Does this asm operand have a single letter operand modifier?
665 if (ExtraCode && ExtraCode[0]) {
666 if (ExtraCode[1] != 0)
667 return true; // Unknown modifier.
668
669 switch (ExtraCode[0]) {
670 default:
671 return true; // Unknown modifier.
672 case 'w': // Print W register
673 case 'x': // Print X register
674 if (MO.isReg())
675 return printAsmMRegister(MO, ExtraCode[0], O);
676 if (MO.isImm() && MO.getImm() == 0) {
677 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
678 O << AArch64InstPrinter::getRegisterName(Reg);
679 return false;
680 }
681 printOperand(MI, OpNum, O);
682 return false;
683 case 'b': // Print B register.
684 case 'h': // Print H register.
685 case 's': // Print S register.
686 case 'd': // Print D register.
687 case 'q': // Print Q register.
688 case 'z': // Print Z register.
689 if (MO.isReg()) {
690 const TargetRegisterClass *RC;
691 switch (ExtraCode[0]) {
692 case 'b':
693 RC = &AArch64::FPR8RegClass;
694 break;
695 case 'h':
696 RC = &AArch64::FPR16RegClass;
697 break;
698 case 's':
699 RC = &AArch64::FPR32RegClass;
700 break;
701 case 'd':
702 RC = &AArch64::FPR64RegClass;
703 break;
704 case 'q':
705 RC = &AArch64::FPR128RegClass;
706 break;
707 case 'z':
708 RC = &AArch64::ZPRRegClass;
709 break;
710 default:
711 return true;
712 }
713 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
714 }
715 printOperand(MI, OpNum, O);
716 return false;
717 }
718 }
719
720 // According to ARM, we should emit x and v registers unless we have a
721 // modifier.
722 if (MO.isReg()) {
723 Register Reg = MO.getReg();
724
725 // If this is a w or x register, print an x register.
726 if (AArch64::GPR32allRegClass.contains(Reg) ||
727 AArch64::GPR64allRegClass.contains(Reg))
728 return printAsmMRegister(MO, 'x', O);
729
730 unsigned AltName = AArch64::NoRegAltName;
731 const TargetRegisterClass *RegClass;
732 if (AArch64::ZPRRegClass.contains(Reg)) {
733 RegClass = &AArch64::ZPRRegClass;
734 } else if (AArch64::PPRRegClass.contains(Reg)) {
735 RegClass = &AArch64::PPRRegClass;
736 } else {
737 RegClass = &AArch64::FPR128RegClass;
738 AltName = AArch64::vreg;
739 }
740
741 // If this is a b, h, s, d, or q register, print it as a v register.
742 return printAsmRegInClass(MO, RegClass, AltName, O);
743 }
744
745 printOperand(MI, OpNum, O);
746 return false;
747 }
748
PrintAsmMemoryOperand(const MachineInstr * MI,unsigned OpNum,const char * ExtraCode,raw_ostream & O)749 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
750 unsigned OpNum,
751 const char *ExtraCode,
752 raw_ostream &O) {
753 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
754 return true; // Unknown modifier.
755
756 const MachineOperand &MO = MI->getOperand(OpNum);
757 assert(MO.isReg() && "unexpected inline asm memory operand");
758 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
759 return false;
760 }
761
PrintDebugValueComment(const MachineInstr * MI,raw_ostream & OS)762 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
763 raw_ostream &OS) {
764 unsigned NOps = MI->getNumOperands();
765 assert(NOps == 4);
766 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
767 // cast away const; DIetc do not take const operands for some reason.
768 OS << MI->getDebugVariable()->getName();
769 OS << " <- ";
770 // Frame address. Currently handles register +- offset only.
771 assert(MI->getDebugOperand(0).isReg() && MI->isDebugOffsetImm());
772 OS << '[';
773 printOperand(MI, 0, OS);
774 OS << '+';
775 printOperand(MI, 1, OS);
776 OS << ']';
777 OS << "+";
778 printOperand(MI, NOps - 2, OS);
779 }
780
emitJumpTableInfo()781 void AArch64AsmPrinter::emitJumpTableInfo() {
782 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
783 if (!MJTI) return;
784
785 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
786 if (JT.empty()) return;
787
788 const Function &F = MF->getFunction();
789 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
790 bool JTInDiffSection =
791 !STI->isTargetCOFF() ||
792 !TLOF.shouldPutJumpTableInFunctionSection(
793 MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
794 F);
795 if (JTInDiffSection) {
796 // Drop it in the readonly section.
797 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
798 OutStreamer->SwitchSection(ReadOnlySec);
799 }
800
801 auto AFI = MF->getInfo<AArch64FunctionInfo>();
802 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
803 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
804
805 // If this jump table was deleted, ignore it.
806 if (JTBBs.empty()) continue;
807
808 unsigned Size = AFI->getJumpTableEntrySize(JTI);
809 emitAlignment(Align(Size));
810 OutStreamer->emitLabel(GetJTISymbol(JTI));
811
812 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
813 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
814
815 for (auto *JTBB : JTBBs) {
816 const MCExpr *Value =
817 MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
818
819 // Each entry is:
820 // .byte/.hword (LBB - Lbase)>>2
821 // or plain:
822 // .word LBB - Lbase
823 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
824 if (Size != 4)
825 Value = MCBinaryExpr::createLShr(
826 Value, MCConstantExpr::create(2, OutContext), OutContext);
827
828 OutStreamer->emitValue(Value, Size);
829 }
830 }
831 }
832
emitFunctionEntryLabel()833 void AArch64AsmPrinter::emitFunctionEntryLabel() {
834 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
835 MF->getFunction().getCallingConv() ==
836 CallingConv::AArch64_SVE_VectorCall ||
837 STI->getRegisterInfo()->hasSVEArgsOrReturn(MF)) {
838 auto *TS =
839 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
840 TS->emitDirectiveVariantPCS(CurrentFnSym);
841 }
842
843 return AsmPrinter::emitFunctionEntryLabel();
844 }
845
846 /// Small jump tables contain an unsigned byte or half, representing the offset
847 /// from the lowest-addressed possible destination to the desired basic
848 /// block. Since all instructions are 4-byte aligned, this is further compressed
849 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
850 /// materialize the correct destination we need:
851 ///
852 /// adr xDest, .LBB0_0
853 /// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
854 /// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
LowerJumpTableDest(llvm::MCStreamer & OutStreamer,const llvm::MachineInstr & MI)855 void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
856 const llvm::MachineInstr &MI) {
857 Register DestReg = MI.getOperand(0).getReg();
858 Register ScratchReg = MI.getOperand(1).getReg();
859 Register ScratchRegW =
860 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
861 Register TableReg = MI.getOperand(2).getReg();
862 Register EntryReg = MI.getOperand(3).getReg();
863 int JTIdx = MI.getOperand(4).getIndex();
864 int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
865
866 // This has to be first because the compression pass based its reachability
867 // calculations on the start of the JumpTableDest instruction.
868 auto Label =
869 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
870
871 // If we don't already have a symbol to use as the base, use the ADR
872 // instruction itself.
873 if (!Label) {
874 Label = MF->getContext().createTempSymbol();
875 AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
876 OutStreamer.emitLabel(Label);
877 }
878
879 auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
880 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
881 .addReg(DestReg)
882 .addExpr(LabelExpr));
883
884 // Load the number of instruction-steps to offset from the label.
885 unsigned LdrOpcode;
886 switch (Size) {
887 case 1: LdrOpcode = AArch64::LDRBBroX; break;
888 case 2: LdrOpcode = AArch64::LDRHHroX; break;
889 case 4: LdrOpcode = AArch64::LDRSWroX; break;
890 default:
891 llvm_unreachable("Unknown jump table size");
892 }
893
894 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
895 .addReg(Size == 4 ? ScratchReg : ScratchRegW)
896 .addReg(TableReg)
897 .addReg(EntryReg)
898 .addImm(0)
899 .addImm(Size == 1 ? 0 : 1));
900
901 // Add to the already materialized base label address, multiplying by 4 if
902 // compressed.
903 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
904 .addReg(DestReg)
905 .addReg(DestReg)
906 .addReg(ScratchReg)
907 .addImm(Size == 4 ? 0 : 2));
908 }
909
LowerSTACKMAP(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)910 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
911 const MachineInstr &MI) {
912 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
913
914 auto &Ctx = OutStreamer.getContext();
915 MCSymbol *MILabel = Ctx.createTempSymbol();
916 OutStreamer.emitLabel(MILabel);
917
918 SM.recordStackMap(*MILabel, MI);
919 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
920
921 // Scan ahead to trim the shadow.
922 const MachineBasicBlock &MBB = *MI.getParent();
923 MachineBasicBlock::const_iterator MII(MI);
924 ++MII;
925 while (NumNOPBytes > 0) {
926 if (MII == MBB.end() || MII->isCall() ||
927 MII->getOpcode() == AArch64::DBG_VALUE ||
928 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
929 MII->getOpcode() == TargetOpcode::STACKMAP)
930 break;
931 ++MII;
932 NumNOPBytes -= 4;
933 }
934
935 // Emit nops.
936 for (unsigned i = 0; i < NumNOPBytes; i += 4)
937 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
938 }
939
940 // Lower a patchpoint of the form:
941 // [<def>], <id>, <numBytes>, <target>, <numArgs>
LowerPATCHPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)942 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
943 const MachineInstr &MI) {
944 auto &Ctx = OutStreamer.getContext();
945 MCSymbol *MILabel = Ctx.createTempSymbol();
946 OutStreamer.emitLabel(MILabel);
947 SM.recordPatchPoint(*MILabel, MI);
948
949 PatchPointOpers Opers(&MI);
950
951 int64_t CallTarget = Opers.getCallTarget().getImm();
952 unsigned EncodedBytes = 0;
953 if (CallTarget) {
954 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
955 "High 16 bits of call target should be zero.");
956 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
957 EncodedBytes = 16;
958 // Materialize the jump address:
959 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
960 .addReg(ScratchReg)
961 .addImm((CallTarget >> 32) & 0xFFFF)
962 .addImm(32));
963 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
964 .addReg(ScratchReg)
965 .addReg(ScratchReg)
966 .addImm((CallTarget >> 16) & 0xFFFF)
967 .addImm(16));
968 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
969 .addReg(ScratchReg)
970 .addReg(ScratchReg)
971 .addImm(CallTarget & 0xFFFF)
972 .addImm(0));
973 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
974 }
975 // Emit padding.
976 unsigned NumBytes = Opers.getNumPatchBytes();
977 assert(NumBytes >= EncodedBytes &&
978 "Patchpoint can't request size less than the length of a call.");
979 assert((NumBytes - EncodedBytes) % 4 == 0 &&
980 "Invalid number of NOP bytes requested!");
981 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
982 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
983 }
984
LowerSTATEPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)985 void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
986 const MachineInstr &MI) {
987 StatepointOpers SOpers(&MI);
988 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
989 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
990 for (unsigned i = 0; i < PatchBytes; i += 4)
991 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
992 } else {
993 // Lower call target and choose correct opcode
994 const MachineOperand &CallTarget = SOpers.getCallTarget();
995 MCOperand CallTargetMCOp;
996 unsigned CallOpcode;
997 switch (CallTarget.getType()) {
998 case MachineOperand::MO_GlobalAddress:
999 case MachineOperand::MO_ExternalSymbol:
1000 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1001 CallOpcode = AArch64::BL;
1002 break;
1003 case MachineOperand::MO_Immediate:
1004 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1005 CallOpcode = AArch64::BL;
1006 break;
1007 case MachineOperand::MO_Register:
1008 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1009 CallOpcode = AArch64::BLR;
1010 break;
1011 default:
1012 llvm_unreachable("Unsupported operand type in statepoint call target");
1013 break;
1014 }
1015
1016 EmitToStreamer(OutStreamer,
1017 MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1018 }
1019
1020 auto &Ctx = OutStreamer.getContext();
1021 MCSymbol *MILabel = Ctx.createTempSymbol();
1022 OutStreamer.emitLabel(MILabel);
1023 SM.recordStatepoint(*MILabel, MI);
1024 }
1025
LowerFAULTING_OP(const MachineInstr & FaultingMI)1026 void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1027 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1028 // <opcode>, <operands>
1029
1030 Register DefRegister = FaultingMI.getOperand(0).getReg();
1031 FaultMaps::FaultKind FK =
1032 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1033 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1034 unsigned Opcode = FaultingMI.getOperand(3).getImm();
1035 unsigned OperandsBeginIdx = 4;
1036
1037 auto &Ctx = OutStreamer->getContext();
1038 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1039 OutStreamer->emitLabel(FaultingLabel);
1040
1041 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1042 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1043
1044 MCInst MI;
1045 MI.setOpcode(Opcode);
1046
1047 if (DefRegister != (Register)0)
1048 MI.addOperand(MCOperand::createReg(DefRegister));
1049
1050 for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx,
1051 E = FaultingMI.operands_end();
1052 I != E; ++I) {
1053 MCOperand Dest;
1054 lowerOperand(*I, Dest);
1055 MI.addOperand(Dest);
1056 }
1057
1058 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1059 OutStreamer->emitInstruction(MI, getSubtargetInfo());
1060 }
1061
EmitFMov0(const MachineInstr & MI)1062 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
1063 Register DestReg = MI.getOperand(0).getReg();
1064 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
1065 // Convert H/S/D register to corresponding Q register
1066 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1067 DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
1068 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1069 DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
1070 else {
1071 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1072 DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
1073 }
1074 MCInst MOVI;
1075 MOVI.setOpcode(AArch64::MOVIv2d_ns);
1076 MOVI.addOperand(MCOperand::createReg(DestReg));
1077 MOVI.addOperand(MCOperand::createImm(0));
1078 EmitToStreamer(*OutStreamer, MOVI);
1079 } else {
1080 MCInst FMov;
1081 switch (MI.getOpcode()) {
1082 default: llvm_unreachable("Unexpected opcode");
1083 case AArch64::FMOVH0:
1084 FMov.setOpcode(AArch64::FMOVWHr);
1085 FMov.addOperand(MCOperand::createReg(DestReg));
1086 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1087 break;
1088 case AArch64::FMOVS0:
1089 FMov.setOpcode(AArch64::FMOVWSr);
1090 FMov.addOperand(MCOperand::createReg(DestReg));
1091 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1092 break;
1093 case AArch64::FMOVD0:
1094 FMov.setOpcode(AArch64::FMOVXDr);
1095 FMov.addOperand(MCOperand::createReg(DestReg));
1096 FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1097 break;
1098 }
1099 EmitToStreamer(*OutStreamer, FMov);
1100 }
1101 }
1102
1103 // Simple pseudo-instructions have their lowering (with expansion to real
1104 // instructions) auto-generated.
1105 #include "AArch64GenMCPseudoLowering.inc"
1106
emitInstruction(const MachineInstr * MI)1107 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1108 // Do any auto-generated pseudo lowerings.
1109 if (emitPseudoExpansionLowering(*OutStreamer, MI))
1110 return;
1111
1112 if (AArch64FI->getLOHRelated().count(MI)) {
1113 // Generate a label for LOH related instruction
1114 MCSymbol *LOHLabel = createTempSymbol("loh");
1115 // Associate the instruction with the label
1116 LOHInstToLabel[MI] = LOHLabel;
1117 OutStreamer->emitLabel(LOHLabel);
1118 }
1119
1120 AArch64TargetStreamer *TS =
1121 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1122 // Do any manual lowerings.
1123 switch (MI->getOpcode()) {
1124 default:
1125 break;
1126 case AArch64::HINT: {
1127 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1128 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1129 // non-empty. If MI is the initial BTI, place the
1130 // __patchable_function_entries label after BTI.
1131 if (CurrentPatchableFunctionEntrySym &&
1132 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1133 MI == &MF->front().front()) {
1134 int64_t Imm = MI->getOperand(0).getImm();
1135 if ((Imm & 32) && (Imm & 6)) {
1136 MCInst Inst;
1137 MCInstLowering.Lower(MI, Inst);
1138 EmitToStreamer(*OutStreamer, Inst);
1139 CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1140 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1141 return;
1142 }
1143 }
1144 break;
1145 }
1146 case AArch64::MOVMCSym: {
1147 Register DestReg = MI->getOperand(0).getReg();
1148 const MachineOperand &MO_Sym = MI->getOperand(1);
1149 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1150 MCOperand Hi_MCSym, Lo_MCSym;
1151
1152 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1153 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1154
1155 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1156 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1157
1158 MCInst MovZ;
1159 MovZ.setOpcode(AArch64::MOVZXi);
1160 MovZ.addOperand(MCOperand::createReg(DestReg));
1161 MovZ.addOperand(Hi_MCSym);
1162 MovZ.addOperand(MCOperand::createImm(16));
1163 EmitToStreamer(*OutStreamer, MovZ);
1164
1165 MCInst MovK;
1166 MovK.setOpcode(AArch64::MOVKXi);
1167 MovK.addOperand(MCOperand::createReg(DestReg));
1168 MovK.addOperand(MCOperand::createReg(DestReg));
1169 MovK.addOperand(Lo_MCSym);
1170 MovK.addOperand(MCOperand::createImm(0));
1171 EmitToStreamer(*OutStreamer, MovK);
1172 return;
1173 }
1174 case AArch64::MOVIv2d_ns:
1175 // If the target has <rdar://problem/16473581>, lower this
1176 // instruction to movi.16b instead.
1177 if (STI->hasZeroCycleZeroingFPWorkaround() &&
1178 MI->getOperand(1).getImm() == 0) {
1179 MCInst TmpInst;
1180 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1181 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1182 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1183 EmitToStreamer(*OutStreamer, TmpInst);
1184 return;
1185 }
1186 break;
1187
1188 case AArch64::DBG_VALUE: {
1189 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1190 SmallString<128> TmpStr;
1191 raw_svector_ostream OS(TmpStr);
1192 PrintDebugValueComment(MI, OS);
1193 OutStreamer->emitRawText(StringRef(OS.str()));
1194 }
1195 return;
1196
1197 case AArch64::EMITBKEY: {
1198 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1199 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1200 ExceptionHandlingType != ExceptionHandling::ARM)
1201 return;
1202
1203 if (needsCFIMoves() == CFI_M_None)
1204 return;
1205
1206 OutStreamer->emitCFIBKeyFrame();
1207 return;
1208 }
1209 }
1210
1211 // Tail calls use pseudo instructions so they have the proper code-gen
1212 // attributes (isCall, isReturn, etc.). We lower them to the real
1213 // instruction here.
1214 case AArch64::TCRETURNri:
1215 case AArch64::TCRETURNriBTI:
1216 case AArch64::TCRETURNriALL: {
1217 MCInst TmpInst;
1218 TmpInst.setOpcode(AArch64::BR);
1219 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1220 EmitToStreamer(*OutStreamer, TmpInst);
1221 return;
1222 }
1223 case AArch64::TCRETURNdi: {
1224 MCOperand Dest;
1225 MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1226 MCInst TmpInst;
1227 TmpInst.setOpcode(AArch64::B);
1228 TmpInst.addOperand(Dest);
1229 EmitToStreamer(*OutStreamer, TmpInst);
1230 return;
1231 }
1232 case AArch64::SpeculationBarrierISBDSBEndBB: {
1233 // Print DSB SYS + ISB
1234 MCInst TmpInstDSB;
1235 TmpInstDSB.setOpcode(AArch64::DSB);
1236 TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1237 EmitToStreamer(*OutStreamer, TmpInstDSB);
1238 MCInst TmpInstISB;
1239 TmpInstISB.setOpcode(AArch64::ISB);
1240 TmpInstISB.addOperand(MCOperand::createImm(0xf));
1241 EmitToStreamer(*OutStreamer, TmpInstISB);
1242 return;
1243 }
1244 case AArch64::SpeculationBarrierSBEndBB: {
1245 // Print SB
1246 MCInst TmpInstSB;
1247 TmpInstSB.setOpcode(AArch64::SB);
1248 EmitToStreamer(*OutStreamer, TmpInstSB);
1249 return;
1250 }
1251 case AArch64::TLSDESC_CALLSEQ: {
1252 /// lower this to:
1253 /// adrp x0, :tlsdesc:var
1254 /// ldr x1, [x0, #:tlsdesc_lo12:var]
1255 /// add x0, x0, #:tlsdesc_lo12:var
1256 /// .tlsdesccall var
1257 /// blr x1
1258 /// (TPIDR_EL0 offset now in x0)
1259 const MachineOperand &MO_Sym = MI->getOperand(0);
1260 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1261 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1262 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1263 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1264 MCInstLowering.lowerOperand(MO_Sym, Sym);
1265 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1266 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1267
1268 MCInst Adrp;
1269 Adrp.setOpcode(AArch64::ADRP);
1270 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1271 Adrp.addOperand(SymTLSDesc);
1272 EmitToStreamer(*OutStreamer, Adrp);
1273
1274 MCInst Ldr;
1275 if (STI->isTargetILP32()) {
1276 Ldr.setOpcode(AArch64::LDRWui);
1277 Ldr.addOperand(MCOperand::createReg(AArch64::W1));
1278 } else {
1279 Ldr.setOpcode(AArch64::LDRXui);
1280 Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1281 }
1282 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1283 Ldr.addOperand(SymTLSDescLo12);
1284 Ldr.addOperand(MCOperand::createImm(0));
1285 EmitToStreamer(*OutStreamer, Ldr);
1286
1287 MCInst Add;
1288 if (STI->isTargetILP32()) {
1289 Add.setOpcode(AArch64::ADDWri);
1290 Add.addOperand(MCOperand::createReg(AArch64::W0));
1291 Add.addOperand(MCOperand::createReg(AArch64::W0));
1292 } else {
1293 Add.setOpcode(AArch64::ADDXri);
1294 Add.addOperand(MCOperand::createReg(AArch64::X0));
1295 Add.addOperand(MCOperand::createReg(AArch64::X0));
1296 }
1297 Add.addOperand(SymTLSDescLo12);
1298 Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1299 EmitToStreamer(*OutStreamer, Add);
1300
1301 // Emit a relocation-annotation. This expands to no code, but requests
1302 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1303 MCInst TLSDescCall;
1304 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1305 TLSDescCall.addOperand(Sym);
1306 EmitToStreamer(*OutStreamer, TLSDescCall);
1307
1308 MCInst Blr;
1309 Blr.setOpcode(AArch64::BLR);
1310 Blr.addOperand(MCOperand::createReg(AArch64::X1));
1311 EmitToStreamer(*OutStreamer, Blr);
1312
1313 return;
1314 }
1315
1316 case AArch64::JumpTableDest32:
1317 case AArch64::JumpTableDest16:
1318 case AArch64::JumpTableDest8:
1319 LowerJumpTableDest(*OutStreamer, *MI);
1320 return;
1321
1322 case AArch64::FMOVH0:
1323 case AArch64::FMOVS0:
1324 case AArch64::FMOVD0:
1325 EmitFMov0(*MI);
1326 return;
1327
1328 case TargetOpcode::STACKMAP:
1329 return LowerSTACKMAP(*OutStreamer, SM, *MI);
1330
1331 case TargetOpcode::PATCHPOINT:
1332 return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1333
1334 case TargetOpcode::STATEPOINT:
1335 return LowerSTATEPOINT(*OutStreamer, SM, *MI);
1336
1337 case TargetOpcode::FAULTING_OP:
1338 return LowerFAULTING_OP(*MI);
1339
1340 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1341 LowerPATCHABLE_FUNCTION_ENTER(*MI);
1342 return;
1343
1344 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1345 LowerPATCHABLE_FUNCTION_EXIT(*MI);
1346 return;
1347
1348 case TargetOpcode::PATCHABLE_TAIL_CALL:
1349 LowerPATCHABLE_TAIL_CALL(*MI);
1350 return;
1351
1352 case AArch64::HWASAN_CHECK_MEMACCESS:
1353 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1354 LowerHWASAN_CHECK_MEMACCESS(*MI);
1355 return;
1356
1357 case AArch64::SEH_StackAlloc:
1358 TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1359 return;
1360
1361 case AArch64::SEH_SaveFPLR:
1362 TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1363 return;
1364
1365 case AArch64::SEH_SaveFPLR_X:
1366 assert(MI->getOperand(0).getImm() < 0 &&
1367 "Pre increment SEH opcode must have a negative offset");
1368 TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1369 return;
1370
1371 case AArch64::SEH_SaveReg:
1372 TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1373 MI->getOperand(1).getImm());
1374 return;
1375
1376 case AArch64::SEH_SaveReg_X:
1377 assert(MI->getOperand(1).getImm() < 0 &&
1378 "Pre increment SEH opcode must have a negative offset");
1379 TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1380 -MI->getOperand(1).getImm());
1381 return;
1382
1383 case AArch64::SEH_SaveRegP:
1384 if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
1385 MI->getOperand(0).getImm() <= 28) {
1386 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1387 "Register paired with LR must be odd");
1388 TS->EmitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
1389 MI->getOperand(2).getImm());
1390 return;
1391 }
1392 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1393 "Non-consecutive registers not allowed for save_regp");
1394 TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1395 MI->getOperand(2).getImm());
1396 return;
1397
1398 case AArch64::SEH_SaveRegP_X:
1399 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1400 "Non-consecutive registers not allowed for save_regp_x");
1401 assert(MI->getOperand(2).getImm() < 0 &&
1402 "Pre increment SEH opcode must have a negative offset");
1403 TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1404 -MI->getOperand(2).getImm());
1405 return;
1406
1407 case AArch64::SEH_SaveFReg:
1408 TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1409 MI->getOperand(1).getImm());
1410 return;
1411
1412 case AArch64::SEH_SaveFReg_X:
1413 assert(MI->getOperand(1).getImm() < 0 &&
1414 "Pre increment SEH opcode must have a negative offset");
1415 TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1416 -MI->getOperand(1).getImm());
1417 return;
1418
1419 case AArch64::SEH_SaveFRegP:
1420 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1421 "Non-consecutive registers not allowed for save_regp");
1422 TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1423 MI->getOperand(2).getImm());
1424 return;
1425
1426 case AArch64::SEH_SaveFRegP_X:
1427 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1428 "Non-consecutive registers not allowed for save_regp_x");
1429 assert(MI->getOperand(2).getImm() < 0 &&
1430 "Pre increment SEH opcode must have a negative offset");
1431 TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1432 -MI->getOperand(2).getImm());
1433 return;
1434
1435 case AArch64::SEH_SetFP:
1436 TS->EmitARM64WinCFISetFP();
1437 return;
1438
1439 case AArch64::SEH_AddFP:
1440 TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1441 return;
1442
1443 case AArch64::SEH_Nop:
1444 TS->EmitARM64WinCFINop();
1445 return;
1446
1447 case AArch64::SEH_PrologEnd:
1448 TS->EmitARM64WinCFIPrologEnd();
1449 return;
1450
1451 case AArch64::SEH_EpilogStart:
1452 TS->EmitARM64WinCFIEpilogStart();
1453 return;
1454
1455 case AArch64::SEH_EpilogEnd:
1456 TS->EmitARM64WinCFIEpilogEnd();
1457 return;
1458 }
1459
1460 // Finally, do the automated lowerings for everything else.
1461 MCInst TmpInst;
1462 MCInstLowering.Lower(MI, TmpInst);
1463 EmitToStreamer(*OutStreamer, TmpInst);
1464 }
1465
1466 // Force static initialization.
LLVMInitializeAArch64AsmPrinter()1467 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1468 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1469 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1470 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1471 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1472 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1473 }
1474