1 //===-- RISCVAsmPrinter.cpp - RISC-V LLVM assembly writer -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the RISC-V assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "MCTargetDesc/RISCVBaseInfo.h"
15 #include "MCTargetDesc/RISCVInstPrinter.h"
16 #include "MCTargetDesc/RISCVMCExpr.h"
17 #include "MCTargetDesc/RISCVTargetStreamer.h"
18 #include "RISCV.h"
19 #include "RISCVMachineFunctionInfo.h"
20 #include "RISCVTargetMachine.h"
21 #include "TargetInfo/RISCVTargetInfo.h"
22 #include "llvm/ADT/APInt.h"
23 #include "llvm/ADT/Statistic.h"
24 #include "llvm/BinaryFormat/ELF.h"
25 #include "llvm/CodeGen/AsmPrinter.h"
26 #include "llvm/CodeGen/MachineConstantPool.h"
27 #include "llvm/CodeGen/MachineFunctionPass.h"
28 #include "llvm/CodeGen/MachineInstr.h"
29 #include "llvm/CodeGen/MachineModuleInfo.h"
30 #include "llvm/MC/MCAsmInfo.h"
31 #include "llvm/MC/MCContext.h"
32 #include "llvm/MC/MCInst.h"
33 #include "llvm/MC/MCInstBuilder.h"
34 #include "llvm/MC/MCObjectFileInfo.h"
35 #include "llvm/MC/MCSectionELF.h"
36 #include "llvm/MC/MCStreamer.h"
37 #include "llvm/MC/MCSymbol.h"
38 #include "llvm/MC/TargetRegistry.h"
39 #include "llvm/Support/RISCVISAInfo.h"
40 #include "llvm/Support/raw_ostream.h"
41 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
42
43 using namespace llvm;
44
45 #define DEBUG_TYPE "asm-printer"
46
47 STATISTIC(RISCVNumInstrsCompressed,
48 "Number of RISC-V Compressed instructions emitted");
49
50 namespace llvm {
51 extern const SubtargetFeatureKV RISCVFeatureKV[RISCV::NumSubtargetFeatures];
52 } // namespace llvm
53
54 namespace {
55 class RISCVAsmPrinter : public AsmPrinter {
56 const RISCVSubtarget *STI;
57
58 public:
RISCVAsmPrinter(TargetMachine & TM,std::unique_ptr<MCStreamer> Streamer)59 explicit RISCVAsmPrinter(TargetMachine &TM,
60 std::unique_ptr<MCStreamer> Streamer)
61 : AsmPrinter(TM, std::move(Streamer)) {}
62
getPassName() const63 StringRef getPassName() const override { return "RISC-V Assembly Printer"; }
64
65 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
66 const MachineInstr &MI);
67
68 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
69 const MachineInstr &MI);
70
71 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
72 const MachineInstr &MI);
73
74 bool runOnMachineFunction(MachineFunction &MF) override;
75
76 void emitInstruction(const MachineInstr *MI) override;
77
78 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
79 const char *ExtraCode, raw_ostream &OS) override;
80 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
81 const char *ExtraCode, raw_ostream &OS) override;
82
83 void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
84 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
85 const MachineInstr *MI);
86
87 typedef std::tuple<unsigned, uint32_t> HwasanMemaccessTuple;
88 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
89 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
90 void LowerKCFI_CHECK(const MachineInstr &MI);
91 void EmitHwasanMemaccessSymbols(Module &M);
92
93 // Wrapper needed for tblgenned pseudo lowering.
94 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const;
95
96 void emitStartOfAsmFile(Module &M) override;
97 void emitEndOfAsmFile(Module &M) override;
98
99 void emitFunctionEntryLabel() override;
100 bool emitDirectiveOptionArch();
101
102 private:
103 void emitAttributes(const MCSubtargetInfo &SubtargetInfo);
104
105 void emitNTLHint(const MachineInstr *MI);
106
107 bool lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
108 };
109 }
110
LowerSTACKMAP(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)111 void RISCVAsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
112 const MachineInstr &MI) {
113 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
114 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
115
116 auto &Ctx = OutStreamer.getContext();
117 MCSymbol *MILabel = Ctx.createTempSymbol();
118 OutStreamer.emitLabel(MILabel);
119
120 SM.recordStackMap(*MILabel, MI);
121 assert(NumNOPBytes % NOPBytes == 0 &&
122 "Invalid number of NOP bytes requested!");
123
124 // Scan ahead to trim the shadow.
125 const MachineBasicBlock &MBB = *MI.getParent();
126 MachineBasicBlock::const_iterator MII(MI);
127 ++MII;
128 while (NumNOPBytes > 0) {
129 if (MII == MBB.end() || MII->isCall() ||
130 MII->getOpcode() == RISCV::DBG_VALUE ||
131 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
132 MII->getOpcode() == TargetOpcode::STACKMAP)
133 break;
134 ++MII;
135 NumNOPBytes -= 4;
136 }
137
138 // Emit nops.
139 emitNops(NumNOPBytes / NOPBytes);
140 }
141
142 // Lower a patchpoint of the form:
143 // [<def>], <id>, <numBytes>, <target>, <numArgs>
LowerPATCHPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)144 void RISCVAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
145 const MachineInstr &MI) {
146 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
147
148 auto &Ctx = OutStreamer.getContext();
149 MCSymbol *MILabel = Ctx.createTempSymbol();
150 OutStreamer.emitLabel(MILabel);
151 SM.recordPatchPoint(*MILabel, MI);
152
153 PatchPointOpers Opers(&MI);
154
155 unsigned EncodedBytes = 0;
156
157 // Emit padding.
158 unsigned NumBytes = Opers.getNumPatchBytes();
159 assert(NumBytes >= EncodedBytes &&
160 "Patchpoint can't request size less than the length of a call.");
161 assert((NumBytes - EncodedBytes) % NOPBytes == 0 &&
162 "Invalid number of NOP bytes requested!");
163 emitNops((NumBytes - EncodedBytes) / NOPBytes);
164 }
165
LowerSTATEPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)166 void RISCVAsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
167 const MachineInstr &MI) {
168 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
169
170 StatepointOpers SOpers(&MI);
171 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
172 assert(PatchBytes % NOPBytes == 0 &&
173 "Invalid number of NOP bytes requested!");
174 emitNops(PatchBytes / NOPBytes);
175 }
176
177 auto &Ctx = OutStreamer.getContext();
178 MCSymbol *MILabel = Ctx.createTempSymbol();
179 OutStreamer.emitLabel(MILabel);
180 SM.recordStatepoint(*MILabel, MI);
181 }
182
EmitToStreamer(MCStreamer & S,const MCInst & Inst)183 void RISCVAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
184 MCInst CInst;
185 bool Res = RISCVRVC::compress(CInst, Inst, *STI);
186 if (Res)
187 ++RISCVNumInstrsCompressed;
188 AsmPrinter::EmitToStreamer(*OutStreamer, Res ? CInst : Inst);
189 }
190
191 // Simple pseudo-instructions have their lowering (with expansion to real
192 // instructions) auto-generated.
193 #include "RISCVGenMCPseudoLowering.inc"
194
195 // If the target supports Zihintntl and the instruction has a nontemporal
196 // MachineMemOperand, emit an NTLH hint instruction before it.
emitNTLHint(const MachineInstr * MI)197 void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) {
198 if (!STI->hasStdExtZihintntl())
199 return;
200
201 if (MI->memoperands_empty())
202 return;
203
204 MachineMemOperand *MMO = *(MI->memoperands_begin());
205 if (!MMO->isNonTemporal())
206 return;
207
208 unsigned NontemporalMode = 0;
209 if (MMO->getFlags() & MONontemporalBit0)
210 NontemporalMode += 0b1;
211 if (MMO->getFlags() & MONontemporalBit1)
212 NontemporalMode += 0b10;
213
214 MCInst Hint;
215 if (STI->hasStdExtCOrZca() && STI->enableRVCHintInstrs())
216 Hint.setOpcode(RISCV::C_ADD_HINT);
217 else
218 Hint.setOpcode(RISCV::ADD);
219
220 Hint.addOperand(MCOperand::createReg(RISCV::X0));
221 Hint.addOperand(MCOperand::createReg(RISCV::X0));
222 Hint.addOperand(MCOperand::createReg(RISCV::X2 + NontemporalMode));
223
224 EmitToStreamer(*OutStreamer, Hint);
225 }
226
emitInstruction(const MachineInstr * MI)227 void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
228 RISCV_MC::verifyInstructionPredicates(MI->getOpcode(),
229 getSubtargetInfo().getFeatureBits());
230
231 emitNTLHint(MI);
232
233 // Do any auto-generated pseudo lowerings.
234 if (emitPseudoExpansionLowering(*OutStreamer, MI))
235 return;
236
237
238 switch (MI->getOpcode()) {
239 case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
240 LowerHWASAN_CHECK_MEMACCESS(*MI);
241 return;
242 case RISCV::KCFI_CHECK:
243 LowerKCFI_CHECK(*MI);
244 return;
245 case RISCV::PseudoRVVInitUndefM1:
246 case RISCV::PseudoRVVInitUndefM2:
247 case RISCV::PseudoRVVInitUndefM4:
248 case RISCV::PseudoRVVInitUndefM8:
249 return;
250 case TargetOpcode::STACKMAP:
251 return LowerSTACKMAP(*OutStreamer, SM, *MI);
252 case TargetOpcode::PATCHPOINT:
253 return LowerPATCHPOINT(*OutStreamer, SM, *MI);
254 case TargetOpcode::STATEPOINT:
255 return LowerSTATEPOINT(*OutStreamer, SM, *MI);
256 }
257
258 MCInst OutInst;
259 if (!lowerToMCInst(MI, OutInst))
260 EmitToStreamer(*OutStreamer, OutInst);
261 }
262
PrintAsmOperand(const MachineInstr * MI,unsigned OpNo,const char * ExtraCode,raw_ostream & OS)263 bool RISCVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
264 const char *ExtraCode, raw_ostream &OS) {
265 // First try the generic code, which knows about modifiers like 'c' and 'n'.
266 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS))
267 return false;
268
269 const MachineOperand &MO = MI->getOperand(OpNo);
270 if (ExtraCode && ExtraCode[0]) {
271 if (ExtraCode[1] != 0)
272 return true; // Unknown modifier.
273
274 switch (ExtraCode[0]) {
275 default:
276 return true; // Unknown modifier.
277 case 'z': // Print zero register if zero, regular printing otherwise.
278 if (MO.isImm() && MO.getImm() == 0) {
279 OS << RISCVInstPrinter::getRegisterName(RISCV::X0);
280 return false;
281 }
282 break;
283 case 'i': // Literal 'i' if operand is not a register.
284 if (!MO.isReg())
285 OS << 'i';
286 return false;
287 }
288 }
289
290 switch (MO.getType()) {
291 case MachineOperand::MO_Immediate:
292 OS << MO.getImm();
293 return false;
294 case MachineOperand::MO_Register:
295 OS << RISCVInstPrinter::getRegisterName(MO.getReg());
296 return false;
297 case MachineOperand::MO_GlobalAddress:
298 PrintSymbolOperand(MO, OS);
299 return false;
300 case MachineOperand::MO_BlockAddress: {
301 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
302 Sym->print(OS, MAI);
303 return false;
304 }
305 default:
306 break;
307 }
308
309 return true;
310 }
311
PrintAsmMemoryOperand(const MachineInstr * MI,unsigned OpNo,const char * ExtraCode,raw_ostream & OS)312 bool RISCVAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
313 unsigned OpNo,
314 const char *ExtraCode,
315 raw_ostream &OS) {
316 if (ExtraCode)
317 return AsmPrinter::PrintAsmMemoryOperand(MI, OpNo, ExtraCode, OS);
318
319 const MachineOperand &AddrReg = MI->getOperand(OpNo);
320 assert(MI->getNumOperands() > OpNo + 1 && "Expected additional operand");
321 const MachineOperand &Offset = MI->getOperand(OpNo + 1);
322 // All memory operands should have a register and an immediate operand (see
323 // RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand).
324 if (!AddrReg.isReg())
325 return true;
326 if (!Offset.isImm() && !Offset.isGlobal() && !Offset.isBlockAddress() &&
327 !Offset.isMCSymbol())
328 return true;
329
330 MCOperand MCO;
331 if (!lowerOperand(Offset, MCO))
332 return true;
333
334 if (Offset.isImm())
335 OS << MCO.getImm();
336 else if (Offset.isGlobal() || Offset.isBlockAddress() || Offset.isMCSymbol())
337 OS << *MCO.getExpr();
338 OS << "(" << RISCVInstPrinter::getRegisterName(AddrReg.getReg()) << ")";
339 return false;
340 }
341
emitDirectiveOptionArch()342 bool RISCVAsmPrinter::emitDirectiveOptionArch() {
343 RISCVTargetStreamer &RTS =
344 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
345 SmallVector<RISCVOptionArchArg> NeedEmitStdOptionArgs;
346 const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo();
347 for (const auto &Feature : RISCVFeatureKV) {
348 if (STI->hasFeature(Feature.Value) == MCSTI.hasFeature(Feature.Value))
349 continue;
350
351 if (!llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key))
352 continue;
353
354 auto Delta = STI->hasFeature(Feature.Value) ? RISCVOptionArchArgType::Plus
355 : RISCVOptionArchArgType::Minus;
356 NeedEmitStdOptionArgs.emplace_back(Delta, Feature.Key);
357 }
358 if (!NeedEmitStdOptionArgs.empty()) {
359 RTS.emitDirectiveOptionPush();
360 RTS.emitDirectiveOptionArch(NeedEmitStdOptionArgs);
361 return true;
362 }
363
364 return false;
365 }
366
runOnMachineFunction(MachineFunction & MF)367 bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
368 STI = &MF.getSubtarget<RISCVSubtarget>();
369 RISCVTargetStreamer &RTS =
370 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
371
372 bool EmittedOptionArch = emitDirectiveOptionArch();
373
374 SetupMachineFunction(MF);
375 emitFunctionBody();
376
377 if (EmittedOptionArch)
378 RTS.emitDirectiveOptionPop();
379 return false;
380 }
381
emitStartOfAsmFile(Module & M)382 void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) {
383 RISCVTargetStreamer &RTS =
384 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
385 if (const MDString *ModuleTargetABI =
386 dyn_cast_or_null<MDString>(M.getModuleFlag("target-abi")))
387 RTS.setTargetABI(RISCVABI::getTargetABI(ModuleTargetABI->getString()));
388
389 MCSubtargetInfo SubtargetInfo = *TM.getMCSubtargetInfo();
390
391 // Use module flag to update feature bits.
392 if (auto *MD = dyn_cast_or_null<MDNode>(M.getModuleFlag("riscv-isa"))) {
393 for (auto &ISA : MD->operands()) {
394 if (auto *ISAString = dyn_cast_or_null<MDString>(ISA)) {
395 auto ParseResult = llvm::RISCVISAInfo::parseArchString(
396 ISAString->getString(), /*EnableExperimentalExtension=*/true,
397 /*ExperimentalExtensionVersionCheck=*/true);
398 if (!errorToBool(ParseResult.takeError())) {
399 auto &ISAInfo = *ParseResult;
400 for (const auto &Feature : RISCVFeatureKV) {
401 if (ISAInfo->hasExtension(Feature.Key) &&
402 !SubtargetInfo.hasFeature(Feature.Value))
403 SubtargetInfo.ToggleFeature(Feature.Key);
404 }
405 }
406 }
407 }
408
409 RTS.setFlagsFromFeatures(SubtargetInfo);
410 }
411
412 if (TM.getTargetTriple().isOSBinFormatELF())
413 emitAttributes(SubtargetInfo);
414 }
415
emitEndOfAsmFile(Module & M)416 void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) {
417 RISCVTargetStreamer &RTS =
418 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
419
420 if (TM.getTargetTriple().isOSBinFormatELF())
421 RTS.finishAttributeSection();
422 EmitHwasanMemaccessSymbols(M);
423 }
424
emitAttributes(const MCSubtargetInfo & SubtargetInfo)425 void RISCVAsmPrinter::emitAttributes(const MCSubtargetInfo &SubtargetInfo) {
426 RISCVTargetStreamer &RTS =
427 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
428 // Use MCSubtargetInfo from TargetMachine. Individual functions may have
429 // attributes that differ from other functions in the module and we have no
430 // way to know which function is correct.
431 RTS.emitTargetAttributes(SubtargetInfo, /*EmitStackAlign*/ true);
432 }
433
emitFunctionEntryLabel()434 void RISCVAsmPrinter::emitFunctionEntryLabel() {
435 const auto *RMFI = MF->getInfo<RISCVMachineFunctionInfo>();
436 if (RMFI->isVectorCall()) {
437 auto &RTS =
438 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
439 RTS.emitDirectiveVariantCC(*CurrentFnSym);
440 }
441 return AsmPrinter::emitFunctionEntryLabel();
442 }
443
444 // Force static initialization.
LLVMInitializeRISCVAsmPrinter()445 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVAsmPrinter() {
446 RegisterAsmPrinter<RISCVAsmPrinter> X(getTheRISCV32Target());
447 RegisterAsmPrinter<RISCVAsmPrinter> Y(getTheRISCV64Target());
448 }
449
LowerHWASAN_CHECK_MEMACCESS(const MachineInstr & MI)450 void RISCVAsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
451 Register Reg = MI.getOperand(0).getReg();
452 uint32_t AccessInfo = MI.getOperand(1).getImm();
453 MCSymbol *&Sym =
454 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, AccessInfo)];
455 if (!Sym) {
456 // FIXME: Make this work on non-ELF.
457 if (!TM.getTargetTriple().isOSBinFormatELF())
458 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
459
460 std::string SymName = "__hwasan_check_x" + utostr(Reg - RISCV::X0) + "_" +
461 utostr(AccessInfo) + "_short";
462 Sym = OutContext.getOrCreateSymbol(SymName);
463 }
464 auto Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, OutContext);
465 auto Expr = RISCVMCExpr::create(Res, RISCVMCExpr::VK_RISCV_CALL, OutContext);
466
467 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr));
468 }
469
LowerKCFI_CHECK(const MachineInstr & MI)470 void RISCVAsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
471 Register AddrReg = MI.getOperand(0).getReg();
472 assert(std::next(MI.getIterator())->isCall() &&
473 "KCFI_CHECK not followed by a call instruction");
474 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
475 "KCFI_CHECK call target doesn't match call operand");
476
477 // Temporary registers for comparing the hashes. If a register is used
478 // for the call target, or reserved by the user, we can clobber another
479 // temporary register as the check is immediately followed by the
480 // call. The check defaults to X6/X7, but can fall back to X28-X31 if
481 // needed.
482 unsigned ScratchRegs[] = {RISCV::X6, RISCV::X7};
483 unsigned NextReg = RISCV::X28;
484 auto isRegAvailable = [&](unsigned Reg) {
485 return Reg != AddrReg && !STI->isRegisterReservedByUser(Reg);
486 };
487 for (auto &Reg : ScratchRegs) {
488 if (isRegAvailable(Reg))
489 continue;
490 while (!isRegAvailable(NextReg))
491 ++NextReg;
492 Reg = NextReg++;
493 if (Reg > RISCV::X31)
494 report_fatal_error("Unable to find scratch registers for KCFI_CHECK");
495 }
496
497 if (AddrReg == RISCV::X0) {
498 // Checking X0 makes no sense. Instead of emitting a load, zero
499 // ScratchRegs[0].
500 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI)
501 .addReg(ScratchRegs[0])
502 .addReg(RISCV::X0)
503 .addImm(0));
504 } else {
505 // Adjust the offset for patchable-function-prefix. This assumes that
506 // patchable-function-prefix is the same for all functions.
507 int NopSize = STI->hasStdExtCOrZca() ? 2 : 4;
508 int64_t PrefixNops = 0;
509 (void)MI.getMF()
510 ->getFunction()
511 .getFnAttribute("patchable-function-prefix")
512 .getValueAsString()
513 .getAsInteger(10, PrefixNops);
514
515 // Load the target function type hash.
516 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::LW)
517 .addReg(ScratchRegs[0])
518 .addReg(AddrReg)
519 .addImm(-(PrefixNops * NopSize + 4)));
520 }
521
522 // Load the expected 32-bit type hash.
523 const int64_t Type = MI.getOperand(1).getImm();
524 const int64_t Hi20 = ((Type + 0x800) >> 12) & 0xFFFFF;
525 const int64_t Lo12 = SignExtend64<12>(Type);
526 if (Hi20) {
527 EmitToStreamer(
528 *OutStreamer,
529 MCInstBuilder(RISCV::LUI).addReg(ScratchRegs[1]).addImm(Hi20));
530 }
531 if (Lo12 || Hi20 == 0) {
532 EmitToStreamer(*OutStreamer,
533 MCInstBuilder((STI->hasFeature(RISCV::Feature64Bit) && Hi20)
534 ? RISCV::ADDIW
535 : RISCV::ADDI)
536 .addReg(ScratchRegs[1])
537 .addReg(ScratchRegs[1])
538 .addImm(Lo12));
539 }
540
541 // Compare the hashes and trap if there's a mismatch.
542 MCSymbol *Pass = OutContext.createTempSymbol();
543 EmitToStreamer(*OutStreamer,
544 MCInstBuilder(RISCV::BEQ)
545 .addReg(ScratchRegs[0])
546 .addReg(ScratchRegs[1])
547 .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
548
549 MCSymbol *Trap = OutContext.createTempSymbol();
550 OutStreamer->emitLabel(Trap);
551 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::EBREAK));
552 emitKCFITrapEntry(*MI.getMF(), Trap);
553 OutStreamer->emitLabel(Pass);
554 }
555
EmitHwasanMemaccessSymbols(Module & M)556 void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
557 if (HwasanMemaccessSymbols.empty())
558 return;
559
560 assert(TM.getTargetTriple().isOSBinFormatELF());
561 // Use MCSubtargetInfo from TargetMachine. Individual functions may have
562 // attributes that differ from other functions in the module and we have no
563 // way to know which function is correct.
564 const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo();
565
566 MCSymbol *HwasanTagMismatchV2Sym =
567 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
568 // Annotate symbol as one having incompatible calling convention, so
569 // run-time linkers can instead eagerly bind this function.
570 auto &RTS =
571 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
572 RTS.emitDirectiveVariantCC(*HwasanTagMismatchV2Sym);
573
574 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
575 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
576 auto Expr = RISCVMCExpr::create(HwasanTagMismatchV2Ref,
577 RISCVMCExpr::VK_RISCV_CALL, OutContext);
578
579 for (auto &P : HwasanMemaccessSymbols) {
580 unsigned Reg = std::get<0>(P.first);
581 uint32_t AccessInfo = std::get<1>(P.first);
582 MCSymbol *Sym = P.second;
583
584 unsigned Size =
585 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
586 OutStreamer->switchSection(OutContext.getELFSection(
587 ".text.hot", ELF::SHT_PROGBITS,
588 ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(),
589 /*IsComdat=*/true));
590
591 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
592 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
593 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
594 OutStreamer->emitLabel(Sym);
595
596 // Extract shadow offset from ptr
597 OutStreamer->emitInstruction(
598 MCInstBuilder(RISCV::SLLI).addReg(RISCV::X6).addReg(Reg).addImm(8),
599 MCSTI);
600 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SRLI)
601 .addReg(RISCV::X6)
602 .addReg(RISCV::X6)
603 .addImm(12),
604 MCSTI);
605 // load shadow tag in X6, X5 contains shadow base
606 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADD)
607 .addReg(RISCV::X6)
608 .addReg(RISCV::X5)
609 .addReg(RISCV::X6),
610 MCSTI);
611 OutStreamer->emitInstruction(
612 MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0),
613 MCSTI);
614 // Extract tag from X5 and compare it with loaded tag from shadow
615 OutStreamer->emitInstruction(
616 MCInstBuilder(RISCV::SRLI).addReg(RISCV::X7).addReg(Reg).addImm(56),
617 MCSTI);
618 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
619 // X7 contains tag from memory, while X6 contains tag from the pointer
620 OutStreamer->emitInstruction(
621 MCInstBuilder(RISCV::BNE)
622 .addReg(RISCV::X7)
623 .addReg(RISCV::X6)
624 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
625 OutContext)),
626 MCSTI);
627 MCSymbol *ReturnSym = OutContext.createTempSymbol();
628 OutStreamer->emitLabel(ReturnSym);
629 OutStreamer->emitInstruction(MCInstBuilder(RISCV::JALR)
630 .addReg(RISCV::X0)
631 .addReg(RISCV::X1)
632 .addImm(0),
633 MCSTI);
634 OutStreamer->emitLabel(HandleMismatchOrPartialSym);
635
636 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
637 .addReg(RISCV::X28)
638 .addReg(RISCV::X0)
639 .addImm(16),
640 MCSTI);
641 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
642 OutStreamer->emitInstruction(
643 MCInstBuilder(RISCV::BGEU)
644 .addReg(RISCV::X6)
645 .addReg(RISCV::X28)
646 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
647 MCSTI);
648
649 OutStreamer->emitInstruction(
650 MCInstBuilder(RISCV::ANDI).addReg(RISCV::X28).addReg(Reg).addImm(0xF),
651 MCSTI);
652
653 if (Size != 1)
654 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
655 .addReg(RISCV::X28)
656 .addReg(RISCV::X28)
657 .addImm(Size - 1),
658 MCSTI);
659 OutStreamer->emitInstruction(
660 MCInstBuilder(RISCV::BGE)
661 .addReg(RISCV::X28)
662 .addReg(RISCV::X6)
663 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
664 MCSTI);
665
666 OutStreamer->emitInstruction(
667 MCInstBuilder(RISCV::ORI).addReg(RISCV::X6).addReg(Reg).addImm(0xF),
668 MCSTI);
669 OutStreamer->emitInstruction(
670 MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0),
671 MCSTI);
672 OutStreamer->emitInstruction(
673 MCInstBuilder(RISCV::BEQ)
674 .addReg(RISCV::X6)
675 .addReg(RISCV::X7)
676 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
677 MCSTI);
678
679 OutStreamer->emitLabel(HandleMismatchSym);
680
681 // | Previous stack frames... |
682 // +=================================+ <-- [SP + 256]
683 // | ... |
684 // | |
685 // | Stack frame space for x12 - x31.|
686 // | |
687 // | ... |
688 // +---------------------------------+ <-- [SP + 96]
689 // | Saved x11(arg1), as |
690 // | __hwasan_check_* clobbers it. |
691 // +---------------------------------+ <-- [SP + 88]
692 // | Saved x10(arg0), as |
693 // | __hwasan_check_* clobbers it. |
694 // +---------------------------------+ <-- [SP + 80]
695 // | |
696 // | Stack frame space for x9. |
697 // +---------------------------------+ <-- [SP + 72]
698 // | |
699 // | Saved x8(fp), as |
700 // | __hwasan_check_* clobbers it. |
701 // +---------------------------------+ <-- [SP + 64]
702 // | ... |
703 // | |
704 // | Stack frame space for x2 - x7. |
705 // | |
706 // | ... |
707 // +---------------------------------+ <-- [SP + 16]
708 // | Return address (x1) for caller |
709 // | of __hwasan_check_*. |
710 // +---------------------------------+ <-- [SP + 8]
711 // | Reserved place for x0, possibly |
712 // | junk, since we don't save it. |
713 // +---------------------------------+ <-- [x2 / SP]
714
715 // Adjust sp
716 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
717 .addReg(RISCV::X2)
718 .addReg(RISCV::X2)
719 .addImm(-256),
720 MCSTI);
721
722 // store x10(arg0) by new sp
723 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD)
724 .addReg(RISCV::X10)
725 .addReg(RISCV::X2)
726 .addImm(8 * 10),
727 MCSTI);
728 // store x11(arg1) by new sp
729 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD)
730 .addReg(RISCV::X11)
731 .addReg(RISCV::X2)
732 .addImm(8 * 11),
733 MCSTI);
734
735 // store x8(fp) by new sp
736 OutStreamer->emitInstruction(
737 MCInstBuilder(RISCV::SD).addReg(RISCV::X8).addReg(RISCV::X2).addImm(8 *
738 8),
739 MCSTI);
740 // store x1(ra) by new sp
741 OutStreamer->emitInstruction(
742 MCInstBuilder(RISCV::SD).addReg(RISCV::X1).addReg(RISCV::X2).addImm(1 *
743 8),
744 MCSTI);
745 if (Reg != RISCV::X10)
746 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
747 .addReg(RISCV::X10)
748 .addReg(Reg)
749 .addImm(0),
750 MCSTI);
751 OutStreamer->emitInstruction(
752 MCInstBuilder(RISCV::ADDI)
753 .addReg(RISCV::X11)
754 .addReg(RISCV::X0)
755 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask),
756 MCSTI);
757
758 OutStreamer->emitInstruction(MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr),
759 MCSTI);
760 }
761 }
762
lowerSymbolOperand(const MachineOperand & MO,MCSymbol * Sym,const AsmPrinter & AP)763 static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym,
764 const AsmPrinter &AP) {
765 MCContext &Ctx = AP.OutContext;
766 RISCVMCExpr::VariantKind Kind;
767
768 switch (MO.getTargetFlags()) {
769 default:
770 llvm_unreachable("Unknown target flag on GV operand");
771 case RISCVII::MO_None:
772 Kind = RISCVMCExpr::VK_RISCV_None;
773 break;
774 case RISCVII::MO_CALL:
775 Kind = RISCVMCExpr::VK_RISCV_CALL_PLT;
776 break;
777 case RISCVII::MO_LO:
778 Kind = RISCVMCExpr::VK_RISCV_LO;
779 break;
780 case RISCVII::MO_HI:
781 Kind = RISCVMCExpr::VK_RISCV_HI;
782 break;
783 case RISCVII::MO_PCREL_LO:
784 Kind = RISCVMCExpr::VK_RISCV_PCREL_LO;
785 break;
786 case RISCVII::MO_PCREL_HI:
787 Kind = RISCVMCExpr::VK_RISCV_PCREL_HI;
788 break;
789 case RISCVII::MO_GOT_HI:
790 Kind = RISCVMCExpr::VK_RISCV_GOT_HI;
791 break;
792 case RISCVII::MO_TPREL_LO:
793 Kind = RISCVMCExpr::VK_RISCV_TPREL_LO;
794 break;
795 case RISCVII::MO_TPREL_HI:
796 Kind = RISCVMCExpr::VK_RISCV_TPREL_HI;
797 break;
798 case RISCVII::MO_TPREL_ADD:
799 Kind = RISCVMCExpr::VK_RISCV_TPREL_ADD;
800 break;
801 case RISCVII::MO_TLS_GOT_HI:
802 Kind = RISCVMCExpr::VK_RISCV_TLS_GOT_HI;
803 break;
804 case RISCVII::MO_TLS_GD_HI:
805 Kind = RISCVMCExpr::VK_RISCV_TLS_GD_HI;
806 break;
807 case RISCVII::MO_TLSDESC_HI:
808 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_HI;
809 break;
810 case RISCVII::MO_TLSDESC_LOAD_LO:
811 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_LOAD_LO;
812 break;
813 case RISCVII::MO_TLSDESC_ADD_LO:
814 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_ADD_LO;
815 break;
816 case RISCVII::MO_TLSDESC_CALL:
817 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_CALL;
818 break;
819 }
820
821 const MCExpr *ME =
822 MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Ctx);
823
824 if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
825 ME = MCBinaryExpr::createAdd(
826 ME, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
827
828 if (Kind != RISCVMCExpr::VK_RISCV_None)
829 ME = RISCVMCExpr::create(ME, Kind, Ctx);
830 return MCOperand::createExpr(ME);
831 }
832
lowerOperand(const MachineOperand & MO,MCOperand & MCOp) const833 bool RISCVAsmPrinter::lowerOperand(const MachineOperand &MO,
834 MCOperand &MCOp) const {
835 switch (MO.getType()) {
836 default:
837 report_fatal_error("lowerOperand: unknown operand type");
838 case MachineOperand::MO_Register:
839 // Ignore all implicit register operands.
840 if (MO.isImplicit())
841 return false;
842 MCOp = MCOperand::createReg(MO.getReg());
843 break;
844 case MachineOperand::MO_RegisterMask:
845 // Regmasks are like implicit defs.
846 return false;
847 case MachineOperand::MO_Immediate:
848 MCOp = MCOperand::createImm(MO.getImm());
849 break;
850 case MachineOperand::MO_MachineBasicBlock:
851 MCOp = lowerSymbolOperand(MO, MO.getMBB()->getSymbol(), *this);
852 break;
853 case MachineOperand::MO_GlobalAddress:
854 MCOp = lowerSymbolOperand(MO, getSymbolPreferLocal(*MO.getGlobal()), *this);
855 break;
856 case MachineOperand::MO_BlockAddress:
857 MCOp = lowerSymbolOperand(MO, GetBlockAddressSymbol(MO.getBlockAddress()),
858 *this);
859 break;
860 case MachineOperand::MO_ExternalSymbol:
861 MCOp = lowerSymbolOperand(MO, GetExternalSymbolSymbol(MO.getSymbolName()),
862 *this);
863 break;
864 case MachineOperand::MO_ConstantPoolIndex:
865 MCOp = lowerSymbolOperand(MO, GetCPISymbol(MO.getIndex()), *this);
866 break;
867 case MachineOperand::MO_JumpTableIndex:
868 MCOp = lowerSymbolOperand(MO, GetJTISymbol(MO.getIndex()), *this);
869 break;
870 case MachineOperand::MO_MCSymbol:
871 MCOp = lowerSymbolOperand(MO, MO.getMCSymbol(), *this);
872 break;
873 }
874 return true;
875 }
876
lowerRISCVVMachineInstrToMCInst(const MachineInstr * MI,MCInst & OutMI)877 static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
878 MCInst &OutMI) {
879 const RISCVVPseudosTable::PseudoInfo *RVV =
880 RISCVVPseudosTable::getPseudoInfo(MI->getOpcode());
881 if (!RVV)
882 return false;
883
884 OutMI.setOpcode(RVV->BaseInstr);
885
886 const MachineBasicBlock *MBB = MI->getParent();
887 assert(MBB && "MI expected to be in a basic block");
888 const MachineFunction *MF = MBB->getParent();
889 assert(MF && "MBB expected to be in a machine function");
890
891 const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>();
892 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
893 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
894 assert(TRI && "TargetRegisterInfo expected");
895
896 const MCInstrDesc &MCID = MI->getDesc();
897 uint64_t TSFlags = MCID.TSFlags;
898 unsigned NumOps = MI->getNumExplicitOperands();
899
900 // Skip policy, SEW, VL, VXRM/FRM operands which are the last operands if
901 // present.
902 if (RISCVII::hasVecPolicyOp(TSFlags))
903 --NumOps;
904 if (RISCVII::hasSEWOp(TSFlags))
905 --NumOps;
906 if (RISCVII::hasVLOp(TSFlags))
907 --NumOps;
908 if (RISCVII::hasRoundModeOp(TSFlags))
909 --NumOps;
910
911 bool hasVLOutput = RISCV::isFaultFirstLoad(*MI);
912 for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) {
913 const MachineOperand &MO = MI->getOperand(OpNo);
914 // Skip vl ouput. It should be the second output.
915 if (hasVLOutput && OpNo == 1)
916 continue;
917
918 // Skip merge op. It should be the first operand after the defs.
919 if (OpNo == MI->getNumExplicitDefs() && MO.isReg() && MO.isTied()) {
920 assert(MCID.getOperandConstraint(OpNo, MCOI::TIED_TO) == 0 &&
921 "Expected tied to first def.");
922 const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode());
923 // Skip if the next operand in OutMI is not supposed to be tied. Unless it
924 // is a _TIED instruction.
925 if (OutMCID.getOperandConstraint(OutMI.getNumOperands(), MCOI::TIED_TO) <
926 0 &&
927 !RISCVII::isTiedPseudo(TSFlags))
928 continue;
929 }
930
931 MCOperand MCOp;
932 switch (MO.getType()) {
933 default:
934 llvm_unreachable("Unknown operand type");
935 case MachineOperand::MO_Register: {
936 Register Reg = MO.getReg();
937
938 if (RISCV::VRM2RegClass.contains(Reg) ||
939 RISCV::VRM4RegClass.contains(Reg) ||
940 RISCV::VRM8RegClass.contains(Reg)) {
941 Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0);
942 assert(Reg && "Subregister does not exist");
943 } else if (RISCV::FPR16RegClass.contains(Reg)) {
944 Reg =
945 TRI->getMatchingSuperReg(Reg, RISCV::sub_16, &RISCV::FPR32RegClass);
946 assert(Reg && "Subregister does not exist");
947 } else if (RISCV::FPR64RegClass.contains(Reg)) {
948 Reg = TRI->getSubReg(Reg, RISCV::sub_32);
949 assert(Reg && "Superregister does not exist");
950 } else if (RISCV::VRN2M1RegClass.contains(Reg) ||
951 RISCV::VRN2M2RegClass.contains(Reg) ||
952 RISCV::VRN2M4RegClass.contains(Reg) ||
953 RISCV::VRN3M1RegClass.contains(Reg) ||
954 RISCV::VRN3M2RegClass.contains(Reg) ||
955 RISCV::VRN4M1RegClass.contains(Reg) ||
956 RISCV::VRN4M2RegClass.contains(Reg) ||
957 RISCV::VRN5M1RegClass.contains(Reg) ||
958 RISCV::VRN6M1RegClass.contains(Reg) ||
959 RISCV::VRN7M1RegClass.contains(Reg) ||
960 RISCV::VRN8M1RegClass.contains(Reg)) {
961 Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0);
962 assert(Reg && "Subregister does not exist");
963 }
964
965 MCOp = MCOperand::createReg(Reg);
966 break;
967 }
968 case MachineOperand::MO_Immediate:
969 MCOp = MCOperand::createImm(MO.getImm());
970 break;
971 }
972 OutMI.addOperand(MCOp);
973 }
974
975 // Unmasked pseudo instructions need to append dummy mask operand to
976 // V instructions. All V instructions are modeled as the masked version.
977 const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode());
978 if (OutMI.getNumOperands() < OutMCID.getNumOperands()) {
979 assert(OutMCID.operands()[OutMI.getNumOperands()].RegClass ==
980 RISCV::VMV0RegClassID &&
981 "Expected only mask operand to be missing");
982 OutMI.addOperand(MCOperand::createReg(RISCV::NoRegister));
983 }
984
985 assert(OutMI.getNumOperands() == OutMCID.getNumOperands());
986 return true;
987 }
988
lowerToMCInst(const MachineInstr * MI,MCInst & OutMI)989 bool RISCVAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
990 if (lowerRISCVVMachineInstrToMCInst(MI, OutMI))
991 return false;
992
993 OutMI.setOpcode(MI->getOpcode());
994
995 for (const MachineOperand &MO : MI->operands()) {
996 MCOperand MCOp;
997 if (lowerOperand(MO, MCOp))
998 OutMI.addOperand(MCOp);
999 }
1000
1001 switch (OutMI.getOpcode()) {
1002 case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
1003 const Function &F = MI->getParent()->getParent()->getFunction();
1004 if (F.hasFnAttribute("patchable-function-entry")) {
1005 unsigned Num;
1006 if (F.getFnAttribute("patchable-function-entry")
1007 .getValueAsString()
1008 .getAsInteger(10, Num))
1009 return false;
1010 emitNops(Num);
1011 return true;
1012 }
1013 break;
1014 }
1015 }
1016 return false;
1017 }
1018