1 //===-- RISCVAsmPrinter.cpp - RISC-V LLVM assembly writer -----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains a printer that converts from our internal representation 10 // of machine-dependent LLVM code to the RISC-V assembly language. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "MCTargetDesc/RISCVBaseInfo.h" 15 #include "MCTargetDesc/RISCVInstPrinter.h" 16 #include "MCTargetDesc/RISCVMCExpr.h" 17 #include "MCTargetDesc/RISCVTargetStreamer.h" 18 #include "RISCV.h" 19 #include "RISCVMachineFunctionInfo.h" 20 #include "RISCVTargetMachine.h" 21 #include "TargetInfo/RISCVTargetInfo.h" 22 #include "llvm/ADT/APInt.h" 23 #include "llvm/ADT/Statistic.h" 24 #include "llvm/BinaryFormat/ELF.h" 25 #include "llvm/CodeGen/AsmPrinter.h" 26 #include "llvm/CodeGen/MachineConstantPool.h" 27 #include "llvm/CodeGen/MachineFunctionPass.h" 28 #include "llvm/CodeGen/MachineInstr.h" 29 #include "llvm/CodeGen/MachineModuleInfo.h" 30 #include "llvm/MC/MCAsmInfo.h" 31 #include "llvm/MC/MCContext.h" 32 #include "llvm/MC/MCInst.h" 33 #include "llvm/MC/MCInstBuilder.h" 34 #include "llvm/MC/MCObjectFileInfo.h" 35 #include "llvm/MC/MCSectionELF.h" 36 #include "llvm/MC/MCStreamer.h" 37 #include "llvm/MC/MCSymbol.h" 38 #include "llvm/MC/TargetRegistry.h" 39 #include "llvm/Support/RISCVISAInfo.h" 40 #include "llvm/Support/raw_ostream.h" 41 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" 42 43 using namespace llvm; 44 45 #define DEBUG_TYPE "asm-printer" 46 47 STATISTIC(RISCVNumInstrsCompressed, 48 "Number of RISC-V Compressed instructions emitted"); 49 50 namespace llvm { 51 extern const SubtargetFeatureKV RISCVFeatureKV[RISCV::NumSubtargetFeatures]; 52 } // namespace llvm 53 54 namespace { 55 class RISCVAsmPrinter : public AsmPrinter { 56 const RISCVSubtarget *STI; 57 58 public: 59 explicit RISCVAsmPrinter(TargetMachine &TM, 60 std::unique_ptr<MCStreamer> Streamer) 61 : AsmPrinter(TM, std::move(Streamer)) {} 62 63 StringRef getPassName() const override { return "RISC-V Assembly Printer"; } 64 65 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 66 const MachineInstr &MI); 67 68 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 69 const MachineInstr &MI); 70 71 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 72 const MachineInstr &MI); 73 74 bool runOnMachineFunction(MachineFunction &MF) override; 75 76 void emitInstruction(const MachineInstr *MI) override; 77 78 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, 79 const char *ExtraCode, raw_ostream &OS) override; 80 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, 81 const char *ExtraCode, raw_ostream &OS) override; 82 83 void EmitToStreamer(MCStreamer &S, const MCInst &Inst); 84 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, 85 const MachineInstr *MI); 86 87 typedef std::tuple<unsigned, uint32_t> HwasanMemaccessTuple; 88 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols; 89 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); 90 void LowerKCFI_CHECK(const MachineInstr &MI); 91 void EmitHwasanMemaccessSymbols(Module &M); 92 93 // Wrapper needed for tblgenned pseudo lowering. 94 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const; 95 96 void emitStartOfAsmFile(Module &M) override; 97 void emitEndOfAsmFile(Module &M) override; 98 99 void emitFunctionEntryLabel() override; 100 bool emitDirectiveOptionArch(); 101 102 private: 103 void emitAttributes(); 104 105 void emitNTLHint(const MachineInstr *MI); 106 107 bool lowerToMCInst(const MachineInstr *MI, MCInst &OutMI); 108 }; 109 } 110 111 void RISCVAsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 112 const MachineInstr &MI) { 113 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4; 114 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); 115 116 auto &Ctx = OutStreamer.getContext(); 117 MCSymbol *MILabel = Ctx.createTempSymbol(); 118 OutStreamer.emitLabel(MILabel); 119 120 SM.recordStackMap(*MILabel, MI); 121 assert(NumNOPBytes % NOPBytes == 0 && 122 "Invalid number of NOP bytes requested!"); 123 124 // Scan ahead to trim the shadow. 125 const MachineBasicBlock &MBB = *MI.getParent(); 126 MachineBasicBlock::const_iterator MII(MI); 127 ++MII; 128 while (NumNOPBytes > 0) { 129 if (MII == MBB.end() || MII->isCall() || 130 MII->getOpcode() == RISCV::DBG_VALUE || 131 MII->getOpcode() == TargetOpcode::PATCHPOINT || 132 MII->getOpcode() == TargetOpcode::STACKMAP) 133 break; 134 ++MII; 135 NumNOPBytes -= 4; 136 } 137 138 // Emit nops. 139 emitNops(NumNOPBytes / NOPBytes); 140 } 141 142 // Lower a patchpoint of the form: 143 // [<def>], <id>, <numBytes>, <target>, <numArgs> 144 void RISCVAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 145 const MachineInstr &MI) { 146 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4; 147 148 auto &Ctx = OutStreamer.getContext(); 149 MCSymbol *MILabel = Ctx.createTempSymbol(); 150 OutStreamer.emitLabel(MILabel); 151 SM.recordPatchPoint(*MILabel, MI); 152 153 PatchPointOpers Opers(&MI); 154 155 unsigned EncodedBytes = 0; 156 157 // Emit padding. 158 unsigned NumBytes = Opers.getNumPatchBytes(); 159 assert(NumBytes >= EncodedBytes && 160 "Patchpoint can't request size less than the length of a call."); 161 assert((NumBytes - EncodedBytes) % NOPBytes == 0 && 162 "Invalid number of NOP bytes requested!"); 163 emitNops((NumBytes - EncodedBytes) / NOPBytes); 164 } 165 166 void RISCVAsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 167 const MachineInstr &MI) { 168 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4; 169 170 StatepointOpers SOpers(&MI); 171 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) { 172 assert(PatchBytes % NOPBytes == 0 && 173 "Invalid number of NOP bytes requested!"); 174 emitNops(PatchBytes / NOPBytes); 175 } 176 177 auto &Ctx = OutStreamer.getContext(); 178 MCSymbol *MILabel = Ctx.createTempSymbol(); 179 OutStreamer.emitLabel(MILabel); 180 SM.recordStatepoint(*MILabel, MI); 181 } 182 183 void RISCVAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) { 184 MCInst CInst; 185 bool Res = RISCVRVC::compress(CInst, Inst, *STI); 186 if (Res) 187 ++RISCVNumInstrsCompressed; 188 AsmPrinter::EmitToStreamer(*OutStreamer, Res ? CInst : Inst); 189 } 190 191 // Simple pseudo-instructions have their lowering (with expansion to real 192 // instructions) auto-generated. 193 #include "RISCVGenMCPseudoLowering.inc" 194 195 // If the target supports Zihintntl and the instruction has a nontemporal 196 // MachineMemOperand, emit an NTLH hint instruction before it. 197 void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) { 198 if (!STI->hasStdExtZihintntl()) 199 return; 200 201 if (MI->memoperands_empty()) 202 return; 203 204 MachineMemOperand *MMO = *(MI->memoperands_begin()); 205 if (!MMO->isNonTemporal()) 206 return; 207 208 unsigned NontemporalMode = 0; 209 if (MMO->getFlags() & MONontemporalBit0) 210 NontemporalMode += 0b1; 211 if (MMO->getFlags() & MONontemporalBit1) 212 NontemporalMode += 0b10; 213 214 MCInst Hint; 215 if (STI->hasStdExtCOrZca() && STI->enableRVCHintInstrs()) 216 Hint.setOpcode(RISCV::C_ADD_HINT); 217 else 218 Hint.setOpcode(RISCV::ADD); 219 220 Hint.addOperand(MCOperand::createReg(RISCV::X0)); 221 Hint.addOperand(MCOperand::createReg(RISCV::X0)); 222 Hint.addOperand(MCOperand::createReg(RISCV::X2 + NontemporalMode)); 223 224 EmitToStreamer(*OutStreamer, Hint); 225 } 226 227 void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) { 228 RISCV_MC::verifyInstructionPredicates(MI->getOpcode(), 229 getSubtargetInfo().getFeatureBits()); 230 231 emitNTLHint(MI); 232 233 // Do any auto-generated pseudo lowerings. 234 if (emitPseudoExpansionLowering(*OutStreamer, MI)) 235 return; 236 237 238 switch (MI->getOpcode()) { 239 case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES: 240 LowerHWASAN_CHECK_MEMACCESS(*MI); 241 return; 242 case RISCV::KCFI_CHECK: 243 LowerKCFI_CHECK(*MI); 244 return; 245 case RISCV::PseudoRVVInitUndefM1: 246 case RISCV::PseudoRVVInitUndefM2: 247 case RISCV::PseudoRVVInitUndefM4: 248 case RISCV::PseudoRVVInitUndefM8: 249 return; 250 case TargetOpcode::STACKMAP: 251 return LowerSTACKMAP(*OutStreamer, SM, *MI); 252 case TargetOpcode::PATCHPOINT: 253 return LowerPATCHPOINT(*OutStreamer, SM, *MI); 254 case TargetOpcode::STATEPOINT: 255 return LowerSTATEPOINT(*OutStreamer, SM, *MI); 256 } 257 258 MCInst OutInst; 259 if (!lowerToMCInst(MI, OutInst)) 260 EmitToStreamer(*OutStreamer, OutInst); 261 } 262 263 bool RISCVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, 264 const char *ExtraCode, raw_ostream &OS) { 265 // First try the generic code, which knows about modifiers like 'c' and 'n'. 266 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS)) 267 return false; 268 269 const MachineOperand &MO = MI->getOperand(OpNo); 270 if (ExtraCode && ExtraCode[0]) { 271 if (ExtraCode[1] != 0) 272 return true; // Unknown modifier. 273 274 switch (ExtraCode[0]) { 275 default: 276 return true; // Unknown modifier. 277 case 'z': // Print zero register if zero, regular printing otherwise. 278 if (MO.isImm() && MO.getImm() == 0) { 279 OS << RISCVInstPrinter::getRegisterName(RISCV::X0); 280 return false; 281 } 282 break; 283 case 'i': // Literal 'i' if operand is not a register. 284 if (!MO.isReg()) 285 OS << 'i'; 286 return false; 287 } 288 } 289 290 switch (MO.getType()) { 291 case MachineOperand::MO_Immediate: 292 OS << MO.getImm(); 293 return false; 294 case MachineOperand::MO_Register: 295 OS << RISCVInstPrinter::getRegisterName(MO.getReg()); 296 return false; 297 case MachineOperand::MO_GlobalAddress: 298 PrintSymbolOperand(MO, OS); 299 return false; 300 case MachineOperand::MO_BlockAddress: { 301 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress()); 302 Sym->print(OS, MAI); 303 return false; 304 } 305 default: 306 break; 307 } 308 309 return true; 310 } 311 312 bool RISCVAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, 313 unsigned OpNo, 314 const char *ExtraCode, 315 raw_ostream &OS) { 316 if (ExtraCode) 317 return AsmPrinter::PrintAsmMemoryOperand(MI, OpNo, ExtraCode, OS); 318 319 const MachineOperand &AddrReg = MI->getOperand(OpNo); 320 assert(MI->getNumOperands() > OpNo + 1 && "Expected additional operand"); 321 const MachineOperand &Offset = MI->getOperand(OpNo + 1); 322 // All memory operands should have a register and an immediate operand (see 323 // RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand). 324 if (!AddrReg.isReg()) 325 return true; 326 if (!Offset.isImm() && !Offset.isGlobal() && !Offset.isBlockAddress() && 327 !Offset.isMCSymbol()) 328 return true; 329 330 MCOperand MCO; 331 if (!lowerOperand(Offset, MCO)) 332 return true; 333 334 if (Offset.isImm()) 335 OS << MCO.getImm(); 336 else if (Offset.isGlobal() || Offset.isBlockAddress() || Offset.isMCSymbol()) 337 OS << *MCO.getExpr(); 338 OS << "(" << RISCVInstPrinter::getRegisterName(AddrReg.getReg()) << ")"; 339 return false; 340 } 341 342 bool RISCVAsmPrinter::emitDirectiveOptionArch() { 343 RISCVTargetStreamer &RTS = 344 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 345 SmallVector<RISCVOptionArchArg> NeedEmitStdOptionArgs; 346 const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo(); 347 for (const auto &Feature : RISCVFeatureKV) { 348 if (STI->hasFeature(Feature.Value) == MCSTI.hasFeature(Feature.Value)) 349 continue; 350 351 if (!llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key)) 352 continue; 353 354 auto Delta = STI->hasFeature(Feature.Value) ? RISCVOptionArchArgType::Plus 355 : RISCVOptionArchArgType::Minus; 356 NeedEmitStdOptionArgs.emplace_back(Delta, Feature.Key); 357 } 358 if (!NeedEmitStdOptionArgs.empty()) { 359 RTS.emitDirectiveOptionPush(); 360 RTS.emitDirectiveOptionArch(NeedEmitStdOptionArgs); 361 return true; 362 } 363 364 return false; 365 } 366 367 bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) { 368 STI = &MF.getSubtarget<RISCVSubtarget>(); 369 RISCVTargetStreamer &RTS = 370 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 371 372 bool EmittedOptionArch = emitDirectiveOptionArch(); 373 374 SetupMachineFunction(MF); 375 emitFunctionBody(); 376 377 if (EmittedOptionArch) 378 RTS.emitDirectiveOptionPop(); 379 return false; 380 } 381 382 void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) { 383 RISCVTargetStreamer &RTS = 384 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 385 if (const MDString *ModuleTargetABI = 386 dyn_cast_or_null<MDString>(M.getModuleFlag("target-abi"))) 387 RTS.setTargetABI(RISCVABI::getTargetABI(ModuleTargetABI->getString())); 388 if (TM.getTargetTriple().isOSBinFormatELF()) 389 emitAttributes(); 390 } 391 392 void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) { 393 RISCVTargetStreamer &RTS = 394 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 395 396 if (TM.getTargetTriple().isOSBinFormatELF()) 397 RTS.finishAttributeSection(); 398 EmitHwasanMemaccessSymbols(M); 399 } 400 401 void RISCVAsmPrinter::emitAttributes() { 402 RISCVTargetStreamer &RTS = 403 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 404 // Use MCSubtargetInfo from TargetMachine. Individual functions may have 405 // attributes that differ from other functions in the module and we have no 406 // way to know which function is correct. 407 RTS.emitTargetAttributes(*TM.getMCSubtargetInfo(), /*EmitStackAlign*/ true); 408 } 409 410 void RISCVAsmPrinter::emitFunctionEntryLabel() { 411 const auto *RMFI = MF->getInfo<RISCVMachineFunctionInfo>(); 412 if (RMFI->isVectorCall()) { 413 auto &RTS = 414 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 415 RTS.emitDirectiveVariantCC(*CurrentFnSym); 416 } 417 return AsmPrinter::emitFunctionEntryLabel(); 418 } 419 420 // Force static initialization. 421 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVAsmPrinter() { 422 RegisterAsmPrinter<RISCVAsmPrinter> X(getTheRISCV32Target()); 423 RegisterAsmPrinter<RISCVAsmPrinter> Y(getTheRISCV64Target()); 424 } 425 426 void RISCVAsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { 427 Register Reg = MI.getOperand(0).getReg(); 428 uint32_t AccessInfo = MI.getOperand(1).getImm(); 429 MCSymbol *&Sym = 430 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, AccessInfo)]; 431 if (!Sym) { 432 // FIXME: Make this work on non-ELF. 433 if (!TM.getTargetTriple().isOSBinFormatELF()) 434 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF"); 435 436 std::string SymName = "__hwasan_check_x" + utostr(Reg - RISCV::X0) + "_" + 437 utostr(AccessInfo) + "_short"; 438 Sym = OutContext.getOrCreateSymbol(SymName); 439 } 440 auto Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, OutContext); 441 auto Expr = RISCVMCExpr::create(Res, RISCVMCExpr::VK_RISCV_CALL, OutContext); 442 443 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr)); 444 } 445 446 void RISCVAsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { 447 Register AddrReg = MI.getOperand(0).getReg(); 448 assert(std::next(MI.getIterator())->isCall() && 449 "KCFI_CHECK not followed by a call instruction"); 450 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg && 451 "KCFI_CHECK call target doesn't match call operand"); 452 453 // Temporary registers for comparing the hashes. If a register is used 454 // for the call target, or reserved by the user, we can clobber another 455 // temporary register as the check is immediately followed by the 456 // call. The check defaults to X6/X7, but can fall back to X28-X31 if 457 // needed. 458 unsigned ScratchRegs[] = {RISCV::X6, RISCV::X7}; 459 unsigned NextReg = RISCV::X28; 460 auto isRegAvailable = [&](unsigned Reg) { 461 return Reg != AddrReg && !STI->isRegisterReservedByUser(Reg); 462 }; 463 for (auto &Reg : ScratchRegs) { 464 if (isRegAvailable(Reg)) 465 continue; 466 while (!isRegAvailable(NextReg)) 467 ++NextReg; 468 Reg = NextReg++; 469 if (Reg > RISCV::X31) 470 report_fatal_error("Unable to find scratch registers for KCFI_CHECK"); 471 } 472 473 if (AddrReg == RISCV::X0) { 474 // Checking X0 makes no sense. Instead of emitting a load, zero 475 // ScratchRegs[0]. 476 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI) 477 .addReg(ScratchRegs[0]) 478 .addReg(RISCV::X0) 479 .addImm(0)); 480 } else { 481 // Adjust the offset for patchable-function-prefix. This assumes that 482 // patchable-function-prefix is the same for all functions. 483 int NopSize = STI->hasStdExtCOrZca() ? 2 : 4; 484 int64_t PrefixNops = 0; 485 (void)MI.getMF() 486 ->getFunction() 487 .getFnAttribute("patchable-function-prefix") 488 .getValueAsString() 489 .getAsInteger(10, PrefixNops); 490 491 // Load the target function type hash. 492 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::LW) 493 .addReg(ScratchRegs[0]) 494 .addReg(AddrReg) 495 .addImm(-(PrefixNops * NopSize + 4))); 496 } 497 498 // Load the expected 32-bit type hash. 499 const int64_t Type = MI.getOperand(1).getImm(); 500 const int64_t Hi20 = ((Type + 0x800) >> 12) & 0xFFFFF; 501 const int64_t Lo12 = SignExtend64<12>(Type); 502 if (Hi20) { 503 EmitToStreamer( 504 *OutStreamer, 505 MCInstBuilder(RISCV::LUI).addReg(ScratchRegs[1]).addImm(Hi20)); 506 } 507 if (Lo12 || Hi20 == 0) { 508 EmitToStreamer(*OutStreamer, 509 MCInstBuilder((STI->hasFeature(RISCV::Feature64Bit) && Hi20) 510 ? RISCV::ADDIW 511 : RISCV::ADDI) 512 .addReg(ScratchRegs[1]) 513 .addReg(ScratchRegs[1]) 514 .addImm(Lo12)); 515 } 516 517 // Compare the hashes and trap if there's a mismatch. 518 MCSymbol *Pass = OutContext.createTempSymbol(); 519 EmitToStreamer(*OutStreamer, 520 MCInstBuilder(RISCV::BEQ) 521 .addReg(ScratchRegs[0]) 522 .addReg(ScratchRegs[1]) 523 .addExpr(MCSymbolRefExpr::create(Pass, OutContext))); 524 525 MCSymbol *Trap = OutContext.createTempSymbol(); 526 OutStreamer->emitLabel(Trap); 527 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::EBREAK)); 528 emitKCFITrapEntry(*MI.getMF(), Trap); 529 OutStreamer->emitLabel(Pass); 530 } 531 532 void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) { 533 if (HwasanMemaccessSymbols.empty()) 534 return; 535 536 assert(TM.getTargetTriple().isOSBinFormatELF()); 537 // Use MCSubtargetInfo from TargetMachine. Individual functions may have 538 // attributes that differ from other functions in the module and we have no 539 // way to know which function is correct. 540 const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo(); 541 542 MCSymbol *HwasanTagMismatchV2Sym = 543 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2"); 544 // Annotate symbol as one having incompatible calling convention, so 545 // run-time linkers can instead eagerly bind this function. 546 auto &RTS = 547 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 548 RTS.emitDirectiveVariantCC(*HwasanTagMismatchV2Sym); 549 550 const MCSymbolRefExpr *HwasanTagMismatchV2Ref = 551 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext); 552 auto Expr = RISCVMCExpr::create(HwasanTagMismatchV2Ref, 553 RISCVMCExpr::VK_RISCV_CALL, OutContext); 554 555 for (auto &P : HwasanMemaccessSymbols) { 556 unsigned Reg = std::get<0>(P.first); 557 uint32_t AccessInfo = std::get<1>(P.first); 558 MCSymbol *Sym = P.second; 559 560 unsigned Size = 561 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf); 562 OutStreamer->switchSection(OutContext.getELFSection( 563 ".text.hot", ELF::SHT_PROGBITS, 564 ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(), 565 /*IsComdat=*/true)); 566 567 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction); 568 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak); 569 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden); 570 OutStreamer->emitLabel(Sym); 571 572 // Extract shadow offset from ptr 573 OutStreamer->emitInstruction( 574 MCInstBuilder(RISCV::SLLI).addReg(RISCV::X6).addReg(Reg).addImm(8), 575 MCSTI); 576 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SRLI) 577 .addReg(RISCV::X6) 578 .addReg(RISCV::X6) 579 .addImm(12), 580 MCSTI); 581 // load shadow tag in X6, X5 contains shadow base 582 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADD) 583 .addReg(RISCV::X6) 584 .addReg(RISCV::X5) 585 .addReg(RISCV::X6), 586 MCSTI); 587 OutStreamer->emitInstruction( 588 MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0), 589 MCSTI); 590 // Extract tag from X5 and compare it with loaded tag from shadow 591 OutStreamer->emitInstruction( 592 MCInstBuilder(RISCV::SRLI).addReg(RISCV::X7).addReg(Reg).addImm(56), 593 MCSTI); 594 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol(); 595 // X7 contains tag from memory, while X6 contains tag from the pointer 596 OutStreamer->emitInstruction( 597 MCInstBuilder(RISCV::BNE) 598 .addReg(RISCV::X7) 599 .addReg(RISCV::X6) 600 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym, 601 OutContext)), 602 MCSTI); 603 MCSymbol *ReturnSym = OutContext.createTempSymbol(); 604 OutStreamer->emitLabel(ReturnSym); 605 OutStreamer->emitInstruction(MCInstBuilder(RISCV::JALR) 606 .addReg(RISCV::X0) 607 .addReg(RISCV::X1) 608 .addImm(0), 609 MCSTI); 610 OutStreamer->emitLabel(HandleMismatchOrPartialSym); 611 612 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 613 .addReg(RISCV::X28) 614 .addReg(RISCV::X0) 615 .addImm(16), 616 MCSTI); 617 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol(); 618 OutStreamer->emitInstruction( 619 MCInstBuilder(RISCV::BGEU) 620 .addReg(RISCV::X6) 621 .addReg(RISCV::X28) 622 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 623 MCSTI); 624 625 OutStreamer->emitInstruction( 626 MCInstBuilder(RISCV::ANDI).addReg(RISCV::X28).addReg(Reg).addImm(0xF), 627 MCSTI); 628 629 if (Size != 1) 630 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 631 .addReg(RISCV::X28) 632 .addReg(RISCV::X28) 633 .addImm(Size - 1), 634 MCSTI); 635 OutStreamer->emitInstruction( 636 MCInstBuilder(RISCV::BGE) 637 .addReg(RISCV::X28) 638 .addReg(RISCV::X6) 639 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 640 MCSTI); 641 642 OutStreamer->emitInstruction( 643 MCInstBuilder(RISCV::ORI).addReg(RISCV::X6).addReg(Reg).addImm(0xF), 644 MCSTI); 645 OutStreamer->emitInstruction( 646 MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0), 647 MCSTI); 648 OutStreamer->emitInstruction( 649 MCInstBuilder(RISCV::BEQ) 650 .addReg(RISCV::X6) 651 .addReg(RISCV::X7) 652 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), 653 MCSTI); 654 655 OutStreamer->emitLabel(HandleMismatchSym); 656 657 // | Previous stack frames... | 658 // +=================================+ <-- [SP + 256] 659 // | ... | 660 // | | 661 // | Stack frame space for x12 - x31.| 662 // | | 663 // | ... | 664 // +---------------------------------+ <-- [SP + 96] 665 // | Saved x11(arg1), as | 666 // | __hwasan_check_* clobbers it. | 667 // +---------------------------------+ <-- [SP + 88] 668 // | Saved x10(arg0), as | 669 // | __hwasan_check_* clobbers it. | 670 // +---------------------------------+ <-- [SP + 80] 671 // | | 672 // | Stack frame space for x9. | 673 // +---------------------------------+ <-- [SP + 72] 674 // | | 675 // | Saved x8(fp), as | 676 // | __hwasan_check_* clobbers it. | 677 // +---------------------------------+ <-- [SP + 64] 678 // | ... | 679 // | | 680 // | Stack frame space for x2 - x7. | 681 // | | 682 // | ... | 683 // +---------------------------------+ <-- [SP + 16] 684 // | Return address (x1) for caller | 685 // | of __hwasan_check_*. | 686 // +---------------------------------+ <-- [SP + 8] 687 // | Reserved place for x0, possibly | 688 // | junk, since we don't save it. | 689 // +---------------------------------+ <-- [x2 / SP] 690 691 // Adjust sp 692 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 693 .addReg(RISCV::X2) 694 .addReg(RISCV::X2) 695 .addImm(-256), 696 MCSTI); 697 698 // store x10(arg0) by new sp 699 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD) 700 .addReg(RISCV::X10) 701 .addReg(RISCV::X2) 702 .addImm(8 * 10), 703 MCSTI); 704 // store x11(arg1) by new sp 705 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD) 706 .addReg(RISCV::X11) 707 .addReg(RISCV::X2) 708 .addImm(8 * 11), 709 MCSTI); 710 711 // store x8(fp) by new sp 712 OutStreamer->emitInstruction( 713 MCInstBuilder(RISCV::SD).addReg(RISCV::X8).addReg(RISCV::X2).addImm(8 * 714 8), 715 MCSTI); 716 // store x1(ra) by new sp 717 OutStreamer->emitInstruction( 718 MCInstBuilder(RISCV::SD).addReg(RISCV::X1).addReg(RISCV::X2).addImm(1 * 719 8), 720 MCSTI); 721 if (Reg != RISCV::X10) 722 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 723 .addReg(RISCV::X10) 724 .addReg(Reg) 725 .addImm(0), 726 MCSTI); 727 OutStreamer->emitInstruction( 728 MCInstBuilder(RISCV::ADDI) 729 .addReg(RISCV::X11) 730 .addReg(RISCV::X0) 731 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask), 732 MCSTI); 733 734 OutStreamer->emitInstruction(MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr), 735 MCSTI); 736 } 737 } 738 739 static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym, 740 const AsmPrinter &AP) { 741 MCContext &Ctx = AP.OutContext; 742 RISCVMCExpr::VariantKind Kind; 743 744 switch (MO.getTargetFlags()) { 745 default: 746 llvm_unreachable("Unknown target flag on GV operand"); 747 case RISCVII::MO_None: 748 Kind = RISCVMCExpr::VK_RISCV_None; 749 break; 750 case RISCVII::MO_CALL: 751 Kind = RISCVMCExpr::VK_RISCV_CALL_PLT; 752 break; 753 case RISCVII::MO_LO: 754 Kind = RISCVMCExpr::VK_RISCV_LO; 755 break; 756 case RISCVII::MO_HI: 757 Kind = RISCVMCExpr::VK_RISCV_HI; 758 break; 759 case RISCVII::MO_PCREL_LO: 760 Kind = RISCVMCExpr::VK_RISCV_PCREL_LO; 761 break; 762 case RISCVII::MO_PCREL_HI: 763 Kind = RISCVMCExpr::VK_RISCV_PCREL_HI; 764 break; 765 case RISCVII::MO_GOT_HI: 766 Kind = RISCVMCExpr::VK_RISCV_GOT_HI; 767 break; 768 case RISCVII::MO_TPREL_LO: 769 Kind = RISCVMCExpr::VK_RISCV_TPREL_LO; 770 break; 771 case RISCVII::MO_TPREL_HI: 772 Kind = RISCVMCExpr::VK_RISCV_TPREL_HI; 773 break; 774 case RISCVII::MO_TPREL_ADD: 775 Kind = RISCVMCExpr::VK_RISCV_TPREL_ADD; 776 break; 777 case RISCVII::MO_TLS_GOT_HI: 778 Kind = RISCVMCExpr::VK_RISCV_TLS_GOT_HI; 779 break; 780 case RISCVII::MO_TLS_GD_HI: 781 Kind = RISCVMCExpr::VK_RISCV_TLS_GD_HI; 782 break; 783 case RISCVII::MO_TLSDESC_HI: 784 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_HI; 785 break; 786 case RISCVII::MO_TLSDESC_LOAD_LO: 787 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_LOAD_LO; 788 break; 789 case RISCVII::MO_TLSDESC_ADD_LO: 790 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_ADD_LO; 791 break; 792 case RISCVII::MO_TLSDESC_CALL: 793 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_CALL; 794 break; 795 } 796 797 const MCExpr *ME = 798 MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Ctx); 799 800 if (!MO.isJTI() && !MO.isMBB() && MO.getOffset()) 801 ME = MCBinaryExpr::createAdd( 802 ME, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx); 803 804 if (Kind != RISCVMCExpr::VK_RISCV_None) 805 ME = RISCVMCExpr::create(ME, Kind, Ctx); 806 return MCOperand::createExpr(ME); 807 } 808 809 bool RISCVAsmPrinter::lowerOperand(const MachineOperand &MO, 810 MCOperand &MCOp) const { 811 switch (MO.getType()) { 812 default: 813 report_fatal_error("lowerOperand: unknown operand type"); 814 case MachineOperand::MO_Register: 815 // Ignore all implicit register operands. 816 if (MO.isImplicit()) 817 return false; 818 MCOp = MCOperand::createReg(MO.getReg()); 819 break; 820 case MachineOperand::MO_RegisterMask: 821 // Regmasks are like implicit defs. 822 return false; 823 case MachineOperand::MO_Immediate: 824 MCOp = MCOperand::createImm(MO.getImm()); 825 break; 826 case MachineOperand::MO_MachineBasicBlock: 827 MCOp = lowerSymbolOperand(MO, MO.getMBB()->getSymbol(), *this); 828 break; 829 case MachineOperand::MO_GlobalAddress: 830 MCOp = lowerSymbolOperand(MO, getSymbolPreferLocal(*MO.getGlobal()), *this); 831 break; 832 case MachineOperand::MO_BlockAddress: 833 MCOp = lowerSymbolOperand(MO, GetBlockAddressSymbol(MO.getBlockAddress()), 834 *this); 835 break; 836 case MachineOperand::MO_ExternalSymbol: 837 MCOp = lowerSymbolOperand(MO, GetExternalSymbolSymbol(MO.getSymbolName()), 838 *this); 839 break; 840 case MachineOperand::MO_ConstantPoolIndex: 841 MCOp = lowerSymbolOperand(MO, GetCPISymbol(MO.getIndex()), *this); 842 break; 843 case MachineOperand::MO_JumpTableIndex: 844 MCOp = lowerSymbolOperand(MO, GetJTISymbol(MO.getIndex()), *this); 845 break; 846 case MachineOperand::MO_MCSymbol: 847 MCOp = lowerSymbolOperand(MO, MO.getMCSymbol(), *this); 848 break; 849 } 850 return true; 851 } 852 853 static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI, 854 MCInst &OutMI) { 855 const RISCVVPseudosTable::PseudoInfo *RVV = 856 RISCVVPseudosTable::getPseudoInfo(MI->getOpcode()); 857 if (!RVV) 858 return false; 859 860 OutMI.setOpcode(RVV->BaseInstr); 861 862 const MachineBasicBlock *MBB = MI->getParent(); 863 assert(MBB && "MI expected to be in a basic block"); 864 const MachineFunction *MF = MBB->getParent(); 865 assert(MF && "MBB expected to be in a machine function"); 866 867 const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>(); 868 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 869 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 870 assert(TRI && "TargetRegisterInfo expected"); 871 872 const MCInstrDesc &MCID = MI->getDesc(); 873 uint64_t TSFlags = MCID.TSFlags; 874 unsigned NumOps = MI->getNumExplicitOperands(); 875 876 // Skip policy, SEW, VL, VXRM/FRM operands which are the last operands if 877 // present. 878 if (RISCVII::hasVecPolicyOp(TSFlags)) 879 --NumOps; 880 if (RISCVII::hasSEWOp(TSFlags)) 881 --NumOps; 882 if (RISCVII::hasVLOp(TSFlags)) 883 --NumOps; 884 if (RISCVII::hasRoundModeOp(TSFlags)) 885 --NumOps; 886 887 bool hasVLOutput = RISCV::isFaultFirstLoad(*MI); 888 for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) { 889 const MachineOperand &MO = MI->getOperand(OpNo); 890 // Skip vl ouput. It should be the second output. 891 if (hasVLOutput && OpNo == 1) 892 continue; 893 894 // Skip merge op. It should be the first operand after the defs. 895 if (OpNo == MI->getNumExplicitDefs() && MO.isReg() && MO.isTied()) { 896 assert(MCID.getOperandConstraint(OpNo, MCOI::TIED_TO) == 0 && 897 "Expected tied to first def."); 898 const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode()); 899 // Skip if the next operand in OutMI is not supposed to be tied. Unless it 900 // is a _TIED instruction. 901 if (OutMCID.getOperandConstraint(OutMI.getNumOperands(), MCOI::TIED_TO) < 902 0 && 903 !RISCVII::isTiedPseudo(TSFlags)) 904 continue; 905 } 906 907 MCOperand MCOp; 908 switch (MO.getType()) { 909 default: 910 llvm_unreachable("Unknown operand type"); 911 case MachineOperand::MO_Register: { 912 Register Reg = MO.getReg(); 913 914 if (RISCV::VRM2RegClass.contains(Reg) || 915 RISCV::VRM4RegClass.contains(Reg) || 916 RISCV::VRM8RegClass.contains(Reg)) { 917 Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0); 918 assert(Reg && "Subregister does not exist"); 919 } else if (RISCV::FPR16RegClass.contains(Reg)) { 920 Reg = 921 TRI->getMatchingSuperReg(Reg, RISCV::sub_16, &RISCV::FPR32RegClass); 922 assert(Reg && "Subregister does not exist"); 923 } else if (RISCV::FPR64RegClass.contains(Reg)) { 924 Reg = TRI->getSubReg(Reg, RISCV::sub_32); 925 assert(Reg && "Superregister does not exist"); 926 } else if (RISCV::VRN2M1RegClass.contains(Reg) || 927 RISCV::VRN2M2RegClass.contains(Reg) || 928 RISCV::VRN2M4RegClass.contains(Reg) || 929 RISCV::VRN3M1RegClass.contains(Reg) || 930 RISCV::VRN3M2RegClass.contains(Reg) || 931 RISCV::VRN4M1RegClass.contains(Reg) || 932 RISCV::VRN4M2RegClass.contains(Reg) || 933 RISCV::VRN5M1RegClass.contains(Reg) || 934 RISCV::VRN6M1RegClass.contains(Reg) || 935 RISCV::VRN7M1RegClass.contains(Reg) || 936 RISCV::VRN8M1RegClass.contains(Reg)) { 937 Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0); 938 assert(Reg && "Subregister does not exist"); 939 } 940 941 MCOp = MCOperand::createReg(Reg); 942 break; 943 } 944 case MachineOperand::MO_Immediate: 945 MCOp = MCOperand::createImm(MO.getImm()); 946 break; 947 } 948 OutMI.addOperand(MCOp); 949 } 950 951 // Unmasked pseudo instructions need to append dummy mask operand to 952 // V instructions. All V instructions are modeled as the masked version. 953 const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode()); 954 if (OutMI.getNumOperands() < OutMCID.getNumOperands()) { 955 assert(OutMCID.operands()[OutMI.getNumOperands()].RegClass == 956 RISCV::VMV0RegClassID && 957 "Expected only mask operand to be missing"); 958 OutMI.addOperand(MCOperand::createReg(RISCV::NoRegister)); 959 } 960 961 assert(OutMI.getNumOperands() == OutMCID.getNumOperands()); 962 return true; 963 } 964 965 bool RISCVAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) { 966 if (lowerRISCVVMachineInstrToMCInst(MI, OutMI)) 967 return false; 968 969 OutMI.setOpcode(MI->getOpcode()); 970 971 for (const MachineOperand &MO : MI->operands()) { 972 MCOperand MCOp; 973 if (lowerOperand(MO, MCOp)) 974 OutMI.addOperand(MCOp); 975 } 976 977 switch (OutMI.getOpcode()) { 978 case TargetOpcode::PATCHABLE_FUNCTION_ENTER: { 979 const Function &F = MI->getParent()->getParent()->getFunction(); 980 if (F.hasFnAttribute("patchable-function-entry")) { 981 unsigned Num; 982 if (F.getFnAttribute("patchable-function-entry") 983 .getValueAsString() 984 .getAsInteger(10, Num)) 985 return false; 986 emitNops(Num); 987 return true; 988 } 989 break; 990 } 991 } 992 return false; 993 } 994