1 //===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements PseudoProbeInserter pass, which inserts pseudo probe 10 // annotations for call instructions with a pseudo-probe-specific dwarf 11 // discriminator. such discriminator indicates that the call instruction comes 12 // with a pseudo probe, and the discriminator value holds information to 13 // identify the corresponding counter. 14 //===----------------------------------------------------------------------===// 15 16 #include "llvm/CodeGen/MachineBasicBlock.h" 17 #include "llvm/CodeGen/MachineFunctionPass.h" 18 #include "llvm/CodeGen/MachineInstr.h" 19 #include "llvm/CodeGen/TargetInstrInfo.h" 20 #include "llvm/IR/DebugInfoMetadata.h" 21 #include "llvm/IR/PseudoProbe.h" 22 #include "llvm/InitializePasses.h" 23 #include "llvm/MC/MCPseudoProbe.h" 24 #include "llvm/Target/TargetMachine.h" 25 #include <unordered_set> 26 27 #define DEBUG_TYPE "pseudo-probe-inserter" 28 29 using namespace llvm; 30 31 namespace { 32 class PseudoProbeInserter : public MachineFunctionPass { 33 public: 34 static char ID; 35 36 PseudoProbeInserter() : MachineFunctionPass(ID) { 37 initializePseudoProbeInserterPass(*PassRegistry::getPassRegistry()); 38 } 39 40 StringRef getPassName() const override { return "Pseudo Probe Inserter"; } 41 42 void getAnalysisUsage(AnalysisUsage &AU) const override { 43 AU.setPreservesAll(); 44 MachineFunctionPass::getAnalysisUsage(AU); 45 } 46 47 bool runOnMachineFunction(MachineFunction &MF) override { 48 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); 49 bool Changed = false; 50 for (MachineBasicBlock &MBB : MF) { 51 MachineInstr *FirstInstr = nullptr; 52 for (MachineInstr &MI : MBB) { 53 if (!MI.isPseudo()) 54 FirstInstr = &MI; 55 if (MI.isCall()) { 56 if (DILocation *DL = MI.getDebugLoc()) { 57 auto Value = DL->getDiscriminator(); 58 if (DILocation::isPseudoProbeDiscriminator(Value)) { 59 BuildMI(MBB, MI, DL, TII->get(TargetOpcode::PSEUDO_PROBE)) 60 .addImm(getFuncGUID(MF.getFunction().getParent(), DL)) 61 .addImm( 62 PseudoProbeDwarfDiscriminator::extractProbeIndex(Value)) 63 .addImm( 64 PseudoProbeDwarfDiscriminator::extractProbeType(Value)) 65 .addImm(PseudoProbeDwarfDiscriminator::extractProbeAttributes( 66 Value)); 67 Changed = true; 68 } 69 } 70 } 71 } 72 73 // Walk the block backwards, move PSEUDO_PROBE before the first real 74 // instruction to fix out-of-order probes. There is a problem with probes 75 // as the terminator of the block. During the offline counts processing, 76 // the samples collected on the first physical instruction following a 77 // probe will be counted towards the probe. This logically equals to 78 // treating the instruction next to a probe as if it is from the same 79 // block of the probe. This is accurate most of the time unless the 80 // instruction can be reached from multiple flows, which means it actually 81 // starts a new block. Samples collected on such probes may cause 82 // imprecision with the counts inference algorithm. Fortunately, if 83 // there are still other native instructions preceding the probe we can 84 // use them as a place holder to collect samples for the probe. 85 if (FirstInstr) { 86 auto MII = MBB.rbegin(); 87 while (MII != MBB.rend()) { 88 // Skip all pseudo probes followed by a real instruction since they 89 // are not dangling. 90 if (!MII->isPseudo()) 91 break; 92 auto Cur = MII++; 93 if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE) 94 continue; 95 // Move the dangling probe before FirstInstr. 96 auto *ProbeInstr = &*Cur; 97 MBB.remove(ProbeInstr); 98 MBB.insert(FirstInstr, ProbeInstr); 99 Changed = true; 100 } 101 } else { 102 // Probes not surrounded by any real instructions in the same block are 103 // called dangling probes. Since there's no good way to pick up a sample 104 // collection point for dangling probes at compile time, they are being 105 // removed so that the profile correlation tool will not report any 106 // samples collected for them and it's up to the counts inference tool 107 // to get them a reasonable count. 108 SmallVector<MachineInstr *, 4> ToBeRemoved; 109 for (MachineInstr &MI : MBB) { 110 if (MI.isPseudoProbe()) 111 ToBeRemoved.push_back(&MI); 112 } 113 114 for (auto *MI : ToBeRemoved) 115 MI->eraseFromParent(); 116 117 Changed |= !ToBeRemoved.empty(); 118 } 119 } 120 121 return Changed; 122 } 123 124 private: 125 uint64_t getFuncGUID(Module *M, DILocation *DL) { 126 auto *SP = DL->getScope()->getSubprogram(); 127 auto Name = SP->getLinkageName(); 128 if (Name.empty()) 129 Name = SP->getName(); 130 return Function::getGUID(Name); 131 } 132 }; 133 } // namespace 134 135 char PseudoProbeInserter::ID = 0; 136 INITIALIZE_PASS_BEGIN(PseudoProbeInserter, DEBUG_TYPE, 137 "Insert pseudo probe annotations for value profiling", 138 false, false) 139 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) 140 INITIALIZE_PASS_END(PseudoProbeInserter, DEBUG_TYPE, 141 "Insert pseudo probe annotations for value profiling", 142 false, false) 143 144 FunctionPass *llvm::createPseudoProbeInserter() { 145 return new PseudoProbeInserter(); 146 } 147