1 //===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements PseudoProbeInserter pass, which inserts pseudo probe
10 // annotations for call instructions with a pseudo-probe-specific dwarf
11 // discriminator. such discriminator indicates that the call instruction comes
12 // with a pseudo probe, and the discriminator value holds information to
13 // identify the corresponding counter.
14 //===----------------------------------------------------------------------===//
15 
16 #include "llvm/CodeGen/MachineBasicBlock.h"
17 #include "llvm/CodeGen/MachineFunctionPass.h"
18 #include "llvm/CodeGen/MachineInstr.h"
19 #include "llvm/CodeGen/TargetInstrInfo.h"
20 #include "llvm/IR/DebugInfoMetadata.h"
21 #include "llvm/IR/PseudoProbe.h"
22 #include "llvm/InitializePasses.h"
23 #include "llvm/MC/MCPseudoProbe.h"
24 #include "llvm/Target/TargetMachine.h"
25 #include <unordered_set>
26 
27 #define DEBUG_TYPE "pseudo-probe-inserter"
28 
29 using namespace llvm;
30 
31 namespace {
32 class PseudoProbeInserter : public MachineFunctionPass {
33 public:
34   static char ID;
35 
36   PseudoProbeInserter() : MachineFunctionPass(ID) {
37     initializePseudoProbeInserterPass(*PassRegistry::getPassRegistry());
38   }
39 
40   StringRef getPassName() const override { return "Pseudo Probe Inserter"; }
41 
42   void getAnalysisUsage(AnalysisUsage &AU) const override {
43     AU.setPreservesAll();
44     MachineFunctionPass::getAnalysisUsage(AU);
45   }
46 
47   bool runOnMachineFunction(MachineFunction &MF) override {
48     const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
49     bool Changed = false;
50     for (MachineBasicBlock &MBB : MF) {
51       MachineInstr *FirstInstr = nullptr;
52       for (MachineInstr &MI : MBB) {
53         if (!MI.isPseudo())
54           FirstInstr = &MI;
55         if (MI.isCall()) {
56           if (DILocation *DL = MI.getDebugLoc()) {
57             auto Value = DL->getDiscriminator();
58             if (DILocation::isPseudoProbeDiscriminator(Value)) {
59               BuildMI(MBB, MI, DL, TII->get(TargetOpcode::PSEUDO_PROBE))
60                   .addImm(getFuncGUID(MF.getFunction().getParent(), DL))
61                   .addImm(
62                       PseudoProbeDwarfDiscriminator::extractProbeIndex(Value))
63                   .addImm(
64                       PseudoProbeDwarfDiscriminator::extractProbeType(Value))
65                   .addImm(PseudoProbeDwarfDiscriminator::extractProbeAttributes(
66                       Value));
67               Changed = true;
68             }
69           }
70         }
71       }
72 
73       // Walk the block backwards, move PSEUDO_PROBE before the first real
74       // instruction to fix out-of-order probes. There is a problem with probes
75       // as the terminator of the block. During the offline counts processing,
76       // the samples collected on the first physical instruction following a
77       // probe will be counted towards the probe. This logically equals to
78       // treating the instruction next to a probe as if it is from the same
79       // block of the probe. This is accurate most of the time unless the
80       // instruction can be reached from multiple flows, which means it actually
81       // starts a new block. Samples collected on such probes may cause
82       // imprecision with the counts inference algorithm. Fortunately, if
83       // there are still other native instructions preceding the probe we can
84       // use them as a place holder to collect samples for the probe.
85       if (FirstInstr) {
86         auto MII = MBB.rbegin();
87         while (MII != MBB.rend()) {
88           // Skip all pseudo probes followed by a real instruction since they
89           // are not dangling.
90           if (!MII->isPseudo())
91             break;
92           auto Cur = MII++;
93           if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE)
94             continue;
95           // Move the dangling probe before FirstInstr.
96           auto *ProbeInstr = &*Cur;
97           MBB.remove(ProbeInstr);
98           MBB.insert(FirstInstr, ProbeInstr);
99           Changed = true;
100         }
101       } else {
102         // Probes not surrounded by any real instructions in the same block are
103         // called dangling probes. Since there's no good way to pick up a sample
104         // collection point for dangling probes at compile time, they are being
105         // removed so that the profile correlation tool will not report any
106         // samples collected for them and it's up to the counts inference tool
107         // to get them a reasonable count.
108         SmallVector<MachineInstr *, 4> ToBeRemoved;
109         for (MachineInstr &MI : MBB) {
110           if (MI.isPseudoProbe())
111             ToBeRemoved.push_back(&MI);
112         }
113 
114         for (auto *MI : ToBeRemoved)
115           MI->eraseFromParent();
116 
117         Changed |= !ToBeRemoved.empty();
118       }
119     }
120 
121     return Changed;
122   }
123 
124 private:
125   uint64_t getFuncGUID(Module *M, DILocation *DL) {
126     auto *SP = DL->getScope()->getSubprogram();
127     auto Name = SP->getLinkageName();
128     if (Name.empty())
129       Name = SP->getName();
130     return Function::getGUID(Name);
131   }
132 };
133 } // namespace
134 
135 char PseudoProbeInserter::ID = 0;
136 INITIALIZE_PASS_BEGIN(PseudoProbeInserter, DEBUG_TYPE,
137                       "Insert pseudo probe annotations for value profiling",
138                       false, false)
139 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
140 INITIALIZE_PASS_END(PseudoProbeInserter, DEBUG_TYPE,
141                     "Insert pseudo probe annotations for value profiling",
142                     false, false)
143 
144 FunctionPass *llvm::createPseudoProbeInserter() {
145   return new PseudoProbeInserter();
146 }
147