1 //===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements PseudoProbeInserter pass, which inserts pseudo probe
10 // annotations for call instructions with a pseudo-probe-specific dwarf
11 // discriminator. such discriminator indicates that the call instruction comes
12 // with a pseudo probe, and the discriminator value holds information to
13 // identify the corresponding counter.
14 //===----------------------------------------------------------------------===//
15 
16 #include "llvm/CodeGen/MachineBasicBlock.h"
17 #include "llvm/CodeGen/MachineFunctionPass.h"
18 #include "llvm/CodeGen/MachineInstr.h"
19 #include "llvm/CodeGen/TargetInstrInfo.h"
20 #include "llvm/IR/DebugInfoMetadata.h"
21 #include "llvm/IR/Module.h"
22 #include "llvm/IR/PseudoProbe.h"
23 #include "llvm/InitializePasses.h"
24 
25 #define DEBUG_TYPE "pseudo-probe-inserter"
26 
27 using namespace llvm;
28 
29 namespace {
30 class PseudoProbeInserter : public MachineFunctionPass {
31 public:
32   static char ID;
33 
34   PseudoProbeInserter() : MachineFunctionPass(ID) {
35     initializePseudoProbeInserterPass(*PassRegistry::getPassRegistry());
36   }
37 
38   StringRef getPassName() const override { return "Pseudo Probe Inserter"; }
39 
40   void getAnalysisUsage(AnalysisUsage &AU) const override {
41     AU.setPreservesAll();
42     MachineFunctionPass::getAnalysisUsage(AU);
43   }
44 
45   bool doInitialization(Module &M) override {
46     ShouldRun = M.getNamedMetadata(PseudoProbeDescMetadataName);
47     return false;
48   }
49 
50   bool runOnMachineFunction(MachineFunction &MF) override {
51     if (!ShouldRun)
52       return false;
53     const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
54     bool Changed = false;
55     for (MachineBasicBlock &MBB : MF) {
56       MachineInstr *FirstInstr = nullptr;
57       for (MachineInstr &MI : MBB) {
58         if (!MI.isPseudo())
59           FirstInstr = &MI;
60         if (MI.isCall()) {
61           if (DILocation *DL = MI.getDebugLoc()) {
62             auto Value = DL->getDiscriminator();
63             if (DILocation::isPseudoProbeDiscriminator(Value)) {
64               BuildMI(MBB, MI, DL, TII->get(TargetOpcode::PSEUDO_PROBE))
65                   .addImm(getFuncGUID(MF.getFunction().getParent(), DL))
66                   .addImm(
67                       PseudoProbeDwarfDiscriminator::extractProbeIndex(Value))
68                   .addImm(
69                       PseudoProbeDwarfDiscriminator::extractProbeType(Value))
70                   .addImm(PseudoProbeDwarfDiscriminator::extractProbeAttributes(
71                       Value));
72               Changed = true;
73             }
74           }
75         }
76       }
77 
78       // Walk the block backwards, move PSEUDO_PROBE before the first real
79       // instruction to fix out-of-order probes. There is a problem with probes
80       // as the terminator of the block. During the offline counts processing,
81       // the samples collected on the first physical instruction following a
82       // probe will be counted towards the probe. This logically equals to
83       // treating the instruction next to a probe as if it is from the same
84       // block of the probe. This is accurate most of the time unless the
85       // instruction can be reached from multiple flows, which means it actually
86       // starts a new block. Samples collected on such probes may cause
87       // imprecision with the counts inference algorithm. Fortunately, if
88       // there are still other native instructions preceding the probe we can
89       // use them as a place holder to collect samples for the probe.
90       if (FirstInstr) {
91         auto MII = MBB.rbegin();
92         while (MII != MBB.rend()) {
93           // Skip all pseudo probes followed by a real instruction since they
94           // are not dangling.
95           if (!MII->isPseudo())
96             break;
97           auto Cur = MII++;
98           if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE)
99             continue;
100           // Move the dangling probe before FirstInstr.
101           auto *ProbeInstr = &*Cur;
102           MBB.remove(ProbeInstr);
103           MBB.insert(FirstInstr, ProbeInstr);
104           Changed = true;
105         }
106       } else {
107         // Probes not surrounded by any real instructions in the same block are
108         // called dangling probes. Since there's no good way to pick up a sample
109         // collection point for dangling probes at compile time, they are being
110         // removed so that the profile correlation tool will not report any
111         // samples collected for them and it's up to the counts inference tool
112         // to get them a reasonable count.
113         SmallVector<MachineInstr *, 4> ToBeRemoved;
114         for (MachineInstr &MI : MBB) {
115           if (MI.isPseudoProbe())
116             ToBeRemoved.push_back(&MI);
117         }
118 
119         for (auto *MI : ToBeRemoved)
120           MI->eraseFromParent();
121 
122         Changed |= !ToBeRemoved.empty();
123       }
124     }
125 
126     return Changed;
127   }
128 
129 private:
130   uint64_t getFuncGUID(Module *M, DILocation *DL) {
131     auto Name = DL->getSubprogramLinkageName();
132     return Function::getGUID(Name);
133   }
134 
135   bool ShouldRun = false;
136 };
137 } // namespace
138 
139 char PseudoProbeInserter::ID = 0;
140 INITIALIZE_PASS_BEGIN(PseudoProbeInserter, DEBUG_TYPE,
141                       "Insert pseudo probe annotations for value profiling",
142                       false, false)
143 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
144 INITIALIZE_PASS_END(PseudoProbeInserter, DEBUG_TYPE,
145                     "Insert pseudo probe annotations for value profiling",
146                     false, false)
147 
148 FunctionPass *llvm::createPseudoProbeInserter() {
149   return new PseudoProbeInserter();
150 }
151