1 //===- BlockFrequencyInfo.cpp - Block Frequency Analysis ------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Loops should be simplified before this analysis.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/Analysis/BlockFrequencyInfo.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/None.h"
16 #include "llvm/ADT/iterator.h"
17 #include "llvm/Analysis/BlockFrequencyInfoImpl.h"
18 #include "llvm/Analysis/BranchProbabilityInfo.h"
19 #include "llvm/Analysis/LoopInfo.h"
20 #include "llvm/IR/CFG.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/PassManager.h"
23 #include "llvm/Pass.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/GraphWriter.h"
26 #include "llvm/Support/raw_ostream.h"
27 #include <algorithm>
28 #include <cassert>
29 #include <string>
30
31 using namespace llvm;
32
33 #define DEBUG_TYPE "block-freq"
34
35 static cl::opt<GVDAGType> ViewBlockFreqPropagationDAG(
36 "view-block-freq-propagation-dags", cl::Hidden,
37 cl::desc("Pop up a window to show a dag displaying how block "
38 "frequencies propagation through the CFG."),
39 cl::values(clEnumValN(GVDT_None, "none", "do not display graphs."),
40 clEnumValN(GVDT_Fraction, "fraction",
41 "display a graph using the "
42 "fractional block frequency representation."),
43 clEnumValN(GVDT_Integer, "integer",
44 "display a graph using the raw "
45 "integer fractional block frequency representation."),
46 clEnumValN(GVDT_Count, "count", "display a graph using the real "
47 "profile count if available.")));
48
49 cl::opt<std::string>
50 ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden,
51 cl::desc("The option to specify "
52 "the name of the function "
53 "whose CFG will be displayed."));
54
55 cl::opt<unsigned>
56 ViewHotFreqPercent("view-hot-freq-percent", cl::init(10), cl::Hidden,
57 cl::desc("An integer in percent used to specify "
58 "the hot blocks/edges to be displayed "
59 "in red: a block or edge whose frequency "
60 "is no less than the max frequency of the "
61 "function multiplied by this percent."));
62
63 // Command line option to turn on CFG dot or text dump after profile annotation.
64 cl::opt<PGOViewCountsType> PGOViewCounts(
65 "pgo-view-counts", cl::Hidden,
66 cl::desc("A boolean option to show CFG dag or text with "
67 "block profile counts and branch probabilities "
68 "right after PGO profile annotation step. The "
69 "profile counts are computed using branch "
70 "probabilities from the runtime profile data and "
71 "block frequency propagation algorithm. To view "
72 "the raw counts from the profile, use option "
73 "-pgo-view-raw-counts instead. To limit graph "
74 "display to only one function, use filtering option "
75 "-view-bfi-func-name."),
76 cl::values(clEnumValN(PGOVCT_None, "none", "do not show."),
77 clEnumValN(PGOVCT_Graph, "graph", "show a graph."),
78 clEnumValN(PGOVCT_Text, "text", "show in text.")));
79
80 static cl::opt<bool> PrintBlockFreq(
81 "print-bfi", cl::init(false), cl::Hidden,
82 cl::desc("Print the block frequency info."));
83
84 cl::opt<std::string> PrintBlockFreqFuncName(
85 "print-bfi-func-name", cl::Hidden,
86 cl::desc("The option to specify the name of the function "
87 "whose block frequency info is printed."));
88
89 namespace llvm {
90
getGVDT()91 static GVDAGType getGVDT() {
92 if (PGOViewCounts == PGOVCT_Graph)
93 return GVDT_Count;
94 return ViewBlockFreqPropagationDAG;
95 }
96
97 template <>
98 struct GraphTraits<BlockFrequencyInfo *> {
99 using NodeRef = const BasicBlock *;
100 using ChildIteratorType = succ_const_iterator;
101 using nodes_iterator = pointer_iterator<Function::const_iterator>;
102
getEntryNodellvm::GraphTraits103 static NodeRef getEntryNode(const BlockFrequencyInfo *G) {
104 return &G->getFunction()->front();
105 }
106
child_beginllvm::GraphTraits107 static ChildIteratorType child_begin(const NodeRef N) {
108 return succ_begin(N);
109 }
110
child_endllvm::GraphTraits111 static ChildIteratorType child_end(const NodeRef N) { return succ_end(N); }
112
nodes_beginllvm::GraphTraits113 static nodes_iterator nodes_begin(const BlockFrequencyInfo *G) {
114 return nodes_iterator(G->getFunction()->begin());
115 }
116
nodes_endllvm::GraphTraits117 static nodes_iterator nodes_end(const BlockFrequencyInfo *G) {
118 return nodes_iterator(G->getFunction()->end());
119 }
120 };
121
122 using BFIDOTGTraitsBase =
123 BFIDOTGraphTraitsBase<BlockFrequencyInfo, BranchProbabilityInfo>;
124
125 template <>
126 struct DOTGraphTraits<BlockFrequencyInfo *> : public BFIDOTGTraitsBase {
DOTGraphTraitsllvm::DOTGraphTraits127 explicit DOTGraphTraits(bool isSimple = false)
128 : BFIDOTGTraitsBase(isSimple) {}
129
getNodeLabelllvm::DOTGraphTraits130 std::string getNodeLabel(const BasicBlock *Node,
131 const BlockFrequencyInfo *Graph) {
132
133 return BFIDOTGTraitsBase::getNodeLabel(Node, Graph, getGVDT());
134 }
135
getNodeAttributesllvm::DOTGraphTraits136 std::string getNodeAttributes(const BasicBlock *Node,
137 const BlockFrequencyInfo *Graph) {
138 return BFIDOTGTraitsBase::getNodeAttributes(Node, Graph,
139 ViewHotFreqPercent);
140 }
141
getEdgeAttributesllvm::DOTGraphTraits142 std::string getEdgeAttributes(const BasicBlock *Node, EdgeIter EI,
143 const BlockFrequencyInfo *BFI) {
144 return BFIDOTGTraitsBase::getEdgeAttributes(Node, EI, BFI, BFI->getBPI(),
145 ViewHotFreqPercent);
146 }
147 };
148
149 } // end namespace llvm
150
151 BlockFrequencyInfo::BlockFrequencyInfo() = default;
152
BlockFrequencyInfo(const Function & F,const BranchProbabilityInfo & BPI,const LoopInfo & LI)153 BlockFrequencyInfo::BlockFrequencyInfo(const Function &F,
154 const BranchProbabilityInfo &BPI,
155 const LoopInfo &LI) {
156 calculate(F, BPI, LI);
157 }
158
BlockFrequencyInfo(BlockFrequencyInfo && Arg)159 BlockFrequencyInfo::BlockFrequencyInfo(BlockFrequencyInfo &&Arg)
160 : BFI(std::move(Arg.BFI)) {}
161
operator =(BlockFrequencyInfo && RHS)162 BlockFrequencyInfo &BlockFrequencyInfo::operator=(BlockFrequencyInfo &&RHS) {
163 releaseMemory();
164 BFI = std::move(RHS.BFI);
165 return *this;
166 }
167
168 // Explicitly define the default constructor otherwise it would be implicitly
169 // defined at the first ODR-use which is the BFI member in the
170 // LazyBlockFrequencyInfo header. The dtor needs the BlockFrequencyInfoImpl
171 // template instantiated which is not available in the header.
172 BlockFrequencyInfo::~BlockFrequencyInfo() = default;
173
invalidate(Function & F,const PreservedAnalyses & PA,FunctionAnalysisManager::Invalidator &)174 bool BlockFrequencyInfo::invalidate(Function &F, const PreservedAnalyses &PA,
175 FunctionAnalysisManager::Invalidator &) {
176 // Check whether the analysis, all analyses on functions, or the function's
177 // CFG have been preserved.
178 auto PAC = PA.getChecker<BlockFrequencyAnalysis>();
179 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>() ||
180 PAC.preservedSet<CFGAnalyses>());
181 }
182
calculate(const Function & F,const BranchProbabilityInfo & BPI,const LoopInfo & LI)183 void BlockFrequencyInfo::calculate(const Function &F,
184 const BranchProbabilityInfo &BPI,
185 const LoopInfo &LI) {
186 if (!BFI)
187 BFI.reset(new ImplType);
188 BFI->calculate(F, BPI, LI);
189 if (ViewBlockFreqPropagationDAG != GVDT_None &&
190 (ViewBlockFreqFuncName.empty() ||
191 F.getName().equals(ViewBlockFreqFuncName))) {
192 view();
193 }
194 if (PrintBlockFreq &&
195 (PrintBlockFreqFuncName.empty() ||
196 F.getName().equals(PrintBlockFreqFuncName))) {
197 print(dbgs());
198 }
199 }
200
getBlockFreq(const BasicBlock * BB) const201 BlockFrequency BlockFrequencyInfo::getBlockFreq(const BasicBlock *BB) const {
202 return BFI ? BFI->getBlockFreq(BB) : 0;
203 }
204
205 Optional<uint64_t>
getBlockProfileCount(const BasicBlock * BB,bool AllowSynthetic) const206 BlockFrequencyInfo::getBlockProfileCount(const BasicBlock *BB,
207 bool AllowSynthetic) const {
208 if (!BFI)
209 return None;
210
211 return BFI->getBlockProfileCount(*getFunction(), BB, AllowSynthetic);
212 }
213
214 Optional<uint64_t>
getProfileCountFromFreq(uint64_t Freq) const215 BlockFrequencyInfo::getProfileCountFromFreq(uint64_t Freq) const {
216 if (!BFI)
217 return None;
218 return BFI->getProfileCountFromFreq(*getFunction(), Freq);
219 }
220
isIrrLoopHeader(const BasicBlock * BB)221 bool BlockFrequencyInfo::isIrrLoopHeader(const BasicBlock *BB) {
222 assert(BFI && "Expected analysis to be available");
223 return BFI->isIrrLoopHeader(BB);
224 }
225
setBlockFreq(const BasicBlock * BB,uint64_t Freq)226 void BlockFrequencyInfo::setBlockFreq(const BasicBlock *BB, uint64_t Freq) {
227 assert(BFI && "Expected analysis to be available");
228 BFI->setBlockFreq(BB, Freq);
229 }
230
setBlockFreqAndScale(const BasicBlock * ReferenceBB,uint64_t Freq,SmallPtrSetImpl<BasicBlock * > & BlocksToScale)231 void BlockFrequencyInfo::setBlockFreqAndScale(
232 const BasicBlock *ReferenceBB, uint64_t Freq,
233 SmallPtrSetImpl<BasicBlock *> &BlocksToScale) {
234 assert(BFI && "Expected analysis to be available");
235 // Use 128 bits APInt to avoid overflow.
236 APInt NewFreq(128, Freq);
237 APInt OldFreq(128, BFI->getBlockFreq(ReferenceBB).getFrequency());
238 APInt BBFreq(128, 0);
239 for (auto *BB : BlocksToScale) {
240 BBFreq = BFI->getBlockFreq(BB).getFrequency();
241 // Multiply first by NewFreq and then divide by OldFreq
242 // to minimize loss of precision.
243 BBFreq *= NewFreq;
244 // udiv is an expensive operation in the general case. If this ends up being
245 // a hot spot, one of the options proposed in
246 // https://reviews.llvm.org/D28535#650071 could be used to avoid this.
247 BBFreq = BBFreq.udiv(OldFreq);
248 BFI->setBlockFreq(BB, BBFreq.getLimitedValue());
249 }
250 BFI->setBlockFreq(ReferenceBB, Freq);
251 }
252
253 /// Pop up a ghostview window with the current block frequency propagation
254 /// rendered using dot.
view(StringRef title) const255 void BlockFrequencyInfo::view(StringRef title) const {
256 ViewGraph(const_cast<BlockFrequencyInfo *>(this), title);
257 }
258
getFunction() const259 const Function *BlockFrequencyInfo::getFunction() const {
260 return BFI ? BFI->getFunction() : nullptr;
261 }
262
getBPI() const263 const BranchProbabilityInfo *BlockFrequencyInfo::getBPI() const {
264 return BFI ? &BFI->getBPI() : nullptr;
265 }
266
267 raw_ostream &BlockFrequencyInfo::
printBlockFreq(raw_ostream & OS,const BlockFrequency Freq) const268 printBlockFreq(raw_ostream &OS, const BlockFrequency Freq) const {
269 return BFI ? BFI->printBlockFreq(OS, Freq) : OS;
270 }
271
272 raw_ostream &
printBlockFreq(raw_ostream & OS,const BasicBlock * BB) const273 BlockFrequencyInfo::printBlockFreq(raw_ostream &OS,
274 const BasicBlock *BB) const {
275 return BFI ? BFI->printBlockFreq(OS, BB) : OS;
276 }
277
getEntryFreq() const278 uint64_t BlockFrequencyInfo::getEntryFreq() const {
279 return BFI ? BFI->getEntryFreq() : 0;
280 }
281
releaseMemory()282 void BlockFrequencyInfo::releaseMemory() { BFI.reset(); }
283
print(raw_ostream & OS) const284 void BlockFrequencyInfo::print(raw_ostream &OS) const {
285 if (BFI)
286 BFI->print(OS);
287 }
288
289 INITIALIZE_PASS_BEGIN(BlockFrequencyInfoWrapperPass, "block-freq",
290 "Block Frequency Analysis", true, true)
291 INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
292 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
293 INITIALIZE_PASS_END(BlockFrequencyInfoWrapperPass, "block-freq",
294 "Block Frequency Analysis", true, true)
295
296 char BlockFrequencyInfoWrapperPass::ID = 0;
297
BlockFrequencyInfoWrapperPass()298 BlockFrequencyInfoWrapperPass::BlockFrequencyInfoWrapperPass()
299 : FunctionPass(ID) {
300 initializeBlockFrequencyInfoWrapperPassPass(*PassRegistry::getPassRegistry());
301 }
302
303 BlockFrequencyInfoWrapperPass::~BlockFrequencyInfoWrapperPass() = default;
304
print(raw_ostream & OS,const Module *) const305 void BlockFrequencyInfoWrapperPass::print(raw_ostream &OS,
306 const Module *) const {
307 BFI.print(OS);
308 }
309
getAnalysisUsage(AnalysisUsage & AU) const310 void BlockFrequencyInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
311 AU.addRequired<BranchProbabilityInfoWrapperPass>();
312 AU.addRequired<LoopInfoWrapperPass>();
313 AU.setPreservesAll();
314 }
315
releaseMemory()316 void BlockFrequencyInfoWrapperPass::releaseMemory() { BFI.releaseMemory(); }
317
runOnFunction(Function & F)318 bool BlockFrequencyInfoWrapperPass::runOnFunction(Function &F) {
319 BranchProbabilityInfo &BPI =
320 getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
321 LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
322 BFI.calculate(F, BPI, LI);
323 return false;
324 }
325
326 AnalysisKey BlockFrequencyAnalysis::Key;
run(Function & F,FunctionAnalysisManager & AM)327 BlockFrequencyInfo BlockFrequencyAnalysis::run(Function &F,
328 FunctionAnalysisManager &AM) {
329 BlockFrequencyInfo BFI;
330 BFI.calculate(F, AM.getResult<BranchProbabilityAnalysis>(F),
331 AM.getResult<LoopAnalysis>(F));
332 return BFI;
333 }
334
335 PreservedAnalyses
run(Function & F,FunctionAnalysisManager & AM)336 BlockFrequencyPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
337 OS << "Printing analysis results of BFI for function "
338 << "'" << F.getName() << "':"
339 << "\n";
340 AM.getResult<BlockFrequencyAnalysis>(F).print(OS);
341 return PreservedAnalyses::all();
342 }
343