1 //===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // Instrumentation-based profile-guided optimization 11 // 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H 15 #define LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H 16 17 #include "CGBuilder.h" 18 #include "CodeGenModule.h" 19 #include "CodeGenTypes.h" 20 #include "clang/Frontend/CodeGenOptions.h" 21 #include "llvm/ADT/StringMap.h" 22 #include "llvm/Support/MemoryBuffer.h" 23 #include <memory> 24 25 namespace clang { 26 namespace CodeGen { 27 class RegionCounter; 28 29 /// Per-function PGO state. This class should generally not be used directly, 30 /// but instead through the CodeGenFunction and RegionCounter types. 31 class CodeGenPGO { 32 private: 33 CodeGenModule &CGM; 34 std::string FuncName; 35 llvm::GlobalVariable *FuncNameVar; 36 37 unsigned NumRegionCounters; 38 uint64_t FunctionHash; 39 std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap; 40 std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap; 41 std::vector<uint64_t> RegionCounts; 42 uint64_t CurrentRegionCount; 43 /// \brief A flag that is set to true when this function doesn't need 44 /// to have coverage mapping data. 45 bool SkipCoverageMapping; 46 47 public: CodeGenPGO(CodeGenModule & CGM)48 CodeGenPGO(CodeGenModule &CGM) 49 : CGM(CGM), NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0), 50 SkipCoverageMapping(false) {} 51 52 /// Whether or not we have PGO region data for the current function. This is 53 /// false both when we have no data at all and when our data has been 54 /// discarded. haveRegionCounts()55 bool haveRegionCounts() const { return !RegionCounts.empty(); } 56 57 /// Return the counter value of the current region. getCurrentRegionCount()58 uint64_t getCurrentRegionCount() const { return CurrentRegionCount; } 59 60 /// Set the counter value for the current region. This is used to keep track 61 /// of changes to the most recent counter from control flow and non-local 62 /// exits. setCurrentRegionCount(uint64_t Count)63 void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; } 64 65 /// Indicate that the current region is never reached, and thus should have a 66 /// counter value of zero. This is important so that subsequent regions can 67 /// correctly track their parent counts. setCurrentRegionUnreachable()68 void setCurrentRegionUnreachable() { setCurrentRegionCount(0); } 69 70 /// Check if an execution count is known for a given statement. If so, return 71 /// true and put the value in Count; else return false. getStmtCount(const Stmt * S,uint64_t & Count)72 bool getStmtCount(const Stmt *S, uint64_t &Count) { 73 if (!StmtCountMap) 74 return false; 75 llvm::DenseMap<const Stmt*, uint64_t>::const_iterator 76 I = StmtCountMap->find(S); 77 if (I == StmtCountMap->end()) 78 return false; 79 Count = I->second; 80 return true; 81 } 82 83 /// If the execution count for the current statement is known, record that 84 /// as the current count. setCurrentStmt(const Stmt * S)85 void setCurrentStmt(const Stmt *S) { 86 uint64_t Count; 87 if (getStmtCount(S, Count)) 88 setCurrentRegionCount(Count); 89 } 90 91 /// Calculate branch weights appropriate for PGO data 92 llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount); 93 llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights); 94 llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt); 95 96 /// Check if we need to emit coverage mapping for a given declaration 97 void checkGlobalDecl(GlobalDecl GD); 98 /// Assign counters to regions and configure them for PGO of a given 99 /// function. Does nothing if instrumentation is not enabled and either 100 /// generates global variables or associates PGO data with each of the 101 /// counters depending on whether we are generating or using instrumentation. 102 void assignRegionCounters(const Decl *D, llvm::Function *Fn); 103 /// Emit a coverage mapping range with a counter zero 104 /// for an unused declaration. 105 void emitEmptyCounterMapping(const Decl *D, StringRef FuncName, 106 llvm::GlobalValue::LinkageTypes Linkage); 107 private: 108 void setFuncName(llvm::Function *Fn); 109 void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage); 110 void createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage); 111 void mapRegionCounters(const Decl *D); 112 void computeRegionCounts(const Decl *D); 113 void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, 114 llvm::Function *Fn); 115 void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader, 116 bool IsInMainFile); 117 void emitCounterVariables(); 118 void emitCounterRegionMapping(const Decl *D); 119 120 /// Emit code to increment the counter at the given index 121 void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter); 122 123 /// Return the region counter for the given statement. This should only be 124 /// called on statements that have a dedicated counter. getRegionCounter(const Stmt * S)125 unsigned getRegionCounter(const Stmt *S) { 126 if (!RegionCounterMap) 127 return 0; 128 return (*RegionCounterMap)[S]; 129 } 130 131 /// Return the region count for the counter at the given index. getRegionCount(unsigned Counter)132 uint64_t getRegionCount(unsigned Counter) { 133 if (!haveRegionCounts()) 134 return 0; 135 return RegionCounts[Counter]; 136 } 137 138 friend class RegionCounter; 139 }; 140 141 /// A counter for a particular region. This is the primary interface through 142 /// which clients manage PGO counters and their values. 143 class RegionCounter { 144 CodeGenPGO *PGO; 145 unsigned Counter; 146 uint64_t Count; 147 uint64_t ParentCount; 148 uint64_t RegionCount; 149 int64_t Adjust; 150 RegionCounter(CodeGenPGO & PGO,unsigned CounterIndex)151 RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex) 152 : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)), 153 ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} 154 155 public: RegionCounter(CodeGenPGO & PGO,const Stmt * S)156 RegionCounter(CodeGenPGO &PGO, const Stmt *S) 157 : PGO(&PGO), Counter(PGO.getRegionCounter(S)), 158 Count(PGO.getRegionCount(Counter)), 159 ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} 160 161 /// Get the value of the counter. In most cases this is the number of times 162 /// the region of the counter was entered, but for switch labels it's the 163 /// number of direct jumps to that label. getCount()164 uint64_t getCount() const { return Count; } 165 166 /// Get the value of the counter with adjustments applied. Adjustments occur 167 /// when control enters or leaves the region abnormally; i.e., if there is a 168 /// jump to a label within the region, or if the function can return from 169 /// within the region. The adjusted count, then, is the value of the counter 170 /// at the end of the region. getAdjustedCount()171 uint64_t getAdjustedCount() const { 172 return Count + Adjust; 173 } 174 175 /// Get the value of the counter in this region's parent, i.e., the region 176 /// that was active when this region began. This is useful for deriving 177 /// counts in implicitly counted regions, like the false case of a condition 178 /// or the normal exits of a loop. getParentCount()179 uint64_t getParentCount() const { return ParentCount; } 180 181 /// Activate the counter by emitting an increment and starting to track 182 /// adjustments. If AddIncomingFallThrough is true, the current region count 183 /// will be added to the counter for the purposes of tracking the region. 184 void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) { 185 beginRegion(AddIncomingFallThrough); 186 PGO->emitCounterIncrement(Builder, Counter); 187 } 188 void beginRegion(bool AddIncomingFallThrough=false) { 189 RegionCount = Count; 190 if (AddIncomingFallThrough) 191 RegionCount += PGO->getCurrentRegionCount(); 192 PGO->setCurrentRegionCount(RegionCount); 193 } 194 195 /// For counters on boolean branches, begins tracking adjustments for the 196 /// uncounted path. beginElseRegion()197 void beginElseRegion() { 198 RegionCount = ParentCount - Count; 199 PGO->setCurrentRegionCount(RegionCount); 200 } 201 202 /// Reset the current region count. setCurrentRegionCount(uint64_t CurrentCount)203 void setCurrentRegionCount(uint64_t CurrentCount) { 204 RegionCount = CurrentCount; 205 PGO->setCurrentRegionCount(RegionCount); 206 } 207 208 /// Adjust for non-local control flow after emitting a subexpression or 209 /// substatement. This must be called to account for constructs such as gotos, 210 /// labels, and returns, so that we can ensure that our region's count is 211 /// correct in the code that follows. adjustForControlFlow()212 void adjustForControlFlow() { 213 Adjust += PGO->getCurrentRegionCount() - RegionCount; 214 // Reset the region count in case this is called again later. 215 RegionCount = PGO->getCurrentRegionCount(); 216 } 217 218 /// Commit all adjustments to the current region. If the region is a loop, 219 /// the LoopAdjust value should be the count of all the breaks and continues 220 /// from the loop, to compensate for those counts being deducted from the 221 /// adjustments for the body of the loop. applyAdjustmentsToRegion(uint64_t LoopAdjust)222 void applyAdjustmentsToRegion(uint64_t LoopAdjust) { 223 PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust); 224 } 225 }; 226 227 } // end namespace CodeGen 228 } // end namespace clang 229 230 #endif 231