1 //===- InlineAdvisor.h - Inlining decision making abstraction -*- C++ ---*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 #ifndef LLVM_INLINEADVISOR_H_ 10 #define LLVM_INLINEADVISOR_H_ 11 12 #include <memory> 13 #include <unordered_set> 14 #include <vector> 15 16 #include "llvm/Analysis/InlineCost.h" 17 #include "llvm/IR/PassManager.h" 18 19 namespace llvm { 20 class BasicBlock; 21 class CallBase; 22 class Function; 23 class Module; 24 class OptimizationRemarkEmitter; 25 26 /// There are 3 scenarios we can use the InlineAdvisor: 27 /// - Default - use manual heuristics. 28 /// 29 /// - Release mode, the expected mode for production, day to day deployments. 30 /// In this mode, when building the compiler, we also compile a pre-trained ML 31 /// model to native code, and link it as a static library. This mode has low 32 /// overhead and no additional dependencies for the compiler runtime. 33 /// 34 /// - Development mode, for training new models. 35 /// In this mode, we trade off runtime performance for flexibility. This mode 36 /// requires the full C Tensorflow API library, and evaluates models 37 /// dynamically. This mode also permits generating training logs, for offline 38 /// training. 39 enum class InliningAdvisorMode : int { Default, Release, Development }; 40 41 class InlineAdvisor; 42 /// Capture state between an inlining decision having had been made, and 43 /// its impact being observable. When collecting model training data, this 44 /// allows recording features/decisions/partial reward data sets. 45 /// 46 /// Derivations of this type are expected to be tightly coupled with their 47 /// InliningAdvisors. The base type implements the minimal contractual 48 /// obligations. 49 class InlineAdvice { 50 public: 51 InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 52 OptimizationRemarkEmitter &ORE, bool IsInliningRecommended); 53 54 InlineAdvice(InlineAdvice &&) = delete; 55 InlineAdvice(const InlineAdvice &) = delete; ~InlineAdvice()56 virtual ~InlineAdvice() { 57 assert(Recorded && "InlineAdvice should have been informed of the " 58 "inliner's decision in all cases"); 59 } 60 61 /// Exactly one of the record* APIs must be called. Implementers may extend 62 /// behavior by implementing the corresponding record*Impl. 63 /// 64 /// Call after inlining succeeded, and did not result in deleting the callee. recordInlining()65 void recordInlining() { 66 markRecorded(); 67 recordInliningImpl(); 68 } 69 70 /// Call after inlining succeeded, and resulted in deleting the callee. 71 void recordInliningWithCalleeDeleted(); 72 73 /// Call after the decision for a call site was to not inline. recordUnsuccessfulInlining(const InlineResult & Result)74 void recordUnsuccessfulInlining(const InlineResult &Result) { 75 markRecorded(); 76 recordUnsuccessfulInliningImpl(Result); 77 } 78 79 /// Call to indicate inlining was not attempted. recordUnattemptedInlining()80 void recordUnattemptedInlining() { 81 markRecorded(); 82 recordUnattemptedInliningImpl(); 83 } 84 85 /// Get the inlining recommendation. isInliningRecommended()86 bool isInliningRecommended() const { return IsInliningRecommended; } getOriginalCallSiteDebugLoc()87 const DebugLoc &getOriginalCallSiteDebugLoc() const { return DLoc; } getOriginalCallSiteBasicBlock()88 const BasicBlock *getOriginalCallSiteBasicBlock() const { return Block; } 89 90 protected: recordInliningImpl()91 virtual void recordInliningImpl() {} recordInliningWithCalleeDeletedImpl()92 virtual void recordInliningWithCalleeDeletedImpl() {} recordUnsuccessfulInliningImpl(const InlineResult & Result)93 virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result) {} recordUnattemptedInliningImpl()94 virtual void recordUnattemptedInliningImpl() {} 95 96 InlineAdvisor *const Advisor; 97 /// Caller and Callee are pre-inlining. 98 Function *const Caller; 99 Function *const Callee; 100 101 // Capture the context of CB before inlining, as a successful inlining may 102 // change that context, and we want to report success or failure in the 103 // original context. 104 const DebugLoc DLoc; 105 const BasicBlock *const Block; 106 OptimizationRemarkEmitter &ORE; 107 const bool IsInliningRecommended; 108 109 private: markRecorded()110 void markRecorded() { 111 assert(!Recorded && "Recording should happen exactly once"); 112 Recorded = true; 113 } 114 115 bool Recorded = false; 116 }; 117 118 /// Interface for deciding whether to inline a call site or not. 119 class InlineAdvisor { 120 public: 121 InlineAdvisor(InlineAdvisor &&) = delete; ~InlineAdvisor()122 virtual ~InlineAdvisor() { freeDeletedFunctions(); } 123 124 /// Get an InlineAdvice containing a recommendation on whether to 125 /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to 126 /// be up-to-date wrt previous inlining decisions. 127 /// Returns an InlineAdvice with the inlining recommendation. 128 virtual std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB) = 0; 129 130 /// This must be called when the Inliner pass is entered, to allow the 131 /// InlineAdvisor update internal state, as result of function passes run 132 /// between Inliner pass runs (for the same module). onPassEntry()133 virtual void onPassEntry() {} 134 135 /// This must be called when the Inliner pass is exited, as function passes 136 /// may be run subsequently. This allows an implementation of InlineAdvisor 137 /// to prepare for a partial update. onPassExit()138 virtual void onPassExit() {} 139 140 protected: InlineAdvisor(FunctionAnalysisManager & FAM)141 InlineAdvisor(FunctionAnalysisManager &FAM) : FAM(FAM) {} 142 143 FunctionAnalysisManager &FAM; 144 145 /// We may want to defer deleting functions to after the inlining for a whole 146 /// module has finished. This allows us to reliably use function pointers as 147 /// unique identifiers, as an efficient implementation detail of the 148 /// InlineAdvisor. Otherwise, it is possible the memory allocator 149 /// re-allocate Function objects at the same address of a deleted Function; 150 /// and Functions are potentially created during the function passes called 151 /// after each SCC inlining (e.g. argument promotion does that). 152 void freeDeletedFunctions(); 153 isFunctionDeleted(const Function * F)154 bool isFunctionDeleted(const Function *F) const { 155 return DeletedFunctions.count(F); 156 } 157 158 private: 159 friend class InlineAdvice; 160 void markFunctionAsDeleted(Function *F); 161 std::unordered_set<const Function *> DeletedFunctions; 162 }; 163 164 /// The default (manual heuristics) implementation of the InlineAdvisor. This 165 /// implementation does not need to keep state between inliner pass runs, and is 166 /// reusable as-is for inliner pass test scenarios, as well as for regular use. 167 class DefaultInlineAdvisor : public InlineAdvisor { 168 public: DefaultInlineAdvisor(FunctionAnalysisManager & FAM,InlineParams Params)169 DefaultInlineAdvisor(FunctionAnalysisManager &FAM, InlineParams Params) 170 : InlineAdvisor(FAM), Params(Params) {} 171 172 private: 173 std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB) override; 174 onPassExit()175 void onPassExit() override { freeDeletedFunctions(); } 176 177 InlineParams Params; 178 }; 179 180 /// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor 181 /// needs to capture state right before inlining commences over a module. 182 class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> { 183 public: 184 static AnalysisKey Key; 185 InlineAdvisorAnalysis() = default; 186 struct Result { ResultResult187 Result(Module &M, ModuleAnalysisManager &MAM) : M(M), MAM(MAM) {} invalidateResult188 bool invalidate(Module &, const PreservedAnalyses &, 189 ModuleAnalysisManager::Invalidator &) { 190 // InlineAdvisor must be preserved across analysis invalidations. 191 return false; 192 } 193 bool tryCreate(InlineParams Params, InliningAdvisorMode Mode); getAdvisorResult194 InlineAdvisor *getAdvisor() const { return Advisor.get(); } clearResult195 void clear() { Advisor.reset(); } 196 197 private: 198 Module &M; 199 ModuleAnalysisManager &MAM; 200 std::unique_ptr<InlineAdvisor> Advisor; 201 }; 202 run(Module & M,ModuleAnalysisManager & MAM)203 Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); } 204 }; 205 206 #ifdef LLVM_HAVE_TF_AOT 207 std::unique_ptr<InlineAdvisor> 208 getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM); 209 #endif 210 211 // Default (manual policy) decision making helper APIs. Shared with the legacy 212 // pass manager inliner. 213 214 /// Return the cost only if the inliner should attempt to inline at the given 215 /// CallSite. If we return the cost, we will emit an optimisation remark later 216 /// using that cost, so we won't do so from this function. Return None if 217 /// inlining should not be attempted. 218 Optional<InlineCost> 219 shouldInline(CallBase &CB, function_ref<InlineCost(CallBase &CB)> GetInlineCost, 220 OptimizationRemarkEmitter &ORE, bool EnableDeferral = true); 221 222 /// Emit ORE message. 223 void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, 224 const BasicBlock *Block, const Function &Callee, 225 const Function &Caller, const InlineCost &IC, 226 bool ForProfileContext = false, 227 const char *PassName = nullptr); 228 229 /// Add location info to ORE message. 230 void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc); 231 232 /// Set the inline-remark attribute. 233 void setInlineRemark(CallBase &CB, StringRef Message); 234 235 /// Utility for extracting the inline cost message to a string. 236 std::string inlineCostStr(const InlineCost &IC); 237 } // namespace llvm 238 #endif // LLVM_INLINEADVISOR_H_ 239