1 //===- InlineAdvisor.h - Inlining decision making abstraction -*- C++ ---*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 #ifndef LLVM_ANALYSIS_INLINEADVISOR_H 10 #define LLVM_ANALYSIS_INLINEADVISOR_H 11 12 #include "llvm/Analysis/InlineCost.h" 13 #include "llvm/Config/llvm-config.h" 14 #include "llvm/IR/PassManager.h" 15 #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" 16 #include <memory> 17 #include <unordered_set> 18 19 namespace llvm { 20 class BasicBlock; 21 class CallBase; 22 class Function; 23 class Module; 24 class OptimizationRemarkEmitter; 25 26 /// There are 3 scenarios we can use the InlineAdvisor: 27 /// - Default - use manual heuristics. 28 /// 29 /// - Release mode, the expected mode for production, day to day deployments. 30 /// In this mode, when building the compiler, we also compile a pre-trained ML 31 /// model to native code, and link it as a static library. This mode has low 32 /// overhead and no additional dependencies for the compiler runtime. 33 /// 34 /// - Development mode, for training new models. 35 /// In this mode, we trade off runtime performance for flexibility. This mode 36 /// requires the full C Tensorflow API library, and evaluates models 37 /// dynamically. This mode also permits generating training logs, for offline 38 /// training. 39 enum class InliningAdvisorMode : int { 40 Default, 41 Release, 42 Development 43 }; 44 45 class InlineAdvisor; 46 /// Capture state between an inlining decision having had been made, and 47 /// its impact being observable. When collecting model training data, this 48 /// allows recording features/decisions/partial reward data sets. 49 /// 50 /// Derivations of this type are expected to be tightly coupled with their 51 /// InliningAdvisors. The base type implements the minimal contractual 52 /// obligations. 53 class InlineAdvice { 54 public: 55 InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 56 OptimizationRemarkEmitter &ORE, bool IsInliningRecommended); 57 58 InlineAdvice(InlineAdvice &&) = delete; 59 InlineAdvice(const InlineAdvice &) = delete; ~InlineAdvice()60 virtual ~InlineAdvice() { 61 assert(Recorded && "InlineAdvice should have been informed of the " 62 "inliner's decision in all cases"); 63 } 64 65 /// Exactly one of the record* APIs must be called. Implementers may extend 66 /// behavior by implementing the corresponding record*Impl. 67 /// 68 /// Call after inlining succeeded, and did not result in deleting the callee. 69 void recordInlining(); 70 71 /// Call after inlining succeeded, and resulted in deleting the callee. 72 void recordInliningWithCalleeDeleted(); 73 74 /// Call after the decision for a call site was to not inline. recordUnsuccessfulInlining(const InlineResult & Result)75 void recordUnsuccessfulInlining(const InlineResult &Result) { 76 markRecorded(); 77 recordUnsuccessfulInliningImpl(Result); 78 } 79 80 /// Call to indicate inlining was not attempted. recordUnattemptedInlining()81 void recordUnattemptedInlining() { 82 markRecorded(); 83 recordUnattemptedInliningImpl(); 84 } 85 86 /// Get the inlining recommendation. isInliningRecommended()87 bool isInliningRecommended() const { return IsInliningRecommended; } getOriginalCallSiteDebugLoc()88 const DebugLoc &getOriginalCallSiteDebugLoc() const { return DLoc; } getOriginalCallSiteBasicBlock()89 const BasicBlock *getOriginalCallSiteBasicBlock() const { return Block; } 90 91 protected: recordInliningImpl()92 virtual void recordInliningImpl() {} recordInliningWithCalleeDeletedImpl()93 virtual void recordInliningWithCalleeDeletedImpl() {} recordUnsuccessfulInliningImpl(const InlineResult & Result)94 virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result) {} recordUnattemptedInliningImpl()95 virtual void recordUnattemptedInliningImpl() {} 96 97 InlineAdvisor *const Advisor; 98 /// Caller and Callee are pre-inlining. 99 Function *const Caller; 100 Function *const Callee; 101 102 // Capture the context of CB before inlining, as a successful inlining may 103 // change that context, and we want to report success or failure in the 104 // original context. 105 const DebugLoc DLoc; 106 const BasicBlock *const Block; 107 OptimizationRemarkEmitter &ORE; 108 const bool IsInliningRecommended; 109 110 private: markRecorded()111 void markRecorded() { 112 assert(!Recorded && "Recording should happen exactly once"); 113 Recorded = true; 114 } 115 void recordInlineStatsIfNeeded(); 116 117 bool Recorded = false; 118 }; 119 120 class DefaultInlineAdvice : public InlineAdvice { 121 public: 122 DefaultInlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 123 Optional<InlineCost> OIC, OptimizationRemarkEmitter &ORE, 124 bool EmitRemarks = true) 125 : InlineAdvice(Advisor, CB, ORE, OIC.hasValue()), OriginalCB(&CB), 126 OIC(OIC), EmitRemarks(EmitRemarks) {} 127 128 private: 129 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override; 130 void recordInliningWithCalleeDeletedImpl() override; 131 void recordInliningImpl() override; 132 133 private: 134 CallBase *const OriginalCB; 135 Optional<InlineCost> OIC; 136 bool EmitRemarks; 137 }; 138 139 /// Interface for deciding whether to inline a call site or not. 140 class InlineAdvisor { 141 public: 142 InlineAdvisor(InlineAdvisor &&) = delete; 143 virtual ~InlineAdvisor(); 144 145 /// Get an InlineAdvice containing a recommendation on whether to 146 /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to 147 /// be up-to-date wrt previous inlining decisions. \p MandatoryOnly indicates 148 /// only mandatory (always-inline) call sites should be recommended - this 149 /// allows the InlineAdvisor track such inlininings. 150 /// Returns an InlineAdvice with the inlining recommendation. 151 std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB, 152 bool MandatoryOnly = false); 153 154 /// This must be called when the Inliner pass is entered, to allow the 155 /// InlineAdvisor update internal state, as result of function passes run 156 /// between Inliner pass runs (for the same module). onPassEntry()157 virtual void onPassEntry() {} 158 159 /// This must be called when the Inliner pass is exited, as function passes 160 /// may be run subsequently. This allows an implementation of InlineAdvisor 161 /// to prepare for a partial update. onPassExit()162 virtual void onPassExit() {} 163 164 protected: 165 InlineAdvisor(Module &M, FunctionAnalysisManager &FAM); 166 virtual std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) = 0; 167 virtual std::unique_ptr<InlineAdvice> getMandatoryAdvice(CallBase &CB, 168 bool Advice); 169 170 Module &M; 171 FunctionAnalysisManager &FAM; 172 std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats; 173 174 /// We may want to defer deleting functions to after the inlining for a whole 175 /// module has finished. This allows us to reliably use function pointers as 176 /// unique identifiers, as an efficient implementation detail of the 177 /// InlineAdvisor. Otherwise, it is possible the memory allocator 178 /// re-allocate Function objects at the same address of a deleted Function; 179 /// and Functions are potentially created during the function passes called 180 /// after each SCC inlining (e.g. argument promotion does that). 181 void freeDeletedFunctions(); 182 isFunctionDeleted(const Function * F)183 bool isFunctionDeleted(const Function *F) const { 184 return DeletedFunctions.count(F); 185 } 186 187 enum class MandatoryInliningKind { NotMandatory, Always, Never }; 188 189 static MandatoryInliningKind getMandatoryKind(CallBase &CB, 190 FunctionAnalysisManager &FAM, 191 OptimizationRemarkEmitter &ORE); 192 193 OptimizationRemarkEmitter &getCallerORE(CallBase &CB); 194 195 private: 196 friend class InlineAdvice; 197 void markFunctionAsDeleted(Function *F); 198 std::unordered_set<const Function *> DeletedFunctions; 199 }; 200 201 /// The default (manual heuristics) implementation of the InlineAdvisor. This 202 /// implementation does not need to keep state between inliner pass runs, and is 203 /// reusable as-is for inliner pass test scenarios, as well as for regular use. 204 class DefaultInlineAdvisor : public InlineAdvisor { 205 public: DefaultInlineAdvisor(Module & M,FunctionAnalysisManager & FAM,InlineParams Params)206 DefaultInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, 207 InlineParams Params) 208 : InlineAdvisor(M, FAM), Params(Params) {} 209 210 private: 211 std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override; 212 onPassExit()213 void onPassExit() override { freeDeletedFunctions(); } 214 215 InlineParams Params; 216 }; 217 218 /// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor 219 /// needs to capture state right before inlining commences over a module. 220 class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> { 221 public: 222 static AnalysisKey Key; 223 InlineAdvisorAnalysis() = default; 224 struct Result { ResultResult225 Result(Module &M, ModuleAnalysisManager &MAM) : M(M), MAM(MAM) {} invalidateResult226 bool invalidate(Module &, const PreservedAnalyses &, 227 ModuleAnalysisManager::Invalidator &) { 228 // InlineAdvisor must be preserved across analysis invalidations. 229 return false; 230 } 231 bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, 232 StringRef ReplayFile); getAdvisorResult233 InlineAdvisor *getAdvisor() const { return Advisor.get(); } clearResult234 void clear() { Advisor.reset(); } 235 236 private: 237 Module &M; 238 ModuleAnalysisManager &MAM; 239 std::unique_ptr<InlineAdvisor> Advisor; 240 }; 241 run(Module & M,ModuleAnalysisManager & MAM)242 Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); } 243 }; 244 245 #ifdef LLVM_HAVE_TF_AOT 246 std::unique_ptr<InlineAdvisor> 247 getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM); 248 #endif 249 250 #ifdef LLVM_HAVE_TF_API 251 std::unique_ptr<InlineAdvisor> 252 getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, 253 std::function<bool(CallBase &)> GetDefaultAdvice); 254 #endif 255 256 // Default (manual policy) decision making helper APIs. Shared with the legacy 257 // pass manager inliner. 258 259 /// Return the cost only if the inliner should attempt to inline at the given 260 /// CallSite. If we return the cost, we will emit an optimisation remark later 261 /// using that cost, so we won't do so from this function. Return None if 262 /// inlining should not be attempted. 263 Optional<InlineCost> 264 shouldInline(CallBase &CB, function_ref<InlineCost(CallBase &CB)> GetInlineCost, 265 OptimizationRemarkEmitter &ORE, bool EnableDeferral = true); 266 267 /// Emit ORE message. 268 void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, 269 const BasicBlock *Block, const Function &Callee, 270 const Function &Caller, const InlineCost &IC, 271 bool ForProfileContext = false, 272 const char *PassName = nullptr); 273 274 /// get call site location as string 275 std::string getCallSiteLocation(DebugLoc DLoc); 276 277 /// Add location info to ORE message. 278 void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc); 279 280 /// Set the inline-remark attribute. 281 void setInlineRemark(CallBase &CB, StringRef Message); 282 283 /// Utility for extracting the inline cost message to a string. 284 std::string inlineCostStr(const InlineCost &IC); 285 } // namespace llvm 286 #endif // LLVM_ANALYSIS_INLINEADVISOR_H 287