1 //===- InlineAdvisor.h - Inlining decision making abstraction -*- C++ ---*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 #ifndef LLVM_ANALYSIS_INLINEADVISOR_H 10 #define LLVM_ANALYSIS_INLINEADVISOR_H 11 12 #include "llvm/Analysis/InlineCost.h" 13 #include "llvm/Analysis/LazyCallGraph.h" 14 #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" 15 #include "llvm/Config/llvm-config.h" 16 #include "llvm/IR/PassManager.h" 17 #include <memory> 18 #include <unordered_set> 19 20 namespace llvm { 21 class BasicBlock; 22 class CallBase; 23 class Function; 24 class Module; 25 class OptimizationRemarkEmitter; 26 struct ReplayInlinerSettings; 27 28 /// There are 3 scenarios we can use the InlineAdvisor: 29 /// - Default - use manual heuristics. 30 /// 31 /// - Release mode, the expected mode for production, day to day deployments. 32 /// In this mode, when building the compiler, we also compile a pre-trained ML 33 /// model to native code, and link it as a static library. This mode has low 34 /// overhead and no additional dependencies for the compiler runtime. 35 /// 36 /// - Development mode, for training new models. 37 /// In this mode, we trade off runtime performance for flexibility. This mode 38 /// requires the full C Tensorflow API library, and evaluates models 39 /// dynamically. This mode also permits generating training logs, for offline 40 /// training. 41 enum class InliningAdvisorMode : int { Default, Release, Development }; 42 43 class InlineAdvisor; 44 /// Capture state between an inlining decision having had been made, and 45 /// its impact being observable. When collecting model training data, this 46 /// allows recording features/decisions/partial reward data sets. 47 /// 48 /// Derivations of this type are expected to be tightly coupled with their 49 /// InliningAdvisors. The base type implements the minimal contractual 50 /// obligations. 51 class InlineAdvice { 52 public: 53 InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 54 OptimizationRemarkEmitter &ORE, bool IsInliningRecommended); 55 56 InlineAdvice(InlineAdvice &&) = delete; 57 InlineAdvice(const InlineAdvice &) = delete; 58 virtual ~InlineAdvice() { 59 assert(Recorded && "InlineAdvice should have been informed of the " 60 "inliner's decision in all cases"); 61 } 62 63 /// Exactly one of the record* APIs must be called. Implementers may extend 64 /// behavior by implementing the corresponding record*Impl. 65 /// 66 /// Call after inlining succeeded, and did not result in deleting the callee. 67 void recordInlining(); 68 69 /// Call after inlining succeeded, and results in the callee being 70 /// delete-able, meaning, it has no more users, and will be cleaned up 71 /// subsequently. 72 void recordInliningWithCalleeDeleted(); 73 74 /// Call after the decision for a call site was to not inline. 75 void recordUnsuccessfulInlining(const InlineResult &Result) { 76 markRecorded(); 77 recordUnsuccessfulInliningImpl(Result); 78 } 79 80 /// Call to indicate inlining was not attempted. 81 void recordUnattemptedInlining() { 82 markRecorded(); 83 recordUnattemptedInliningImpl(); 84 } 85 86 /// Get the inlining recommendation. 87 bool isInliningRecommended() const { return IsInliningRecommended; } 88 const DebugLoc &getOriginalCallSiteDebugLoc() const { return DLoc; } 89 const BasicBlock *getOriginalCallSiteBasicBlock() const { return Block; } 90 91 protected: 92 virtual void recordInliningImpl() {} 93 virtual void recordInliningWithCalleeDeletedImpl() {} 94 virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result) {} 95 virtual void recordUnattemptedInliningImpl() {} 96 97 InlineAdvisor *const Advisor; 98 /// Caller and Callee are pre-inlining. 99 Function *const Caller; 100 Function *const Callee; 101 102 // Capture the context of CB before inlining, as a successful inlining may 103 // change that context, and we want to report success or failure in the 104 // original context. 105 const DebugLoc DLoc; 106 const BasicBlock *const Block; 107 OptimizationRemarkEmitter &ORE; 108 const bool IsInliningRecommended; 109 110 private: 111 void markRecorded() { 112 assert(!Recorded && "Recording should happen exactly once"); 113 Recorded = true; 114 } 115 void recordInlineStatsIfNeeded(); 116 117 bool Recorded = false; 118 }; 119 120 class DefaultInlineAdvice : public InlineAdvice { 121 public: 122 DefaultInlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 123 Optional<InlineCost> OIC, OptimizationRemarkEmitter &ORE, 124 bool EmitRemarks = true) 125 : InlineAdvice(Advisor, CB, ORE, OIC.hasValue()), OriginalCB(&CB), 126 OIC(OIC), EmitRemarks(EmitRemarks) {} 127 128 private: 129 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override; 130 void recordInliningWithCalleeDeletedImpl() override; 131 void recordInliningImpl() override; 132 133 private: 134 CallBase *const OriginalCB; 135 Optional<InlineCost> OIC; 136 bool EmitRemarks; 137 }; 138 139 /// Interface for deciding whether to inline a call site or not. 140 class InlineAdvisor { 141 public: 142 InlineAdvisor(InlineAdvisor &&) = delete; 143 virtual ~InlineAdvisor(); 144 145 /// Get an InlineAdvice containing a recommendation on whether to 146 /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to 147 /// be up-to-date wrt previous inlining decisions. \p MandatoryOnly indicates 148 /// only mandatory (always-inline) call sites should be recommended - this 149 /// allows the InlineAdvisor track such inlininings. 150 /// Returns: 151 /// - An InlineAdvice with the inlining recommendation. 152 /// - Null when no recommendation is made (https://reviews.llvm.org/D110658). 153 /// TODO: Consider removing the Null return scenario by incorporating the 154 /// SampleProfile inliner into an InlineAdvisor 155 std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB, 156 bool MandatoryOnly = false); 157 158 /// This must be called when the Inliner pass is entered, to allow the 159 /// InlineAdvisor update internal state, as result of function passes run 160 /// between Inliner pass runs (for the same module). 161 virtual void onPassEntry() {} 162 163 /// This must be called when the Inliner pass is exited, as function passes 164 /// may be run subsequently. This allows an implementation of InlineAdvisor 165 /// to prepare for a partial update, based on the optional SCC. 166 virtual void onPassExit(LazyCallGraph::SCC *SCC = nullptr) {} 167 168 /// Support for printer pass 169 virtual void print(raw_ostream &OS) const { 170 OS << "Unimplemented InlineAdvisor print\n"; 171 } 172 173 protected: 174 InlineAdvisor(Module &M, FunctionAnalysisManager &FAM); 175 virtual std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) = 0; 176 virtual std::unique_ptr<InlineAdvice> getMandatoryAdvice(CallBase &CB, 177 bool Advice); 178 179 Module &M; 180 FunctionAnalysisManager &FAM; 181 std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats; 182 183 enum class MandatoryInliningKind { NotMandatory, Always, Never }; 184 185 static MandatoryInliningKind getMandatoryKind(CallBase &CB, 186 FunctionAnalysisManager &FAM, 187 OptimizationRemarkEmitter &ORE); 188 189 OptimizationRemarkEmitter &getCallerORE(CallBase &CB); 190 191 private: 192 friend class InlineAdvice; 193 }; 194 195 /// The default (manual heuristics) implementation of the InlineAdvisor. This 196 /// implementation does not need to keep state between inliner pass runs, and is 197 /// reusable as-is for inliner pass test scenarios, as well as for regular use. 198 class DefaultInlineAdvisor : public InlineAdvisor { 199 public: 200 DefaultInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, 201 InlineParams Params) 202 : InlineAdvisor(M, FAM), Params(Params) {} 203 204 private: 205 std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override; 206 207 InlineParams Params; 208 }; 209 210 /// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor 211 /// needs to capture state right before inlining commences over a module. 212 class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> { 213 public: 214 static AnalysisKey Key; 215 InlineAdvisorAnalysis() = default; 216 struct Result { 217 Result(Module &M, ModuleAnalysisManager &MAM) : M(M), MAM(MAM) {} 218 bool invalidate(Module &, const PreservedAnalyses &PA, 219 ModuleAnalysisManager::Invalidator &) { 220 // Check whether the analysis has been explicitly invalidated. Otherwise, 221 // it's stateless and remains preserved. 222 auto PAC = PA.getChecker<InlineAdvisorAnalysis>(); 223 return !PAC.preservedWhenStateless(); 224 } 225 bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, 226 const ReplayInlinerSettings &ReplaySettings); 227 InlineAdvisor *getAdvisor() const { return Advisor.get(); } 228 229 private: 230 Module &M; 231 ModuleAnalysisManager &MAM; 232 std::unique_ptr<InlineAdvisor> Advisor; 233 }; 234 235 Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); } 236 }; 237 238 /// Printer pass for the FunctionPropertiesAnalysis results. 239 class InlineAdvisorAnalysisPrinterPass 240 : public PassInfoMixin<InlineAdvisorAnalysisPrinterPass> { 241 raw_ostream &OS; 242 243 public: 244 explicit InlineAdvisorAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {} 245 246 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); 247 }; 248 249 std::unique_ptr<InlineAdvisor> 250 getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM); 251 252 std::unique_ptr<InlineAdvisor> 253 getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, 254 std::function<bool(CallBase &)> GetDefaultAdvice); 255 256 // Default (manual policy) decision making helper APIs. Shared with the legacy 257 // pass manager inliner. 258 259 /// Return the cost only if the inliner should attempt to inline at the given 260 /// CallSite. If we return the cost, we will emit an optimisation remark later 261 /// using that cost, so we won't do so from this function. Return None if 262 /// inlining should not be attempted. 263 Optional<InlineCost> 264 shouldInline(CallBase &CB, function_ref<InlineCost(CallBase &CB)> GetInlineCost, 265 OptimizationRemarkEmitter &ORE, bool EnableDeferral = true); 266 267 /// Emit ORE message. 268 void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, 269 const BasicBlock *Block, const Function &Callee, 270 const Function &Caller, bool IsMandatory, 271 function_ref<void(OptimizationRemark &)> ExtraContext = {}, 272 const char *PassName = nullptr); 273 274 /// Emit ORE message based in cost (default heuristic). 275 void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, 276 const BasicBlock *Block, const Function &Callee, 277 const Function &Caller, const InlineCost &IC, 278 bool ForProfileContext = false, 279 const char *PassName = nullptr); 280 281 /// Add location info to ORE message. 282 void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc); 283 284 /// Set the inline-remark attribute. 285 void setInlineRemark(CallBase &CB, StringRef Message); 286 287 /// Utility for extracting the inline cost message to a string. 288 std::string inlineCostStr(const InlineCost &IC); 289 } // namespace llvm 290 #endif // LLVM_ANALYSIS_INLINEADVISOR_H 291