1 //===- InlineAdvisor.h - Inlining decision making abstraction -*- C++ ---*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 #ifndef LLVM_ANALYSIS_INLINEADVISOR_H 10 #define LLVM_ANALYSIS_INLINEADVISOR_H 11 12 #include "llvm/Analysis/CGSCCPassManager.h" 13 #include "llvm/Analysis/InlineCost.h" 14 #include "llvm/Analysis/LazyCallGraph.h" 15 #include "llvm/Config/llvm-config.h" 16 #include "llvm/IR/PassManager.h" 17 #include <memory> 18 19 namespace llvm { 20 class BasicBlock; 21 class CallBase; 22 class Function; 23 class Module; 24 class OptimizationRemark; 25 class ImportedFunctionsInliningStatistics; 26 class OptimizationRemarkEmitter; 27 struct ReplayInlinerSettings; 28 29 /// There are 3 scenarios we can use the InlineAdvisor: 30 /// - Default - use manual heuristics. 31 /// 32 /// - Release mode, the expected mode for production, day to day deployments. 33 /// In this mode, when building the compiler, we also compile a pre-trained ML 34 /// model to native code, and link it as a static library. This mode has low 35 /// overhead and no additional dependencies for the compiler runtime. 36 /// 37 /// - Development mode, for training new models. 38 /// In this mode, we trade off runtime performance for flexibility. This mode 39 /// requires the full C Tensorflow API library, and evaluates models 40 /// dynamically. This mode also permits generating training logs, for offline 41 /// training. 42 enum class InliningAdvisorMode : int { Default, Release, Development }; 43 44 // Each entry represents an inline driver. 45 enum class InlinePass : int { 46 AlwaysInliner, 47 CGSCCInliner, 48 EarlyInliner, 49 ModuleInliner, 50 MLInliner, 51 ReplayCGSCCInliner, 52 ReplaySampleProfileInliner, 53 SampleProfileInliner, 54 }; 55 56 /// Provides context on when an inline advisor is constructed in the pipeline 57 /// (e.g., link phase, inline driver). 58 struct InlineContext { 59 ThinOrFullLTOPhase LTOPhase; 60 61 InlinePass Pass; 62 }; 63 64 std::string AnnotateInlinePassName(InlineContext IC); 65 66 class InlineAdvisor; 67 /// Capture state between an inlining decision having had been made, and 68 /// its impact being observable. When collecting model training data, this 69 /// allows recording features/decisions/partial reward data sets. 70 /// 71 /// Derivations of this type are expected to be tightly coupled with their 72 /// InliningAdvisors. The base type implements the minimal contractual 73 /// obligations. 74 class InlineAdvice { 75 public: 76 InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 77 OptimizationRemarkEmitter &ORE, bool IsInliningRecommended); 78 79 InlineAdvice(InlineAdvice &&) = delete; 80 InlineAdvice(const InlineAdvice &) = delete; 81 virtual ~InlineAdvice() { 82 assert(Recorded && "InlineAdvice should have been informed of the " 83 "inliner's decision in all cases"); 84 } 85 86 /// Exactly one of the record* APIs must be called. Implementers may extend 87 /// behavior by implementing the corresponding record*Impl. 88 /// 89 /// Call after inlining succeeded, and did not result in deleting the callee. 90 void recordInlining(); 91 92 /// Call after inlining succeeded, and results in the callee being 93 /// delete-able, meaning, it has no more users, and will be cleaned up 94 /// subsequently. 95 void recordInliningWithCalleeDeleted(); 96 97 /// Call after the decision for a call site was to not inline. 98 void recordUnsuccessfulInlining(const InlineResult &Result) { 99 markRecorded(); 100 recordUnsuccessfulInliningImpl(Result); 101 } 102 103 /// Call to indicate inlining was not attempted. 104 void recordUnattemptedInlining() { 105 markRecorded(); 106 recordUnattemptedInliningImpl(); 107 } 108 109 /// Get the inlining recommendation. 110 bool isInliningRecommended() const { return IsInliningRecommended; } 111 const DebugLoc &getOriginalCallSiteDebugLoc() const { return DLoc; } 112 const BasicBlock *getOriginalCallSiteBasicBlock() const { return Block; } 113 114 protected: 115 virtual void recordInliningImpl() {} 116 virtual void recordInliningWithCalleeDeletedImpl() {} 117 virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result) {} 118 virtual void recordUnattemptedInliningImpl() {} 119 120 InlineAdvisor *const Advisor; 121 /// Caller and Callee are pre-inlining. 122 Function *const Caller; 123 Function *const Callee; 124 125 // Capture the context of CB before inlining, as a successful inlining may 126 // change that context, and we want to report success or failure in the 127 // original context. 128 const DebugLoc DLoc; 129 const BasicBlock *const Block; 130 OptimizationRemarkEmitter &ORE; 131 const bool IsInliningRecommended; 132 133 private: 134 void markRecorded() { 135 assert(!Recorded && "Recording should happen exactly once"); 136 Recorded = true; 137 } 138 void recordInlineStatsIfNeeded(); 139 140 bool Recorded = false; 141 }; 142 143 class DefaultInlineAdvice : public InlineAdvice { 144 public: 145 DefaultInlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 146 Optional<InlineCost> OIC, OptimizationRemarkEmitter &ORE, 147 bool EmitRemarks = true) 148 : InlineAdvice(Advisor, CB, ORE, OIC.has_value()), OriginalCB(&CB), 149 OIC(OIC), EmitRemarks(EmitRemarks) {} 150 151 private: 152 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override; 153 void recordInliningWithCalleeDeletedImpl() override; 154 void recordInliningImpl() override; 155 156 private: 157 CallBase *const OriginalCB; 158 Optional<InlineCost> OIC; 159 bool EmitRemarks; 160 }; 161 162 /// Interface for deciding whether to inline a call site or not. 163 class InlineAdvisor { 164 public: 165 InlineAdvisor(InlineAdvisor &&) = delete; 166 virtual ~InlineAdvisor(); 167 168 /// Get an InlineAdvice containing a recommendation on whether to 169 /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to 170 /// be up-to-date wrt previous inlining decisions. \p MandatoryOnly indicates 171 /// only mandatory (always-inline) call sites should be recommended - this 172 /// allows the InlineAdvisor track such inlininings. 173 /// Returns: 174 /// - An InlineAdvice with the inlining recommendation. 175 /// - Null when no recommendation is made (https://reviews.llvm.org/D110658). 176 /// TODO: Consider removing the Null return scenario by incorporating the 177 /// SampleProfile inliner into an InlineAdvisor 178 std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB, 179 bool MandatoryOnly = false); 180 181 /// This must be called when the Inliner pass is entered, to allow the 182 /// InlineAdvisor update internal state, as result of function passes run 183 /// between Inliner pass runs (for the same module). 184 virtual void onPassEntry(LazyCallGraph::SCC *SCC = nullptr) {} 185 186 /// This must be called when the Inliner pass is exited, as function passes 187 /// may be run subsequently. This allows an implementation of InlineAdvisor 188 /// to prepare for a partial update, based on the optional SCC. 189 virtual void onPassExit(LazyCallGraph::SCC *SCC = nullptr) {} 190 191 /// Support for printer pass 192 virtual void print(raw_ostream &OS) const { 193 OS << "Unimplemented InlineAdvisor print\n"; 194 } 195 196 /// NOTE pass name is annotated only when inline advisor constructor provides InlineContext. 197 const char *getAnnotatedInlinePassName() const { 198 return AnnotatedInlinePassName.c_str(); 199 } 200 201 protected: 202 InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, 203 Optional<InlineContext> IC = NoneType::None); 204 virtual std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) = 0; 205 virtual std::unique_ptr<InlineAdvice> getMandatoryAdvice(CallBase &CB, 206 bool Advice); 207 208 Module &M; 209 FunctionAnalysisManager &FAM; 210 const Optional<InlineContext> IC; 211 const std::string AnnotatedInlinePassName; 212 std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats; 213 214 enum class MandatoryInliningKind { NotMandatory, Always, Never }; 215 216 static MandatoryInliningKind getMandatoryKind(CallBase &CB, 217 FunctionAnalysisManager &FAM, 218 OptimizationRemarkEmitter &ORE); 219 220 OptimizationRemarkEmitter &getCallerORE(CallBase &CB); 221 222 private: 223 friend class InlineAdvice; 224 }; 225 226 /// The default (manual heuristics) implementation of the InlineAdvisor. This 227 /// implementation does not need to keep state between inliner pass runs, and is 228 /// reusable as-is for inliner pass test scenarios, as well as for regular use. 229 class DefaultInlineAdvisor : public InlineAdvisor { 230 public: 231 DefaultInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, 232 InlineParams Params, InlineContext IC) 233 : InlineAdvisor(M, FAM, IC), Params(Params) {} 234 235 private: 236 std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override; 237 238 InlineParams Params; 239 }; 240 241 /// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor 242 /// needs to capture state right before inlining commences over a module. 243 class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> { 244 public: 245 static AnalysisKey Key; 246 InlineAdvisorAnalysis() = default; 247 struct Result { 248 Result(Module &M, ModuleAnalysisManager &MAM) : M(M), MAM(MAM) {} 249 bool invalidate(Module &, const PreservedAnalyses &PA, 250 ModuleAnalysisManager::Invalidator &) { 251 // Check whether the analysis has been explicitly invalidated. Otherwise, 252 // it's stateless and remains preserved. 253 auto PAC = PA.getChecker<InlineAdvisorAnalysis>(); 254 return !PAC.preservedWhenStateless(); 255 } 256 bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, 257 const ReplayInlinerSettings &ReplaySettings, 258 InlineContext IC); 259 InlineAdvisor *getAdvisor() const { return Advisor.get(); } 260 261 private: 262 Module &M; 263 ModuleAnalysisManager &MAM; 264 std::unique_ptr<InlineAdvisor> Advisor; 265 }; 266 267 Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); } 268 }; 269 270 /// Printer pass for the FunctionPropertiesAnalysis results. 271 class InlineAdvisorAnalysisPrinterPass 272 : public PassInfoMixin<InlineAdvisorAnalysisPrinterPass> { 273 raw_ostream &OS; 274 275 public: 276 explicit InlineAdvisorAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {} 277 278 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); 279 280 PreservedAnalyses run(LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, 281 LazyCallGraph &CG, CGSCCUpdateResult &UR); 282 }; 283 284 std::unique_ptr<InlineAdvisor> 285 getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM); 286 287 std::unique_ptr<InlineAdvisor> 288 getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, 289 std::function<bool(CallBase &)> GetDefaultAdvice); 290 291 // Default (manual policy) decision making helper APIs. Shared with the legacy 292 // pass manager inliner. 293 294 /// Return the cost only if the inliner should attempt to inline at the given 295 /// CallSite. If we return the cost, we will emit an optimisation remark later 296 /// using that cost, so we won't do so from this function. Return None if 297 /// inlining should not be attempted. 298 Optional<InlineCost> 299 shouldInline(CallBase &CB, function_ref<InlineCost(CallBase &CB)> GetInlineCost, 300 OptimizationRemarkEmitter &ORE, bool EnableDeferral = true); 301 302 /// Emit ORE message. 303 void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, 304 const BasicBlock *Block, const Function &Callee, 305 const Function &Caller, bool IsMandatory, 306 function_ref<void(OptimizationRemark &)> ExtraContext = {}, 307 const char *PassName = nullptr); 308 309 /// Emit ORE message based in cost (default heuristic). 310 void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, 311 const BasicBlock *Block, const Function &Callee, 312 const Function &Caller, const InlineCost &IC, 313 bool ForProfileContext = false, 314 const char *PassName = nullptr); 315 316 /// Add location info to ORE message. 317 void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc); 318 319 /// Set the inline-remark attribute. 320 void setInlineRemark(CallBase &CB, StringRef Message); 321 322 /// Utility for extracting the inline cost message to a string. 323 std::string inlineCostStr(const InlineCost &IC); 324 } // namespace llvm 325 #endif // LLVM_ANALYSIS_INLINEADVISOR_H 326