1 //===- Parsing, selection, and construction of pass pipelines -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
13 /// pipelines.
14 ///
15 //===----------------------------------------------------------------------===//
16
17 #include "llvm/Passes/PassBuilder.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
20 #include "llvm/Analysis/AliasSetTracker.h"
21 #include "llvm/Analysis/AssumptionCache.h"
22 #include "llvm/Analysis/BasicAliasAnalysis.h"
23 #include "llvm/Analysis/BlockFrequencyInfo.h"
24 #include "llvm/Analysis/BranchProbabilityInfo.h"
25 #include "llvm/Analysis/CFGPrinter.h"
26 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
27 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
28 #include "llvm/Analysis/CGSCCPassManager.h"
29 #include "llvm/Analysis/CallGraph.h"
30 #include "llvm/Analysis/DDG.h"
31 #include "llvm/Analysis/DDGPrinter.h"
32 #include "llvm/Analysis/Delinearization.h"
33 #include "llvm/Analysis/DemandedBits.h"
34 #include "llvm/Analysis/DependenceAnalysis.h"
35 #include "llvm/Analysis/DivergenceAnalysis.h"
36 #include "llvm/Analysis/DominanceFrontier.h"
37 #include "llvm/Analysis/FunctionPropertiesAnalysis.h"
38 #include "llvm/Analysis/GlobalsModRef.h"
39 #include "llvm/Analysis/IRSimilarityIdentifier.h"
40 #include "llvm/Analysis/IVUsers.h"
41 #include "llvm/Analysis/InlineAdvisor.h"
42 #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
43 #include "llvm/Analysis/InstCount.h"
44 #include "llvm/Analysis/LazyCallGraph.h"
45 #include "llvm/Analysis/LazyValueInfo.h"
46 #include "llvm/Analysis/Lint.h"
47 #include "llvm/Analysis/LoopAccessAnalysis.h"
48 #include "llvm/Analysis/LoopCacheAnalysis.h"
49 #include "llvm/Analysis/LoopInfo.h"
50 #include "llvm/Analysis/LoopNestAnalysis.h"
51 #include "llvm/Analysis/MemDerefPrinter.h"
52 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
53 #include "llvm/Analysis/MemorySSA.h"
54 #include "llvm/Analysis/ModuleDebugInfoPrinter.h"
55 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
56 #include "llvm/Analysis/MustExecute.h"
57 #include "llvm/Analysis/ObjCARCAliasAnalysis.h"
58 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
59 #include "llvm/Analysis/PhiValues.h"
60 #include "llvm/Analysis/PostDominators.h"
61 #include "llvm/Analysis/ProfileSummaryInfo.h"
62 #include "llvm/Analysis/RegionInfo.h"
63 #include "llvm/Analysis/ScalarEvolution.h"
64 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
65 #include "llvm/Analysis/ScopedNoAliasAA.h"
66 #include "llvm/Analysis/StackLifetime.h"
67 #include "llvm/Analysis/StackSafetyAnalysis.h"
68 #include "llvm/Analysis/TargetLibraryInfo.h"
69 #include "llvm/Analysis/TargetTransformInfo.h"
70 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
71 #include "llvm/IR/Dominators.h"
72 #include "llvm/IR/IRPrintingPasses.h"
73 #include "llvm/IR/PassManager.h"
74 #include "llvm/IR/PrintPasses.h"
75 #include "llvm/IR/SafepointIRVerifier.h"
76 #include "llvm/IR/Verifier.h"
77 #include "llvm/Support/CommandLine.h"
78 #include "llvm/Support/Debug.h"
79 #include "llvm/Support/ErrorHandling.h"
80 #include "llvm/Support/FormatVariadic.h"
81 #include "llvm/Support/Regex.h"
82 #include "llvm/Target/TargetMachine.h"
83 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
84 #include "llvm/Transforms/Coroutines/CoroCleanup.h"
85 #include "llvm/Transforms/Coroutines/CoroEarly.h"
86 #include "llvm/Transforms/Coroutines/CoroElide.h"
87 #include "llvm/Transforms/Coroutines/CoroSplit.h"
88 #include "llvm/Transforms/IPO/AlwaysInliner.h"
89 #include "llvm/Transforms/IPO/Annotation2Metadata.h"
90 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
91 #include "llvm/Transforms/IPO/Attributor.h"
92 #include "llvm/Transforms/IPO/BlockExtractor.h"
93 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
94 #include "llvm/Transforms/IPO/ConstantMerge.h"
95 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
96 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
97 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
98 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
99 #include "llvm/Transforms/IPO/FunctionAttrs.h"
100 #include "llvm/Transforms/IPO/FunctionImport.h"
101 #include "llvm/Transforms/IPO/GlobalDCE.h"
102 #include "llvm/Transforms/IPO/GlobalOpt.h"
103 #include "llvm/Transforms/IPO/GlobalSplit.h"
104 #include "llvm/Transforms/IPO/HotColdSplitting.h"
105 #include "llvm/Transforms/IPO/IROutliner.h"
106 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
107 #include "llvm/Transforms/IPO/Inliner.h"
108 #include "llvm/Transforms/IPO/Internalize.h"
109 #include "llvm/Transforms/IPO/LoopExtractor.h"
110 #include "llvm/Transforms/IPO/LowerTypeTests.h"
111 #include "llvm/Transforms/IPO/MergeFunctions.h"
112 #include "llvm/Transforms/IPO/OpenMPOpt.h"
113 #include "llvm/Transforms/IPO/PartialInlining.h"
114 #include "llvm/Transforms/IPO/SCCP.h"
115 #include "llvm/Transforms/IPO/SampleProfile.h"
116 #include "llvm/Transforms/IPO/SampleProfileProbe.h"
117 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
118 #include "llvm/Transforms/IPO/StripSymbols.h"
119 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
120 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
121 #include "llvm/Transforms/InstCombine/InstCombine.h"
122 #include "llvm/Transforms/Instrumentation.h"
123 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
124 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
125 #include "llvm/Transforms/Instrumentation/CGProfile.h"
126 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
127 #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
128 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
129 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
130 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
131 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
132 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
133 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
134 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
135 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
136 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
137 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
138 #include "llvm/Transforms/ObjCARC.h"
139 #include "llvm/Transforms/Scalar/ADCE.h"
140 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
141 #include "llvm/Transforms/Scalar/AnnotationRemarks.h"
142 #include "llvm/Transforms/Scalar/BDCE.h"
143 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
144 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
145 #include "llvm/Transforms/Scalar/ConstraintElimination.h"
146 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
147 #include "llvm/Transforms/Scalar/DCE.h"
148 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
149 #include "llvm/Transforms/Scalar/DivRemPairs.h"
150 #include "llvm/Transforms/Scalar/EarlyCSE.h"
151 #include "llvm/Transforms/Scalar/Float2Int.h"
152 #include "llvm/Transforms/Scalar/GVN.h"
153 #include "llvm/Transforms/Scalar/GuardWidening.h"
154 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
155 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
156 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
157 #include "llvm/Transforms/Scalar/InferAddressSpaces.h"
158 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
159 #include "llvm/Transforms/Scalar/JumpThreading.h"
160 #include "llvm/Transforms/Scalar/LICM.h"
161 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
162 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
163 #include "llvm/Transforms/Scalar/LoopDeletion.h"
164 #include "llvm/Transforms/Scalar/LoopDistribute.h"
165 #include "llvm/Transforms/Scalar/LoopFlatten.h"
166 #include "llvm/Transforms/Scalar/LoopFuse.h"
167 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
168 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
169 #include "llvm/Transforms/Scalar/LoopInterchange.h"
170 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
171 #include "llvm/Transforms/Scalar/LoopPassManager.h"
172 #include "llvm/Transforms/Scalar/LoopPredication.h"
173 #include "llvm/Transforms/Scalar/LoopReroll.h"
174 #include "llvm/Transforms/Scalar/LoopRotation.h"
175 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
176 #include "llvm/Transforms/Scalar/LoopSink.h"
177 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
178 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
179 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
180 #include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
181 #include "llvm/Transforms/Scalar/LowerAtomic.h"
182 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
183 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
184 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
185 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
186 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
187 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
188 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
189 #include "llvm/Transforms/Scalar/MergeICmps.h"
190 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
191 #include "llvm/Transforms/Scalar/NaryReassociate.h"
192 #include "llvm/Transforms/Scalar/NewGVN.h"
193 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
194 #include "llvm/Transforms/Scalar/Reassociate.h"
195 #include "llvm/Transforms/Scalar/Reg2Mem.h"
196 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
197 #include "llvm/Transforms/Scalar/SCCP.h"
198 #include "llvm/Transforms/Scalar/SROA.h"
199 #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
200 #include "llvm/Transforms/Scalar/Scalarizer.h"
201 #include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
202 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
203 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
204 #include "llvm/Transforms/Scalar/Sink.h"
205 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
206 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
207 #include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
208 #include "llvm/Transforms/Scalar/StructurizeCFG.h"
209 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
210 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
211 #include "llvm/Transforms/Utils/AddDiscriminators.h"
212 #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
213 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
214 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
215 #include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
216 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
217 #include "llvm/Transforms/Utils/FixIrreducible.h"
218 #include "llvm/Transforms/Utils/HelloWorld.h"
219 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
220 #include "llvm/Transforms/Utils/InstructionNamer.h"
221 #include "llvm/Transforms/Utils/LCSSA.h"
222 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
223 #include "llvm/Transforms/Utils/LoopSimplify.h"
224 #include "llvm/Transforms/Utils/LoopVersioning.h"
225 #include "llvm/Transforms/Utils/LowerInvoke.h"
226 #include "llvm/Transforms/Utils/LowerSwitch.h"
227 #include "llvm/Transforms/Utils/Mem2Reg.h"
228 #include "llvm/Transforms/Utils/MetaRenamer.h"
229 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
230 #include "llvm/Transforms/Utils/RelLookupTableConverter.h"
231 #include "llvm/Transforms/Utils/StripGCRelocates.h"
232 #include "llvm/Transforms/Utils/StripNonLineTableDebugInfo.h"
233 #include "llvm/Transforms/Utils/SymbolRewriter.h"
234 #include "llvm/Transforms/Utils/UnifyFunctionExitNodes.h"
235 #include "llvm/Transforms/Utils/UnifyLoopExits.h"
236 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
237 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
238 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
239 #include "llvm/Transforms/Vectorize/VectorCombine.h"
240
241 using namespace llvm;
242
243 static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
244 "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
245 cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
246 cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
247 "Heuristics-based inliner version."),
248 clEnumValN(InliningAdvisorMode::Development, "development",
249 "Use development mode (runtime-loadable model)."),
250 clEnumValN(InliningAdvisorMode::Release, "release",
251 "Use release mode (AOT-compiled model).")));
252
253 static cl::opt<bool> EnableSyntheticCounts(
254 "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
255 cl::desc("Run synthetic function entry count generation "
256 "pass"));
257
258 static const Regex DefaultAliasRegex(
259 "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
260
261 /// Flag to enable inline deferral during PGO.
262 static cl::opt<bool>
263 EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
264 cl::Hidden,
265 cl::desc("Enable inline deferral during PGO"));
266
267 static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
268 cl::Hidden, cl::ZeroOrMore,
269 cl::desc("Enable memory profiler"));
270
271 static cl::opt<bool> PerformMandatoryInliningsFirst(
272 "mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore,
273 cl::desc("Perform mandatory inlinings module-wide, before performing "
274 "inlining."));
275
276 static cl::opt<bool> EnableO3NonTrivialUnswitching(
277 "enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden,
278 cl::ZeroOrMore, cl::desc("Enable non-trivial loop unswitching for -O3"));
279
PipelineTuningOptions()280 PipelineTuningOptions::PipelineTuningOptions() {
281 LoopInterleaving = true;
282 LoopVectorization = true;
283 SLPVectorization = false;
284 LoopUnrolling = true;
285 ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
286 Coroutines = false;
287 LicmMssaOptCap = SetLicmMssaOptCap;
288 LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
289 CallGraphProfile = true;
290 MergeFunctions = false;
291 }
292
293 namespace llvm {
294 extern cl::opt<unsigned> MaxDevirtIterations;
295 extern cl::opt<bool> EnableConstraintElimination;
296 extern cl::opt<bool> EnableGVNHoist;
297 extern cl::opt<bool> EnableGVNSink;
298 extern cl::opt<bool> EnableHotColdSplit;
299 extern cl::opt<bool> EnableIROutliner;
300 extern cl::opt<bool> EnableOrderFileInstrumentation;
301 extern cl::opt<bool> EnableCHR;
302 extern cl::opt<bool> EnableLoopInterchange;
303 extern cl::opt<bool> EnableUnrollAndJam;
304 extern cl::opt<bool> EnableLoopFlatten;
305 extern cl::opt<bool> RunNewGVN;
306 extern cl::opt<bool> RunPartialInlining;
307 extern cl::opt<bool> ExtraVectorizerPasses;
308
309 extern cl::opt<bool> FlattenedProfileUsed;
310
311 extern cl::opt<AttributorRunOption> AttributorRun;
312 extern cl::opt<bool> EnableKnowledgeRetention;
313
314 extern cl::opt<bool> EnableMatrix;
315
316 extern cl::opt<bool> DisablePreInliner;
317 extern cl::opt<int> PreInlineThreshold;
318 } // namespace llvm
319
320 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O0 = {
321 /*SpeedLevel*/ 0,
322 /*SizeLevel*/ 0};
323 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O1 = {
324 /*SpeedLevel*/ 1,
325 /*SizeLevel*/ 0};
326 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O2 = {
327 /*SpeedLevel*/ 2,
328 /*SizeLevel*/ 0};
329 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O3 = {
330 /*SpeedLevel*/ 3,
331 /*SizeLevel*/ 0};
332 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Os = {
333 /*SpeedLevel*/ 2,
334 /*SizeLevel*/ 1};
335 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Oz = {
336 /*SpeedLevel*/ 2,
337 /*SizeLevel*/ 2};
338
339 namespace {
340
341 // The following passes/analyses have custom names, otherwise their name will
342 // include `(anonymous namespace)`. These are special since they are only for
343 // testing purposes and don't live in a header file.
344
345 /// No-op module pass which does nothing.
346 struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
run__anone50602ed0111::NoOpModulePass347 PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
348 return PreservedAnalyses::all();
349 }
350
name__anone50602ed0111::NoOpModulePass351 static StringRef name() { return "NoOpModulePass"; }
352 };
353
354 /// No-op module analysis.
355 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
356 friend AnalysisInfoMixin<NoOpModuleAnalysis>;
357 static AnalysisKey Key;
358
359 public:
360 struct Result {};
run(Module &,ModuleAnalysisManager &)361 Result run(Module &, ModuleAnalysisManager &) { return Result(); }
name()362 static StringRef name() { return "NoOpModuleAnalysis"; }
363 };
364
365 /// No-op CGSCC pass which does nothing.
366 struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
run__anone50602ed0111::NoOpCGSCCPass367 PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
368 LazyCallGraph &, CGSCCUpdateResult &UR) {
369 return PreservedAnalyses::all();
370 }
name__anone50602ed0111::NoOpCGSCCPass371 static StringRef name() { return "NoOpCGSCCPass"; }
372 };
373
374 /// No-op CGSCC analysis.
375 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
376 friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
377 static AnalysisKey Key;
378
379 public:
380 struct Result {};
run(LazyCallGraph::SCC &,CGSCCAnalysisManager &,LazyCallGraph & G)381 Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
382 return Result();
383 }
name()384 static StringRef name() { return "NoOpCGSCCAnalysis"; }
385 };
386
387 /// No-op function pass which does nothing.
388 struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
run__anone50602ed0111::NoOpFunctionPass389 PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
390 return PreservedAnalyses::all();
391 }
name__anone50602ed0111::NoOpFunctionPass392 static StringRef name() { return "NoOpFunctionPass"; }
393 };
394
395 /// No-op function analysis.
396 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
397 friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
398 static AnalysisKey Key;
399
400 public:
401 struct Result {};
run(Function &,FunctionAnalysisManager &)402 Result run(Function &, FunctionAnalysisManager &) { return Result(); }
name()403 static StringRef name() { return "NoOpFunctionAnalysis"; }
404 };
405
406 /// No-op loop pass which does nothing.
407 struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
run__anone50602ed0111::NoOpLoopPass408 PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
409 LoopStandardAnalysisResults &, LPMUpdater &) {
410 return PreservedAnalyses::all();
411 }
name__anone50602ed0111::NoOpLoopPass412 static StringRef name() { return "NoOpLoopPass"; }
413 };
414
415 /// No-op loop analysis.
416 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
417 friend AnalysisInfoMixin<NoOpLoopAnalysis>;
418 static AnalysisKey Key;
419
420 public:
421 struct Result {};
run(Loop &,LoopAnalysisManager &,LoopStandardAnalysisResults &)422 Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
423 return Result();
424 }
name()425 static StringRef name() { return "NoOpLoopAnalysis"; }
426 };
427
428 AnalysisKey NoOpModuleAnalysis::Key;
429 AnalysisKey NoOpCGSCCAnalysis::Key;
430 AnalysisKey NoOpFunctionAnalysis::Key;
431 AnalysisKey NoOpLoopAnalysis::Key;
432
433 /// Whether or not we should populate a PassInstrumentationCallbacks's class to
434 /// pass name map.
435 ///
436 /// This is for optimization purposes so we don't populate it if we never use
437 /// it. This should be updated if new pass instrumentation wants to use the map.
438 /// We currently only use this for --print-before/after.
shouldPopulateClassToPassNames()439 bool shouldPopulateClassToPassNames() {
440 return !printBeforePasses().empty() || !printAfterPasses().empty();
441 }
442
443 } // namespace
444
PassBuilder(TargetMachine * TM,PipelineTuningOptions PTO,Optional<PGOOptions> PGOOpt,PassInstrumentationCallbacks * PIC)445 PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO,
446 Optional<PGOOptions> PGOOpt,
447 PassInstrumentationCallbacks *PIC)
448 : TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
449 if (TM)
450 TM->registerPassBuilderCallbacks(*this);
451 if (PIC && shouldPopulateClassToPassNames()) {
452 #define MODULE_PASS(NAME, CREATE_PASS) \
453 PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
454 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
455 PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
456 #define FUNCTION_PASS(NAME, CREATE_PASS) \
457 PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
458 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
459 PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
460 #define LOOP_PASS(NAME, CREATE_PASS) \
461 PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
462 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
463 PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
464 #define CGSCC_PASS(NAME, CREATE_PASS) \
465 PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
466 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
467 PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
468 #include "PassRegistry.def"
469 }
470 }
471
invokePeepholeEPCallbacks(FunctionPassManager & FPM,PassBuilder::OptimizationLevel Level)472 void PassBuilder::invokePeepholeEPCallbacks(
473 FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
474 for (auto &C : PeepholeEPCallbacks)
475 C(FPM, Level);
476 }
477
registerModuleAnalyses(ModuleAnalysisManager & MAM)478 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
479 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
480 MAM.registerPass([&] { return CREATE_PASS; });
481 #include "PassRegistry.def"
482
483 for (auto &C : ModuleAnalysisRegistrationCallbacks)
484 C(MAM);
485 }
486
registerCGSCCAnalyses(CGSCCAnalysisManager & CGAM)487 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
488 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
489 CGAM.registerPass([&] { return CREATE_PASS; });
490 #include "PassRegistry.def"
491
492 for (auto &C : CGSCCAnalysisRegistrationCallbacks)
493 C(CGAM);
494 }
495
registerFunctionAnalyses(FunctionAnalysisManager & FAM)496 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
497 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
498 FAM.registerPass([&] { return CREATE_PASS; });
499 #include "PassRegistry.def"
500
501 for (auto &C : FunctionAnalysisRegistrationCallbacks)
502 C(FAM);
503 }
504
registerLoopAnalyses(LoopAnalysisManager & LAM)505 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
506 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
507 LAM.registerPass([&] { return CREATE_PASS; });
508 #include "PassRegistry.def"
509
510 for (auto &C : LoopAnalysisRegistrationCallbacks)
511 C(LAM);
512 }
513
514 // Helper to add AnnotationRemarksPass.
addAnnotationRemarksPass(ModulePassManager & MPM)515 static void addAnnotationRemarksPass(ModulePassManager &MPM) {
516 FunctionPassManager FPM;
517 FPM.addPass(AnnotationRemarksPass());
518 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
519 }
520
521 // Helper to check if the current compilation phase is preparing for LTO
isLTOPreLink(ThinOrFullLTOPhase Phase)522 static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
523 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink ||
524 Phase == ThinOrFullLTOPhase::FullLTOPreLink;
525 }
526
527 // TODO: Investigate the cost/benefit of tail call elimination on debugging.
528 FunctionPassManager
buildO1FunctionSimplificationPipeline(OptimizationLevel Level,ThinOrFullLTOPhase Phase)529 PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
530 ThinOrFullLTOPhase Phase) {
531
532 FunctionPassManager FPM;
533
534 // Form SSA out of local memory accesses after breaking apart aggregates into
535 // scalars.
536 FPM.addPass(SROA());
537
538 // Catch trivial redundancies
539 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
540
541 // Hoisting of scalars and load expressions.
542 FPM.addPass(SimplifyCFGPass());
543 FPM.addPass(InstCombinePass());
544
545 FPM.addPass(LibCallsShrinkWrapPass());
546
547 invokePeepholeEPCallbacks(FPM, Level);
548
549 FPM.addPass(SimplifyCFGPass());
550
551 // Form canonically associated expression trees, and simplify the trees using
552 // basic mathematical properties. For example, this will form (nearly)
553 // minimal multiplication trees.
554 FPM.addPass(ReassociatePass());
555
556 // Add the primary loop simplification pipeline.
557 // FIXME: Currently this is split into two loop pass pipelines because we run
558 // some function passes in between them. These can and should be removed
559 // and/or replaced by scheduling the loop pass equivalents in the correct
560 // positions. But those equivalent passes aren't powerful enough yet.
561 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
562 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
563 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
564 // `LoopInstSimplify`.
565 LoopPassManager LPM1, LPM2;
566
567 // Simplify the loop body. We do this initially to clean up after other loop
568 // passes run, either when iterating on a loop or on inner loops with
569 // implications on the outer loop.
570 LPM1.addPass(LoopInstSimplifyPass());
571 LPM1.addPass(LoopSimplifyCFGPass());
572
573 // Try to remove as much code from the loop header as possible,
574 // to reduce amount of IR that will have to be duplicated.
575 // TODO: Investigate promotion cap for O1.
576 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
577
578 LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true,
579 isLTOPreLink(Phase)));
580 // TODO: Investigate promotion cap for O1.
581 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
582 LPM1.addPass(SimpleLoopUnswitchPass());
583
584 LPM2.addPass(LoopIdiomRecognizePass());
585 LPM2.addPass(IndVarSimplifyPass());
586
587 for (auto &C : LateLoopOptimizationsEPCallbacks)
588 C(LPM2, Level);
589
590 LPM2.addPass(LoopDeletionPass());
591
592 if (EnableLoopInterchange)
593 LPM2.addPass(LoopInterchangePass());
594
595 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
596 // because it changes IR to makes profile annotation in back compile
597 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
598 // attributes so we need to make sure and allow the full unroll pass to pay
599 // attention to it.
600 if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
601 PGOOpt->Action != PGOOptions::SampleUse)
602 LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
603 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
604 PTO.ForgetAllSCEVInLoopUnroll));
605
606 for (auto &C : LoopOptimizerEndEPCallbacks)
607 C(LPM2, Level);
608
609 // We provide the opt remark emitter pass for LICM to use. We only need to do
610 // this once as it is immutable.
611 FPM.addPass(
612 RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
613 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),
614 EnableMSSALoopDependency,
615 /*UseBlockFrequencyInfo=*/true));
616 FPM.addPass(SimplifyCFGPass());
617 FPM.addPass(InstCombinePass());
618 if (EnableLoopFlatten)
619 FPM.addPass(LoopFlattenPass());
620 // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
621 // *All* loop passes must preserve it, in order to be able to use it.
622 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),
623 /*UseMemorySSA=*/false,
624 /*UseBlockFrequencyInfo=*/false));
625
626 // Delete small array after loop unroll.
627 FPM.addPass(SROA());
628
629 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
630 FPM.addPass(MemCpyOptPass());
631
632 // Sparse conditional constant propagation.
633 // FIXME: It isn't clear why we do this *after* loop passes rather than
634 // before...
635 FPM.addPass(SCCPPass());
636
637 // Delete dead bit computations (instcombine runs after to fold away the dead
638 // computations, and then ADCE will run later to exploit any new DCE
639 // opportunities that creates).
640 FPM.addPass(BDCEPass());
641
642 // Run instcombine after redundancy and dead bit elimination to exploit
643 // opportunities opened up by them.
644 FPM.addPass(InstCombinePass());
645 invokePeepholeEPCallbacks(FPM, Level);
646
647 if (PTO.Coroutines)
648 FPM.addPass(CoroElidePass());
649
650 for (auto &C : ScalarOptimizerLateEPCallbacks)
651 C(FPM, Level);
652
653 // Finally, do an expensive DCE pass to catch all the dead code exposed by
654 // the simplifications and basic cleanup after all the simplifications.
655 // TODO: Investigate if this is too expensive.
656 FPM.addPass(ADCEPass());
657 FPM.addPass(SimplifyCFGPass());
658 FPM.addPass(InstCombinePass());
659 invokePeepholeEPCallbacks(FPM, Level);
660
661 return FPM;
662 }
663
664 FunctionPassManager
buildFunctionSimplificationPipeline(OptimizationLevel Level,ThinOrFullLTOPhase Phase)665 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
666 ThinOrFullLTOPhase Phase) {
667 assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
668
669 // The O1 pipeline has a separate pipeline creation function to simplify
670 // construction readability.
671 if (Level.getSpeedupLevel() == 1)
672 return buildO1FunctionSimplificationPipeline(Level, Phase);
673
674 FunctionPassManager FPM;
675
676 // Form SSA out of local memory accesses after breaking apart aggregates into
677 // scalars.
678 FPM.addPass(SROA());
679
680 // Catch trivial redundancies
681 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
682 if (EnableKnowledgeRetention)
683 FPM.addPass(AssumeSimplifyPass());
684
685 // Hoisting of scalars and load expressions.
686 if (EnableGVNHoist)
687 FPM.addPass(GVNHoistPass());
688
689 // Global value numbering based sinking.
690 if (EnableGVNSink) {
691 FPM.addPass(GVNSinkPass());
692 FPM.addPass(SimplifyCFGPass());
693 }
694
695 if (EnableConstraintElimination)
696 FPM.addPass(ConstraintEliminationPass());
697
698 // Speculative execution if the target has divergent branches; otherwise nop.
699 FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
700
701 // Optimize based on known information about branches, and cleanup afterward.
702 FPM.addPass(JumpThreadingPass());
703 FPM.addPass(CorrelatedValuePropagationPass());
704
705 FPM.addPass(SimplifyCFGPass());
706 if (Level == OptimizationLevel::O3)
707 FPM.addPass(AggressiveInstCombinePass());
708 FPM.addPass(InstCombinePass());
709
710 if (!Level.isOptimizingForSize())
711 FPM.addPass(LibCallsShrinkWrapPass());
712
713 invokePeepholeEPCallbacks(FPM, Level);
714
715 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
716 // using the size value profile. Don't perform this when optimizing for size.
717 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
718 !Level.isOptimizingForSize())
719 FPM.addPass(PGOMemOPSizeOpt());
720
721 FPM.addPass(TailCallElimPass());
722 FPM.addPass(SimplifyCFGPass());
723
724 // Form canonically associated expression trees, and simplify the trees using
725 // basic mathematical properties. For example, this will form (nearly)
726 // minimal multiplication trees.
727 FPM.addPass(ReassociatePass());
728
729 // Add the primary loop simplification pipeline.
730 // FIXME: Currently this is split into two loop pass pipelines because we run
731 // some function passes in between them. These can and should be removed
732 // and/or replaced by scheduling the loop pass equivalents in the correct
733 // positions. But those equivalent passes aren't powerful enough yet.
734 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
735 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
736 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
737 // `LoopInstSimplify`.
738 LoopPassManager LPM1, LPM2;
739
740 // Simplify the loop body. We do this initially to clean up after other loop
741 // passes run, either when iterating on a loop or on inner loops with
742 // implications on the outer loop.
743 LPM1.addPass(LoopInstSimplifyPass());
744 LPM1.addPass(LoopSimplifyCFGPass());
745
746 // Try to remove as much code from the loop header as possible,
747 // to reduce amount of IR that will have to be duplicated.
748 // TODO: Investigate promotion cap for O1.
749 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
750
751 // Disable header duplication in loop rotation at -Oz.
752 LPM1.addPass(
753 LoopRotatePass(Level != OptimizationLevel::Oz, isLTOPreLink(Phase)));
754 // TODO: Investigate promotion cap for O1.
755 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
756 LPM1.addPass(
757 SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3 &&
758 EnableO3NonTrivialUnswitching));
759 LPM2.addPass(LoopIdiomRecognizePass());
760 LPM2.addPass(IndVarSimplifyPass());
761
762 for (auto &C : LateLoopOptimizationsEPCallbacks)
763 C(LPM2, Level);
764
765 LPM2.addPass(LoopDeletionPass());
766
767 if (EnableLoopInterchange)
768 LPM2.addPass(LoopInterchangePass());
769
770 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
771 // because it changes IR to makes profile annotation in back compile
772 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
773 // attributes so we need to make sure and allow the full unroll pass to pay
774 // attention to it.
775 if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
776 PGOOpt->Action != PGOOptions::SampleUse)
777 LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
778 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
779 PTO.ForgetAllSCEVInLoopUnroll));
780
781 for (auto &C : LoopOptimizerEndEPCallbacks)
782 C(LPM2, Level);
783
784 // We provide the opt remark emitter pass for LICM to use. We only need to do
785 // this once as it is immutable.
786 FPM.addPass(
787 RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
788 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),
789 EnableMSSALoopDependency,
790 /*UseBlockFrequencyInfo=*/true));
791 FPM.addPass(SimplifyCFGPass());
792 FPM.addPass(InstCombinePass());
793 if (EnableLoopFlatten)
794 FPM.addPass(LoopFlattenPass());
795 // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
796 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
797 // *All* loop passes must preserve it, in order to be able to use it.
798 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),
799 /*UseMemorySSA=*/false,
800 /*UseBlockFrequencyInfo=*/false));
801
802 // Delete small array after loop unroll.
803 FPM.addPass(SROA());
804
805 // Eliminate redundancies.
806 FPM.addPass(MergedLoadStoreMotionPass());
807 if (RunNewGVN)
808 FPM.addPass(NewGVNPass());
809 else
810 FPM.addPass(GVN());
811
812 // Sparse conditional constant propagation.
813 // FIXME: It isn't clear why we do this *after* loop passes rather than
814 // before...
815 FPM.addPass(SCCPPass());
816
817 // Delete dead bit computations (instcombine runs after to fold away the dead
818 // computations, and then ADCE will run later to exploit any new DCE
819 // opportunities that creates).
820 FPM.addPass(BDCEPass());
821
822 // Run instcombine after redundancy and dead bit elimination to exploit
823 // opportunities opened up by them.
824 FPM.addPass(InstCombinePass());
825 invokePeepholeEPCallbacks(FPM, Level);
826
827 // Re-consider control flow based optimizations after redundancy elimination,
828 // redo DCE, etc.
829 FPM.addPass(JumpThreadingPass());
830 FPM.addPass(CorrelatedValuePropagationPass());
831
832 // Finally, do an expensive DCE pass to catch all the dead code exposed by
833 // the simplifications and basic cleanup after all the simplifications.
834 // TODO: Investigate if this is too expensive.
835 FPM.addPass(ADCEPass());
836
837 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
838 FPM.addPass(MemCpyOptPass());
839
840 FPM.addPass(DSEPass());
841 FPM.addPass(createFunctionToLoopPassAdaptor(
842 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
843 EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true));
844
845 if (PTO.Coroutines)
846 FPM.addPass(CoroElidePass());
847
848 for (auto &C : ScalarOptimizerLateEPCallbacks)
849 C(FPM, Level);
850
851 FPM.addPass(SimplifyCFGPass(
852 SimplifyCFGOptions().hoistCommonInsts(true).sinkCommonInsts(true)));
853 FPM.addPass(InstCombinePass());
854 invokePeepholeEPCallbacks(FPM, Level);
855
856 if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
857 (PGOOpt->Action == PGOOptions::IRUse ||
858 PGOOpt->Action == PGOOptions::SampleUse))
859 FPM.addPass(ControlHeightReductionPass());
860
861 return FPM;
862 }
863
addRequiredLTOPreLinkPasses(ModulePassManager & MPM)864 void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
865 MPM.addPass(CanonicalizeAliasesPass());
866 MPM.addPass(NameAnonGlobalPass());
867 }
868
addPGOInstrPasses(ModulePassManager & MPM,PassBuilder::OptimizationLevel Level,bool RunProfileGen,bool IsCS,std::string ProfileFile,std::string ProfileRemappingFile)869 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
870 PassBuilder::OptimizationLevel Level,
871 bool RunProfileGen, bool IsCS,
872 std::string ProfileFile,
873 std::string ProfileRemappingFile) {
874 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
875 if (!IsCS && !DisablePreInliner) {
876 InlineParams IP;
877
878 IP.DefaultThreshold = PreInlineThreshold;
879
880 // FIXME: The hint threshold has the same value used by the regular inliner
881 // when not optimzing for size. This should probably be lowered after
882 // performance testing.
883 // FIXME: this comment is cargo culted from the old pass manager, revisit).
884 IP.HintThreshold = Level.isOptimizingForSize() ? PreInlineThreshold : 325;
885 ModuleInlinerWrapperPass MIWP(IP);
886 CGSCCPassManager &CGPipeline = MIWP.getPM();
887
888 FunctionPassManager FPM;
889 FPM.addPass(SROA());
890 FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
891 FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
892 FPM.addPass(InstCombinePass()); // Combine silly sequences.
893 invokePeepholeEPCallbacks(FPM, Level);
894
895 CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
896
897 MPM.addPass(std::move(MIWP));
898
899 // Delete anything that is now dead to make sure that we don't instrument
900 // dead code. Instrumentation can end up keeping dead code around and
901 // dramatically increase code size.
902 MPM.addPass(GlobalDCEPass());
903 }
904
905 if (!RunProfileGen) {
906 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
907 MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
908 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
909 // RequireAnalysisPass for PSI before subsequent non-module passes.
910 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
911 return;
912 }
913
914 // Perform PGO instrumentation.
915 MPM.addPass(PGOInstrumentationGen(IsCS));
916
917 FunctionPassManager FPM;
918 // Disable header duplication in loop rotation at -Oz.
919 FPM.addPass(createFunctionToLoopPassAdaptor(
920 LoopRotatePass(Level != OptimizationLevel::Oz), EnableMSSALoopDependency,
921 /*UseBlockFrequencyInfo=*/false));
922 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
923
924 // Add the profile lowering pass.
925 InstrProfOptions Options;
926 if (!ProfileFile.empty())
927 Options.InstrProfileOutput = ProfileFile;
928 // Do counter promotion at Level greater than O0.
929 Options.DoCounterPromotion = true;
930 Options.UseBFIInPromotion = IsCS;
931 MPM.addPass(InstrProfiling(Options, IsCS));
932 }
933
addPGOInstrPassesForO0(ModulePassManager & MPM,bool RunProfileGen,bool IsCS,std::string ProfileFile,std::string ProfileRemappingFile)934 void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
935 bool RunProfileGen, bool IsCS,
936 std::string ProfileFile,
937 std::string ProfileRemappingFile) {
938 if (!RunProfileGen) {
939 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
940 MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
941 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
942 // RequireAnalysisPass for PSI before subsequent non-module passes.
943 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
944 return;
945 }
946
947 // Perform PGO instrumentation.
948 MPM.addPass(PGOInstrumentationGen(IsCS));
949 // Add the profile lowering pass.
950 InstrProfOptions Options;
951 if (!ProfileFile.empty())
952 Options.InstrProfileOutput = ProfileFile;
953 // Do not do counter promotion at O0.
954 Options.DoCounterPromotion = false;
955 Options.UseBFIInPromotion = IsCS;
956 MPM.addPass(InstrProfiling(Options, IsCS));
957 }
958
959 static InlineParams
getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level)960 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
961 return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
962 }
963
964 ModuleInlinerWrapperPass
buildInlinerPipeline(OptimizationLevel Level,ThinOrFullLTOPhase Phase)965 PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
966 ThinOrFullLTOPhase Phase) {
967 InlineParams IP = getInlineParamsFromOptLevel(Level);
968 if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
969 PGOOpt->Action == PGOOptions::SampleUse)
970 IP.HotCallSiteThreshold = 0;
971
972 if (PGOOpt)
973 IP.EnableDeferral = EnablePGOInlineDeferral;
974
975 ModuleInlinerWrapperPass MIWP(IP, PerformMandatoryInliningsFirst,
976 UseInlineAdvisor, MaxDevirtIterations);
977
978 // Require the GlobalsAA analysis for the module so we can query it within
979 // the CGSCC pipeline.
980 MIWP.addModulePass(RequireAnalysisPass<GlobalsAA, Module>());
981 // Invalidate AAManager so it can be recreated and pick up the newly available
982 // GlobalsAA.
983 MIWP.addModulePass(
984 createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));
985
986 // Require the ProfileSummaryAnalysis for the module so we can query it within
987 // the inliner pass.
988 MIWP.addModulePass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
989
990 // Now begin the main postorder CGSCC pipeline.
991 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
992 // manager and trying to emulate its precise behavior. Much of this doesn't
993 // make a lot of sense and we should revisit the core CGSCC structure.
994 CGSCCPassManager &MainCGPipeline = MIWP.getPM();
995
996 // Note: historically, the PruneEH pass was run first to deduce nounwind and
997 // generally clean up exception handling overhead. It isn't clear this is
998 // valuable as the inliner doesn't currently care whether it is inlining an
999 // invoke or a call.
1000
1001 if (AttributorRun & AttributorRunOption::CGSCC)
1002 MainCGPipeline.addPass(AttributorCGSCCPass());
1003
1004 if (PTO.Coroutines)
1005 MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
1006
1007 // Now deduce any function attributes based in the current code.
1008 MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
1009
1010 // When at O3 add argument promotion to the pass pipeline.
1011 // FIXME: It isn't at all clear why this should be limited to O3.
1012 if (Level == OptimizationLevel::O3)
1013 MainCGPipeline.addPass(ArgumentPromotionPass());
1014
1015 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
1016 // there are no OpenMP runtime calls present in the module.
1017 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
1018 MainCGPipeline.addPass(OpenMPOptCGSCCPass());
1019
1020 for (auto &C : CGSCCOptimizerLateEPCallbacks)
1021 C(MainCGPipeline, Level);
1022
1023 // Lastly, add the core function simplification pipeline nested inside the
1024 // CGSCC walk.
1025 MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
1026 buildFunctionSimplificationPipeline(Level, Phase)));
1027
1028 return MIWP;
1029 }
1030
1031 ModulePassManager
buildModuleSimplificationPipeline(OptimizationLevel Level,ThinOrFullLTOPhase Phase)1032 PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
1033 ThinOrFullLTOPhase Phase) {
1034 ModulePassManager MPM;
1035
1036 // Place pseudo probe instrumentation as the first pass of the pipeline to
1037 // minimize the impact of optimization changes.
1038 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1039 Phase != ThinOrFullLTOPhase::ThinLTOPostLink)
1040 MPM.addPass(SampleProfileProbePass(TM));
1041
1042 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
1043
1044 // In ThinLTO mode, when flattened profile is used, all the available
1045 // profile information will be annotated in PreLink phase so there is
1046 // no need to load the profile again in PostLink.
1047 bool LoadSampleProfile =
1048 HasSampleProfile &&
1049 !(FlattenedProfileUsed && Phase == ThinOrFullLTOPhase::ThinLTOPostLink);
1050
1051 // During the ThinLTO backend phase we perform early indirect call promotion
1052 // here, before globalopt. Otherwise imported available_externally functions
1053 // look unreferenced and are removed. If we are going to load the sample
1054 // profile then defer until later.
1055 // TODO: See if we can move later and consolidate with the location where
1056 // we perform ICP when we are loading a sample profile.
1057 // TODO: We pass HasSampleProfile (whether there was a sample profile file
1058 // passed to the compile) to the SamplePGO flag of ICP. This is used to
1059 // determine whether the new direct calls are annotated with prof metadata.
1060 // Ideally this should be determined from whether the IR is annotated with
1061 // sample profile, and not whether the a sample profile was provided on the
1062 // command line. E.g. for flattened profiles where we will not be reloading
1063 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
1064 // provide the sample profile file.
1065 if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink && !LoadSampleProfile)
1066 MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
1067
1068 // Do basic inference of function attributes from known properties of system
1069 // libraries and other oracles.
1070 MPM.addPass(InferFunctionAttrsPass());
1071
1072 // Create an early function pass manager to cleanup the output of the
1073 // frontend.
1074 FunctionPassManager EarlyFPM;
1075 // Lower llvm.expect to metadata before attempting transforms.
1076 // Compare/branch metadata may alter the behavior of passes like SimplifyCFG.
1077 EarlyFPM.addPass(LowerExpectIntrinsicPass());
1078 EarlyFPM.addPass(SimplifyCFGPass());
1079 EarlyFPM.addPass(SROA());
1080 EarlyFPM.addPass(EarlyCSEPass());
1081 if (PTO.Coroutines)
1082 EarlyFPM.addPass(CoroEarlyPass());
1083 if (Level == OptimizationLevel::O3)
1084 EarlyFPM.addPass(CallSiteSplittingPass());
1085
1086 // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
1087 // to convert bitcast to direct calls so that they can be inlined during the
1088 // profile annotation prepration step.
1089 // More details about SamplePGO design can be found in:
1090 // https://research.google.com/pubs/pub45290.html
1091 // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
1092 if (LoadSampleProfile)
1093 EarlyFPM.addPass(InstCombinePass());
1094 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1095
1096 if (LoadSampleProfile) {
1097 // Annotate sample profile right after early FPM to ensure freshness of
1098 // the debug info.
1099 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1100 PGOOpt->ProfileRemappingFile, Phase));
1101 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1102 // RequireAnalysisPass for PSI before subsequent non-module passes.
1103 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1104 // Do not invoke ICP in the LTOPrelink phase as it makes it hard
1105 // for the profile annotation to be accurate in the LTO backend.
1106 if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink &&
1107 Phase != ThinOrFullLTOPhase::FullLTOPreLink)
1108 // We perform early indirect call promotion here, before globalopt.
1109 // This is important for the ThinLTO backend phase because otherwise
1110 // imported available_externally functions look unreferenced and are
1111 // removed.
1112 MPM.addPass(
1113 PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));
1114 }
1115
1116 // Try to perform OpenMP specific optimizations on the module. This is a
1117 // (quick!) no-op if there are no OpenMP runtime calls present in the module.
1118 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
1119 MPM.addPass(OpenMPOptPass());
1120
1121 if (AttributorRun & AttributorRunOption::MODULE)
1122 MPM.addPass(AttributorPass());
1123
1124 // Lower type metadata and the type.test intrinsic in the ThinLTO
1125 // post link pipeline after ICP. This is to enable usage of the type
1126 // tests in ICP sequences.
1127 if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink)
1128 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1129
1130 for (auto &C : PipelineEarlySimplificationEPCallbacks)
1131 C(MPM, Level);
1132
1133 // Interprocedural constant propagation now that basic cleanup has occurred
1134 // and prior to optimizing globals.
1135 // FIXME: This position in the pipeline hasn't been carefully considered in
1136 // years, it should be re-analyzed.
1137 MPM.addPass(IPSCCPPass());
1138
1139 // Attach metadata to indirect call sites indicating the set of functions
1140 // they may target at run-time. This should follow IPSCCP.
1141 MPM.addPass(CalledValuePropagationPass());
1142
1143 // Optimize globals to try and fold them into constants.
1144 MPM.addPass(GlobalOptPass());
1145
1146 // Promote any localized globals to SSA registers.
1147 // FIXME: Should this instead by a run of SROA?
1148 // FIXME: We should probably run instcombine and simplify-cfg afterward to
1149 // delete control flows that are dead once globals have been folded to
1150 // constants.
1151 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1152
1153 // Remove any dead arguments exposed by cleanups and constant folding
1154 // globals.
1155 MPM.addPass(DeadArgumentEliminationPass());
1156
1157 // Create a small function pass pipeline to cleanup after all the global
1158 // optimizations.
1159 FunctionPassManager GlobalCleanupPM;
1160 GlobalCleanupPM.addPass(InstCombinePass());
1161 invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
1162
1163 GlobalCleanupPM.addPass(SimplifyCFGPass());
1164 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
1165
1166 // Add all the requested passes for instrumentation PGO, if requested.
1167 if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
1168 (PGOOpt->Action == PGOOptions::IRInstr ||
1169 PGOOpt->Action == PGOOptions::IRUse)) {
1170 addPGOInstrPasses(MPM, Level,
1171 /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
1172 /* IsCS */ false, PGOOpt->ProfileFile,
1173 PGOOpt->ProfileRemappingFile);
1174 MPM.addPass(PGOIndirectCallPromotion(false, false));
1175 }
1176 if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
1177 PGOOpt->CSAction == PGOOptions::CSIRInstr)
1178 MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
1179
1180 // Synthesize function entry counts for non-PGO compilation.
1181 if (EnableSyntheticCounts && !PGOOpt)
1182 MPM.addPass(SyntheticCountsPropagation());
1183
1184 MPM.addPass(buildInlinerPipeline(Level, Phase));
1185
1186 if (EnableMemProfiler && Phase != ThinOrFullLTOPhase::ThinLTOPreLink) {
1187 MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
1188 MPM.addPass(ModuleMemProfilerPass());
1189 }
1190
1191 return MPM;
1192 }
1193
1194 /// TODO: Should LTO cause any differences to this set of passes?
addVectorPasses(OptimizationLevel Level,FunctionPassManager & FPM,bool IsLTO)1195 void PassBuilder::addVectorPasses(OptimizationLevel Level,
1196 FunctionPassManager &FPM, bool IsLTO) {
1197 FPM.addPass(LoopVectorizePass(
1198 LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1199
1200 if (IsLTO) {
1201 // The vectorizer may have significantly shortened a loop body; unroll
1202 // again. Unroll small loops to hide loop backedge latency and saturate any
1203 // parallel execution resources of an out-of-order processor. We also then
1204 // need to clean up redundancies and loop invariant code.
1205 // FIXME: It would be really good to use a loop-integrated instruction
1206 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1207 // across the loop nests.
1208 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1209 if (EnableUnrollAndJam && PTO.LoopUnrolling)
1210 FPM.addPass(createFunctionToLoopPassAdaptor(
1211 LoopUnrollAndJamPass(Level.getSpeedupLevel())));
1212 FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
1213 Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1214 PTO.ForgetAllSCEVInLoopUnroll)));
1215 FPM.addPass(WarnMissedTransformationsPass());
1216 }
1217
1218 if (!IsLTO) {
1219 // Eliminate loads by forwarding stores from the previous iteration to loads
1220 // of the current iteration.
1221 FPM.addPass(LoopLoadEliminationPass());
1222 }
1223 // Cleanup after the loop optimization passes.
1224 FPM.addPass(InstCombinePass());
1225
1226 if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1227 // At higher optimization levels, try to clean up any runtime overlap and
1228 // alignment checks inserted by the vectorizer. We want to track correlated
1229 // runtime checks for two inner loops in the same outer loop, fold any
1230 // common computations, hoist loop-invariant aspects out of any outer loop,
1231 // and unswitch the runtime checks if possible. Once hoisted, we may have
1232 // dead (or speculatable) control flows or more combining opportunities.
1233 FPM.addPass(EarlyCSEPass());
1234 FPM.addPass(CorrelatedValuePropagationPass());
1235 FPM.addPass(InstCombinePass());
1236 LoopPassManager LPM;
1237 LPM.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
1238 LPM.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
1239 OptimizationLevel::O3));
1240 FPM.addPass(
1241 RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
1242 FPM.addPass(createFunctionToLoopPassAdaptor(
1243 std::move(LPM), EnableMSSALoopDependency,
1244 /*UseBlockFrequencyInfo=*/true));
1245 FPM.addPass(SimplifyCFGPass());
1246 FPM.addPass(InstCombinePass());
1247 }
1248
1249 if (IsLTO) {
1250 FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true)));
1251 } else {
1252 // Now that we've formed fast to execute loop structures, we do further
1253 // optimizations. These are run afterward as they might block doing complex
1254 // analyses and transforms such as what are needed for loop vectorization.
1255
1256 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1257 // GVN, loop transforms, and others have already run, so it's now better to
1258 // convert to more optimized IR using more aggressive simplify CFG options.
1259 // The extra sinking transform can create larger basic blocks, so do this
1260 // before SLP vectorization.
1261 FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
1262 .forwardSwitchCondToPhi(true)
1263 .convertSwitchToLookupTable(true)
1264 .needCanonicalLoops(false)
1265 .hoistCommonInsts(true)
1266 .sinkCommonInsts(true)));
1267 }
1268 if (IsLTO) {
1269 FPM.addPass(SCCPPass());
1270 FPM.addPass(InstCombinePass());
1271 FPM.addPass(BDCEPass());
1272 }
1273
1274 // Optimize parallel scalar instruction chains into SIMD instructions.
1275 if (PTO.SLPVectorization) {
1276 FPM.addPass(SLPVectorizerPass());
1277 if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1278 FPM.addPass(EarlyCSEPass());
1279 }
1280 }
1281 // Enhance/cleanup vector code.
1282 FPM.addPass(VectorCombinePass());
1283
1284 if (!IsLTO) {
1285 FPM.addPass(InstCombinePass());
1286 // Unroll small loops to hide loop backedge latency and saturate any
1287 // parallel execution resources of an out-of-order processor. We also then
1288 // need to clean up redundancies and loop invariant code.
1289 // FIXME: It would be really good to use a loop-integrated instruction
1290 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1291 // across the loop nests.
1292 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1293 if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1294 FPM.addPass(createFunctionToLoopPassAdaptor(
1295 LoopUnrollAndJamPass(Level.getSpeedupLevel())));
1296 }
1297 FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
1298 Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1299 PTO.ForgetAllSCEVInLoopUnroll)));
1300 FPM.addPass(WarnMissedTransformationsPass());
1301 FPM.addPass(InstCombinePass());
1302 FPM.addPass(
1303 RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
1304 FPM.addPass(createFunctionToLoopPassAdaptor(
1305 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
1306 EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true));
1307 }
1308
1309 // Now that we've vectorized and unrolled loops, we may have more refined
1310 // alignment information, try to re-derive it here.
1311 FPM.addPass(AlignmentFromAssumptionsPass());
1312
1313 if (IsLTO)
1314 FPM.addPass(InstCombinePass());
1315 }
1316
1317 ModulePassManager
buildModuleOptimizationPipeline(OptimizationLevel Level,bool LTOPreLink)1318 PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
1319 bool LTOPreLink) {
1320 ModulePassManager MPM;
1321
1322 // Optimize globals now that the module is fully simplified.
1323 MPM.addPass(GlobalOptPass());
1324 MPM.addPass(GlobalDCEPass());
1325
1326 // Run partial inlining pass to partially inline functions that have
1327 // large bodies.
1328 if (RunPartialInlining)
1329 MPM.addPass(PartialInlinerPass());
1330
1331 // Remove avail extern fns and globals definitions since we aren't compiling
1332 // an object file for later LTO. For LTO we want to preserve these so they
1333 // are eligible for inlining at link-time. Note if they are unreferenced they
1334 // will be removed by GlobalDCE later, so this only impacts referenced
1335 // available externally globals. Eventually they will be suppressed during
1336 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1337 // may make globals referenced by available external functions dead and saves
1338 // running remaining passes on the eliminated functions. These should be
1339 // preserved during prelinking for link-time inlining decisions.
1340 if (!LTOPreLink)
1341 MPM.addPass(EliminateAvailableExternallyPass());
1342
1343 if (EnableOrderFileInstrumentation)
1344 MPM.addPass(InstrOrderFilePass());
1345
1346 // Do RPO function attribute inference across the module to forward-propagate
1347 // attributes where applicable.
1348 // FIXME: Is this really an optimization rather than a canonicalization?
1349 MPM.addPass(ReversePostOrderFunctionAttrsPass());
1350
1351 // Do a post inline PGO instrumentation and use pass. This is a context
1352 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1353 // cross-module inline has not been done yet. The context sensitive
1354 // instrumentation is after all the inlines are done.
1355 if (!LTOPreLink && PGOOpt) {
1356 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1357 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1358 /* IsCS */ true, PGOOpt->CSProfileGenFile,
1359 PGOOpt->ProfileRemappingFile);
1360 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1361 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1362 /* IsCS */ true, PGOOpt->ProfileFile,
1363 PGOOpt->ProfileRemappingFile);
1364 }
1365
1366 // Re-require GloblasAA here prior to function passes. This is particularly
1367 // useful as the above will have inlined, DCE'ed, and function-attr
1368 // propagated everything. We should at this point have a reasonably minimal
1369 // and richly annotated call graph. By computing aliasing and mod/ref
1370 // information for all local globals here, the late loop passes and notably
1371 // the vectorizer will be able to use them to help recognize vectorizable
1372 // memory operations.
1373 MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
1374
1375 FunctionPassManager OptimizePM;
1376 OptimizePM.addPass(Float2IntPass());
1377 OptimizePM.addPass(LowerConstantIntrinsicsPass());
1378
1379 if (EnableMatrix) {
1380 OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1381 OptimizePM.addPass(EarlyCSEPass());
1382 }
1383
1384 // FIXME: We need to run some loop optimizations to re-rotate loops after
1385 // simplify-cfg and others undo their rotation.
1386
1387 // Optimize the loop execution. These passes operate on entire loop nests
1388 // rather than on each loop in an inside-out manner, and so they are actually
1389 // function passes.
1390
1391 for (auto &C : VectorizerStartEPCallbacks)
1392 C(OptimizePM, Level);
1393
1394 // First rotate loops that may have been un-rotated by prior passes.
1395 // Disable header duplication at -Oz.
1396 OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1397 LoopRotatePass(Level != OptimizationLevel::Oz, LTOPreLink),
1398 EnableMSSALoopDependency,
1399 /*UseBlockFrequencyInfo=*/false));
1400
1401 // Distribute loops to allow partial vectorization. I.e. isolate dependences
1402 // into separate loop that would otherwise inhibit vectorization. This is
1403 // currently only performed for loops marked with the metadata
1404 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1405 OptimizePM.addPass(LoopDistributePass());
1406
1407 // Populates the VFABI attribute with the scalar-to-vector mappings
1408 // from the TargetLibraryInfo.
1409 OptimizePM.addPass(InjectTLIMappings());
1410
1411 addVectorPasses(Level, OptimizePM, /* IsLTO */ false);
1412
1413 // Split out cold code. Splitting is done late to avoid hiding context from
1414 // other optimizations and inadvertently regressing performance. The tradeoff
1415 // is that this has a higher code size cost than splitting early.
1416 if (EnableHotColdSplit && !LTOPreLink)
1417 MPM.addPass(HotColdSplittingPass());
1418
1419 // Search the code for similar regions of code. If enough similar regions can
1420 // be found where extracting the regions into their own function will decrease
1421 // the size of the program, we extract the regions, a deduplicate the
1422 // structurally similar regions.
1423 if (EnableIROutliner)
1424 MPM.addPass(IROutlinerPass());
1425
1426 // Merge functions if requested.
1427 if (PTO.MergeFunctions)
1428 MPM.addPass(MergeFunctionsPass());
1429
1430 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1431 // canonicalization pass that enables other optimizations. As a result,
1432 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1433 // result too early.
1434 OptimizePM.addPass(LoopSinkPass());
1435
1436 // And finally clean up LCSSA form before generating code.
1437 OptimizePM.addPass(InstSimplifyPass());
1438
1439 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1440 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1441 // flattening of blocks.
1442 OptimizePM.addPass(DivRemPairsPass());
1443
1444 // LoopSink (and other loop passes since the last simplifyCFG) might have
1445 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1446 OptimizePM.addPass(SimplifyCFGPass());
1447
1448 // Optimize PHIs by speculating around them when profitable. Note that this
1449 // pass needs to be run after any PRE or similar pass as it is essentially
1450 // inserting redundancies into the program. This even includes SimplifyCFG.
1451 OptimizePM.addPass(SpeculateAroundPHIsPass());
1452
1453 if (PTO.Coroutines)
1454 OptimizePM.addPass(CoroCleanupPass());
1455
1456 // Add the core optimizing pipeline.
1457 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
1458
1459 for (auto &C : OptimizerLastEPCallbacks)
1460 C(MPM, Level);
1461
1462 if (PTO.CallGraphProfile)
1463 MPM.addPass(CGProfilePass());
1464
1465 // Now we need to do some global optimization transforms.
1466 // FIXME: It would seem like these should come first in the optimization
1467 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1468 // ordering here.
1469 MPM.addPass(GlobalDCEPass());
1470 MPM.addPass(ConstantMergePass());
1471
1472 // TODO: Relative look table converter pass caused an issue when full lto is
1473 // enabled. See https://reviews.llvm.org/D94355 for more details.
1474 // Until the issue fixed, disable this pass during pre-linking phase.
1475 if (!LTOPreLink)
1476 MPM.addPass(RelLookupTableConverterPass());
1477
1478 return MPM;
1479 }
1480
1481 ModulePassManager
buildPerModuleDefaultPipeline(OptimizationLevel Level,bool LTOPreLink)1482 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1483 bool LTOPreLink) {
1484 assert(Level != OptimizationLevel::O0 &&
1485 "Must request optimizations for the default pipeline!");
1486
1487 ModulePassManager MPM;
1488
1489 // Convert @llvm.global.annotations to !annotation metadata.
1490 MPM.addPass(Annotation2MetadataPass());
1491
1492 // Force any function attributes we want the rest of the pipeline to observe.
1493 MPM.addPass(ForceFunctionAttrsPass());
1494
1495 // Apply module pipeline start EP callback.
1496 for (auto &C : PipelineStartEPCallbacks)
1497 C(MPM, Level);
1498
1499 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1500 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1501
1502 // Add the core simplification pipeline.
1503 MPM.addPass(buildModuleSimplificationPipeline(
1504 Level, LTOPreLink ? ThinOrFullLTOPhase::FullLTOPreLink
1505 : ThinOrFullLTOPhase::None));
1506
1507 // Now add the optimization pipeline.
1508 MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPreLink));
1509
1510 if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
1511 MPM.addPass(PseudoProbeUpdatePass());
1512
1513 // Emit annotation remarks.
1514 addAnnotationRemarksPass(MPM);
1515
1516 if (LTOPreLink)
1517 addRequiredLTOPreLinkPasses(MPM);
1518
1519 return MPM;
1520 }
1521
1522 ModulePassManager
buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level)1523 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1524 assert(Level != OptimizationLevel::O0 &&
1525 "Must request optimizations for the default pipeline!");
1526
1527 ModulePassManager MPM;
1528
1529 // Convert @llvm.global.annotations to !annotation metadata.
1530 MPM.addPass(Annotation2MetadataPass());
1531
1532 // Force any function attributes we want the rest of the pipeline to observe.
1533 MPM.addPass(ForceFunctionAttrsPass());
1534
1535 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1536 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1537
1538 // Apply module pipeline start EP callback.
1539 for (auto &C : PipelineStartEPCallbacks)
1540 C(MPM, Level);
1541
1542 // If we are planning to perform ThinLTO later, we don't bloat the code with
1543 // unrolling/vectorization/... now. Just simplify the module as much as we
1544 // can.
1545 MPM.addPass(buildModuleSimplificationPipeline(
1546 Level, ThinOrFullLTOPhase::ThinLTOPreLink));
1547
1548 // Run partial inlining pass to partially inline functions that have
1549 // large bodies.
1550 // FIXME: It isn't clear whether this is really the right place to run this
1551 // in ThinLTO. Because there is another canonicalization and simplification
1552 // phase that will run after the thin link, running this here ends up with
1553 // less information than will be available later and it may grow functions in
1554 // ways that aren't beneficial.
1555 if (RunPartialInlining)
1556 MPM.addPass(PartialInlinerPass());
1557
1558 // Reduce the size of the IR as much as possible.
1559 MPM.addPass(GlobalOptPass());
1560
1561 // Module simplification splits coroutines, but does not fully clean up
1562 // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up
1563 // on these, we schedule the cleanup here.
1564 if (PTO.Coroutines)
1565 MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
1566
1567 if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
1568 MPM.addPass(PseudoProbeUpdatePass());
1569
1570 // Handle OptimizerLastEPCallbacks added by clang on PreLink. Actual
1571 // optimization is going to be done in PostLink stage, but clang can't
1572 // add callbacks there in case of in-process ThinLTO called by linker.
1573 for (auto &C : OptimizerLastEPCallbacks)
1574 C(MPM, Level);
1575
1576 // Emit annotation remarks.
1577 addAnnotationRemarksPass(MPM);
1578
1579 addRequiredLTOPreLinkPasses(MPM);
1580
1581 return MPM;
1582 }
1583
buildThinLTODefaultPipeline(OptimizationLevel Level,const ModuleSummaryIndex * ImportSummary)1584 ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1585 OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1586 ModulePassManager MPM;
1587
1588 // Convert @llvm.global.annotations to !annotation metadata.
1589 MPM.addPass(Annotation2MetadataPass());
1590
1591 if (ImportSummary) {
1592 // These passes import type identifier resolutions for whole-program
1593 // devirtualization and CFI. They must run early because other passes may
1594 // disturb the specific instruction patterns that these passes look for,
1595 // creating dependencies on resolutions that may not appear in the summary.
1596 //
1597 // For example, GVN may transform the pattern assume(type.test) appearing in
1598 // two basic blocks into assume(phi(type.test, type.test)), which would
1599 // transform a dependency on a WPD resolution into a dependency on a type
1600 // identifier resolution for CFI.
1601 //
1602 // Also, WPD has access to more precise information than ICP and can
1603 // devirtualize more effectively, so it should operate on the IR first.
1604 //
1605 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1606 // metadata and intrinsics.
1607 MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1608 MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1609 }
1610
1611 if (Level == OptimizationLevel::O0) {
1612 // Run a second time to clean up any type tests left behind by WPD for use
1613 // in ICP.
1614 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1615 // Drop available_externally and unreferenced globals. This is necessary
1616 // with ThinLTO in order to avoid leaving undefined references to dead
1617 // globals in the object file.
1618 MPM.addPass(EliminateAvailableExternallyPass());
1619 MPM.addPass(GlobalDCEPass());
1620 return MPM;
1621 }
1622
1623 // Force any function attributes we want the rest of the pipeline to observe.
1624 MPM.addPass(ForceFunctionAttrsPass());
1625
1626 // Add the core simplification pipeline.
1627 MPM.addPass(buildModuleSimplificationPipeline(
1628 Level, ThinOrFullLTOPhase::ThinLTOPostLink));
1629
1630 // Now add the optimization pipeline.
1631 MPM.addPass(buildModuleOptimizationPipeline(Level));
1632
1633 // Emit annotation remarks.
1634 addAnnotationRemarksPass(MPM);
1635
1636 return MPM;
1637 }
1638
1639 ModulePassManager
buildLTOPreLinkDefaultPipeline(OptimizationLevel Level)1640 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1641 assert(Level != OptimizationLevel::O0 &&
1642 "Must request optimizations for the default pipeline!");
1643 // FIXME: We should use a customized pre-link pipeline!
1644 return buildPerModuleDefaultPipeline(Level,
1645 /* LTOPreLink */ true);
1646 }
1647
1648 ModulePassManager
buildLTODefaultPipeline(OptimizationLevel Level,ModuleSummaryIndex * ExportSummary)1649 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
1650 ModuleSummaryIndex *ExportSummary) {
1651 ModulePassManager MPM;
1652
1653 // Convert @llvm.global.annotations to !annotation metadata.
1654 MPM.addPass(Annotation2MetadataPass());
1655
1656 if (Level == OptimizationLevel::O0) {
1657 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1658 // metadata and intrinsics.
1659 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1660 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1661 // Run a second time to clean up any type tests left behind by WPD for use
1662 // in ICP.
1663 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1664
1665 // Emit annotation remarks.
1666 addAnnotationRemarksPass(MPM);
1667
1668 return MPM;
1669 }
1670
1671 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1672 // Load sample profile before running the LTO optimization pipeline.
1673 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1674 PGOOpt->ProfileRemappingFile,
1675 ThinOrFullLTOPhase::FullLTOPostLink));
1676 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1677 // RequireAnalysisPass for PSI before subsequent non-module passes.
1678 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1679 }
1680
1681 // Remove unused virtual tables to improve the quality of code generated by
1682 // whole-program devirtualization and bitset lowering.
1683 MPM.addPass(GlobalDCEPass());
1684
1685 // Force any function attributes we want the rest of the pipeline to observe.
1686 MPM.addPass(ForceFunctionAttrsPass());
1687
1688 // Do basic inference of function attributes from known properties of system
1689 // libraries and other oracles.
1690 MPM.addPass(InferFunctionAttrsPass());
1691
1692 if (Level.getSpeedupLevel() > 1) {
1693 FunctionPassManager EarlyFPM;
1694 EarlyFPM.addPass(CallSiteSplittingPass());
1695 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1696
1697 // Indirect call promotion. This should promote all the targets that are
1698 // left by the earlier promotion pass that promotes intra-module targets.
1699 // This two-step promotion is to save the compile time. For LTO, it should
1700 // produce the same result as if we only do promotion here.
1701 MPM.addPass(PGOIndirectCallPromotion(
1702 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1703 // Propagate constants at call sites into the functions they call. This
1704 // opens opportunities for globalopt (and inlining) by substituting function
1705 // pointers passed as arguments to direct uses of functions.
1706 MPM.addPass(IPSCCPPass());
1707
1708 // Attach metadata to indirect call sites indicating the set of functions
1709 // they may target at run-time. This should follow IPSCCP.
1710 MPM.addPass(CalledValuePropagationPass());
1711 }
1712
1713 // Now deduce any function attributes based in the current code.
1714 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1715 PostOrderFunctionAttrsPass()));
1716
1717 // Do RPO function attribute inference across the module to forward-propagate
1718 // attributes where applicable.
1719 // FIXME: Is this really an optimization rather than a canonicalization?
1720 MPM.addPass(ReversePostOrderFunctionAttrsPass());
1721
1722 // Use in-range annotations on GEP indices to split globals where beneficial.
1723 MPM.addPass(GlobalSplitPass());
1724
1725 // Run whole program optimization of virtual call when the list of callees
1726 // is fixed.
1727 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1728
1729 // Stop here at -O1.
1730 if (Level == OptimizationLevel::O1) {
1731 // The LowerTypeTestsPass needs to run to lower type metadata and the
1732 // type.test intrinsics. The pass does nothing if CFI is disabled.
1733 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1734 // Run a second time to clean up any type tests left behind by WPD for use
1735 // in ICP (which is performed earlier than this in the regular LTO
1736 // pipeline).
1737 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1738
1739 // Emit annotation remarks.
1740 addAnnotationRemarksPass(MPM);
1741
1742 return MPM;
1743 }
1744
1745 // Optimize globals to try and fold them into constants.
1746 MPM.addPass(GlobalOptPass());
1747
1748 // Promote any localized globals to SSA registers.
1749 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1750
1751 // Linking modules together can lead to duplicate global constant, only
1752 // keep one copy of each constant.
1753 MPM.addPass(ConstantMergePass());
1754
1755 // Remove unused arguments from functions.
1756 MPM.addPass(DeadArgumentEliminationPass());
1757
1758 // Reduce the code after globalopt and ipsccp. Both can open up significant
1759 // simplification opportunities, and both can propagate functions through
1760 // function pointers. When this happens, we often have to resolve varargs
1761 // calls, etc, so let instcombine do this.
1762 FunctionPassManager PeepholeFPM;
1763 if (Level == OptimizationLevel::O3)
1764 PeepholeFPM.addPass(AggressiveInstCombinePass());
1765 PeepholeFPM.addPass(InstCombinePass());
1766 invokePeepholeEPCallbacks(PeepholeFPM, Level);
1767
1768 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
1769
1770 // Note: historically, the PruneEH pass was run first to deduce nounwind and
1771 // generally clean up exception handling overhead. It isn't clear this is
1772 // valuable as the inliner doesn't currently care whether it is inlining an
1773 // invoke or a call.
1774 // Run the inliner now.
1775 MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level)));
1776
1777 // Optimize globals again after we ran the inliner.
1778 MPM.addPass(GlobalOptPass());
1779
1780 // Garbage collect dead functions.
1781 // FIXME: Add ArgumentPromotion pass after once it's ported.
1782 MPM.addPass(GlobalDCEPass());
1783
1784 FunctionPassManager FPM;
1785 // The IPO Passes may leave cruft around. Clean up after them.
1786 FPM.addPass(InstCombinePass());
1787 invokePeepholeEPCallbacks(FPM, Level);
1788
1789 FPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1790
1791 // Do a post inline PGO instrumentation and use pass. This is a context
1792 // sensitive PGO pass.
1793 if (PGOOpt) {
1794 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1795 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1796 /* IsCS */ true, PGOOpt->CSProfileGenFile,
1797 PGOOpt->ProfileRemappingFile);
1798 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1799 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1800 /* IsCS */ true, PGOOpt->ProfileFile,
1801 PGOOpt->ProfileRemappingFile);
1802 }
1803
1804 // Break up allocas
1805 FPM.addPass(SROA());
1806
1807 // LTO provides additional opportunities for tailcall elimination due to
1808 // link-time inlining, and visibility of nocapture attribute.
1809 FPM.addPass(TailCallElimPass());
1810
1811 // Run a few AA driver optimizations here and now to cleanup the code.
1812 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1813
1814 MPM.addPass(
1815 createModuleToPostOrderCGSCCPassAdaptor(PostOrderFunctionAttrsPass()));
1816
1817 // Require the GlobalsAA analysis for the module so we can query it within
1818 // MainFPM.
1819 MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
1820 // Invalidate AAManager so it can be recreated and pick up the newly available
1821 // GlobalsAA.
1822 MPM.addPass(
1823 createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));
1824
1825 FunctionPassManager MainFPM;
1826 MainFPM.addPass(createFunctionToLoopPassAdaptor(
1827 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
1828 EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true));
1829
1830 if (RunNewGVN)
1831 MainFPM.addPass(NewGVNPass());
1832 else
1833 MainFPM.addPass(GVN());
1834
1835 // Remove dead memcpy()'s.
1836 MainFPM.addPass(MemCpyOptPass());
1837
1838 // Nuke dead stores.
1839 MainFPM.addPass(DSEPass());
1840 MainFPM.addPass(MergedLoadStoreMotionPass());
1841
1842 // More loops are countable; try to optimize them.
1843 if (EnableLoopFlatten && Level.getSpeedupLevel() > 1)
1844 MainFPM.addPass(LoopFlattenPass());
1845
1846 if (EnableConstraintElimination)
1847 MainFPM.addPass(ConstraintEliminationPass());
1848
1849 LoopPassManager LPM;
1850 LPM.addPass(IndVarSimplifyPass());
1851 LPM.addPass(LoopDeletionPass());
1852 // FIXME: Add loop interchange.
1853
1854 // Unroll small loops and perform peeling.
1855 LPM.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
1856 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
1857 PTO.ForgetAllSCEVInLoopUnroll));
1858 // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
1859 // *All* loop passes must preserve it, in order to be able to use it.
1860 MainFPM.addPass(createFunctionToLoopPassAdaptor(
1861 std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/true));
1862
1863 MainFPM.addPass(LoopDistributePass());
1864
1865 addVectorPasses(Level, MainFPM, /* IsLTO */ true);
1866
1867 invokePeepholeEPCallbacks(MainFPM, Level);
1868 MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1869 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1870
1871 // Create a function that performs CFI checks for cross-DSO calls with
1872 // targets in the current module.
1873 MPM.addPass(CrossDSOCFIPass());
1874
1875 // Lower type metadata and the type.test intrinsic. This pass supports
1876 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1877 // to be run at link time if CFI is enabled. This pass does nothing if
1878 // CFI is disabled.
1879 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1880 // Run a second time to clean up any type tests left behind by WPD for use
1881 // in ICP (which is performed earlier than this in the regular LTO pipeline).
1882 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1883
1884 // Enable splitting late in the FullLTO post-link pipeline. This is done in
1885 // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1886 if (EnableHotColdSplit)
1887 MPM.addPass(HotColdSplittingPass());
1888
1889 // Add late LTO optimization passes.
1890 // Delete basic blocks, which optimization passes may have killed.
1891 MPM.addPass(createModuleToFunctionPassAdaptor(
1892 SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true))));
1893
1894 // Drop bodies of available eternally objects to improve GlobalDCE.
1895 MPM.addPass(EliminateAvailableExternallyPass());
1896
1897 // Now that we have optimized the program, discard unreachable functions.
1898 MPM.addPass(GlobalDCEPass());
1899
1900 if (PTO.MergeFunctions)
1901 MPM.addPass(MergeFunctionsPass());
1902
1903 // Emit annotation remarks.
1904 addAnnotationRemarksPass(MPM);
1905
1906 return MPM;
1907 }
1908
buildO0DefaultPipeline(OptimizationLevel Level,bool LTOPreLink)1909 ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
1910 bool LTOPreLink) {
1911 assert(Level == OptimizationLevel::O0 &&
1912 "buildO0DefaultPipeline should only be used with O0");
1913
1914 ModulePassManager MPM;
1915
1916 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
1917 PGOOpt->Action == PGOOptions::IRUse))
1918 addPGOInstrPassesForO0(
1919 MPM,
1920 /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
1921 /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
1922
1923 for (auto &C : PipelineStartEPCallbacks)
1924 C(MPM, Level);
1925 for (auto &C : PipelineEarlySimplificationEPCallbacks)
1926 C(MPM, Level);
1927
1928 // Build a minimal pipeline based on the semantics required by LLVM,
1929 // which is just that always inlining occurs. Further, disable generating
1930 // lifetime intrinsics to avoid enabling further optimizations during
1931 // code generation.
1932 // However, we need to insert lifetime intrinsics to avoid invalid access
1933 // caused by multithreaded coroutines.
1934 MPM.addPass(AlwaysInlinerPass(
1935 /*InsertLifetimeIntrinsics=*/PTO.Coroutines));
1936
1937 if (PTO.MergeFunctions)
1938 MPM.addPass(MergeFunctionsPass());
1939
1940 if (EnableMatrix)
1941 MPM.addPass(
1942 createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass(true)));
1943
1944 if (!CGSCCOptimizerLateEPCallbacks.empty()) {
1945 CGSCCPassManager CGPM;
1946 for (auto &C : CGSCCOptimizerLateEPCallbacks)
1947 C(CGPM, Level);
1948 if (!CGPM.isEmpty())
1949 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
1950 }
1951 if (!LateLoopOptimizationsEPCallbacks.empty()) {
1952 LoopPassManager LPM;
1953 for (auto &C : LateLoopOptimizationsEPCallbacks)
1954 C(LPM, Level);
1955 if (!LPM.isEmpty()) {
1956 MPM.addPass(createModuleToFunctionPassAdaptor(
1957 createFunctionToLoopPassAdaptor(std::move(LPM))));
1958 }
1959 }
1960 if (!LoopOptimizerEndEPCallbacks.empty()) {
1961 LoopPassManager LPM;
1962 for (auto &C : LoopOptimizerEndEPCallbacks)
1963 C(LPM, Level);
1964 if (!LPM.isEmpty()) {
1965 MPM.addPass(createModuleToFunctionPassAdaptor(
1966 createFunctionToLoopPassAdaptor(std::move(LPM))));
1967 }
1968 }
1969 if (!ScalarOptimizerLateEPCallbacks.empty()) {
1970 FunctionPassManager FPM;
1971 for (auto &C : ScalarOptimizerLateEPCallbacks)
1972 C(FPM, Level);
1973 if (!FPM.isEmpty())
1974 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1975 }
1976 if (!VectorizerStartEPCallbacks.empty()) {
1977 FunctionPassManager FPM;
1978 for (auto &C : VectorizerStartEPCallbacks)
1979 C(FPM, Level);
1980 if (!FPM.isEmpty())
1981 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1982 }
1983
1984 if (PTO.Coroutines) {
1985 MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass()));
1986
1987 CGSCCPassManager CGPM;
1988 CGPM.addPass(CoroSplitPass());
1989 CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass()));
1990 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
1991
1992 MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
1993 }
1994
1995 for (auto &C : OptimizerLastEPCallbacks)
1996 C(MPM, Level);
1997
1998 if (LTOPreLink)
1999 addRequiredLTOPreLinkPasses(MPM);
2000
2001 return MPM;
2002 }
2003
buildDefaultAAPipeline()2004 AAManager PassBuilder::buildDefaultAAPipeline() {
2005 AAManager AA;
2006
2007 // The order in which these are registered determines their priority when
2008 // being queried.
2009
2010 // First we register the basic alias analysis that provides the majority of
2011 // per-function local AA logic. This is a stateless, on-demand local set of
2012 // AA techniques.
2013 AA.registerFunctionAnalysis<BasicAA>();
2014
2015 // Next we query fast, specialized alias analyses that wrap IR-embedded
2016 // information about aliasing.
2017 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
2018 AA.registerFunctionAnalysis<TypeBasedAA>();
2019
2020 // Add support for querying global aliasing information when available.
2021 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
2022 // analysis, all that the `AAManager` can do is query for any *cached*
2023 // results from `GlobalsAA` through a readonly proxy.
2024 AA.registerModuleAnalysis<GlobalsAA>();
2025
2026 // Add target-specific alias analyses.
2027 if (TM)
2028 TM->registerDefaultAliasAnalyses(AA);
2029
2030 return AA;
2031 }
2032
parseRepeatPassName(StringRef Name)2033 static Optional<int> parseRepeatPassName(StringRef Name) {
2034 if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
2035 return None;
2036 int Count;
2037 if (Name.getAsInteger(0, Count) || Count <= 0)
2038 return None;
2039 return Count;
2040 }
2041
parseDevirtPassName(StringRef Name)2042 static Optional<int> parseDevirtPassName(StringRef Name) {
2043 if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
2044 return None;
2045 int Count;
2046 if (Name.getAsInteger(0, Count) || Count < 0)
2047 return None;
2048 return Count;
2049 }
2050
checkParametrizedPassName(StringRef Name,StringRef PassName)2051 static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
2052 if (!Name.consume_front(PassName))
2053 return false;
2054 // normal pass name w/o parameters == default parameters
2055 if (Name.empty())
2056 return true;
2057 return Name.startswith("<") && Name.endswith(">");
2058 }
2059
2060 namespace {
2061
2062 /// This performs customized parsing of pass name with parameters.
2063 ///
2064 /// We do not need parametrization of passes in textual pipeline very often,
2065 /// yet on a rare occasion ability to specify parameters right there can be
2066 /// useful.
2067 ///
2068 /// \p Name - parameterized specification of a pass from a textual pipeline
2069 /// is a string in a form of :
2070 /// PassName '<' parameter-list '>'
2071 ///
2072 /// Parameter list is being parsed by the parser callable argument, \p Parser,
2073 /// It takes a string-ref of parameters and returns either StringError or a
2074 /// parameter list in a form of a custom parameters type, all wrapped into
2075 /// Expected<> template class.
2076 ///
2077 template <typename ParametersParseCallableT>
parsePassParameters(ParametersParseCallableT && Parser,StringRef Name,StringRef PassName)2078 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
2079 StringRef PassName) -> decltype(Parser(StringRef{})) {
2080 using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
2081
2082 StringRef Params = Name;
2083 if (!Params.consume_front(PassName)) {
2084 assert(false &&
2085 "unable to strip pass name from parametrized pass specification");
2086 }
2087 if (Params.empty())
2088 return ParametersT{};
2089 if (!Params.consume_front("<") || !Params.consume_back(">")) {
2090 assert(false && "invalid format for parametrized pass name");
2091 }
2092
2093 Expected<ParametersT> Result = Parser(Params);
2094 assert((Result || Result.template errorIsA<StringError>()) &&
2095 "Pass parameter parser can only return StringErrors.");
2096 return Result;
2097 }
2098
2099 /// Parser of parameters for LoopUnroll pass.
parseLoopUnrollOptions(StringRef Params)2100 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
2101 LoopUnrollOptions UnrollOpts;
2102 while (!Params.empty()) {
2103 StringRef ParamName;
2104 std::tie(ParamName, Params) = Params.split(';');
2105 int OptLevel = StringSwitch<int>(ParamName)
2106 .Case("O0", 0)
2107 .Case("O1", 1)
2108 .Case("O2", 2)
2109 .Case("O3", 3)
2110 .Default(-1);
2111 if (OptLevel >= 0) {
2112 UnrollOpts.setOptLevel(OptLevel);
2113 continue;
2114 }
2115 if (ParamName.consume_front("full-unroll-max=")) {
2116 int Count;
2117 if (ParamName.getAsInteger(0, Count))
2118 return make_error<StringError>(
2119 formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
2120 inconvertibleErrorCode());
2121 UnrollOpts.setFullUnrollMaxCount(Count);
2122 continue;
2123 }
2124
2125 bool Enable = !ParamName.consume_front("no-");
2126 if (ParamName == "partial") {
2127 UnrollOpts.setPartial(Enable);
2128 } else if (ParamName == "peeling") {
2129 UnrollOpts.setPeeling(Enable);
2130 } else if (ParamName == "profile-peeling") {
2131 UnrollOpts.setProfileBasedPeeling(Enable);
2132 } else if (ParamName == "runtime") {
2133 UnrollOpts.setRuntime(Enable);
2134 } else if (ParamName == "upperbound") {
2135 UnrollOpts.setUpperBound(Enable);
2136 } else {
2137 return make_error<StringError>(
2138 formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
2139 inconvertibleErrorCode());
2140 }
2141 }
2142 return UnrollOpts;
2143 }
2144
parseMSanPassOptions(StringRef Params)2145 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
2146 MemorySanitizerOptions Result;
2147 while (!Params.empty()) {
2148 StringRef ParamName;
2149 std::tie(ParamName, Params) = Params.split(';');
2150
2151 if (ParamName == "recover") {
2152 Result.Recover = true;
2153 } else if (ParamName == "kernel") {
2154 Result.Kernel = true;
2155 } else if (ParamName.consume_front("track-origins=")) {
2156 if (ParamName.getAsInteger(0, Result.TrackOrigins))
2157 return make_error<StringError>(
2158 formatv("invalid argument to MemorySanitizer pass track-origins "
2159 "parameter: '{0}' ",
2160 ParamName)
2161 .str(),
2162 inconvertibleErrorCode());
2163 } else {
2164 return make_error<StringError>(
2165 formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
2166 .str(),
2167 inconvertibleErrorCode());
2168 }
2169 }
2170 return Result;
2171 }
2172
2173 /// Parser of parameters for SimplifyCFG pass.
parseSimplifyCFGOptions(StringRef Params)2174 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
2175 SimplifyCFGOptions Result;
2176 while (!Params.empty()) {
2177 StringRef ParamName;
2178 std::tie(ParamName, Params) = Params.split(';');
2179
2180 bool Enable = !ParamName.consume_front("no-");
2181 if (ParamName == "forward-switch-cond") {
2182 Result.forwardSwitchCondToPhi(Enable);
2183 } else if (ParamName == "switch-to-lookup") {
2184 Result.convertSwitchToLookupTable(Enable);
2185 } else if (ParamName == "keep-loops") {
2186 Result.needCanonicalLoops(Enable);
2187 } else if (ParamName == "hoist-common-insts") {
2188 Result.hoistCommonInsts(Enable);
2189 } else if (ParamName == "sink-common-insts") {
2190 Result.sinkCommonInsts(Enable);
2191 } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
2192 APInt BonusInstThreshold;
2193 if (ParamName.getAsInteger(0, BonusInstThreshold))
2194 return make_error<StringError>(
2195 formatv("invalid argument to SimplifyCFG pass bonus-threshold "
2196 "parameter: '{0}' ",
2197 ParamName).str(),
2198 inconvertibleErrorCode());
2199 Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
2200 } else {
2201 return make_error<StringError>(
2202 formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
2203 inconvertibleErrorCode());
2204 }
2205 }
2206 return Result;
2207 }
2208
2209 /// Parser of parameters for LoopVectorize pass.
parseLoopVectorizeOptions(StringRef Params)2210 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
2211 LoopVectorizeOptions Opts;
2212 while (!Params.empty()) {
2213 StringRef ParamName;
2214 std::tie(ParamName, Params) = Params.split(';');
2215
2216 bool Enable = !ParamName.consume_front("no-");
2217 if (ParamName == "interleave-forced-only") {
2218 Opts.setInterleaveOnlyWhenForced(Enable);
2219 } else if (ParamName == "vectorize-forced-only") {
2220 Opts.setVectorizeOnlyWhenForced(Enable);
2221 } else {
2222 return make_error<StringError>(
2223 formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
2224 inconvertibleErrorCode());
2225 }
2226 }
2227 return Opts;
2228 }
2229
parseLoopUnswitchOptions(StringRef Params)2230 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
2231 bool Result = false;
2232 while (!Params.empty()) {
2233 StringRef ParamName;
2234 std::tie(ParamName, Params) = Params.split(';');
2235
2236 bool Enable = !ParamName.consume_front("no-");
2237 if (ParamName == "nontrivial") {
2238 Result = Enable;
2239 } else {
2240 return make_error<StringError>(
2241 formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
2242 .str(),
2243 inconvertibleErrorCode());
2244 }
2245 }
2246 return Result;
2247 }
2248
parseMergedLoadStoreMotionOptions(StringRef Params)2249 Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
2250 bool Result = false;
2251 while (!Params.empty()) {
2252 StringRef ParamName;
2253 std::tie(ParamName, Params) = Params.split(';');
2254
2255 bool Enable = !ParamName.consume_front("no-");
2256 if (ParamName == "split-footer-bb") {
2257 Result = Enable;
2258 } else {
2259 return make_error<StringError>(
2260 formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
2261 ParamName)
2262 .str(),
2263 inconvertibleErrorCode());
2264 }
2265 }
2266 return Result;
2267 }
2268
parseGVNOptions(StringRef Params)2269 Expected<GVNOptions> parseGVNOptions(StringRef Params) {
2270 GVNOptions Result;
2271 while (!Params.empty()) {
2272 StringRef ParamName;
2273 std::tie(ParamName, Params) = Params.split(';');
2274
2275 bool Enable = !ParamName.consume_front("no-");
2276 if (ParamName == "pre") {
2277 Result.setPRE(Enable);
2278 } else if (ParamName == "load-pre") {
2279 Result.setLoadPRE(Enable);
2280 } else if (ParamName == "split-backedge-load-pre") {
2281 Result.setLoadPRESplitBackedge(Enable);
2282 } else if (ParamName == "memdep") {
2283 Result.setMemDep(Enable);
2284 } else {
2285 return make_error<StringError>(
2286 formatv("invalid GVN pass parameter '{0}' ", ParamName).str(),
2287 inconvertibleErrorCode());
2288 }
2289 }
2290 return Result;
2291 }
2292
2293 Expected<StackLifetime::LivenessType>
parseStackLifetimeOptions(StringRef Params)2294 parseStackLifetimeOptions(StringRef Params) {
2295 StackLifetime::LivenessType Result = StackLifetime::LivenessType::May;
2296 while (!Params.empty()) {
2297 StringRef ParamName;
2298 std::tie(ParamName, Params) = Params.split(';');
2299
2300 if (ParamName == "may") {
2301 Result = StackLifetime::LivenessType::May;
2302 } else if (ParamName == "must") {
2303 Result = StackLifetime::LivenessType::Must;
2304 } else {
2305 return make_error<StringError>(
2306 formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(),
2307 inconvertibleErrorCode());
2308 }
2309 }
2310 return Result;
2311 }
2312
2313 } // namespace
2314
2315 /// Tests whether a pass name starts with a valid prefix for a default pipeline
2316 /// alias.
startsWithDefaultPipelineAliasPrefix(StringRef Name)2317 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
2318 return Name.startswith("default") || Name.startswith("thinlto") ||
2319 Name.startswith("lto");
2320 }
2321
2322 /// Tests whether registered callbacks will accept a given pass name.
2323 ///
2324 /// When parsing a pipeline text, the type of the outermost pipeline may be
2325 /// omitted, in which case the type is automatically determined from the first
2326 /// pass name in the text. This may be a name that is handled through one of the
2327 /// callbacks. We check this through the oridinary parsing callbacks by setting
2328 /// up a dummy PassManager in order to not force the client to also handle this
2329 /// type of query.
2330 template <typename PassManagerT, typename CallbacksT>
callbacksAcceptPassName(StringRef Name,CallbacksT & Callbacks)2331 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
2332 if (!Callbacks.empty()) {
2333 PassManagerT DummyPM;
2334 for (auto &CB : Callbacks)
2335 if (CB(Name, DummyPM, {}))
2336 return true;
2337 }
2338 return false;
2339 }
2340
2341 template <typename CallbacksT>
isModulePassName(StringRef Name,CallbacksT & Callbacks)2342 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
2343 // Manually handle aliases for pre-configured pipeline fragments.
2344 if (startsWithDefaultPipelineAliasPrefix(Name))
2345 return DefaultAliasRegex.match(Name);
2346
2347 // Explicitly handle pass manager names.
2348 if (Name == "module")
2349 return true;
2350 if (Name == "cgscc")
2351 return true;
2352 if (Name == "function")
2353 return true;
2354
2355 // Explicitly handle custom-parsed pass names.
2356 if (parseRepeatPassName(Name))
2357 return true;
2358
2359 #define MODULE_PASS(NAME, CREATE_PASS) \
2360 if (Name == NAME) \
2361 return true;
2362 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2363 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2364 return true;
2365 #include "PassRegistry.def"
2366
2367 return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
2368 }
2369
2370 template <typename CallbacksT>
isCGSCCPassName(StringRef Name,CallbacksT & Callbacks)2371 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
2372 // Explicitly handle pass manager names.
2373 if (Name == "cgscc")
2374 return true;
2375 if (Name == "function")
2376 return true;
2377
2378 // Explicitly handle custom-parsed pass names.
2379 if (parseRepeatPassName(Name))
2380 return true;
2381 if (parseDevirtPassName(Name))
2382 return true;
2383
2384 #define CGSCC_PASS(NAME, CREATE_PASS) \
2385 if (Name == NAME) \
2386 return true;
2387 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2388 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2389 return true;
2390 #include "PassRegistry.def"
2391
2392 return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
2393 }
2394
2395 template <typename CallbacksT>
isFunctionPassName(StringRef Name,CallbacksT & Callbacks)2396 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
2397 // Explicitly handle pass manager names.
2398 if (Name == "function")
2399 return true;
2400 if (Name == "loop" || Name == "loop-mssa")
2401 return true;
2402
2403 // Explicitly handle custom-parsed pass names.
2404 if (parseRepeatPassName(Name))
2405 return true;
2406
2407 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2408 if (Name == NAME) \
2409 return true;
2410 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2411 if (checkParametrizedPassName(Name, NAME)) \
2412 return true;
2413 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2414 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2415 return true;
2416 #include "PassRegistry.def"
2417
2418 return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
2419 }
2420
2421 template <typename CallbacksT>
isLoopPassName(StringRef Name,CallbacksT & Callbacks)2422 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
2423 // Explicitly handle pass manager names.
2424 if (Name == "loop" || Name == "loop-mssa")
2425 return true;
2426
2427 // Explicitly handle custom-parsed pass names.
2428 if (parseRepeatPassName(Name))
2429 return true;
2430
2431 #define LOOP_PASS(NAME, CREATE_PASS) \
2432 if (Name == NAME) \
2433 return true;
2434 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2435 if (checkParametrizedPassName(Name, NAME)) \
2436 return true;
2437 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2438 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2439 return true;
2440 #include "PassRegistry.def"
2441
2442 return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
2443 }
2444
2445 Optional<std::vector<PassBuilder::PipelineElement>>
parsePipelineText(StringRef Text)2446 PassBuilder::parsePipelineText(StringRef Text) {
2447 std::vector<PipelineElement> ResultPipeline;
2448
2449 SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
2450 &ResultPipeline};
2451 for (;;) {
2452 std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
2453 size_t Pos = Text.find_first_of(",()");
2454 Pipeline.push_back({Text.substr(0, Pos), {}});
2455
2456 // If we have a single terminating name, we're done.
2457 if (Pos == Text.npos)
2458 break;
2459
2460 char Sep = Text[Pos];
2461 Text = Text.substr(Pos + 1);
2462 if (Sep == ',')
2463 // Just a name ending in a comma, continue.
2464 continue;
2465
2466 if (Sep == '(') {
2467 // Push the inner pipeline onto the stack to continue processing.
2468 PipelineStack.push_back(&Pipeline.back().InnerPipeline);
2469 continue;
2470 }
2471
2472 assert(Sep == ')' && "Bogus separator!");
2473 // When handling the close parenthesis, we greedily consume them to avoid
2474 // empty strings in the pipeline.
2475 do {
2476 // If we try to pop the outer pipeline we have unbalanced parentheses.
2477 if (PipelineStack.size() == 1)
2478 return None;
2479
2480 PipelineStack.pop_back();
2481 } while (Text.consume_front(")"));
2482
2483 // Check if we've finished parsing.
2484 if (Text.empty())
2485 break;
2486
2487 // Otherwise, the end of an inner pipeline always has to be followed by
2488 // a comma, and then we can continue.
2489 if (!Text.consume_front(","))
2490 return None;
2491 }
2492
2493 if (PipelineStack.size() > 1)
2494 // Unbalanced paretheses.
2495 return None;
2496
2497 assert(PipelineStack.back() == &ResultPipeline &&
2498 "Wrong pipeline at the bottom of the stack!");
2499 return {std::move(ResultPipeline)};
2500 }
2501
parseModulePass(ModulePassManager & MPM,const PipelineElement & E)2502 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
2503 const PipelineElement &E) {
2504 auto &Name = E.Name;
2505 auto &InnerPipeline = E.InnerPipeline;
2506
2507 // First handle complex passes like the pass managers which carry pipelines.
2508 if (!InnerPipeline.empty()) {
2509 if (Name == "module") {
2510 ModulePassManager NestedMPM;
2511 if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
2512 return Err;
2513 MPM.addPass(std::move(NestedMPM));
2514 return Error::success();
2515 }
2516 if (Name == "cgscc") {
2517 CGSCCPassManager CGPM;
2518 if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline))
2519 return Err;
2520 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2521 return Error::success();
2522 }
2523 if (Name == "function") {
2524 FunctionPassManager FPM;
2525 if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
2526 return Err;
2527 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
2528 return Error::success();
2529 }
2530 if (auto Count = parseRepeatPassName(Name)) {
2531 ModulePassManager NestedMPM;
2532 if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
2533 return Err;
2534 MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
2535 return Error::success();
2536 }
2537
2538 for (auto &C : ModulePipelineParsingCallbacks)
2539 if (C(Name, MPM, InnerPipeline))
2540 return Error::success();
2541
2542 // Normal passes can't have pipelines.
2543 return make_error<StringError>(
2544 formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
2545 inconvertibleErrorCode());
2546 ;
2547 }
2548
2549 // Manually handle aliases for pre-configured pipeline fragments.
2550 if (startsWithDefaultPipelineAliasPrefix(Name)) {
2551 SmallVector<StringRef, 3> Matches;
2552 if (!DefaultAliasRegex.match(Name, &Matches))
2553 return make_error<StringError>(
2554 formatv("unknown default pipeline alias '{0}'", Name).str(),
2555 inconvertibleErrorCode());
2556
2557 assert(Matches.size() == 3 && "Must capture two matched strings!");
2558
2559 OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
2560 .Case("O0", OptimizationLevel::O0)
2561 .Case("O1", OptimizationLevel::O1)
2562 .Case("O2", OptimizationLevel::O2)
2563 .Case("O3", OptimizationLevel::O3)
2564 .Case("Os", OptimizationLevel::Os)
2565 .Case("Oz", OptimizationLevel::Oz);
2566 if (L == OptimizationLevel::O0 && Matches[1] != "thinlto" &&
2567 Matches[1] != "lto") {
2568 MPM.addPass(buildO0DefaultPipeline(L, Matches[1] == "thinlto-pre-link" ||
2569 Matches[1] == "lto-pre-link"));
2570 return Error::success();
2571 }
2572
2573 // This is consistent with old pass manager invoked via opt, but
2574 // inconsistent with clang. Clang doesn't enable loop vectorization
2575 // but does enable slp vectorization at Oz.
2576 PTO.LoopVectorization =
2577 L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2578 PTO.SLPVectorization =
2579 L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2580
2581 if (Matches[1] == "default") {
2582 MPM.addPass(buildPerModuleDefaultPipeline(L));
2583 } else if (Matches[1] == "thinlto-pre-link") {
2584 MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L));
2585 } else if (Matches[1] == "thinlto") {
2586 MPM.addPass(buildThinLTODefaultPipeline(L, nullptr));
2587 } else if (Matches[1] == "lto-pre-link") {
2588 MPM.addPass(buildLTOPreLinkDefaultPipeline(L));
2589 } else {
2590 assert(Matches[1] == "lto" && "Not one of the matched options!");
2591 MPM.addPass(buildLTODefaultPipeline(L, nullptr));
2592 }
2593 return Error::success();
2594 }
2595
2596 // Finally expand the basic registered passes from the .inc file.
2597 #define MODULE_PASS(NAME, CREATE_PASS) \
2598 if (Name == NAME) { \
2599 MPM.addPass(CREATE_PASS); \
2600 return Error::success(); \
2601 }
2602 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2603 if (Name == "require<" NAME ">") { \
2604 MPM.addPass( \
2605 RequireAnalysisPass< \
2606 std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \
2607 return Error::success(); \
2608 } \
2609 if (Name == "invalidate<" NAME ">") { \
2610 MPM.addPass(InvalidateAnalysisPass< \
2611 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2612 return Error::success(); \
2613 }
2614 #define CGSCC_PASS(NAME, CREATE_PASS) \
2615 if (Name == NAME) { \
2616 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS)); \
2617 return Error::success(); \
2618 }
2619 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2620 if (Name == NAME) { \
2621 MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS)); \
2622 return Error::success(); \
2623 }
2624 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2625 if (checkParametrizedPassName(Name, NAME)) { \
2626 auto Params = parsePassParameters(PARSER, Name, NAME); \
2627 if (!Params) \
2628 return Params.takeError(); \
2629 MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2630 return Error::success(); \
2631 }
2632 #define LOOP_PASS(NAME, CREATE_PASS) \
2633 if (Name == NAME) { \
2634 MPM.addPass(createModuleToFunctionPassAdaptor( \
2635 createFunctionToLoopPassAdaptor(CREATE_PASS, false, false))); \
2636 return Error::success(); \
2637 }
2638 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2639 if (checkParametrizedPassName(Name, NAME)) { \
2640 auto Params = parsePassParameters(PARSER, Name, NAME); \
2641 if (!Params) \
2642 return Params.takeError(); \
2643 MPM.addPass( \
2644 createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2645 CREATE_PASS(Params.get()), false, false))); \
2646 return Error::success(); \
2647 }
2648 #include "PassRegistry.def"
2649
2650 for (auto &C : ModulePipelineParsingCallbacks)
2651 if (C(Name, MPM, InnerPipeline))
2652 return Error::success();
2653 return make_error<StringError>(
2654 formatv("unknown module pass '{0}'", Name).str(),
2655 inconvertibleErrorCode());
2656 }
2657
parseCGSCCPass(CGSCCPassManager & CGPM,const PipelineElement & E)2658 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
2659 const PipelineElement &E) {
2660 auto &Name = E.Name;
2661 auto &InnerPipeline = E.InnerPipeline;
2662
2663 // First handle complex passes like the pass managers which carry pipelines.
2664 if (!InnerPipeline.empty()) {
2665 if (Name == "cgscc") {
2666 CGSCCPassManager NestedCGPM;
2667 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2668 return Err;
2669 // Add the nested pass manager with the appropriate adaptor.
2670 CGPM.addPass(std::move(NestedCGPM));
2671 return Error::success();
2672 }
2673 if (Name == "function") {
2674 FunctionPassManager FPM;
2675 if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
2676 return Err;
2677 // Add the nested pass manager with the appropriate adaptor.
2678 CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
2679 return Error::success();
2680 }
2681 if (auto Count = parseRepeatPassName(Name)) {
2682 CGSCCPassManager NestedCGPM;
2683 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2684 return Err;
2685 CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
2686 return Error::success();
2687 }
2688 if (auto MaxRepetitions = parseDevirtPassName(Name)) {
2689 CGSCCPassManager NestedCGPM;
2690 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2691 return Err;
2692 CGPM.addPass(
2693 createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
2694 return Error::success();
2695 }
2696
2697 for (auto &C : CGSCCPipelineParsingCallbacks)
2698 if (C(Name, CGPM, InnerPipeline))
2699 return Error::success();
2700
2701 // Normal passes can't have pipelines.
2702 return make_error<StringError>(
2703 formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
2704 inconvertibleErrorCode());
2705 }
2706
2707 // Now expand the basic registered passes from the .inc file.
2708 #define CGSCC_PASS(NAME, CREATE_PASS) \
2709 if (Name == NAME) { \
2710 CGPM.addPass(CREATE_PASS); \
2711 return Error::success(); \
2712 }
2713 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2714 if (Name == "require<" NAME ">") { \
2715 CGPM.addPass(RequireAnalysisPass< \
2716 std::remove_reference<decltype(CREATE_PASS)>::type, \
2717 LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
2718 CGSCCUpdateResult &>()); \
2719 return Error::success(); \
2720 } \
2721 if (Name == "invalidate<" NAME ">") { \
2722 CGPM.addPass(InvalidateAnalysisPass< \
2723 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2724 return Error::success(); \
2725 }
2726 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2727 if (Name == NAME) { \
2728 CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS)); \
2729 return Error::success(); \
2730 }
2731 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2732 if (checkParametrizedPassName(Name, NAME)) { \
2733 auto Params = parsePassParameters(PARSER, Name, NAME); \
2734 if (!Params) \
2735 return Params.takeError(); \
2736 CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2737 return Error::success(); \
2738 }
2739 #define LOOP_PASS(NAME, CREATE_PASS) \
2740 if (Name == NAME) { \
2741 CGPM.addPass(createCGSCCToFunctionPassAdaptor( \
2742 createFunctionToLoopPassAdaptor(CREATE_PASS, false, false))); \
2743 return Error::success(); \
2744 }
2745 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2746 if (checkParametrizedPassName(Name, NAME)) { \
2747 auto Params = parsePassParameters(PARSER, Name, NAME); \
2748 if (!Params) \
2749 return Params.takeError(); \
2750 CGPM.addPass( \
2751 createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2752 CREATE_PASS(Params.get()), false, false))); \
2753 return Error::success(); \
2754 }
2755 #include "PassRegistry.def"
2756
2757 for (auto &C : CGSCCPipelineParsingCallbacks)
2758 if (C(Name, CGPM, InnerPipeline))
2759 return Error::success();
2760 return make_error<StringError>(
2761 formatv("unknown cgscc pass '{0}'", Name).str(),
2762 inconvertibleErrorCode());
2763 }
2764
parseFunctionPass(FunctionPassManager & FPM,const PipelineElement & E)2765 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
2766 const PipelineElement &E) {
2767 auto &Name = E.Name;
2768 auto &InnerPipeline = E.InnerPipeline;
2769
2770 // First handle complex passes like the pass managers which carry pipelines.
2771 if (!InnerPipeline.empty()) {
2772 if (Name == "function") {
2773 FunctionPassManager NestedFPM;
2774 if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
2775 return Err;
2776 // Add the nested pass manager with the appropriate adaptor.
2777 FPM.addPass(std::move(NestedFPM));
2778 return Error::success();
2779 }
2780 if (Name == "loop" || Name == "loop-mssa") {
2781 LoopPassManager LPM;
2782 if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline))
2783 return Err;
2784 // Add the nested pass manager with the appropriate adaptor.
2785 bool UseMemorySSA = (Name == "loop-mssa");
2786 bool UseBFI = llvm::any_of(
2787 InnerPipeline, [](auto Pipeline) { return Pipeline.Name == "licm"; });
2788 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
2789 UseBFI));
2790 return Error::success();
2791 }
2792 if (auto Count = parseRepeatPassName(Name)) {
2793 FunctionPassManager NestedFPM;
2794 if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
2795 return Err;
2796 FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
2797 return Error::success();
2798 }
2799
2800 for (auto &C : FunctionPipelineParsingCallbacks)
2801 if (C(Name, FPM, InnerPipeline))
2802 return Error::success();
2803
2804 // Normal passes can't have pipelines.
2805 return make_error<StringError>(
2806 formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
2807 inconvertibleErrorCode());
2808 }
2809
2810 // Now expand the basic registered passes from the .inc file.
2811 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2812 if (Name == NAME) { \
2813 FPM.addPass(CREATE_PASS); \
2814 return Error::success(); \
2815 }
2816 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2817 if (checkParametrizedPassName(Name, NAME)) { \
2818 auto Params = parsePassParameters(PARSER, Name, NAME); \
2819 if (!Params) \
2820 return Params.takeError(); \
2821 FPM.addPass(CREATE_PASS(Params.get())); \
2822 return Error::success(); \
2823 }
2824 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2825 if (Name == "require<" NAME ">") { \
2826 FPM.addPass( \
2827 RequireAnalysisPass< \
2828 std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \
2829 return Error::success(); \
2830 } \
2831 if (Name == "invalidate<" NAME ">") { \
2832 FPM.addPass(InvalidateAnalysisPass< \
2833 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2834 return Error::success(); \
2835 }
2836 // FIXME: UseMemorySSA is set to false. Maybe we could do things like:
2837 // bool UseMemorySSA = !("canon-freeze" || "loop-predication" ||
2838 // "guard-widening");
2839 // The risk is that it may become obsolete if we're not careful.
2840 #define LOOP_PASS(NAME, CREATE_PASS) \
2841 if (Name == NAME) { \
2842 FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false)); \
2843 return Error::success(); \
2844 }
2845 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2846 if (checkParametrizedPassName(Name, NAME)) { \
2847 auto Params = parsePassParameters(PARSER, Name, NAME); \
2848 if (!Params) \
2849 return Params.takeError(); \
2850 FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \
2851 false, false)); \
2852 return Error::success(); \
2853 }
2854 #include "PassRegistry.def"
2855
2856 for (auto &C : FunctionPipelineParsingCallbacks)
2857 if (C(Name, FPM, InnerPipeline))
2858 return Error::success();
2859 return make_error<StringError>(
2860 formatv("unknown function pass '{0}'", Name).str(),
2861 inconvertibleErrorCode());
2862 }
2863
parseLoopPass(LoopPassManager & LPM,const PipelineElement & E)2864 Error PassBuilder::parseLoopPass(LoopPassManager &LPM,
2865 const PipelineElement &E) {
2866 StringRef Name = E.Name;
2867 auto &InnerPipeline = E.InnerPipeline;
2868
2869 // First handle complex passes like the pass managers which carry pipelines.
2870 if (!InnerPipeline.empty()) {
2871 if (Name == "loop") {
2872 LoopPassManager NestedLPM;
2873 if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
2874 return Err;
2875 // Add the nested pass manager with the appropriate adaptor.
2876 LPM.addPass(std::move(NestedLPM));
2877 return Error::success();
2878 }
2879 if (auto Count = parseRepeatPassName(Name)) {
2880 LoopPassManager NestedLPM;
2881 if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
2882 return Err;
2883 LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2884 return Error::success();
2885 }
2886
2887 for (auto &C : LoopPipelineParsingCallbacks)
2888 if (C(Name, LPM, InnerPipeline))
2889 return Error::success();
2890
2891 // Normal passes can't have pipelines.
2892 return make_error<StringError>(
2893 formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2894 inconvertibleErrorCode());
2895 }
2896
2897 // Now expand the basic registered passes from the .inc file.
2898 #define LOOP_PASS(NAME, CREATE_PASS) \
2899 if (Name == NAME) { \
2900 LPM.addPass(CREATE_PASS); \
2901 return Error::success(); \
2902 }
2903 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2904 if (checkParametrizedPassName(Name, NAME)) { \
2905 auto Params = parsePassParameters(PARSER, Name, NAME); \
2906 if (!Params) \
2907 return Params.takeError(); \
2908 LPM.addPass(CREATE_PASS(Params.get())); \
2909 return Error::success(); \
2910 }
2911 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2912 if (Name == "require<" NAME ">") { \
2913 LPM.addPass(RequireAnalysisPass< \
2914 std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \
2915 LoopAnalysisManager, LoopStandardAnalysisResults &, \
2916 LPMUpdater &>()); \
2917 return Error::success(); \
2918 } \
2919 if (Name == "invalidate<" NAME ">") { \
2920 LPM.addPass(InvalidateAnalysisPass< \
2921 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2922 return Error::success(); \
2923 }
2924 #include "PassRegistry.def"
2925
2926 for (auto &C : LoopPipelineParsingCallbacks)
2927 if (C(Name, LPM, InnerPipeline))
2928 return Error::success();
2929 return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2930 inconvertibleErrorCode());
2931 }
2932
parseAAPassName(AAManager & AA,StringRef Name)2933 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2934 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2935 if (Name == NAME) { \
2936 AA.registerModuleAnalysis< \
2937 std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2938 return true; \
2939 }
2940 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2941 if (Name == NAME) { \
2942 AA.registerFunctionAnalysis< \
2943 std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2944 return true; \
2945 }
2946 #include "PassRegistry.def"
2947
2948 for (auto &C : AAParsingCallbacks)
2949 if (C(Name, AA))
2950 return true;
2951 return false;
2952 }
2953
parseLoopPassPipeline(LoopPassManager & LPM,ArrayRef<PipelineElement> Pipeline)2954 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2955 ArrayRef<PipelineElement> Pipeline) {
2956 for (const auto &Element : Pipeline) {
2957 if (auto Err = parseLoopPass(LPM, Element))
2958 return Err;
2959 }
2960 return Error::success();
2961 }
2962
parseFunctionPassPipeline(FunctionPassManager & FPM,ArrayRef<PipelineElement> Pipeline)2963 Error PassBuilder::parseFunctionPassPipeline(
2964 FunctionPassManager &FPM, ArrayRef<PipelineElement> Pipeline) {
2965 for (const auto &Element : Pipeline) {
2966 if (auto Err = parseFunctionPass(FPM, Element))
2967 return Err;
2968 }
2969 return Error::success();
2970 }
2971
parseCGSCCPassPipeline(CGSCCPassManager & CGPM,ArrayRef<PipelineElement> Pipeline)2972 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2973 ArrayRef<PipelineElement> Pipeline) {
2974 for (const auto &Element : Pipeline) {
2975 if (auto Err = parseCGSCCPass(CGPM, Element))
2976 return Err;
2977 }
2978 return Error::success();
2979 }
2980
crossRegisterProxies(LoopAnalysisManager & LAM,FunctionAnalysisManager & FAM,CGSCCAnalysisManager & CGAM,ModuleAnalysisManager & MAM)2981 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
2982 FunctionAnalysisManager &FAM,
2983 CGSCCAnalysisManager &CGAM,
2984 ModuleAnalysisManager &MAM) {
2985 MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2986 MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2987 CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2988 FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2989 FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2990 FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2991 LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2992 }
2993
parseModulePassPipeline(ModulePassManager & MPM,ArrayRef<PipelineElement> Pipeline)2994 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2995 ArrayRef<PipelineElement> Pipeline) {
2996 for (const auto &Element : Pipeline) {
2997 if (auto Err = parseModulePass(MPM, Element))
2998 return Err;
2999 }
3000 return Error::success();
3001 }
3002
3003 // Primary pass pipeline description parsing routine for a \c ModulePassManager
3004 // FIXME: Should this routine accept a TargetMachine or require the caller to
3005 // pre-populate the analysis managers with target-specific stuff?
parsePassPipeline(ModulePassManager & MPM,StringRef PipelineText)3006 Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
3007 StringRef PipelineText) {
3008 auto Pipeline = parsePipelineText(PipelineText);
3009 if (!Pipeline || Pipeline->empty())
3010 return make_error<StringError>(
3011 formatv("invalid pipeline '{0}'", PipelineText).str(),
3012 inconvertibleErrorCode());
3013
3014 // If the first name isn't at the module layer, wrap the pipeline up
3015 // automatically.
3016 StringRef FirstName = Pipeline->front().Name;
3017
3018 if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
3019 if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
3020 Pipeline = {{"cgscc", std::move(*Pipeline)}};
3021 } else if (isFunctionPassName(FirstName,
3022 FunctionPipelineParsingCallbacks)) {
3023 Pipeline = {{"function", std::move(*Pipeline)}};
3024 } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
3025 Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
3026 } else {
3027 for (auto &C : TopLevelPipelineParsingCallbacks)
3028 if (C(MPM, *Pipeline))
3029 return Error::success();
3030
3031 // Unknown pass or pipeline name!
3032 auto &InnerPipeline = Pipeline->front().InnerPipeline;
3033 return make_error<StringError>(
3034 formatv("unknown {0} name '{1}'",
3035 (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
3036 .str(),
3037 inconvertibleErrorCode());
3038 }
3039 }
3040
3041 if (auto Err = parseModulePassPipeline(MPM, *Pipeline))
3042 return Err;
3043 return Error::success();
3044 }
3045
3046 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
parsePassPipeline(CGSCCPassManager & CGPM,StringRef PipelineText)3047 Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
3048 StringRef PipelineText) {
3049 auto Pipeline = parsePipelineText(PipelineText);
3050 if (!Pipeline || Pipeline->empty())
3051 return make_error<StringError>(
3052 formatv("invalid pipeline '{0}'", PipelineText).str(),
3053 inconvertibleErrorCode());
3054
3055 StringRef FirstName = Pipeline->front().Name;
3056 if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
3057 return make_error<StringError>(
3058 formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
3059 PipelineText)
3060 .str(),
3061 inconvertibleErrorCode());
3062
3063 if (auto Err = parseCGSCCPassPipeline(CGPM, *Pipeline))
3064 return Err;
3065 return Error::success();
3066 }
3067
3068 // Primary pass pipeline description parsing routine for a \c
3069 // FunctionPassManager
parsePassPipeline(FunctionPassManager & FPM,StringRef PipelineText)3070 Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
3071 StringRef PipelineText) {
3072 auto Pipeline = parsePipelineText(PipelineText);
3073 if (!Pipeline || Pipeline->empty())
3074 return make_error<StringError>(
3075 formatv("invalid pipeline '{0}'", PipelineText).str(),
3076 inconvertibleErrorCode());
3077
3078 StringRef FirstName = Pipeline->front().Name;
3079 if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
3080 return make_error<StringError>(
3081 formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
3082 PipelineText)
3083 .str(),
3084 inconvertibleErrorCode());
3085
3086 if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline))
3087 return Err;
3088 return Error::success();
3089 }
3090
3091 // Primary pass pipeline description parsing routine for a \c LoopPassManager
parsePassPipeline(LoopPassManager & CGPM,StringRef PipelineText)3092 Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
3093 StringRef PipelineText) {
3094 auto Pipeline = parsePipelineText(PipelineText);
3095 if (!Pipeline || Pipeline->empty())
3096 return make_error<StringError>(
3097 formatv("invalid pipeline '{0}'", PipelineText).str(),
3098 inconvertibleErrorCode());
3099
3100 if (auto Err = parseLoopPassPipeline(CGPM, *Pipeline))
3101 return Err;
3102
3103 return Error::success();
3104 }
3105
parseAAPipeline(AAManager & AA,StringRef PipelineText)3106 Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
3107 // If the pipeline just consists of the word 'default' just replace the AA
3108 // manager with our default one.
3109 if (PipelineText == "default") {
3110 AA = buildDefaultAAPipeline();
3111 return Error::success();
3112 }
3113
3114 while (!PipelineText.empty()) {
3115 StringRef Name;
3116 std::tie(Name, PipelineText) = PipelineText.split(',');
3117 if (!parseAAPassName(AA, Name))
3118 return make_error<StringError>(
3119 formatv("unknown alias analysis name '{0}'", Name).str(),
3120 inconvertibleErrorCode());
3121 }
3122
3123 return Error::success();
3124 }
3125
isAAPassName(StringRef PassName)3126 bool PassBuilder::isAAPassName(StringRef PassName) {
3127 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3128 if (PassName == NAME) \
3129 return true;
3130 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3131 if (PassName == NAME) \
3132 return true;
3133 #include "PassRegistry.def"
3134 return false;
3135 }
3136
isAnalysisPassName(StringRef PassName)3137 bool PassBuilder::isAnalysisPassName(StringRef PassName) {
3138 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
3139 if (PassName == NAME) \
3140 return true;
3141 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
3142 if (PassName == NAME) \
3143 return true;
3144 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
3145 if (PassName == NAME) \
3146 return true;
3147 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
3148 if (PassName == NAME) \
3149 return true;
3150 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3151 if (PassName == NAME) \
3152 return true;
3153 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3154 if (PassName == NAME) \
3155 return true;
3156 #include "PassRegistry.def"
3157 return false;
3158 }
3159
printPassName(StringRef PassName,raw_ostream & OS)3160 static void printPassName(StringRef PassName, raw_ostream &OS) {
3161 OS << " " << PassName << "\n";
3162 }
3163
printPassNames(raw_ostream & OS)3164 void PassBuilder::printPassNames(raw_ostream &OS) {
3165 // TODO: print pass descriptions when they are available
3166
3167 OS << "Module passes:\n";
3168 #define MODULE_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3169 #include "PassRegistry.def"
3170
3171 OS << "Module analyses:\n";
3172 #define MODULE_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3173 #include "PassRegistry.def"
3174
3175 OS << "Module alias analyses:\n";
3176 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3177 #include "PassRegistry.def"
3178
3179 OS << "CGSCC passes:\n";
3180 #define CGSCC_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3181 #include "PassRegistry.def"
3182
3183 OS << "CGSCC analyses:\n";
3184 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3185 #include "PassRegistry.def"
3186
3187 OS << "Function passes:\n";
3188 #define FUNCTION_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3189 #include "PassRegistry.def"
3190
3191 OS << "Function analyses:\n";
3192 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3193 #include "PassRegistry.def"
3194
3195 OS << "Function alias analyses:\n";
3196 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3197 #include "PassRegistry.def"
3198
3199 OS << "Loop passes:\n";
3200 #define LOOP_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3201 #include "PassRegistry.def"
3202
3203 OS << "Loop analyses:\n";
3204 #define LOOP_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3205 #include "PassRegistry.def"
3206 }
3207
registerParseTopLevelPipelineCallback(const std::function<bool (ModulePassManager &,ArrayRef<PipelineElement>)> & C)3208 void PassBuilder::registerParseTopLevelPipelineCallback(
3209 const std::function<bool(ModulePassManager &, ArrayRef<PipelineElement>)>
3210 &C) {
3211 TopLevelPipelineParsingCallbacks.push_back(C);
3212 }
3213