1 //===- Parsing, selection, and construction of pass pipelines -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
13 /// pipelines.
14 ///
15 //===----------------------------------------------------------------------===//
16 
17 #include "llvm/Passes/PassBuilder.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/Analysis/AliasAnalysis.h"
20 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
21 #include "llvm/Analysis/AssumptionCache.h"
22 #include "llvm/Analysis/BasicAliasAnalysis.h"
23 #include "llvm/Analysis/BlockFrequencyInfo.h"
24 #include "llvm/Analysis/BranchProbabilityInfo.h"
25 #include "llvm/Analysis/CFGPrinter.h"
26 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
27 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
28 #include "llvm/Analysis/CGSCCPassManager.h"
29 #include "llvm/Analysis/CallGraph.h"
30 #include "llvm/Analysis/DDG.h"
31 #include "llvm/Analysis/DemandedBits.h"
32 #include "llvm/Analysis/DependenceAnalysis.h"
33 #include "llvm/Analysis/DominanceFrontier.h"
34 #include "llvm/Analysis/GlobalsModRef.h"
35 #include "llvm/Analysis/IVUsers.h"
36 #include "llvm/Analysis/LazyCallGraph.h"
37 #include "llvm/Analysis/LazyValueInfo.h"
38 #include "llvm/Analysis/LoopAccessAnalysis.h"
39 #include "llvm/Analysis/LoopCacheAnalysis.h"
40 #include "llvm/Analysis/LoopInfo.h"
41 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
42 #include "llvm/Analysis/MemorySSA.h"
43 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
44 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
45 #include "llvm/Analysis/PhiValues.h"
46 #include "llvm/Analysis/PostDominators.h"
47 #include "llvm/Analysis/ProfileSummaryInfo.h"
48 #include "llvm/Analysis/RegionInfo.h"
49 #include "llvm/Analysis/ScalarEvolution.h"
50 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
51 #include "llvm/Analysis/ScopedNoAliasAA.h"
52 #include "llvm/Analysis/StackSafetyAnalysis.h"
53 #include "llvm/Analysis/TargetLibraryInfo.h"
54 #include "llvm/Analysis/TargetTransformInfo.h"
55 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
56 #include "llvm/CodeGen/MachineModuleInfo.h"
57 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
58 #include "llvm/CodeGen/UnreachableBlockElim.h"
59 #include "llvm/IR/Dominators.h"
60 #include "llvm/IR/IRPrintingPasses.h"
61 #include "llvm/IR/PassManager.h"
62 #include "llvm/IR/SafepointIRVerifier.h"
63 #include "llvm/IR/Verifier.h"
64 #include "llvm/Support/CommandLine.h"
65 #include "llvm/Support/Debug.h"
66 #include "llvm/Support/FormatVariadic.h"
67 #include "llvm/Support/Regex.h"
68 #include "llvm/Target/TargetMachine.h"
69 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
70 #include "llvm/Transforms/IPO/AlwaysInliner.h"
71 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
72 #include "llvm/Transforms/IPO/Attributor.h"
73 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
74 #include "llvm/Transforms/IPO/ConstantMerge.h"
75 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
76 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
77 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
78 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
79 #include "llvm/Transforms/IPO/FunctionAttrs.h"
80 #include "llvm/Transforms/IPO/FunctionImport.h"
81 #include "llvm/Transforms/IPO/GlobalDCE.h"
82 #include "llvm/Transforms/IPO/GlobalOpt.h"
83 #include "llvm/Transforms/IPO/GlobalSplit.h"
84 #include "llvm/Transforms/IPO/HotColdSplitting.h"
85 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
86 #include "llvm/Transforms/IPO/Inliner.h"
87 #include "llvm/Transforms/IPO/Internalize.h"
88 #include "llvm/Transforms/IPO/LowerTypeTests.h"
89 #include "llvm/Transforms/IPO/MergeFunctions.h"
90 #include "llvm/Transforms/IPO/PartialInlining.h"
91 #include "llvm/Transforms/IPO/SCCP.h"
92 #include "llvm/Transforms/IPO/SampleProfile.h"
93 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
94 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
95 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
96 #include "llvm/Transforms/InstCombine/InstCombine.h"
97 #include "llvm/Transforms/Instrumentation.h"
98 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
99 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
100 #include "llvm/Transforms/Instrumentation/CGProfile.h"
101 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
102 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
103 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
104 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
105 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
106 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
107 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
108 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
109 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
110 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
111 #include "llvm/Transforms/Scalar/ADCE.h"
112 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
113 #include "llvm/Transforms/Scalar/BDCE.h"
114 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
115 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
116 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
117 #include "llvm/Transforms/Scalar/DCE.h"
118 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
119 #include "llvm/Transforms/Scalar/DivRemPairs.h"
120 #include "llvm/Transforms/Scalar/EarlyCSE.h"
121 #include "llvm/Transforms/Scalar/Float2Int.h"
122 #include "llvm/Transforms/Scalar/GVN.h"
123 #include "llvm/Transforms/Scalar/GuardWidening.h"
124 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
125 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
126 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
127 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
128 #include "llvm/Transforms/Scalar/JumpThreading.h"
129 #include "llvm/Transforms/Scalar/LICM.h"
130 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
131 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
132 #include "llvm/Transforms/Scalar/LoopDeletion.h"
133 #include "llvm/Transforms/Scalar/LoopDistribute.h"
134 #include "llvm/Transforms/Scalar/LoopFuse.h"
135 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
136 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
137 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
138 #include "llvm/Transforms/Scalar/LoopPassManager.h"
139 #include "llvm/Transforms/Scalar/LoopPredication.h"
140 #include "llvm/Transforms/Scalar/LoopRotation.h"
141 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
142 #include "llvm/Transforms/Scalar/LoopSink.h"
143 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
144 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
145 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
146 #include "llvm/Transforms/Scalar/LowerAtomic.h"
147 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
148 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
149 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
150 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
151 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
152 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
153 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
154 #include "llvm/Transforms/Scalar/MergeICmps.h"
155 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
156 #include "llvm/Transforms/Scalar/NaryReassociate.h"
157 #include "llvm/Transforms/Scalar/NewGVN.h"
158 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
159 #include "llvm/Transforms/Scalar/Reassociate.h"
160 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
161 #include "llvm/Transforms/Scalar/SCCP.h"
162 #include "llvm/Transforms/Scalar/SROA.h"
163 #include "llvm/Transforms/Scalar/Scalarizer.h"
164 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
165 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
166 #include "llvm/Transforms/Scalar/Sink.h"
167 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
168 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
169 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
170 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
171 #include "llvm/Transforms/Utils/AddDiscriminators.h"
172 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
173 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
174 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
175 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
176 #include "llvm/Transforms/Utils/LCSSA.h"
177 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
178 #include "llvm/Transforms/Utils/LoopSimplify.h"
179 #include "llvm/Transforms/Utils/LowerInvoke.h"
180 #include "llvm/Transforms/Utils/Mem2Reg.h"
181 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
182 #include "llvm/Transforms/Utils/SymbolRewriter.h"
183 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
184 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
185 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
186 
187 using namespace llvm;
188 
189 static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
190                                              cl::ReallyHidden, cl::init(4));
191 static cl::opt<bool>
192     RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
193                        cl::Hidden, cl::ZeroOrMore,
194                        cl::desc("Run Partial inlinining pass"));
195 
196 static cl::opt<int> PreInlineThreshold(
197     "npm-preinline-threshold", cl::Hidden, cl::init(75), cl::ZeroOrMore,
198     cl::desc("Control the amount of inlining in pre-instrumentation inliner "
199              "(default = 75)"));
200 
201 static cl::opt<bool>
202     RunNewGVN("enable-npm-newgvn", cl::init(false),
203               cl::Hidden, cl::ZeroOrMore,
204               cl::desc("Run NewGVN instead of GVN"));
205 
206 static cl::opt<bool> EnableGVNHoist(
207     "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
208     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
209 
210 static cl::opt<bool> EnableGVNSink(
211     "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
212     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
213 
214 static cl::opt<bool> EnableUnrollAndJam(
215     "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden,
216     cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
217 
218 static cl::opt<bool> EnableSyntheticCounts(
219     "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
220     cl::desc("Run synthetic function entry count generation "
221              "pass"));
222 
223 static const Regex DefaultAliasRegex(
224     "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
225 
226 // This option is used in simplifying testing SampleFDO optimizations for
227 // profile loading.
228 static cl::opt<bool>
229     EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
230               cl::desc("Enable control height reduction optimization (CHR)"));
231 
232 PipelineTuningOptions::PipelineTuningOptions() {
233   LoopInterleaving = EnableLoopInterleaving;
234   LoopVectorization = EnableLoopVectorization;
235   SLPVectorization = RunSLPVectorization;
236   LoopUnrolling = true;
237   ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
238   LicmMssaOptCap = SetLicmMssaOptCap;
239   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
240 }
241 
242 extern cl::opt<bool> EnableHotColdSplit;
243 extern cl::opt<bool> EnableOrderFileInstrumentation;
244 
245 extern cl::opt<bool> FlattenedProfileUsed;
246 
247 static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level) {
248   switch (Level) {
249   case PassBuilder::O0:
250   case PassBuilder::O1:
251   case PassBuilder::O2:
252   case PassBuilder::O3:
253     return false;
254 
255   case PassBuilder::Os:
256   case PassBuilder::Oz:
257     return true;
258   }
259   llvm_unreachable("Invalid optimization level!");
260 }
261 
262 namespace {
263 
264 /// No-op module pass which does nothing.
265 struct NoOpModulePass {
266   PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
267     return PreservedAnalyses::all();
268   }
269   static StringRef name() { return "NoOpModulePass"; }
270 };
271 
272 /// No-op module analysis.
273 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
274   friend AnalysisInfoMixin<NoOpModuleAnalysis>;
275   static AnalysisKey Key;
276 
277 public:
278   struct Result {};
279   Result run(Module &, ModuleAnalysisManager &) { return Result(); }
280   static StringRef name() { return "NoOpModuleAnalysis"; }
281 };
282 
283 /// No-op CGSCC pass which does nothing.
284 struct NoOpCGSCCPass {
285   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
286                         LazyCallGraph &, CGSCCUpdateResult &UR) {
287     return PreservedAnalyses::all();
288   }
289   static StringRef name() { return "NoOpCGSCCPass"; }
290 };
291 
292 /// No-op CGSCC analysis.
293 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
294   friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
295   static AnalysisKey Key;
296 
297 public:
298   struct Result {};
299   Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
300     return Result();
301   }
302   static StringRef name() { return "NoOpCGSCCAnalysis"; }
303 };
304 
305 /// No-op function pass which does nothing.
306 struct NoOpFunctionPass {
307   PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
308     return PreservedAnalyses::all();
309   }
310   static StringRef name() { return "NoOpFunctionPass"; }
311 };
312 
313 /// No-op function analysis.
314 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
315   friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
316   static AnalysisKey Key;
317 
318 public:
319   struct Result {};
320   Result run(Function &, FunctionAnalysisManager &) { return Result(); }
321   static StringRef name() { return "NoOpFunctionAnalysis"; }
322 };
323 
324 /// No-op loop pass which does nothing.
325 struct NoOpLoopPass {
326   PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
327                         LoopStandardAnalysisResults &, LPMUpdater &) {
328     return PreservedAnalyses::all();
329   }
330   static StringRef name() { return "NoOpLoopPass"; }
331 };
332 
333 /// No-op loop analysis.
334 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
335   friend AnalysisInfoMixin<NoOpLoopAnalysis>;
336   static AnalysisKey Key;
337 
338 public:
339   struct Result {};
340   Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
341     return Result();
342   }
343   static StringRef name() { return "NoOpLoopAnalysis"; }
344 };
345 
346 AnalysisKey NoOpModuleAnalysis::Key;
347 AnalysisKey NoOpCGSCCAnalysis::Key;
348 AnalysisKey NoOpFunctionAnalysis::Key;
349 AnalysisKey NoOpLoopAnalysis::Key;
350 
351 } // End anonymous namespace.
352 
353 void PassBuilder::invokePeepholeEPCallbacks(
354     FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
355   for (auto &C : PeepholeEPCallbacks)
356     C(FPM, Level);
357 }
358 
359 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
360 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
361   MAM.registerPass([&] { return CREATE_PASS; });
362 #include "PassRegistry.def"
363 
364   for (auto &C : ModuleAnalysisRegistrationCallbacks)
365     C(MAM);
366 }
367 
368 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
369 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
370   CGAM.registerPass([&] { return CREATE_PASS; });
371 #include "PassRegistry.def"
372 
373   for (auto &C : CGSCCAnalysisRegistrationCallbacks)
374     C(CGAM);
375 }
376 
377 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
378 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
379   FAM.registerPass([&] { return CREATE_PASS; });
380 #include "PassRegistry.def"
381 
382   for (auto &C : FunctionAnalysisRegistrationCallbacks)
383     C(FAM);
384 }
385 
386 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
387 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
388   LAM.registerPass([&] { return CREATE_PASS; });
389 #include "PassRegistry.def"
390 
391   for (auto &C : LoopAnalysisRegistrationCallbacks)
392     C(LAM);
393 }
394 
395 FunctionPassManager
396 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
397                                                  ThinLTOPhase Phase,
398                                                  bool DebugLogging) {
399   assert(Level != O0 && "Must request optimizations!");
400   FunctionPassManager FPM(DebugLogging);
401 
402   // Form SSA out of local memory accesses after breaking apart aggregates into
403   // scalars.
404   FPM.addPass(SROA());
405 
406   // Catch trivial redundancies
407   FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
408 
409   // Hoisting of scalars and load expressions.
410   if (Level > O1) {
411     if (EnableGVNHoist)
412       FPM.addPass(GVNHoistPass());
413 
414     // Global value numbering based sinking.
415     if (EnableGVNSink) {
416       FPM.addPass(GVNSinkPass());
417       FPM.addPass(SimplifyCFGPass());
418     }
419   }
420 
421   // Speculative execution if the target has divergent branches; otherwise nop.
422   if (Level > O1) {
423     FPM.addPass(SpeculativeExecutionPass());
424 
425     // Optimize based on known information about branches, and cleanup afterward.
426     FPM.addPass(JumpThreadingPass());
427     FPM.addPass(CorrelatedValuePropagationPass());
428   }
429   FPM.addPass(SimplifyCFGPass());
430   if (Level == O3)
431     FPM.addPass(AggressiveInstCombinePass());
432   FPM.addPass(InstCombinePass());
433 
434   if (!isOptimizingForSize(Level))
435     FPM.addPass(LibCallsShrinkWrapPass());
436 
437   invokePeepholeEPCallbacks(FPM, Level);
438 
439   // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
440   // using the size value profile. Don't perform this when optimizing for size.
441   if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
442       !isOptimizingForSize(Level) && Level > O1)
443     FPM.addPass(PGOMemOPSizeOpt());
444 
445   // TODO: Investigate the cost/benefit of tail call elimination on debugging.
446   if (Level > O1)
447     FPM.addPass(TailCallElimPass());
448   FPM.addPass(SimplifyCFGPass());
449 
450   // Form canonically associated expression trees, and simplify the trees using
451   // basic mathematical properties. For example, this will form (nearly)
452   // minimal multiplication trees.
453   FPM.addPass(ReassociatePass());
454 
455   // Add the primary loop simplification pipeline.
456   // FIXME: Currently this is split into two loop pass pipelines because we run
457   // some function passes in between them. These can and should be removed
458   // and/or replaced by scheduling the loop pass equivalents in the correct
459   // positions. But those equivalent passes aren't powerful enough yet.
460   // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
461   // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
462   // fully replace `SimplifyCFGPass`, and the closest to the other we have is
463   // `LoopInstSimplify`.
464   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
465 
466   // Simplify the loop body. We do this initially to clean up after other loop
467   // passes run, either when iterating on a loop or on inner loops with
468   // implications on the outer loop.
469   LPM1.addPass(LoopInstSimplifyPass());
470   LPM1.addPass(LoopSimplifyCFGPass());
471 
472   // Rotate Loop - disable header duplication at -Oz
473   LPM1.addPass(LoopRotatePass(Level != Oz));
474   // TODO: Investigate promotion cap for O1.
475   LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
476   LPM1.addPass(SimpleLoopUnswitchPass());
477   LPM2.addPass(IndVarSimplifyPass());
478   LPM2.addPass(LoopIdiomRecognizePass());
479 
480   for (auto &C : LateLoopOptimizationsEPCallbacks)
481     C(LPM2, Level);
482 
483   LPM2.addPass(LoopDeletionPass());
484   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
485   // because it changes IR to makes profile annotation in back compile
486   // inaccurate.
487   if ((Phase != ThinLTOPhase::PreLink || !PGOOpt ||
488        PGOOpt->Action != PGOOptions::SampleUse) &&
489       PTO.LoopUnrolling)
490     LPM2.addPass(LoopFullUnrollPass(Level, /*OnlyWhenForced=*/false,
491                                     PTO.ForgetAllSCEVInLoopUnroll));
492 
493   for (auto &C : LoopOptimizerEndEPCallbacks)
494     C(LPM2, Level);
495 
496   // We provide the opt remark emitter pass for LICM to use. We only need to do
497   // this once as it is immutable.
498   FPM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
499   FPM.addPass(createFunctionToLoopPassAdaptor(
500       std::move(LPM1), EnableMSSALoopDependency, DebugLogging));
501   FPM.addPass(SimplifyCFGPass());
502   FPM.addPass(InstCombinePass());
503   // The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass,
504   // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
505   // *All* loop passes must preserve it, in order to be able to use it.
506   FPM.addPass(createFunctionToLoopPassAdaptor(
507       std::move(LPM2), /*UseMemorySSA=*/false, DebugLogging));
508 
509   // Delete small array after loop unroll.
510   FPM.addPass(SROA());
511 
512   // Eliminate redundancies.
513   if (Level != O1) {
514     // These passes add substantial compile time so skip them at O1.
515     FPM.addPass(MergedLoadStoreMotionPass());
516     if (RunNewGVN)
517       FPM.addPass(NewGVNPass());
518     else
519       FPM.addPass(GVN());
520   }
521 
522   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
523   FPM.addPass(MemCpyOptPass());
524 
525   // Sparse conditional constant propagation.
526   // FIXME: It isn't clear why we do this *after* loop passes rather than
527   // before...
528   FPM.addPass(SCCPPass());
529 
530   // Delete dead bit computations (instcombine runs after to fold away the dead
531   // computations, and then ADCE will run later to exploit any new DCE
532   // opportunities that creates).
533   FPM.addPass(BDCEPass());
534 
535   // Run instcombine after redundancy and dead bit elimination to exploit
536   // opportunities opened up by them.
537   FPM.addPass(InstCombinePass());
538   invokePeepholeEPCallbacks(FPM, Level);
539 
540   // Re-consider control flow based optimizations after redundancy elimination,
541   // redo DCE, etc.
542   if (Level > O1) {
543     FPM.addPass(JumpThreadingPass());
544     FPM.addPass(CorrelatedValuePropagationPass());
545     FPM.addPass(DSEPass());
546     FPM.addPass(createFunctionToLoopPassAdaptor(
547         LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
548         EnableMSSALoopDependency, DebugLogging));
549   }
550 
551   for (auto &C : ScalarOptimizerLateEPCallbacks)
552     C(FPM, Level);
553 
554   // Finally, do an expensive DCE pass to catch all the dead code exposed by
555   // the simplifications and basic cleanup after all the simplifications.
556   // TODO: Investigate if this is too expensive.
557   FPM.addPass(ADCEPass());
558   FPM.addPass(SimplifyCFGPass());
559   FPM.addPass(InstCombinePass());
560   invokePeepholeEPCallbacks(FPM, Level);
561 
562   if (EnableCHR && Level == O3 && PGOOpt &&
563       (PGOOpt->Action == PGOOptions::IRUse ||
564        PGOOpt->Action == PGOOptions::SampleUse))
565     FPM.addPass(ControlHeightReductionPass());
566 
567   return FPM;
568 }
569 
570 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
571                                     PassBuilder::OptimizationLevel Level,
572                                     bool RunProfileGen, bool IsCS,
573                                     std::string ProfileFile,
574                                     std::string ProfileRemappingFile) {
575   assert(Level != O0 && "Not expecting O0 here!");
576   // Generally running simplification passes and the inliner with an high
577   // threshold results in smaller executables, but there may be cases where
578   // the size grows, so let's be conservative here and skip this simplification
579   // at -Os/Oz. We will not do this  inline for context sensistive PGO (when
580   // IsCS is true).
581   if (!isOptimizingForSize(Level) && !IsCS) {
582     InlineParams IP;
583 
584     IP.DefaultThreshold = PreInlineThreshold;
585 
586     // FIXME: The hint threshold has the same value used by the regular inliner.
587     // This should probably be lowered after performance testing.
588     // FIXME: this comment is cargo culted from the old pass manager, revisit).
589     IP.HintThreshold = 325;
590 
591     CGSCCPassManager CGPipeline(DebugLogging);
592 
593     CGPipeline.addPass(InlinerPass(IP));
594 
595     FunctionPassManager FPM;
596     FPM.addPass(SROA());
597     FPM.addPass(EarlyCSEPass());    // Catch trivial redundancies.
598     FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
599     FPM.addPass(InstCombinePass()); // Combine silly sequences.
600     invokePeepholeEPCallbacks(FPM, Level);
601 
602     CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
603 
604     MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline)));
605 
606     // Delete anything that is now dead to make sure that we don't instrument
607     // dead code. Instrumentation can end up keeping dead code around and
608     // dramatically increase code size.
609     MPM.addPass(GlobalDCEPass());
610   }
611 
612   if (!RunProfileGen) {
613     assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
614     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
615     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
616     // RequireAnalysisPass for PSI before subsequent non-module passes.
617     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
618     return;
619   }
620 
621   // Perform PGO instrumentation.
622   MPM.addPass(PGOInstrumentationGen(IsCS));
623 
624   FunctionPassManager FPM;
625   FPM.addPass(createFunctionToLoopPassAdaptor(
626       LoopRotatePass(), EnableMSSALoopDependency, DebugLogging));
627   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
628 
629   // Add the profile lowering pass.
630   InstrProfOptions Options;
631   if (!ProfileFile.empty())
632     Options.InstrProfileOutput = ProfileFile;
633   // Do counter promotion at Level greater than O0.
634   Options.DoCounterPromotion = true;
635   Options.UseBFIInPromotion = IsCS;
636   MPM.addPass(InstrProfiling(Options, IsCS));
637 }
638 
639 void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
640                                          bool DebugLogging, bool RunProfileGen,
641                                          bool IsCS, std::string ProfileFile,
642                                          std::string ProfileRemappingFile) {
643   if (!RunProfileGen) {
644     assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
645     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
646     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
647     // RequireAnalysisPass for PSI before subsequent non-module passes.
648     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
649     return;
650   }
651 
652   // Perform PGO instrumentation.
653   MPM.addPass(PGOInstrumentationGen(IsCS));
654   // Add the profile lowering pass.
655   InstrProfOptions Options;
656   if (!ProfileFile.empty())
657     Options.InstrProfileOutput = ProfileFile;
658   // Do not do counter promotion at O0.
659   Options.DoCounterPromotion = false;
660   Options.UseBFIInPromotion = IsCS;
661   MPM.addPass(InstrProfiling(Options, IsCS));
662 }
663 
664 static InlineParams
665 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
666   auto O3 = PassBuilder::O3;
667   unsigned OptLevel = Level > O3 ? 2 : Level;
668   unsigned SizeLevel = Level > O3 ? Level - O3 : 0;
669   return getInlineParams(OptLevel, SizeLevel);
670 }
671 
672 ModulePassManager
673 PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
674                                                ThinLTOPhase Phase,
675                                                bool DebugLogging) {
676   ModulePassManager MPM(DebugLogging);
677 
678   bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
679 
680   // In ThinLTO mode, when flattened profile is used, all the available
681   // profile information will be annotated in PreLink phase so there is
682   // no need to load the profile again in PostLink.
683   bool LoadSampleProfile =
684       HasSampleProfile &&
685       !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink);
686 
687   // During the ThinLTO backend phase we perform early indirect call promotion
688   // here, before globalopt. Otherwise imported available_externally functions
689   // look unreferenced and are removed. If we are going to load the sample
690   // profile then defer until later.
691   // TODO: See if we can move later and consolidate with the location where
692   // we perform ICP when we are loading a sample profile.
693   // TODO: We pass HasSampleProfile (whether there was a sample profile file
694   // passed to the compile) to the SamplePGO flag of ICP. This is used to
695   // determine whether the new direct calls are annotated with prof metadata.
696   // Ideally this should be determined from whether the IR is annotated with
697   // sample profile, and not whether the a sample profile was provided on the
698   // command line. E.g. for flattened profiles where we will not be reloading
699   // the sample profile in the ThinLTO backend, we ideally shouldn't have to
700   // provide the sample profile file.
701   if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile)
702     MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
703 
704   // Do basic inference of function attributes from known properties of system
705   // libraries and other oracles.
706   MPM.addPass(InferFunctionAttrsPass());
707 
708   // Create an early function pass manager to cleanup the output of the
709   // frontend.
710   FunctionPassManager EarlyFPM(DebugLogging);
711   EarlyFPM.addPass(SimplifyCFGPass());
712   EarlyFPM.addPass(SROA());
713   EarlyFPM.addPass(EarlyCSEPass());
714   EarlyFPM.addPass(LowerExpectIntrinsicPass());
715   if (Level == O3)
716     EarlyFPM.addPass(CallSiteSplittingPass());
717 
718   // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
719   // to convert bitcast to direct calls so that they can be inlined during the
720   // profile annotation prepration step.
721   // More details about SamplePGO design can be found in:
722   // https://research.google.com/pubs/pub45290.html
723   // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
724   if (LoadSampleProfile)
725     EarlyFPM.addPass(InstCombinePass());
726   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
727 
728   if (LoadSampleProfile) {
729     // Annotate sample profile right after early FPM to ensure freshness of
730     // the debug info.
731     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
732                                         PGOOpt->ProfileRemappingFile,
733                                         Phase == ThinLTOPhase::PreLink));
734     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
735     // RequireAnalysisPass for PSI before subsequent non-module passes.
736     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
737     // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
738     // for the profile annotation to be accurate in the ThinLTO backend.
739     if (Phase != ThinLTOPhase::PreLink)
740       // We perform early indirect call promotion here, before globalopt.
741       // This is important for the ThinLTO backend phase because otherwise
742       // imported available_externally functions look unreferenced and are
743       // removed.
744       MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
745                                            true /* SamplePGO */));
746   }
747 
748   // Interprocedural constant propagation now that basic cleanup has occurred
749   // and prior to optimizing globals.
750   // FIXME: This position in the pipeline hasn't been carefully considered in
751   // years, it should be re-analyzed.
752   MPM.addPass(IPSCCPPass());
753 
754   // Attach metadata to indirect call sites indicating the set of functions
755   // they may target at run-time. This should follow IPSCCP.
756   MPM.addPass(CalledValuePropagationPass());
757 
758   // Optimize globals to try and fold them into constants.
759   MPM.addPass(GlobalOptPass());
760 
761   // Promote any localized globals to SSA registers.
762   // FIXME: Should this instead by a run of SROA?
763   // FIXME: We should probably run instcombine and simplify-cfg afterward to
764   // delete control flows that are dead once globals have been folded to
765   // constants.
766   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
767 
768   // Remove any dead arguments exposed by cleanups and constand folding
769   // globals.
770   MPM.addPass(DeadArgumentEliminationPass());
771 
772   // Create a small function pass pipeline to cleanup after all the global
773   // optimizations.
774   FunctionPassManager GlobalCleanupPM(DebugLogging);
775   GlobalCleanupPM.addPass(InstCombinePass());
776   invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
777 
778   GlobalCleanupPM.addPass(SimplifyCFGPass());
779   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
780 
781   // Add all the requested passes for instrumentation PGO, if requested.
782   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
783       (PGOOpt->Action == PGOOptions::IRInstr ||
784        PGOOpt->Action == PGOOptions::IRUse)) {
785     addPGOInstrPasses(MPM, DebugLogging, Level,
786                       /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
787                       /* IsCS */ false, PGOOpt->ProfileFile,
788                       PGOOpt->ProfileRemappingFile);
789     MPM.addPass(PGOIndirectCallPromotion(false, false));
790   }
791   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
792       PGOOpt->CSAction == PGOOptions::CSIRInstr)
793     MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
794 
795   // Synthesize function entry counts for non-PGO compilation.
796   if (EnableSyntheticCounts && !PGOOpt)
797     MPM.addPass(SyntheticCountsPropagation());
798 
799   // Require the GlobalsAA analysis for the module so we can query it within
800   // the CGSCC pipeline.
801   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
802 
803   // Require the ProfileSummaryAnalysis for the module so we can query it within
804   // the inliner pass.
805   MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
806 
807   // Now begin the main postorder CGSCC pipeline.
808   // FIXME: The current CGSCC pipeline has its origins in the legacy pass
809   // manager and trying to emulate its precise behavior. Much of this doesn't
810   // make a lot of sense and we should revisit the core CGSCC structure.
811   CGSCCPassManager MainCGPipeline(DebugLogging);
812 
813   // Note: historically, the PruneEH pass was run first to deduce nounwind and
814   // generally clean up exception handling overhead. It isn't clear this is
815   // valuable as the inliner doesn't currently care whether it is inlining an
816   // invoke or a call.
817 
818   // Run the inliner first. The theory is that we are walking bottom-up and so
819   // the callees have already been fully optimized, and we want to inline them
820   // into the callers so that our optimizations can reflect that.
821   // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
822   // because it makes profile annotation in the backend inaccurate.
823   InlineParams IP = getInlineParamsFromOptLevel(Level);
824   if (Phase == ThinLTOPhase::PreLink && PGOOpt &&
825       PGOOpt->Action == PGOOptions::SampleUse)
826     IP.HotCallSiteThreshold = 0;
827   MainCGPipeline.addPass(InlinerPass(IP));
828 
829   // Now deduce any function attributes based in the current code.
830   MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
831 
832   // When at O3 add argument promotion to the pass pipeline.
833   // FIXME: It isn't at all clear why this should be limited to O3.
834   if (Level == O3)
835     MainCGPipeline.addPass(ArgumentPromotionPass());
836 
837   // Lastly, add the core function simplification pipeline nested inside the
838   // CGSCC walk.
839   MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
840       buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
841 
842   for (auto &C : CGSCCOptimizerLateEPCallbacks)
843     C(MainCGPipeline, Level);
844 
845   // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
846   // to detect when we devirtualize indirect calls and iterate the SCC passes
847   // in that case to try and catch knock-on inlining or function attrs
848   // opportunities. Then we add it to the module pipeline by walking the SCCs
849   // in postorder (or bottom-up).
850   MPM.addPass(
851       createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
852           std::move(MainCGPipeline), MaxDevirtIterations)));
853 
854   return MPM;
855 }
856 
857 ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
858     OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) {
859   ModulePassManager MPM(DebugLogging);
860 
861   // Optimize globals now that the module is fully simplified.
862   MPM.addPass(GlobalOptPass());
863   MPM.addPass(GlobalDCEPass());
864 
865   // Run partial inlining pass to partially inline functions that have
866   // large bodies.
867   if (RunPartialInlining)
868     MPM.addPass(PartialInlinerPass());
869 
870   // Remove avail extern fns and globals definitions since we aren't compiling
871   // an object file for later LTO. For LTO we want to preserve these so they
872   // are eligible for inlining at link-time. Note if they are unreferenced they
873   // will be removed by GlobalDCE later, so this only impacts referenced
874   // available externally globals. Eventually they will be suppressed during
875   // codegen, but eliminating here enables more opportunity for GlobalDCE as it
876   // may make globals referenced by available external functions dead and saves
877   // running remaining passes on the eliminated functions. These should be
878   // preserved during prelinking for link-time inlining decisions.
879   if (!LTOPreLink)
880     MPM.addPass(EliminateAvailableExternallyPass());
881 
882   if (EnableOrderFileInstrumentation)
883     MPM.addPass(InstrOrderFilePass());
884 
885   // Do RPO function attribute inference across the module to forward-propagate
886   // attributes where applicable.
887   // FIXME: Is this really an optimization rather than a canonicalization?
888   MPM.addPass(ReversePostOrderFunctionAttrsPass());
889 
890   // Do a post inline PGO instrumentation and use pass. This is a context
891   // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
892   // cross-module inline has not been done yet. The context sensitive
893   // instrumentation is after all the inlines are done.
894   if (!LTOPreLink && PGOOpt) {
895     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
896       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
897                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
898                         PGOOpt->ProfileRemappingFile);
899     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
900       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
901                         /* IsCS */ true, PGOOpt->ProfileFile,
902                         PGOOpt->ProfileRemappingFile);
903   }
904 
905   // Re-require GloblasAA here prior to function passes. This is particularly
906   // useful as the above will have inlined, DCE'ed, and function-attr
907   // propagated everything. We should at this point have a reasonably minimal
908   // and richly annotated call graph. By computing aliasing and mod/ref
909   // information for all local globals here, the late loop passes and notably
910   // the vectorizer will be able to use them to help recognize vectorizable
911   // memory operations.
912   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
913 
914   FunctionPassManager OptimizePM(DebugLogging);
915   OptimizePM.addPass(Float2IntPass());
916   OptimizePM.addPass(LowerConstantIntrinsicsPass());
917 
918   // FIXME: We need to run some loop optimizations to re-rotate loops after
919   // simplify-cfg and others undo their rotation.
920 
921   // Optimize the loop execution. These passes operate on entire loop nests
922   // rather than on each loop in an inside-out manner, and so they are actually
923   // function passes.
924 
925   for (auto &C : VectorizerStartEPCallbacks)
926     C(OptimizePM, Level);
927 
928   // First rotate loops that may have been un-rotated by prior passes.
929   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
930       LoopRotatePass(), EnableMSSALoopDependency, DebugLogging));
931 
932   // Distribute loops to allow partial vectorization.  I.e. isolate dependences
933   // into separate loop that would otherwise inhibit vectorization.  This is
934   // currently only performed for loops marked with the metadata
935   // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
936   OptimizePM.addPass(LoopDistributePass());
937 
938   // Now run the core loop vectorizer.
939   OptimizePM.addPass(LoopVectorizePass(
940       LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
941 
942   // Eliminate loads by forwarding stores from the previous iteration to loads
943   // of the current iteration.
944   OptimizePM.addPass(LoopLoadEliminationPass());
945 
946   // Cleanup after the loop optimization passes.
947   OptimizePM.addPass(InstCombinePass());
948 
949   // Now that we've formed fast to execute loop structures, we do further
950   // optimizations. These are run afterward as they might block doing complex
951   // analyses and transforms such as what are needed for loop vectorization.
952 
953   // Cleanup after loop vectorization, etc. Simplification passes like CVP and
954   // GVN, loop transforms, and others have already run, so it's now better to
955   // convert to more optimized IR using more aggressive simplify CFG options.
956   // The extra sinking transform can create larger basic blocks, so do this
957   // before SLP vectorization.
958   OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions().
959                                      forwardSwitchCondToPhi(true).
960                                      convertSwitchToLookupTable(true).
961                                      needCanonicalLoops(false).
962                                      sinkCommonInsts(true)));
963 
964   // Optimize parallel scalar instruction chains into SIMD instructions.
965   if (PTO.SLPVectorization)
966     OptimizePM.addPass(SLPVectorizerPass());
967 
968   OptimizePM.addPass(InstCombinePass());
969 
970   // Unroll small loops to hide loop backedge latency and saturate any parallel
971   // execution resources of an out-of-order processor. We also then need to
972   // clean up redundancies and loop invariant code.
973   // FIXME: It would be really good to use a loop-integrated instruction
974   // combiner for cleanup here so that the unrolling and LICM can be pipelined
975   // across the loop nests.
976   // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
977   if (EnableUnrollAndJam && PTO.LoopUnrolling) {
978     OptimizePM.addPass(LoopUnrollAndJamPass(Level));
979   }
980   OptimizePM.addPass(LoopUnrollPass(
981       LoopUnrollOptions(Level, /*OnlyWhenForced=*/!PTO.LoopUnrolling,
982                         PTO.ForgetAllSCEVInLoopUnroll)));
983   OptimizePM.addPass(WarnMissedTransformationsPass());
984   OptimizePM.addPass(InstCombinePass());
985   OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
986   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
987       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
988       EnableMSSALoopDependency, DebugLogging));
989 
990   // Now that we've vectorized and unrolled loops, we may have more refined
991   // alignment information, try to re-derive it here.
992   OptimizePM.addPass(AlignmentFromAssumptionsPass());
993 
994   // Split out cold code. Splitting is done late to avoid hiding context from
995   // other optimizations and inadvertently regressing performance. The tradeoff
996   // is that this has a higher code size cost than splitting early.
997   if (EnableHotColdSplit && !LTOPreLink)
998     MPM.addPass(HotColdSplittingPass());
999 
1000   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1001   // canonicalization pass that enables other optimizations. As a result,
1002   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1003   // result too early.
1004   OptimizePM.addPass(LoopSinkPass());
1005 
1006   // And finally clean up LCSSA form before generating code.
1007   OptimizePM.addPass(InstSimplifyPass());
1008 
1009   // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1010   // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1011   // flattening of blocks.
1012   OptimizePM.addPass(DivRemPairsPass());
1013 
1014   // LoopSink (and other loop passes since the last simplifyCFG) might have
1015   // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1016   OptimizePM.addPass(SimplifyCFGPass());
1017 
1018   // Optimize PHIs by speculating around them when profitable. Note that this
1019   // pass needs to be run after any PRE or similar pass as it is essentially
1020   // inserting redundancies into the program. This even includes SimplifyCFG.
1021   OptimizePM.addPass(SpeculateAroundPHIsPass());
1022 
1023   for (auto &C : OptimizerLastEPCallbacks)
1024     C(OptimizePM, Level);
1025 
1026   // Add the core optimizing pipeline.
1027   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
1028 
1029   MPM.addPass(CGProfilePass());
1030 
1031   // Now we need to do some global optimization transforms.
1032   // FIXME: It would seem like these should come first in the optimization
1033   // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1034   // ordering here.
1035   MPM.addPass(GlobalDCEPass());
1036   MPM.addPass(ConstantMergePass());
1037 
1038   return MPM;
1039 }
1040 
1041 ModulePassManager
1042 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1043                                            bool DebugLogging, bool LTOPreLink) {
1044   assert(Level != O0 && "Must request optimizations for the default pipeline!");
1045 
1046   ModulePassManager MPM(DebugLogging);
1047 
1048   // Force any function attributes we want the rest of the pipeline to observe.
1049   MPM.addPass(ForceFunctionAttrsPass());
1050 
1051   // Apply module pipeline start EP callback.
1052   for (auto &C : PipelineStartEPCallbacks)
1053     C(MPM);
1054 
1055   if (PGOOpt && PGOOpt->SamplePGOSupport)
1056     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1057 
1058   // Add the core simplification pipeline.
1059   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None,
1060                                                 DebugLogging));
1061 
1062   // Now add the optimization pipeline.
1063   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging, LTOPreLink));
1064 
1065   return MPM;
1066 }
1067 
1068 ModulePassManager
1069 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1070                                                 bool DebugLogging) {
1071   assert(Level != O0 && "Must request optimizations for the default pipeline!");
1072 
1073   ModulePassManager MPM(DebugLogging);
1074 
1075   // Force any function attributes we want the rest of the pipeline to observe.
1076   MPM.addPass(ForceFunctionAttrsPass());
1077 
1078   if (PGOOpt && PGOOpt->SamplePGOSupport)
1079     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1080 
1081   // Apply module pipeline start EP callback.
1082   for (auto &C : PipelineStartEPCallbacks)
1083     C(MPM);
1084 
1085   // If we are planning to perform ThinLTO later, we don't bloat the code with
1086   // unrolling/vectorization/... now. Just simplify the module as much as we
1087   // can.
1088   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink,
1089                                                 DebugLogging));
1090 
1091   // Run partial inlining pass to partially inline functions that have
1092   // large bodies.
1093   // FIXME: It isn't clear whether this is really the right place to run this
1094   // in ThinLTO. Because there is another canonicalization and simplification
1095   // phase that will run after the thin link, running this here ends up with
1096   // less information than will be available later and it may grow functions in
1097   // ways that aren't beneficial.
1098   if (RunPartialInlining)
1099     MPM.addPass(PartialInlinerPass());
1100 
1101   // Reduce the size of the IR as much as possible.
1102   MPM.addPass(GlobalOptPass());
1103 
1104   return MPM;
1105 }
1106 
1107 ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1108     OptimizationLevel Level, bool DebugLogging,
1109     const ModuleSummaryIndex *ImportSummary) {
1110   ModulePassManager MPM(DebugLogging);
1111 
1112   if (ImportSummary) {
1113     // These passes import type identifier resolutions for whole-program
1114     // devirtualization and CFI. They must run early because other passes may
1115     // disturb the specific instruction patterns that these passes look for,
1116     // creating dependencies on resolutions that may not appear in the summary.
1117     //
1118     // For example, GVN may transform the pattern assume(type.test) appearing in
1119     // two basic blocks into assume(phi(type.test, type.test)), which would
1120     // transform a dependency on a WPD resolution into a dependency on a type
1121     // identifier resolution for CFI.
1122     //
1123     // Also, WPD has access to more precise information than ICP and can
1124     // devirtualize more effectively, so it should operate on the IR first.
1125     //
1126     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1127     // metadata and intrinsics.
1128     MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1129     MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1130   }
1131 
1132   if (Level == O0)
1133     return MPM;
1134 
1135   // Force any function attributes we want the rest of the pipeline to observe.
1136   MPM.addPass(ForceFunctionAttrsPass());
1137 
1138   // Add the core simplification pipeline.
1139   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
1140                                                 DebugLogging));
1141 
1142   // Now add the optimization pipeline.
1143   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
1144 
1145   return MPM;
1146 }
1147 
1148 ModulePassManager
1149 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1150                                             bool DebugLogging) {
1151   assert(Level != O0 && "Must request optimizations for the default pipeline!");
1152   // FIXME: We should use a customized pre-link pipeline!
1153   return buildPerModuleDefaultPipeline(Level, DebugLogging,
1154                                        /* LTOPreLink */true);
1155 }
1156 
1157 ModulePassManager
1158 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
1159                                      ModuleSummaryIndex *ExportSummary) {
1160   ModulePassManager MPM(DebugLogging);
1161 
1162   if (Level == O0) {
1163     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1164     // metadata and intrinsics.
1165     MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1166     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1167     return MPM;
1168   }
1169 
1170   if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1171     // Load sample profile before running the LTO optimization pipeline.
1172     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1173                                         PGOOpt->ProfileRemappingFile,
1174                                         false /* ThinLTOPhase::PreLink */));
1175     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1176     // RequireAnalysisPass for PSI before subsequent non-module passes.
1177     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1178   }
1179 
1180   // Remove unused virtual tables to improve the quality of code generated by
1181   // whole-program devirtualization and bitset lowering.
1182   MPM.addPass(GlobalDCEPass());
1183 
1184   // Force any function attributes we want the rest of the pipeline to observe.
1185   MPM.addPass(ForceFunctionAttrsPass());
1186 
1187   // Do basic inference of function attributes from known properties of system
1188   // libraries and other oracles.
1189   MPM.addPass(InferFunctionAttrsPass());
1190 
1191   if (Level > 1) {
1192     FunctionPassManager EarlyFPM(DebugLogging);
1193     EarlyFPM.addPass(CallSiteSplittingPass());
1194     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1195 
1196     // Indirect call promotion. This should promote all the targets that are
1197     // left by the earlier promotion pass that promotes intra-module targets.
1198     // This two-step promotion is to save the compile time. For LTO, it should
1199     // produce the same result as if we only do promotion here.
1200     MPM.addPass(PGOIndirectCallPromotion(
1201         true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1202     // Propagate constants at call sites into the functions they call.  This
1203     // opens opportunities for globalopt (and inlining) by substituting function
1204     // pointers passed as arguments to direct uses of functions.
1205    MPM.addPass(IPSCCPPass());
1206 
1207    // Attach metadata to indirect call sites indicating the set of functions
1208    // they may target at run-time. This should follow IPSCCP.
1209    MPM.addPass(CalledValuePropagationPass());
1210   }
1211 
1212   // Now deduce any function attributes based in the current code.
1213   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1214               PostOrderFunctionAttrsPass()));
1215 
1216   // Do RPO function attribute inference across the module to forward-propagate
1217   // attributes where applicable.
1218   // FIXME: Is this really an optimization rather than a canonicalization?
1219   MPM.addPass(ReversePostOrderFunctionAttrsPass());
1220 
1221   // Use in-range annotations on GEP indices to split globals where beneficial.
1222   MPM.addPass(GlobalSplitPass());
1223 
1224   // Run whole program optimization of virtual call when the list of callees
1225   // is fixed.
1226   MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1227 
1228   // Stop here at -O1.
1229   if (Level == 1) {
1230     // The LowerTypeTestsPass needs to run to lower type metadata and the
1231     // type.test intrinsics. The pass does nothing if CFI is disabled.
1232     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1233     return MPM;
1234   }
1235 
1236   // Optimize globals to try and fold them into constants.
1237   MPM.addPass(GlobalOptPass());
1238 
1239   // Promote any localized globals to SSA registers.
1240   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1241 
1242   // Linking modules together can lead to duplicate global constant, only
1243   // keep one copy of each constant.
1244   MPM.addPass(ConstantMergePass());
1245 
1246   // Remove unused arguments from functions.
1247   MPM.addPass(DeadArgumentEliminationPass());
1248 
1249   // Reduce the code after globalopt and ipsccp.  Both can open up significant
1250   // simplification opportunities, and both can propagate functions through
1251   // function pointers.  When this happens, we often have to resolve varargs
1252   // calls, etc, so let instcombine do this.
1253   FunctionPassManager PeepholeFPM(DebugLogging);
1254   if (Level == O3)
1255     PeepholeFPM.addPass(AggressiveInstCombinePass());
1256   PeepholeFPM.addPass(InstCombinePass());
1257   invokePeepholeEPCallbacks(PeepholeFPM, Level);
1258 
1259   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
1260 
1261   // Note: historically, the PruneEH pass was run first to deduce nounwind and
1262   // generally clean up exception handling overhead. It isn't clear this is
1263   // valuable as the inliner doesn't currently care whether it is inlining an
1264   // invoke or a call.
1265   // Run the inliner now.
1266   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1267       InlinerPass(getInlineParamsFromOptLevel(Level))));
1268 
1269   // Optimize globals again after we ran the inliner.
1270   MPM.addPass(GlobalOptPass());
1271 
1272   // Garbage collect dead functions.
1273   // FIXME: Add ArgumentPromotion pass after once it's ported.
1274   MPM.addPass(GlobalDCEPass());
1275 
1276   FunctionPassManager FPM(DebugLogging);
1277   // The IPO Passes may leave cruft around. Clean up after them.
1278   FPM.addPass(InstCombinePass());
1279   invokePeepholeEPCallbacks(FPM, Level);
1280 
1281   FPM.addPass(JumpThreadingPass());
1282 
1283   // Do a post inline PGO instrumentation and use pass. This is a context
1284   // sensitive PGO pass.
1285   if (PGOOpt) {
1286     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1287       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1288                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
1289                         PGOOpt->ProfileRemappingFile);
1290     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1291       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1292                         /* IsCS */ true, PGOOpt->ProfileFile,
1293                         PGOOpt->ProfileRemappingFile);
1294   }
1295 
1296   // Break up allocas
1297   FPM.addPass(SROA());
1298 
1299   // LTO provides additional opportunities for tailcall elimination due to
1300   // link-time inlining, and visibility of nocapture attribute.
1301   FPM.addPass(TailCallElimPass());
1302 
1303   // Run a few AA driver optimizations here and now to cleanup the code.
1304   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1305 
1306   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1307               PostOrderFunctionAttrsPass()));
1308   // FIXME: here we run IP alias analysis in the legacy PM.
1309 
1310   FunctionPassManager MainFPM;
1311 
1312   // FIXME: once we fix LoopPass Manager, add LICM here.
1313   // FIXME: once we provide support for enabling MLSM, add it here.
1314   if (RunNewGVN)
1315     MainFPM.addPass(NewGVNPass());
1316   else
1317     MainFPM.addPass(GVN());
1318 
1319   // Remove dead memcpy()'s.
1320   MainFPM.addPass(MemCpyOptPass());
1321 
1322   // Nuke dead stores.
1323   MainFPM.addPass(DSEPass());
1324 
1325   // FIXME: at this point, we run a bunch of loop passes:
1326   // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
1327   // loopVectorize. Enable them once the remaining issue with LPM
1328   // are sorted out.
1329 
1330   MainFPM.addPass(InstCombinePass());
1331   MainFPM.addPass(SimplifyCFGPass());
1332   MainFPM.addPass(SCCPPass());
1333   MainFPM.addPass(InstCombinePass());
1334   MainFPM.addPass(BDCEPass());
1335 
1336   // FIXME: We may want to run SLPVectorizer here.
1337   // After vectorization, assume intrinsics may tell us more
1338   // about pointer alignments.
1339 #if 0
1340   MainFPM.add(AlignmentFromAssumptionsPass());
1341 #endif
1342 
1343   // FIXME: Conditionally run LoadCombine here, after it's ported
1344   // (in case we still have this pass, given its questionable usefulness).
1345 
1346   MainFPM.addPass(InstCombinePass());
1347   invokePeepholeEPCallbacks(MainFPM, Level);
1348   MainFPM.addPass(JumpThreadingPass());
1349   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1350 
1351   // Create a function that performs CFI checks for cross-DSO calls with
1352   // targets in the current module.
1353   MPM.addPass(CrossDSOCFIPass());
1354 
1355   // Lower type metadata and the type.test intrinsic. This pass supports
1356   // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1357   // to be run at link time if CFI is enabled. This pass does nothing if
1358   // CFI is disabled.
1359   MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1360 
1361   // Enable splitting late in the FullLTO post-link pipeline. This is done in
1362   // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1363   if (EnableHotColdSplit)
1364     MPM.addPass(HotColdSplittingPass());
1365 
1366   // Add late LTO optimization passes.
1367   // Delete basic blocks, which optimization passes may have killed.
1368   MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1369 
1370   // Drop bodies of available eternally objects to improve GlobalDCE.
1371   MPM.addPass(EliminateAvailableExternallyPass());
1372 
1373   // Now that we have optimized the program, discard unreachable functions.
1374   MPM.addPass(GlobalDCEPass());
1375 
1376   // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
1377   return MPM;
1378 }
1379 
1380 AAManager PassBuilder::buildDefaultAAPipeline() {
1381   AAManager AA;
1382 
1383   // The order in which these are registered determines their priority when
1384   // being queried.
1385 
1386   // First we register the basic alias analysis that provides the majority of
1387   // per-function local AA logic. This is a stateless, on-demand local set of
1388   // AA techniques.
1389   AA.registerFunctionAnalysis<BasicAA>();
1390 
1391   // Next we query fast, specialized alias analyses that wrap IR-embedded
1392   // information about aliasing.
1393   AA.registerFunctionAnalysis<ScopedNoAliasAA>();
1394   AA.registerFunctionAnalysis<TypeBasedAA>();
1395 
1396   // Add support for querying global aliasing information when available.
1397   // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1398   // analysis, all that the `AAManager` can do is query for any *cached*
1399   // results from `GlobalsAA` through a readonly proxy.
1400   AA.registerModuleAnalysis<GlobalsAA>();
1401 
1402   return AA;
1403 }
1404 
1405 static Optional<int> parseRepeatPassName(StringRef Name) {
1406   if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1407     return None;
1408   int Count;
1409   if (Name.getAsInteger(0, Count) || Count <= 0)
1410     return None;
1411   return Count;
1412 }
1413 
1414 static Optional<int> parseDevirtPassName(StringRef Name) {
1415   if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1416     return None;
1417   int Count;
1418   if (Name.getAsInteger(0, Count) || Count <= 0)
1419     return None;
1420   return Count;
1421 }
1422 
1423 static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
1424   if (!Name.consume_front(PassName))
1425     return false;
1426   // normal pass name w/o parameters == default parameters
1427   if (Name.empty())
1428     return true;
1429   return Name.startswith("<") && Name.endswith(">");
1430 }
1431 
1432 namespace {
1433 
1434 /// This performs customized parsing of pass name with parameters.
1435 ///
1436 /// We do not need parametrization of passes in textual pipeline very often,
1437 /// yet on a rare occasion ability to specify parameters right there can be
1438 /// useful.
1439 ///
1440 /// \p Name - parameterized specification of a pass from a textual pipeline
1441 /// is a string in a form of :
1442 ///      PassName '<' parameter-list '>'
1443 ///
1444 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1445 /// It takes a string-ref of parameters and returns either StringError or a
1446 /// parameter list in a form of a custom parameters type, all wrapped into
1447 /// Expected<> template class.
1448 ///
1449 template <typename ParametersParseCallableT>
1450 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
1451                          StringRef PassName) -> decltype(Parser(StringRef{})) {
1452   using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
1453 
1454   StringRef Params = Name;
1455   if (!Params.consume_front(PassName)) {
1456     assert(false &&
1457            "unable to strip pass name from parametrized pass specification");
1458   }
1459   if (Params.empty())
1460     return ParametersT{};
1461   if (!Params.consume_front("<") || !Params.consume_back(">")) {
1462     assert(false && "invalid format for parametrized pass name");
1463   }
1464 
1465   Expected<ParametersT> Result = Parser(Params);
1466   assert((Result || Result.template errorIsA<StringError>()) &&
1467          "Pass parameter parser can only return StringErrors.");
1468   return Result;
1469 }
1470 
1471 /// Parser of parameters for LoopUnroll pass.
1472 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
1473   LoopUnrollOptions UnrollOpts;
1474   while (!Params.empty()) {
1475     StringRef ParamName;
1476     std::tie(ParamName, Params) = Params.split(';');
1477     int OptLevel = StringSwitch<int>(ParamName)
1478                        .Case("O0", 0)
1479                        .Case("O1", 1)
1480                        .Case("O2", 2)
1481                        .Case("O3", 3)
1482                        .Default(-1);
1483     if (OptLevel >= 0) {
1484       UnrollOpts.setOptLevel(OptLevel);
1485       continue;
1486     }
1487     if (ParamName.consume_front("full-unroll-max=")) {
1488       int Count;
1489       if (ParamName.getAsInteger(0, Count))
1490         return make_error<StringError>(
1491             formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1492             inconvertibleErrorCode());
1493       UnrollOpts.setFullUnrollMaxCount(Count);
1494       continue;
1495     }
1496 
1497     bool Enable = !ParamName.consume_front("no-");
1498     if (ParamName == "partial") {
1499       UnrollOpts.setPartial(Enable);
1500     } else if (ParamName == "peeling") {
1501       UnrollOpts.setPeeling(Enable);
1502     } else if (ParamName == "profile-peeling") {
1503       UnrollOpts.setProfileBasedPeeling(Enable);
1504     } else if (ParamName == "runtime") {
1505       UnrollOpts.setRuntime(Enable);
1506     } else if (ParamName == "upperbound") {
1507       UnrollOpts.setUpperBound(Enable);
1508     } else {
1509       return make_error<StringError>(
1510           formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1511           inconvertibleErrorCode());
1512     }
1513   }
1514   return UnrollOpts;
1515 }
1516 
1517 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
1518   MemorySanitizerOptions Result;
1519   while (!Params.empty()) {
1520     StringRef ParamName;
1521     std::tie(ParamName, Params) = Params.split(';');
1522 
1523     if (ParamName == "recover") {
1524       Result.Recover = true;
1525     } else if (ParamName == "kernel") {
1526       Result.Kernel = true;
1527     } else if (ParamName.consume_front("track-origins=")) {
1528       if (ParamName.getAsInteger(0, Result.TrackOrigins))
1529         return make_error<StringError>(
1530             formatv("invalid argument to MemorySanitizer pass track-origins "
1531                     "parameter: '{0}' ",
1532                     ParamName)
1533                 .str(),
1534             inconvertibleErrorCode());
1535     } else {
1536       return make_error<StringError>(
1537           formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
1538               .str(),
1539           inconvertibleErrorCode());
1540     }
1541   }
1542   return Result;
1543 }
1544 
1545 /// Parser of parameters for SimplifyCFG pass.
1546 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
1547   SimplifyCFGOptions Result;
1548   while (!Params.empty()) {
1549     StringRef ParamName;
1550     std::tie(ParamName, Params) = Params.split(';');
1551 
1552     bool Enable = !ParamName.consume_front("no-");
1553     if (ParamName == "forward-switch-cond") {
1554       Result.forwardSwitchCondToPhi(Enable);
1555     } else if (ParamName == "switch-to-lookup") {
1556       Result.convertSwitchToLookupTable(Enable);
1557     } else if (ParamName == "keep-loops") {
1558       Result.needCanonicalLoops(Enable);
1559     } else if (ParamName == "sink-common-insts") {
1560       Result.sinkCommonInsts(Enable);
1561     } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
1562       APInt BonusInstThreshold;
1563       if (ParamName.getAsInteger(0, BonusInstThreshold))
1564         return make_error<StringError>(
1565             formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1566                     "parameter: '{0}' ",
1567                     ParamName).str(),
1568             inconvertibleErrorCode());
1569       Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
1570     } else {
1571       return make_error<StringError>(
1572           formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
1573           inconvertibleErrorCode());
1574     }
1575   }
1576   return Result;
1577 }
1578 
1579 /// Parser of parameters for LoopVectorize pass.
1580 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
1581   LoopVectorizeOptions Opts;
1582   while (!Params.empty()) {
1583     StringRef ParamName;
1584     std::tie(ParamName, Params) = Params.split(';');
1585 
1586     bool Enable = !ParamName.consume_front("no-");
1587     if (ParamName == "interleave-forced-only") {
1588       Opts.setInterleaveOnlyWhenForced(Enable);
1589     } else if (ParamName == "vectorize-forced-only") {
1590       Opts.setVectorizeOnlyWhenForced(Enable);
1591     } else {
1592       return make_error<StringError>(
1593           formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
1594           inconvertibleErrorCode());
1595     }
1596   }
1597   return Opts;
1598 }
1599 
1600 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
1601   bool Result = false;
1602   while (!Params.empty()) {
1603     StringRef ParamName;
1604     std::tie(ParamName, Params) = Params.split(';');
1605 
1606     bool Enable = !ParamName.consume_front("no-");
1607     if (ParamName == "nontrivial") {
1608       Result = Enable;
1609     } else {
1610       return make_error<StringError>(
1611           formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
1612               .str(),
1613           inconvertibleErrorCode());
1614     }
1615   }
1616   return Result;
1617 }
1618 
1619 Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
1620   bool Result = false;
1621   while (!Params.empty()) {
1622     StringRef ParamName;
1623     std::tie(ParamName, Params) = Params.split(';');
1624 
1625     bool Enable = !ParamName.consume_front("no-");
1626     if (ParamName == "split-footer-bb") {
1627       Result = Enable;
1628     } else {
1629       return make_error<StringError>(
1630           formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
1631                   ParamName)
1632               .str(),
1633           inconvertibleErrorCode());
1634     }
1635   }
1636   return Result;
1637 }
1638 } // namespace
1639 
1640 /// Tests whether a pass name starts with a valid prefix for a default pipeline
1641 /// alias.
1642 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
1643   return Name.startswith("default") || Name.startswith("thinlto") ||
1644          Name.startswith("lto");
1645 }
1646 
1647 /// Tests whether registered callbacks will accept a given pass name.
1648 ///
1649 /// When parsing a pipeline text, the type of the outermost pipeline may be
1650 /// omitted, in which case the type is automatically determined from the first
1651 /// pass name in the text. This may be a name that is handled through one of the
1652 /// callbacks. We check this through the oridinary parsing callbacks by setting
1653 /// up a dummy PassManager in order to not force the client to also handle this
1654 /// type of query.
1655 template <typename PassManagerT, typename CallbacksT>
1656 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
1657   if (!Callbacks.empty()) {
1658     PassManagerT DummyPM;
1659     for (auto &CB : Callbacks)
1660       if (CB(Name, DummyPM, {}))
1661         return true;
1662   }
1663   return false;
1664 }
1665 
1666 template <typename CallbacksT>
1667 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
1668   // Manually handle aliases for pre-configured pipeline fragments.
1669   if (startsWithDefaultPipelineAliasPrefix(Name))
1670     return DefaultAliasRegex.match(Name);
1671 
1672   // Explicitly handle pass manager names.
1673   if (Name == "module")
1674     return true;
1675   if (Name == "cgscc")
1676     return true;
1677   if (Name == "function")
1678     return true;
1679 
1680   // Explicitly handle custom-parsed pass names.
1681   if (parseRepeatPassName(Name))
1682     return true;
1683 
1684 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1685   if (Name == NAME)                                                            \
1686     return true;
1687 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1688   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1689     return true;
1690 #include "PassRegistry.def"
1691 
1692   return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
1693 }
1694 
1695 template <typename CallbacksT>
1696 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
1697   // Explicitly handle pass manager names.
1698   if (Name == "cgscc")
1699     return true;
1700   if (Name == "function")
1701     return true;
1702 
1703   // Explicitly handle custom-parsed pass names.
1704   if (parseRepeatPassName(Name))
1705     return true;
1706   if (parseDevirtPassName(Name))
1707     return true;
1708 
1709 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
1710   if (Name == NAME)                                                            \
1711     return true;
1712 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
1713   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1714     return true;
1715 #include "PassRegistry.def"
1716 
1717   return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
1718 }
1719 
1720 template <typename CallbacksT>
1721 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
1722   // Explicitly handle pass manager names.
1723   if (Name == "function")
1724     return true;
1725   if (Name == "loop" || Name == "loop-mssa")
1726     return true;
1727 
1728   // Explicitly handle custom-parsed pass names.
1729   if (parseRepeatPassName(Name))
1730     return true;
1731 
1732 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
1733   if (Name == NAME)                                                            \
1734     return true;
1735 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
1736   if (checkParametrizedPassName(Name, NAME))                                   \
1737     return true;
1738 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
1739   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1740     return true;
1741 #include "PassRegistry.def"
1742 
1743   return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
1744 }
1745 
1746 template <typename CallbacksT>
1747 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
1748   // Explicitly handle pass manager names.
1749   if (Name == "loop" || Name == "loop-mssa")
1750     return true;
1751 
1752   // Explicitly handle custom-parsed pass names.
1753   if (parseRepeatPassName(Name))
1754     return true;
1755 
1756 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
1757   if (Name == NAME)                                                            \
1758     return true;
1759 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
1760   if (checkParametrizedPassName(Name, NAME))                                   \
1761     return true;
1762 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
1763   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1764     return true;
1765 #include "PassRegistry.def"
1766 
1767   return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
1768 }
1769 
1770 Optional<std::vector<PassBuilder::PipelineElement>>
1771 PassBuilder::parsePipelineText(StringRef Text) {
1772   std::vector<PipelineElement> ResultPipeline;
1773 
1774   SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
1775       &ResultPipeline};
1776   for (;;) {
1777     std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
1778     size_t Pos = Text.find_first_of(",()");
1779     Pipeline.push_back({Text.substr(0, Pos), {}});
1780 
1781     // If we have a single terminating name, we're done.
1782     if (Pos == Text.npos)
1783       break;
1784 
1785     char Sep = Text[Pos];
1786     Text = Text.substr(Pos + 1);
1787     if (Sep == ',')
1788       // Just a name ending in a comma, continue.
1789       continue;
1790 
1791     if (Sep == '(') {
1792       // Push the inner pipeline onto the stack to continue processing.
1793       PipelineStack.push_back(&Pipeline.back().InnerPipeline);
1794       continue;
1795     }
1796 
1797     assert(Sep == ')' && "Bogus separator!");
1798     // When handling the close parenthesis, we greedily consume them to avoid
1799     // empty strings in the pipeline.
1800     do {
1801       // If we try to pop the outer pipeline we have unbalanced parentheses.
1802       if (PipelineStack.size() == 1)
1803         return None;
1804 
1805       PipelineStack.pop_back();
1806     } while (Text.consume_front(")"));
1807 
1808     // Check if we've finished parsing.
1809     if (Text.empty())
1810       break;
1811 
1812     // Otherwise, the end of an inner pipeline always has to be followed by
1813     // a comma, and then we can continue.
1814     if (!Text.consume_front(","))
1815       return None;
1816   }
1817 
1818   if (PipelineStack.size() > 1)
1819     // Unbalanced paretheses.
1820     return None;
1821 
1822   assert(PipelineStack.back() == &ResultPipeline &&
1823          "Wrong pipeline at the bottom of the stack!");
1824   return {std::move(ResultPipeline)};
1825 }
1826 
1827 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
1828                                    const PipelineElement &E,
1829                                    bool VerifyEachPass, bool DebugLogging) {
1830   auto &Name = E.Name;
1831   auto &InnerPipeline = E.InnerPipeline;
1832 
1833   // First handle complex passes like the pass managers which carry pipelines.
1834   if (!InnerPipeline.empty()) {
1835     if (Name == "module") {
1836       ModulePassManager NestedMPM(DebugLogging);
1837       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1838                                              VerifyEachPass, DebugLogging))
1839         return Err;
1840       MPM.addPass(std::move(NestedMPM));
1841       return Error::success();
1842     }
1843     if (Name == "cgscc") {
1844       CGSCCPassManager CGPM(DebugLogging);
1845       if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass,
1846                                             DebugLogging))
1847         return Err;
1848       MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
1849       return Error::success();
1850     }
1851     if (Name == "function") {
1852       FunctionPassManager FPM(DebugLogging);
1853       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1854                                                VerifyEachPass, DebugLogging))
1855         return Err;
1856       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1857       return Error::success();
1858     }
1859     if (auto Count = parseRepeatPassName(Name)) {
1860       ModulePassManager NestedMPM(DebugLogging);
1861       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1862                                              VerifyEachPass, DebugLogging))
1863         return Err;
1864       MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
1865       return Error::success();
1866     }
1867 
1868     for (auto &C : ModulePipelineParsingCallbacks)
1869       if (C(Name, MPM, InnerPipeline))
1870         return Error::success();
1871 
1872     // Normal passes can't have pipelines.
1873     return make_error<StringError>(
1874         formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
1875         inconvertibleErrorCode());
1876     ;
1877   }
1878 
1879   // Manually handle aliases for pre-configured pipeline fragments.
1880   if (startsWithDefaultPipelineAliasPrefix(Name)) {
1881     SmallVector<StringRef, 3> Matches;
1882     if (!DefaultAliasRegex.match(Name, &Matches))
1883       return make_error<StringError>(
1884           formatv("unknown default pipeline alias '{0}'", Name).str(),
1885           inconvertibleErrorCode());
1886 
1887     assert(Matches.size() == 3 && "Must capture two matched strings!");
1888 
1889     OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
1890                               .Case("O0", O0)
1891                               .Case("O1", O1)
1892                               .Case("O2", O2)
1893                               .Case("O3", O3)
1894                               .Case("Os", Os)
1895                               .Case("Oz", Oz);
1896     if (L == O0) {
1897       // Add instrumentation PGO passes -- at O0 we can still do PGO.
1898       if (PGOOpt && Matches[1] != "thinlto" &&
1899           (PGOOpt->Action == PGOOptions::IRInstr ||
1900            PGOOpt->Action == PGOOptions::IRUse))
1901         addPGOInstrPassesForO0(
1902             MPM, DebugLogging,
1903             /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
1904             /* IsCS */ false, PGOOpt->ProfileFile,
1905             PGOOpt->ProfileRemappingFile);
1906       // Do nothing else at all!
1907       return Error::success();
1908     }
1909 
1910     // This is consistent with old pass manager invoked via opt, but
1911     // inconsistent with clang. Clang doesn't enable loop vectorization
1912     // but does enable slp vectorization at Oz.
1913     PTO.LoopVectorization = L > O1 && L < Oz;
1914     PTO.SLPVectorization = L > O1 && L < Oz;
1915 
1916     if (Matches[1] == "default") {
1917       MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging));
1918     } else if (Matches[1] == "thinlto-pre-link") {
1919       MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging));
1920     } else if (Matches[1] == "thinlto") {
1921       MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr));
1922     } else if (Matches[1] == "lto-pre-link") {
1923       MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging));
1924     } else {
1925       assert(Matches[1] == "lto" && "Not one of the matched options!");
1926       MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr));
1927     }
1928     return Error::success();
1929   }
1930 
1931   // Finally expand the basic registered passes from the .inc file.
1932 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1933   if (Name == NAME) {                                                          \
1934     MPM.addPass(CREATE_PASS);                                                  \
1935     return Error::success();                                                   \
1936   }
1937 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1938   if (Name == "require<" NAME ">") {                                           \
1939     MPM.addPass(                                                               \
1940         RequireAnalysisPass<                                                   \
1941             std::remove_reference<decltype(CREATE_PASS)>::type, Module>());    \
1942     return Error::success();                                                   \
1943   }                                                                            \
1944   if (Name == "invalidate<" NAME ">") {                                        \
1945     MPM.addPass(InvalidateAnalysisPass<                                        \
1946                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
1947     return Error::success();                                                   \
1948   }
1949 #include "PassRegistry.def"
1950 
1951   for (auto &C : ModulePipelineParsingCallbacks)
1952     if (C(Name, MPM, InnerPipeline))
1953       return Error::success();
1954   return make_error<StringError>(
1955       formatv("unknown module pass '{0}'", Name).str(),
1956       inconvertibleErrorCode());
1957 }
1958 
1959 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
1960                                   const PipelineElement &E, bool VerifyEachPass,
1961                                   bool DebugLogging) {
1962   auto &Name = E.Name;
1963   auto &InnerPipeline = E.InnerPipeline;
1964 
1965   // First handle complex passes like the pass managers which carry pipelines.
1966   if (!InnerPipeline.empty()) {
1967     if (Name == "cgscc") {
1968       CGSCCPassManager NestedCGPM(DebugLogging);
1969       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1970                                             VerifyEachPass, DebugLogging))
1971         return Err;
1972       // Add the nested pass manager with the appropriate adaptor.
1973       CGPM.addPass(std::move(NestedCGPM));
1974       return Error::success();
1975     }
1976     if (Name == "function") {
1977       FunctionPassManager FPM(DebugLogging);
1978       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1979                                                VerifyEachPass, DebugLogging))
1980         return Err;
1981       // Add the nested pass manager with the appropriate adaptor.
1982       CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
1983       return Error::success();
1984     }
1985     if (auto Count = parseRepeatPassName(Name)) {
1986       CGSCCPassManager NestedCGPM(DebugLogging);
1987       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1988                                             VerifyEachPass, DebugLogging))
1989         return Err;
1990       CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
1991       return Error::success();
1992     }
1993     if (auto MaxRepetitions = parseDevirtPassName(Name)) {
1994       CGSCCPassManager NestedCGPM(DebugLogging);
1995       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1996                                             VerifyEachPass, DebugLogging))
1997         return Err;
1998       CGPM.addPass(
1999           createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
2000       return Error::success();
2001     }
2002 
2003     for (auto &C : CGSCCPipelineParsingCallbacks)
2004       if (C(Name, CGPM, InnerPipeline))
2005         return Error::success();
2006 
2007     // Normal passes can't have pipelines.
2008     return make_error<StringError>(
2009         formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
2010         inconvertibleErrorCode());
2011   }
2012 
2013 // Now expand the basic registered passes from the .inc file.
2014 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
2015   if (Name == NAME) {                                                          \
2016     CGPM.addPass(CREATE_PASS);                                                 \
2017     return Error::success();                                                   \
2018   }
2019 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
2020   if (Name == "require<" NAME ">") {                                           \
2021     CGPM.addPass(RequireAnalysisPass<                                          \
2022                  std::remove_reference<decltype(CREATE_PASS)>::type,           \
2023                  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,    \
2024                  CGSCCUpdateResult &>());                                      \
2025     return Error::success();                                                   \
2026   }                                                                            \
2027   if (Name == "invalidate<" NAME ">") {                                        \
2028     CGPM.addPass(InvalidateAnalysisPass<                                       \
2029                  std::remove_reference<decltype(CREATE_PASS)>::type>());       \
2030     return Error::success();                                                   \
2031   }
2032 #include "PassRegistry.def"
2033 
2034   for (auto &C : CGSCCPipelineParsingCallbacks)
2035     if (C(Name, CGPM, InnerPipeline))
2036       return Error::success();
2037   return make_error<StringError>(
2038       formatv("unknown cgscc pass '{0}'", Name).str(),
2039       inconvertibleErrorCode());
2040 }
2041 
2042 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
2043                                      const PipelineElement &E,
2044                                      bool VerifyEachPass, bool DebugLogging) {
2045   auto &Name = E.Name;
2046   auto &InnerPipeline = E.InnerPipeline;
2047 
2048   // First handle complex passes like the pass managers which carry pipelines.
2049   if (!InnerPipeline.empty()) {
2050     if (Name == "function") {
2051       FunctionPassManager NestedFPM(DebugLogging);
2052       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
2053                                                VerifyEachPass, DebugLogging))
2054         return Err;
2055       // Add the nested pass manager with the appropriate adaptor.
2056       FPM.addPass(std::move(NestedFPM));
2057       return Error::success();
2058     }
2059     if (Name == "loop" || Name == "loop-mssa") {
2060       LoopPassManager LPM(DebugLogging);
2061       if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass,
2062                                            DebugLogging))
2063         return Err;
2064       // Add the nested pass manager with the appropriate adaptor.
2065       bool UseMemorySSA = (Name == "loop-mssa");
2066       FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
2067                                                   DebugLogging));
2068       return Error::success();
2069     }
2070     if (auto Count = parseRepeatPassName(Name)) {
2071       FunctionPassManager NestedFPM(DebugLogging);
2072       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
2073                                                VerifyEachPass, DebugLogging))
2074         return Err;
2075       FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
2076       return Error::success();
2077     }
2078 
2079     for (auto &C : FunctionPipelineParsingCallbacks)
2080       if (C(Name, FPM, InnerPipeline))
2081         return Error::success();
2082 
2083     // Normal passes can't have pipelines.
2084     return make_error<StringError>(
2085         formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
2086         inconvertibleErrorCode());
2087   }
2088 
2089 // Now expand the basic registered passes from the .inc file.
2090 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2091   if (Name == NAME) {                                                          \
2092     FPM.addPass(CREATE_PASS);                                                  \
2093     return Error::success();                                                   \
2094   }
2095 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2096   if (checkParametrizedPassName(Name, NAME)) {                                 \
2097     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2098     if (!Params)                                                               \
2099       return Params.takeError();                                               \
2100     FPM.addPass(CREATE_PASS(Params.get()));                                    \
2101     return Error::success();                                                   \
2102   }
2103 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
2104   if (Name == "require<" NAME ">") {                                           \
2105     FPM.addPass(                                                               \
2106         RequireAnalysisPass<                                                   \
2107             std::remove_reference<decltype(CREATE_PASS)>::type, Function>());  \
2108     return Error::success();                                                   \
2109   }                                                                            \
2110   if (Name == "invalidate<" NAME ">") {                                        \
2111     FPM.addPass(InvalidateAnalysisPass<                                        \
2112                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2113     return Error::success();                                                   \
2114   }
2115 #include "PassRegistry.def"
2116 
2117   for (auto &C : FunctionPipelineParsingCallbacks)
2118     if (C(Name, FPM, InnerPipeline))
2119       return Error::success();
2120   return make_error<StringError>(
2121       formatv("unknown function pass '{0}'", Name).str(),
2122       inconvertibleErrorCode());
2123 }
2124 
2125 Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E,
2126                                  bool VerifyEachPass, bool DebugLogging) {
2127   StringRef Name = E.Name;
2128   auto &InnerPipeline = E.InnerPipeline;
2129 
2130   // First handle complex passes like the pass managers which carry pipelines.
2131   if (!InnerPipeline.empty()) {
2132     if (Name == "loop") {
2133       LoopPassManager NestedLPM(DebugLogging);
2134       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2135                                            VerifyEachPass, DebugLogging))
2136         return Err;
2137       // Add the nested pass manager with the appropriate adaptor.
2138       LPM.addPass(std::move(NestedLPM));
2139       return Error::success();
2140     }
2141     if (auto Count = parseRepeatPassName(Name)) {
2142       LoopPassManager NestedLPM(DebugLogging);
2143       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2144                                            VerifyEachPass, DebugLogging))
2145         return Err;
2146       LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2147       return Error::success();
2148     }
2149 
2150     for (auto &C : LoopPipelineParsingCallbacks)
2151       if (C(Name, LPM, InnerPipeline))
2152         return Error::success();
2153 
2154     // Normal passes can't have pipelines.
2155     return make_error<StringError>(
2156         formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2157         inconvertibleErrorCode());
2158   }
2159 
2160 // Now expand the basic registered passes from the .inc file.
2161 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2162   if (Name == NAME) {                                                          \
2163     LPM.addPass(CREATE_PASS);                                                  \
2164     return Error::success();                                                   \
2165   }
2166 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2167   if (checkParametrizedPassName(Name, NAME)) {                                 \
2168     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2169     if (!Params)                                                               \
2170       return Params.takeError();                                               \
2171     LPM.addPass(CREATE_PASS(Params.get()));                                    \
2172     return Error::success();                                                   \
2173   }
2174 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2175   if (Name == "require<" NAME ">") {                                           \
2176     LPM.addPass(RequireAnalysisPass<                                           \
2177                 std::remove_reference<decltype(CREATE_PASS)>::type, Loop,      \
2178                 LoopAnalysisManager, LoopStandardAnalysisResults &,            \
2179                 LPMUpdater &>());                                              \
2180     return Error::success();                                                   \
2181   }                                                                            \
2182   if (Name == "invalidate<" NAME ">") {                                        \
2183     LPM.addPass(InvalidateAnalysisPass<                                        \
2184                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2185     return Error::success();                                                   \
2186   }
2187 #include "PassRegistry.def"
2188 
2189   for (auto &C : LoopPipelineParsingCallbacks)
2190     if (C(Name, LPM, InnerPipeline))
2191       return Error::success();
2192   return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2193                                  inconvertibleErrorCode());
2194 }
2195 
2196 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2197 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2198   if (Name == NAME) {                                                          \
2199     AA.registerModuleAnalysis<                                                 \
2200         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2201     return true;                                                               \
2202   }
2203 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2204   if (Name == NAME) {                                                          \
2205     AA.registerFunctionAnalysis<                                               \
2206         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2207     return true;                                                               \
2208   }
2209 #include "PassRegistry.def"
2210 
2211   for (auto &C : AAParsingCallbacks)
2212     if (C(Name, AA))
2213       return true;
2214   return false;
2215 }
2216 
2217 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2218                                          ArrayRef<PipelineElement> Pipeline,
2219                                          bool VerifyEachPass,
2220                                          bool DebugLogging) {
2221   for (const auto &Element : Pipeline) {
2222     if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging))
2223       return Err;
2224     // FIXME: No verifier support for Loop passes!
2225   }
2226   return Error::success();
2227 }
2228 
2229 Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM,
2230                                              ArrayRef<PipelineElement> Pipeline,
2231                                              bool VerifyEachPass,
2232                                              bool DebugLogging) {
2233   for (const auto &Element : Pipeline) {
2234     if (auto Err =
2235             parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging))
2236       return Err;
2237     if (VerifyEachPass)
2238       FPM.addPass(VerifierPass());
2239   }
2240   return Error::success();
2241 }
2242 
2243 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2244                                           ArrayRef<PipelineElement> Pipeline,
2245                                           bool VerifyEachPass,
2246                                           bool DebugLogging) {
2247   for (const auto &Element : Pipeline) {
2248     if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging))
2249       return Err;
2250     // FIXME: No verifier support for CGSCC passes!
2251   }
2252   return Error::success();
2253 }
2254 
2255 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
2256                                        FunctionAnalysisManager &FAM,
2257                                        CGSCCAnalysisManager &CGAM,
2258                                        ModuleAnalysisManager &MAM) {
2259   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2260   MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2261   CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2262   FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2263   FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2264   FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2265   LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2266 }
2267 
2268 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2269                                            ArrayRef<PipelineElement> Pipeline,
2270                                            bool VerifyEachPass,
2271                                            bool DebugLogging) {
2272   for (const auto &Element : Pipeline) {
2273     if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging))
2274       return Err;
2275     if (VerifyEachPass)
2276       MPM.addPass(VerifierPass());
2277   }
2278   return Error::success();
2279 }
2280 
2281 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2282 // FIXME: Should this routine accept a TargetMachine or require the caller to
2283 // pre-populate the analysis managers with target-specific stuff?
2284 Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
2285                                      StringRef PipelineText,
2286                                      bool VerifyEachPass, bool DebugLogging) {
2287   auto Pipeline = parsePipelineText(PipelineText);
2288   if (!Pipeline || Pipeline->empty())
2289     return make_error<StringError>(
2290         formatv("invalid pipeline '{0}'", PipelineText).str(),
2291         inconvertibleErrorCode());
2292 
2293   // If the first name isn't at the module layer, wrap the pipeline up
2294   // automatically.
2295   StringRef FirstName = Pipeline->front().Name;
2296 
2297   if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2298     if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
2299       Pipeline = {{"cgscc", std::move(*Pipeline)}};
2300     } else if (isFunctionPassName(FirstName,
2301                                   FunctionPipelineParsingCallbacks)) {
2302       Pipeline = {{"function", std::move(*Pipeline)}};
2303     } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
2304       Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
2305     } else {
2306       for (auto &C : TopLevelPipelineParsingCallbacks)
2307         if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2308           return Error::success();
2309 
2310       // Unknown pass or pipeline name!
2311       auto &InnerPipeline = Pipeline->front().InnerPipeline;
2312       return make_error<StringError>(
2313           formatv("unknown {0} name '{1}'",
2314                   (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
2315               .str(),
2316           inconvertibleErrorCode());
2317     }
2318   }
2319 
2320   if (auto Err =
2321           parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2322     return Err;
2323   return Error::success();
2324 }
2325 
2326 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
2327 Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
2328                                      StringRef PipelineText,
2329                                      bool VerifyEachPass, bool DebugLogging) {
2330   auto Pipeline = parsePipelineText(PipelineText);
2331   if (!Pipeline || Pipeline->empty())
2332     return make_error<StringError>(
2333         formatv("invalid pipeline '{0}'", PipelineText).str(),
2334         inconvertibleErrorCode());
2335 
2336   StringRef FirstName = Pipeline->front().Name;
2337   if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
2338     return make_error<StringError>(
2339         formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
2340                 PipelineText)
2341             .str(),
2342         inconvertibleErrorCode());
2343 
2344   if (auto Err =
2345           parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2346     return Err;
2347   return Error::success();
2348 }
2349 
2350 // Primary pass pipeline description parsing routine for a \c
2351 // FunctionPassManager
2352 Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
2353                                      StringRef PipelineText,
2354                                      bool VerifyEachPass, bool DebugLogging) {
2355   auto Pipeline = parsePipelineText(PipelineText);
2356   if (!Pipeline || Pipeline->empty())
2357     return make_error<StringError>(
2358         formatv("invalid pipeline '{0}'", PipelineText).str(),
2359         inconvertibleErrorCode());
2360 
2361   StringRef FirstName = Pipeline->front().Name;
2362   if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
2363     return make_error<StringError>(
2364         formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
2365                 PipelineText)
2366             .str(),
2367         inconvertibleErrorCode());
2368 
2369   if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass,
2370                                            DebugLogging))
2371     return Err;
2372   return Error::success();
2373 }
2374 
2375 // Primary pass pipeline description parsing routine for a \c LoopPassManager
2376 Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
2377                                      StringRef PipelineText,
2378                                      bool VerifyEachPass, bool DebugLogging) {
2379   auto Pipeline = parsePipelineText(PipelineText);
2380   if (!Pipeline || Pipeline->empty())
2381     return make_error<StringError>(
2382         formatv("invalid pipeline '{0}'", PipelineText).str(),
2383         inconvertibleErrorCode());
2384 
2385   if (auto Err =
2386           parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2387     return Err;
2388 
2389   return Error::success();
2390 }
2391 
2392 Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
2393   // If the pipeline just consists of the word 'default' just replace the AA
2394   // manager with our default one.
2395   if (PipelineText == "default") {
2396     AA = buildDefaultAAPipeline();
2397     return Error::success();
2398   }
2399 
2400   while (!PipelineText.empty()) {
2401     StringRef Name;
2402     std::tie(Name, PipelineText) = PipelineText.split(',');
2403     if (!parseAAPassName(AA, Name))
2404       return make_error<StringError>(
2405           formatv("unknown alias analysis name '{0}'", Name).str(),
2406           inconvertibleErrorCode());
2407   }
2408 
2409   return Error::success();
2410 }
2411