1 //===- Parsing, selection, and construction of pass pipelines -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
13 /// pipelines.
14 ///
15 //===----------------------------------------------------------------------===//
16
17 #include "llvm/Passes/PassBuilder.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/Analysis/AliasAnalysis.h"
20 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
21 #include "llvm/Analysis/AliasSetTracker.h"
22 #include "llvm/Analysis/AssumptionCache.h"
23 #include "llvm/Analysis/BasicAliasAnalysis.h"
24 #include "llvm/Analysis/BlockFrequencyInfo.h"
25 #include "llvm/Analysis/BranchProbabilityInfo.h"
26 #include "llvm/Analysis/CFGPrinter.h"
27 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
28 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
29 #include "llvm/Analysis/CGSCCPassManager.h"
30 #include "llvm/Analysis/CallGraph.h"
31 #include "llvm/Analysis/DDG.h"
32 #include "llvm/Analysis/Delinearization.h"
33 #include "llvm/Analysis/DemandedBits.h"
34 #include "llvm/Analysis/DependenceAnalysis.h"
35 #include "llvm/Analysis/DominanceFrontier.h"
36 #include "llvm/Analysis/FunctionPropertiesAnalysis.h"
37 #include "llvm/Analysis/GlobalsModRef.h"
38 #include "llvm/Analysis/IRSimilarityIdentifier.h"
39 #include "llvm/Analysis/IVUsers.h"
40 #include "llvm/Analysis/InlineAdvisor.h"
41 #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
42 #include "llvm/Analysis/InstCount.h"
43 #include "llvm/Analysis/LazyCallGraph.h"
44 #include "llvm/Analysis/LazyValueInfo.h"
45 #include "llvm/Analysis/Lint.h"
46 #include "llvm/Analysis/LoopAccessAnalysis.h"
47 #include "llvm/Analysis/LoopCacheAnalysis.h"
48 #include "llvm/Analysis/LoopInfo.h"
49 #include "llvm/Analysis/LoopNestAnalysis.h"
50 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
51 #include "llvm/Analysis/MemorySSA.h"
52 #include "llvm/Analysis/ModuleDebugInfoPrinter.h"
53 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
54 #include "llvm/Analysis/MustExecute.h"
55 #include "llvm/Analysis/ObjCARCAliasAnalysis.h"
56 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
57 #include "llvm/Analysis/PhiValues.h"
58 #include "llvm/Analysis/PostDominators.h"
59 #include "llvm/Analysis/ProfileSummaryInfo.h"
60 #include "llvm/Analysis/RegionInfo.h"
61 #include "llvm/Analysis/ScalarEvolution.h"
62 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
63 #include "llvm/Analysis/ScopedNoAliasAA.h"
64 #include "llvm/Analysis/StackLifetime.h"
65 #include "llvm/Analysis/StackSafetyAnalysis.h"
66 #include "llvm/Analysis/TargetLibraryInfo.h"
67 #include "llvm/Analysis/TargetTransformInfo.h"
68 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
69 #include "llvm/IR/Dominators.h"
70 #include "llvm/IR/IRPrintingPasses.h"
71 #include "llvm/IR/PassManager.h"
72 #include "llvm/IR/SafepointIRVerifier.h"
73 #include "llvm/IR/Verifier.h"
74 #include "llvm/Support/CommandLine.h"
75 #include "llvm/Support/Debug.h"
76 #include "llvm/Support/FormatVariadic.h"
77 #include "llvm/Support/Regex.h"
78 #include "llvm/Target/TargetMachine.h"
79 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
80 #include "llvm/Transforms/Coroutines/CoroCleanup.h"
81 #include "llvm/Transforms/Coroutines/CoroEarly.h"
82 #include "llvm/Transforms/Coroutines/CoroElide.h"
83 #include "llvm/Transforms/Coroutines/CoroSplit.h"
84 #include "llvm/Transforms/HelloNew/HelloWorld.h"
85 #include "llvm/Transforms/IPO/AlwaysInliner.h"
86 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
87 #include "llvm/Transforms/IPO/Attributor.h"
88 #include "llvm/Transforms/IPO/BlockExtractor.h"
89 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
90 #include "llvm/Transforms/IPO/ConstantMerge.h"
91 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
92 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
93 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
94 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
95 #include "llvm/Transforms/IPO/FunctionAttrs.h"
96 #include "llvm/Transforms/IPO/FunctionImport.h"
97 #include "llvm/Transforms/IPO/GlobalDCE.h"
98 #include "llvm/Transforms/IPO/GlobalOpt.h"
99 #include "llvm/Transforms/IPO/GlobalSplit.h"
100 #include "llvm/Transforms/IPO/HotColdSplitting.h"
101 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
102 #include "llvm/Transforms/IPO/Inliner.h"
103 #include "llvm/Transforms/IPO/Internalize.h"
104 #include "llvm/Transforms/IPO/LoopExtractor.h"
105 #include "llvm/Transforms/IPO/LowerTypeTests.h"
106 #include "llvm/Transforms/IPO/MergeFunctions.h"
107 #include "llvm/Transforms/IPO/OpenMPOpt.h"
108 #include "llvm/Transforms/IPO/PartialInlining.h"
109 #include "llvm/Transforms/IPO/SCCP.h"
110 #include "llvm/Transforms/IPO/SampleProfile.h"
111 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
112 #include "llvm/Transforms/IPO/StripSymbols.h"
113 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
114 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
115 #include "llvm/Transforms/InstCombine/InstCombine.h"
116 #include "llvm/Transforms/Instrumentation.h"
117 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
118 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
119 #include "llvm/Transforms/Instrumentation/CGProfile.h"
120 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
121 #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
122 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
123 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
124 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
125 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
126 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
127 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
128 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
129 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
130 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
131 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
132 #include "llvm/Transforms/ObjCARC.h"
133 #include "llvm/Transforms/Scalar/ADCE.h"
134 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
135 #include "llvm/Transforms/Scalar/BDCE.h"
136 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
137 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
138 #include "llvm/Transforms/Scalar/ConstraintElimination.h"
139 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
140 #include "llvm/Transforms/Scalar/DCE.h"
141 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
142 #include "llvm/Transforms/Scalar/DivRemPairs.h"
143 #include "llvm/Transforms/Scalar/EarlyCSE.h"
144 #include "llvm/Transforms/Scalar/Float2Int.h"
145 #include "llvm/Transforms/Scalar/GVN.h"
146 #include "llvm/Transforms/Scalar/GuardWidening.h"
147 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
148 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
149 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
150 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
151 #include "llvm/Transforms/Scalar/JumpThreading.h"
152 #include "llvm/Transforms/Scalar/LICM.h"
153 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
154 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
155 #include "llvm/Transforms/Scalar/LoopDeletion.h"
156 #include "llvm/Transforms/Scalar/LoopDistribute.h"
157 #include "llvm/Transforms/Scalar/LoopFlatten.h"
158 #include "llvm/Transforms/Scalar/LoopFuse.h"
159 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
160 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
161 #include "llvm/Transforms/Scalar/LoopInterchange.h"
162 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
163 #include "llvm/Transforms/Scalar/LoopPassManager.h"
164 #include "llvm/Transforms/Scalar/LoopPredication.h"
165 #include "llvm/Transforms/Scalar/LoopReroll.h"
166 #include "llvm/Transforms/Scalar/LoopRotation.h"
167 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
168 #include "llvm/Transforms/Scalar/LoopSink.h"
169 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
170 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
171 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
172 #include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
173 #include "llvm/Transforms/Scalar/LowerAtomic.h"
174 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
175 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
176 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
177 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
178 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
179 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
180 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
181 #include "llvm/Transforms/Scalar/MergeICmps.h"
182 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
183 #include "llvm/Transforms/Scalar/NaryReassociate.h"
184 #include "llvm/Transforms/Scalar/NewGVN.h"
185 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
186 #include "llvm/Transforms/Scalar/Reassociate.h"
187 #include "llvm/Transforms/Scalar/Reg2Mem.h"
188 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
189 #include "llvm/Transforms/Scalar/SCCP.h"
190 #include "llvm/Transforms/Scalar/SROA.h"
191 #include "llvm/Transforms/Scalar/Scalarizer.h"
192 #include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
193 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
194 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
195 #include "llvm/Transforms/Scalar/Sink.h"
196 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
197 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
198 #include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
199 #include "llvm/Transforms/Scalar/StructurizeCFG.h"
200 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
201 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
202 #include "llvm/Transforms/Utils/AddDiscriminators.h"
203 #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
204 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
205 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
206 #include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
207 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
208 #include "llvm/Transforms/Utils/FixIrreducible.h"
209 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
210 #include "llvm/Transforms/Utils/InstructionNamer.h"
211 #include "llvm/Transforms/Utils/LCSSA.h"
212 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
213 #include "llvm/Transforms/Utils/LoopSimplify.h"
214 #include "llvm/Transforms/Utils/LoopVersioning.h"
215 #include "llvm/Transforms/Utils/LowerInvoke.h"
216 #include "llvm/Transforms/Utils/LowerSwitch.h"
217 #include "llvm/Transforms/Utils/Mem2Reg.h"
218 #include "llvm/Transforms/Utils/MetaRenamer.h"
219 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
220 #include "llvm/Transforms/Utils/StripGCRelocates.h"
221 #include "llvm/Transforms/Utils/StripNonLineTableDebugInfo.h"
222 #include "llvm/Transforms/Utils/SymbolRewriter.h"
223 #include "llvm/Transforms/Utils/UnifyFunctionExitNodes.h"
224 #include "llvm/Transforms/Utils/UnifyLoopExits.h"
225 #include "llvm/Transforms/Utils/UniqueInternalLinkageNames.h"
226 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
227 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
228 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
229 #include "llvm/Transforms/Vectorize/VectorCombine.h"
230
231 using namespace llvm;
232
233 static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
234 cl::ReallyHidden, cl::init(4));
235 static cl::opt<bool>
236 RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
237 cl::Hidden, cl::ZeroOrMore,
238 cl::desc("Run Partial inlinining pass"));
239
240 static cl::opt<int> PreInlineThreshold(
241 "npm-preinline-threshold", cl::Hidden, cl::init(75), cl::ZeroOrMore,
242 cl::desc("Control the amount of inlining in pre-instrumentation inliner "
243 "(default = 75)"));
244
245 static cl::opt<bool>
246 RunNewGVN("enable-npm-newgvn", cl::init(false),
247 cl::Hidden, cl::ZeroOrMore,
248 cl::desc("Run NewGVN instead of GVN"));
249
250 static cl::opt<bool> EnableGVNHoist(
251 "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
252 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
253
254 static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
255 "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
256 cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
257 cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
258 "Heuristics-based inliner version."),
259 clEnumValN(InliningAdvisorMode::Development, "development",
260 "Use development mode (runtime-loadable model)."),
261 clEnumValN(InliningAdvisorMode::Release, "release",
262 "Use release mode (AOT-compiled model).")));
263
264 static cl::opt<bool> EnableGVNSink(
265 "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
266 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
267
268 static cl::opt<bool> EnableUnrollAndJam(
269 "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden,
270 cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
271
272 static cl::opt<bool> EnableLoopFlatten(
273 "enable-npm-loop-flatten", cl::init(false), cl::Hidden,
274 cl::desc("Enable the Loop flattening pass for the new PM (default = off)"));
275
276 static cl::opt<bool> EnableSyntheticCounts(
277 "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
278 cl::desc("Run synthetic function entry count generation "
279 "pass"));
280
281 static const Regex DefaultAliasRegex(
282 "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
283
284 // This option is used in simplifying testing SampleFDO optimizations for
285 // profile loading.
286 static cl::opt<bool>
287 EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
288 cl::desc("Enable control height reduction optimization (CHR)"));
289
290 /// Flag to enable inline deferral during PGO.
291 static cl::opt<bool>
292 EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
293 cl::Hidden,
294 cl::desc("Enable inline deferral during PGO"));
295
296 static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
297 cl::Hidden, cl::ZeroOrMore,
298 cl::desc("Enable memory profiler"));
299
PipelineTuningOptions()300 PipelineTuningOptions::PipelineTuningOptions() {
301 LoopInterleaving = true;
302 LoopVectorization = true;
303 SLPVectorization = false;
304 LoopUnrolling = true;
305 ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
306 Coroutines = false;
307 LicmMssaOptCap = SetLicmMssaOptCap;
308 LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
309 CallGraphProfile = true;
310 }
311
312 extern cl::opt<bool> EnableConstraintElimination;
313 extern cl::opt<bool> EnableHotColdSplit;
314 extern cl::opt<bool> EnableOrderFileInstrumentation;
315
316 extern cl::opt<bool> FlattenedProfileUsed;
317
318 extern cl::opt<AttributorRunOption> AttributorRun;
319 extern cl::opt<bool> EnableKnowledgeRetention;
320
321 extern cl::opt<bool> EnableMatrix;
322
323 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O0 = {
324 /*SpeedLevel*/ 0,
325 /*SizeLevel*/ 0};
326 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O1 = {
327 /*SpeedLevel*/ 1,
328 /*SizeLevel*/ 0};
329 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O2 = {
330 /*SpeedLevel*/ 2,
331 /*SizeLevel*/ 0};
332 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O3 = {
333 /*SpeedLevel*/ 3,
334 /*SizeLevel*/ 0};
335 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Os = {
336 /*SpeedLevel*/ 2,
337 /*SizeLevel*/ 1};
338 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Oz = {
339 /*SpeedLevel*/ 2,
340 /*SizeLevel*/ 2};
341
342 namespace {
343
344 // The following passes/analyses have custom names, otherwise their name will
345 // include `(anonymous namespace)`. These are special since they are only for
346 // testing purposes and don't live in a header file.
347
348 /// No-op module pass which does nothing.
349 struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
run__anon3cf1f6a50111::NoOpModulePass350 PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
351 return PreservedAnalyses::all();
352 }
353
name__anon3cf1f6a50111::NoOpModulePass354 static StringRef name() { return "NoOpModulePass"; }
355 };
356
357 /// No-op module analysis.
358 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
359 friend AnalysisInfoMixin<NoOpModuleAnalysis>;
360 static AnalysisKey Key;
361
362 public:
363 struct Result {};
run(Module &,ModuleAnalysisManager &)364 Result run(Module &, ModuleAnalysisManager &) { return Result(); }
name()365 static StringRef name() { return "NoOpModuleAnalysis"; }
366 };
367
368 /// No-op CGSCC pass which does nothing.
369 struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
run__anon3cf1f6a50111::NoOpCGSCCPass370 PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
371 LazyCallGraph &, CGSCCUpdateResult &UR) {
372 return PreservedAnalyses::all();
373 }
name__anon3cf1f6a50111::NoOpCGSCCPass374 static StringRef name() { return "NoOpCGSCCPass"; }
375 };
376
377 /// No-op CGSCC analysis.
378 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
379 friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
380 static AnalysisKey Key;
381
382 public:
383 struct Result {};
run(LazyCallGraph::SCC &,CGSCCAnalysisManager &,LazyCallGraph & G)384 Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
385 return Result();
386 }
name()387 static StringRef name() { return "NoOpCGSCCAnalysis"; }
388 };
389
390 /// No-op function pass which does nothing.
391 struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
run__anon3cf1f6a50111::NoOpFunctionPass392 PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
393 return PreservedAnalyses::all();
394 }
name__anon3cf1f6a50111::NoOpFunctionPass395 static StringRef name() { return "NoOpFunctionPass"; }
396 };
397
398 /// No-op function analysis.
399 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
400 friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
401 static AnalysisKey Key;
402
403 public:
404 struct Result {};
run(Function &,FunctionAnalysisManager &)405 Result run(Function &, FunctionAnalysisManager &) { return Result(); }
name()406 static StringRef name() { return "NoOpFunctionAnalysis"; }
407 };
408
409 /// No-op loop pass which does nothing.
410 struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
run__anon3cf1f6a50111::NoOpLoopPass411 PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
412 LoopStandardAnalysisResults &, LPMUpdater &) {
413 return PreservedAnalyses::all();
414 }
name__anon3cf1f6a50111::NoOpLoopPass415 static StringRef name() { return "NoOpLoopPass"; }
416 };
417
418 /// No-op loop analysis.
419 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
420 friend AnalysisInfoMixin<NoOpLoopAnalysis>;
421 static AnalysisKey Key;
422
423 public:
424 struct Result {};
run(Loop &,LoopAnalysisManager &,LoopStandardAnalysisResults &)425 Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
426 return Result();
427 }
name()428 static StringRef name() { return "NoOpLoopAnalysis"; }
429 };
430
431 AnalysisKey NoOpModuleAnalysis::Key;
432 AnalysisKey NoOpCGSCCAnalysis::Key;
433 AnalysisKey NoOpFunctionAnalysis::Key;
434 AnalysisKey NoOpLoopAnalysis::Key;
435
436 } // namespace
437
PassBuilder(bool DebugLogging,TargetMachine * TM,PipelineTuningOptions PTO,Optional<PGOOptions> PGOOpt,PassInstrumentationCallbacks * PIC)438 PassBuilder::PassBuilder(bool DebugLogging, TargetMachine *TM,
439 PipelineTuningOptions PTO, Optional<PGOOptions> PGOOpt,
440 PassInstrumentationCallbacks *PIC)
441 : DebugLogging(DebugLogging), TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
442 if (TM)
443 TM->registerPassBuilderCallbacks(*this, DebugLogging);
444 }
445
invokePeepholeEPCallbacks(FunctionPassManager & FPM,PassBuilder::OptimizationLevel Level)446 void PassBuilder::invokePeepholeEPCallbacks(
447 FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
448 for (auto &C : PeepholeEPCallbacks)
449 C(FPM, Level);
450 }
451
registerModuleAnalyses(ModuleAnalysisManager & MAM)452 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
453 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
454 MAM.registerPass([&] { return CREATE_PASS; });
455 #include "PassRegistry.def"
456
457 for (auto &C : ModuleAnalysisRegistrationCallbacks)
458 C(MAM);
459 }
460
registerCGSCCAnalyses(CGSCCAnalysisManager & CGAM)461 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
462 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
463 CGAM.registerPass([&] { return CREATE_PASS; });
464 #include "PassRegistry.def"
465
466 for (auto &C : CGSCCAnalysisRegistrationCallbacks)
467 C(CGAM);
468 }
469
registerFunctionAnalyses(FunctionAnalysisManager & FAM)470 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
471 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
472 FAM.registerPass([&] { return CREATE_PASS; });
473 #include "PassRegistry.def"
474
475 for (auto &C : FunctionAnalysisRegistrationCallbacks)
476 C(FAM);
477 }
478
registerLoopAnalyses(LoopAnalysisManager & LAM)479 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
480 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
481 LAM.registerPass([&] { return CREATE_PASS; });
482 #include "PassRegistry.def"
483
484 for (auto &C : LoopAnalysisRegistrationCallbacks)
485 C(LAM);
486 }
487
488 // TODO: Investigate the cost/benefit of tail call elimination on debugging.
489 FunctionPassManager
buildO1FunctionSimplificationPipeline(OptimizationLevel Level,ThinLTOPhase Phase)490 PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
491 ThinLTOPhase Phase) {
492
493 FunctionPassManager FPM(DebugLogging);
494
495 // Form SSA out of local memory accesses after breaking apart aggregates into
496 // scalars.
497 FPM.addPass(SROA());
498
499 // Catch trivial redundancies
500 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
501
502 // Hoisting of scalars and load expressions.
503 FPM.addPass(SimplifyCFGPass());
504 FPM.addPass(InstCombinePass());
505
506 FPM.addPass(LibCallsShrinkWrapPass());
507
508 invokePeepholeEPCallbacks(FPM, Level);
509
510 FPM.addPass(SimplifyCFGPass());
511
512 // Form canonically associated expression trees, and simplify the trees using
513 // basic mathematical properties. For example, this will form (nearly)
514 // minimal multiplication trees.
515 FPM.addPass(ReassociatePass());
516
517 // Add the primary loop simplification pipeline.
518 // FIXME: Currently this is split into two loop pass pipelines because we run
519 // some function passes in between them. These can and should be removed
520 // and/or replaced by scheduling the loop pass equivalents in the correct
521 // positions. But those equivalent passes aren't powerful enough yet.
522 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
523 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
524 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
525 // `LoopInstSimplify`.
526 LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
527
528 // Simplify the loop body. We do this initially to clean up after other loop
529 // passes run, either when iterating on a loop or on inner loops with
530 // implications on the outer loop.
531 LPM1.addPass(LoopInstSimplifyPass());
532 LPM1.addPass(LoopSimplifyCFGPass());
533
534 LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true));
535 // TODO: Investigate promotion cap for O1.
536 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
537 LPM1.addPass(SimpleLoopUnswitchPass());
538
539 if (EnableLoopFlatten)
540 FPM.addPass(LoopFlattenPass());
541 LPM2.addPass(IndVarSimplifyPass());
542 LPM2.addPass(LoopIdiomRecognizePass());
543
544 for (auto &C : LateLoopOptimizationsEPCallbacks)
545 C(LPM2, Level);
546
547 LPM2.addPass(LoopDeletionPass());
548 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
549 // because it changes IR to makes profile annotation in back compile
550 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
551 // attributes so we need to make sure and allow the full unroll pass to pay
552 // attention to it.
553 if (Phase != ThinLTOPhase::PreLink || !PGOOpt ||
554 PGOOpt->Action != PGOOptions::SampleUse)
555 LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
556 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
557 PTO.ForgetAllSCEVInLoopUnroll));
558
559 for (auto &C : LoopOptimizerEndEPCallbacks)
560 C(LPM2, Level);
561
562 // We provide the opt remark emitter pass for LICM to use. We only need to do
563 // this once as it is immutable.
564 FPM.addPass(
565 RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
566 FPM.addPass(createFunctionToLoopPassAdaptor(
567 std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
568 DebugLogging));
569 FPM.addPass(SimplifyCFGPass());
570 FPM.addPass(InstCombinePass());
571 // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
572 // *All* loop passes must preserve it, in order to be able to use it.
573 FPM.addPass(createFunctionToLoopPassAdaptor(
574 std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
575 DebugLogging));
576
577 // Delete small array after loop unroll.
578 FPM.addPass(SROA());
579
580 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
581 FPM.addPass(MemCpyOptPass());
582
583 // Sparse conditional constant propagation.
584 // FIXME: It isn't clear why we do this *after* loop passes rather than
585 // before...
586 FPM.addPass(SCCPPass());
587
588 // Delete dead bit computations (instcombine runs after to fold away the dead
589 // computations, and then ADCE will run later to exploit any new DCE
590 // opportunities that creates).
591 FPM.addPass(BDCEPass());
592
593 // Run instcombine after redundancy and dead bit elimination to exploit
594 // opportunities opened up by them.
595 FPM.addPass(InstCombinePass());
596 invokePeepholeEPCallbacks(FPM, Level);
597
598 if (PTO.Coroutines)
599 FPM.addPass(CoroElidePass());
600
601 for (auto &C : ScalarOptimizerLateEPCallbacks)
602 C(FPM, Level);
603
604 // Finally, do an expensive DCE pass to catch all the dead code exposed by
605 // the simplifications and basic cleanup after all the simplifications.
606 // TODO: Investigate if this is too expensive.
607 FPM.addPass(ADCEPass());
608 FPM.addPass(SimplifyCFGPass());
609 FPM.addPass(InstCombinePass());
610 invokePeepholeEPCallbacks(FPM, Level);
611
612 return FPM;
613 }
614
615 FunctionPassManager
buildFunctionSimplificationPipeline(OptimizationLevel Level,ThinLTOPhase Phase)616 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
617 ThinLTOPhase Phase) {
618 assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
619
620 // The O1 pipeline has a separate pipeline creation function to simplify
621 // construction readability.
622 if (Level.getSpeedupLevel() == 1)
623 return buildO1FunctionSimplificationPipeline(Level, Phase);
624
625 FunctionPassManager FPM(DebugLogging);
626
627 // Form SSA out of local memory accesses after breaking apart aggregates into
628 // scalars.
629 FPM.addPass(SROA());
630
631 // Catch trivial redundancies
632 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
633 if (EnableKnowledgeRetention)
634 FPM.addPass(AssumeSimplifyPass());
635
636 // Hoisting of scalars and load expressions.
637 if (EnableGVNHoist)
638 FPM.addPass(GVNHoistPass());
639
640 // Global value numbering based sinking.
641 if (EnableGVNSink) {
642 FPM.addPass(GVNSinkPass());
643 FPM.addPass(SimplifyCFGPass());
644 }
645
646 if (EnableConstraintElimination)
647 FPM.addPass(ConstraintEliminationPass());
648
649 // Speculative execution if the target has divergent branches; otherwise nop.
650 FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
651
652 // Optimize based on known information about branches, and cleanup afterward.
653 FPM.addPass(JumpThreadingPass());
654 FPM.addPass(CorrelatedValuePropagationPass());
655
656 FPM.addPass(SimplifyCFGPass());
657 if (Level == OptimizationLevel::O3)
658 FPM.addPass(AggressiveInstCombinePass());
659 FPM.addPass(InstCombinePass());
660
661 if (!Level.isOptimizingForSize())
662 FPM.addPass(LibCallsShrinkWrapPass());
663
664 invokePeepholeEPCallbacks(FPM, Level);
665
666 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
667 // using the size value profile. Don't perform this when optimizing for size.
668 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
669 !Level.isOptimizingForSize())
670 FPM.addPass(PGOMemOPSizeOpt());
671
672 FPM.addPass(TailCallElimPass());
673 FPM.addPass(SimplifyCFGPass());
674
675 // Form canonically associated expression trees, and simplify the trees using
676 // basic mathematical properties. For example, this will form (nearly)
677 // minimal multiplication trees.
678 FPM.addPass(ReassociatePass());
679
680 // Add the primary loop simplification pipeline.
681 // FIXME: Currently this is split into two loop pass pipelines because we run
682 // some function passes in between them. These can and should be removed
683 // and/or replaced by scheduling the loop pass equivalents in the correct
684 // positions. But those equivalent passes aren't powerful enough yet.
685 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
686 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
687 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
688 // `LoopInstSimplify`.
689 LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
690
691 // Simplify the loop body. We do this initially to clean up after other loop
692 // passes run, either when iterating on a loop or on inner loops with
693 // implications on the outer loop.
694 LPM1.addPass(LoopInstSimplifyPass());
695 LPM1.addPass(LoopSimplifyCFGPass());
696
697 // Disable header duplication in loop rotation at -Oz.
698 LPM1.addPass(LoopRotatePass(Level != OptimizationLevel::Oz));
699 // TODO: Investigate promotion cap for O1.
700 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
701 LPM1.addPass(SimpleLoopUnswitchPass());
702 LPM2.addPass(IndVarSimplifyPass());
703 LPM2.addPass(LoopIdiomRecognizePass());
704
705 for (auto &C : LateLoopOptimizationsEPCallbacks)
706 C(LPM2, Level);
707
708 LPM2.addPass(LoopDeletionPass());
709 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
710 // because it changes IR to makes profile annotation in back compile
711 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
712 // attributes so we need to make sure and allow the full unroll pass to pay
713 // attention to it.
714 if (Phase != ThinLTOPhase::PreLink || !PGOOpt ||
715 PGOOpt->Action != PGOOptions::SampleUse)
716 LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
717 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
718 PTO.ForgetAllSCEVInLoopUnroll));
719
720 for (auto &C : LoopOptimizerEndEPCallbacks)
721 C(LPM2, Level);
722
723 // We provide the opt remark emitter pass for LICM to use. We only need to do
724 // this once as it is immutable.
725 FPM.addPass(
726 RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
727 FPM.addPass(createFunctionToLoopPassAdaptor(
728 std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
729 DebugLogging));
730 FPM.addPass(SimplifyCFGPass());
731 FPM.addPass(InstCombinePass());
732 // The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass,
733 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
734 // *All* loop passes must preserve it, in order to be able to use it.
735 FPM.addPass(createFunctionToLoopPassAdaptor(
736 std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
737 DebugLogging));
738
739 // Delete small array after loop unroll.
740 FPM.addPass(SROA());
741
742 // Eliminate redundancies.
743 FPM.addPass(MergedLoadStoreMotionPass());
744 if (RunNewGVN)
745 FPM.addPass(NewGVNPass());
746 else
747 FPM.addPass(GVN());
748
749 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
750 FPM.addPass(MemCpyOptPass());
751
752 // Sparse conditional constant propagation.
753 // FIXME: It isn't clear why we do this *after* loop passes rather than
754 // before...
755 FPM.addPass(SCCPPass());
756
757 // Delete dead bit computations (instcombine runs after to fold away the dead
758 // computations, and then ADCE will run later to exploit any new DCE
759 // opportunities that creates).
760 FPM.addPass(BDCEPass());
761
762 // Run instcombine after redundancy and dead bit elimination to exploit
763 // opportunities opened up by them.
764 FPM.addPass(InstCombinePass());
765 invokePeepholeEPCallbacks(FPM, Level);
766
767 // Re-consider control flow based optimizations after redundancy elimination,
768 // redo DCE, etc.
769 FPM.addPass(JumpThreadingPass());
770 FPM.addPass(CorrelatedValuePropagationPass());
771
772 // Finally, do an expensive DCE pass to catch all the dead code exposed by
773 // the simplifications and basic cleanup after all the simplifications.
774 // TODO: Investigate if this is too expensive.
775 FPM.addPass(ADCEPass());
776
777 FPM.addPass(DSEPass());
778 FPM.addPass(createFunctionToLoopPassAdaptor(
779 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
780 EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
781
782 if (PTO.Coroutines)
783 FPM.addPass(CoroElidePass());
784
785 for (auto &C : ScalarOptimizerLateEPCallbacks)
786 C(FPM, Level);
787
788 FPM.addPass(SimplifyCFGPass());
789 FPM.addPass(InstCombinePass());
790 invokePeepholeEPCallbacks(FPM, Level);
791
792 if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
793 (PGOOpt->Action == PGOOptions::IRUse ||
794 PGOOpt->Action == PGOOptions::SampleUse))
795 FPM.addPass(ControlHeightReductionPass());
796
797 return FPM;
798 }
799
addPGOInstrPasses(ModulePassManager & MPM,PassBuilder::OptimizationLevel Level,bool RunProfileGen,bool IsCS,std::string ProfileFile,std::string ProfileRemappingFile)800 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
801 PassBuilder::OptimizationLevel Level,
802 bool RunProfileGen, bool IsCS,
803 std::string ProfileFile,
804 std::string ProfileRemappingFile) {
805 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
806 // Generally running simplification passes and the inliner with an high
807 // threshold results in smaller executables, but there may be cases where
808 // the size grows, so let's be conservative here and skip this simplification
809 // at -Os/Oz. We will not do this inline for context sensistive PGO (when
810 // IsCS is true).
811 if (!Level.isOptimizingForSize() && !IsCS) {
812 InlineParams IP;
813
814 IP.DefaultThreshold = PreInlineThreshold;
815
816 // FIXME: The hint threshold has the same value used by the regular inliner.
817 // This should probably be lowered after performance testing.
818 // FIXME: this comment is cargo culted from the old pass manager, revisit).
819 IP.HintThreshold = 325;
820 ModuleInlinerWrapperPass MIWP(IP, DebugLogging);
821 CGSCCPassManager &CGPipeline = MIWP.getPM();
822
823 FunctionPassManager FPM;
824 FPM.addPass(SROA());
825 FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
826 FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
827 FPM.addPass(InstCombinePass()); // Combine silly sequences.
828 invokePeepholeEPCallbacks(FPM, Level);
829
830 CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
831
832 MPM.addPass(std::move(MIWP));
833
834 // Delete anything that is now dead to make sure that we don't instrument
835 // dead code. Instrumentation can end up keeping dead code around and
836 // dramatically increase code size.
837 MPM.addPass(GlobalDCEPass());
838 }
839
840 if (!RunProfileGen) {
841 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
842 MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
843 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
844 // RequireAnalysisPass for PSI before subsequent non-module passes.
845 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
846 return;
847 }
848
849 // Perform PGO instrumentation.
850 MPM.addPass(PGOInstrumentationGen(IsCS));
851
852 FunctionPassManager FPM;
853 // Disable header duplication in loop rotation at -Oz.
854 FPM.addPass(createFunctionToLoopPassAdaptor(
855 LoopRotatePass(Level != OptimizationLevel::Oz), EnableMSSALoopDependency,
856 /*UseBlockFrequencyInfo=*/false, DebugLogging));
857 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
858
859 // Add the profile lowering pass.
860 InstrProfOptions Options;
861 if (!ProfileFile.empty())
862 Options.InstrProfileOutput = ProfileFile;
863 // Do counter promotion at Level greater than O0.
864 Options.DoCounterPromotion = true;
865 Options.UseBFIInPromotion = IsCS;
866 MPM.addPass(InstrProfiling(Options, IsCS));
867 }
868
addPGOInstrPassesForO0(ModulePassManager & MPM,bool RunProfileGen,bool IsCS,std::string ProfileFile,std::string ProfileRemappingFile)869 void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
870 bool RunProfileGen, bool IsCS,
871 std::string ProfileFile,
872 std::string ProfileRemappingFile) {
873 if (!RunProfileGen) {
874 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
875 MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
876 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
877 // RequireAnalysisPass for PSI before subsequent non-module passes.
878 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
879 return;
880 }
881
882 // Perform PGO instrumentation.
883 MPM.addPass(PGOInstrumentationGen(IsCS));
884 // Add the profile lowering pass.
885 InstrProfOptions Options;
886 if (!ProfileFile.empty())
887 Options.InstrProfileOutput = ProfileFile;
888 // Do not do counter promotion at O0.
889 Options.DoCounterPromotion = false;
890 Options.UseBFIInPromotion = IsCS;
891 MPM.addPass(InstrProfiling(Options, IsCS));
892 }
893
894 static InlineParams
getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level)895 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
896 return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
897 }
898
899 ModuleInlinerWrapperPass
buildInlinerPipeline(OptimizationLevel Level,ThinLTOPhase Phase)900 PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase) {
901 InlineParams IP = getInlineParamsFromOptLevel(Level);
902 if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt &&
903 PGOOpt->Action == PGOOptions::SampleUse)
904 IP.HotCallSiteThreshold = 0;
905
906 if (PGOOpt)
907 IP.EnableDeferral = EnablePGOInlineDeferral;
908
909 ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor,
910 MaxDevirtIterations);
911
912 // Require the GlobalsAA analysis for the module so we can query it within
913 // the CGSCC pipeline.
914 MIWP.addRequiredModuleAnalysis<GlobalsAA>();
915
916 // Require the ProfileSummaryAnalysis for the module so we can query it within
917 // the inliner pass.
918 MIWP.addRequiredModuleAnalysis<ProfileSummaryAnalysis>();
919
920 // Now begin the main postorder CGSCC pipeline.
921 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
922 // manager and trying to emulate its precise behavior. Much of this doesn't
923 // make a lot of sense and we should revisit the core CGSCC structure.
924 CGSCCPassManager &MainCGPipeline = MIWP.getPM();
925
926 // Note: historically, the PruneEH pass was run first to deduce nounwind and
927 // generally clean up exception handling overhead. It isn't clear this is
928 // valuable as the inliner doesn't currently care whether it is inlining an
929 // invoke or a call.
930
931 if (AttributorRun & AttributorRunOption::CGSCC)
932 MainCGPipeline.addPass(AttributorCGSCCPass());
933
934 if (PTO.Coroutines)
935 MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
936
937 // Now deduce any function attributes based in the current code.
938 MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
939
940 // When at O3 add argument promotion to the pass pipeline.
941 // FIXME: It isn't at all clear why this should be limited to O3.
942 if (Level == OptimizationLevel::O3)
943 MainCGPipeline.addPass(ArgumentPromotionPass());
944
945 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
946 // there are no OpenMP runtime calls present in the module.
947 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
948 MainCGPipeline.addPass(OpenMPOptPass());
949
950 // Lastly, add the core function simplification pipeline nested inside the
951 // CGSCC walk.
952 MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
953 buildFunctionSimplificationPipeline(Level, Phase)));
954
955 for (auto &C : CGSCCOptimizerLateEPCallbacks)
956 C(MainCGPipeline, Level);
957
958 return MIWP;
959 }
960
961 ModulePassManager
buildModuleSimplificationPipeline(OptimizationLevel Level,ThinLTOPhase Phase)962 PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
963 ThinLTOPhase Phase) {
964 ModulePassManager MPM(DebugLogging);
965
966 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
967
968 // In ThinLTO mode, when flattened profile is used, all the available
969 // profile information will be annotated in PreLink phase so there is
970 // no need to load the profile again in PostLink.
971 bool LoadSampleProfile =
972 HasSampleProfile &&
973 !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink);
974
975 // During the ThinLTO backend phase we perform early indirect call promotion
976 // here, before globalopt. Otherwise imported available_externally functions
977 // look unreferenced and are removed. If we are going to load the sample
978 // profile then defer until later.
979 // TODO: See if we can move later and consolidate with the location where
980 // we perform ICP when we are loading a sample profile.
981 // TODO: We pass HasSampleProfile (whether there was a sample profile file
982 // passed to the compile) to the SamplePGO flag of ICP. This is used to
983 // determine whether the new direct calls are annotated with prof metadata.
984 // Ideally this should be determined from whether the IR is annotated with
985 // sample profile, and not whether the a sample profile was provided on the
986 // command line. E.g. for flattened profiles where we will not be reloading
987 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
988 // provide the sample profile file.
989 if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile)
990 MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
991
992 // Do basic inference of function attributes from known properties of system
993 // libraries and other oracles.
994 MPM.addPass(InferFunctionAttrsPass());
995
996 // Create an early function pass manager to cleanup the output of the
997 // frontend.
998 FunctionPassManager EarlyFPM(DebugLogging);
999 EarlyFPM.addPass(SimplifyCFGPass());
1000 EarlyFPM.addPass(SROA());
1001 EarlyFPM.addPass(EarlyCSEPass());
1002 EarlyFPM.addPass(LowerExpectIntrinsicPass());
1003 if (PTO.Coroutines)
1004 EarlyFPM.addPass(CoroEarlyPass());
1005 if (Level == OptimizationLevel::O3)
1006 EarlyFPM.addPass(CallSiteSplittingPass());
1007
1008 // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
1009 // to convert bitcast to direct calls so that they can be inlined during the
1010 // profile annotation prepration step.
1011 // More details about SamplePGO design can be found in:
1012 // https://research.google.com/pubs/pub45290.html
1013 // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
1014 if (LoadSampleProfile)
1015 EarlyFPM.addPass(InstCombinePass());
1016 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1017
1018 if (LoadSampleProfile) {
1019 // Annotate sample profile right after early FPM to ensure freshness of
1020 // the debug info.
1021 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1022 PGOOpt->ProfileRemappingFile,
1023 Phase == ThinLTOPhase::PreLink));
1024 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1025 // RequireAnalysisPass for PSI before subsequent non-module passes.
1026 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1027 // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
1028 // for the profile annotation to be accurate in the ThinLTO backend.
1029 if (Phase != ThinLTOPhase::PreLink)
1030 // We perform early indirect call promotion here, before globalopt.
1031 // This is important for the ThinLTO backend phase because otherwise
1032 // imported available_externally functions look unreferenced and are
1033 // removed.
1034 MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
1035 true /* SamplePGO */));
1036 }
1037
1038 if (AttributorRun & AttributorRunOption::MODULE)
1039 MPM.addPass(AttributorPass());
1040
1041 // Lower type metadata and the type.test intrinsic in the ThinLTO
1042 // post link pipeline after ICP. This is to enable usage of the type
1043 // tests in ICP sequences.
1044 if (Phase == ThinLTOPhase::PostLink)
1045 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1046
1047 // Interprocedural constant propagation now that basic cleanup has occurred
1048 // and prior to optimizing globals.
1049 // FIXME: This position in the pipeline hasn't been carefully considered in
1050 // years, it should be re-analyzed.
1051 MPM.addPass(IPSCCPPass());
1052
1053 // Attach metadata to indirect call sites indicating the set of functions
1054 // they may target at run-time. This should follow IPSCCP.
1055 MPM.addPass(CalledValuePropagationPass());
1056
1057 // Optimize globals to try and fold them into constants.
1058 MPM.addPass(GlobalOptPass());
1059
1060 // Promote any localized globals to SSA registers.
1061 // FIXME: Should this instead by a run of SROA?
1062 // FIXME: We should probably run instcombine and simplify-cfg afterward to
1063 // delete control flows that are dead once globals have been folded to
1064 // constants.
1065 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1066
1067 // Remove any dead arguments exposed by cleanups and constant folding
1068 // globals.
1069 MPM.addPass(DeadArgumentEliminationPass());
1070
1071 // Create a small function pass pipeline to cleanup after all the global
1072 // optimizations.
1073 FunctionPassManager GlobalCleanupPM(DebugLogging);
1074 GlobalCleanupPM.addPass(InstCombinePass());
1075 invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
1076
1077 GlobalCleanupPM.addPass(SimplifyCFGPass());
1078 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
1079
1080 // Add all the requested passes for instrumentation PGO, if requested.
1081 if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
1082 (PGOOpt->Action == PGOOptions::IRInstr ||
1083 PGOOpt->Action == PGOOptions::IRUse)) {
1084 addPGOInstrPasses(MPM, Level,
1085 /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
1086 /* IsCS */ false, PGOOpt->ProfileFile,
1087 PGOOpt->ProfileRemappingFile);
1088 MPM.addPass(PGOIndirectCallPromotion(false, false));
1089 }
1090 if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
1091 PGOOpt->CSAction == PGOOptions::CSIRInstr)
1092 MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
1093
1094 // Synthesize function entry counts for non-PGO compilation.
1095 if (EnableSyntheticCounts && !PGOOpt)
1096 MPM.addPass(SyntheticCountsPropagation());
1097
1098 MPM.addPass(buildInlinerPipeline(Level, Phase));
1099
1100 if (EnableMemProfiler && Phase != ThinLTOPhase::PreLink) {
1101 MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
1102 MPM.addPass(ModuleMemProfilerPass());
1103 }
1104
1105 return MPM;
1106 }
1107
1108 ModulePassManager
buildModuleOptimizationPipeline(OptimizationLevel Level,bool LTOPreLink)1109 PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
1110 bool LTOPreLink) {
1111 ModulePassManager MPM(DebugLogging);
1112
1113 // Optimize globals now that the module is fully simplified.
1114 MPM.addPass(GlobalOptPass());
1115 MPM.addPass(GlobalDCEPass());
1116
1117 // Run partial inlining pass to partially inline functions that have
1118 // large bodies.
1119 if (RunPartialInlining)
1120 MPM.addPass(PartialInlinerPass());
1121
1122 // Remove avail extern fns and globals definitions since we aren't compiling
1123 // an object file for later LTO. For LTO we want to preserve these so they
1124 // are eligible for inlining at link-time. Note if they are unreferenced they
1125 // will be removed by GlobalDCE later, so this only impacts referenced
1126 // available externally globals. Eventually they will be suppressed during
1127 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1128 // may make globals referenced by available external functions dead and saves
1129 // running remaining passes on the eliminated functions. These should be
1130 // preserved during prelinking for link-time inlining decisions.
1131 if (!LTOPreLink)
1132 MPM.addPass(EliminateAvailableExternallyPass());
1133
1134 if (EnableOrderFileInstrumentation)
1135 MPM.addPass(InstrOrderFilePass());
1136
1137 // Do RPO function attribute inference across the module to forward-propagate
1138 // attributes where applicable.
1139 // FIXME: Is this really an optimization rather than a canonicalization?
1140 MPM.addPass(ReversePostOrderFunctionAttrsPass());
1141
1142 // Do a post inline PGO instrumentation and use pass. This is a context
1143 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1144 // cross-module inline has not been done yet. The context sensitive
1145 // instrumentation is after all the inlines are done.
1146 if (!LTOPreLink && PGOOpt) {
1147 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1148 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1149 /* IsCS */ true, PGOOpt->CSProfileGenFile,
1150 PGOOpt->ProfileRemappingFile);
1151 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1152 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1153 /* IsCS */ true, PGOOpt->ProfileFile,
1154 PGOOpt->ProfileRemappingFile);
1155 }
1156
1157 // Re-require GloblasAA here prior to function passes. This is particularly
1158 // useful as the above will have inlined, DCE'ed, and function-attr
1159 // propagated everything. We should at this point have a reasonably minimal
1160 // and richly annotated call graph. By computing aliasing and mod/ref
1161 // information for all local globals here, the late loop passes and notably
1162 // the vectorizer will be able to use them to help recognize vectorizable
1163 // memory operations.
1164 MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
1165
1166 FunctionPassManager OptimizePM(DebugLogging);
1167 OptimizePM.addPass(Float2IntPass());
1168 OptimizePM.addPass(LowerConstantIntrinsicsPass());
1169
1170 if (EnableMatrix) {
1171 OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1172 OptimizePM.addPass(EarlyCSEPass());
1173 }
1174
1175 // FIXME: We need to run some loop optimizations to re-rotate loops after
1176 // simplify-cfg and others undo their rotation.
1177
1178 // Optimize the loop execution. These passes operate on entire loop nests
1179 // rather than on each loop in an inside-out manner, and so they are actually
1180 // function passes.
1181
1182 for (auto &C : VectorizerStartEPCallbacks)
1183 C(OptimizePM, Level);
1184
1185 // First rotate loops that may have been un-rotated by prior passes.
1186 // Disable header duplication at -Oz.
1187 OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1188 LoopRotatePass(Level != OptimizationLevel::Oz), EnableMSSALoopDependency,
1189 /*UseBlockFrequencyInfo=*/false, DebugLogging));
1190
1191 // Distribute loops to allow partial vectorization. I.e. isolate dependences
1192 // into separate loop that would otherwise inhibit vectorization. This is
1193 // currently only performed for loops marked with the metadata
1194 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1195 OptimizePM.addPass(LoopDistributePass());
1196
1197 // Populates the VFABI attribute with the scalar-to-vector mappings
1198 // from the TargetLibraryInfo.
1199 OptimizePM.addPass(InjectTLIMappings());
1200
1201 // Now run the core loop vectorizer.
1202 OptimizePM.addPass(LoopVectorizePass(
1203 LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1204
1205 // Eliminate loads by forwarding stores from the previous iteration to loads
1206 // of the current iteration.
1207 OptimizePM.addPass(LoopLoadEliminationPass());
1208
1209 // Cleanup after the loop optimization passes.
1210 OptimizePM.addPass(InstCombinePass());
1211
1212 // Now that we've formed fast to execute loop structures, we do further
1213 // optimizations. These are run afterward as they might block doing complex
1214 // analyses and transforms such as what are needed for loop vectorization.
1215
1216 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1217 // GVN, loop transforms, and others have already run, so it's now better to
1218 // convert to more optimized IR using more aggressive simplify CFG options.
1219 // The extra sinking transform can create larger basic blocks, so do this
1220 // before SLP vectorization.
1221 // FIXME: study whether hoisting and/or sinking of common instructions should
1222 // be delayed until after SLP vectorizer.
1223 OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
1224 .forwardSwitchCondToPhi(true)
1225 .convertSwitchToLookupTable(true)
1226 .needCanonicalLoops(false)
1227 .hoistCommonInsts(true)
1228 .sinkCommonInsts(true)));
1229
1230 // Optimize parallel scalar instruction chains into SIMD instructions.
1231 if (PTO.SLPVectorization)
1232 OptimizePM.addPass(SLPVectorizerPass());
1233
1234 // Enhance/cleanup vector code.
1235 OptimizePM.addPass(VectorCombinePass());
1236 OptimizePM.addPass(InstCombinePass());
1237
1238 // Unroll small loops to hide loop backedge latency and saturate any parallel
1239 // execution resources of an out-of-order processor. We also then need to
1240 // clean up redundancies and loop invariant code.
1241 // FIXME: It would be really good to use a loop-integrated instruction
1242 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1243 // across the loop nests.
1244 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1245 if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1246 OptimizePM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel()));
1247 }
1248 OptimizePM.addPass(LoopUnrollPass(LoopUnrollOptions(
1249 Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1250 PTO.ForgetAllSCEVInLoopUnroll)));
1251 OptimizePM.addPass(WarnMissedTransformationsPass());
1252 OptimizePM.addPass(InstCombinePass());
1253 OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
1254 OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1255 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
1256 EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
1257
1258 // Now that we've vectorized and unrolled loops, we may have more refined
1259 // alignment information, try to re-derive it here.
1260 OptimizePM.addPass(AlignmentFromAssumptionsPass());
1261
1262 // Split out cold code. Splitting is done late to avoid hiding context from
1263 // other optimizations and inadvertently regressing performance. The tradeoff
1264 // is that this has a higher code size cost than splitting early.
1265 if (EnableHotColdSplit && !LTOPreLink)
1266 MPM.addPass(HotColdSplittingPass());
1267
1268 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1269 // canonicalization pass that enables other optimizations. As a result,
1270 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1271 // result too early.
1272 OptimizePM.addPass(LoopSinkPass());
1273
1274 // And finally clean up LCSSA form before generating code.
1275 OptimizePM.addPass(InstSimplifyPass());
1276
1277 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1278 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1279 // flattening of blocks.
1280 OptimizePM.addPass(DivRemPairsPass());
1281
1282 // LoopSink (and other loop passes since the last simplifyCFG) might have
1283 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1284 OptimizePM.addPass(SimplifyCFGPass());
1285
1286 // Optimize PHIs by speculating around them when profitable. Note that this
1287 // pass needs to be run after any PRE or similar pass as it is essentially
1288 // inserting redundancies into the program. This even includes SimplifyCFG.
1289 OptimizePM.addPass(SpeculateAroundPHIsPass());
1290
1291 if (PTO.Coroutines)
1292 OptimizePM.addPass(CoroCleanupPass());
1293
1294 // Add the core optimizing pipeline.
1295 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
1296
1297 for (auto &C : OptimizerLastEPCallbacks)
1298 C(MPM, Level);
1299
1300 if (PTO.CallGraphProfile)
1301 MPM.addPass(CGProfilePass());
1302
1303 // Now we need to do some global optimization transforms.
1304 // FIXME: It would seem like these should come first in the optimization
1305 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1306 // ordering here.
1307 MPM.addPass(GlobalDCEPass());
1308 MPM.addPass(ConstantMergePass());
1309
1310 return MPM;
1311 }
1312
1313 ModulePassManager
buildPerModuleDefaultPipeline(OptimizationLevel Level,bool LTOPreLink)1314 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1315 bool LTOPreLink) {
1316 assert(Level != OptimizationLevel::O0 &&
1317 "Must request optimizations for the default pipeline!");
1318
1319 ModulePassManager MPM(DebugLogging);
1320
1321 // Force any function attributes we want the rest of the pipeline to observe.
1322 MPM.addPass(ForceFunctionAttrsPass());
1323
1324 // Apply module pipeline start EP callback.
1325 for (auto &C : PipelineStartEPCallbacks)
1326 C(MPM, Level);
1327
1328 if (PGOOpt && PGOOpt->SamplePGOSupport)
1329 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1330
1331 // Add the core simplification pipeline.
1332 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None));
1333
1334 // Now add the optimization pipeline.
1335 MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPreLink));
1336
1337 return MPM;
1338 }
1339
1340 ModulePassManager
buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level)1341 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1342 assert(Level != OptimizationLevel::O0 &&
1343 "Must request optimizations for the default pipeline!");
1344
1345 ModulePassManager MPM(DebugLogging);
1346
1347 // Force any function attributes we want the rest of the pipeline to observe.
1348 MPM.addPass(ForceFunctionAttrsPass());
1349
1350 if (PGOOpt && PGOOpt->SamplePGOSupport)
1351 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1352
1353 // Apply module pipeline start EP callback.
1354 for (auto &C : PipelineStartEPCallbacks)
1355 C(MPM, Level);
1356
1357 // If we are planning to perform ThinLTO later, we don't bloat the code with
1358 // unrolling/vectorization/... now. Just simplify the module as much as we
1359 // can.
1360 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink));
1361
1362 // Run partial inlining pass to partially inline functions that have
1363 // large bodies.
1364 // FIXME: It isn't clear whether this is really the right place to run this
1365 // in ThinLTO. Because there is another canonicalization and simplification
1366 // phase that will run after the thin link, running this here ends up with
1367 // less information than will be available later and it may grow functions in
1368 // ways that aren't beneficial.
1369 if (RunPartialInlining)
1370 MPM.addPass(PartialInlinerPass());
1371
1372 // Reduce the size of the IR as much as possible.
1373 MPM.addPass(GlobalOptPass());
1374
1375 // Module simplification splits coroutines, but does not fully clean up
1376 // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up
1377 // on these, we schedule the cleanup here.
1378 if (PTO.Coroutines)
1379 MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
1380
1381 return MPM;
1382 }
1383
buildThinLTODefaultPipeline(OptimizationLevel Level,const ModuleSummaryIndex * ImportSummary)1384 ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1385 OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1386 ModulePassManager MPM(DebugLogging);
1387
1388 if (ImportSummary) {
1389 // These passes import type identifier resolutions for whole-program
1390 // devirtualization and CFI. They must run early because other passes may
1391 // disturb the specific instruction patterns that these passes look for,
1392 // creating dependencies on resolutions that may not appear in the summary.
1393 //
1394 // For example, GVN may transform the pattern assume(type.test) appearing in
1395 // two basic blocks into assume(phi(type.test, type.test)), which would
1396 // transform a dependency on a WPD resolution into a dependency on a type
1397 // identifier resolution for CFI.
1398 //
1399 // Also, WPD has access to more precise information than ICP and can
1400 // devirtualize more effectively, so it should operate on the IR first.
1401 //
1402 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1403 // metadata and intrinsics.
1404 MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1405 MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1406 }
1407
1408 if (Level == OptimizationLevel::O0)
1409 return MPM;
1410
1411 // Force any function attributes we want the rest of the pipeline to observe.
1412 MPM.addPass(ForceFunctionAttrsPass());
1413
1414 // Add the core simplification pipeline.
1415 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink));
1416
1417 // Now add the optimization pipeline.
1418 MPM.addPass(buildModuleOptimizationPipeline(Level));
1419
1420 return MPM;
1421 }
1422
1423 ModulePassManager
buildLTOPreLinkDefaultPipeline(OptimizationLevel Level)1424 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1425 assert(Level != OptimizationLevel::O0 &&
1426 "Must request optimizations for the default pipeline!");
1427 // FIXME: We should use a customized pre-link pipeline!
1428 return buildPerModuleDefaultPipeline(Level,
1429 /* LTOPreLink */ true);
1430 }
1431
1432 ModulePassManager
buildLTODefaultPipeline(OptimizationLevel Level,ModuleSummaryIndex * ExportSummary)1433 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
1434 ModuleSummaryIndex *ExportSummary) {
1435 ModulePassManager MPM(DebugLogging);
1436
1437 if (Level == OptimizationLevel::O0) {
1438 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1439 // metadata and intrinsics.
1440 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1441 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1442 // Run a second time to clean up any type tests left behind by WPD for use
1443 // in ICP.
1444 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1445 return MPM;
1446 }
1447
1448 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1449 // Load sample profile before running the LTO optimization pipeline.
1450 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1451 PGOOpt->ProfileRemappingFile,
1452 false /* ThinLTOPhase::PreLink */));
1453 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1454 // RequireAnalysisPass for PSI before subsequent non-module passes.
1455 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1456 }
1457
1458 // Remove unused virtual tables to improve the quality of code generated by
1459 // whole-program devirtualization and bitset lowering.
1460 MPM.addPass(GlobalDCEPass());
1461
1462 // Force any function attributes we want the rest of the pipeline to observe.
1463 MPM.addPass(ForceFunctionAttrsPass());
1464
1465 // Do basic inference of function attributes from known properties of system
1466 // libraries and other oracles.
1467 MPM.addPass(InferFunctionAttrsPass());
1468
1469 if (Level.getSpeedupLevel() > 1) {
1470 FunctionPassManager EarlyFPM(DebugLogging);
1471 EarlyFPM.addPass(CallSiteSplittingPass());
1472 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1473
1474 // Indirect call promotion. This should promote all the targets that are
1475 // left by the earlier promotion pass that promotes intra-module targets.
1476 // This two-step promotion is to save the compile time. For LTO, it should
1477 // produce the same result as if we only do promotion here.
1478 MPM.addPass(PGOIndirectCallPromotion(
1479 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1480 // Propagate constants at call sites into the functions they call. This
1481 // opens opportunities for globalopt (and inlining) by substituting function
1482 // pointers passed as arguments to direct uses of functions.
1483 MPM.addPass(IPSCCPPass());
1484
1485 // Attach metadata to indirect call sites indicating the set of functions
1486 // they may target at run-time. This should follow IPSCCP.
1487 MPM.addPass(CalledValuePropagationPass());
1488 }
1489
1490 // Now deduce any function attributes based in the current code.
1491 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1492 PostOrderFunctionAttrsPass()));
1493
1494 // Do RPO function attribute inference across the module to forward-propagate
1495 // attributes where applicable.
1496 // FIXME: Is this really an optimization rather than a canonicalization?
1497 MPM.addPass(ReversePostOrderFunctionAttrsPass());
1498
1499 // Use in-range annotations on GEP indices to split globals where beneficial.
1500 MPM.addPass(GlobalSplitPass());
1501
1502 // Run whole program optimization of virtual call when the list of callees
1503 // is fixed.
1504 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1505
1506 // Stop here at -O1.
1507 if (Level == OptimizationLevel::O1) {
1508 // The LowerTypeTestsPass needs to run to lower type metadata and the
1509 // type.test intrinsics. The pass does nothing if CFI is disabled.
1510 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1511 // Run a second time to clean up any type tests left behind by WPD for use
1512 // in ICP (which is performed earlier than this in the regular LTO
1513 // pipeline).
1514 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1515 return MPM;
1516 }
1517
1518 // Optimize globals to try and fold them into constants.
1519 MPM.addPass(GlobalOptPass());
1520
1521 // Promote any localized globals to SSA registers.
1522 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1523
1524 // Linking modules together can lead to duplicate global constant, only
1525 // keep one copy of each constant.
1526 MPM.addPass(ConstantMergePass());
1527
1528 // Remove unused arguments from functions.
1529 MPM.addPass(DeadArgumentEliminationPass());
1530
1531 // Reduce the code after globalopt and ipsccp. Both can open up significant
1532 // simplification opportunities, and both can propagate functions through
1533 // function pointers. When this happens, we often have to resolve varargs
1534 // calls, etc, so let instcombine do this.
1535 FunctionPassManager PeepholeFPM(DebugLogging);
1536 if (Level == OptimizationLevel::O3)
1537 PeepholeFPM.addPass(AggressiveInstCombinePass());
1538 PeepholeFPM.addPass(InstCombinePass());
1539 invokePeepholeEPCallbacks(PeepholeFPM, Level);
1540
1541 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
1542
1543 // Note: historically, the PruneEH pass was run first to deduce nounwind and
1544 // generally clean up exception handling overhead. It isn't clear this is
1545 // valuable as the inliner doesn't currently care whether it is inlining an
1546 // invoke or a call.
1547 // Run the inliner now.
1548 MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level),
1549 DebugLogging));
1550
1551 // Optimize globals again after we ran the inliner.
1552 MPM.addPass(GlobalOptPass());
1553
1554 // Garbage collect dead functions.
1555 // FIXME: Add ArgumentPromotion pass after once it's ported.
1556 MPM.addPass(GlobalDCEPass());
1557
1558 FunctionPassManager FPM(DebugLogging);
1559 // The IPO Passes may leave cruft around. Clean up after them.
1560 FPM.addPass(InstCombinePass());
1561 invokePeepholeEPCallbacks(FPM, Level);
1562
1563 FPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1564
1565 // Do a post inline PGO instrumentation and use pass. This is a context
1566 // sensitive PGO pass.
1567 if (PGOOpt) {
1568 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1569 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1570 /* IsCS */ true, PGOOpt->CSProfileGenFile,
1571 PGOOpt->ProfileRemappingFile);
1572 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1573 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1574 /* IsCS */ true, PGOOpt->ProfileFile,
1575 PGOOpt->ProfileRemappingFile);
1576 }
1577
1578 // Break up allocas
1579 FPM.addPass(SROA());
1580
1581 // LTO provides additional opportunities for tailcall elimination due to
1582 // link-time inlining, and visibility of nocapture attribute.
1583 FPM.addPass(TailCallElimPass());
1584
1585 // Run a few AA driver optimizations here and now to cleanup the code.
1586 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1587
1588 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1589 PostOrderFunctionAttrsPass()));
1590 // FIXME: here we run IP alias analysis in the legacy PM.
1591
1592 FunctionPassManager MainFPM;
1593
1594 // FIXME: once we fix LoopPass Manager, add LICM here.
1595 // FIXME: once we provide support for enabling MLSM, add it here.
1596 if (RunNewGVN)
1597 MainFPM.addPass(NewGVNPass());
1598 else
1599 MainFPM.addPass(GVN());
1600
1601 // Remove dead memcpy()'s.
1602 MainFPM.addPass(MemCpyOptPass());
1603
1604 // Nuke dead stores.
1605 MainFPM.addPass(DSEPass());
1606
1607 // FIXME: at this point, we run a bunch of loop passes:
1608 // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
1609 // loopVectorize. Enable them once the remaining issue with LPM
1610 // are sorted out.
1611
1612 MainFPM.addPass(InstCombinePass());
1613 MainFPM.addPass(SimplifyCFGPass());
1614 MainFPM.addPass(SCCPPass());
1615 MainFPM.addPass(InstCombinePass());
1616 MainFPM.addPass(BDCEPass());
1617
1618 // FIXME: We may want to run SLPVectorizer here.
1619 // After vectorization, assume intrinsics may tell us more
1620 // about pointer alignments.
1621 #if 0
1622 MainFPM.add(AlignmentFromAssumptionsPass());
1623 #endif
1624
1625 // FIXME: Conditionally run LoadCombine here, after it's ported
1626 // (in case we still have this pass, given its questionable usefulness).
1627
1628 MainFPM.addPass(InstCombinePass());
1629 invokePeepholeEPCallbacks(MainFPM, Level);
1630 MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1631 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1632
1633 // Create a function that performs CFI checks for cross-DSO calls with
1634 // targets in the current module.
1635 MPM.addPass(CrossDSOCFIPass());
1636
1637 // Lower type metadata and the type.test intrinsic. This pass supports
1638 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1639 // to be run at link time if CFI is enabled. This pass does nothing if
1640 // CFI is disabled.
1641 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1642 // Run a second time to clean up any type tests left behind by WPD for use
1643 // in ICP (which is performed earlier than this in the regular LTO pipeline).
1644 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1645
1646 // Enable splitting late in the FullLTO post-link pipeline. This is done in
1647 // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1648 if (EnableHotColdSplit)
1649 MPM.addPass(HotColdSplittingPass());
1650
1651 // Add late LTO optimization passes.
1652 // Delete basic blocks, which optimization passes may have killed.
1653 MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1654
1655 // Drop bodies of available eternally objects to improve GlobalDCE.
1656 MPM.addPass(EliminateAvailableExternallyPass());
1657
1658 // Now that we have optimized the program, discard unreachable functions.
1659 MPM.addPass(GlobalDCEPass());
1660
1661 // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
1662 return MPM;
1663 }
1664
buildDefaultAAPipeline()1665 AAManager PassBuilder::buildDefaultAAPipeline() {
1666 AAManager AA;
1667
1668 // The order in which these are registered determines their priority when
1669 // being queried.
1670
1671 // First we register the basic alias analysis that provides the majority of
1672 // per-function local AA logic. This is a stateless, on-demand local set of
1673 // AA techniques.
1674 AA.registerFunctionAnalysis<BasicAA>();
1675
1676 // Next we query fast, specialized alias analyses that wrap IR-embedded
1677 // information about aliasing.
1678 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
1679 AA.registerFunctionAnalysis<TypeBasedAA>();
1680
1681 // Add support for querying global aliasing information when available.
1682 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1683 // analysis, all that the `AAManager` can do is query for any *cached*
1684 // results from `GlobalsAA` through a readonly proxy.
1685 AA.registerModuleAnalysis<GlobalsAA>();
1686
1687 return AA;
1688 }
1689
parseRepeatPassName(StringRef Name)1690 static Optional<int> parseRepeatPassName(StringRef Name) {
1691 if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1692 return None;
1693 int Count;
1694 if (Name.getAsInteger(0, Count) || Count <= 0)
1695 return None;
1696 return Count;
1697 }
1698
parseDevirtPassName(StringRef Name)1699 static Optional<int> parseDevirtPassName(StringRef Name) {
1700 if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1701 return None;
1702 int Count;
1703 if (Name.getAsInteger(0, Count) || Count <= 0)
1704 return None;
1705 return Count;
1706 }
1707
checkParametrizedPassName(StringRef Name,StringRef PassName)1708 static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
1709 if (!Name.consume_front(PassName))
1710 return false;
1711 // normal pass name w/o parameters == default parameters
1712 if (Name.empty())
1713 return true;
1714 return Name.startswith("<") && Name.endswith(">");
1715 }
1716
1717 namespace {
1718
1719 /// This performs customized parsing of pass name with parameters.
1720 ///
1721 /// We do not need parametrization of passes in textual pipeline very often,
1722 /// yet on a rare occasion ability to specify parameters right there can be
1723 /// useful.
1724 ///
1725 /// \p Name - parameterized specification of a pass from a textual pipeline
1726 /// is a string in a form of :
1727 /// PassName '<' parameter-list '>'
1728 ///
1729 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1730 /// It takes a string-ref of parameters and returns either StringError or a
1731 /// parameter list in a form of a custom parameters type, all wrapped into
1732 /// Expected<> template class.
1733 ///
1734 template <typename ParametersParseCallableT>
parsePassParameters(ParametersParseCallableT && Parser,StringRef Name,StringRef PassName)1735 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
1736 StringRef PassName) -> decltype(Parser(StringRef{})) {
1737 using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
1738
1739 StringRef Params = Name;
1740 if (!Params.consume_front(PassName)) {
1741 assert(false &&
1742 "unable to strip pass name from parametrized pass specification");
1743 }
1744 if (Params.empty())
1745 return ParametersT{};
1746 if (!Params.consume_front("<") || !Params.consume_back(">")) {
1747 assert(false && "invalid format for parametrized pass name");
1748 }
1749
1750 Expected<ParametersT> Result = Parser(Params);
1751 assert((Result || Result.template errorIsA<StringError>()) &&
1752 "Pass parameter parser can only return StringErrors.");
1753 return Result;
1754 }
1755
1756 /// Parser of parameters for LoopUnroll pass.
parseLoopUnrollOptions(StringRef Params)1757 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
1758 LoopUnrollOptions UnrollOpts;
1759 while (!Params.empty()) {
1760 StringRef ParamName;
1761 std::tie(ParamName, Params) = Params.split(';');
1762 int OptLevel = StringSwitch<int>(ParamName)
1763 .Case("O0", 0)
1764 .Case("O1", 1)
1765 .Case("O2", 2)
1766 .Case("O3", 3)
1767 .Default(-1);
1768 if (OptLevel >= 0) {
1769 UnrollOpts.setOptLevel(OptLevel);
1770 continue;
1771 }
1772 if (ParamName.consume_front("full-unroll-max=")) {
1773 int Count;
1774 if (ParamName.getAsInteger(0, Count))
1775 return make_error<StringError>(
1776 formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1777 inconvertibleErrorCode());
1778 UnrollOpts.setFullUnrollMaxCount(Count);
1779 continue;
1780 }
1781
1782 bool Enable = !ParamName.consume_front("no-");
1783 if (ParamName == "partial") {
1784 UnrollOpts.setPartial(Enable);
1785 } else if (ParamName == "peeling") {
1786 UnrollOpts.setPeeling(Enable);
1787 } else if (ParamName == "profile-peeling") {
1788 UnrollOpts.setProfileBasedPeeling(Enable);
1789 } else if (ParamName == "runtime") {
1790 UnrollOpts.setRuntime(Enable);
1791 } else if (ParamName == "upperbound") {
1792 UnrollOpts.setUpperBound(Enable);
1793 } else {
1794 return make_error<StringError>(
1795 formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1796 inconvertibleErrorCode());
1797 }
1798 }
1799 return UnrollOpts;
1800 }
1801
parseMSanPassOptions(StringRef Params)1802 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
1803 MemorySanitizerOptions Result;
1804 while (!Params.empty()) {
1805 StringRef ParamName;
1806 std::tie(ParamName, Params) = Params.split(';');
1807
1808 if (ParamName == "recover") {
1809 Result.Recover = true;
1810 } else if (ParamName == "kernel") {
1811 Result.Kernel = true;
1812 } else if (ParamName.consume_front("track-origins=")) {
1813 if (ParamName.getAsInteger(0, Result.TrackOrigins))
1814 return make_error<StringError>(
1815 formatv("invalid argument to MemorySanitizer pass track-origins "
1816 "parameter: '{0}' ",
1817 ParamName)
1818 .str(),
1819 inconvertibleErrorCode());
1820 } else {
1821 return make_error<StringError>(
1822 formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
1823 .str(),
1824 inconvertibleErrorCode());
1825 }
1826 }
1827 return Result;
1828 }
1829
1830 /// Parser of parameters for SimplifyCFG pass.
parseSimplifyCFGOptions(StringRef Params)1831 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
1832 SimplifyCFGOptions Result;
1833 while (!Params.empty()) {
1834 StringRef ParamName;
1835 std::tie(ParamName, Params) = Params.split(';');
1836
1837 bool Enable = !ParamName.consume_front("no-");
1838 if (ParamName == "forward-switch-cond") {
1839 Result.forwardSwitchCondToPhi(Enable);
1840 } else if (ParamName == "switch-to-lookup") {
1841 Result.convertSwitchToLookupTable(Enable);
1842 } else if (ParamName == "keep-loops") {
1843 Result.needCanonicalLoops(Enable);
1844 } else if (ParamName == "hoist-common-insts") {
1845 Result.hoistCommonInsts(Enable);
1846 } else if (ParamName == "sink-common-insts") {
1847 Result.sinkCommonInsts(Enable);
1848 } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
1849 APInt BonusInstThreshold;
1850 if (ParamName.getAsInteger(0, BonusInstThreshold))
1851 return make_error<StringError>(
1852 formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1853 "parameter: '{0}' ",
1854 ParamName).str(),
1855 inconvertibleErrorCode());
1856 Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
1857 } else {
1858 return make_error<StringError>(
1859 formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
1860 inconvertibleErrorCode());
1861 }
1862 }
1863 return Result;
1864 }
1865
1866 /// Parser of parameters for LoopVectorize pass.
parseLoopVectorizeOptions(StringRef Params)1867 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
1868 LoopVectorizeOptions Opts;
1869 while (!Params.empty()) {
1870 StringRef ParamName;
1871 std::tie(ParamName, Params) = Params.split(';');
1872
1873 bool Enable = !ParamName.consume_front("no-");
1874 if (ParamName == "interleave-forced-only") {
1875 Opts.setInterleaveOnlyWhenForced(Enable);
1876 } else if (ParamName == "vectorize-forced-only") {
1877 Opts.setVectorizeOnlyWhenForced(Enable);
1878 } else {
1879 return make_error<StringError>(
1880 formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
1881 inconvertibleErrorCode());
1882 }
1883 }
1884 return Opts;
1885 }
1886
parseLoopUnswitchOptions(StringRef Params)1887 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
1888 bool Result = false;
1889 while (!Params.empty()) {
1890 StringRef ParamName;
1891 std::tie(ParamName, Params) = Params.split(';');
1892
1893 bool Enable = !ParamName.consume_front("no-");
1894 if (ParamName == "nontrivial") {
1895 Result = Enable;
1896 } else {
1897 return make_error<StringError>(
1898 formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
1899 .str(),
1900 inconvertibleErrorCode());
1901 }
1902 }
1903 return Result;
1904 }
1905
parseMergedLoadStoreMotionOptions(StringRef Params)1906 Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
1907 bool Result = false;
1908 while (!Params.empty()) {
1909 StringRef ParamName;
1910 std::tie(ParamName, Params) = Params.split(';');
1911
1912 bool Enable = !ParamName.consume_front("no-");
1913 if (ParamName == "split-footer-bb") {
1914 Result = Enable;
1915 } else {
1916 return make_error<StringError>(
1917 formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
1918 ParamName)
1919 .str(),
1920 inconvertibleErrorCode());
1921 }
1922 }
1923 return Result;
1924 }
1925
parseGVNOptions(StringRef Params)1926 Expected<GVNOptions> parseGVNOptions(StringRef Params) {
1927 GVNOptions Result;
1928 while (!Params.empty()) {
1929 StringRef ParamName;
1930 std::tie(ParamName, Params) = Params.split(';');
1931
1932 bool Enable = !ParamName.consume_front("no-");
1933 if (ParamName == "pre") {
1934 Result.setPRE(Enable);
1935 } else if (ParamName == "load-pre") {
1936 Result.setLoadPRE(Enable);
1937 } else if (ParamName == "split-backedge-load-pre") {
1938 Result.setLoadPRESplitBackedge(Enable);
1939 } else if (ParamName == "memdep") {
1940 Result.setMemDep(Enable);
1941 } else {
1942 return make_error<StringError>(
1943 formatv("invalid GVN pass parameter '{0}' ", ParamName).str(),
1944 inconvertibleErrorCode());
1945 }
1946 }
1947 return Result;
1948 }
1949
1950 Expected<StackLifetime::LivenessType>
parseStackLifetimeOptions(StringRef Params)1951 parseStackLifetimeOptions(StringRef Params) {
1952 StackLifetime::LivenessType Result = StackLifetime::LivenessType::May;
1953 while (!Params.empty()) {
1954 StringRef ParamName;
1955 std::tie(ParamName, Params) = Params.split(';');
1956
1957 if (ParamName == "may") {
1958 Result = StackLifetime::LivenessType::May;
1959 } else if (ParamName == "must") {
1960 Result = StackLifetime::LivenessType::Must;
1961 } else {
1962 return make_error<StringError>(
1963 formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(),
1964 inconvertibleErrorCode());
1965 }
1966 }
1967 return Result;
1968 }
1969
1970 } // namespace
1971
1972 /// Tests whether a pass name starts with a valid prefix for a default pipeline
1973 /// alias.
startsWithDefaultPipelineAliasPrefix(StringRef Name)1974 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
1975 return Name.startswith("default") || Name.startswith("thinlto") ||
1976 Name.startswith("lto");
1977 }
1978
1979 /// Tests whether registered callbacks will accept a given pass name.
1980 ///
1981 /// When parsing a pipeline text, the type of the outermost pipeline may be
1982 /// omitted, in which case the type is automatically determined from the first
1983 /// pass name in the text. This may be a name that is handled through one of the
1984 /// callbacks. We check this through the oridinary parsing callbacks by setting
1985 /// up a dummy PassManager in order to not force the client to also handle this
1986 /// type of query.
1987 template <typename PassManagerT, typename CallbacksT>
callbacksAcceptPassName(StringRef Name,CallbacksT & Callbacks)1988 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
1989 if (!Callbacks.empty()) {
1990 PassManagerT DummyPM;
1991 for (auto &CB : Callbacks)
1992 if (CB(Name, DummyPM, {}))
1993 return true;
1994 }
1995 return false;
1996 }
1997
1998 template <typename CallbacksT>
isModulePassName(StringRef Name,CallbacksT & Callbacks)1999 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
2000 // Manually handle aliases for pre-configured pipeline fragments.
2001 if (startsWithDefaultPipelineAliasPrefix(Name))
2002 return DefaultAliasRegex.match(Name);
2003
2004 // Explicitly handle pass manager names.
2005 if (Name == "module")
2006 return true;
2007 if (Name == "cgscc")
2008 return true;
2009 if (Name == "function")
2010 return true;
2011
2012 // Explicitly handle custom-parsed pass names.
2013 if (parseRepeatPassName(Name))
2014 return true;
2015
2016 #define MODULE_PASS(NAME, CREATE_PASS) \
2017 if (Name == NAME) \
2018 return true;
2019 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2020 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2021 return true;
2022 #include "PassRegistry.def"
2023
2024 return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
2025 }
2026
2027 template <typename CallbacksT>
isCGSCCPassName(StringRef Name,CallbacksT & Callbacks)2028 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
2029 // Explicitly handle pass manager names.
2030 if (Name == "cgscc")
2031 return true;
2032 if (Name == "function")
2033 return true;
2034
2035 // Explicitly handle custom-parsed pass names.
2036 if (parseRepeatPassName(Name))
2037 return true;
2038 if (parseDevirtPassName(Name))
2039 return true;
2040
2041 #define CGSCC_PASS(NAME, CREATE_PASS) \
2042 if (Name == NAME) \
2043 return true;
2044 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2045 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2046 return true;
2047 #include "PassRegistry.def"
2048
2049 return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
2050 }
2051
2052 template <typename CallbacksT>
isFunctionPassName(StringRef Name,CallbacksT & Callbacks)2053 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
2054 // Explicitly handle pass manager names.
2055 if (Name == "function")
2056 return true;
2057 if (Name == "loop" || Name == "loop-mssa")
2058 return true;
2059
2060 // Explicitly handle custom-parsed pass names.
2061 if (parseRepeatPassName(Name))
2062 return true;
2063
2064 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2065 if (Name == NAME) \
2066 return true;
2067 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2068 if (checkParametrizedPassName(Name, NAME)) \
2069 return true;
2070 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2071 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2072 return true;
2073 #include "PassRegistry.def"
2074
2075 return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
2076 }
2077
2078 template <typename CallbacksT>
isLoopPassName(StringRef Name,CallbacksT & Callbacks)2079 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
2080 // Explicitly handle pass manager names.
2081 if (Name == "loop" || Name == "loop-mssa")
2082 return true;
2083
2084 // Explicitly handle custom-parsed pass names.
2085 if (parseRepeatPassName(Name))
2086 return true;
2087
2088 #define LOOP_PASS(NAME, CREATE_PASS) \
2089 if (Name == NAME) \
2090 return true;
2091 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2092 if (checkParametrizedPassName(Name, NAME)) \
2093 return true;
2094 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2095 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2096 return true;
2097 #include "PassRegistry.def"
2098
2099 return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
2100 }
2101
2102 Optional<std::vector<PassBuilder::PipelineElement>>
parsePipelineText(StringRef Text)2103 PassBuilder::parsePipelineText(StringRef Text) {
2104 std::vector<PipelineElement> ResultPipeline;
2105
2106 SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
2107 &ResultPipeline};
2108 for (;;) {
2109 std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
2110 size_t Pos = Text.find_first_of(",()");
2111 Pipeline.push_back({Text.substr(0, Pos), {}});
2112
2113 // If we have a single terminating name, we're done.
2114 if (Pos == Text.npos)
2115 break;
2116
2117 char Sep = Text[Pos];
2118 Text = Text.substr(Pos + 1);
2119 if (Sep == ',')
2120 // Just a name ending in a comma, continue.
2121 continue;
2122
2123 if (Sep == '(') {
2124 // Push the inner pipeline onto the stack to continue processing.
2125 PipelineStack.push_back(&Pipeline.back().InnerPipeline);
2126 continue;
2127 }
2128
2129 assert(Sep == ')' && "Bogus separator!");
2130 // When handling the close parenthesis, we greedily consume them to avoid
2131 // empty strings in the pipeline.
2132 do {
2133 // If we try to pop the outer pipeline we have unbalanced parentheses.
2134 if (PipelineStack.size() == 1)
2135 return None;
2136
2137 PipelineStack.pop_back();
2138 } while (Text.consume_front(")"));
2139
2140 // Check if we've finished parsing.
2141 if (Text.empty())
2142 break;
2143
2144 // Otherwise, the end of an inner pipeline always has to be followed by
2145 // a comma, and then we can continue.
2146 if (!Text.consume_front(","))
2147 return None;
2148 }
2149
2150 if (PipelineStack.size() > 1)
2151 // Unbalanced paretheses.
2152 return None;
2153
2154 assert(PipelineStack.back() == &ResultPipeline &&
2155 "Wrong pipeline at the bottom of the stack!");
2156 return {std::move(ResultPipeline)};
2157 }
2158
parseModulePass(ModulePassManager & MPM,const PipelineElement & E)2159 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
2160 const PipelineElement &E) {
2161 auto &Name = E.Name;
2162 auto &InnerPipeline = E.InnerPipeline;
2163
2164 // First handle complex passes like the pass managers which carry pipelines.
2165 if (!InnerPipeline.empty()) {
2166 if (Name == "module") {
2167 ModulePassManager NestedMPM(DebugLogging);
2168 if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
2169 return Err;
2170 MPM.addPass(std::move(NestedMPM));
2171 return Error::success();
2172 }
2173 if (Name == "cgscc") {
2174 CGSCCPassManager CGPM(DebugLogging);
2175 if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline))
2176 return Err;
2177 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2178 return Error::success();
2179 }
2180 if (Name == "function") {
2181 FunctionPassManager FPM(DebugLogging);
2182 if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
2183 return Err;
2184 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
2185 return Error::success();
2186 }
2187 if (auto Count = parseRepeatPassName(Name)) {
2188 ModulePassManager NestedMPM(DebugLogging);
2189 if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
2190 return Err;
2191 MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
2192 return Error::success();
2193 }
2194
2195 for (auto &C : ModulePipelineParsingCallbacks)
2196 if (C(Name, MPM, InnerPipeline))
2197 return Error::success();
2198
2199 // Normal passes can't have pipelines.
2200 return make_error<StringError>(
2201 formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
2202 inconvertibleErrorCode());
2203 ;
2204 }
2205
2206 // Manually handle aliases for pre-configured pipeline fragments.
2207 if (startsWithDefaultPipelineAliasPrefix(Name)) {
2208 SmallVector<StringRef, 3> Matches;
2209 if (!DefaultAliasRegex.match(Name, &Matches))
2210 return make_error<StringError>(
2211 formatv("unknown default pipeline alias '{0}'", Name).str(),
2212 inconvertibleErrorCode());
2213
2214 assert(Matches.size() == 3 && "Must capture two matched strings!");
2215
2216 OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
2217 .Case("O0", OptimizationLevel::O0)
2218 .Case("O1", OptimizationLevel::O1)
2219 .Case("O2", OptimizationLevel::O2)
2220 .Case("O3", OptimizationLevel::O3)
2221 .Case("Os", OptimizationLevel::Os)
2222 .Case("Oz", OptimizationLevel::Oz);
2223 if (L == OptimizationLevel::O0) {
2224 // Add instrumentation PGO passes -- at O0 we can still do PGO.
2225 if (PGOOpt && Matches[1] != "thinlto" &&
2226 (PGOOpt->Action == PGOOptions::IRInstr ||
2227 PGOOpt->Action == PGOOptions::IRUse))
2228 addPGOInstrPassesForO0(
2229 MPM,
2230 /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
2231 /* IsCS */ false, PGOOpt->ProfileFile,
2232 PGOOpt->ProfileRemappingFile);
2233
2234 // For IR that makes use of coroutines intrinsics, coroutine passes must
2235 // be run, even at -O0.
2236 if (PTO.Coroutines) {
2237 MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass()));
2238
2239 CGSCCPassManager CGPM(DebugLogging);
2240 CGPM.addPass(CoroSplitPass());
2241 CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass()));
2242 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2243
2244 MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
2245 }
2246
2247 // Do nothing else at all!
2248 return Error::success();
2249 }
2250
2251 // This is consistent with old pass manager invoked via opt, but
2252 // inconsistent with clang. Clang doesn't enable loop vectorization
2253 // but does enable slp vectorization at Oz.
2254 PTO.LoopVectorization =
2255 L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2256 PTO.SLPVectorization =
2257 L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2258
2259 if (Matches[1] == "default") {
2260 MPM.addPass(buildPerModuleDefaultPipeline(L));
2261 } else if (Matches[1] == "thinlto-pre-link") {
2262 MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L));
2263 } else if (Matches[1] == "thinlto") {
2264 MPM.addPass(buildThinLTODefaultPipeline(L, nullptr));
2265 } else if (Matches[1] == "lto-pre-link") {
2266 MPM.addPass(buildLTOPreLinkDefaultPipeline(L));
2267 } else {
2268 assert(Matches[1] == "lto" && "Not one of the matched options!");
2269 MPM.addPass(buildLTODefaultPipeline(L, nullptr));
2270 }
2271 return Error::success();
2272 }
2273
2274 // Finally expand the basic registered passes from the .inc file.
2275 #define MODULE_PASS(NAME, CREATE_PASS) \
2276 if (Name == NAME) { \
2277 MPM.addPass(CREATE_PASS); \
2278 return Error::success(); \
2279 }
2280 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2281 if (Name == "require<" NAME ">") { \
2282 MPM.addPass( \
2283 RequireAnalysisPass< \
2284 std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \
2285 return Error::success(); \
2286 } \
2287 if (Name == "invalidate<" NAME ">") { \
2288 MPM.addPass(InvalidateAnalysisPass< \
2289 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2290 return Error::success(); \
2291 }
2292 #define CGSCC_PASS(NAME, CREATE_PASS) \
2293 if (Name == NAME) { \
2294 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS)); \
2295 return Error::success(); \
2296 }
2297 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2298 if (Name == NAME) { \
2299 MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS)); \
2300 return Error::success(); \
2301 }
2302 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2303 if (checkParametrizedPassName(Name, NAME)) { \
2304 auto Params = parsePassParameters(PARSER, Name, NAME); \
2305 if (!Params) \
2306 return Params.takeError(); \
2307 MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2308 return Error::success(); \
2309 }
2310 #define LOOP_PASS(NAME, CREATE_PASS) \
2311 if (Name == NAME) { \
2312 MPM.addPass( \
2313 createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2314 CREATE_PASS, false, false, DebugLogging))); \
2315 return Error::success(); \
2316 }
2317 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2318 if (checkParametrizedPassName(Name, NAME)) { \
2319 auto Params = parsePassParameters(PARSER, Name, NAME); \
2320 if (!Params) \
2321 return Params.takeError(); \
2322 MPM.addPass( \
2323 createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2324 CREATE_PASS(Params.get()), false, false, DebugLogging))); \
2325 return Error::success(); \
2326 }
2327 #include "PassRegistry.def"
2328
2329 for (auto &C : ModulePipelineParsingCallbacks)
2330 if (C(Name, MPM, InnerPipeline))
2331 return Error::success();
2332 return make_error<StringError>(
2333 formatv("unknown module pass '{0}'", Name).str(),
2334 inconvertibleErrorCode());
2335 }
2336
parseCGSCCPass(CGSCCPassManager & CGPM,const PipelineElement & E)2337 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
2338 const PipelineElement &E) {
2339 auto &Name = E.Name;
2340 auto &InnerPipeline = E.InnerPipeline;
2341
2342 // First handle complex passes like the pass managers which carry pipelines.
2343 if (!InnerPipeline.empty()) {
2344 if (Name == "cgscc") {
2345 CGSCCPassManager NestedCGPM(DebugLogging);
2346 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2347 return Err;
2348 // Add the nested pass manager with the appropriate adaptor.
2349 CGPM.addPass(std::move(NestedCGPM));
2350 return Error::success();
2351 }
2352 if (Name == "function") {
2353 FunctionPassManager FPM(DebugLogging);
2354 if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
2355 return Err;
2356 // Add the nested pass manager with the appropriate adaptor.
2357 CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
2358 return Error::success();
2359 }
2360 if (auto Count = parseRepeatPassName(Name)) {
2361 CGSCCPassManager NestedCGPM(DebugLogging);
2362 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2363 return Err;
2364 CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
2365 return Error::success();
2366 }
2367 if (auto MaxRepetitions = parseDevirtPassName(Name)) {
2368 CGSCCPassManager NestedCGPM(DebugLogging);
2369 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2370 return Err;
2371 CGPM.addPass(
2372 createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
2373 return Error::success();
2374 }
2375
2376 for (auto &C : CGSCCPipelineParsingCallbacks)
2377 if (C(Name, CGPM, InnerPipeline))
2378 return Error::success();
2379
2380 // Normal passes can't have pipelines.
2381 return make_error<StringError>(
2382 formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
2383 inconvertibleErrorCode());
2384 }
2385
2386 // Now expand the basic registered passes from the .inc file.
2387 #define CGSCC_PASS(NAME, CREATE_PASS) \
2388 if (Name == NAME) { \
2389 CGPM.addPass(CREATE_PASS); \
2390 return Error::success(); \
2391 }
2392 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2393 if (Name == "require<" NAME ">") { \
2394 CGPM.addPass(RequireAnalysisPass< \
2395 std::remove_reference<decltype(CREATE_PASS)>::type, \
2396 LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
2397 CGSCCUpdateResult &>()); \
2398 return Error::success(); \
2399 } \
2400 if (Name == "invalidate<" NAME ">") { \
2401 CGPM.addPass(InvalidateAnalysisPass< \
2402 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2403 return Error::success(); \
2404 }
2405 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2406 if (Name == NAME) { \
2407 CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS)); \
2408 return Error::success(); \
2409 }
2410 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2411 if (checkParametrizedPassName(Name, NAME)) { \
2412 auto Params = parsePassParameters(PARSER, Name, NAME); \
2413 if (!Params) \
2414 return Params.takeError(); \
2415 CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2416 return Error::success(); \
2417 }
2418 #define LOOP_PASS(NAME, CREATE_PASS) \
2419 if (Name == NAME) { \
2420 CGPM.addPass( \
2421 createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2422 CREATE_PASS, false, false, DebugLogging))); \
2423 return Error::success(); \
2424 }
2425 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2426 if (checkParametrizedPassName(Name, NAME)) { \
2427 auto Params = parsePassParameters(PARSER, Name, NAME); \
2428 if (!Params) \
2429 return Params.takeError(); \
2430 CGPM.addPass( \
2431 createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2432 CREATE_PASS(Params.get()), false, false, DebugLogging))); \
2433 return Error::success(); \
2434 }
2435 #include "PassRegistry.def"
2436
2437 for (auto &C : CGSCCPipelineParsingCallbacks)
2438 if (C(Name, CGPM, InnerPipeline))
2439 return Error::success();
2440 return make_error<StringError>(
2441 formatv("unknown cgscc pass '{0}'", Name).str(),
2442 inconvertibleErrorCode());
2443 }
2444
parseFunctionPass(FunctionPassManager & FPM,const PipelineElement & E)2445 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
2446 const PipelineElement &E) {
2447 auto &Name = E.Name;
2448 auto &InnerPipeline = E.InnerPipeline;
2449
2450 // First handle complex passes like the pass managers which carry pipelines.
2451 if (!InnerPipeline.empty()) {
2452 if (Name == "function") {
2453 FunctionPassManager NestedFPM(DebugLogging);
2454 if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
2455 return Err;
2456 // Add the nested pass manager with the appropriate adaptor.
2457 FPM.addPass(std::move(NestedFPM));
2458 return Error::success();
2459 }
2460 if (Name == "loop" || Name == "loop-mssa") {
2461 LoopPassManager LPM(DebugLogging);
2462 if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline))
2463 return Err;
2464 // Add the nested pass manager with the appropriate adaptor.
2465 bool UseMemorySSA = (Name == "loop-mssa");
2466 bool UseBFI =
2467 std::any_of(InnerPipeline.begin(), InnerPipeline.end(),
2468 [](auto Pipeline) { return Pipeline.Name == "licm"; });
2469 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
2470 UseBFI, DebugLogging));
2471 return Error::success();
2472 }
2473 if (auto Count = parseRepeatPassName(Name)) {
2474 FunctionPassManager NestedFPM(DebugLogging);
2475 if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
2476 return Err;
2477 FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
2478 return Error::success();
2479 }
2480
2481 for (auto &C : FunctionPipelineParsingCallbacks)
2482 if (C(Name, FPM, InnerPipeline))
2483 return Error::success();
2484
2485 // Normal passes can't have pipelines.
2486 return make_error<StringError>(
2487 formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
2488 inconvertibleErrorCode());
2489 }
2490
2491 // Now expand the basic registered passes from the .inc file.
2492 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2493 if (Name == NAME) { \
2494 FPM.addPass(CREATE_PASS); \
2495 return Error::success(); \
2496 }
2497 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2498 if (checkParametrizedPassName(Name, NAME)) { \
2499 auto Params = parsePassParameters(PARSER, Name, NAME); \
2500 if (!Params) \
2501 return Params.takeError(); \
2502 FPM.addPass(CREATE_PASS(Params.get())); \
2503 return Error::success(); \
2504 }
2505 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2506 if (Name == "require<" NAME ">") { \
2507 FPM.addPass( \
2508 RequireAnalysisPass< \
2509 std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \
2510 return Error::success(); \
2511 } \
2512 if (Name == "invalidate<" NAME ">") { \
2513 FPM.addPass(InvalidateAnalysisPass< \
2514 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2515 return Error::success(); \
2516 }
2517 // FIXME: UseMemorySSA is set to false. Maybe we could do things like:
2518 // bool UseMemorySSA = !("canon-freeze" || "loop-predication" ||
2519 // "guard-widening");
2520 // The risk is that it may become obsolete if we're not careful.
2521 #define LOOP_PASS(NAME, CREATE_PASS) \
2522 if (Name == NAME) { \
2523 FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false, \
2524 DebugLogging)); \
2525 return Error::success(); \
2526 }
2527 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2528 if (checkParametrizedPassName(Name, NAME)) { \
2529 auto Params = parsePassParameters(PARSER, Name, NAME); \
2530 if (!Params) \
2531 return Params.takeError(); \
2532 FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \
2533 false, false, DebugLogging)); \
2534 return Error::success(); \
2535 }
2536 #include "PassRegistry.def"
2537
2538 for (auto &C : FunctionPipelineParsingCallbacks)
2539 if (C(Name, FPM, InnerPipeline))
2540 return Error::success();
2541 return make_error<StringError>(
2542 formatv("unknown function pass '{0}'", Name).str(),
2543 inconvertibleErrorCode());
2544 }
2545
parseLoopPass(LoopPassManager & LPM,const PipelineElement & E)2546 Error PassBuilder::parseLoopPass(LoopPassManager &LPM,
2547 const PipelineElement &E) {
2548 StringRef Name = E.Name;
2549 auto &InnerPipeline = E.InnerPipeline;
2550
2551 // First handle complex passes like the pass managers which carry pipelines.
2552 if (!InnerPipeline.empty()) {
2553 if (Name == "loop") {
2554 LoopPassManager NestedLPM(DebugLogging);
2555 if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
2556 return Err;
2557 // Add the nested pass manager with the appropriate adaptor.
2558 LPM.addPass(std::move(NestedLPM));
2559 return Error::success();
2560 }
2561 if (auto Count = parseRepeatPassName(Name)) {
2562 LoopPassManager NestedLPM(DebugLogging);
2563 if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
2564 return Err;
2565 LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2566 return Error::success();
2567 }
2568
2569 for (auto &C : LoopPipelineParsingCallbacks)
2570 if (C(Name, LPM, InnerPipeline))
2571 return Error::success();
2572
2573 // Normal passes can't have pipelines.
2574 return make_error<StringError>(
2575 formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2576 inconvertibleErrorCode());
2577 }
2578
2579 // Now expand the basic registered passes from the .inc file.
2580 #define LOOP_PASS(NAME, CREATE_PASS) \
2581 if (Name == NAME) { \
2582 LPM.addPass(CREATE_PASS); \
2583 return Error::success(); \
2584 }
2585 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2586 if (checkParametrizedPassName(Name, NAME)) { \
2587 auto Params = parsePassParameters(PARSER, Name, NAME); \
2588 if (!Params) \
2589 return Params.takeError(); \
2590 LPM.addPass(CREATE_PASS(Params.get())); \
2591 return Error::success(); \
2592 }
2593 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2594 if (Name == "require<" NAME ">") { \
2595 LPM.addPass(RequireAnalysisPass< \
2596 std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \
2597 LoopAnalysisManager, LoopStandardAnalysisResults &, \
2598 LPMUpdater &>()); \
2599 return Error::success(); \
2600 } \
2601 if (Name == "invalidate<" NAME ">") { \
2602 LPM.addPass(InvalidateAnalysisPass< \
2603 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2604 return Error::success(); \
2605 }
2606 #include "PassRegistry.def"
2607
2608 for (auto &C : LoopPipelineParsingCallbacks)
2609 if (C(Name, LPM, InnerPipeline))
2610 return Error::success();
2611 return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2612 inconvertibleErrorCode());
2613 }
2614
parseAAPassName(AAManager & AA,StringRef Name)2615 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2616 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2617 if (Name == NAME) { \
2618 AA.registerModuleAnalysis< \
2619 std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2620 return true; \
2621 }
2622 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2623 if (Name == NAME) { \
2624 AA.registerFunctionAnalysis< \
2625 std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2626 return true; \
2627 }
2628 #include "PassRegistry.def"
2629
2630 for (auto &C : AAParsingCallbacks)
2631 if (C(Name, AA))
2632 return true;
2633 return false;
2634 }
2635
parseLoopPassPipeline(LoopPassManager & LPM,ArrayRef<PipelineElement> Pipeline)2636 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2637 ArrayRef<PipelineElement> Pipeline) {
2638 for (const auto &Element : Pipeline) {
2639 if (auto Err = parseLoopPass(LPM, Element))
2640 return Err;
2641 }
2642 return Error::success();
2643 }
2644
parseFunctionPassPipeline(FunctionPassManager & FPM,ArrayRef<PipelineElement> Pipeline)2645 Error PassBuilder::parseFunctionPassPipeline(
2646 FunctionPassManager &FPM, ArrayRef<PipelineElement> Pipeline) {
2647 for (const auto &Element : Pipeline) {
2648 if (auto Err = parseFunctionPass(FPM, Element))
2649 return Err;
2650 }
2651 return Error::success();
2652 }
2653
parseCGSCCPassPipeline(CGSCCPassManager & CGPM,ArrayRef<PipelineElement> Pipeline)2654 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2655 ArrayRef<PipelineElement> Pipeline) {
2656 for (const auto &Element : Pipeline) {
2657 if (auto Err = parseCGSCCPass(CGPM, Element))
2658 return Err;
2659 }
2660 return Error::success();
2661 }
2662
crossRegisterProxies(LoopAnalysisManager & LAM,FunctionAnalysisManager & FAM,CGSCCAnalysisManager & CGAM,ModuleAnalysisManager & MAM)2663 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
2664 FunctionAnalysisManager &FAM,
2665 CGSCCAnalysisManager &CGAM,
2666 ModuleAnalysisManager &MAM) {
2667 MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2668 MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2669 CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2670 FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2671 FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2672 FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2673 LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2674 }
2675
parseModulePassPipeline(ModulePassManager & MPM,ArrayRef<PipelineElement> Pipeline)2676 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2677 ArrayRef<PipelineElement> Pipeline) {
2678 for (const auto &Element : Pipeline) {
2679 if (auto Err = parseModulePass(MPM, Element))
2680 return Err;
2681 }
2682 return Error::success();
2683 }
2684
2685 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2686 // FIXME: Should this routine accept a TargetMachine or require the caller to
2687 // pre-populate the analysis managers with target-specific stuff?
parsePassPipeline(ModulePassManager & MPM,StringRef PipelineText)2688 Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
2689 StringRef PipelineText) {
2690 auto Pipeline = parsePipelineText(PipelineText);
2691 if (!Pipeline || Pipeline->empty())
2692 return make_error<StringError>(
2693 formatv("invalid pipeline '{0}'", PipelineText).str(),
2694 inconvertibleErrorCode());
2695
2696 // If the first name isn't at the module layer, wrap the pipeline up
2697 // automatically.
2698 StringRef FirstName = Pipeline->front().Name;
2699
2700 if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2701 if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
2702 Pipeline = {{"cgscc", std::move(*Pipeline)}};
2703 } else if (isFunctionPassName(FirstName,
2704 FunctionPipelineParsingCallbacks)) {
2705 Pipeline = {{"function", std::move(*Pipeline)}};
2706 } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
2707 Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
2708 } else {
2709 for (auto &C : TopLevelPipelineParsingCallbacks)
2710 if (C(MPM, *Pipeline, DebugLogging))
2711 return Error::success();
2712
2713 // Unknown pass or pipeline name!
2714 auto &InnerPipeline = Pipeline->front().InnerPipeline;
2715 return make_error<StringError>(
2716 formatv("unknown {0} name '{1}'",
2717 (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
2718 .str(),
2719 inconvertibleErrorCode());
2720 }
2721 }
2722
2723 if (auto Err = parseModulePassPipeline(MPM, *Pipeline))
2724 return Err;
2725 return Error::success();
2726 }
2727
2728 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
parsePassPipeline(CGSCCPassManager & CGPM,StringRef PipelineText)2729 Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
2730 StringRef PipelineText) {
2731 auto Pipeline = parsePipelineText(PipelineText);
2732 if (!Pipeline || Pipeline->empty())
2733 return make_error<StringError>(
2734 formatv("invalid pipeline '{0}'", PipelineText).str(),
2735 inconvertibleErrorCode());
2736
2737 StringRef FirstName = Pipeline->front().Name;
2738 if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
2739 return make_error<StringError>(
2740 formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
2741 PipelineText)
2742 .str(),
2743 inconvertibleErrorCode());
2744
2745 if (auto Err = parseCGSCCPassPipeline(CGPM, *Pipeline))
2746 return Err;
2747 return Error::success();
2748 }
2749
2750 // Primary pass pipeline description parsing routine for a \c
2751 // FunctionPassManager
parsePassPipeline(FunctionPassManager & FPM,StringRef PipelineText)2752 Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
2753 StringRef PipelineText) {
2754 auto Pipeline = parsePipelineText(PipelineText);
2755 if (!Pipeline || Pipeline->empty())
2756 return make_error<StringError>(
2757 formatv("invalid pipeline '{0}'", PipelineText).str(),
2758 inconvertibleErrorCode());
2759
2760 StringRef FirstName = Pipeline->front().Name;
2761 if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
2762 return make_error<StringError>(
2763 formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
2764 PipelineText)
2765 .str(),
2766 inconvertibleErrorCode());
2767
2768 if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline))
2769 return Err;
2770 return Error::success();
2771 }
2772
2773 // Primary pass pipeline description parsing routine for a \c LoopPassManager
parsePassPipeline(LoopPassManager & CGPM,StringRef PipelineText)2774 Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
2775 StringRef PipelineText) {
2776 auto Pipeline = parsePipelineText(PipelineText);
2777 if (!Pipeline || Pipeline->empty())
2778 return make_error<StringError>(
2779 formatv("invalid pipeline '{0}'", PipelineText).str(),
2780 inconvertibleErrorCode());
2781
2782 if (auto Err = parseLoopPassPipeline(CGPM, *Pipeline))
2783 return Err;
2784
2785 return Error::success();
2786 }
2787
parseAAPipeline(AAManager & AA,StringRef PipelineText)2788 Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
2789 // If the pipeline just consists of the word 'default' just replace the AA
2790 // manager with our default one.
2791 if (PipelineText == "default") {
2792 AA = buildDefaultAAPipeline();
2793 return Error::success();
2794 }
2795
2796 while (!PipelineText.empty()) {
2797 StringRef Name;
2798 std::tie(Name, PipelineText) = PipelineText.split(',');
2799 if (!parseAAPassName(AA, Name))
2800 return make_error<StringError>(
2801 formatv("unknown alias analysis name '{0}'", Name).str(),
2802 inconvertibleErrorCode());
2803 }
2804
2805 return Error::success();
2806 }
2807
isAAPassName(StringRef PassName)2808 bool PassBuilder::isAAPassName(StringRef PassName) {
2809 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2810 if (PassName == NAME) \
2811 return true;
2812 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2813 if (PassName == NAME) \
2814 return true;
2815 #include "PassRegistry.def"
2816 return false;
2817 }
2818
isAnalysisPassName(StringRef PassName)2819 bool PassBuilder::isAnalysisPassName(StringRef PassName) {
2820 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2821 if (PassName == NAME) \
2822 return true;
2823 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2824 if (PassName == NAME) \
2825 return true;
2826 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2827 if (PassName == NAME) \
2828 return true;
2829 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2830 if (PassName == NAME) \
2831 return true;
2832 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2833 if (PassName == NAME) \
2834 return true;
2835 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2836 if (PassName == NAME) \
2837 return true;
2838 #include "PassRegistry.def"
2839 return false;
2840 }
2841
registerParseTopLevelPipelineCallback(const std::function<bool (ModulePassManager &,ArrayRef<PipelineElement>,bool DebugLogging)> & C)2842 void PassBuilder::registerParseTopLevelPipelineCallback(
2843 const std::function<bool(ModulePassManager &, ArrayRef<PipelineElement>,
2844 bool DebugLogging)> &C) {
2845 TopLevelPipelineParsingCallbacks.push_back(C);
2846 }
2847