106f32e7eSjoerg //===- Parsing, selection, and construction of pass pipelines -------------===//
206f32e7eSjoerg //
306f32e7eSjoerg // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
406f32e7eSjoerg // See https://llvm.org/LICENSE.txt for license information.
506f32e7eSjoerg // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
606f32e7eSjoerg //
706f32e7eSjoerg //===----------------------------------------------------------------------===//
806f32e7eSjoerg /// \file
906f32e7eSjoerg ///
1006f32e7eSjoerg /// This file provides the implementation of the PassBuilder based on our
1106f32e7eSjoerg /// static pass registry as well as related functionality. It also provides
1206f32e7eSjoerg /// helpers to aid in analyzing, debugging, and testing passes and pass
1306f32e7eSjoerg /// pipelines.
1406f32e7eSjoerg ///
1506f32e7eSjoerg //===----------------------------------------------------------------------===//
1606f32e7eSjoerg
1706f32e7eSjoerg #include "llvm/Passes/PassBuilder.h"
1806f32e7eSjoerg #include "llvm/ADT/StringSwitch.h"
1906f32e7eSjoerg #include "llvm/Analysis/AliasAnalysisEvaluator.h"
20*da58b97aSjoerg #include "llvm/Analysis/AliasSetTracker.h"
2106f32e7eSjoerg #include "llvm/Analysis/AssumptionCache.h"
2206f32e7eSjoerg #include "llvm/Analysis/BasicAliasAnalysis.h"
2306f32e7eSjoerg #include "llvm/Analysis/BlockFrequencyInfo.h"
2406f32e7eSjoerg #include "llvm/Analysis/BranchProbabilityInfo.h"
2506f32e7eSjoerg #include "llvm/Analysis/CFGPrinter.h"
2606f32e7eSjoerg #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
2706f32e7eSjoerg #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
2806f32e7eSjoerg #include "llvm/Analysis/CGSCCPassManager.h"
2906f32e7eSjoerg #include "llvm/Analysis/CallGraph.h"
3006f32e7eSjoerg #include "llvm/Analysis/DDG.h"
31*da58b97aSjoerg #include "llvm/Analysis/DDGPrinter.h"
32*da58b97aSjoerg #include "llvm/Analysis/Delinearization.h"
3306f32e7eSjoerg #include "llvm/Analysis/DemandedBits.h"
3406f32e7eSjoerg #include "llvm/Analysis/DependenceAnalysis.h"
35*da58b97aSjoerg #include "llvm/Analysis/DivergenceAnalysis.h"
3606f32e7eSjoerg #include "llvm/Analysis/DominanceFrontier.h"
37*da58b97aSjoerg #include "llvm/Analysis/FunctionPropertiesAnalysis.h"
3806f32e7eSjoerg #include "llvm/Analysis/GlobalsModRef.h"
39*da58b97aSjoerg #include "llvm/Analysis/IRSimilarityIdentifier.h"
4006f32e7eSjoerg #include "llvm/Analysis/IVUsers.h"
41*da58b97aSjoerg #include "llvm/Analysis/InlineAdvisor.h"
42*da58b97aSjoerg #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
43*da58b97aSjoerg #include "llvm/Analysis/InstCount.h"
4406f32e7eSjoerg #include "llvm/Analysis/LazyCallGraph.h"
4506f32e7eSjoerg #include "llvm/Analysis/LazyValueInfo.h"
46*da58b97aSjoerg #include "llvm/Analysis/Lint.h"
4706f32e7eSjoerg #include "llvm/Analysis/LoopAccessAnalysis.h"
4806f32e7eSjoerg #include "llvm/Analysis/LoopCacheAnalysis.h"
4906f32e7eSjoerg #include "llvm/Analysis/LoopInfo.h"
50*da58b97aSjoerg #include "llvm/Analysis/LoopNestAnalysis.h"
51*da58b97aSjoerg #include "llvm/Analysis/MemDerefPrinter.h"
5206f32e7eSjoerg #include "llvm/Analysis/MemoryDependenceAnalysis.h"
5306f32e7eSjoerg #include "llvm/Analysis/MemorySSA.h"
54*da58b97aSjoerg #include "llvm/Analysis/ModuleDebugInfoPrinter.h"
5506f32e7eSjoerg #include "llvm/Analysis/ModuleSummaryAnalysis.h"
56*da58b97aSjoerg #include "llvm/Analysis/MustExecute.h"
57*da58b97aSjoerg #include "llvm/Analysis/ObjCARCAliasAnalysis.h"
5806f32e7eSjoerg #include "llvm/Analysis/OptimizationRemarkEmitter.h"
5906f32e7eSjoerg #include "llvm/Analysis/PhiValues.h"
6006f32e7eSjoerg #include "llvm/Analysis/PostDominators.h"
6106f32e7eSjoerg #include "llvm/Analysis/ProfileSummaryInfo.h"
6206f32e7eSjoerg #include "llvm/Analysis/RegionInfo.h"
6306f32e7eSjoerg #include "llvm/Analysis/ScalarEvolution.h"
6406f32e7eSjoerg #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
6506f32e7eSjoerg #include "llvm/Analysis/ScopedNoAliasAA.h"
66*da58b97aSjoerg #include "llvm/Analysis/StackLifetime.h"
6706f32e7eSjoerg #include "llvm/Analysis/StackSafetyAnalysis.h"
6806f32e7eSjoerg #include "llvm/Analysis/TargetLibraryInfo.h"
6906f32e7eSjoerg #include "llvm/Analysis/TargetTransformInfo.h"
7006f32e7eSjoerg #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
7106f32e7eSjoerg #include "llvm/IR/Dominators.h"
7206f32e7eSjoerg #include "llvm/IR/IRPrintingPasses.h"
7306f32e7eSjoerg #include "llvm/IR/PassManager.h"
74*da58b97aSjoerg #include "llvm/IR/PrintPasses.h"
7506f32e7eSjoerg #include "llvm/IR/SafepointIRVerifier.h"
7606f32e7eSjoerg #include "llvm/IR/Verifier.h"
77*da58b97aSjoerg #include "llvm/Support/CommandLine.h"
7806f32e7eSjoerg #include "llvm/Support/Debug.h"
79*da58b97aSjoerg #include "llvm/Support/ErrorHandling.h"
8006f32e7eSjoerg #include "llvm/Support/FormatVariadic.h"
8106f32e7eSjoerg #include "llvm/Support/Regex.h"
8206f32e7eSjoerg #include "llvm/Target/TargetMachine.h"
8306f32e7eSjoerg #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
84*da58b97aSjoerg #include "llvm/Transforms/Coroutines/CoroCleanup.h"
85*da58b97aSjoerg #include "llvm/Transforms/Coroutines/CoroEarly.h"
86*da58b97aSjoerg #include "llvm/Transforms/Coroutines/CoroElide.h"
87*da58b97aSjoerg #include "llvm/Transforms/Coroutines/CoroSplit.h"
8806f32e7eSjoerg #include "llvm/Transforms/IPO/AlwaysInliner.h"
89*da58b97aSjoerg #include "llvm/Transforms/IPO/Annotation2Metadata.h"
9006f32e7eSjoerg #include "llvm/Transforms/IPO/ArgumentPromotion.h"
9106f32e7eSjoerg #include "llvm/Transforms/IPO/Attributor.h"
92*da58b97aSjoerg #include "llvm/Transforms/IPO/BlockExtractor.h"
9306f32e7eSjoerg #include "llvm/Transforms/IPO/CalledValuePropagation.h"
9406f32e7eSjoerg #include "llvm/Transforms/IPO/ConstantMerge.h"
9506f32e7eSjoerg #include "llvm/Transforms/IPO/CrossDSOCFI.h"
9606f32e7eSjoerg #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
9706f32e7eSjoerg #include "llvm/Transforms/IPO/ElimAvailExtern.h"
9806f32e7eSjoerg #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
9906f32e7eSjoerg #include "llvm/Transforms/IPO/FunctionAttrs.h"
10006f32e7eSjoerg #include "llvm/Transforms/IPO/FunctionImport.h"
10106f32e7eSjoerg #include "llvm/Transforms/IPO/GlobalDCE.h"
10206f32e7eSjoerg #include "llvm/Transforms/IPO/GlobalOpt.h"
10306f32e7eSjoerg #include "llvm/Transforms/IPO/GlobalSplit.h"
10406f32e7eSjoerg #include "llvm/Transforms/IPO/HotColdSplitting.h"
105*da58b97aSjoerg #include "llvm/Transforms/IPO/IROutliner.h"
10606f32e7eSjoerg #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
10706f32e7eSjoerg #include "llvm/Transforms/IPO/Inliner.h"
10806f32e7eSjoerg #include "llvm/Transforms/IPO/Internalize.h"
109*da58b97aSjoerg #include "llvm/Transforms/IPO/LoopExtractor.h"
11006f32e7eSjoerg #include "llvm/Transforms/IPO/LowerTypeTests.h"
111*da58b97aSjoerg #include "llvm/Transforms/IPO/MergeFunctions.h"
112*da58b97aSjoerg #include "llvm/Transforms/IPO/OpenMPOpt.h"
11306f32e7eSjoerg #include "llvm/Transforms/IPO/PartialInlining.h"
11406f32e7eSjoerg #include "llvm/Transforms/IPO/SCCP.h"
11506f32e7eSjoerg #include "llvm/Transforms/IPO/SampleProfile.h"
116*da58b97aSjoerg #include "llvm/Transforms/IPO/SampleProfileProbe.h"
11706f32e7eSjoerg #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
118*da58b97aSjoerg #include "llvm/Transforms/IPO/StripSymbols.h"
11906f32e7eSjoerg #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
12006f32e7eSjoerg #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
12106f32e7eSjoerg #include "llvm/Transforms/InstCombine/InstCombine.h"
12206f32e7eSjoerg #include "llvm/Transforms/Instrumentation.h"
12306f32e7eSjoerg #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
12406f32e7eSjoerg #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
12506f32e7eSjoerg #include "llvm/Transforms/Instrumentation/CGProfile.h"
12606f32e7eSjoerg #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
127*da58b97aSjoerg #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
12806f32e7eSjoerg #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
12906f32e7eSjoerg #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
13006f32e7eSjoerg #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
13106f32e7eSjoerg #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
132*da58b97aSjoerg #include "llvm/Transforms/Instrumentation/MemProfiler.h"
13306f32e7eSjoerg #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
13406f32e7eSjoerg #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
13506f32e7eSjoerg #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
13606f32e7eSjoerg #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
13706f32e7eSjoerg #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
138*da58b97aSjoerg #include "llvm/Transforms/ObjCARC.h"
13906f32e7eSjoerg #include "llvm/Transforms/Scalar/ADCE.h"
14006f32e7eSjoerg #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
141*da58b97aSjoerg #include "llvm/Transforms/Scalar/AnnotationRemarks.h"
14206f32e7eSjoerg #include "llvm/Transforms/Scalar/BDCE.h"
14306f32e7eSjoerg #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
14406f32e7eSjoerg #include "llvm/Transforms/Scalar/ConstantHoisting.h"
145*da58b97aSjoerg #include "llvm/Transforms/Scalar/ConstraintElimination.h"
14606f32e7eSjoerg #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
14706f32e7eSjoerg #include "llvm/Transforms/Scalar/DCE.h"
14806f32e7eSjoerg #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
14906f32e7eSjoerg #include "llvm/Transforms/Scalar/DivRemPairs.h"
15006f32e7eSjoerg #include "llvm/Transforms/Scalar/EarlyCSE.h"
15106f32e7eSjoerg #include "llvm/Transforms/Scalar/Float2Int.h"
15206f32e7eSjoerg #include "llvm/Transforms/Scalar/GVN.h"
15306f32e7eSjoerg #include "llvm/Transforms/Scalar/GuardWidening.h"
15406f32e7eSjoerg #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
15506f32e7eSjoerg #include "llvm/Transforms/Scalar/IndVarSimplify.h"
15606f32e7eSjoerg #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
157*da58b97aSjoerg #include "llvm/Transforms/Scalar/InferAddressSpaces.h"
15806f32e7eSjoerg #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
15906f32e7eSjoerg #include "llvm/Transforms/Scalar/JumpThreading.h"
16006f32e7eSjoerg #include "llvm/Transforms/Scalar/LICM.h"
16106f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
16206f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
16306f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopDeletion.h"
16406f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopDistribute.h"
165*da58b97aSjoerg #include "llvm/Transforms/Scalar/LoopFlatten.h"
16606f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopFuse.h"
16706f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
16806f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
169*da58b97aSjoerg #include "llvm/Transforms/Scalar/LoopInterchange.h"
17006f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
17106f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopPassManager.h"
17206f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopPredication.h"
173*da58b97aSjoerg #include "llvm/Transforms/Scalar/LoopReroll.h"
17406f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopRotation.h"
17506f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
17606f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopSink.h"
17706f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
17806f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
17906f32e7eSjoerg #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
180*da58b97aSjoerg #include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
18106f32e7eSjoerg #include "llvm/Transforms/Scalar/LowerAtomic.h"
18206f32e7eSjoerg #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
18306f32e7eSjoerg #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
18406f32e7eSjoerg #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
185*da58b97aSjoerg #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
18606f32e7eSjoerg #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
18706f32e7eSjoerg #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
18806f32e7eSjoerg #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
18906f32e7eSjoerg #include "llvm/Transforms/Scalar/MergeICmps.h"
19006f32e7eSjoerg #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
19106f32e7eSjoerg #include "llvm/Transforms/Scalar/NaryReassociate.h"
19206f32e7eSjoerg #include "llvm/Transforms/Scalar/NewGVN.h"
19306f32e7eSjoerg #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
19406f32e7eSjoerg #include "llvm/Transforms/Scalar/Reassociate.h"
195*da58b97aSjoerg #include "llvm/Transforms/Scalar/Reg2Mem.h"
19606f32e7eSjoerg #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
19706f32e7eSjoerg #include "llvm/Transforms/Scalar/SCCP.h"
19806f32e7eSjoerg #include "llvm/Transforms/Scalar/SROA.h"
199*da58b97aSjoerg #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
20006f32e7eSjoerg #include "llvm/Transforms/Scalar/Scalarizer.h"
201*da58b97aSjoerg #include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
20206f32e7eSjoerg #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
20306f32e7eSjoerg #include "llvm/Transforms/Scalar/SimplifyCFG.h"
20406f32e7eSjoerg #include "llvm/Transforms/Scalar/Sink.h"
20506f32e7eSjoerg #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
20606f32e7eSjoerg #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
207*da58b97aSjoerg #include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
208*da58b97aSjoerg #include "llvm/Transforms/Scalar/StructurizeCFG.h"
20906f32e7eSjoerg #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
21006f32e7eSjoerg #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
21106f32e7eSjoerg #include "llvm/Transforms/Utils/AddDiscriminators.h"
212*da58b97aSjoerg #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
21306f32e7eSjoerg #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
21406f32e7eSjoerg #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
215*da58b97aSjoerg #include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
21606f32e7eSjoerg #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
217*da58b97aSjoerg #include "llvm/Transforms/Utils/FixIrreducible.h"
218*da58b97aSjoerg #include "llvm/Transforms/Utils/HelloWorld.h"
219*da58b97aSjoerg #include "llvm/Transforms/Utils/InjectTLIMappings.h"
220*da58b97aSjoerg #include "llvm/Transforms/Utils/InstructionNamer.h"
22106f32e7eSjoerg #include "llvm/Transforms/Utils/LCSSA.h"
22206f32e7eSjoerg #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
22306f32e7eSjoerg #include "llvm/Transforms/Utils/LoopSimplify.h"
224*da58b97aSjoerg #include "llvm/Transforms/Utils/LoopVersioning.h"
22506f32e7eSjoerg #include "llvm/Transforms/Utils/LowerInvoke.h"
226*da58b97aSjoerg #include "llvm/Transforms/Utils/LowerSwitch.h"
22706f32e7eSjoerg #include "llvm/Transforms/Utils/Mem2Reg.h"
228*da58b97aSjoerg #include "llvm/Transforms/Utils/MetaRenamer.h"
22906f32e7eSjoerg #include "llvm/Transforms/Utils/NameAnonGlobals.h"
230*da58b97aSjoerg #include "llvm/Transforms/Utils/RelLookupTableConverter.h"
231*da58b97aSjoerg #include "llvm/Transforms/Utils/StripGCRelocates.h"
232*da58b97aSjoerg #include "llvm/Transforms/Utils/StripNonLineTableDebugInfo.h"
23306f32e7eSjoerg #include "llvm/Transforms/Utils/SymbolRewriter.h"
234*da58b97aSjoerg #include "llvm/Transforms/Utils/UnifyFunctionExitNodes.h"
235*da58b97aSjoerg #include "llvm/Transforms/Utils/UnifyLoopExits.h"
23606f32e7eSjoerg #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
23706f32e7eSjoerg #include "llvm/Transforms/Vectorize/LoopVectorize.h"
23806f32e7eSjoerg #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
239*da58b97aSjoerg #include "llvm/Transforms/Vectorize/VectorCombine.h"
24006f32e7eSjoerg
24106f32e7eSjoerg using namespace llvm;
24206f32e7eSjoerg
243*da58b97aSjoerg static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
244*da58b97aSjoerg "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
245*da58b97aSjoerg cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
246*da58b97aSjoerg cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
247*da58b97aSjoerg "Heuristics-based inliner version."),
248*da58b97aSjoerg clEnumValN(InliningAdvisorMode::Development, "development",
249*da58b97aSjoerg "Use development mode (runtime-loadable model)."),
250*da58b97aSjoerg clEnumValN(InliningAdvisorMode::Release, "release",
251*da58b97aSjoerg "Use release mode (AOT-compiled model).")));
25206f32e7eSjoerg
25306f32e7eSjoerg static cl::opt<bool> EnableSyntheticCounts(
25406f32e7eSjoerg "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
25506f32e7eSjoerg cl::desc("Run synthetic function entry count generation "
25606f32e7eSjoerg "pass"));
25706f32e7eSjoerg
25806f32e7eSjoerg static const Regex DefaultAliasRegex(
25906f32e7eSjoerg "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
26006f32e7eSjoerg
261*da58b97aSjoerg /// Flag to enable inline deferral during PGO.
26206f32e7eSjoerg static cl::opt<bool>
263*da58b97aSjoerg EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
264*da58b97aSjoerg cl::Hidden,
265*da58b97aSjoerg cl::desc("Enable inline deferral during PGO"));
266*da58b97aSjoerg
267*da58b97aSjoerg static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
268*da58b97aSjoerg cl::Hidden, cl::ZeroOrMore,
269*da58b97aSjoerg cl::desc("Enable memory profiler"));
270*da58b97aSjoerg
271*da58b97aSjoerg static cl::opt<bool> PerformMandatoryInliningsFirst(
272*da58b97aSjoerg "mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore,
273*da58b97aSjoerg cl::desc("Perform mandatory inlinings module-wide, before performing "
274*da58b97aSjoerg "inlining."));
275*da58b97aSjoerg
276*da58b97aSjoerg static cl::opt<bool> EnableO3NonTrivialUnswitching(
277*da58b97aSjoerg "enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden,
278*da58b97aSjoerg cl::ZeroOrMore, cl::desc("Enable non-trivial loop unswitching for -O3"));
27906f32e7eSjoerg
PipelineTuningOptions()28006f32e7eSjoerg PipelineTuningOptions::PipelineTuningOptions() {
281*da58b97aSjoerg LoopInterleaving = true;
282*da58b97aSjoerg LoopVectorization = true;
283*da58b97aSjoerg SLPVectorization = false;
28406f32e7eSjoerg LoopUnrolling = true;
28506f32e7eSjoerg ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
286*da58b97aSjoerg Coroutines = false;
28706f32e7eSjoerg LicmMssaOptCap = SetLicmMssaOptCap;
28806f32e7eSjoerg LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
289*da58b97aSjoerg CallGraphProfile = true;
290*da58b97aSjoerg MergeFunctions = false;
29106f32e7eSjoerg }
29206f32e7eSjoerg
293*da58b97aSjoerg namespace llvm {
294*da58b97aSjoerg extern cl::opt<unsigned> MaxDevirtIterations;
295*da58b97aSjoerg extern cl::opt<bool> EnableConstraintElimination;
296*da58b97aSjoerg extern cl::opt<bool> EnableGVNHoist;
297*da58b97aSjoerg extern cl::opt<bool> EnableGVNSink;
29806f32e7eSjoerg extern cl::opt<bool> EnableHotColdSplit;
299*da58b97aSjoerg extern cl::opt<bool> EnableIROutliner;
30006f32e7eSjoerg extern cl::opt<bool> EnableOrderFileInstrumentation;
301*da58b97aSjoerg extern cl::opt<bool> EnableCHR;
302*da58b97aSjoerg extern cl::opt<bool> EnableLoopInterchange;
303*da58b97aSjoerg extern cl::opt<bool> EnableUnrollAndJam;
304*da58b97aSjoerg extern cl::opt<bool> EnableLoopFlatten;
305*da58b97aSjoerg extern cl::opt<bool> RunNewGVN;
306*da58b97aSjoerg extern cl::opt<bool> RunPartialInlining;
307*da58b97aSjoerg extern cl::opt<bool> ExtraVectorizerPasses;
30806f32e7eSjoerg
30906f32e7eSjoerg extern cl::opt<bool> FlattenedProfileUsed;
31006f32e7eSjoerg
311*da58b97aSjoerg extern cl::opt<AttributorRunOption> AttributorRun;
312*da58b97aSjoerg extern cl::opt<bool> EnableKnowledgeRetention;
31306f32e7eSjoerg
314*da58b97aSjoerg extern cl::opt<bool> EnableMatrix;
315*da58b97aSjoerg
316*da58b97aSjoerg extern cl::opt<bool> DisablePreInliner;
317*da58b97aSjoerg extern cl::opt<int> PreInlineThreshold;
318*da58b97aSjoerg } // namespace llvm
319*da58b97aSjoerg
320*da58b97aSjoerg const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O0 = {
321*da58b97aSjoerg /*SpeedLevel*/ 0,
322*da58b97aSjoerg /*SizeLevel*/ 0};
323*da58b97aSjoerg const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O1 = {
324*da58b97aSjoerg /*SpeedLevel*/ 1,
325*da58b97aSjoerg /*SizeLevel*/ 0};
326*da58b97aSjoerg const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O2 = {
327*da58b97aSjoerg /*SpeedLevel*/ 2,
328*da58b97aSjoerg /*SizeLevel*/ 0};
329*da58b97aSjoerg const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O3 = {
330*da58b97aSjoerg /*SpeedLevel*/ 3,
331*da58b97aSjoerg /*SizeLevel*/ 0};
332*da58b97aSjoerg const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Os = {
333*da58b97aSjoerg /*SpeedLevel*/ 2,
334*da58b97aSjoerg /*SizeLevel*/ 1};
335*da58b97aSjoerg const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Oz = {
336*da58b97aSjoerg /*SpeedLevel*/ 2,
337*da58b97aSjoerg /*SizeLevel*/ 2};
33806f32e7eSjoerg
33906f32e7eSjoerg namespace {
34006f32e7eSjoerg
341*da58b97aSjoerg // The following passes/analyses have custom names, otherwise their name will
342*da58b97aSjoerg // include `(anonymous namespace)`. These are special since they are only for
343*da58b97aSjoerg // testing purposes and don't live in a header file.
344*da58b97aSjoerg
34506f32e7eSjoerg /// No-op module pass which does nothing.
346*da58b97aSjoerg struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
run__anone50602ed0111::NoOpModulePass34706f32e7eSjoerg PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
34806f32e7eSjoerg return PreservedAnalyses::all();
34906f32e7eSjoerg }
350*da58b97aSjoerg
name__anone50602ed0111::NoOpModulePass35106f32e7eSjoerg static StringRef name() { return "NoOpModulePass"; }
35206f32e7eSjoerg };
35306f32e7eSjoerg
35406f32e7eSjoerg /// No-op module analysis.
35506f32e7eSjoerg class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
35606f32e7eSjoerg friend AnalysisInfoMixin<NoOpModuleAnalysis>;
35706f32e7eSjoerg static AnalysisKey Key;
35806f32e7eSjoerg
35906f32e7eSjoerg public:
36006f32e7eSjoerg struct Result {};
run(Module &,ModuleAnalysisManager &)36106f32e7eSjoerg Result run(Module &, ModuleAnalysisManager &) { return Result(); }
name()36206f32e7eSjoerg static StringRef name() { return "NoOpModuleAnalysis"; }
36306f32e7eSjoerg };
36406f32e7eSjoerg
36506f32e7eSjoerg /// No-op CGSCC pass which does nothing.
366*da58b97aSjoerg struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
run__anone50602ed0111::NoOpCGSCCPass36706f32e7eSjoerg PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
36806f32e7eSjoerg LazyCallGraph &, CGSCCUpdateResult &UR) {
36906f32e7eSjoerg return PreservedAnalyses::all();
37006f32e7eSjoerg }
name__anone50602ed0111::NoOpCGSCCPass37106f32e7eSjoerg static StringRef name() { return "NoOpCGSCCPass"; }
37206f32e7eSjoerg };
37306f32e7eSjoerg
37406f32e7eSjoerg /// No-op CGSCC analysis.
37506f32e7eSjoerg class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
37606f32e7eSjoerg friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
37706f32e7eSjoerg static AnalysisKey Key;
37806f32e7eSjoerg
37906f32e7eSjoerg public:
38006f32e7eSjoerg struct Result {};
run(LazyCallGraph::SCC &,CGSCCAnalysisManager &,LazyCallGraph & G)38106f32e7eSjoerg Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
38206f32e7eSjoerg return Result();
38306f32e7eSjoerg }
name()38406f32e7eSjoerg static StringRef name() { return "NoOpCGSCCAnalysis"; }
38506f32e7eSjoerg };
38606f32e7eSjoerg
38706f32e7eSjoerg /// No-op function pass which does nothing.
388*da58b97aSjoerg struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
run__anone50602ed0111::NoOpFunctionPass38906f32e7eSjoerg PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
39006f32e7eSjoerg return PreservedAnalyses::all();
39106f32e7eSjoerg }
name__anone50602ed0111::NoOpFunctionPass39206f32e7eSjoerg static StringRef name() { return "NoOpFunctionPass"; }
39306f32e7eSjoerg };
39406f32e7eSjoerg
39506f32e7eSjoerg /// No-op function analysis.
39606f32e7eSjoerg class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
39706f32e7eSjoerg friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
39806f32e7eSjoerg static AnalysisKey Key;
39906f32e7eSjoerg
40006f32e7eSjoerg public:
40106f32e7eSjoerg struct Result {};
run(Function &,FunctionAnalysisManager &)40206f32e7eSjoerg Result run(Function &, FunctionAnalysisManager &) { return Result(); }
name()40306f32e7eSjoerg static StringRef name() { return "NoOpFunctionAnalysis"; }
40406f32e7eSjoerg };
40506f32e7eSjoerg
40606f32e7eSjoerg /// No-op loop pass which does nothing.
407*da58b97aSjoerg struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
run__anone50602ed0111::NoOpLoopPass40806f32e7eSjoerg PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
40906f32e7eSjoerg LoopStandardAnalysisResults &, LPMUpdater &) {
41006f32e7eSjoerg return PreservedAnalyses::all();
41106f32e7eSjoerg }
name__anone50602ed0111::NoOpLoopPass41206f32e7eSjoerg static StringRef name() { return "NoOpLoopPass"; }
41306f32e7eSjoerg };
41406f32e7eSjoerg
41506f32e7eSjoerg /// No-op loop analysis.
41606f32e7eSjoerg class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
41706f32e7eSjoerg friend AnalysisInfoMixin<NoOpLoopAnalysis>;
41806f32e7eSjoerg static AnalysisKey Key;
41906f32e7eSjoerg
42006f32e7eSjoerg public:
42106f32e7eSjoerg struct Result {};
run(Loop &,LoopAnalysisManager &,LoopStandardAnalysisResults &)42206f32e7eSjoerg Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
42306f32e7eSjoerg return Result();
42406f32e7eSjoerg }
name()42506f32e7eSjoerg static StringRef name() { return "NoOpLoopAnalysis"; }
42606f32e7eSjoerg };
42706f32e7eSjoerg
42806f32e7eSjoerg AnalysisKey NoOpModuleAnalysis::Key;
42906f32e7eSjoerg AnalysisKey NoOpCGSCCAnalysis::Key;
43006f32e7eSjoerg AnalysisKey NoOpFunctionAnalysis::Key;
43106f32e7eSjoerg AnalysisKey NoOpLoopAnalysis::Key;
43206f32e7eSjoerg
433*da58b97aSjoerg /// Whether or not we should populate a PassInstrumentationCallbacks's class to
434*da58b97aSjoerg /// pass name map.
435*da58b97aSjoerg ///
436*da58b97aSjoerg /// This is for optimization purposes so we don't populate it if we never use
437*da58b97aSjoerg /// it. This should be updated if new pass instrumentation wants to use the map.
438*da58b97aSjoerg /// We currently only use this for --print-before/after.
shouldPopulateClassToPassNames()439*da58b97aSjoerg bool shouldPopulateClassToPassNames() {
440*da58b97aSjoerg return !printBeforePasses().empty() || !printAfterPasses().empty();
441*da58b97aSjoerg }
442*da58b97aSjoerg
443*da58b97aSjoerg } // namespace
444*da58b97aSjoerg
PassBuilder(TargetMachine * TM,PipelineTuningOptions PTO,Optional<PGOOptions> PGOOpt,PassInstrumentationCallbacks * PIC)445*da58b97aSjoerg PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO,
446*da58b97aSjoerg Optional<PGOOptions> PGOOpt,
447*da58b97aSjoerg PassInstrumentationCallbacks *PIC)
448*da58b97aSjoerg : TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
449*da58b97aSjoerg if (TM)
450*da58b97aSjoerg TM->registerPassBuilderCallbacks(*this);
451*da58b97aSjoerg if (PIC && shouldPopulateClassToPassNames()) {
452*da58b97aSjoerg #define MODULE_PASS(NAME, CREATE_PASS) \
453*da58b97aSjoerg PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
454*da58b97aSjoerg #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
455*da58b97aSjoerg PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
456*da58b97aSjoerg #define FUNCTION_PASS(NAME, CREATE_PASS) \
457*da58b97aSjoerg PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
458*da58b97aSjoerg #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
459*da58b97aSjoerg PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
460*da58b97aSjoerg #define LOOP_PASS(NAME, CREATE_PASS) \
461*da58b97aSjoerg PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
462*da58b97aSjoerg #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
463*da58b97aSjoerg PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
464*da58b97aSjoerg #define CGSCC_PASS(NAME, CREATE_PASS) \
465*da58b97aSjoerg PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
466*da58b97aSjoerg #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
467*da58b97aSjoerg PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
468*da58b97aSjoerg #include "PassRegistry.def"
469*da58b97aSjoerg }
470*da58b97aSjoerg }
47106f32e7eSjoerg
invokePeepholeEPCallbacks(FunctionPassManager & FPM,PassBuilder::OptimizationLevel Level)47206f32e7eSjoerg void PassBuilder::invokePeepholeEPCallbacks(
47306f32e7eSjoerg FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
47406f32e7eSjoerg for (auto &C : PeepholeEPCallbacks)
47506f32e7eSjoerg C(FPM, Level);
47606f32e7eSjoerg }
47706f32e7eSjoerg
registerModuleAnalyses(ModuleAnalysisManager & MAM)47806f32e7eSjoerg void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
47906f32e7eSjoerg #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
48006f32e7eSjoerg MAM.registerPass([&] { return CREATE_PASS; });
48106f32e7eSjoerg #include "PassRegistry.def"
48206f32e7eSjoerg
48306f32e7eSjoerg for (auto &C : ModuleAnalysisRegistrationCallbacks)
48406f32e7eSjoerg C(MAM);
48506f32e7eSjoerg }
48606f32e7eSjoerg
registerCGSCCAnalyses(CGSCCAnalysisManager & CGAM)48706f32e7eSjoerg void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
48806f32e7eSjoerg #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
48906f32e7eSjoerg CGAM.registerPass([&] { return CREATE_PASS; });
49006f32e7eSjoerg #include "PassRegistry.def"
49106f32e7eSjoerg
49206f32e7eSjoerg for (auto &C : CGSCCAnalysisRegistrationCallbacks)
49306f32e7eSjoerg C(CGAM);
49406f32e7eSjoerg }
49506f32e7eSjoerg
registerFunctionAnalyses(FunctionAnalysisManager & FAM)49606f32e7eSjoerg void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
49706f32e7eSjoerg #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
49806f32e7eSjoerg FAM.registerPass([&] { return CREATE_PASS; });
49906f32e7eSjoerg #include "PassRegistry.def"
50006f32e7eSjoerg
50106f32e7eSjoerg for (auto &C : FunctionAnalysisRegistrationCallbacks)
50206f32e7eSjoerg C(FAM);
50306f32e7eSjoerg }
50406f32e7eSjoerg
registerLoopAnalyses(LoopAnalysisManager & LAM)50506f32e7eSjoerg void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
50606f32e7eSjoerg #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
50706f32e7eSjoerg LAM.registerPass([&] { return CREATE_PASS; });
50806f32e7eSjoerg #include "PassRegistry.def"
50906f32e7eSjoerg
51006f32e7eSjoerg for (auto &C : LoopAnalysisRegistrationCallbacks)
51106f32e7eSjoerg C(LAM);
51206f32e7eSjoerg }
51306f32e7eSjoerg
514*da58b97aSjoerg // Helper to add AnnotationRemarksPass.
addAnnotationRemarksPass(ModulePassManager & MPM)515*da58b97aSjoerg static void addAnnotationRemarksPass(ModulePassManager &MPM) {
516*da58b97aSjoerg FunctionPassManager FPM;
517*da58b97aSjoerg FPM.addPass(AnnotationRemarksPass());
518*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
519*da58b97aSjoerg }
520*da58b97aSjoerg
521*da58b97aSjoerg // Helper to check if the current compilation phase is preparing for LTO
isLTOPreLink(ThinOrFullLTOPhase Phase)522*da58b97aSjoerg static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
523*da58b97aSjoerg return Phase == ThinOrFullLTOPhase::ThinLTOPreLink ||
524*da58b97aSjoerg Phase == ThinOrFullLTOPhase::FullLTOPreLink;
525*da58b97aSjoerg }
526*da58b97aSjoerg
527*da58b97aSjoerg // TODO: Investigate the cost/benefit of tail call elimination on debugging.
52806f32e7eSjoerg FunctionPassManager
buildO1FunctionSimplificationPipeline(OptimizationLevel Level,ThinOrFullLTOPhase Phase)529*da58b97aSjoerg PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
530*da58b97aSjoerg ThinOrFullLTOPhase Phase) {
531*da58b97aSjoerg
532*da58b97aSjoerg FunctionPassManager FPM;
53306f32e7eSjoerg
53406f32e7eSjoerg // Form SSA out of local memory accesses after breaking apart aggregates into
53506f32e7eSjoerg // scalars.
53606f32e7eSjoerg FPM.addPass(SROA());
53706f32e7eSjoerg
53806f32e7eSjoerg // Catch trivial redundancies
53906f32e7eSjoerg FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
54006f32e7eSjoerg
54106f32e7eSjoerg // Hoisting of scalars and load expressions.
542*da58b97aSjoerg FPM.addPass(SimplifyCFGPass());
543*da58b97aSjoerg FPM.addPass(InstCombinePass());
544*da58b97aSjoerg
545*da58b97aSjoerg FPM.addPass(LibCallsShrinkWrapPass());
546*da58b97aSjoerg
547*da58b97aSjoerg invokePeepholeEPCallbacks(FPM, Level);
548*da58b97aSjoerg
549*da58b97aSjoerg FPM.addPass(SimplifyCFGPass());
550*da58b97aSjoerg
551*da58b97aSjoerg // Form canonically associated expression trees, and simplify the trees using
552*da58b97aSjoerg // basic mathematical properties. For example, this will form (nearly)
553*da58b97aSjoerg // minimal multiplication trees.
554*da58b97aSjoerg FPM.addPass(ReassociatePass());
555*da58b97aSjoerg
556*da58b97aSjoerg // Add the primary loop simplification pipeline.
557*da58b97aSjoerg // FIXME: Currently this is split into two loop pass pipelines because we run
558*da58b97aSjoerg // some function passes in between them. These can and should be removed
559*da58b97aSjoerg // and/or replaced by scheduling the loop pass equivalents in the correct
560*da58b97aSjoerg // positions. But those equivalent passes aren't powerful enough yet.
561*da58b97aSjoerg // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
562*da58b97aSjoerg // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
563*da58b97aSjoerg // fully replace `SimplifyCFGPass`, and the closest to the other we have is
564*da58b97aSjoerg // `LoopInstSimplify`.
565*da58b97aSjoerg LoopPassManager LPM1, LPM2;
566*da58b97aSjoerg
567*da58b97aSjoerg // Simplify the loop body. We do this initially to clean up after other loop
568*da58b97aSjoerg // passes run, either when iterating on a loop or on inner loops with
569*da58b97aSjoerg // implications on the outer loop.
570*da58b97aSjoerg LPM1.addPass(LoopInstSimplifyPass());
571*da58b97aSjoerg LPM1.addPass(LoopSimplifyCFGPass());
572*da58b97aSjoerg
573*da58b97aSjoerg // Try to remove as much code from the loop header as possible,
574*da58b97aSjoerg // to reduce amount of IR that will have to be duplicated.
575*da58b97aSjoerg // TODO: Investigate promotion cap for O1.
576*da58b97aSjoerg LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
577*da58b97aSjoerg
578*da58b97aSjoerg LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true,
579*da58b97aSjoerg isLTOPreLink(Phase)));
580*da58b97aSjoerg // TODO: Investigate promotion cap for O1.
581*da58b97aSjoerg LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
582*da58b97aSjoerg LPM1.addPass(SimpleLoopUnswitchPass());
583*da58b97aSjoerg
584*da58b97aSjoerg LPM2.addPass(LoopIdiomRecognizePass());
585*da58b97aSjoerg LPM2.addPass(IndVarSimplifyPass());
586*da58b97aSjoerg
587*da58b97aSjoerg for (auto &C : LateLoopOptimizationsEPCallbacks)
588*da58b97aSjoerg C(LPM2, Level);
589*da58b97aSjoerg
590*da58b97aSjoerg LPM2.addPass(LoopDeletionPass());
591*da58b97aSjoerg
592*da58b97aSjoerg if (EnableLoopInterchange)
593*da58b97aSjoerg LPM2.addPass(LoopInterchangePass());
594*da58b97aSjoerg
595*da58b97aSjoerg // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
596*da58b97aSjoerg // because it changes IR to makes profile annotation in back compile
597*da58b97aSjoerg // inaccurate. The normal unroller doesn't pay attention to forced full unroll
598*da58b97aSjoerg // attributes so we need to make sure and allow the full unroll pass to pay
599*da58b97aSjoerg // attention to it.
600*da58b97aSjoerg if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
601*da58b97aSjoerg PGOOpt->Action != PGOOptions::SampleUse)
602*da58b97aSjoerg LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
603*da58b97aSjoerg /* OnlyWhenForced= */ !PTO.LoopUnrolling,
604*da58b97aSjoerg PTO.ForgetAllSCEVInLoopUnroll));
605*da58b97aSjoerg
606*da58b97aSjoerg for (auto &C : LoopOptimizerEndEPCallbacks)
607*da58b97aSjoerg C(LPM2, Level);
608*da58b97aSjoerg
609*da58b97aSjoerg // We provide the opt remark emitter pass for LICM to use. We only need to do
610*da58b97aSjoerg // this once as it is immutable.
611*da58b97aSjoerg FPM.addPass(
612*da58b97aSjoerg RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
613*da58b97aSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),
614*da58b97aSjoerg EnableMSSALoopDependency,
615*da58b97aSjoerg /*UseBlockFrequencyInfo=*/true));
616*da58b97aSjoerg FPM.addPass(SimplifyCFGPass());
617*da58b97aSjoerg FPM.addPass(InstCombinePass());
618*da58b97aSjoerg if (EnableLoopFlatten)
619*da58b97aSjoerg FPM.addPass(LoopFlattenPass());
620*da58b97aSjoerg // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
621*da58b97aSjoerg // *All* loop passes must preserve it, in order to be able to use it.
622*da58b97aSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),
623*da58b97aSjoerg /*UseMemorySSA=*/false,
624*da58b97aSjoerg /*UseBlockFrequencyInfo=*/false));
625*da58b97aSjoerg
626*da58b97aSjoerg // Delete small array after loop unroll.
627*da58b97aSjoerg FPM.addPass(SROA());
628*da58b97aSjoerg
629*da58b97aSjoerg // Specially optimize memory movement as it doesn't look like dataflow in SSA.
630*da58b97aSjoerg FPM.addPass(MemCpyOptPass());
631*da58b97aSjoerg
632*da58b97aSjoerg // Sparse conditional constant propagation.
633*da58b97aSjoerg // FIXME: It isn't clear why we do this *after* loop passes rather than
634*da58b97aSjoerg // before...
635*da58b97aSjoerg FPM.addPass(SCCPPass());
636*da58b97aSjoerg
637*da58b97aSjoerg // Delete dead bit computations (instcombine runs after to fold away the dead
638*da58b97aSjoerg // computations, and then ADCE will run later to exploit any new DCE
639*da58b97aSjoerg // opportunities that creates).
640*da58b97aSjoerg FPM.addPass(BDCEPass());
641*da58b97aSjoerg
642*da58b97aSjoerg // Run instcombine after redundancy and dead bit elimination to exploit
643*da58b97aSjoerg // opportunities opened up by them.
644*da58b97aSjoerg FPM.addPass(InstCombinePass());
645*da58b97aSjoerg invokePeepholeEPCallbacks(FPM, Level);
646*da58b97aSjoerg
647*da58b97aSjoerg if (PTO.Coroutines)
648*da58b97aSjoerg FPM.addPass(CoroElidePass());
649*da58b97aSjoerg
650*da58b97aSjoerg for (auto &C : ScalarOptimizerLateEPCallbacks)
651*da58b97aSjoerg C(FPM, Level);
652*da58b97aSjoerg
653*da58b97aSjoerg // Finally, do an expensive DCE pass to catch all the dead code exposed by
654*da58b97aSjoerg // the simplifications and basic cleanup after all the simplifications.
655*da58b97aSjoerg // TODO: Investigate if this is too expensive.
656*da58b97aSjoerg FPM.addPass(ADCEPass());
657*da58b97aSjoerg FPM.addPass(SimplifyCFGPass());
658*da58b97aSjoerg FPM.addPass(InstCombinePass());
659*da58b97aSjoerg invokePeepholeEPCallbacks(FPM, Level);
660*da58b97aSjoerg
661*da58b97aSjoerg return FPM;
662*da58b97aSjoerg }
663*da58b97aSjoerg
664*da58b97aSjoerg FunctionPassManager
buildFunctionSimplificationPipeline(OptimizationLevel Level,ThinOrFullLTOPhase Phase)665*da58b97aSjoerg PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
666*da58b97aSjoerg ThinOrFullLTOPhase Phase) {
667*da58b97aSjoerg assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
668*da58b97aSjoerg
669*da58b97aSjoerg // The O1 pipeline has a separate pipeline creation function to simplify
670*da58b97aSjoerg // construction readability.
671*da58b97aSjoerg if (Level.getSpeedupLevel() == 1)
672*da58b97aSjoerg return buildO1FunctionSimplificationPipeline(Level, Phase);
673*da58b97aSjoerg
674*da58b97aSjoerg FunctionPassManager FPM;
675*da58b97aSjoerg
676*da58b97aSjoerg // Form SSA out of local memory accesses after breaking apart aggregates into
677*da58b97aSjoerg // scalars.
678*da58b97aSjoerg FPM.addPass(SROA());
679*da58b97aSjoerg
680*da58b97aSjoerg // Catch trivial redundancies
681*da58b97aSjoerg FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
682*da58b97aSjoerg if (EnableKnowledgeRetention)
683*da58b97aSjoerg FPM.addPass(AssumeSimplifyPass());
684*da58b97aSjoerg
685*da58b97aSjoerg // Hoisting of scalars and load expressions.
68606f32e7eSjoerg if (EnableGVNHoist)
68706f32e7eSjoerg FPM.addPass(GVNHoistPass());
68806f32e7eSjoerg
68906f32e7eSjoerg // Global value numbering based sinking.
69006f32e7eSjoerg if (EnableGVNSink) {
69106f32e7eSjoerg FPM.addPass(GVNSinkPass());
69206f32e7eSjoerg FPM.addPass(SimplifyCFGPass());
69306f32e7eSjoerg }
69406f32e7eSjoerg
695*da58b97aSjoerg if (EnableConstraintElimination)
696*da58b97aSjoerg FPM.addPass(ConstraintEliminationPass());
697*da58b97aSjoerg
69806f32e7eSjoerg // Speculative execution if the target has divergent branches; otherwise nop.
699*da58b97aSjoerg FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
70006f32e7eSjoerg
70106f32e7eSjoerg // Optimize based on known information about branches, and cleanup afterward.
70206f32e7eSjoerg FPM.addPass(JumpThreadingPass());
70306f32e7eSjoerg FPM.addPass(CorrelatedValuePropagationPass());
704*da58b97aSjoerg
70506f32e7eSjoerg FPM.addPass(SimplifyCFGPass());
706*da58b97aSjoerg if (Level == OptimizationLevel::O3)
70706f32e7eSjoerg FPM.addPass(AggressiveInstCombinePass());
70806f32e7eSjoerg FPM.addPass(InstCombinePass());
70906f32e7eSjoerg
710*da58b97aSjoerg if (!Level.isOptimizingForSize())
71106f32e7eSjoerg FPM.addPass(LibCallsShrinkWrapPass());
71206f32e7eSjoerg
71306f32e7eSjoerg invokePeepholeEPCallbacks(FPM, Level);
71406f32e7eSjoerg
71506f32e7eSjoerg // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
71606f32e7eSjoerg // using the size value profile. Don't perform this when optimizing for size.
71706f32e7eSjoerg if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
718*da58b97aSjoerg !Level.isOptimizingForSize())
71906f32e7eSjoerg FPM.addPass(PGOMemOPSizeOpt());
72006f32e7eSjoerg
72106f32e7eSjoerg FPM.addPass(TailCallElimPass());
72206f32e7eSjoerg FPM.addPass(SimplifyCFGPass());
72306f32e7eSjoerg
72406f32e7eSjoerg // Form canonically associated expression trees, and simplify the trees using
72506f32e7eSjoerg // basic mathematical properties. For example, this will form (nearly)
72606f32e7eSjoerg // minimal multiplication trees.
72706f32e7eSjoerg FPM.addPass(ReassociatePass());
72806f32e7eSjoerg
72906f32e7eSjoerg // Add the primary loop simplification pipeline.
73006f32e7eSjoerg // FIXME: Currently this is split into two loop pass pipelines because we run
73106f32e7eSjoerg // some function passes in between them. These can and should be removed
73206f32e7eSjoerg // and/or replaced by scheduling the loop pass equivalents in the correct
73306f32e7eSjoerg // positions. But those equivalent passes aren't powerful enough yet.
73406f32e7eSjoerg // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
73506f32e7eSjoerg // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
73606f32e7eSjoerg // fully replace `SimplifyCFGPass`, and the closest to the other we have is
73706f32e7eSjoerg // `LoopInstSimplify`.
738*da58b97aSjoerg LoopPassManager LPM1, LPM2;
73906f32e7eSjoerg
74006f32e7eSjoerg // Simplify the loop body. We do this initially to clean up after other loop
74106f32e7eSjoerg // passes run, either when iterating on a loop or on inner loops with
74206f32e7eSjoerg // implications on the outer loop.
74306f32e7eSjoerg LPM1.addPass(LoopInstSimplifyPass());
74406f32e7eSjoerg LPM1.addPass(LoopSimplifyCFGPass());
74506f32e7eSjoerg
746*da58b97aSjoerg // Try to remove as much code from the loop header as possible,
747*da58b97aSjoerg // to reduce amount of IR that will have to be duplicated.
748*da58b97aSjoerg // TODO: Investigate promotion cap for O1.
74906f32e7eSjoerg LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
750*da58b97aSjoerg
751*da58b97aSjoerg // Disable header duplication in loop rotation at -Oz.
752*da58b97aSjoerg LPM1.addPass(
753*da58b97aSjoerg LoopRotatePass(Level != OptimizationLevel::Oz, isLTOPreLink(Phase)));
754*da58b97aSjoerg // TODO: Investigate promotion cap for O1.
755*da58b97aSjoerg LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
756*da58b97aSjoerg LPM1.addPass(
757*da58b97aSjoerg SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3 &&
758*da58b97aSjoerg EnableO3NonTrivialUnswitching));
75906f32e7eSjoerg LPM2.addPass(LoopIdiomRecognizePass());
760*da58b97aSjoerg LPM2.addPass(IndVarSimplifyPass());
76106f32e7eSjoerg
76206f32e7eSjoerg for (auto &C : LateLoopOptimizationsEPCallbacks)
76306f32e7eSjoerg C(LPM2, Level);
76406f32e7eSjoerg
76506f32e7eSjoerg LPM2.addPass(LoopDeletionPass());
766*da58b97aSjoerg
767*da58b97aSjoerg if (EnableLoopInterchange)
768*da58b97aSjoerg LPM2.addPass(LoopInterchangePass());
769*da58b97aSjoerg
77006f32e7eSjoerg // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
77106f32e7eSjoerg // because it changes IR to makes profile annotation in back compile
772*da58b97aSjoerg // inaccurate. The normal unroller doesn't pay attention to forced full unroll
773*da58b97aSjoerg // attributes so we need to make sure and allow the full unroll pass to pay
774*da58b97aSjoerg // attention to it.
775*da58b97aSjoerg if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
776*da58b97aSjoerg PGOOpt->Action != PGOOptions::SampleUse)
777*da58b97aSjoerg LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
778*da58b97aSjoerg /* OnlyWhenForced= */ !PTO.LoopUnrolling,
77906f32e7eSjoerg PTO.ForgetAllSCEVInLoopUnroll));
78006f32e7eSjoerg
78106f32e7eSjoerg for (auto &C : LoopOptimizerEndEPCallbacks)
78206f32e7eSjoerg C(LPM2, Level);
78306f32e7eSjoerg
78406f32e7eSjoerg // We provide the opt remark emitter pass for LICM to use. We only need to do
78506f32e7eSjoerg // this once as it is immutable.
786*da58b97aSjoerg FPM.addPass(
787*da58b97aSjoerg RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
788*da58b97aSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),
789*da58b97aSjoerg EnableMSSALoopDependency,
790*da58b97aSjoerg /*UseBlockFrequencyInfo=*/true));
79106f32e7eSjoerg FPM.addPass(SimplifyCFGPass());
79206f32e7eSjoerg FPM.addPass(InstCombinePass());
793*da58b97aSjoerg if (EnableLoopFlatten)
794*da58b97aSjoerg FPM.addPass(LoopFlattenPass());
795*da58b97aSjoerg // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
79606f32e7eSjoerg // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
79706f32e7eSjoerg // *All* loop passes must preserve it, in order to be able to use it.
798*da58b97aSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),
799*da58b97aSjoerg /*UseMemorySSA=*/false,
800*da58b97aSjoerg /*UseBlockFrequencyInfo=*/false));
801*da58b97aSjoerg
802*da58b97aSjoerg // Delete small array after loop unroll.
803*da58b97aSjoerg FPM.addPass(SROA());
80406f32e7eSjoerg
80506f32e7eSjoerg // Eliminate redundancies.
80606f32e7eSjoerg FPM.addPass(MergedLoadStoreMotionPass());
80706f32e7eSjoerg if (RunNewGVN)
80806f32e7eSjoerg FPM.addPass(NewGVNPass());
80906f32e7eSjoerg else
81006f32e7eSjoerg FPM.addPass(GVN());
81106f32e7eSjoerg
81206f32e7eSjoerg // Sparse conditional constant propagation.
81306f32e7eSjoerg // FIXME: It isn't clear why we do this *after* loop passes rather than
81406f32e7eSjoerg // before...
81506f32e7eSjoerg FPM.addPass(SCCPPass());
81606f32e7eSjoerg
81706f32e7eSjoerg // Delete dead bit computations (instcombine runs after to fold away the dead
81806f32e7eSjoerg // computations, and then ADCE will run later to exploit any new DCE
81906f32e7eSjoerg // opportunities that creates).
82006f32e7eSjoerg FPM.addPass(BDCEPass());
82106f32e7eSjoerg
82206f32e7eSjoerg // Run instcombine after redundancy and dead bit elimination to exploit
82306f32e7eSjoerg // opportunities opened up by them.
82406f32e7eSjoerg FPM.addPass(InstCombinePass());
82506f32e7eSjoerg invokePeepholeEPCallbacks(FPM, Level);
82606f32e7eSjoerg
82706f32e7eSjoerg // Re-consider control flow based optimizations after redundancy elimination,
82806f32e7eSjoerg // redo DCE, etc.
82906f32e7eSjoerg FPM.addPass(JumpThreadingPass());
83006f32e7eSjoerg FPM.addPass(CorrelatedValuePropagationPass());
831*da58b97aSjoerg
832*da58b97aSjoerg // Finally, do an expensive DCE pass to catch all the dead code exposed by
833*da58b97aSjoerg // the simplifications and basic cleanup after all the simplifications.
834*da58b97aSjoerg // TODO: Investigate if this is too expensive.
835*da58b97aSjoerg FPM.addPass(ADCEPass());
836*da58b97aSjoerg
837*da58b97aSjoerg // Specially optimize memory movement as it doesn't look like dataflow in SSA.
838*da58b97aSjoerg FPM.addPass(MemCpyOptPass());
839*da58b97aSjoerg
84006f32e7eSjoerg FPM.addPass(DSEPass());
84106f32e7eSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(
84206f32e7eSjoerg LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
843*da58b97aSjoerg EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true));
844*da58b97aSjoerg
845*da58b97aSjoerg if (PTO.Coroutines)
846*da58b97aSjoerg FPM.addPass(CoroElidePass());
84706f32e7eSjoerg
84806f32e7eSjoerg for (auto &C : ScalarOptimizerLateEPCallbacks)
84906f32e7eSjoerg C(FPM, Level);
85006f32e7eSjoerg
851*da58b97aSjoerg FPM.addPass(SimplifyCFGPass(
852*da58b97aSjoerg SimplifyCFGOptions().hoistCommonInsts(true).sinkCommonInsts(true)));
85306f32e7eSjoerg FPM.addPass(InstCombinePass());
85406f32e7eSjoerg invokePeepholeEPCallbacks(FPM, Level);
85506f32e7eSjoerg
856*da58b97aSjoerg if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
85706f32e7eSjoerg (PGOOpt->Action == PGOOptions::IRUse ||
85806f32e7eSjoerg PGOOpt->Action == PGOOptions::SampleUse))
85906f32e7eSjoerg FPM.addPass(ControlHeightReductionPass());
86006f32e7eSjoerg
86106f32e7eSjoerg return FPM;
86206f32e7eSjoerg }
86306f32e7eSjoerg
addRequiredLTOPreLinkPasses(ModulePassManager & MPM)864*da58b97aSjoerg void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
865*da58b97aSjoerg MPM.addPass(CanonicalizeAliasesPass());
866*da58b97aSjoerg MPM.addPass(NameAnonGlobalPass());
867*da58b97aSjoerg }
868*da58b97aSjoerg
addPGOInstrPasses(ModulePassManager & MPM,PassBuilder::OptimizationLevel Level,bool RunProfileGen,bool IsCS,std::string ProfileFile,std::string ProfileRemappingFile)869*da58b97aSjoerg void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
87006f32e7eSjoerg PassBuilder::OptimizationLevel Level,
87106f32e7eSjoerg bool RunProfileGen, bool IsCS,
87206f32e7eSjoerg std::string ProfileFile,
87306f32e7eSjoerg std::string ProfileRemappingFile) {
874*da58b97aSjoerg assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
875*da58b97aSjoerg if (!IsCS && !DisablePreInliner) {
87606f32e7eSjoerg InlineParams IP;
87706f32e7eSjoerg
878*da58b97aSjoerg IP.DefaultThreshold = PreInlineThreshold;
87906f32e7eSjoerg
880*da58b97aSjoerg // FIXME: The hint threshold has the same value used by the regular inliner
881*da58b97aSjoerg // when not optimzing for size. This should probably be lowered after
882*da58b97aSjoerg // performance testing.
88306f32e7eSjoerg // FIXME: this comment is cargo culted from the old pass manager, revisit).
884*da58b97aSjoerg IP.HintThreshold = Level.isOptimizingForSize() ? PreInlineThreshold : 325;
885*da58b97aSjoerg ModuleInlinerWrapperPass MIWP(IP);
886*da58b97aSjoerg CGSCCPassManager &CGPipeline = MIWP.getPM();
88706f32e7eSjoerg
88806f32e7eSjoerg FunctionPassManager FPM;
88906f32e7eSjoerg FPM.addPass(SROA());
89006f32e7eSjoerg FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
89106f32e7eSjoerg FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
89206f32e7eSjoerg FPM.addPass(InstCombinePass()); // Combine silly sequences.
89306f32e7eSjoerg invokePeepholeEPCallbacks(FPM, Level);
89406f32e7eSjoerg
89506f32e7eSjoerg CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
89606f32e7eSjoerg
897*da58b97aSjoerg MPM.addPass(std::move(MIWP));
89806f32e7eSjoerg
89906f32e7eSjoerg // Delete anything that is now dead to make sure that we don't instrument
90006f32e7eSjoerg // dead code. Instrumentation can end up keeping dead code around and
90106f32e7eSjoerg // dramatically increase code size.
90206f32e7eSjoerg MPM.addPass(GlobalDCEPass());
90306f32e7eSjoerg }
90406f32e7eSjoerg
90506f32e7eSjoerg if (!RunProfileGen) {
90606f32e7eSjoerg assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
90706f32e7eSjoerg MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
90806f32e7eSjoerg // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
90906f32e7eSjoerg // RequireAnalysisPass for PSI before subsequent non-module passes.
91006f32e7eSjoerg MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
91106f32e7eSjoerg return;
91206f32e7eSjoerg }
91306f32e7eSjoerg
91406f32e7eSjoerg // Perform PGO instrumentation.
91506f32e7eSjoerg MPM.addPass(PGOInstrumentationGen(IsCS));
91606f32e7eSjoerg
91706f32e7eSjoerg FunctionPassManager FPM;
918*da58b97aSjoerg // Disable header duplication in loop rotation at -Oz.
91906f32e7eSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(
920*da58b97aSjoerg LoopRotatePass(Level != OptimizationLevel::Oz), EnableMSSALoopDependency,
921*da58b97aSjoerg /*UseBlockFrequencyInfo=*/false));
92206f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
92306f32e7eSjoerg
92406f32e7eSjoerg // Add the profile lowering pass.
92506f32e7eSjoerg InstrProfOptions Options;
92606f32e7eSjoerg if (!ProfileFile.empty())
92706f32e7eSjoerg Options.InstrProfileOutput = ProfileFile;
92806f32e7eSjoerg // Do counter promotion at Level greater than O0.
92906f32e7eSjoerg Options.DoCounterPromotion = true;
93006f32e7eSjoerg Options.UseBFIInPromotion = IsCS;
93106f32e7eSjoerg MPM.addPass(InstrProfiling(Options, IsCS));
93206f32e7eSjoerg }
93306f32e7eSjoerg
addPGOInstrPassesForO0(ModulePassManager & MPM,bool RunProfileGen,bool IsCS,std::string ProfileFile,std::string ProfileRemappingFile)93406f32e7eSjoerg void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
935*da58b97aSjoerg bool RunProfileGen, bool IsCS,
936*da58b97aSjoerg std::string ProfileFile,
93706f32e7eSjoerg std::string ProfileRemappingFile) {
93806f32e7eSjoerg if (!RunProfileGen) {
93906f32e7eSjoerg assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
94006f32e7eSjoerg MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
94106f32e7eSjoerg // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
94206f32e7eSjoerg // RequireAnalysisPass for PSI before subsequent non-module passes.
94306f32e7eSjoerg MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
94406f32e7eSjoerg return;
94506f32e7eSjoerg }
94606f32e7eSjoerg
94706f32e7eSjoerg // Perform PGO instrumentation.
94806f32e7eSjoerg MPM.addPass(PGOInstrumentationGen(IsCS));
94906f32e7eSjoerg // Add the profile lowering pass.
95006f32e7eSjoerg InstrProfOptions Options;
95106f32e7eSjoerg if (!ProfileFile.empty())
95206f32e7eSjoerg Options.InstrProfileOutput = ProfileFile;
95306f32e7eSjoerg // Do not do counter promotion at O0.
95406f32e7eSjoerg Options.DoCounterPromotion = false;
95506f32e7eSjoerg Options.UseBFIInPromotion = IsCS;
95606f32e7eSjoerg MPM.addPass(InstrProfiling(Options, IsCS));
95706f32e7eSjoerg }
95806f32e7eSjoerg
95906f32e7eSjoerg static InlineParams
getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level)96006f32e7eSjoerg getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
961*da58b97aSjoerg return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
962*da58b97aSjoerg }
963*da58b97aSjoerg
964*da58b97aSjoerg ModuleInlinerWrapperPass
buildInlinerPipeline(OptimizationLevel Level,ThinOrFullLTOPhase Phase)965*da58b97aSjoerg PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
966*da58b97aSjoerg ThinOrFullLTOPhase Phase) {
967*da58b97aSjoerg InlineParams IP = getInlineParamsFromOptLevel(Level);
968*da58b97aSjoerg if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
969*da58b97aSjoerg PGOOpt->Action == PGOOptions::SampleUse)
970*da58b97aSjoerg IP.HotCallSiteThreshold = 0;
971*da58b97aSjoerg
972*da58b97aSjoerg if (PGOOpt)
973*da58b97aSjoerg IP.EnableDeferral = EnablePGOInlineDeferral;
974*da58b97aSjoerg
975*da58b97aSjoerg ModuleInlinerWrapperPass MIWP(IP, PerformMandatoryInliningsFirst,
976*da58b97aSjoerg UseInlineAdvisor, MaxDevirtIterations);
977*da58b97aSjoerg
978*da58b97aSjoerg // Require the GlobalsAA analysis for the module so we can query it within
979*da58b97aSjoerg // the CGSCC pipeline.
980*da58b97aSjoerg MIWP.addModulePass(RequireAnalysisPass<GlobalsAA, Module>());
981*da58b97aSjoerg // Invalidate AAManager so it can be recreated and pick up the newly available
982*da58b97aSjoerg // GlobalsAA.
983*da58b97aSjoerg MIWP.addModulePass(
984*da58b97aSjoerg createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));
985*da58b97aSjoerg
986*da58b97aSjoerg // Require the ProfileSummaryAnalysis for the module so we can query it within
987*da58b97aSjoerg // the inliner pass.
988*da58b97aSjoerg MIWP.addModulePass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
989*da58b97aSjoerg
990*da58b97aSjoerg // Now begin the main postorder CGSCC pipeline.
991*da58b97aSjoerg // FIXME: The current CGSCC pipeline has its origins in the legacy pass
992*da58b97aSjoerg // manager and trying to emulate its precise behavior. Much of this doesn't
993*da58b97aSjoerg // make a lot of sense and we should revisit the core CGSCC structure.
994*da58b97aSjoerg CGSCCPassManager &MainCGPipeline = MIWP.getPM();
995*da58b97aSjoerg
996*da58b97aSjoerg // Note: historically, the PruneEH pass was run first to deduce nounwind and
997*da58b97aSjoerg // generally clean up exception handling overhead. It isn't clear this is
998*da58b97aSjoerg // valuable as the inliner doesn't currently care whether it is inlining an
999*da58b97aSjoerg // invoke or a call.
1000*da58b97aSjoerg
1001*da58b97aSjoerg if (AttributorRun & AttributorRunOption::CGSCC)
1002*da58b97aSjoerg MainCGPipeline.addPass(AttributorCGSCCPass());
1003*da58b97aSjoerg
1004*da58b97aSjoerg if (PTO.Coroutines)
1005*da58b97aSjoerg MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
1006*da58b97aSjoerg
1007*da58b97aSjoerg // Now deduce any function attributes based in the current code.
1008*da58b97aSjoerg MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
1009*da58b97aSjoerg
1010*da58b97aSjoerg // When at O3 add argument promotion to the pass pipeline.
1011*da58b97aSjoerg // FIXME: It isn't at all clear why this should be limited to O3.
1012*da58b97aSjoerg if (Level == OptimizationLevel::O3)
1013*da58b97aSjoerg MainCGPipeline.addPass(ArgumentPromotionPass());
1014*da58b97aSjoerg
1015*da58b97aSjoerg // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
1016*da58b97aSjoerg // there are no OpenMP runtime calls present in the module.
1017*da58b97aSjoerg if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
1018*da58b97aSjoerg MainCGPipeline.addPass(OpenMPOptCGSCCPass());
1019*da58b97aSjoerg
1020*da58b97aSjoerg for (auto &C : CGSCCOptimizerLateEPCallbacks)
1021*da58b97aSjoerg C(MainCGPipeline, Level);
1022*da58b97aSjoerg
1023*da58b97aSjoerg // Lastly, add the core function simplification pipeline nested inside the
1024*da58b97aSjoerg // CGSCC walk.
1025*da58b97aSjoerg MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
1026*da58b97aSjoerg buildFunctionSimplificationPipeline(Level, Phase)));
1027*da58b97aSjoerg
1028*da58b97aSjoerg return MIWP;
102906f32e7eSjoerg }
103006f32e7eSjoerg
103106f32e7eSjoerg ModulePassManager
buildModuleSimplificationPipeline(OptimizationLevel Level,ThinOrFullLTOPhase Phase)103206f32e7eSjoerg PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
1033*da58b97aSjoerg ThinOrFullLTOPhase Phase) {
1034*da58b97aSjoerg ModulePassManager MPM;
1035*da58b97aSjoerg
1036*da58b97aSjoerg // Place pseudo probe instrumentation as the first pass of the pipeline to
1037*da58b97aSjoerg // minimize the impact of optimization changes.
1038*da58b97aSjoerg if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1039*da58b97aSjoerg Phase != ThinOrFullLTOPhase::ThinLTOPostLink)
1040*da58b97aSjoerg MPM.addPass(SampleProfileProbePass(TM));
104106f32e7eSjoerg
104206f32e7eSjoerg bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
104306f32e7eSjoerg
104406f32e7eSjoerg // In ThinLTO mode, when flattened profile is used, all the available
104506f32e7eSjoerg // profile information will be annotated in PreLink phase so there is
104606f32e7eSjoerg // no need to load the profile again in PostLink.
104706f32e7eSjoerg bool LoadSampleProfile =
104806f32e7eSjoerg HasSampleProfile &&
1049*da58b97aSjoerg !(FlattenedProfileUsed && Phase == ThinOrFullLTOPhase::ThinLTOPostLink);
105006f32e7eSjoerg
105106f32e7eSjoerg // During the ThinLTO backend phase we perform early indirect call promotion
105206f32e7eSjoerg // here, before globalopt. Otherwise imported available_externally functions
105306f32e7eSjoerg // look unreferenced and are removed. If we are going to load the sample
105406f32e7eSjoerg // profile then defer until later.
105506f32e7eSjoerg // TODO: See if we can move later and consolidate with the location where
105606f32e7eSjoerg // we perform ICP when we are loading a sample profile.
105706f32e7eSjoerg // TODO: We pass HasSampleProfile (whether there was a sample profile file
105806f32e7eSjoerg // passed to the compile) to the SamplePGO flag of ICP. This is used to
105906f32e7eSjoerg // determine whether the new direct calls are annotated with prof metadata.
106006f32e7eSjoerg // Ideally this should be determined from whether the IR is annotated with
106106f32e7eSjoerg // sample profile, and not whether the a sample profile was provided on the
106206f32e7eSjoerg // command line. E.g. for flattened profiles where we will not be reloading
106306f32e7eSjoerg // the sample profile in the ThinLTO backend, we ideally shouldn't have to
106406f32e7eSjoerg // provide the sample profile file.
1065*da58b97aSjoerg if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink && !LoadSampleProfile)
106606f32e7eSjoerg MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
106706f32e7eSjoerg
106806f32e7eSjoerg // Do basic inference of function attributes from known properties of system
106906f32e7eSjoerg // libraries and other oracles.
107006f32e7eSjoerg MPM.addPass(InferFunctionAttrsPass());
107106f32e7eSjoerg
107206f32e7eSjoerg // Create an early function pass manager to cleanup the output of the
107306f32e7eSjoerg // frontend.
1074*da58b97aSjoerg FunctionPassManager EarlyFPM;
1075*da58b97aSjoerg // Lower llvm.expect to metadata before attempting transforms.
1076*da58b97aSjoerg // Compare/branch metadata may alter the behavior of passes like SimplifyCFG.
1077*da58b97aSjoerg EarlyFPM.addPass(LowerExpectIntrinsicPass());
107806f32e7eSjoerg EarlyFPM.addPass(SimplifyCFGPass());
107906f32e7eSjoerg EarlyFPM.addPass(SROA());
108006f32e7eSjoerg EarlyFPM.addPass(EarlyCSEPass());
1081*da58b97aSjoerg if (PTO.Coroutines)
1082*da58b97aSjoerg EarlyFPM.addPass(CoroEarlyPass());
1083*da58b97aSjoerg if (Level == OptimizationLevel::O3)
108406f32e7eSjoerg EarlyFPM.addPass(CallSiteSplittingPass());
108506f32e7eSjoerg
108606f32e7eSjoerg // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
108706f32e7eSjoerg // to convert bitcast to direct calls so that they can be inlined during the
108806f32e7eSjoerg // profile annotation prepration step.
108906f32e7eSjoerg // More details about SamplePGO design can be found in:
109006f32e7eSjoerg // https://research.google.com/pubs/pub45290.html
109106f32e7eSjoerg // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
109206f32e7eSjoerg if (LoadSampleProfile)
109306f32e7eSjoerg EarlyFPM.addPass(InstCombinePass());
109406f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
109506f32e7eSjoerg
109606f32e7eSjoerg if (LoadSampleProfile) {
109706f32e7eSjoerg // Annotate sample profile right after early FPM to ensure freshness of
109806f32e7eSjoerg // the debug info.
109906f32e7eSjoerg MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1100*da58b97aSjoerg PGOOpt->ProfileRemappingFile, Phase));
110106f32e7eSjoerg // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
110206f32e7eSjoerg // RequireAnalysisPass for PSI before subsequent non-module passes.
110306f32e7eSjoerg MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1104*da58b97aSjoerg // Do not invoke ICP in the LTOPrelink phase as it makes it hard
1105*da58b97aSjoerg // for the profile annotation to be accurate in the LTO backend.
1106*da58b97aSjoerg if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink &&
1107*da58b97aSjoerg Phase != ThinOrFullLTOPhase::FullLTOPreLink)
110806f32e7eSjoerg // We perform early indirect call promotion here, before globalopt.
110906f32e7eSjoerg // This is important for the ThinLTO backend phase because otherwise
111006f32e7eSjoerg // imported available_externally functions look unreferenced and are
111106f32e7eSjoerg // removed.
1112*da58b97aSjoerg MPM.addPass(
1113*da58b97aSjoerg PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));
111406f32e7eSjoerg }
111506f32e7eSjoerg
1116*da58b97aSjoerg // Try to perform OpenMP specific optimizations on the module. This is a
1117*da58b97aSjoerg // (quick!) no-op if there are no OpenMP runtime calls present in the module.
1118*da58b97aSjoerg if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
1119*da58b97aSjoerg MPM.addPass(OpenMPOptPass());
1120*da58b97aSjoerg
1121*da58b97aSjoerg if (AttributorRun & AttributorRunOption::MODULE)
1122*da58b97aSjoerg MPM.addPass(AttributorPass());
1123*da58b97aSjoerg
1124*da58b97aSjoerg // Lower type metadata and the type.test intrinsic in the ThinLTO
1125*da58b97aSjoerg // post link pipeline after ICP. This is to enable usage of the type
1126*da58b97aSjoerg // tests in ICP sequences.
1127*da58b97aSjoerg if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink)
1128*da58b97aSjoerg MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1129*da58b97aSjoerg
1130*da58b97aSjoerg for (auto &C : PipelineEarlySimplificationEPCallbacks)
1131*da58b97aSjoerg C(MPM, Level);
1132*da58b97aSjoerg
113306f32e7eSjoerg // Interprocedural constant propagation now that basic cleanup has occurred
113406f32e7eSjoerg // and prior to optimizing globals.
113506f32e7eSjoerg // FIXME: This position in the pipeline hasn't been carefully considered in
113606f32e7eSjoerg // years, it should be re-analyzed.
113706f32e7eSjoerg MPM.addPass(IPSCCPPass());
113806f32e7eSjoerg
113906f32e7eSjoerg // Attach metadata to indirect call sites indicating the set of functions
114006f32e7eSjoerg // they may target at run-time. This should follow IPSCCP.
114106f32e7eSjoerg MPM.addPass(CalledValuePropagationPass());
114206f32e7eSjoerg
114306f32e7eSjoerg // Optimize globals to try and fold them into constants.
114406f32e7eSjoerg MPM.addPass(GlobalOptPass());
114506f32e7eSjoerg
114606f32e7eSjoerg // Promote any localized globals to SSA registers.
114706f32e7eSjoerg // FIXME: Should this instead by a run of SROA?
114806f32e7eSjoerg // FIXME: We should probably run instcombine and simplify-cfg afterward to
114906f32e7eSjoerg // delete control flows that are dead once globals have been folded to
115006f32e7eSjoerg // constants.
115106f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
115206f32e7eSjoerg
1153*da58b97aSjoerg // Remove any dead arguments exposed by cleanups and constant folding
115406f32e7eSjoerg // globals.
115506f32e7eSjoerg MPM.addPass(DeadArgumentEliminationPass());
115606f32e7eSjoerg
115706f32e7eSjoerg // Create a small function pass pipeline to cleanup after all the global
115806f32e7eSjoerg // optimizations.
1159*da58b97aSjoerg FunctionPassManager GlobalCleanupPM;
116006f32e7eSjoerg GlobalCleanupPM.addPass(InstCombinePass());
116106f32e7eSjoerg invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
116206f32e7eSjoerg
116306f32e7eSjoerg GlobalCleanupPM.addPass(SimplifyCFGPass());
116406f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
116506f32e7eSjoerg
116606f32e7eSjoerg // Add all the requested passes for instrumentation PGO, if requested.
1167*da58b97aSjoerg if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
116806f32e7eSjoerg (PGOOpt->Action == PGOOptions::IRInstr ||
116906f32e7eSjoerg PGOOpt->Action == PGOOptions::IRUse)) {
1170*da58b97aSjoerg addPGOInstrPasses(MPM, Level,
117106f32e7eSjoerg /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
117206f32e7eSjoerg /* IsCS */ false, PGOOpt->ProfileFile,
117306f32e7eSjoerg PGOOpt->ProfileRemappingFile);
117406f32e7eSjoerg MPM.addPass(PGOIndirectCallPromotion(false, false));
117506f32e7eSjoerg }
1176*da58b97aSjoerg if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
117706f32e7eSjoerg PGOOpt->CSAction == PGOOptions::CSIRInstr)
117806f32e7eSjoerg MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
117906f32e7eSjoerg
118006f32e7eSjoerg // Synthesize function entry counts for non-PGO compilation.
118106f32e7eSjoerg if (EnableSyntheticCounts && !PGOOpt)
118206f32e7eSjoerg MPM.addPass(SyntheticCountsPropagation());
118306f32e7eSjoerg
1184*da58b97aSjoerg MPM.addPass(buildInlinerPipeline(Level, Phase));
118506f32e7eSjoerg
1186*da58b97aSjoerg if (EnableMemProfiler && Phase != ThinOrFullLTOPhase::ThinLTOPreLink) {
1187*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
1188*da58b97aSjoerg MPM.addPass(ModuleMemProfilerPass());
1189*da58b97aSjoerg }
119006f32e7eSjoerg
119106f32e7eSjoerg return MPM;
119206f32e7eSjoerg }
119306f32e7eSjoerg
1194*da58b97aSjoerg /// TODO: Should LTO cause any differences to this set of passes?
addVectorPasses(OptimizationLevel Level,FunctionPassManager & FPM,bool IsLTO)1195*da58b97aSjoerg void PassBuilder::addVectorPasses(OptimizationLevel Level,
1196*da58b97aSjoerg FunctionPassManager &FPM, bool IsLTO) {
1197*da58b97aSjoerg FPM.addPass(LoopVectorizePass(
1198*da58b97aSjoerg LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1199*da58b97aSjoerg
1200*da58b97aSjoerg if (IsLTO) {
1201*da58b97aSjoerg // The vectorizer may have significantly shortened a loop body; unroll
1202*da58b97aSjoerg // again. Unroll small loops to hide loop backedge latency and saturate any
1203*da58b97aSjoerg // parallel execution resources of an out-of-order processor. We also then
1204*da58b97aSjoerg // need to clean up redundancies and loop invariant code.
1205*da58b97aSjoerg // FIXME: It would be really good to use a loop-integrated instruction
1206*da58b97aSjoerg // combiner for cleanup here so that the unrolling and LICM can be pipelined
1207*da58b97aSjoerg // across the loop nests.
1208*da58b97aSjoerg // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1209*da58b97aSjoerg if (EnableUnrollAndJam && PTO.LoopUnrolling)
1210*da58b97aSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(
1211*da58b97aSjoerg LoopUnrollAndJamPass(Level.getSpeedupLevel())));
1212*da58b97aSjoerg FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
1213*da58b97aSjoerg Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1214*da58b97aSjoerg PTO.ForgetAllSCEVInLoopUnroll)));
1215*da58b97aSjoerg FPM.addPass(WarnMissedTransformationsPass());
1216*da58b97aSjoerg }
1217*da58b97aSjoerg
1218*da58b97aSjoerg if (!IsLTO) {
1219*da58b97aSjoerg // Eliminate loads by forwarding stores from the previous iteration to loads
1220*da58b97aSjoerg // of the current iteration.
1221*da58b97aSjoerg FPM.addPass(LoopLoadEliminationPass());
1222*da58b97aSjoerg }
1223*da58b97aSjoerg // Cleanup after the loop optimization passes.
1224*da58b97aSjoerg FPM.addPass(InstCombinePass());
1225*da58b97aSjoerg
1226*da58b97aSjoerg if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1227*da58b97aSjoerg // At higher optimization levels, try to clean up any runtime overlap and
1228*da58b97aSjoerg // alignment checks inserted by the vectorizer. We want to track correlated
1229*da58b97aSjoerg // runtime checks for two inner loops in the same outer loop, fold any
1230*da58b97aSjoerg // common computations, hoist loop-invariant aspects out of any outer loop,
1231*da58b97aSjoerg // and unswitch the runtime checks if possible. Once hoisted, we may have
1232*da58b97aSjoerg // dead (or speculatable) control flows or more combining opportunities.
1233*da58b97aSjoerg FPM.addPass(EarlyCSEPass());
1234*da58b97aSjoerg FPM.addPass(CorrelatedValuePropagationPass());
1235*da58b97aSjoerg FPM.addPass(InstCombinePass());
1236*da58b97aSjoerg LoopPassManager LPM;
1237*da58b97aSjoerg LPM.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
1238*da58b97aSjoerg LPM.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
1239*da58b97aSjoerg OptimizationLevel::O3));
1240*da58b97aSjoerg FPM.addPass(
1241*da58b97aSjoerg RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
1242*da58b97aSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(
1243*da58b97aSjoerg std::move(LPM), EnableMSSALoopDependency,
1244*da58b97aSjoerg /*UseBlockFrequencyInfo=*/true));
1245*da58b97aSjoerg FPM.addPass(SimplifyCFGPass());
1246*da58b97aSjoerg FPM.addPass(InstCombinePass());
1247*da58b97aSjoerg }
1248*da58b97aSjoerg
1249*da58b97aSjoerg if (IsLTO) {
1250*da58b97aSjoerg FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true)));
1251*da58b97aSjoerg } else {
1252*da58b97aSjoerg // Now that we've formed fast to execute loop structures, we do further
1253*da58b97aSjoerg // optimizations. These are run afterward as they might block doing complex
1254*da58b97aSjoerg // analyses and transforms such as what are needed for loop vectorization.
1255*da58b97aSjoerg
1256*da58b97aSjoerg // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1257*da58b97aSjoerg // GVN, loop transforms, and others have already run, so it's now better to
1258*da58b97aSjoerg // convert to more optimized IR using more aggressive simplify CFG options.
1259*da58b97aSjoerg // The extra sinking transform can create larger basic blocks, so do this
1260*da58b97aSjoerg // before SLP vectorization.
1261*da58b97aSjoerg FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
1262*da58b97aSjoerg .forwardSwitchCondToPhi(true)
1263*da58b97aSjoerg .convertSwitchToLookupTable(true)
1264*da58b97aSjoerg .needCanonicalLoops(false)
1265*da58b97aSjoerg .hoistCommonInsts(true)
1266*da58b97aSjoerg .sinkCommonInsts(true)));
1267*da58b97aSjoerg }
1268*da58b97aSjoerg if (IsLTO) {
1269*da58b97aSjoerg FPM.addPass(SCCPPass());
1270*da58b97aSjoerg FPM.addPass(InstCombinePass());
1271*da58b97aSjoerg FPM.addPass(BDCEPass());
1272*da58b97aSjoerg }
1273*da58b97aSjoerg
1274*da58b97aSjoerg // Optimize parallel scalar instruction chains into SIMD instructions.
1275*da58b97aSjoerg if (PTO.SLPVectorization) {
1276*da58b97aSjoerg FPM.addPass(SLPVectorizerPass());
1277*da58b97aSjoerg if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1278*da58b97aSjoerg FPM.addPass(EarlyCSEPass());
1279*da58b97aSjoerg }
1280*da58b97aSjoerg }
1281*da58b97aSjoerg // Enhance/cleanup vector code.
1282*da58b97aSjoerg FPM.addPass(VectorCombinePass());
1283*da58b97aSjoerg
1284*da58b97aSjoerg if (!IsLTO) {
1285*da58b97aSjoerg FPM.addPass(InstCombinePass());
1286*da58b97aSjoerg // Unroll small loops to hide loop backedge latency and saturate any
1287*da58b97aSjoerg // parallel execution resources of an out-of-order processor. We also then
1288*da58b97aSjoerg // need to clean up redundancies and loop invariant code.
1289*da58b97aSjoerg // FIXME: It would be really good to use a loop-integrated instruction
1290*da58b97aSjoerg // combiner for cleanup here so that the unrolling and LICM can be pipelined
1291*da58b97aSjoerg // across the loop nests.
1292*da58b97aSjoerg // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1293*da58b97aSjoerg if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1294*da58b97aSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(
1295*da58b97aSjoerg LoopUnrollAndJamPass(Level.getSpeedupLevel())));
1296*da58b97aSjoerg }
1297*da58b97aSjoerg FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
1298*da58b97aSjoerg Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1299*da58b97aSjoerg PTO.ForgetAllSCEVInLoopUnroll)));
1300*da58b97aSjoerg FPM.addPass(WarnMissedTransformationsPass());
1301*da58b97aSjoerg FPM.addPass(InstCombinePass());
1302*da58b97aSjoerg FPM.addPass(
1303*da58b97aSjoerg RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
1304*da58b97aSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(
1305*da58b97aSjoerg LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
1306*da58b97aSjoerg EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true));
1307*da58b97aSjoerg }
1308*da58b97aSjoerg
1309*da58b97aSjoerg // Now that we've vectorized and unrolled loops, we may have more refined
1310*da58b97aSjoerg // alignment information, try to re-derive it here.
1311*da58b97aSjoerg FPM.addPass(AlignmentFromAssumptionsPass());
1312*da58b97aSjoerg
1313*da58b97aSjoerg if (IsLTO)
1314*da58b97aSjoerg FPM.addPass(InstCombinePass());
1315*da58b97aSjoerg }
1316*da58b97aSjoerg
1317*da58b97aSjoerg ModulePassManager
buildModuleOptimizationPipeline(OptimizationLevel Level,bool LTOPreLink)1318*da58b97aSjoerg PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
1319*da58b97aSjoerg bool LTOPreLink) {
1320*da58b97aSjoerg ModulePassManager MPM;
132106f32e7eSjoerg
132206f32e7eSjoerg // Optimize globals now that the module is fully simplified.
132306f32e7eSjoerg MPM.addPass(GlobalOptPass());
132406f32e7eSjoerg MPM.addPass(GlobalDCEPass());
132506f32e7eSjoerg
132606f32e7eSjoerg // Run partial inlining pass to partially inline functions that have
132706f32e7eSjoerg // large bodies.
132806f32e7eSjoerg if (RunPartialInlining)
132906f32e7eSjoerg MPM.addPass(PartialInlinerPass());
133006f32e7eSjoerg
133106f32e7eSjoerg // Remove avail extern fns and globals definitions since we aren't compiling
133206f32e7eSjoerg // an object file for later LTO. For LTO we want to preserve these so they
133306f32e7eSjoerg // are eligible for inlining at link-time. Note if they are unreferenced they
133406f32e7eSjoerg // will be removed by GlobalDCE later, so this only impacts referenced
133506f32e7eSjoerg // available externally globals. Eventually they will be suppressed during
133606f32e7eSjoerg // codegen, but eliminating here enables more opportunity for GlobalDCE as it
133706f32e7eSjoerg // may make globals referenced by available external functions dead and saves
133806f32e7eSjoerg // running remaining passes on the eliminated functions. These should be
133906f32e7eSjoerg // preserved during prelinking for link-time inlining decisions.
134006f32e7eSjoerg if (!LTOPreLink)
134106f32e7eSjoerg MPM.addPass(EliminateAvailableExternallyPass());
134206f32e7eSjoerg
134306f32e7eSjoerg if (EnableOrderFileInstrumentation)
134406f32e7eSjoerg MPM.addPass(InstrOrderFilePass());
134506f32e7eSjoerg
134606f32e7eSjoerg // Do RPO function attribute inference across the module to forward-propagate
134706f32e7eSjoerg // attributes where applicable.
134806f32e7eSjoerg // FIXME: Is this really an optimization rather than a canonicalization?
134906f32e7eSjoerg MPM.addPass(ReversePostOrderFunctionAttrsPass());
135006f32e7eSjoerg
135106f32e7eSjoerg // Do a post inline PGO instrumentation and use pass. This is a context
135206f32e7eSjoerg // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
135306f32e7eSjoerg // cross-module inline has not been done yet. The context sensitive
135406f32e7eSjoerg // instrumentation is after all the inlines are done.
135506f32e7eSjoerg if (!LTOPreLink && PGOOpt) {
135606f32e7eSjoerg if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1357*da58b97aSjoerg addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
135806f32e7eSjoerg /* IsCS */ true, PGOOpt->CSProfileGenFile,
135906f32e7eSjoerg PGOOpt->ProfileRemappingFile);
136006f32e7eSjoerg else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1361*da58b97aSjoerg addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
136206f32e7eSjoerg /* IsCS */ true, PGOOpt->ProfileFile,
136306f32e7eSjoerg PGOOpt->ProfileRemappingFile);
136406f32e7eSjoerg }
136506f32e7eSjoerg
136606f32e7eSjoerg // Re-require GloblasAA here prior to function passes. This is particularly
136706f32e7eSjoerg // useful as the above will have inlined, DCE'ed, and function-attr
136806f32e7eSjoerg // propagated everything. We should at this point have a reasonably minimal
136906f32e7eSjoerg // and richly annotated call graph. By computing aliasing and mod/ref
137006f32e7eSjoerg // information for all local globals here, the late loop passes and notably
137106f32e7eSjoerg // the vectorizer will be able to use them to help recognize vectorizable
137206f32e7eSjoerg // memory operations.
137306f32e7eSjoerg MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
137406f32e7eSjoerg
1375*da58b97aSjoerg FunctionPassManager OptimizePM;
137606f32e7eSjoerg OptimizePM.addPass(Float2IntPass());
137706f32e7eSjoerg OptimizePM.addPass(LowerConstantIntrinsicsPass());
137806f32e7eSjoerg
1379*da58b97aSjoerg if (EnableMatrix) {
1380*da58b97aSjoerg OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1381*da58b97aSjoerg OptimizePM.addPass(EarlyCSEPass());
1382*da58b97aSjoerg }
1383*da58b97aSjoerg
138406f32e7eSjoerg // FIXME: We need to run some loop optimizations to re-rotate loops after
138506f32e7eSjoerg // simplify-cfg and others undo their rotation.
138606f32e7eSjoerg
138706f32e7eSjoerg // Optimize the loop execution. These passes operate on entire loop nests
138806f32e7eSjoerg // rather than on each loop in an inside-out manner, and so they are actually
138906f32e7eSjoerg // function passes.
139006f32e7eSjoerg
139106f32e7eSjoerg for (auto &C : VectorizerStartEPCallbacks)
139206f32e7eSjoerg C(OptimizePM, Level);
139306f32e7eSjoerg
139406f32e7eSjoerg // First rotate loops that may have been un-rotated by prior passes.
1395*da58b97aSjoerg // Disable header duplication at -Oz.
139606f32e7eSjoerg OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1397*da58b97aSjoerg LoopRotatePass(Level != OptimizationLevel::Oz, LTOPreLink),
1398*da58b97aSjoerg EnableMSSALoopDependency,
1399*da58b97aSjoerg /*UseBlockFrequencyInfo=*/false));
140006f32e7eSjoerg
140106f32e7eSjoerg // Distribute loops to allow partial vectorization. I.e. isolate dependences
140206f32e7eSjoerg // into separate loop that would otherwise inhibit vectorization. This is
140306f32e7eSjoerg // currently only performed for loops marked with the metadata
140406f32e7eSjoerg // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
140506f32e7eSjoerg OptimizePM.addPass(LoopDistributePass());
140606f32e7eSjoerg
1407*da58b97aSjoerg // Populates the VFABI attribute with the scalar-to-vector mappings
1408*da58b97aSjoerg // from the TargetLibraryInfo.
1409*da58b97aSjoerg OptimizePM.addPass(InjectTLIMappings());
141006f32e7eSjoerg
1411*da58b97aSjoerg addVectorPasses(Level, OptimizePM, /* IsLTO */ false);
141206f32e7eSjoerg
141306f32e7eSjoerg // Split out cold code. Splitting is done late to avoid hiding context from
141406f32e7eSjoerg // other optimizations and inadvertently regressing performance. The tradeoff
141506f32e7eSjoerg // is that this has a higher code size cost than splitting early.
141606f32e7eSjoerg if (EnableHotColdSplit && !LTOPreLink)
141706f32e7eSjoerg MPM.addPass(HotColdSplittingPass());
141806f32e7eSjoerg
1419*da58b97aSjoerg // Search the code for similar regions of code. If enough similar regions can
1420*da58b97aSjoerg // be found where extracting the regions into their own function will decrease
1421*da58b97aSjoerg // the size of the program, we extract the regions, a deduplicate the
1422*da58b97aSjoerg // structurally similar regions.
1423*da58b97aSjoerg if (EnableIROutliner)
1424*da58b97aSjoerg MPM.addPass(IROutlinerPass());
1425*da58b97aSjoerg
1426*da58b97aSjoerg // Merge functions if requested.
1427*da58b97aSjoerg if (PTO.MergeFunctions)
1428*da58b97aSjoerg MPM.addPass(MergeFunctionsPass());
1429*da58b97aSjoerg
143006f32e7eSjoerg // LoopSink pass sinks instructions hoisted by LICM, which serves as a
143106f32e7eSjoerg // canonicalization pass that enables other optimizations. As a result,
143206f32e7eSjoerg // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
143306f32e7eSjoerg // result too early.
143406f32e7eSjoerg OptimizePM.addPass(LoopSinkPass());
143506f32e7eSjoerg
143606f32e7eSjoerg // And finally clean up LCSSA form before generating code.
143706f32e7eSjoerg OptimizePM.addPass(InstSimplifyPass());
143806f32e7eSjoerg
143906f32e7eSjoerg // This hoists/decomposes div/rem ops. It should run after other sink/hoist
144006f32e7eSjoerg // passes to avoid re-sinking, but before SimplifyCFG because it can allow
144106f32e7eSjoerg // flattening of blocks.
144206f32e7eSjoerg OptimizePM.addPass(DivRemPairsPass());
144306f32e7eSjoerg
144406f32e7eSjoerg // LoopSink (and other loop passes since the last simplifyCFG) might have
144506f32e7eSjoerg // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
144606f32e7eSjoerg OptimizePM.addPass(SimplifyCFGPass());
144706f32e7eSjoerg
144806f32e7eSjoerg // Optimize PHIs by speculating around them when profitable. Note that this
144906f32e7eSjoerg // pass needs to be run after any PRE or similar pass as it is essentially
145006f32e7eSjoerg // inserting redundancies into the program. This even includes SimplifyCFG.
145106f32e7eSjoerg OptimizePM.addPass(SpeculateAroundPHIsPass());
145206f32e7eSjoerg
1453*da58b97aSjoerg if (PTO.Coroutines)
1454*da58b97aSjoerg OptimizePM.addPass(CoroCleanupPass());
145506f32e7eSjoerg
145606f32e7eSjoerg // Add the core optimizing pipeline.
145706f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
145806f32e7eSjoerg
1459*da58b97aSjoerg for (auto &C : OptimizerLastEPCallbacks)
1460*da58b97aSjoerg C(MPM, Level);
1461*da58b97aSjoerg
1462*da58b97aSjoerg if (PTO.CallGraphProfile)
146306f32e7eSjoerg MPM.addPass(CGProfilePass());
146406f32e7eSjoerg
146506f32e7eSjoerg // Now we need to do some global optimization transforms.
146606f32e7eSjoerg // FIXME: It would seem like these should come first in the optimization
146706f32e7eSjoerg // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
146806f32e7eSjoerg // ordering here.
146906f32e7eSjoerg MPM.addPass(GlobalDCEPass());
147006f32e7eSjoerg MPM.addPass(ConstantMergePass());
147106f32e7eSjoerg
1472*da58b97aSjoerg // TODO: Relative look table converter pass caused an issue when full lto is
1473*da58b97aSjoerg // enabled. See https://reviews.llvm.org/D94355 for more details.
1474*da58b97aSjoerg // Until the issue fixed, disable this pass during pre-linking phase.
1475*da58b97aSjoerg if (!LTOPreLink)
1476*da58b97aSjoerg MPM.addPass(RelLookupTableConverterPass());
1477*da58b97aSjoerg
147806f32e7eSjoerg return MPM;
147906f32e7eSjoerg }
148006f32e7eSjoerg
148106f32e7eSjoerg ModulePassManager
buildPerModuleDefaultPipeline(OptimizationLevel Level,bool LTOPreLink)148206f32e7eSjoerg PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1483*da58b97aSjoerg bool LTOPreLink) {
1484*da58b97aSjoerg assert(Level != OptimizationLevel::O0 &&
1485*da58b97aSjoerg "Must request optimizations for the default pipeline!");
148606f32e7eSjoerg
1487*da58b97aSjoerg ModulePassManager MPM;
1488*da58b97aSjoerg
1489*da58b97aSjoerg // Convert @llvm.global.annotations to !annotation metadata.
1490*da58b97aSjoerg MPM.addPass(Annotation2MetadataPass());
149106f32e7eSjoerg
149206f32e7eSjoerg // Force any function attributes we want the rest of the pipeline to observe.
149306f32e7eSjoerg MPM.addPass(ForceFunctionAttrsPass());
149406f32e7eSjoerg
149506f32e7eSjoerg // Apply module pipeline start EP callback.
149606f32e7eSjoerg for (auto &C : PipelineStartEPCallbacks)
1497*da58b97aSjoerg C(MPM, Level);
149806f32e7eSjoerg
1499*da58b97aSjoerg if (PGOOpt && PGOOpt->DebugInfoForProfiling)
150006f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
150106f32e7eSjoerg
150206f32e7eSjoerg // Add the core simplification pipeline.
1503*da58b97aSjoerg MPM.addPass(buildModuleSimplificationPipeline(
1504*da58b97aSjoerg Level, LTOPreLink ? ThinOrFullLTOPhase::FullLTOPreLink
1505*da58b97aSjoerg : ThinOrFullLTOPhase::None));
150606f32e7eSjoerg
150706f32e7eSjoerg // Now add the optimization pipeline.
1508*da58b97aSjoerg MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPreLink));
1509*da58b97aSjoerg
1510*da58b97aSjoerg if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
1511*da58b97aSjoerg MPM.addPass(PseudoProbeUpdatePass());
1512*da58b97aSjoerg
1513*da58b97aSjoerg // Emit annotation remarks.
1514*da58b97aSjoerg addAnnotationRemarksPass(MPM);
1515*da58b97aSjoerg
1516*da58b97aSjoerg if (LTOPreLink)
1517*da58b97aSjoerg addRequiredLTOPreLinkPasses(MPM);
151806f32e7eSjoerg
151906f32e7eSjoerg return MPM;
152006f32e7eSjoerg }
152106f32e7eSjoerg
152206f32e7eSjoerg ModulePassManager
buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level)1523*da58b97aSjoerg PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1524*da58b97aSjoerg assert(Level != OptimizationLevel::O0 &&
1525*da58b97aSjoerg "Must request optimizations for the default pipeline!");
152606f32e7eSjoerg
1527*da58b97aSjoerg ModulePassManager MPM;
1528*da58b97aSjoerg
1529*da58b97aSjoerg // Convert @llvm.global.annotations to !annotation metadata.
1530*da58b97aSjoerg MPM.addPass(Annotation2MetadataPass());
153106f32e7eSjoerg
153206f32e7eSjoerg // Force any function attributes we want the rest of the pipeline to observe.
153306f32e7eSjoerg MPM.addPass(ForceFunctionAttrsPass());
153406f32e7eSjoerg
1535*da58b97aSjoerg if (PGOOpt && PGOOpt->DebugInfoForProfiling)
153606f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
153706f32e7eSjoerg
153806f32e7eSjoerg // Apply module pipeline start EP callback.
153906f32e7eSjoerg for (auto &C : PipelineStartEPCallbacks)
1540*da58b97aSjoerg C(MPM, Level);
154106f32e7eSjoerg
154206f32e7eSjoerg // If we are planning to perform ThinLTO later, we don't bloat the code with
154306f32e7eSjoerg // unrolling/vectorization/... now. Just simplify the module as much as we
154406f32e7eSjoerg // can.
1545*da58b97aSjoerg MPM.addPass(buildModuleSimplificationPipeline(
1546*da58b97aSjoerg Level, ThinOrFullLTOPhase::ThinLTOPreLink));
154706f32e7eSjoerg
154806f32e7eSjoerg // Run partial inlining pass to partially inline functions that have
154906f32e7eSjoerg // large bodies.
155006f32e7eSjoerg // FIXME: It isn't clear whether this is really the right place to run this
155106f32e7eSjoerg // in ThinLTO. Because there is another canonicalization and simplification
155206f32e7eSjoerg // phase that will run after the thin link, running this here ends up with
155306f32e7eSjoerg // less information than will be available later and it may grow functions in
155406f32e7eSjoerg // ways that aren't beneficial.
155506f32e7eSjoerg if (RunPartialInlining)
155606f32e7eSjoerg MPM.addPass(PartialInlinerPass());
155706f32e7eSjoerg
155806f32e7eSjoerg // Reduce the size of the IR as much as possible.
155906f32e7eSjoerg MPM.addPass(GlobalOptPass());
156006f32e7eSjoerg
1561*da58b97aSjoerg // Module simplification splits coroutines, but does not fully clean up
1562*da58b97aSjoerg // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up
1563*da58b97aSjoerg // on these, we schedule the cleanup here.
1564*da58b97aSjoerg if (PTO.Coroutines)
1565*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
1566*da58b97aSjoerg
1567*da58b97aSjoerg if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
1568*da58b97aSjoerg MPM.addPass(PseudoProbeUpdatePass());
1569*da58b97aSjoerg
1570*da58b97aSjoerg // Handle OptimizerLastEPCallbacks added by clang on PreLink. Actual
1571*da58b97aSjoerg // optimization is going to be done in PostLink stage, but clang can't
1572*da58b97aSjoerg // add callbacks there in case of in-process ThinLTO called by linker.
1573*da58b97aSjoerg for (auto &C : OptimizerLastEPCallbacks)
1574*da58b97aSjoerg C(MPM, Level);
1575*da58b97aSjoerg
1576*da58b97aSjoerg // Emit annotation remarks.
1577*da58b97aSjoerg addAnnotationRemarksPass(MPM);
1578*da58b97aSjoerg
1579*da58b97aSjoerg addRequiredLTOPreLinkPasses(MPM);
1580*da58b97aSjoerg
158106f32e7eSjoerg return MPM;
158206f32e7eSjoerg }
158306f32e7eSjoerg
buildThinLTODefaultPipeline(OptimizationLevel Level,const ModuleSummaryIndex * ImportSummary)158406f32e7eSjoerg ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1585*da58b97aSjoerg OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1586*da58b97aSjoerg ModulePassManager MPM;
1587*da58b97aSjoerg
1588*da58b97aSjoerg // Convert @llvm.global.annotations to !annotation metadata.
1589*da58b97aSjoerg MPM.addPass(Annotation2MetadataPass());
159006f32e7eSjoerg
159106f32e7eSjoerg if (ImportSummary) {
159206f32e7eSjoerg // These passes import type identifier resolutions for whole-program
159306f32e7eSjoerg // devirtualization and CFI. They must run early because other passes may
159406f32e7eSjoerg // disturb the specific instruction patterns that these passes look for,
159506f32e7eSjoerg // creating dependencies on resolutions that may not appear in the summary.
159606f32e7eSjoerg //
159706f32e7eSjoerg // For example, GVN may transform the pattern assume(type.test) appearing in
159806f32e7eSjoerg // two basic blocks into assume(phi(type.test, type.test)), which would
159906f32e7eSjoerg // transform a dependency on a WPD resolution into a dependency on a type
160006f32e7eSjoerg // identifier resolution for CFI.
160106f32e7eSjoerg //
160206f32e7eSjoerg // Also, WPD has access to more precise information than ICP and can
160306f32e7eSjoerg // devirtualize more effectively, so it should operate on the IR first.
160406f32e7eSjoerg //
160506f32e7eSjoerg // The WPD and LowerTypeTest passes need to run at -O0 to lower type
160606f32e7eSjoerg // metadata and intrinsics.
160706f32e7eSjoerg MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
160806f32e7eSjoerg MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
160906f32e7eSjoerg }
161006f32e7eSjoerg
1611*da58b97aSjoerg if (Level == OptimizationLevel::O0) {
1612*da58b97aSjoerg // Run a second time to clean up any type tests left behind by WPD for use
1613*da58b97aSjoerg // in ICP.
1614*da58b97aSjoerg MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1615*da58b97aSjoerg // Drop available_externally and unreferenced globals. This is necessary
1616*da58b97aSjoerg // with ThinLTO in order to avoid leaving undefined references to dead
1617*da58b97aSjoerg // globals in the object file.
1618*da58b97aSjoerg MPM.addPass(EliminateAvailableExternallyPass());
1619*da58b97aSjoerg MPM.addPass(GlobalDCEPass());
162006f32e7eSjoerg return MPM;
1621*da58b97aSjoerg }
162206f32e7eSjoerg
162306f32e7eSjoerg // Force any function attributes we want the rest of the pipeline to observe.
162406f32e7eSjoerg MPM.addPass(ForceFunctionAttrsPass());
162506f32e7eSjoerg
162606f32e7eSjoerg // Add the core simplification pipeline.
1627*da58b97aSjoerg MPM.addPass(buildModuleSimplificationPipeline(
1628*da58b97aSjoerg Level, ThinOrFullLTOPhase::ThinLTOPostLink));
162906f32e7eSjoerg
163006f32e7eSjoerg // Now add the optimization pipeline.
1631*da58b97aSjoerg MPM.addPass(buildModuleOptimizationPipeline(Level));
1632*da58b97aSjoerg
1633*da58b97aSjoerg // Emit annotation remarks.
1634*da58b97aSjoerg addAnnotationRemarksPass(MPM);
163506f32e7eSjoerg
163606f32e7eSjoerg return MPM;
163706f32e7eSjoerg }
163806f32e7eSjoerg
163906f32e7eSjoerg ModulePassManager
buildLTOPreLinkDefaultPipeline(OptimizationLevel Level)1640*da58b97aSjoerg PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
1641*da58b97aSjoerg assert(Level != OptimizationLevel::O0 &&
1642*da58b97aSjoerg "Must request optimizations for the default pipeline!");
164306f32e7eSjoerg // FIXME: We should use a customized pre-link pipeline!
1644*da58b97aSjoerg return buildPerModuleDefaultPipeline(Level,
164506f32e7eSjoerg /* LTOPreLink */ true);
164606f32e7eSjoerg }
164706f32e7eSjoerg
164806f32e7eSjoerg ModulePassManager
buildLTODefaultPipeline(OptimizationLevel Level,ModuleSummaryIndex * ExportSummary)1649*da58b97aSjoerg PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
165006f32e7eSjoerg ModuleSummaryIndex *ExportSummary) {
1651*da58b97aSjoerg ModulePassManager MPM;
165206f32e7eSjoerg
1653*da58b97aSjoerg // Convert @llvm.global.annotations to !annotation metadata.
1654*da58b97aSjoerg MPM.addPass(Annotation2MetadataPass());
1655*da58b97aSjoerg
1656*da58b97aSjoerg if (Level == OptimizationLevel::O0) {
165706f32e7eSjoerg // The WPD and LowerTypeTest passes need to run at -O0 to lower type
165806f32e7eSjoerg // metadata and intrinsics.
165906f32e7eSjoerg MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
166006f32e7eSjoerg MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1661*da58b97aSjoerg // Run a second time to clean up any type tests left behind by WPD for use
1662*da58b97aSjoerg // in ICP.
1663*da58b97aSjoerg MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1664*da58b97aSjoerg
1665*da58b97aSjoerg // Emit annotation remarks.
1666*da58b97aSjoerg addAnnotationRemarksPass(MPM);
1667*da58b97aSjoerg
166806f32e7eSjoerg return MPM;
166906f32e7eSjoerg }
167006f32e7eSjoerg
167106f32e7eSjoerg if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
167206f32e7eSjoerg // Load sample profile before running the LTO optimization pipeline.
167306f32e7eSjoerg MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
167406f32e7eSjoerg PGOOpt->ProfileRemappingFile,
1675*da58b97aSjoerg ThinOrFullLTOPhase::FullLTOPostLink));
167606f32e7eSjoerg // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
167706f32e7eSjoerg // RequireAnalysisPass for PSI before subsequent non-module passes.
167806f32e7eSjoerg MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
167906f32e7eSjoerg }
168006f32e7eSjoerg
168106f32e7eSjoerg // Remove unused virtual tables to improve the quality of code generated by
168206f32e7eSjoerg // whole-program devirtualization and bitset lowering.
168306f32e7eSjoerg MPM.addPass(GlobalDCEPass());
168406f32e7eSjoerg
168506f32e7eSjoerg // Force any function attributes we want the rest of the pipeline to observe.
168606f32e7eSjoerg MPM.addPass(ForceFunctionAttrsPass());
168706f32e7eSjoerg
168806f32e7eSjoerg // Do basic inference of function attributes from known properties of system
168906f32e7eSjoerg // libraries and other oracles.
169006f32e7eSjoerg MPM.addPass(InferFunctionAttrsPass());
169106f32e7eSjoerg
1692*da58b97aSjoerg if (Level.getSpeedupLevel() > 1) {
1693*da58b97aSjoerg FunctionPassManager EarlyFPM;
169406f32e7eSjoerg EarlyFPM.addPass(CallSiteSplittingPass());
169506f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
169606f32e7eSjoerg
169706f32e7eSjoerg // Indirect call promotion. This should promote all the targets that are
169806f32e7eSjoerg // left by the earlier promotion pass that promotes intra-module targets.
169906f32e7eSjoerg // This two-step promotion is to save the compile time. For LTO, it should
170006f32e7eSjoerg // produce the same result as if we only do promotion here.
170106f32e7eSjoerg MPM.addPass(PGOIndirectCallPromotion(
170206f32e7eSjoerg true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
170306f32e7eSjoerg // Propagate constants at call sites into the functions they call. This
170406f32e7eSjoerg // opens opportunities for globalopt (and inlining) by substituting function
170506f32e7eSjoerg // pointers passed as arguments to direct uses of functions.
170606f32e7eSjoerg MPM.addPass(IPSCCPPass());
170706f32e7eSjoerg
170806f32e7eSjoerg // Attach metadata to indirect call sites indicating the set of functions
170906f32e7eSjoerg // they may target at run-time. This should follow IPSCCP.
171006f32e7eSjoerg MPM.addPass(CalledValuePropagationPass());
171106f32e7eSjoerg }
171206f32e7eSjoerg
171306f32e7eSjoerg // Now deduce any function attributes based in the current code.
171406f32e7eSjoerg MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
171506f32e7eSjoerg PostOrderFunctionAttrsPass()));
171606f32e7eSjoerg
171706f32e7eSjoerg // Do RPO function attribute inference across the module to forward-propagate
171806f32e7eSjoerg // attributes where applicable.
171906f32e7eSjoerg // FIXME: Is this really an optimization rather than a canonicalization?
172006f32e7eSjoerg MPM.addPass(ReversePostOrderFunctionAttrsPass());
172106f32e7eSjoerg
172206f32e7eSjoerg // Use in-range annotations on GEP indices to split globals where beneficial.
172306f32e7eSjoerg MPM.addPass(GlobalSplitPass());
172406f32e7eSjoerg
172506f32e7eSjoerg // Run whole program optimization of virtual call when the list of callees
172606f32e7eSjoerg // is fixed.
172706f32e7eSjoerg MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
172806f32e7eSjoerg
172906f32e7eSjoerg // Stop here at -O1.
1730*da58b97aSjoerg if (Level == OptimizationLevel::O1) {
173106f32e7eSjoerg // The LowerTypeTestsPass needs to run to lower type metadata and the
173206f32e7eSjoerg // type.test intrinsics. The pass does nothing if CFI is disabled.
173306f32e7eSjoerg MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1734*da58b97aSjoerg // Run a second time to clean up any type tests left behind by WPD for use
1735*da58b97aSjoerg // in ICP (which is performed earlier than this in the regular LTO
1736*da58b97aSjoerg // pipeline).
1737*da58b97aSjoerg MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1738*da58b97aSjoerg
1739*da58b97aSjoerg // Emit annotation remarks.
1740*da58b97aSjoerg addAnnotationRemarksPass(MPM);
1741*da58b97aSjoerg
174206f32e7eSjoerg return MPM;
174306f32e7eSjoerg }
174406f32e7eSjoerg
174506f32e7eSjoerg // Optimize globals to try and fold them into constants.
174606f32e7eSjoerg MPM.addPass(GlobalOptPass());
174706f32e7eSjoerg
174806f32e7eSjoerg // Promote any localized globals to SSA registers.
174906f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
175006f32e7eSjoerg
175106f32e7eSjoerg // Linking modules together can lead to duplicate global constant, only
175206f32e7eSjoerg // keep one copy of each constant.
175306f32e7eSjoerg MPM.addPass(ConstantMergePass());
175406f32e7eSjoerg
175506f32e7eSjoerg // Remove unused arguments from functions.
175606f32e7eSjoerg MPM.addPass(DeadArgumentEliminationPass());
175706f32e7eSjoerg
175806f32e7eSjoerg // Reduce the code after globalopt and ipsccp. Both can open up significant
175906f32e7eSjoerg // simplification opportunities, and both can propagate functions through
176006f32e7eSjoerg // function pointers. When this happens, we often have to resolve varargs
176106f32e7eSjoerg // calls, etc, so let instcombine do this.
1762*da58b97aSjoerg FunctionPassManager PeepholeFPM;
1763*da58b97aSjoerg if (Level == OptimizationLevel::O3)
176406f32e7eSjoerg PeepholeFPM.addPass(AggressiveInstCombinePass());
176506f32e7eSjoerg PeepholeFPM.addPass(InstCombinePass());
176606f32e7eSjoerg invokePeepholeEPCallbacks(PeepholeFPM, Level);
176706f32e7eSjoerg
176806f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
176906f32e7eSjoerg
177006f32e7eSjoerg // Note: historically, the PruneEH pass was run first to deduce nounwind and
177106f32e7eSjoerg // generally clean up exception handling overhead. It isn't clear this is
177206f32e7eSjoerg // valuable as the inliner doesn't currently care whether it is inlining an
177306f32e7eSjoerg // invoke or a call.
177406f32e7eSjoerg // Run the inliner now.
1775*da58b97aSjoerg MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level)));
177606f32e7eSjoerg
177706f32e7eSjoerg // Optimize globals again after we ran the inliner.
177806f32e7eSjoerg MPM.addPass(GlobalOptPass());
177906f32e7eSjoerg
178006f32e7eSjoerg // Garbage collect dead functions.
178106f32e7eSjoerg // FIXME: Add ArgumentPromotion pass after once it's ported.
178206f32e7eSjoerg MPM.addPass(GlobalDCEPass());
178306f32e7eSjoerg
1784*da58b97aSjoerg FunctionPassManager FPM;
178506f32e7eSjoerg // The IPO Passes may leave cruft around. Clean up after them.
178606f32e7eSjoerg FPM.addPass(InstCombinePass());
178706f32e7eSjoerg invokePeepholeEPCallbacks(FPM, Level);
178806f32e7eSjoerg
1789*da58b97aSjoerg FPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
179006f32e7eSjoerg
179106f32e7eSjoerg // Do a post inline PGO instrumentation and use pass. This is a context
179206f32e7eSjoerg // sensitive PGO pass.
179306f32e7eSjoerg if (PGOOpt) {
179406f32e7eSjoerg if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1795*da58b97aSjoerg addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
179606f32e7eSjoerg /* IsCS */ true, PGOOpt->CSProfileGenFile,
179706f32e7eSjoerg PGOOpt->ProfileRemappingFile);
179806f32e7eSjoerg else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1799*da58b97aSjoerg addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
180006f32e7eSjoerg /* IsCS */ true, PGOOpt->ProfileFile,
180106f32e7eSjoerg PGOOpt->ProfileRemappingFile);
180206f32e7eSjoerg }
180306f32e7eSjoerg
180406f32e7eSjoerg // Break up allocas
180506f32e7eSjoerg FPM.addPass(SROA());
180606f32e7eSjoerg
180706f32e7eSjoerg // LTO provides additional opportunities for tailcall elimination due to
180806f32e7eSjoerg // link-time inlining, and visibility of nocapture attribute.
180906f32e7eSjoerg FPM.addPass(TailCallElimPass());
181006f32e7eSjoerg
181106f32e7eSjoerg // Run a few AA driver optimizations here and now to cleanup the code.
181206f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
181306f32e7eSjoerg
1814*da58b97aSjoerg MPM.addPass(
1815*da58b97aSjoerg createModuleToPostOrderCGSCCPassAdaptor(PostOrderFunctionAttrsPass()));
1816*da58b97aSjoerg
1817*da58b97aSjoerg // Require the GlobalsAA analysis for the module so we can query it within
1818*da58b97aSjoerg // MainFPM.
1819*da58b97aSjoerg MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
1820*da58b97aSjoerg // Invalidate AAManager so it can be recreated and pick up the newly available
1821*da58b97aSjoerg // GlobalsAA.
1822*da58b97aSjoerg MPM.addPass(
1823*da58b97aSjoerg createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));
182406f32e7eSjoerg
182506f32e7eSjoerg FunctionPassManager MainFPM;
1826*da58b97aSjoerg MainFPM.addPass(createFunctionToLoopPassAdaptor(
1827*da58b97aSjoerg LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
1828*da58b97aSjoerg EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true));
182906f32e7eSjoerg
183006f32e7eSjoerg if (RunNewGVN)
183106f32e7eSjoerg MainFPM.addPass(NewGVNPass());
183206f32e7eSjoerg else
183306f32e7eSjoerg MainFPM.addPass(GVN());
183406f32e7eSjoerg
183506f32e7eSjoerg // Remove dead memcpy()'s.
183606f32e7eSjoerg MainFPM.addPass(MemCpyOptPass());
183706f32e7eSjoerg
183806f32e7eSjoerg // Nuke dead stores.
183906f32e7eSjoerg MainFPM.addPass(DSEPass());
1840*da58b97aSjoerg MainFPM.addPass(MergedLoadStoreMotionPass());
184106f32e7eSjoerg
1842*da58b97aSjoerg // More loops are countable; try to optimize them.
1843*da58b97aSjoerg if (EnableLoopFlatten && Level.getSpeedupLevel() > 1)
1844*da58b97aSjoerg MainFPM.addPass(LoopFlattenPass());
184506f32e7eSjoerg
1846*da58b97aSjoerg if (EnableConstraintElimination)
1847*da58b97aSjoerg MainFPM.addPass(ConstraintEliminationPass());
184806f32e7eSjoerg
1849*da58b97aSjoerg LoopPassManager LPM;
1850*da58b97aSjoerg LPM.addPass(IndVarSimplifyPass());
1851*da58b97aSjoerg LPM.addPass(LoopDeletionPass());
1852*da58b97aSjoerg // FIXME: Add loop interchange.
185306f32e7eSjoerg
1854*da58b97aSjoerg // Unroll small loops and perform peeling.
1855*da58b97aSjoerg LPM.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
1856*da58b97aSjoerg /* OnlyWhenForced= */ !PTO.LoopUnrolling,
1857*da58b97aSjoerg PTO.ForgetAllSCEVInLoopUnroll));
1858*da58b97aSjoerg // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
1859*da58b97aSjoerg // *All* loop passes must preserve it, in order to be able to use it.
1860*da58b97aSjoerg MainFPM.addPass(createFunctionToLoopPassAdaptor(
1861*da58b97aSjoerg std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/true));
186206f32e7eSjoerg
1863*da58b97aSjoerg MainFPM.addPass(LoopDistributePass());
1864*da58b97aSjoerg
1865*da58b97aSjoerg addVectorPasses(Level, MainFPM, /* IsLTO */ true);
1866*da58b97aSjoerg
186706f32e7eSjoerg invokePeepholeEPCallbacks(MainFPM, Level);
1868*da58b97aSjoerg MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
186906f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
187006f32e7eSjoerg
187106f32e7eSjoerg // Create a function that performs CFI checks for cross-DSO calls with
187206f32e7eSjoerg // targets in the current module.
187306f32e7eSjoerg MPM.addPass(CrossDSOCFIPass());
187406f32e7eSjoerg
187506f32e7eSjoerg // Lower type metadata and the type.test intrinsic. This pass supports
187606f32e7eSjoerg // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
187706f32e7eSjoerg // to be run at link time if CFI is enabled. This pass does nothing if
187806f32e7eSjoerg // CFI is disabled.
187906f32e7eSjoerg MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1880*da58b97aSjoerg // Run a second time to clean up any type tests left behind by WPD for use
1881*da58b97aSjoerg // in ICP (which is performed earlier than this in the regular LTO pipeline).
1882*da58b97aSjoerg MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
188306f32e7eSjoerg
188406f32e7eSjoerg // Enable splitting late in the FullLTO post-link pipeline. This is done in
188506f32e7eSjoerg // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
188606f32e7eSjoerg if (EnableHotColdSplit)
188706f32e7eSjoerg MPM.addPass(HotColdSplittingPass());
188806f32e7eSjoerg
188906f32e7eSjoerg // Add late LTO optimization passes.
189006f32e7eSjoerg // Delete basic blocks, which optimization passes may have killed.
1891*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(
1892*da58b97aSjoerg SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true))));
189306f32e7eSjoerg
189406f32e7eSjoerg // Drop bodies of available eternally objects to improve GlobalDCE.
189506f32e7eSjoerg MPM.addPass(EliminateAvailableExternallyPass());
189606f32e7eSjoerg
189706f32e7eSjoerg // Now that we have optimized the program, discard unreachable functions.
189806f32e7eSjoerg MPM.addPass(GlobalDCEPass());
189906f32e7eSjoerg
1900*da58b97aSjoerg if (PTO.MergeFunctions)
1901*da58b97aSjoerg MPM.addPass(MergeFunctionsPass());
1902*da58b97aSjoerg
1903*da58b97aSjoerg // Emit annotation remarks.
1904*da58b97aSjoerg addAnnotationRemarksPass(MPM);
1905*da58b97aSjoerg
1906*da58b97aSjoerg return MPM;
1907*da58b97aSjoerg }
1908*da58b97aSjoerg
buildO0DefaultPipeline(OptimizationLevel Level,bool LTOPreLink)1909*da58b97aSjoerg ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
1910*da58b97aSjoerg bool LTOPreLink) {
1911*da58b97aSjoerg assert(Level == OptimizationLevel::O0 &&
1912*da58b97aSjoerg "buildO0DefaultPipeline should only be used with O0");
1913*da58b97aSjoerg
1914*da58b97aSjoerg ModulePassManager MPM;
1915*da58b97aSjoerg
1916*da58b97aSjoerg if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
1917*da58b97aSjoerg PGOOpt->Action == PGOOptions::IRUse))
1918*da58b97aSjoerg addPGOInstrPassesForO0(
1919*da58b97aSjoerg MPM,
1920*da58b97aSjoerg /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
1921*da58b97aSjoerg /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
1922*da58b97aSjoerg
1923*da58b97aSjoerg for (auto &C : PipelineStartEPCallbacks)
1924*da58b97aSjoerg C(MPM, Level);
1925*da58b97aSjoerg for (auto &C : PipelineEarlySimplificationEPCallbacks)
1926*da58b97aSjoerg C(MPM, Level);
1927*da58b97aSjoerg
1928*da58b97aSjoerg // Build a minimal pipeline based on the semantics required by LLVM,
1929*da58b97aSjoerg // which is just that always inlining occurs. Further, disable generating
1930*da58b97aSjoerg // lifetime intrinsics to avoid enabling further optimizations during
1931*da58b97aSjoerg // code generation.
1932*da58b97aSjoerg // However, we need to insert lifetime intrinsics to avoid invalid access
1933*da58b97aSjoerg // caused by multithreaded coroutines.
1934*da58b97aSjoerg MPM.addPass(AlwaysInlinerPass(
1935*da58b97aSjoerg /*InsertLifetimeIntrinsics=*/PTO.Coroutines));
1936*da58b97aSjoerg
1937*da58b97aSjoerg if (PTO.MergeFunctions)
1938*da58b97aSjoerg MPM.addPass(MergeFunctionsPass());
1939*da58b97aSjoerg
1940*da58b97aSjoerg if (EnableMatrix)
1941*da58b97aSjoerg MPM.addPass(
1942*da58b97aSjoerg createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass(true)));
1943*da58b97aSjoerg
1944*da58b97aSjoerg if (!CGSCCOptimizerLateEPCallbacks.empty()) {
1945*da58b97aSjoerg CGSCCPassManager CGPM;
1946*da58b97aSjoerg for (auto &C : CGSCCOptimizerLateEPCallbacks)
1947*da58b97aSjoerg C(CGPM, Level);
1948*da58b97aSjoerg if (!CGPM.isEmpty())
1949*da58b97aSjoerg MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
1950*da58b97aSjoerg }
1951*da58b97aSjoerg if (!LateLoopOptimizationsEPCallbacks.empty()) {
1952*da58b97aSjoerg LoopPassManager LPM;
1953*da58b97aSjoerg for (auto &C : LateLoopOptimizationsEPCallbacks)
1954*da58b97aSjoerg C(LPM, Level);
1955*da58b97aSjoerg if (!LPM.isEmpty()) {
1956*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(
1957*da58b97aSjoerg createFunctionToLoopPassAdaptor(std::move(LPM))));
1958*da58b97aSjoerg }
1959*da58b97aSjoerg }
1960*da58b97aSjoerg if (!LoopOptimizerEndEPCallbacks.empty()) {
1961*da58b97aSjoerg LoopPassManager LPM;
1962*da58b97aSjoerg for (auto &C : LoopOptimizerEndEPCallbacks)
1963*da58b97aSjoerg C(LPM, Level);
1964*da58b97aSjoerg if (!LPM.isEmpty()) {
1965*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(
1966*da58b97aSjoerg createFunctionToLoopPassAdaptor(std::move(LPM))));
1967*da58b97aSjoerg }
1968*da58b97aSjoerg }
1969*da58b97aSjoerg if (!ScalarOptimizerLateEPCallbacks.empty()) {
1970*da58b97aSjoerg FunctionPassManager FPM;
1971*da58b97aSjoerg for (auto &C : ScalarOptimizerLateEPCallbacks)
1972*da58b97aSjoerg C(FPM, Level);
1973*da58b97aSjoerg if (!FPM.isEmpty())
1974*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1975*da58b97aSjoerg }
1976*da58b97aSjoerg if (!VectorizerStartEPCallbacks.empty()) {
1977*da58b97aSjoerg FunctionPassManager FPM;
1978*da58b97aSjoerg for (auto &C : VectorizerStartEPCallbacks)
1979*da58b97aSjoerg C(FPM, Level);
1980*da58b97aSjoerg if (!FPM.isEmpty())
1981*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1982*da58b97aSjoerg }
1983*da58b97aSjoerg
1984*da58b97aSjoerg if (PTO.Coroutines) {
1985*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass()));
1986*da58b97aSjoerg
1987*da58b97aSjoerg CGSCCPassManager CGPM;
1988*da58b97aSjoerg CGPM.addPass(CoroSplitPass());
1989*da58b97aSjoerg CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass()));
1990*da58b97aSjoerg MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
1991*da58b97aSjoerg
1992*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
1993*da58b97aSjoerg }
1994*da58b97aSjoerg
1995*da58b97aSjoerg for (auto &C : OptimizerLastEPCallbacks)
1996*da58b97aSjoerg C(MPM, Level);
1997*da58b97aSjoerg
1998*da58b97aSjoerg if (LTOPreLink)
1999*da58b97aSjoerg addRequiredLTOPreLinkPasses(MPM);
2000*da58b97aSjoerg
200106f32e7eSjoerg return MPM;
200206f32e7eSjoerg }
200306f32e7eSjoerg
buildDefaultAAPipeline()200406f32e7eSjoerg AAManager PassBuilder::buildDefaultAAPipeline() {
200506f32e7eSjoerg AAManager AA;
200606f32e7eSjoerg
200706f32e7eSjoerg // The order in which these are registered determines their priority when
200806f32e7eSjoerg // being queried.
200906f32e7eSjoerg
201006f32e7eSjoerg // First we register the basic alias analysis that provides the majority of
201106f32e7eSjoerg // per-function local AA logic. This is a stateless, on-demand local set of
201206f32e7eSjoerg // AA techniques.
201306f32e7eSjoerg AA.registerFunctionAnalysis<BasicAA>();
201406f32e7eSjoerg
201506f32e7eSjoerg // Next we query fast, specialized alias analyses that wrap IR-embedded
201606f32e7eSjoerg // information about aliasing.
201706f32e7eSjoerg AA.registerFunctionAnalysis<ScopedNoAliasAA>();
201806f32e7eSjoerg AA.registerFunctionAnalysis<TypeBasedAA>();
201906f32e7eSjoerg
202006f32e7eSjoerg // Add support for querying global aliasing information when available.
202106f32e7eSjoerg // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
202206f32e7eSjoerg // analysis, all that the `AAManager` can do is query for any *cached*
202306f32e7eSjoerg // results from `GlobalsAA` through a readonly proxy.
202406f32e7eSjoerg AA.registerModuleAnalysis<GlobalsAA>();
202506f32e7eSjoerg
2026*da58b97aSjoerg // Add target-specific alias analyses.
2027*da58b97aSjoerg if (TM)
2028*da58b97aSjoerg TM->registerDefaultAliasAnalyses(AA);
2029*da58b97aSjoerg
203006f32e7eSjoerg return AA;
203106f32e7eSjoerg }
203206f32e7eSjoerg
parseRepeatPassName(StringRef Name)203306f32e7eSjoerg static Optional<int> parseRepeatPassName(StringRef Name) {
203406f32e7eSjoerg if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
203506f32e7eSjoerg return None;
203606f32e7eSjoerg int Count;
203706f32e7eSjoerg if (Name.getAsInteger(0, Count) || Count <= 0)
203806f32e7eSjoerg return None;
203906f32e7eSjoerg return Count;
204006f32e7eSjoerg }
204106f32e7eSjoerg
parseDevirtPassName(StringRef Name)204206f32e7eSjoerg static Optional<int> parseDevirtPassName(StringRef Name) {
204306f32e7eSjoerg if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
204406f32e7eSjoerg return None;
204506f32e7eSjoerg int Count;
2046*da58b97aSjoerg if (Name.getAsInteger(0, Count) || Count < 0)
204706f32e7eSjoerg return None;
204806f32e7eSjoerg return Count;
204906f32e7eSjoerg }
205006f32e7eSjoerg
checkParametrizedPassName(StringRef Name,StringRef PassName)205106f32e7eSjoerg static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
205206f32e7eSjoerg if (!Name.consume_front(PassName))
205306f32e7eSjoerg return false;
205406f32e7eSjoerg // normal pass name w/o parameters == default parameters
205506f32e7eSjoerg if (Name.empty())
205606f32e7eSjoerg return true;
205706f32e7eSjoerg return Name.startswith("<") && Name.endswith(">");
205806f32e7eSjoerg }
205906f32e7eSjoerg
206006f32e7eSjoerg namespace {
206106f32e7eSjoerg
206206f32e7eSjoerg /// This performs customized parsing of pass name with parameters.
206306f32e7eSjoerg ///
206406f32e7eSjoerg /// We do not need parametrization of passes in textual pipeline very often,
206506f32e7eSjoerg /// yet on a rare occasion ability to specify parameters right there can be
206606f32e7eSjoerg /// useful.
206706f32e7eSjoerg ///
206806f32e7eSjoerg /// \p Name - parameterized specification of a pass from a textual pipeline
206906f32e7eSjoerg /// is a string in a form of :
207006f32e7eSjoerg /// PassName '<' parameter-list '>'
207106f32e7eSjoerg ///
207206f32e7eSjoerg /// Parameter list is being parsed by the parser callable argument, \p Parser,
207306f32e7eSjoerg /// It takes a string-ref of parameters and returns either StringError or a
207406f32e7eSjoerg /// parameter list in a form of a custom parameters type, all wrapped into
207506f32e7eSjoerg /// Expected<> template class.
207606f32e7eSjoerg ///
207706f32e7eSjoerg template <typename ParametersParseCallableT>
parsePassParameters(ParametersParseCallableT && Parser,StringRef Name,StringRef PassName)207806f32e7eSjoerg auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
207906f32e7eSjoerg StringRef PassName) -> decltype(Parser(StringRef{})) {
208006f32e7eSjoerg using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
208106f32e7eSjoerg
208206f32e7eSjoerg StringRef Params = Name;
208306f32e7eSjoerg if (!Params.consume_front(PassName)) {
208406f32e7eSjoerg assert(false &&
208506f32e7eSjoerg "unable to strip pass name from parametrized pass specification");
208606f32e7eSjoerg }
208706f32e7eSjoerg if (Params.empty())
208806f32e7eSjoerg return ParametersT{};
208906f32e7eSjoerg if (!Params.consume_front("<") || !Params.consume_back(">")) {
209006f32e7eSjoerg assert(false && "invalid format for parametrized pass name");
209106f32e7eSjoerg }
209206f32e7eSjoerg
209306f32e7eSjoerg Expected<ParametersT> Result = Parser(Params);
209406f32e7eSjoerg assert((Result || Result.template errorIsA<StringError>()) &&
209506f32e7eSjoerg "Pass parameter parser can only return StringErrors.");
209606f32e7eSjoerg return Result;
209706f32e7eSjoerg }
209806f32e7eSjoerg
209906f32e7eSjoerg /// Parser of parameters for LoopUnroll pass.
parseLoopUnrollOptions(StringRef Params)210006f32e7eSjoerg Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
210106f32e7eSjoerg LoopUnrollOptions UnrollOpts;
210206f32e7eSjoerg while (!Params.empty()) {
210306f32e7eSjoerg StringRef ParamName;
210406f32e7eSjoerg std::tie(ParamName, Params) = Params.split(';');
210506f32e7eSjoerg int OptLevel = StringSwitch<int>(ParamName)
210606f32e7eSjoerg .Case("O0", 0)
210706f32e7eSjoerg .Case("O1", 1)
210806f32e7eSjoerg .Case("O2", 2)
210906f32e7eSjoerg .Case("O3", 3)
211006f32e7eSjoerg .Default(-1);
211106f32e7eSjoerg if (OptLevel >= 0) {
211206f32e7eSjoerg UnrollOpts.setOptLevel(OptLevel);
211306f32e7eSjoerg continue;
211406f32e7eSjoerg }
211506f32e7eSjoerg if (ParamName.consume_front("full-unroll-max=")) {
211606f32e7eSjoerg int Count;
211706f32e7eSjoerg if (ParamName.getAsInteger(0, Count))
211806f32e7eSjoerg return make_error<StringError>(
211906f32e7eSjoerg formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
212006f32e7eSjoerg inconvertibleErrorCode());
212106f32e7eSjoerg UnrollOpts.setFullUnrollMaxCount(Count);
212206f32e7eSjoerg continue;
212306f32e7eSjoerg }
212406f32e7eSjoerg
212506f32e7eSjoerg bool Enable = !ParamName.consume_front("no-");
212606f32e7eSjoerg if (ParamName == "partial") {
212706f32e7eSjoerg UnrollOpts.setPartial(Enable);
212806f32e7eSjoerg } else if (ParamName == "peeling") {
212906f32e7eSjoerg UnrollOpts.setPeeling(Enable);
213006f32e7eSjoerg } else if (ParamName == "profile-peeling") {
213106f32e7eSjoerg UnrollOpts.setProfileBasedPeeling(Enable);
213206f32e7eSjoerg } else if (ParamName == "runtime") {
213306f32e7eSjoerg UnrollOpts.setRuntime(Enable);
213406f32e7eSjoerg } else if (ParamName == "upperbound") {
213506f32e7eSjoerg UnrollOpts.setUpperBound(Enable);
213606f32e7eSjoerg } else {
213706f32e7eSjoerg return make_error<StringError>(
213806f32e7eSjoerg formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
213906f32e7eSjoerg inconvertibleErrorCode());
214006f32e7eSjoerg }
214106f32e7eSjoerg }
214206f32e7eSjoerg return UnrollOpts;
214306f32e7eSjoerg }
214406f32e7eSjoerg
parseMSanPassOptions(StringRef Params)214506f32e7eSjoerg Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
214606f32e7eSjoerg MemorySanitizerOptions Result;
214706f32e7eSjoerg while (!Params.empty()) {
214806f32e7eSjoerg StringRef ParamName;
214906f32e7eSjoerg std::tie(ParamName, Params) = Params.split(';');
215006f32e7eSjoerg
215106f32e7eSjoerg if (ParamName == "recover") {
215206f32e7eSjoerg Result.Recover = true;
215306f32e7eSjoerg } else if (ParamName == "kernel") {
215406f32e7eSjoerg Result.Kernel = true;
215506f32e7eSjoerg } else if (ParamName.consume_front("track-origins=")) {
215606f32e7eSjoerg if (ParamName.getAsInteger(0, Result.TrackOrigins))
215706f32e7eSjoerg return make_error<StringError>(
215806f32e7eSjoerg formatv("invalid argument to MemorySanitizer pass track-origins "
215906f32e7eSjoerg "parameter: '{0}' ",
216006f32e7eSjoerg ParamName)
216106f32e7eSjoerg .str(),
216206f32e7eSjoerg inconvertibleErrorCode());
216306f32e7eSjoerg } else {
216406f32e7eSjoerg return make_error<StringError>(
216506f32e7eSjoerg formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
216606f32e7eSjoerg .str(),
216706f32e7eSjoerg inconvertibleErrorCode());
216806f32e7eSjoerg }
216906f32e7eSjoerg }
217006f32e7eSjoerg return Result;
217106f32e7eSjoerg }
217206f32e7eSjoerg
217306f32e7eSjoerg /// Parser of parameters for SimplifyCFG pass.
parseSimplifyCFGOptions(StringRef Params)217406f32e7eSjoerg Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
217506f32e7eSjoerg SimplifyCFGOptions Result;
217606f32e7eSjoerg while (!Params.empty()) {
217706f32e7eSjoerg StringRef ParamName;
217806f32e7eSjoerg std::tie(ParamName, Params) = Params.split(';');
217906f32e7eSjoerg
218006f32e7eSjoerg bool Enable = !ParamName.consume_front("no-");
218106f32e7eSjoerg if (ParamName == "forward-switch-cond") {
218206f32e7eSjoerg Result.forwardSwitchCondToPhi(Enable);
218306f32e7eSjoerg } else if (ParamName == "switch-to-lookup") {
218406f32e7eSjoerg Result.convertSwitchToLookupTable(Enable);
218506f32e7eSjoerg } else if (ParamName == "keep-loops") {
218606f32e7eSjoerg Result.needCanonicalLoops(Enable);
2187*da58b97aSjoerg } else if (ParamName == "hoist-common-insts") {
2188*da58b97aSjoerg Result.hoistCommonInsts(Enable);
218906f32e7eSjoerg } else if (ParamName == "sink-common-insts") {
219006f32e7eSjoerg Result.sinkCommonInsts(Enable);
219106f32e7eSjoerg } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
219206f32e7eSjoerg APInt BonusInstThreshold;
219306f32e7eSjoerg if (ParamName.getAsInteger(0, BonusInstThreshold))
219406f32e7eSjoerg return make_error<StringError>(
219506f32e7eSjoerg formatv("invalid argument to SimplifyCFG pass bonus-threshold "
219606f32e7eSjoerg "parameter: '{0}' ",
219706f32e7eSjoerg ParamName).str(),
219806f32e7eSjoerg inconvertibleErrorCode());
219906f32e7eSjoerg Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
220006f32e7eSjoerg } else {
220106f32e7eSjoerg return make_error<StringError>(
220206f32e7eSjoerg formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
220306f32e7eSjoerg inconvertibleErrorCode());
220406f32e7eSjoerg }
220506f32e7eSjoerg }
220606f32e7eSjoerg return Result;
220706f32e7eSjoerg }
220806f32e7eSjoerg
220906f32e7eSjoerg /// Parser of parameters for LoopVectorize pass.
parseLoopVectorizeOptions(StringRef Params)221006f32e7eSjoerg Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
221106f32e7eSjoerg LoopVectorizeOptions Opts;
221206f32e7eSjoerg while (!Params.empty()) {
221306f32e7eSjoerg StringRef ParamName;
221406f32e7eSjoerg std::tie(ParamName, Params) = Params.split(';');
221506f32e7eSjoerg
221606f32e7eSjoerg bool Enable = !ParamName.consume_front("no-");
221706f32e7eSjoerg if (ParamName == "interleave-forced-only") {
221806f32e7eSjoerg Opts.setInterleaveOnlyWhenForced(Enable);
221906f32e7eSjoerg } else if (ParamName == "vectorize-forced-only") {
222006f32e7eSjoerg Opts.setVectorizeOnlyWhenForced(Enable);
222106f32e7eSjoerg } else {
222206f32e7eSjoerg return make_error<StringError>(
222306f32e7eSjoerg formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
222406f32e7eSjoerg inconvertibleErrorCode());
222506f32e7eSjoerg }
222606f32e7eSjoerg }
222706f32e7eSjoerg return Opts;
222806f32e7eSjoerg }
222906f32e7eSjoerg
parseLoopUnswitchOptions(StringRef Params)223006f32e7eSjoerg Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
223106f32e7eSjoerg bool Result = false;
223206f32e7eSjoerg while (!Params.empty()) {
223306f32e7eSjoerg StringRef ParamName;
223406f32e7eSjoerg std::tie(ParamName, Params) = Params.split(';');
223506f32e7eSjoerg
223606f32e7eSjoerg bool Enable = !ParamName.consume_front("no-");
223706f32e7eSjoerg if (ParamName == "nontrivial") {
223806f32e7eSjoerg Result = Enable;
223906f32e7eSjoerg } else {
224006f32e7eSjoerg return make_error<StringError>(
224106f32e7eSjoerg formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
224206f32e7eSjoerg .str(),
224306f32e7eSjoerg inconvertibleErrorCode());
224406f32e7eSjoerg }
224506f32e7eSjoerg }
224606f32e7eSjoerg return Result;
224706f32e7eSjoerg }
224806f32e7eSjoerg
parseMergedLoadStoreMotionOptions(StringRef Params)224906f32e7eSjoerg Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
225006f32e7eSjoerg bool Result = false;
225106f32e7eSjoerg while (!Params.empty()) {
225206f32e7eSjoerg StringRef ParamName;
225306f32e7eSjoerg std::tie(ParamName, Params) = Params.split(';');
225406f32e7eSjoerg
225506f32e7eSjoerg bool Enable = !ParamName.consume_front("no-");
225606f32e7eSjoerg if (ParamName == "split-footer-bb") {
225706f32e7eSjoerg Result = Enable;
225806f32e7eSjoerg } else {
225906f32e7eSjoerg return make_error<StringError>(
226006f32e7eSjoerg formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
226106f32e7eSjoerg ParamName)
226206f32e7eSjoerg .str(),
226306f32e7eSjoerg inconvertibleErrorCode());
226406f32e7eSjoerg }
226506f32e7eSjoerg }
226606f32e7eSjoerg return Result;
226706f32e7eSjoerg }
2268*da58b97aSjoerg
parseGVNOptions(StringRef Params)2269*da58b97aSjoerg Expected<GVNOptions> parseGVNOptions(StringRef Params) {
2270*da58b97aSjoerg GVNOptions Result;
2271*da58b97aSjoerg while (!Params.empty()) {
2272*da58b97aSjoerg StringRef ParamName;
2273*da58b97aSjoerg std::tie(ParamName, Params) = Params.split(';');
2274*da58b97aSjoerg
2275*da58b97aSjoerg bool Enable = !ParamName.consume_front("no-");
2276*da58b97aSjoerg if (ParamName == "pre") {
2277*da58b97aSjoerg Result.setPRE(Enable);
2278*da58b97aSjoerg } else if (ParamName == "load-pre") {
2279*da58b97aSjoerg Result.setLoadPRE(Enable);
2280*da58b97aSjoerg } else if (ParamName == "split-backedge-load-pre") {
2281*da58b97aSjoerg Result.setLoadPRESplitBackedge(Enable);
2282*da58b97aSjoerg } else if (ParamName == "memdep") {
2283*da58b97aSjoerg Result.setMemDep(Enable);
2284*da58b97aSjoerg } else {
2285*da58b97aSjoerg return make_error<StringError>(
2286*da58b97aSjoerg formatv("invalid GVN pass parameter '{0}' ", ParamName).str(),
2287*da58b97aSjoerg inconvertibleErrorCode());
2288*da58b97aSjoerg }
2289*da58b97aSjoerg }
2290*da58b97aSjoerg return Result;
2291*da58b97aSjoerg }
2292*da58b97aSjoerg
2293*da58b97aSjoerg Expected<StackLifetime::LivenessType>
parseStackLifetimeOptions(StringRef Params)2294*da58b97aSjoerg parseStackLifetimeOptions(StringRef Params) {
2295*da58b97aSjoerg StackLifetime::LivenessType Result = StackLifetime::LivenessType::May;
2296*da58b97aSjoerg while (!Params.empty()) {
2297*da58b97aSjoerg StringRef ParamName;
2298*da58b97aSjoerg std::tie(ParamName, Params) = Params.split(';');
2299*da58b97aSjoerg
2300*da58b97aSjoerg if (ParamName == "may") {
2301*da58b97aSjoerg Result = StackLifetime::LivenessType::May;
2302*da58b97aSjoerg } else if (ParamName == "must") {
2303*da58b97aSjoerg Result = StackLifetime::LivenessType::Must;
2304*da58b97aSjoerg } else {
2305*da58b97aSjoerg return make_error<StringError>(
2306*da58b97aSjoerg formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(),
2307*da58b97aSjoerg inconvertibleErrorCode());
2308*da58b97aSjoerg }
2309*da58b97aSjoerg }
2310*da58b97aSjoerg return Result;
2311*da58b97aSjoerg }
2312*da58b97aSjoerg
231306f32e7eSjoerg } // namespace
231406f32e7eSjoerg
231506f32e7eSjoerg /// Tests whether a pass name starts with a valid prefix for a default pipeline
231606f32e7eSjoerg /// alias.
startsWithDefaultPipelineAliasPrefix(StringRef Name)231706f32e7eSjoerg static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
231806f32e7eSjoerg return Name.startswith("default") || Name.startswith("thinlto") ||
231906f32e7eSjoerg Name.startswith("lto");
232006f32e7eSjoerg }
232106f32e7eSjoerg
232206f32e7eSjoerg /// Tests whether registered callbacks will accept a given pass name.
232306f32e7eSjoerg ///
232406f32e7eSjoerg /// When parsing a pipeline text, the type of the outermost pipeline may be
232506f32e7eSjoerg /// omitted, in which case the type is automatically determined from the first
232606f32e7eSjoerg /// pass name in the text. This may be a name that is handled through one of the
232706f32e7eSjoerg /// callbacks. We check this through the oridinary parsing callbacks by setting
232806f32e7eSjoerg /// up a dummy PassManager in order to not force the client to also handle this
232906f32e7eSjoerg /// type of query.
233006f32e7eSjoerg template <typename PassManagerT, typename CallbacksT>
callbacksAcceptPassName(StringRef Name,CallbacksT & Callbacks)233106f32e7eSjoerg static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
233206f32e7eSjoerg if (!Callbacks.empty()) {
233306f32e7eSjoerg PassManagerT DummyPM;
233406f32e7eSjoerg for (auto &CB : Callbacks)
233506f32e7eSjoerg if (CB(Name, DummyPM, {}))
233606f32e7eSjoerg return true;
233706f32e7eSjoerg }
233806f32e7eSjoerg return false;
233906f32e7eSjoerg }
234006f32e7eSjoerg
234106f32e7eSjoerg template <typename CallbacksT>
isModulePassName(StringRef Name,CallbacksT & Callbacks)234206f32e7eSjoerg static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
234306f32e7eSjoerg // Manually handle aliases for pre-configured pipeline fragments.
234406f32e7eSjoerg if (startsWithDefaultPipelineAliasPrefix(Name))
234506f32e7eSjoerg return DefaultAliasRegex.match(Name);
234606f32e7eSjoerg
234706f32e7eSjoerg // Explicitly handle pass manager names.
234806f32e7eSjoerg if (Name == "module")
234906f32e7eSjoerg return true;
235006f32e7eSjoerg if (Name == "cgscc")
235106f32e7eSjoerg return true;
235206f32e7eSjoerg if (Name == "function")
235306f32e7eSjoerg return true;
235406f32e7eSjoerg
235506f32e7eSjoerg // Explicitly handle custom-parsed pass names.
235606f32e7eSjoerg if (parseRepeatPassName(Name))
235706f32e7eSjoerg return true;
235806f32e7eSjoerg
235906f32e7eSjoerg #define MODULE_PASS(NAME, CREATE_PASS) \
236006f32e7eSjoerg if (Name == NAME) \
236106f32e7eSjoerg return true;
236206f32e7eSjoerg #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
236306f32e7eSjoerg if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
236406f32e7eSjoerg return true;
236506f32e7eSjoerg #include "PassRegistry.def"
236606f32e7eSjoerg
236706f32e7eSjoerg return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
236806f32e7eSjoerg }
236906f32e7eSjoerg
237006f32e7eSjoerg template <typename CallbacksT>
isCGSCCPassName(StringRef Name,CallbacksT & Callbacks)237106f32e7eSjoerg static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
237206f32e7eSjoerg // Explicitly handle pass manager names.
237306f32e7eSjoerg if (Name == "cgscc")
237406f32e7eSjoerg return true;
237506f32e7eSjoerg if (Name == "function")
237606f32e7eSjoerg return true;
237706f32e7eSjoerg
237806f32e7eSjoerg // Explicitly handle custom-parsed pass names.
237906f32e7eSjoerg if (parseRepeatPassName(Name))
238006f32e7eSjoerg return true;
238106f32e7eSjoerg if (parseDevirtPassName(Name))
238206f32e7eSjoerg return true;
238306f32e7eSjoerg
238406f32e7eSjoerg #define CGSCC_PASS(NAME, CREATE_PASS) \
238506f32e7eSjoerg if (Name == NAME) \
238606f32e7eSjoerg return true;
238706f32e7eSjoerg #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
238806f32e7eSjoerg if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
238906f32e7eSjoerg return true;
239006f32e7eSjoerg #include "PassRegistry.def"
239106f32e7eSjoerg
239206f32e7eSjoerg return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
239306f32e7eSjoerg }
239406f32e7eSjoerg
239506f32e7eSjoerg template <typename CallbacksT>
isFunctionPassName(StringRef Name,CallbacksT & Callbacks)239606f32e7eSjoerg static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
239706f32e7eSjoerg // Explicitly handle pass manager names.
239806f32e7eSjoerg if (Name == "function")
239906f32e7eSjoerg return true;
240006f32e7eSjoerg if (Name == "loop" || Name == "loop-mssa")
240106f32e7eSjoerg return true;
240206f32e7eSjoerg
240306f32e7eSjoerg // Explicitly handle custom-parsed pass names.
240406f32e7eSjoerg if (parseRepeatPassName(Name))
240506f32e7eSjoerg return true;
240606f32e7eSjoerg
240706f32e7eSjoerg #define FUNCTION_PASS(NAME, CREATE_PASS) \
240806f32e7eSjoerg if (Name == NAME) \
240906f32e7eSjoerg return true;
241006f32e7eSjoerg #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
241106f32e7eSjoerg if (checkParametrizedPassName(Name, NAME)) \
241206f32e7eSjoerg return true;
241306f32e7eSjoerg #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
241406f32e7eSjoerg if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
241506f32e7eSjoerg return true;
241606f32e7eSjoerg #include "PassRegistry.def"
241706f32e7eSjoerg
241806f32e7eSjoerg return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
241906f32e7eSjoerg }
242006f32e7eSjoerg
242106f32e7eSjoerg template <typename CallbacksT>
isLoopPassName(StringRef Name,CallbacksT & Callbacks)242206f32e7eSjoerg static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
242306f32e7eSjoerg // Explicitly handle pass manager names.
242406f32e7eSjoerg if (Name == "loop" || Name == "loop-mssa")
242506f32e7eSjoerg return true;
242606f32e7eSjoerg
242706f32e7eSjoerg // Explicitly handle custom-parsed pass names.
242806f32e7eSjoerg if (parseRepeatPassName(Name))
242906f32e7eSjoerg return true;
243006f32e7eSjoerg
243106f32e7eSjoerg #define LOOP_PASS(NAME, CREATE_PASS) \
243206f32e7eSjoerg if (Name == NAME) \
243306f32e7eSjoerg return true;
243406f32e7eSjoerg #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
243506f32e7eSjoerg if (checkParametrizedPassName(Name, NAME)) \
243606f32e7eSjoerg return true;
243706f32e7eSjoerg #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
243806f32e7eSjoerg if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
243906f32e7eSjoerg return true;
244006f32e7eSjoerg #include "PassRegistry.def"
244106f32e7eSjoerg
244206f32e7eSjoerg return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
244306f32e7eSjoerg }
244406f32e7eSjoerg
244506f32e7eSjoerg Optional<std::vector<PassBuilder::PipelineElement>>
parsePipelineText(StringRef Text)244606f32e7eSjoerg PassBuilder::parsePipelineText(StringRef Text) {
244706f32e7eSjoerg std::vector<PipelineElement> ResultPipeline;
244806f32e7eSjoerg
244906f32e7eSjoerg SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
245006f32e7eSjoerg &ResultPipeline};
245106f32e7eSjoerg for (;;) {
245206f32e7eSjoerg std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
245306f32e7eSjoerg size_t Pos = Text.find_first_of(",()");
245406f32e7eSjoerg Pipeline.push_back({Text.substr(0, Pos), {}});
245506f32e7eSjoerg
245606f32e7eSjoerg // If we have a single terminating name, we're done.
245706f32e7eSjoerg if (Pos == Text.npos)
245806f32e7eSjoerg break;
245906f32e7eSjoerg
246006f32e7eSjoerg char Sep = Text[Pos];
246106f32e7eSjoerg Text = Text.substr(Pos + 1);
246206f32e7eSjoerg if (Sep == ',')
246306f32e7eSjoerg // Just a name ending in a comma, continue.
246406f32e7eSjoerg continue;
246506f32e7eSjoerg
246606f32e7eSjoerg if (Sep == '(') {
246706f32e7eSjoerg // Push the inner pipeline onto the stack to continue processing.
246806f32e7eSjoerg PipelineStack.push_back(&Pipeline.back().InnerPipeline);
246906f32e7eSjoerg continue;
247006f32e7eSjoerg }
247106f32e7eSjoerg
247206f32e7eSjoerg assert(Sep == ')' && "Bogus separator!");
247306f32e7eSjoerg // When handling the close parenthesis, we greedily consume them to avoid
247406f32e7eSjoerg // empty strings in the pipeline.
247506f32e7eSjoerg do {
247606f32e7eSjoerg // If we try to pop the outer pipeline we have unbalanced parentheses.
247706f32e7eSjoerg if (PipelineStack.size() == 1)
247806f32e7eSjoerg return None;
247906f32e7eSjoerg
248006f32e7eSjoerg PipelineStack.pop_back();
248106f32e7eSjoerg } while (Text.consume_front(")"));
248206f32e7eSjoerg
248306f32e7eSjoerg // Check if we've finished parsing.
248406f32e7eSjoerg if (Text.empty())
248506f32e7eSjoerg break;
248606f32e7eSjoerg
248706f32e7eSjoerg // Otherwise, the end of an inner pipeline always has to be followed by
248806f32e7eSjoerg // a comma, and then we can continue.
248906f32e7eSjoerg if (!Text.consume_front(","))
249006f32e7eSjoerg return None;
249106f32e7eSjoerg }
249206f32e7eSjoerg
249306f32e7eSjoerg if (PipelineStack.size() > 1)
249406f32e7eSjoerg // Unbalanced paretheses.
249506f32e7eSjoerg return None;
249606f32e7eSjoerg
249706f32e7eSjoerg assert(PipelineStack.back() == &ResultPipeline &&
249806f32e7eSjoerg "Wrong pipeline at the bottom of the stack!");
249906f32e7eSjoerg return {std::move(ResultPipeline)};
250006f32e7eSjoerg }
250106f32e7eSjoerg
parseModulePass(ModulePassManager & MPM,const PipelineElement & E)250206f32e7eSjoerg Error PassBuilder::parseModulePass(ModulePassManager &MPM,
2503*da58b97aSjoerg const PipelineElement &E) {
250406f32e7eSjoerg auto &Name = E.Name;
250506f32e7eSjoerg auto &InnerPipeline = E.InnerPipeline;
250606f32e7eSjoerg
250706f32e7eSjoerg // First handle complex passes like the pass managers which carry pipelines.
250806f32e7eSjoerg if (!InnerPipeline.empty()) {
250906f32e7eSjoerg if (Name == "module") {
2510*da58b97aSjoerg ModulePassManager NestedMPM;
2511*da58b97aSjoerg if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
251206f32e7eSjoerg return Err;
251306f32e7eSjoerg MPM.addPass(std::move(NestedMPM));
251406f32e7eSjoerg return Error::success();
251506f32e7eSjoerg }
251606f32e7eSjoerg if (Name == "cgscc") {
2517*da58b97aSjoerg CGSCCPassManager CGPM;
2518*da58b97aSjoerg if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline))
251906f32e7eSjoerg return Err;
252006f32e7eSjoerg MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
252106f32e7eSjoerg return Error::success();
252206f32e7eSjoerg }
252306f32e7eSjoerg if (Name == "function") {
2524*da58b97aSjoerg FunctionPassManager FPM;
2525*da58b97aSjoerg if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
252606f32e7eSjoerg return Err;
252706f32e7eSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
252806f32e7eSjoerg return Error::success();
252906f32e7eSjoerg }
253006f32e7eSjoerg if (auto Count = parseRepeatPassName(Name)) {
2531*da58b97aSjoerg ModulePassManager NestedMPM;
2532*da58b97aSjoerg if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
253306f32e7eSjoerg return Err;
253406f32e7eSjoerg MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
253506f32e7eSjoerg return Error::success();
253606f32e7eSjoerg }
253706f32e7eSjoerg
253806f32e7eSjoerg for (auto &C : ModulePipelineParsingCallbacks)
253906f32e7eSjoerg if (C(Name, MPM, InnerPipeline))
254006f32e7eSjoerg return Error::success();
254106f32e7eSjoerg
254206f32e7eSjoerg // Normal passes can't have pipelines.
254306f32e7eSjoerg return make_error<StringError>(
254406f32e7eSjoerg formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
254506f32e7eSjoerg inconvertibleErrorCode());
254606f32e7eSjoerg ;
254706f32e7eSjoerg }
254806f32e7eSjoerg
254906f32e7eSjoerg // Manually handle aliases for pre-configured pipeline fragments.
255006f32e7eSjoerg if (startsWithDefaultPipelineAliasPrefix(Name)) {
255106f32e7eSjoerg SmallVector<StringRef, 3> Matches;
255206f32e7eSjoerg if (!DefaultAliasRegex.match(Name, &Matches))
255306f32e7eSjoerg return make_error<StringError>(
255406f32e7eSjoerg formatv("unknown default pipeline alias '{0}'", Name).str(),
255506f32e7eSjoerg inconvertibleErrorCode());
255606f32e7eSjoerg
255706f32e7eSjoerg assert(Matches.size() == 3 && "Must capture two matched strings!");
255806f32e7eSjoerg
255906f32e7eSjoerg OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
2560*da58b97aSjoerg .Case("O0", OptimizationLevel::O0)
2561*da58b97aSjoerg .Case("O1", OptimizationLevel::O1)
2562*da58b97aSjoerg .Case("O2", OptimizationLevel::O2)
2563*da58b97aSjoerg .Case("O3", OptimizationLevel::O3)
2564*da58b97aSjoerg .Case("Os", OptimizationLevel::Os)
2565*da58b97aSjoerg .Case("Oz", OptimizationLevel::Oz);
2566*da58b97aSjoerg if (L == OptimizationLevel::O0 && Matches[1] != "thinlto" &&
2567*da58b97aSjoerg Matches[1] != "lto") {
2568*da58b97aSjoerg MPM.addPass(buildO0DefaultPipeline(L, Matches[1] == "thinlto-pre-link" ||
2569*da58b97aSjoerg Matches[1] == "lto-pre-link"));
257006f32e7eSjoerg return Error::success();
257106f32e7eSjoerg }
257206f32e7eSjoerg
2573*da58b97aSjoerg // This is consistent with old pass manager invoked via opt, but
2574*da58b97aSjoerg // inconsistent with clang. Clang doesn't enable loop vectorization
2575*da58b97aSjoerg // but does enable slp vectorization at Oz.
2576*da58b97aSjoerg PTO.LoopVectorization =
2577*da58b97aSjoerg L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2578*da58b97aSjoerg PTO.SLPVectorization =
2579*da58b97aSjoerg L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2580*da58b97aSjoerg
258106f32e7eSjoerg if (Matches[1] == "default") {
2582*da58b97aSjoerg MPM.addPass(buildPerModuleDefaultPipeline(L));
258306f32e7eSjoerg } else if (Matches[1] == "thinlto-pre-link") {
2584*da58b97aSjoerg MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L));
258506f32e7eSjoerg } else if (Matches[1] == "thinlto") {
2586*da58b97aSjoerg MPM.addPass(buildThinLTODefaultPipeline(L, nullptr));
258706f32e7eSjoerg } else if (Matches[1] == "lto-pre-link") {
2588*da58b97aSjoerg MPM.addPass(buildLTOPreLinkDefaultPipeline(L));
258906f32e7eSjoerg } else {
259006f32e7eSjoerg assert(Matches[1] == "lto" && "Not one of the matched options!");
2591*da58b97aSjoerg MPM.addPass(buildLTODefaultPipeline(L, nullptr));
259206f32e7eSjoerg }
259306f32e7eSjoerg return Error::success();
259406f32e7eSjoerg }
259506f32e7eSjoerg
259606f32e7eSjoerg // Finally expand the basic registered passes from the .inc file.
259706f32e7eSjoerg #define MODULE_PASS(NAME, CREATE_PASS) \
259806f32e7eSjoerg if (Name == NAME) { \
259906f32e7eSjoerg MPM.addPass(CREATE_PASS); \
260006f32e7eSjoerg return Error::success(); \
260106f32e7eSjoerg }
260206f32e7eSjoerg #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
260306f32e7eSjoerg if (Name == "require<" NAME ">") { \
260406f32e7eSjoerg MPM.addPass( \
260506f32e7eSjoerg RequireAnalysisPass< \
260606f32e7eSjoerg std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \
260706f32e7eSjoerg return Error::success(); \
260806f32e7eSjoerg } \
260906f32e7eSjoerg if (Name == "invalidate<" NAME ">") { \
261006f32e7eSjoerg MPM.addPass(InvalidateAnalysisPass< \
261106f32e7eSjoerg std::remove_reference<decltype(CREATE_PASS)>::type>()); \
261206f32e7eSjoerg return Error::success(); \
261306f32e7eSjoerg }
2614*da58b97aSjoerg #define CGSCC_PASS(NAME, CREATE_PASS) \
2615*da58b97aSjoerg if (Name == NAME) { \
2616*da58b97aSjoerg MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS)); \
2617*da58b97aSjoerg return Error::success(); \
2618*da58b97aSjoerg }
2619*da58b97aSjoerg #define FUNCTION_PASS(NAME, CREATE_PASS) \
2620*da58b97aSjoerg if (Name == NAME) { \
2621*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS)); \
2622*da58b97aSjoerg return Error::success(); \
2623*da58b97aSjoerg }
2624*da58b97aSjoerg #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2625*da58b97aSjoerg if (checkParametrizedPassName(Name, NAME)) { \
2626*da58b97aSjoerg auto Params = parsePassParameters(PARSER, Name, NAME); \
2627*da58b97aSjoerg if (!Params) \
2628*da58b97aSjoerg return Params.takeError(); \
2629*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2630*da58b97aSjoerg return Error::success(); \
2631*da58b97aSjoerg }
2632*da58b97aSjoerg #define LOOP_PASS(NAME, CREATE_PASS) \
2633*da58b97aSjoerg if (Name == NAME) { \
2634*da58b97aSjoerg MPM.addPass(createModuleToFunctionPassAdaptor( \
2635*da58b97aSjoerg createFunctionToLoopPassAdaptor(CREATE_PASS, false, false))); \
2636*da58b97aSjoerg return Error::success(); \
2637*da58b97aSjoerg }
2638*da58b97aSjoerg #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2639*da58b97aSjoerg if (checkParametrizedPassName(Name, NAME)) { \
2640*da58b97aSjoerg auto Params = parsePassParameters(PARSER, Name, NAME); \
2641*da58b97aSjoerg if (!Params) \
2642*da58b97aSjoerg return Params.takeError(); \
2643*da58b97aSjoerg MPM.addPass( \
2644*da58b97aSjoerg createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2645*da58b97aSjoerg CREATE_PASS(Params.get()), false, false))); \
2646*da58b97aSjoerg return Error::success(); \
2647*da58b97aSjoerg }
264806f32e7eSjoerg #include "PassRegistry.def"
264906f32e7eSjoerg
265006f32e7eSjoerg for (auto &C : ModulePipelineParsingCallbacks)
265106f32e7eSjoerg if (C(Name, MPM, InnerPipeline))
265206f32e7eSjoerg return Error::success();
265306f32e7eSjoerg return make_error<StringError>(
265406f32e7eSjoerg formatv("unknown module pass '{0}'", Name).str(),
265506f32e7eSjoerg inconvertibleErrorCode());
265606f32e7eSjoerg }
265706f32e7eSjoerg
parseCGSCCPass(CGSCCPassManager & CGPM,const PipelineElement & E)265806f32e7eSjoerg Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
2659*da58b97aSjoerg const PipelineElement &E) {
266006f32e7eSjoerg auto &Name = E.Name;
266106f32e7eSjoerg auto &InnerPipeline = E.InnerPipeline;
266206f32e7eSjoerg
266306f32e7eSjoerg // First handle complex passes like the pass managers which carry pipelines.
266406f32e7eSjoerg if (!InnerPipeline.empty()) {
266506f32e7eSjoerg if (Name == "cgscc") {
2666*da58b97aSjoerg CGSCCPassManager NestedCGPM;
2667*da58b97aSjoerg if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
266806f32e7eSjoerg return Err;
266906f32e7eSjoerg // Add the nested pass manager with the appropriate adaptor.
267006f32e7eSjoerg CGPM.addPass(std::move(NestedCGPM));
267106f32e7eSjoerg return Error::success();
267206f32e7eSjoerg }
267306f32e7eSjoerg if (Name == "function") {
2674*da58b97aSjoerg FunctionPassManager FPM;
2675*da58b97aSjoerg if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
267606f32e7eSjoerg return Err;
267706f32e7eSjoerg // Add the nested pass manager with the appropriate adaptor.
267806f32e7eSjoerg CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
267906f32e7eSjoerg return Error::success();
268006f32e7eSjoerg }
268106f32e7eSjoerg if (auto Count = parseRepeatPassName(Name)) {
2682*da58b97aSjoerg CGSCCPassManager NestedCGPM;
2683*da58b97aSjoerg if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
268406f32e7eSjoerg return Err;
268506f32e7eSjoerg CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
268606f32e7eSjoerg return Error::success();
268706f32e7eSjoerg }
268806f32e7eSjoerg if (auto MaxRepetitions = parseDevirtPassName(Name)) {
2689*da58b97aSjoerg CGSCCPassManager NestedCGPM;
2690*da58b97aSjoerg if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
269106f32e7eSjoerg return Err;
269206f32e7eSjoerg CGPM.addPass(
269306f32e7eSjoerg createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
269406f32e7eSjoerg return Error::success();
269506f32e7eSjoerg }
269606f32e7eSjoerg
269706f32e7eSjoerg for (auto &C : CGSCCPipelineParsingCallbacks)
269806f32e7eSjoerg if (C(Name, CGPM, InnerPipeline))
269906f32e7eSjoerg return Error::success();
270006f32e7eSjoerg
270106f32e7eSjoerg // Normal passes can't have pipelines.
270206f32e7eSjoerg return make_error<StringError>(
270306f32e7eSjoerg formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
270406f32e7eSjoerg inconvertibleErrorCode());
270506f32e7eSjoerg }
270606f32e7eSjoerg
270706f32e7eSjoerg // Now expand the basic registered passes from the .inc file.
270806f32e7eSjoerg #define CGSCC_PASS(NAME, CREATE_PASS) \
270906f32e7eSjoerg if (Name == NAME) { \
271006f32e7eSjoerg CGPM.addPass(CREATE_PASS); \
271106f32e7eSjoerg return Error::success(); \
271206f32e7eSjoerg }
271306f32e7eSjoerg #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
271406f32e7eSjoerg if (Name == "require<" NAME ">") { \
271506f32e7eSjoerg CGPM.addPass(RequireAnalysisPass< \
271606f32e7eSjoerg std::remove_reference<decltype(CREATE_PASS)>::type, \
271706f32e7eSjoerg LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
271806f32e7eSjoerg CGSCCUpdateResult &>()); \
271906f32e7eSjoerg return Error::success(); \
272006f32e7eSjoerg } \
272106f32e7eSjoerg if (Name == "invalidate<" NAME ">") { \
272206f32e7eSjoerg CGPM.addPass(InvalidateAnalysisPass< \
272306f32e7eSjoerg std::remove_reference<decltype(CREATE_PASS)>::type>()); \
272406f32e7eSjoerg return Error::success(); \
272506f32e7eSjoerg }
2726*da58b97aSjoerg #define FUNCTION_PASS(NAME, CREATE_PASS) \
2727*da58b97aSjoerg if (Name == NAME) { \
2728*da58b97aSjoerg CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS)); \
2729*da58b97aSjoerg return Error::success(); \
2730*da58b97aSjoerg }
2731*da58b97aSjoerg #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2732*da58b97aSjoerg if (checkParametrizedPassName(Name, NAME)) { \
2733*da58b97aSjoerg auto Params = parsePassParameters(PARSER, Name, NAME); \
2734*da58b97aSjoerg if (!Params) \
2735*da58b97aSjoerg return Params.takeError(); \
2736*da58b97aSjoerg CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2737*da58b97aSjoerg return Error::success(); \
2738*da58b97aSjoerg }
2739*da58b97aSjoerg #define LOOP_PASS(NAME, CREATE_PASS) \
2740*da58b97aSjoerg if (Name == NAME) { \
2741*da58b97aSjoerg CGPM.addPass(createCGSCCToFunctionPassAdaptor( \
2742*da58b97aSjoerg createFunctionToLoopPassAdaptor(CREATE_PASS, false, false))); \
2743*da58b97aSjoerg return Error::success(); \
2744*da58b97aSjoerg }
2745*da58b97aSjoerg #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2746*da58b97aSjoerg if (checkParametrizedPassName(Name, NAME)) { \
2747*da58b97aSjoerg auto Params = parsePassParameters(PARSER, Name, NAME); \
2748*da58b97aSjoerg if (!Params) \
2749*da58b97aSjoerg return Params.takeError(); \
2750*da58b97aSjoerg CGPM.addPass( \
2751*da58b97aSjoerg createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2752*da58b97aSjoerg CREATE_PASS(Params.get()), false, false))); \
2753*da58b97aSjoerg return Error::success(); \
2754*da58b97aSjoerg }
275506f32e7eSjoerg #include "PassRegistry.def"
275606f32e7eSjoerg
275706f32e7eSjoerg for (auto &C : CGSCCPipelineParsingCallbacks)
275806f32e7eSjoerg if (C(Name, CGPM, InnerPipeline))
275906f32e7eSjoerg return Error::success();
276006f32e7eSjoerg return make_error<StringError>(
276106f32e7eSjoerg formatv("unknown cgscc pass '{0}'", Name).str(),
276206f32e7eSjoerg inconvertibleErrorCode());
276306f32e7eSjoerg }
276406f32e7eSjoerg
parseFunctionPass(FunctionPassManager & FPM,const PipelineElement & E)276506f32e7eSjoerg Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
2766*da58b97aSjoerg const PipelineElement &E) {
276706f32e7eSjoerg auto &Name = E.Name;
276806f32e7eSjoerg auto &InnerPipeline = E.InnerPipeline;
276906f32e7eSjoerg
277006f32e7eSjoerg // First handle complex passes like the pass managers which carry pipelines.
277106f32e7eSjoerg if (!InnerPipeline.empty()) {
277206f32e7eSjoerg if (Name == "function") {
2773*da58b97aSjoerg FunctionPassManager NestedFPM;
2774*da58b97aSjoerg if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
277506f32e7eSjoerg return Err;
277606f32e7eSjoerg // Add the nested pass manager with the appropriate adaptor.
277706f32e7eSjoerg FPM.addPass(std::move(NestedFPM));
277806f32e7eSjoerg return Error::success();
277906f32e7eSjoerg }
278006f32e7eSjoerg if (Name == "loop" || Name == "loop-mssa") {
2781*da58b97aSjoerg LoopPassManager LPM;
2782*da58b97aSjoerg if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline))
278306f32e7eSjoerg return Err;
278406f32e7eSjoerg // Add the nested pass manager with the appropriate adaptor.
278506f32e7eSjoerg bool UseMemorySSA = (Name == "loop-mssa");
2786*da58b97aSjoerg bool UseBFI = llvm::any_of(
2787*da58b97aSjoerg InnerPipeline, [](auto Pipeline) { return Pipeline.Name == "licm"; });
278806f32e7eSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
2789*da58b97aSjoerg UseBFI));
279006f32e7eSjoerg return Error::success();
279106f32e7eSjoerg }
279206f32e7eSjoerg if (auto Count = parseRepeatPassName(Name)) {
2793*da58b97aSjoerg FunctionPassManager NestedFPM;
2794*da58b97aSjoerg if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
279506f32e7eSjoerg return Err;
279606f32e7eSjoerg FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
279706f32e7eSjoerg return Error::success();
279806f32e7eSjoerg }
279906f32e7eSjoerg
280006f32e7eSjoerg for (auto &C : FunctionPipelineParsingCallbacks)
280106f32e7eSjoerg if (C(Name, FPM, InnerPipeline))
280206f32e7eSjoerg return Error::success();
280306f32e7eSjoerg
280406f32e7eSjoerg // Normal passes can't have pipelines.
280506f32e7eSjoerg return make_error<StringError>(
280606f32e7eSjoerg formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
280706f32e7eSjoerg inconvertibleErrorCode());
280806f32e7eSjoerg }
280906f32e7eSjoerg
281006f32e7eSjoerg // Now expand the basic registered passes from the .inc file.
281106f32e7eSjoerg #define FUNCTION_PASS(NAME, CREATE_PASS) \
281206f32e7eSjoerg if (Name == NAME) { \
281306f32e7eSjoerg FPM.addPass(CREATE_PASS); \
281406f32e7eSjoerg return Error::success(); \
281506f32e7eSjoerg }
281606f32e7eSjoerg #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
281706f32e7eSjoerg if (checkParametrizedPassName(Name, NAME)) { \
281806f32e7eSjoerg auto Params = parsePassParameters(PARSER, Name, NAME); \
281906f32e7eSjoerg if (!Params) \
282006f32e7eSjoerg return Params.takeError(); \
282106f32e7eSjoerg FPM.addPass(CREATE_PASS(Params.get())); \
282206f32e7eSjoerg return Error::success(); \
282306f32e7eSjoerg }
282406f32e7eSjoerg #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
282506f32e7eSjoerg if (Name == "require<" NAME ">") { \
282606f32e7eSjoerg FPM.addPass( \
282706f32e7eSjoerg RequireAnalysisPass< \
282806f32e7eSjoerg std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \
282906f32e7eSjoerg return Error::success(); \
283006f32e7eSjoerg } \
283106f32e7eSjoerg if (Name == "invalidate<" NAME ">") { \
283206f32e7eSjoerg FPM.addPass(InvalidateAnalysisPass< \
283306f32e7eSjoerg std::remove_reference<decltype(CREATE_PASS)>::type>()); \
283406f32e7eSjoerg return Error::success(); \
283506f32e7eSjoerg }
2836*da58b97aSjoerg // FIXME: UseMemorySSA is set to false. Maybe we could do things like:
2837*da58b97aSjoerg // bool UseMemorySSA = !("canon-freeze" || "loop-predication" ||
2838*da58b97aSjoerg // "guard-widening");
2839*da58b97aSjoerg // The risk is that it may become obsolete if we're not careful.
2840*da58b97aSjoerg #define LOOP_PASS(NAME, CREATE_PASS) \
2841*da58b97aSjoerg if (Name == NAME) { \
2842*da58b97aSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false)); \
2843*da58b97aSjoerg return Error::success(); \
2844*da58b97aSjoerg }
2845*da58b97aSjoerg #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2846*da58b97aSjoerg if (checkParametrizedPassName(Name, NAME)) { \
2847*da58b97aSjoerg auto Params = parsePassParameters(PARSER, Name, NAME); \
2848*da58b97aSjoerg if (!Params) \
2849*da58b97aSjoerg return Params.takeError(); \
2850*da58b97aSjoerg FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \
2851*da58b97aSjoerg false, false)); \
2852*da58b97aSjoerg return Error::success(); \
2853*da58b97aSjoerg }
285406f32e7eSjoerg #include "PassRegistry.def"
285506f32e7eSjoerg
285606f32e7eSjoerg for (auto &C : FunctionPipelineParsingCallbacks)
285706f32e7eSjoerg if (C(Name, FPM, InnerPipeline))
285806f32e7eSjoerg return Error::success();
285906f32e7eSjoerg return make_error<StringError>(
286006f32e7eSjoerg formatv("unknown function pass '{0}'", Name).str(),
286106f32e7eSjoerg inconvertibleErrorCode());
286206f32e7eSjoerg }
286306f32e7eSjoerg
parseLoopPass(LoopPassManager & LPM,const PipelineElement & E)2864*da58b97aSjoerg Error PassBuilder::parseLoopPass(LoopPassManager &LPM,
2865*da58b97aSjoerg const PipelineElement &E) {
286606f32e7eSjoerg StringRef Name = E.Name;
286706f32e7eSjoerg auto &InnerPipeline = E.InnerPipeline;
286806f32e7eSjoerg
286906f32e7eSjoerg // First handle complex passes like the pass managers which carry pipelines.
287006f32e7eSjoerg if (!InnerPipeline.empty()) {
287106f32e7eSjoerg if (Name == "loop") {
2872*da58b97aSjoerg LoopPassManager NestedLPM;
2873*da58b97aSjoerg if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
287406f32e7eSjoerg return Err;
287506f32e7eSjoerg // Add the nested pass manager with the appropriate adaptor.
287606f32e7eSjoerg LPM.addPass(std::move(NestedLPM));
287706f32e7eSjoerg return Error::success();
287806f32e7eSjoerg }
287906f32e7eSjoerg if (auto Count = parseRepeatPassName(Name)) {
2880*da58b97aSjoerg LoopPassManager NestedLPM;
2881*da58b97aSjoerg if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
288206f32e7eSjoerg return Err;
288306f32e7eSjoerg LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
288406f32e7eSjoerg return Error::success();
288506f32e7eSjoerg }
288606f32e7eSjoerg
288706f32e7eSjoerg for (auto &C : LoopPipelineParsingCallbacks)
288806f32e7eSjoerg if (C(Name, LPM, InnerPipeline))
288906f32e7eSjoerg return Error::success();
289006f32e7eSjoerg
289106f32e7eSjoerg // Normal passes can't have pipelines.
289206f32e7eSjoerg return make_error<StringError>(
289306f32e7eSjoerg formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
289406f32e7eSjoerg inconvertibleErrorCode());
289506f32e7eSjoerg }
289606f32e7eSjoerg
289706f32e7eSjoerg // Now expand the basic registered passes from the .inc file.
289806f32e7eSjoerg #define LOOP_PASS(NAME, CREATE_PASS) \
289906f32e7eSjoerg if (Name == NAME) { \
290006f32e7eSjoerg LPM.addPass(CREATE_PASS); \
290106f32e7eSjoerg return Error::success(); \
290206f32e7eSjoerg }
290306f32e7eSjoerg #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
290406f32e7eSjoerg if (checkParametrizedPassName(Name, NAME)) { \
290506f32e7eSjoerg auto Params = parsePassParameters(PARSER, Name, NAME); \
290606f32e7eSjoerg if (!Params) \
290706f32e7eSjoerg return Params.takeError(); \
290806f32e7eSjoerg LPM.addPass(CREATE_PASS(Params.get())); \
290906f32e7eSjoerg return Error::success(); \
291006f32e7eSjoerg }
291106f32e7eSjoerg #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
291206f32e7eSjoerg if (Name == "require<" NAME ">") { \
291306f32e7eSjoerg LPM.addPass(RequireAnalysisPass< \
291406f32e7eSjoerg std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \
291506f32e7eSjoerg LoopAnalysisManager, LoopStandardAnalysisResults &, \
291606f32e7eSjoerg LPMUpdater &>()); \
291706f32e7eSjoerg return Error::success(); \
291806f32e7eSjoerg } \
291906f32e7eSjoerg if (Name == "invalidate<" NAME ">") { \
292006f32e7eSjoerg LPM.addPass(InvalidateAnalysisPass< \
292106f32e7eSjoerg std::remove_reference<decltype(CREATE_PASS)>::type>()); \
292206f32e7eSjoerg return Error::success(); \
292306f32e7eSjoerg }
292406f32e7eSjoerg #include "PassRegistry.def"
292506f32e7eSjoerg
292606f32e7eSjoerg for (auto &C : LoopPipelineParsingCallbacks)
292706f32e7eSjoerg if (C(Name, LPM, InnerPipeline))
292806f32e7eSjoerg return Error::success();
292906f32e7eSjoerg return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
293006f32e7eSjoerg inconvertibleErrorCode());
293106f32e7eSjoerg }
293206f32e7eSjoerg
parseAAPassName(AAManager & AA,StringRef Name)293306f32e7eSjoerg bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
293406f32e7eSjoerg #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
293506f32e7eSjoerg if (Name == NAME) { \
293606f32e7eSjoerg AA.registerModuleAnalysis< \
293706f32e7eSjoerg std::remove_reference<decltype(CREATE_PASS)>::type>(); \
293806f32e7eSjoerg return true; \
293906f32e7eSjoerg }
294006f32e7eSjoerg #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
294106f32e7eSjoerg if (Name == NAME) { \
294206f32e7eSjoerg AA.registerFunctionAnalysis< \
294306f32e7eSjoerg std::remove_reference<decltype(CREATE_PASS)>::type>(); \
294406f32e7eSjoerg return true; \
294506f32e7eSjoerg }
294606f32e7eSjoerg #include "PassRegistry.def"
294706f32e7eSjoerg
294806f32e7eSjoerg for (auto &C : AAParsingCallbacks)
294906f32e7eSjoerg if (C(Name, AA))
295006f32e7eSjoerg return true;
295106f32e7eSjoerg return false;
295206f32e7eSjoerg }
295306f32e7eSjoerg
parseLoopPassPipeline(LoopPassManager & LPM,ArrayRef<PipelineElement> Pipeline)295406f32e7eSjoerg Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2955*da58b97aSjoerg ArrayRef<PipelineElement> Pipeline) {
295606f32e7eSjoerg for (const auto &Element : Pipeline) {
2957*da58b97aSjoerg if (auto Err = parseLoopPass(LPM, Element))
295806f32e7eSjoerg return Err;
295906f32e7eSjoerg }
296006f32e7eSjoerg return Error::success();
296106f32e7eSjoerg }
296206f32e7eSjoerg
parseFunctionPassPipeline(FunctionPassManager & FPM,ArrayRef<PipelineElement> Pipeline)2963*da58b97aSjoerg Error PassBuilder::parseFunctionPassPipeline(
2964*da58b97aSjoerg FunctionPassManager &FPM, ArrayRef<PipelineElement> Pipeline) {
296506f32e7eSjoerg for (const auto &Element : Pipeline) {
2966*da58b97aSjoerg if (auto Err = parseFunctionPass(FPM, Element))
296706f32e7eSjoerg return Err;
296806f32e7eSjoerg }
296906f32e7eSjoerg return Error::success();
297006f32e7eSjoerg }
297106f32e7eSjoerg
parseCGSCCPassPipeline(CGSCCPassManager & CGPM,ArrayRef<PipelineElement> Pipeline)297206f32e7eSjoerg Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2973*da58b97aSjoerg ArrayRef<PipelineElement> Pipeline) {
297406f32e7eSjoerg for (const auto &Element : Pipeline) {
2975*da58b97aSjoerg if (auto Err = parseCGSCCPass(CGPM, Element))
297606f32e7eSjoerg return Err;
297706f32e7eSjoerg }
297806f32e7eSjoerg return Error::success();
297906f32e7eSjoerg }
298006f32e7eSjoerg
crossRegisterProxies(LoopAnalysisManager & LAM,FunctionAnalysisManager & FAM,CGSCCAnalysisManager & CGAM,ModuleAnalysisManager & MAM)298106f32e7eSjoerg void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
298206f32e7eSjoerg FunctionAnalysisManager &FAM,
298306f32e7eSjoerg CGSCCAnalysisManager &CGAM,
298406f32e7eSjoerg ModuleAnalysisManager &MAM) {
298506f32e7eSjoerg MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
298606f32e7eSjoerg MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
298706f32e7eSjoerg CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
298806f32e7eSjoerg FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
298906f32e7eSjoerg FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
299006f32e7eSjoerg FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
299106f32e7eSjoerg LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
299206f32e7eSjoerg }
299306f32e7eSjoerg
parseModulePassPipeline(ModulePassManager & MPM,ArrayRef<PipelineElement> Pipeline)299406f32e7eSjoerg Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2995*da58b97aSjoerg ArrayRef<PipelineElement> Pipeline) {
299606f32e7eSjoerg for (const auto &Element : Pipeline) {
2997*da58b97aSjoerg if (auto Err = parseModulePass(MPM, Element))
299806f32e7eSjoerg return Err;
299906f32e7eSjoerg }
300006f32e7eSjoerg return Error::success();
300106f32e7eSjoerg }
300206f32e7eSjoerg
300306f32e7eSjoerg // Primary pass pipeline description parsing routine for a \c ModulePassManager
300406f32e7eSjoerg // FIXME: Should this routine accept a TargetMachine or require the caller to
300506f32e7eSjoerg // pre-populate the analysis managers with target-specific stuff?
parsePassPipeline(ModulePassManager & MPM,StringRef PipelineText)300606f32e7eSjoerg Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
3007*da58b97aSjoerg StringRef PipelineText) {
300806f32e7eSjoerg auto Pipeline = parsePipelineText(PipelineText);
300906f32e7eSjoerg if (!Pipeline || Pipeline->empty())
301006f32e7eSjoerg return make_error<StringError>(
301106f32e7eSjoerg formatv("invalid pipeline '{0}'", PipelineText).str(),
301206f32e7eSjoerg inconvertibleErrorCode());
301306f32e7eSjoerg
301406f32e7eSjoerg // If the first name isn't at the module layer, wrap the pipeline up
301506f32e7eSjoerg // automatically.
301606f32e7eSjoerg StringRef FirstName = Pipeline->front().Name;
301706f32e7eSjoerg
301806f32e7eSjoerg if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
301906f32e7eSjoerg if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
302006f32e7eSjoerg Pipeline = {{"cgscc", std::move(*Pipeline)}};
302106f32e7eSjoerg } else if (isFunctionPassName(FirstName,
302206f32e7eSjoerg FunctionPipelineParsingCallbacks)) {
302306f32e7eSjoerg Pipeline = {{"function", std::move(*Pipeline)}};
302406f32e7eSjoerg } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
302506f32e7eSjoerg Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
302606f32e7eSjoerg } else {
302706f32e7eSjoerg for (auto &C : TopLevelPipelineParsingCallbacks)
3028*da58b97aSjoerg if (C(MPM, *Pipeline))
302906f32e7eSjoerg return Error::success();
303006f32e7eSjoerg
303106f32e7eSjoerg // Unknown pass or pipeline name!
303206f32e7eSjoerg auto &InnerPipeline = Pipeline->front().InnerPipeline;
303306f32e7eSjoerg return make_error<StringError>(
303406f32e7eSjoerg formatv("unknown {0} name '{1}'",
303506f32e7eSjoerg (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
303606f32e7eSjoerg .str(),
303706f32e7eSjoerg inconvertibleErrorCode());
303806f32e7eSjoerg }
303906f32e7eSjoerg }
304006f32e7eSjoerg
3041*da58b97aSjoerg if (auto Err = parseModulePassPipeline(MPM, *Pipeline))
304206f32e7eSjoerg return Err;
304306f32e7eSjoerg return Error::success();
304406f32e7eSjoerg }
304506f32e7eSjoerg
304606f32e7eSjoerg // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
parsePassPipeline(CGSCCPassManager & CGPM,StringRef PipelineText)304706f32e7eSjoerg Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
3048*da58b97aSjoerg StringRef PipelineText) {
304906f32e7eSjoerg auto Pipeline = parsePipelineText(PipelineText);
305006f32e7eSjoerg if (!Pipeline || Pipeline->empty())
305106f32e7eSjoerg return make_error<StringError>(
305206f32e7eSjoerg formatv("invalid pipeline '{0}'", PipelineText).str(),
305306f32e7eSjoerg inconvertibleErrorCode());
305406f32e7eSjoerg
305506f32e7eSjoerg StringRef FirstName = Pipeline->front().Name;
305606f32e7eSjoerg if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
305706f32e7eSjoerg return make_error<StringError>(
305806f32e7eSjoerg formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
305906f32e7eSjoerg PipelineText)
306006f32e7eSjoerg .str(),
306106f32e7eSjoerg inconvertibleErrorCode());
306206f32e7eSjoerg
3063*da58b97aSjoerg if (auto Err = parseCGSCCPassPipeline(CGPM, *Pipeline))
306406f32e7eSjoerg return Err;
306506f32e7eSjoerg return Error::success();
306606f32e7eSjoerg }
306706f32e7eSjoerg
306806f32e7eSjoerg // Primary pass pipeline description parsing routine for a \c
306906f32e7eSjoerg // FunctionPassManager
parsePassPipeline(FunctionPassManager & FPM,StringRef PipelineText)307006f32e7eSjoerg Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
3071*da58b97aSjoerg StringRef PipelineText) {
307206f32e7eSjoerg auto Pipeline = parsePipelineText(PipelineText);
307306f32e7eSjoerg if (!Pipeline || Pipeline->empty())
307406f32e7eSjoerg return make_error<StringError>(
307506f32e7eSjoerg formatv("invalid pipeline '{0}'", PipelineText).str(),
307606f32e7eSjoerg inconvertibleErrorCode());
307706f32e7eSjoerg
307806f32e7eSjoerg StringRef FirstName = Pipeline->front().Name;
307906f32e7eSjoerg if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
308006f32e7eSjoerg return make_error<StringError>(
308106f32e7eSjoerg formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
308206f32e7eSjoerg PipelineText)
308306f32e7eSjoerg .str(),
308406f32e7eSjoerg inconvertibleErrorCode());
308506f32e7eSjoerg
3086*da58b97aSjoerg if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline))
308706f32e7eSjoerg return Err;
308806f32e7eSjoerg return Error::success();
308906f32e7eSjoerg }
309006f32e7eSjoerg
309106f32e7eSjoerg // Primary pass pipeline description parsing routine for a \c LoopPassManager
parsePassPipeline(LoopPassManager & CGPM,StringRef PipelineText)309206f32e7eSjoerg Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
3093*da58b97aSjoerg StringRef PipelineText) {
309406f32e7eSjoerg auto Pipeline = parsePipelineText(PipelineText);
309506f32e7eSjoerg if (!Pipeline || Pipeline->empty())
309606f32e7eSjoerg return make_error<StringError>(
309706f32e7eSjoerg formatv("invalid pipeline '{0}'", PipelineText).str(),
309806f32e7eSjoerg inconvertibleErrorCode());
309906f32e7eSjoerg
3100*da58b97aSjoerg if (auto Err = parseLoopPassPipeline(CGPM, *Pipeline))
310106f32e7eSjoerg return Err;
310206f32e7eSjoerg
310306f32e7eSjoerg return Error::success();
310406f32e7eSjoerg }
310506f32e7eSjoerg
parseAAPipeline(AAManager & AA,StringRef PipelineText)310606f32e7eSjoerg Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
310706f32e7eSjoerg // If the pipeline just consists of the word 'default' just replace the AA
310806f32e7eSjoerg // manager with our default one.
310906f32e7eSjoerg if (PipelineText == "default") {
311006f32e7eSjoerg AA = buildDefaultAAPipeline();
311106f32e7eSjoerg return Error::success();
311206f32e7eSjoerg }
311306f32e7eSjoerg
311406f32e7eSjoerg while (!PipelineText.empty()) {
311506f32e7eSjoerg StringRef Name;
311606f32e7eSjoerg std::tie(Name, PipelineText) = PipelineText.split(',');
311706f32e7eSjoerg if (!parseAAPassName(AA, Name))
311806f32e7eSjoerg return make_error<StringError>(
311906f32e7eSjoerg formatv("unknown alias analysis name '{0}'", Name).str(),
312006f32e7eSjoerg inconvertibleErrorCode());
312106f32e7eSjoerg }
312206f32e7eSjoerg
312306f32e7eSjoerg return Error::success();
312406f32e7eSjoerg }
3125*da58b97aSjoerg
isAAPassName(StringRef PassName)3126*da58b97aSjoerg bool PassBuilder::isAAPassName(StringRef PassName) {
3127*da58b97aSjoerg #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3128*da58b97aSjoerg if (PassName == NAME) \
3129*da58b97aSjoerg return true;
3130*da58b97aSjoerg #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3131*da58b97aSjoerg if (PassName == NAME) \
3132*da58b97aSjoerg return true;
3133*da58b97aSjoerg #include "PassRegistry.def"
3134*da58b97aSjoerg return false;
3135*da58b97aSjoerg }
3136*da58b97aSjoerg
isAnalysisPassName(StringRef PassName)3137*da58b97aSjoerg bool PassBuilder::isAnalysisPassName(StringRef PassName) {
3138*da58b97aSjoerg #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
3139*da58b97aSjoerg if (PassName == NAME) \
3140*da58b97aSjoerg return true;
3141*da58b97aSjoerg #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
3142*da58b97aSjoerg if (PassName == NAME) \
3143*da58b97aSjoerg return true;
3144*da58b97aSjoerg #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
3145*da58b97aSjoerg if (PassName == NAME) \
3146*da58b97aSjoerg return true;
3147*da58b97aSjoerg #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
3148*da58b97aSjoerg if (PassName == NAME) \
3149*da58b97aSjoerg return true;
3150*da58b97aSjoerg #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3151*da58b97aSjoerg if (PassName == NAME) \
3152*da58b97aSjoerg return true;
3153*da58b97aSjoerg #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3154*da58b97aSjoerg if (PassName == NAME) \
3155*da58b97aSjoerg return true;
3156*da58b97aSjoerg #include "PassRegistry.def"
3157*da58b97aSjoerg return false;
3158*da58b97aSjoerg }
3159*da58b97aSjoerg
printPassName(StringRef PassName,raw_ostream & OS)3160*da58b97aSjoerg static void printPassName(StringRef PassName, raw_ostream &OS) {
3161*da58b97aSjoerg OS << " " << PassName << "\n";
3162*da58b97aSjoerg }
3163*da58b97aSjoerg
printPassNames(raw_ostream & OS)3164*da58b97aSjoerg void PassBuilder::printPassNames(raw_ostream &OS) {
3165*da58b97aSjoerg // TODO: print pass descriptions when they are available
3166*da58b97aSjoerg
3167*da58b97aSjoerg OS << "Module passes:\n";
3168*da58b97aSjoerg #define MODULE_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3169*da58b97aSjoerg #include "PassRegistry.def"
3170*da58b97aSjoerg
3171*da58b97aSjoerg OS << "Module analyses:\n";
3172*da58b97aSjoerg #define MODULE_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3173*da58b97aSjoerg #include "PassRegistry.def"
3174*da58b97aSjoerg
3175*da58b97aSjoerg OS << "Module alias analyses:\n";
3176*da58b97aSjoerg #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3177*da58b97aSjoerg #include "PassRegistry.def"
3178*da58b97aSjoerg
3179*da58b97aSjoerg OS << "CGSCC passes:\n";
3180*da58b97aSjoerg #define CGSCC_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3181*da58b97aSjoerg #include "PassRegistry.def"
3182*da58b97aSjoerg
3183*da58b97aSjoerg OS << "CGSCC analyses:\n";
3184*da58b97aSjoerg #define CGSCC_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3185*da58b97aSjoerg #include "PassRegistry.def"
3186*da58b97aSjoerg
3187*da58b97aSjoerg OS << "Function passes:\n";
3188*da58b97aSjoerg #define FUNCTION_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3189*da58b97aSjoerg #include "PassRegistry.def"
3190*da58b97aSjoerg
3191*da58b97aSjoerg OS << "Function analyses:\n";
3192*da58b97aSjoerg #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3193*da58b97aSjoerg #include "PassRegistry.def"
3194*da58b97aSjoerg
3195*da58b97aSjoerg OS << "Function alias analyses:\n";
3196*da58b97aSjoerg #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3197*da58b97aSjoerg #include "PassRegistry.def"
3198*da58b97aSjoerg
3199*da58b97aSjoerg OS << "Loop passes:\n";
3200*da58b97aSjoerg #define LOOP_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3201*da58b97aSjoerg #include "PassRegistry.def"
3202*da58b97aSjoerg
3203*da58b97aSjoerg OS << "Loop analyses:\n";
3204*da58b97aSjoerg #define LOOP_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3205*da58b97aSjoerg #include "PassRegistry.def"
3206*da58b97aSjoerg }
3207*da58b97aSjoerg
registerParseTopLevelPipelineCallback(const std::function<bool (ModulePassManager &,ArrayRef<PipelineElement>)> & C)3208*da58b97aSjoerg void PassBuilder::registerParseTopLevelPipelineCallback(
3209*da58b97aSjoerg const std::function<bool(ModulePassManager &, ArrayRef<PipelineElement>)>
3210*da58b97aSjoerg &C) {
3211*da58b97aSjoerg TopLevelPipelineParsingCallbacks.push_back(C);
3212*da58b97aSjoerg }
3213