1 //===- PassManagerBuilder.cpp - Build Standard Pass -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the PassManagerBuilder class, which is used to set up a
10 // "standard" optimization sequence suitable for languages like C and C++.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
15 #include "llvm-c/Transforms/PassManagerBuilder.h"
16 #include "llvm/ADT/STLExtras.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/Analysis/BasicAliasAnalysis.h"
19 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
20 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
21 #include "llvm/Analysis/GlobalsModRef.h"
22 #include "llvm/Analysis/InlineCost.h"
23 #include "llvm/Analysis/Passes.h"
24 #include "llvm/Analysis/ScopedNoAliasAA.h"
25 #include "llvm/Analysis/TargetLibraryInfo.h"
26 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
27 #include "llvm/IR/DataLayout.h"
28 #include "llvm/IR/LegacyPassManager.h"
29 #include "llvm/IR/Verifier.h"
30 #include "llvm/Support/CommandLine.h"
31 #include "llvm/Support/ManagedStatic.h"
32 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
33 #include "llvm/Transforms/IPO.h"
34 #include "llvm/Transforms/IPO/Attributor.h"
35 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
36 #include "llvm/Transforms/IPO/FunctionAttrs.h"
37 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
38 #include "llvm/Transforms/InstCombine/InstCombine.h"
39 #include "llvm/Transforms/Instrumentation.h"
40 #include "llvm/Transforms/Scalar.h"
41 #include "llvm/Transforms/Scalar/GVN.h"
42 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
43 #include "llvm/Transforms/Scalar/LICM.h"
44 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
45 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
46 #include "llvm/Transforms/Utils.h"
47 #include "llvm/Transforms/Vectorize.h"
48 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
49 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
50 #include "llvm/Transforms/Vectorize/VectorCombine.h"
51
52 using namespace llvm;
53
54 static cl::opt<bool>
55 RunPartialInlining("enable-partial-inlining", cl::init(false), cl::Hidden,
56 cl::ZeroOrMore, cl::desc("Run Partial inlinining pass"));
57
58 static cl::opt<bool>
59 UseGVNAfterVectorization("use-gvn-after-vectorization",
60 cl::init(false), cl::Hidden,
61 cl::desc("Run GVN instead of Early CSE after vectorization passes"));
62
63 static cl::opt<bool> ExtraVectorizerPasses(
64 "extra-vectorizer-passes", cl::init(false), cl::Hidden,
65 cl::desc("Run cleanup optimization passes after vectorization."));
66
67 static cl::opt<bool>
68 RunLoopRerolling("reroll-loops", cl::Hidden,
69 cl::desc("Run the loop rerolling pass"));
70
71 static cl::opt<bool> RunNewGVN("enable-newgvn", cl::init(false), cl::Hidden,
72 cl::desc("Run the NewGVN pass"));
73
74 // Experimental option to use CFL-AA
75 enum class CFLAAType { None, Steensgaard, Andersen, Both };
76 static cl::opt<CFLAAType>
77 UseCFLAA("use-cfl-aa", cl::init(CFLAAType::None), cl::Hidden,
78 cl::desc("Enable the new, experimental CFL alias analysis"),
79 cl::values(clEnumValN(CFLAAType::None, "none", "Disable CFL-AA"),
80 clEnumValN(CFLAAType::Steensgaard, "steens",
81 "Enable unification-based CFL-AA"),
82 clEnumValN(CFLAAType::Andersen, "anders",
83 "Enable inclusion-based CFL-AA"),
84 clEnumValN(CFLAAType::Both, "both",
85 "Enable both variants of CFL-AA")));
86
87 static cl::opt<bool> EnableLoopInterchange(
88 "enable-loopinterchange", cl::init(false), cl::Hidden,
89 cl::desc("Enable the new, experimental LoopInterchange Pass"));
90
91 static cl::opt<bool> EnableUnrollAndJam("enable-unroll-and-jam",
92 cl::init(false), cl::Hidden,
93 cl::desc("Enable Unroll And Jam Pass"));
94
95 static cl::opt<bool>
96 EnablePrepareForThinLTO("prepare-for-thinlto", cl::init(false), cl::Hidden,
97 cl::desc("Enable preparation for ThinLTO."));
98
99 static cl::opt<bool>
100 EnablePerformThinLTO("perform-thinlto", cl::init(false), cl::Hidden,
101 cl::desc("Enable performing ThinLTO."));
102
103 cl::opt<bool> EnableHotColdSplit("hot-cold-split", cl::init(false),
104 cl::ZeroOrMore, cl::desc("Enable hot-cold splitting pass"));
105
106 static cl::opt<bool> UseLoopVersioningLICM(
107 "enable-loop-versioning-licm", cl::init(false), cl::Hidden,
108 cl::desc("Enable the experimental Loop Versioning LICM pass"));
109
110 static cl::opt<bool>
111 DisablePreInliner("disable-preinline", cl::init(false), cl::Hidden,
112 cl::desc("Disable pre-instrumentation inliner"));
113
114 static cl::opt<int> PreInlineThreshold(
115 "preinline-threshold", cl::Hidden, cl::init(75), cl::ZeroOrMore,
116 cl::desc("Control the amount of inlining in pre-instrumentation inliner "
117 "(default = 75)"));
118
119 static cl::opt<bool> EnableGVNHoist(
120 "enable-gvn-hoist", cl::init(false), cl::ZeroOrMore,
121 cl::desc("Enable the GVN hoisting pass (default = off)"));
122
123 static cl::opt<bool>
124 DisableLibCallsShrinkWrap("disable-libcalls-shrinkwrap", cl::init(false),
125 cl::Hidden,
126 cl::desc("Disable shrink-wrap library calls"));
127
128 static cl::opt<bool> EnableSimpleLoopUnswitch(
129 "enable-simple-loop-unswitch", cl::init(false), cl::Hidden,
130 cl::desc("Enable the simple loop unswitch pass. Also enables independent "
131 "cleanup passes integrated into the loop pass manager pipeline."));
132
133 static cl::opt<bool> EnableGVNSink(
134 "enable-gvn-sink", cl::init(false), cl::ZeroOrMore,
135 cl::desc("Enable the GVN sinking pass (default = off)"));
136
137 // This option is used in simplifying testing SampleFDO optimizations for
138 // profile loading.
139 static cl::opt<bool>
140 EnableCHR("enable-chr", cl::init(true), cl::Hidden,
141 cl::desc("Enable control height reduction optimization (CHR)"));
142
143 cl::opt<bool> FlattenedProfileUsed(
144 "flattened-profile-used", cl::init(false), cl::Hidden,
145 cl::desc("Indicate the sample profile being used is flattened, i.e., "
146 "no inline hierachy exists in the profile. "));
147
148 cl::opt<bool> EnableOrderFileInstrumentation(
149 "enable-order-file-instrumentation", cl::init(false), cl::Hidden,
150 cl::desc("Enable order file instrumentation (default = off)"));
151
152 static cl::opt<bool>
153 EnableMatrix("enable-matrix", cl::init(false), cl::Hidden,
154 cl::desc("Enable lowering of the matrix intrinsics"));
155
156 cl::opt<AttributorRunOption> AttributorRun(
157 "attributor-enable", cl::Hidden, cl::init(AttributorRunOption::NONE),
158 cl::desc("Enable the attributor inter-procedural deduction pass."),
159 cl::values(clEnumValN(AttributorRunOption::ALL, "all",
160 "enable all attributor runs"),
161 clEnumValN(AttributorRunOption::MODULE, "module",
162 "enable module-wide attributor runs"),
163 clEnumValN(AttributorRunOption::CGSCC, "cgscc",
164 "enable call graph SCC attributor runs"),
165 clEnumValN(AttributorRunOption::NONE, "none",
166 "disable attributor runs")));
167
168 extern cl::opt<bool> EnableKnowledgeRetention;
169
PassManagerBuilder()170 PassManagerBuilder::PassManagerBuilder() {
171 OptLevel = 2;
172 SizeLevel = 0;
173 LibraryInfo = nullptr;
174 Inliner = nullptr;
175 DisableUnrollLoops = false;
176 SLPVectorize = false;
177 LoopVectorize = true;
178 LoopsInterleaved = true;
179 RerollLoops = RunLoopRerolling;
180 NewGVN = RunNewGVN;
181 LicmMssaOptCap = SetLicmMssaOptCap;
182 LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
183 DisableGVNLoadPRE = false;
184 ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
185 VerifyInput = false;
186 VerifyOutput = false;
187 MergeFunctions = false;
188 PrepareForLTO = false;
189 EnablePGOInstrGen = false;
190 EnablePGOCSInstrGen = false;
191 EnablePGOCSInstrUse = false;
192 PGOInstrGen = "";
193 PGOInstrUse = "";
194 PGOSampleUse = "";
195 PrepareForThinLTO = EnablePrepareForThinLTO;
196 PerformThinLTO = EnablePerformThinLTO;
197 DivergentTarget = false;
198 CallGraphProfile = true;
199 }
200
~PassManagerBuilder()201 PassManagerBuilder::~PassManagerBuilder() {
202 delete LibraryInfo;
203 delete Inliner;
204 }
205
206 /// Set of global extensions, automatically added as part of the standard set.
207 static ManagedStatic<
208 SmallVector<std::tuple<PassManagerBuilder::ExtensionPointTy,
209 PassManagerBuilder::ExtensionFn,
210 PassManagerBuilder::GlobalExtensionID>,
211 8>>
212 GlobalExtensions;
213 static PassManagerBuilder::GlobalExtensionID GlobalExtensionsCounter;
214
215 /// Check if GlobalExtensions is constructed and not empty.
216 /// Since GlobalExtensions is a managed static, calling 'empty()' will trigger
217 /// the construction of the object.
GlobalExtensionsNotEmpty()218 static bool GlobalExtensionsNotEmpty() {
219 return GlobalExtensions.isConstructed() && !GlobalExtensions->empty();
220 }
221
222 PassManagerBuilder::GlobalExtensionID
addGlobalExtension(PassManagerBuilder::ExtensionPointTy Ty,PassManagerBuilder::ExtensionFn Fn)223 PassManagerBuilder::addGlobalExtension(PassManagerBuilder::ExtensionPointTy Ty,
224 PassManagerBuilder::ExtensionFn Fn) {
225 auto ExtensionID = GlobalExtensionsCounter++;
226 GlobalExtensions->push_back(std::make_tuple(Ty, std::move(Fn), ExtensionID));
227 return ExtensionID;
228 }
229
removeGlobalExtension(PassManagerBuilder::GlobalExtensionID ExtensionID)230 void PassManagerBuilder::removeGlobalExtension(
231 PassManagerBuilder::GlobalExtensionID ExtensionID) {
232 // RegisterStandardPasses may try to call this function after GlobalExtensions
233 // has already been destroyed; doing so should not generate an error.
234 if (!GlobalExtensions.isConstructed())
235 return;
236
237 auto GlobalExtension =
238 llvm::find_if(*GlobalExtensions, [ExtensionID](const auto &elem) {
239 return std::get<2>(elem) == ExtensionID;
240 });
241 assert(GlobalExtension != GlobalExtensions->end() &&
242 "The extension ID to be removed should always be valid.");
243
244 GlobalExtensions->erase(GlobalExtension);
245 }
246
addExtension(ExtensionPointTy Ty,ExtensionFn Fn)247 void PassManagerBuilder::addExtension(ExtensionPointTy Ty, ExtensionFn Fn) {
248 Extensions.push_back(std::make_pair(Ty, std::move(Fn)));
249 }
250
addExtensionsToPM(ExtensionPointTy ETy,legacy::PassManagerBase & PM) const251 void PassManagerBuilder::addExtensionsToPM(ExtensionPointTy ETy,
252 legacy::PassManagerBase &PM) const {
253 if (GlobalExtensionsNotEmpty()) {
254 for (auto &Ext : *GlobalExtensions) {
255 if (std::get<0>(Ext) == ETy)
256 std::get<1>(Ext)(*this, PM);
257 }
258 }
259 for (unsigned i = 0, e = Extensions.size(); i != e; ++i)
260 if (Extensions[i].first == ETy)
261 Extensions[i].second(*this, PM);
262 }
263
addInitialAliasAnalysisPasses(legacy::PassManagerBase & PM) const264 void PassManagerBuilder::addInitialAliasAnalysisPasses(
265 legacy::PassManagerBase &PM) const {
266 switch (UseCFLAA) {
267 case CFLAAType::Steensgaard:
268 PM.add(createCFLSteensAAWrapperPass());
269 break;
270 case CFLAAType::Andersen:
271 PM.add(createCFLAndersAAWrapperPass());
272 break;
273 case CFLAAType::Both:
274 PM.add(createCFLSteensAAWrapperPass());
275 PM.add(createCFLAndersAAWrapperPass());
276 break;
277 default:
278 break;
279 }
280
281 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
282 // BasicAliasAnalysis wins if they disagree. This is intended to help
283 // support "obvious" type-punning idioms.
284 PM.add(createTypeBasedAAWrapperPass());
285 PM.add(createScopedNoAliasAAWrapperPass());
286 }
287
populateFunctionPassManager(legacy::FunctionPassManager & FPM)288 void PassManagerBuilder::populateFunctionPassManager(
289 legacy::FunctionPassManager &FPM) {
290 addExtensionsToPM(EP_EarlyAsPossible, FPM);
291 FPM.add(createEntryExitInstrumenterPass());
292
293 // Add LibraryInfo if we have some.
294 if (LibraryInfo)
295 FPM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo));
296
297 if (OptLevel == 0) return;
298
299 addInitialAliasAnalysisPasses(FPM);
300
301 FPM.add(createCFGSimplificationPass());
302 FPM.add(createSROAPass());
303 FPM.add(createEarlyCSEPass());
304 FPM.add(createLowerExpectIntrinsicPass());
305 }
306
307 // Do PGO instrumentation generation or use pass as the option specified.
addPGOInstrPasses(legacy::PassManagerBase & MPM,bool IsCS=false)308 void PassManagerBuilder::addPGOInstrPasses(legacy::PassManagerBase &MPM,
309 bool IsCS = false) {
310 if (IsCS) {
311 if (!EnablePGOCSInstrGen && !EnablePGOCSInstrUse)
312 return;
313 } else if (!EnablePGOInstrGen && PGOInstrUse.empty() && PGOSampleUse.empty())
314 return;
315
316 // Perform the preinline and cleanup passes for O1 and above.
317 // And avoid doing them if optimizing for size.
318 // We will not do this inline for context sensitive PGO (when IsCS is true).
319 if (OptLevel > 0 && SizeLevel == 0 && !DisablePreInliner &&
320 PGOSampleUse.empty() && !IsCS) {
321 // Create preinline pass. We construct an InlineParams object and specify
322 // the threshold here to avoid the command line options of the regular
323 // inliner to influence pre-inlining. The only fields of InlineParams we
324 // care about are DefaultThreshold and HintThreshold.
325 InlineParams IP;
326 IP.DefaultThreshold = PreInlineThreshold;
327 // FIXME: The hint threshold has the same value used by the regular inliner.
328 // This should probably be lowered after performance testing.
329 IP.HintThreshold = 325;
330
331 MPM.add(createFunctionInliningPass(IP));
332 MPM.add(createSROAPass());
333 MPM.add(createEarlyCSEPass()); // Catch trivial redundancies
334 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
335 MPM.add(createInstructionCombiningPass()); // Combine silly seq's
336 addExtensionsToPM(EP_Peephole, MPM);
337 }
338 if ((EnablePGOInstrGen && !IsCS) || (EnablePGOCSInstrGen && IsCS)) {
339 MPM.add(createPGOInstrumentationGenLegacyPass(IsCS));
340 // Add the profile lowering pass.
341 InstrProfOptions Options;
342 if (!PGOInstrGen.empty())
343 Options.InstrProfileOutput = PGOInstrGen;
344 Options.DoCounterPromotion = true;
345 Options.UseBFIInPromotion = IsCS;
346 MPM.add(createLoopRotatePass());
347 MPM.add(createInstrProfilingLegacyPass(Options, IsCS));
348 }
349 if (!PGOInstrUse.empty())
350 MPM.add(createPGOInstrumentationUseLegacyPass(PGOInstrUse, IsCS));
351 // Indirect call promotion that promotes intra-module targets only.
352 // For ThinLTO this is done earlier due to interactions with globalopt
353 // for imported functions. We don't run this at -O0.
354 if (OptLevel > 0 && !IsCS)
355 MPM.add(
356 createPGOIndirectCallPromotionLegacyPass(false, !PGOSampleUse.empty()));
357 }
addFunctionSimplificationPasses(legacy::PassManagerBase & MPM)358 void PassManagerBuilder::addFunctionSimplificationPasses(
359 legacy::PassManagerBase &MPM) {
360 // Start of function pass.
361 // Break up aggregate allocas, using SSAUpdater.
362 assert(OptLevel >= 1 && "Calling function optimizer with no optimization level!");
363 MPM.add(createSROAPass());
364 MPM.add(createEarlyCSEPass(true /* Enable mem-ssa. */)); // Catch trivial redundancies
365 if (EnableKnowledgeRetention)
366 MPM.add(createAssumeSimplifyPass());
367
368 if (OptLevel > 1) {
369 if (EnableGVNHoist)
370 MPM.add(createGVNHoistPass());
371 if (EnableGVNSink) {
372 MPM.add(createGVNSinkPass());
373 MPM.add(createCFGSimplificationPass());
374 }
375 }
376
377 if (OptLevel > 1) {
378 // Speculative execution if the target has divergent branches; otherwise nop.
379 MPM.add(createSpeculativeExecutionIfHasBranchDivergencePass());
380
381 MPM.add(createJumpThreadingPass()); // Thread jumps.
382 MPM.add(createCorrelatedValuePropagationPass()); // Propagate conditionals
383 }
384 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
385 // Combine silly seq's
386 if (OptLevel > 2)
387 MPM.add(createAggressiveInstCombinerPass());
388 MPM.add(createInstructionCombiningPass());
389 if (SizeLevel == 0 && !DisableLibCallsShrinkWrap)
390 MPM.add(createLibCallsShrinkWrapPass());
391 addExtensionsToPM(EP_Peephole, MPM);
392
393 // Optimize memory intrinsic calls based on the profiled size information.
394 if (SizeLevel == 0)
395 MPM.add(createPGOMemOPSizeOptLegacyPass());
396
397 // TODO: Investigate the cost/benefit of tail call elimination on debugging.
398 if (OptLevel > 1)
399 MPM.add(createTailCallEliminationPass()); // Eliminate tail calls
400 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
401 MPM.add(createReassociatePass()); // Reassociate expressions
402
403 // Begin the loop pass pipeline.
404 if (EnableSimpleLoopUnswitch) {
405 // The simple loop unswitch pass relies on separate cleanup passes. Schedule
406 // them first so when we re-process a loop they run before other loop
407 // passes.
408 MPM.add(createLoopInstSimplifyPass());
409 MPM.add(createLoopSimplifyCFGPass());
410 }
411 // Rotate Loop - disable header duplication at -Oz
412 MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1));
413 // TODO: Investigate promotion cap for O1.
414 MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
415 if (EnableSimpleLoopUnswitch)
416 MPM.add(createSimpleLoopUnswitchLegacyPass());
417 else
418 MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3, DivergentTarget));
419 // FIXME: We break the loop pass pipeline here in order to do full
420 // simplify-cfg. Eventually loop-simplifycfg should be enhanced to replace the
421 // need for this.
422 MPM.add(createCFGSimplificationPass());
423 MPM.add(createInstructionCombiningPass());
424 // We resume loop passes creating a second loop pipeline here.
425 MPM.add(createIndVarSimplifyPass()); // Canonicalize indvars
426 MPM.add(createLoopIdiomPass()); // Recognize idioms like memset.
427 addExtensionsToPM(EP_LateLoopOptimizations, MPM);
428 MPM.add(createLoopDeletionPass()); // Delete dead loops
429
430 if (EnableLoopInterchange)
431 MPM.add(createLoopInterchangePass()); // Interchange loops
432
433 // Unroll small loops
434 MPM.add(createSimpleLoopUnrollPass(OptLevel, DisableUnrollLoops,
435 ForgetAllSCEVInLoopUnroll));
436 addExtensionsToPM(EP_LoopOptimizerEnd, MPM);
437 // This ends the loop pass pipelines.
438
439 if (OptLevel > 1) {
440 MPM.add(createMergedLoadStoreMotionPass()); // Merge ld/st in diamonds
441 MPM.add(NewGVN ? createNewGVNPass()
442 : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies
443 }
444 MPM.add(createMemCpyOptPass()); // Remove memcpy / form memset
445 MPM.add(createSCCPPass()); // Constant prop with SCCP
446
447 // Delete dead bit computations (instcombine runs after to fold away the dead
448 // computations, and then ADCE will run later to exploit any new DCE
449 // opportunities that creates).
450 MPM.add(createBitTrackingDCEPass()); // Delete dead bit computations
451
452 // Run instcombine after redundancy elimination to exploit opportunities
453 // opened up by them.
454 MPM.add(createInstructionCombiningPass());
455 addExtensionsToPM(EP_Peephole, MPM);
456 if (OptLevel > 1) {
457 MPM.add(createJumpThreadingPass()); // Thread jumps
458 MPM.add(createCorrelatedValuePropagationPass());
459 MPM.add(createDeadStoreEliminationPass()); // Delete dead stores
460 MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
461 }
462
463 addExtensionsToPM(EP_ScalarOptimizerLate, MPM);
464
465 if (RerollLoops)
466 MPM.add(createLoopRerollPass());
467
468 // TODO: Investigate if this is too expensive at O1.
469 MPM.add(createAggressiveDCEPass()); // Delete dead instructions
470 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
471 // Clean up after everything.
472 MPM.add(createInstructionCombiningPass());
473 addExtensionsToPM(EP_Peephole, MPM);
474
475 if (EnableCHR && OptLevel >= 3 &&
476 (!PGOInstrUse.empty() || !PGOSampleUse.empty() || EnablePGOCSInstrGen))
477 MPM.add(createControlHeightReductionLegacyPass());
478 }
479
populateModulePassManager(legacy::PassManagerBase & MPM)480 void PassManagerBuilder::populateModulePassManager(
481 legacy::PassManagerBase &MPM) {
482 // Whether this is a default or *LTO pre-link pipeline. The FullLTO post-link
483 // is handled separately, so just check this is not the ThinLTO post-link.
484 bool DefaultOrPreLinkPipeline = !PerformThinLTO;
485
486 if (!PGOSampleUse.empty()) {
487 MPM.add(createPruneEHPass());
488 // In ThinLTO mode, when flattened profile is used, all the available
489 // profile information will be annotated in PreLink phase so there is
490 // no need to load the profile again in PostLink.
491 if (!(FlattenedProfileUsed && PerformThinLTO))
492 MPM.add(createSampleProfileLoaderPass(PGOSampleUse));
493 }
494
495 // Allow forcing function attributes as a debugging and tuning aid.
496 MPM.add(createForceFunctionAttrsLegacyPass());
497
498 // If all optimizations are disabled, just run the always-inline pass and,
499 // if enabled, the function merging pass.
500 if (OptLevel == 0) {
501 addPGOInstrPasses(MPM);
502 if (Inliner) {
503 MPM.add(Inliner);
504 Inliner = nullptr;
505 }
506
507 // FIXME: The BarrierNoopPass is a HACK! The inliner pass above implicitly
508 // creates a CGSCC pass manager, but we don't want to add extensions into
509 // that pass manager. To prevent this we insert a no-op module pass to reset
510 // the pass manager to get the same behavior as EP_OptimizerLast in non-O0
511 // builds. The function merging pass is
512 if (MergeFunctions)
513 MPM.add(createMergeFunctionsPass());
514 else if (GlobalExtensionsNotEmpty() || !Extensions.empty())
515 MPM.add(createBarrierNoopPass());
516
517 if (PerformThinLTO) {
518 MPM.add(createLowerTypeTestsPass(nullptr, nullptr, true));
519 // Drop available_externally and unreferenced globals. This is necessary
520 // with ThinLTO in order to avoid leaving undefined references to dead
521 // globals in the object file.
522 MPM.add(createEliminateAvailableExternallyPass());
523 MPM.add(createGlobalDCEPass());
524 }
525
526 addExtensionsToPM(EP_EnabledOnOptLevel0, MPM);
527
528 if (PrepareForLTO || PrepareForThinLTO) {
529 MPM.add(createCanonicalizeAliasesPass());
530 // Rename anon globals to be able to export them in the summary.
531 // This has to be done after we add the extensions to the pass manager
532 // as there could be passes (e.g. Adddress sanitizer) which introduce
533 // new unnamed globals.
534 MPM.add(createNameAnonGlobalPass());
535 }
536 return;
537 }
538
539 // Add LibraryInfo if we have some.
540 if (LibraryInfo)
541 MPM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo));
542
543 addInitialAliasAnalysisPasses(MPM);
544
545 // For ThinLTO there are two passes of indirect call promotion. The
546 // first is during the compile phase when PerformThinLTO=false and
547 // intra-module indirect call targets are promoted. The second is during
548 // the ThinLTO backend when PerformThinLTO=true, when we promote imported
549 // inter-module indirect calls. For that we perform indirect call promotion
550 // earlier in the pass pipeline, here before globalopt. Otherwise imported
551 // available_externally functions look unreferenced and are removed.
552 if (PerformThinLTO) {
553 MPM.add(createPGOIndirectCallPromotionLegacyPass(/*InLTO = */ true,
554 !PGOSampleUse.empty()));
555 MPM.add(createLowerTypeTestsPass(nullptr, nullptr, true));
556 }
557
558 // For SamplePGO in ThinLTO compile phase, we do not want to unroll loops
559 // as it will change the CFG too much to make the 2nd profile annotation
560 // in backend more difficult.
561 bool PrepareForThinLTOUsingPGOSampleProfile =
562 PrepareForThinLTO && !PGOSampleUse.empty();
563 if (PrepareForThinLTOUsingPGOSampleProfile)
564 DisableUnrollLoops = true;
565
566 // Infer attributes about declarations if possible.
567 MPM.add(createInferFunctionAttrsLegacyPass());
568
569 // Infer attributes on declarations, call sites, arguments, etc.
570 if (AttributorRun & AttributorRunOption::MODULE)
571 MPM.add(createAttributorLegacyPass());
572
573 addExtensionsToPM(EP_ModuleOptimizerEarly, MPM);
574
575 if (OptLevel > 2)
576 MPM.add(createCallSiteSplittingPass());
577
578 MPM.add(createIPSCCPPass()); // IP SCCP
579 MPM.add(createCalledValuePropagationPass());
580
581 MPM.add(createGlobalOptimizerPass()); // Optimize out global vars
582 // Promote any localized global vars.
583 MPM.add(createPromoteMemoryToRegisterPass());
584
585 MPM.add(createDeadArgEliminationPass()); // Dead argument elimination
586
587 MPM.add(createInstructionCombiningPass()); // Clean up after IPCP & DAE
588 addExtensionsToPM(EP_Peephole, MPM);
589 MPM.add(createCFGSimplificationPass()); // Clean up after IPCP & DAE
590
591 // For SamplePGO in ThinLTO compile phase, we do not want to do indirect
592 // call promotion as it will change the CFG too much to make the 2nd
593 // profile annotation in backend more difficult.
594 // PGO instrumentation is added during the compile phase for ThinLTO, do
595 // not run it a second time
596 if (DefaultOrPreLinkPipeline && !PrepareForThinLTOUsingPGOSampleProfile)
597 addPGOInstrPasses(MPM);
598
599 // Create profile COMDAT variables. Lld linker wants to see all variables
600 // before the LTO/ThinLTO link since it needs to resolve symbols/comdats.
601 if (!PerformThinLTO && EnablePGOCSInstrGen)
602 MPM.add(createPGOInstrumentationGenCreateVarLegacyPass(PGOInstrGen));
603
604 // We add a module alias analysis pass here. In part due to bugs in the
605 // analysis infrastructure this "works" in that the analysis stays alive
606 // for the entire SCC pass run below.
607 MPM.add(createGlobalsAAWrapperPass());
608
609 // Start of CallGraph SCC passes.
610 MPM.add(createPruneEHPass()); // Remove dead EH info
611 bool RunInliner = false;
612 if (Inliner) {
613 MPM.add(Inliner);
614 Inliner = nullptr;
615 RunInliner = true;
616 }
617
618 // Infer attributes on declarations, call sites, arguments, etc. for an SCC.
619 if (AttributorRun & AttributorRunOption::CGSCC)
620 MPM.add(createAttributorCGSCCLegacyPass());
621
622 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
623 // there are no OpenMP runtime calls present in the module.
624 if (OptLevel > 1)
625 MPM.add(createOpenMPOptLegacyPass());
626
627 MPM.add(createPostOrderFunctionAttrsLegacyPass());
628 if (OptLevel > 2)
629 MPM.add(createArgumentPromotionPass()); // Scalarize uninlined fn args
630
631 addExtensionsToPM(EP_CGSCCOptimizerLate, MPM);
632 addFunctionSimplificationPasses(MPM);
633
634 // FIXME: This is a HACK! The inliner pass above implicitly creates a CGSCC
635 // pass manager that we are specifically trying to avoid. To prevent this
636 // we must insert a no-op module pass to reset the pass manager.
637 MPM.add(createBarrierNoopPass());
638
639 if (RunPartialInlining)
640 MPM.add(createPartialInliningPass());
641
642 if (OptLevel > 1 && !PrepareForLTO && !PrepareForThinLTO)
643 // Remove avail extern fns and globals definitions if we aren't
644 // compiling an object file for later LTO. For LTO we want to preserve
645 // these so they are eligible for inlining at link-time. Note if they
646 // are unreferenced they will be removed by GlobalDCE later, so
647 // this only impacts referenced available externally globals.
648 // Eventually they will be suppressed during codegen, but eliminating
649 // here enables more opportunity for GlobalDCE as it may make
650 // globals referenced by available external functions dead
651 // and saves running remaining passes on the eliminated functions.
652 MPM.add(createEliminateAvailableExternallyPass());
653
654 // CSFDO instrumentation and use pass. Don't invoke this for Prepare pass
655 // for LTO and ThinLTO -- The actual pass will be called after all inlines
656 // are performed.
657 // Need to do this after COMDAT variables have been eliminated,
658 // (i.e. after EliminateAvailableExternallyPass).
659 if (!(PrepareForLTO || PrepareForThinLTO))
660 addPGOInstrPasses(MPM, /* IsCS */ true);
661
662 if (EnableOrderFileInstrumentation)
663 MPM.add(createInstrOrderFilePass());
664
665 MPM.add(createReversePostOrderFunctionAttrsPass());
666
667 // The inliner performs some kind of dead code elimination as it goes,
668 // but there are cases that are not really caught by it. We might
669 // at some point consider teaching the inliner about them, but it
670 // is OK for now to run GlobalOpt + GlobalDCE in tandem as their
671 // benefits generally outweight the cost, making the whole pipeline
672 // faster.
673 if (RunInliner) {
674 MPM.add(createGlobalOptimizerPass());
675 MPM.add(createGlobalDCEPass());
676 }
677
678 // If we are planning to perform ThinLTO later, let's not bloat the code with
679 // unrolling/vectorization/... now. We'll first run the inliner + CGSCC passes
680 // during ThinLTO and perform the rest of the optimizations afterward.
681 if (PrepareForThinLTO) {
682 // Ensure we perform any last passes, but do so before renaming anonymous
683 // globals in case the passes add any.
684 addExtensionsToPM(EP_OptimizerLast, MPM);
685 MPM.add(createCanonicalizeAliasesPass());
686 // Rename anon globals to be able to export them in the summary.
687 MPM.add(createNameAnonGlobalPass());
688 return;
689 }
690
691 if (PerformThinLTO)
692 // Optimize globals now when performing ThinLTO, this enables more
693 // optimizations later.
694 MPM.add(createGlobalOptimizerPass());
695
696 // Scheduling LoopVersioningLICM when inlining is over, because after that
697 // we may see more accurate aliasing. Reason to run this late is that too
698 // early versioning may prevent further inlining due to increase of code
699 // size. By placing it just after inlining other optimizations which runs
700 // later might get benefit of no-alias assumption in clone loop.
701 if (UseLoopVersioningLICM) {
702 MPM.add(createLoopVersioningLICMPass()); // Do LoopVersioningLICM
703 MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
704 }
705
706 // We add a fresh GlobalsModRef run at this point. This is particularly
707 // useful as the above will have inlined, DCE'ed, and function-attr
708 // propagated everything. We should at this point have a reasonably minimal
709 // and richly annotated call graph. By computing aliasing and mod/ref
710 // information for all local globals here, the late loop passes and notably
711 // the vectorizer will be able to use them to help recognize vectorizable
712 // memory operations.
713 //
714 // Note that this relies on a bug in the pass manager which preserves
715 // a module analysis into a function pass pipeline (and throughout it) so
716 // long as the first function pass doesn't invalidate the module analysis.
717 // Thus both Float2Int and LoopRotate have to preserve AliasAnalysis for
718 // this to work. Fortunately, it is trivial to preserve AliasAnalysis
719 // (doing nothing preserves it as it is required to be conservatively
720 // correct in the face of IR changes).
721 MPM.add(createGlobalsAAWrapperPass());
722
723 MPM.add(createFloat2IntPass());
724 MPM.add(createLowerConstantIntrinsicsPass());
725
726 if (EnableMatrix) {
727 MPM.add(createLowerMatrixIntrinsicsPass());
728 // CSE the pointer arithmetic of the column vectors. This allows alias
729 // analysis to establish no-aliasing between loads and stores of different
730 // columns of the same matrix.
731 MPM.add(createEarlyCSEPass(false));
732 }
733
734 addExtensionsToPM(EP_VectorizerStart, MPM);
735
736 // Re-rotate loops in all our loop nests. These may have fallout out of
737 // rotated form due to GVN or other transformations, and the vectorizer relies
738 // on the rotated form. Disable header duplication at -Oz.
739 MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1));
740
741 // Distribute loops to allow partial vectorization. I.e. isolate dependences
742 // into separate loop that would otherwise inhibit vectorization. This is
743 // currently only performed for loops marked with the metadata
744 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
745 MPM.add(createLoopDistributePass());
746
747 MPM.add(createLoopVectorizePass(!LoopsInterleaved, !LoopVectorize));
748
749 // Eliminate loads by forwarding stores from the previous iteration to loads
750 // of the current iteration.
751 MPM.add(createLoopLoadEliminationPass());
752
753 // FIXME: Because of #pragma vectorize enable, the passes below are always
754 // inserted in the pipeline, even when the vectorizer doesn't run (ex. when
755 // on -O1 and no #pragma is found). Would be good to have these two passes
756 // as function calls, so that we can only pass them when the vectorizer
757 // changed the code.
758 MPM.add(createInstructionCombiningPass());
759 if (OptLevel > 1 && ExtraVectorizerPasses) {
760 // At higher optimization levels, try to clean up any runtime overlap and
761 // alignment checks inserted by the vectorizer. We want to track correllated
762 // runtime checks for two inner loops in the same outer loop, fold any
763 // common computations, hoist loop-invariant aspects out of any outer loop,
764 // and unswitch the runtime checks if possible. Once hoisted, we may have
765 // dead (or speculatable) control flows or more combining opportunities.
766 MPM.add(createEarlyCSEPass());
767 MPM.add(createCorrelatedValuePropagationPass());
768 MPM.add(createInstructionCombiningPass());
769 MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
770 MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3, DivergentTarget));
771 MPM.add(createCFGSimplificationPass());
772 MPM.add(createInstructionCombiningPass());
773 }
774
775 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
776 // GVN, loop transforms, and others have already run, so it's now better to
777 // convert to more optimized IR using more aggressive simplify CFG options.
778 // The extra sinking transform can create larger basic blocks, so do this
779 // before SLP vectorization.
780 MPM.add(createCFGSimplificationPass(1, true, true, false, true));
781
782 if (SLPVectorize) {
783 MPM.add(createSLPVectorizerPass()); // Vectorize parallel scalar chains.
784 if (OptLevel > 1 && ExtraVectorizerPasses) {
785 MPM.add(createEarlyCSEPass());
786 }
787 }
788
789 // Enhance/cleanup vector code.
790 MPM.add(createVectorCombinePass());
791
792 addExtensionsToPM(EP_Peephole, MPM);
793 MPM.add(createInstructionCombiningPass());
794
795 if (EnableUnrollAndJam && !DisableUnrollLoops) {
796 // Unroll and Jam. We do this before unroll but need to be in a separate
797 // loop pass manager in order for the outer loop to be processed by
798 // unroll and jam before the inner loop is unrolled.
799 MPM.add(createLoopUnrollAndJamPass(OptLevel));
800 }
801
802 // Unroll small loops
803 MPM.add(createLoopUnrollPass(OptLevel, DisableUnrollLoops,
804 ForgetAllSCEVInLoopUnroll));
805
806 if (!DisableUnrollLoops) {
807 // LoopUnroll may generate some redundency to cleanup.
808 MPM.add(createInstructionCombiningPass());
809
810 // Runtime unrolling will introduce runtime check in loop prologue. If the
811 // unrolled loop is a inner loop, then the prologue will be inside the
812 // outer loop. LICM pass can help to promote the runtime check out if the
813 // checked value is loop invariant.
814 MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
815 }
816
817 MPM.add(createWarnMissedTransformationsPass());
818
819 // After vectorization and unrolling, assume intrinsics may tell us more
820 // about pointer alignments.
821 MPM.add(createAlignmentFromAssumptionsPass());
822
823 // FIXME: We shouldn't bother with this anymore.
824 MPM.add(createStripDeadPrototypesPass()); // Get rid of dead prototypes
825
826 // GlobalOpt already deletes dead functions and globals, at -O2 try a
827 // late pass of GlobalDCE. It is capable of deleting dead cycles.
828 if (OptLevel > 1) {
829 MPM.add(createGlobalDCEPass()); // Remove dead fns and globals.
830 MPM.add(createConstantMergePass()); // Merge dup global constants
831 }
832
833 // See comment in the new PM for justification of scheduling splitting at
834 // this stage (\ref buildModuleSimplificationPipeline).
835 if (EnableHotColdSplit && !(PrepareForLTO || PrepareForThinLTO))
836 MPM.add(createHotColdSplittingPass());
837
838 if (MergeFunctions)
839 MPM.add(createMergeFunctionsPass());
840
841 // Add Module flag "CG Profile" based on Branch Frequency Information.
842 if (CallGraphProfile)
843 MPM.add(createCGProfileLegacyPass());
844
845 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
846 // canonicalization pass that enables other optimizations. As a result,
847 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
848 // result too early.
849 MPM.add(createLoopSinkPass());
850 // Get rid of LCSSA nodes.
851 MPM.add(createInstSimplifyLegacyPass());
852
853 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
854 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
855 // flattening of blocks.
856 MPM.add(createDivRemPairsPass());
857
858 // LoopSink (and other loop passes since the last simplifyCFG) might have
859 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
860 MPM.add(createCFGSimplificationPass());
861
862 addExtensionsToPM(EP_OptimizerLast, MPM);
863
864 if (PrepareForLTO) {
865 MPM.add(createCanonicalizeAliasesPass());
866 // Rename anon globals to be able to handle them in the summary
867 MPM.add(createNameAnonGlobalPass());
868 }
869 }
870
addLTOOptimizationPasses(legacy::PassManagerBase & PM)871 void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) {
872 // Load sample profile before running the LTO optimization pipeline.
873 if (!PGOSampleUse.empty()) {
874 PM.add(createPruneEHPass());
875 PM.add(createSampleProfileLoaderPass(PGOSampleUse));
876 }
877
878 // Remove unused virtual tables to improve the quality of code generated by
879 // whole-program devirtualization and bitset lowering.
880 PM.add(createGlobalDCEPass());
881
882 // Provide AliasAnalysis services for optimizations.
883 addInitialAliasAnalysisPasses(PM);
884
885 // Allow forcing function attributes as a debugging and tuning aid.
886 PM.add(createForceFunctionAttrsLegacyPass());
887
888 // Infer attributes about declarations if possible.
889 PM.add(createInferFunctionAttrsLegacyPass());
890
891 if (OptLevel > 1) {
892 // Split call-site with more constrained arguments.
893 PM.add(createCallSiteSplittingPass());
894
895 // Indirect call promotion. This should promote all the targets that are
896 // left by the earlier promotion pass that promotes intra-module targets.
897 // This two-step promotion is to save the compile time. For LTO, it should
898 // produce the same result as if we only do promotion here.
899 PM.add(
900 createPGOIndirectCallPromotionLegacyPass(true, !PGOSampleUse.empty()));
901
902 // Propagate constants at call sites into the functions they call. This
903 // opens opportunities for globalopt (and inlining) by substituting function
904 // pointers passed as arguments to direct uses of functions.
905 PM.add(createIPSCCPPass());
906
907 // Attach metadata to indirect call sites indicating the set of functions
908 // they may target at run-time. This should follow IPSCCP.
909 PM.add(createCalledValuePropagationPass());
910
911 // Infer attributes on declarations, call sites, arguments, etc.
912 if (AttributorRun & AttributorRunOption::MODULE)
913 PM.add(createAttributorLegacyPass());
914 }
915
916 // Infer attributes about definitions. The readnone attribute in particular is
917 // required for virtual constant propagation.
918 PM.add(createPostOrderFunctionAttrsLegacyPass());
919 PM.add(createReversePostOrderFunctionAttrsPass());
920
921 // Split globals using inrange annotations on GEP indices. This can help
922 // improve the quality of generated code when virtual constant propagation or
923 // control flow integrity are enabled.
924 PM.add(createGlobalSplitPass());
925
926 // Apply whole-program devirtualization and virtual constant propagation.
927 PM.add(createWholeProgramDevirtPass(ExportSummary, nullptr));
928
929 // That's all we need at opt level 1.
930 if (OptLevel == 1)
931 return;
932
933 // Now that we internalized some globals, see if we can hack on them!
934 PM.add(createGlobalOptimizerPass());
935 // Promote any localized global vars.
936 PM.add(createPromoteMemoryToRegisterPass());
937
938 // Linking modules together can lead to duplicated global constants, only
939 // keep one copy of each constant.
940 PM.add(createConstantMergePass());
941
942 // Remove unused arguments from functions.
943 PM.add(createDeadArgEliminationPass());
944
945 // Reduce the code after globalopt and ipsccp. Both can open up significant
946 // simplification opportunities, and both can propagate functions through
947 // function pointers. When this happens, we often have to resolve varargs
948 // calls, etc, so let instcombine do this.
949 if (OptLevel > 2)
950 PM.add(createAggressiveInstCombinerPass());
951 PM.add(createInstructionCombiningPass());
952 addExtensionsToPM(EP_Peephole, PM);
953
954 // Inline small functions
955 bool RunInliner = Inliner;
956 if (RunInliner) {
957 PM.add(Inliner);
958 Inliner = nullptr;
959 }
960
961 PM.add(createPruneEHPass()); // Remove dead EH info.
962
963 // CSFDO instrumentation and use pass.
964 addPGOInstrPasses(PM, /* IsCS */ true);
965
966 // Infer attributes on declarations, call sites, arguments, etc. for an SCC.
967 if (AttributorRun & AttributorRunOption::CGSCC)
968 PM.add(createAttributorCGSCCLegacyPass());
969
970 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
971 // there are no OpenMP runtime calls present in the module.
972 if (OptLevel > 1)
973 PM.add(createOpenMPOptLegacyPass());
974
975 // Optimize globals again if we ran the inliner.
976 if (RunInliner)
977 PM.add(createGlobalOptimizerPass());
978 PM.add(createGlobalDCEPass()); // Remove dead functions.
979
980 // If we didn't decide to inline a function, check to see if we can
981 // transform it to pass arguments by value instead of by reference.
982 PM.add(createArgumentPromotionPass());
983
984 // The IPO passes may leave cruft around. Clean up after them.
985 PM.add(createInstructionCombiningPass());
986 addExtensionsToPM(EP_Peephole, PM);
987 PM.add(createJumpThreadingPass());
988
989 // Break up allocas
990 PM.add(createSROAPass());
991
992 // LTO provides additional opportunities for tailcall elimination due to
993 // link-time inlining, and visibility of nocapture attribute.
994 if (OptLevel > 1)
995 PM.add(createTailCallEliminationPass());
996
997 // Infer attributes on declarations, call sites, arguments, etc.
998 PM.add(createPostOrderFunctionAttrsLegacyPass()); // Add nocapture.
999 // Run a few AA driven optimizations here and now, to cleanup the code.
1000 PM.add(createGlobalsAAWrapperPass()); // IP alias analysis.
1001
1002 PM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
1003 PM.add(createMergedLoadStoreMotionPass()); // Merge ld/st in diamonds.
1004 PM.add(NewGVN ? createNewGVNPass()
1005 : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies.
1006 PM.add(createMemCpyOptPass()); // Remove dead memcpys.
1007
1008 // Nuke dead stores.
1009 PM.add(createDeadStoreEliminationPass());
1010
1011 // More loops are countable; try to optimize them.
1012 PM.add(createIndVarSimplifyPass());
1013 PM.add(createLoopDeletionPass());
1014 if (EnableLoopInterchange)
1015 PM.add(createLoopInterchangePass());
1016
1017 // Unroll small loops
1018 PM.add(createSimpleLoopUnrollPass(OptLevel, DisableUnrollLoops,
1019 ForgetAllSCEVInLoopUnroll));
1020 PM.add(createLoopVectorizePass(true, !LoopVectorize));
1021 // The vectorizer may have significantly shortened a loop body; unroll again.
1022 PM.add(createLoopUnrollPass(OptLevel, DisableUnrollLoops,
1023 ForgetAllSCEVInLoopUnroll));
1024
1025 PM.add(createWarnMissedTransformationsPass());
1026
1027 // Now that we've optimized loops (in particular loop induction variables),
1028 // we may have exposed more scalar opportunities. Run parts of the scalar
1029 // optimizer again at this point.
1030 PM.add(createInstructionCombiningPass()); // Initial cleanup
1031 PM.add(createCFGSimplificationPass()); // if-convert
1032 PM.add(createSCCPPass()); // Propagate exposed constants
1033 PM.add(createInstructionCombiningPass()); // Clean up again
1034 PM.add(createBitTrackingDCEPass());
1035
1036 // More scalar chains could be vectorized due to more alias information
1037 if (SLPVectorize)
1038 PM.add(createSLPVectorizerPass()); // Vectorize parallel scalar chains.
1039
1040 PM.add(createVectorCombinePass()); // Clean up partial vectorization.
1041
1042 // After vectorization, assume intrinsics may tell us more about pointer
1043 // alignments.
1044 PM.add(createAlignmentFromAssumptionsPass());
1045
1046 // Cleanup and simplify the code after the scalar optimizations.
1047 PM.add(createInstructionCombiningPass());
1048 addExtensionsToPM(EP_Peephole, PM);
1049
1050 PM.add(createJumpThreadingPass());
1051 }
1052
addLateLTOOptimizationPasses(legacy::PassManagerBase & PM)1053 void PassManagerBuilder::addLateLTOOptimizationPasses(
1054 legacy::PassManagerBase &PM) {
1055 // See comment in the new PM for justification of scheduling splitting at
1056 // this stage (\ref buildLTODefaultPipeline).
1057 if (EnableHotColdSplit)
1058 PM.add(createHotColdSplittingPass());
1059
1060 // Delete basic blocks, which optimization passes may have killed.
1061 PM.add(createCFGSimplificationPass());
1062
1063 // Drop bodies of available externally objects to improve GlobalDCE.
1064 PM.add(createEliminateAvailableExternallyPass());
1065
1066 // Now that we have optimized the program, discard unreachable functions.
1067 PM.add(createGlobalDCEPass());
1068
1069 // FIXME: this is profitable (for compiler time) to do at -O0 too, but
1070 // currently it damages debug info.
1071 if (MergeFunctions)
1072 PM.add(createMergeFunctionsPass());
1073 }
1074
populateThinLTOPassManager(legacy::PassManagerBase & PM)1075 void PassManagerBuilder::populateThinLTOPassManager(
1076 legacy::PassManagerBase &PM) {
1077 PerformThinLTO = true;
1078 if (LibraryInfo)
1079 PM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo));
1080
1081 if (VerifyInput)
1082 PM.add(createVerifierPass());
1083
1084 if (ImportSummary) {
1085 // This pass imports type identifier resolutions for whole-program
1086 // devirtualization and CFI. It must run early because other passes may
1087 // disturb the specific instruction patterns that these passes look for,
1088 // creating dependencies on resolutions that may not appear in the summary.
1089 //
1090 // For example, GVN may transform the pattern assume(type.test) appearing in
1091 // two basic blocks into assume(phi(type.test, type.test)), which would
1092 // transform a dependency on a WPD resolution into a dependency on a type
1093 // identifier resolution for CFI.
1094 //
1095 // Also, WPD has access to more precise information than ICP and can
1096 // devirtualize more effectively, so it should operate on the IR first.
1097 PM.add(createWholeProgramDevirtPass(nullptr, ImportSummary));
1098 PM.add(createLowerTypeTestsPass(nullptr, ImportSummary));
1099 }
1100
1101 populateModulePassManager(PM);
1102
1103 if (VerifyOutput)
1104 PM.add(createVerifierPass());
1105 PerformThinLTO = false;
1106 }
1107
populateLTOPassManager(legacy::PassManagerBase & PM)1108 void PassManagerBuilder::populateLTOPassManager(legacy::PassManagerBase &PM) {
1109 if (LibraryInfo)
1110 PM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo));
1111
1112 if (VerifyInput)
1113 PM.add(createVerifierPass());
1114
1115 addExtensionsToPM(EP_FullLinkTimeOptimizationEarly, PM);
1116
1117 if (OptLevel != 0)
1118 addLTOOptimizationPasses(PM);
1119 else {
1120 // The whole-program-devirt pass needs to run at -O0 because only it knows
1121 // about the llvm.type.checked.load intrinsic: it needs to both lower the
1122 // intrinsic itself and handle it in the summary.
1123 PM.add(createWholeProgramDevirtPass(ExportSummary, nullptr));
1124 }
1125
1126 // Create a function that performs CFI checks for cross-DSO calls with targets
1127 // in the current module.
1128 PM.add(createCrossDSOCFIPass());
1129
1130 // Lower type metadata and the type.test intrinsic. This pass supports Clang's
1131 // control flow integrity mechanisms (-fsanitize=cfi*) and needs to run at
1132 // link time if CFI is enabled. The pass does nothing if CFI is disabled.
1133 PM.add(createLowerTypeTestsPass(ExportSummary, nullptr));
1134 // Run a second time to clean up any type tests left behind by WPD for use
1135 // in ICP (which is performed earlier than this in the regular LTO pipeline).
1136 PM.add(createLowerTypeTestsPass(nullptr, nullptr, true));
1137
1138 if (OptLevel != 0)
1139 addLateLTOOptimizationPasses(PM);
1140
1141 addExtensionsToPM(EP_FullLinkTimeOptimizationLast, PM);
1142
1143 if (VerifyOutput)
1144 PM.add(createVerifierPass());
1145 }
1146
LLVMPassManagerBuilderCreate()1147 LLVMPassManagerBuilderRef LLVMPassManagerBuilderCreate() {
1148 PassManagerBuilder *PMB = new PassManagerBuilder();
1149 return wrap(PMB);
1150 }
1151
LLVMPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB)1152 void LLVMPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) {
1153 PassManagerBuilder *Builder = unwrap(PMB);
1154 delete Builder;
1155 }
1156
1157 void
LLVMPassManagerBuilderSetOptLevel(LLVMPassManagerBuilderRef PMB,unsigned OptLevel)1158 LLVMPassManagerBuilderSetOptLevel(LLVMPassManagerBuilderRef PMB,
1159 unsigned OptLevel) {
1160 PassManagerBuilder *Builder = unwrap(PMB);
1161 Builder->OptLevel = OptLevel;
1162 }
1163
1164 void
LLVMPassManagerBuilderSetSizeLevel(LLVMPassManagerBuilderRef PMB,unsigned SizeLevel)1165 LLVMPassManagerBuilderSetSizeLevel(LLVMPassManagerBuilderRef PMB,
1166 unsigned SizeLevel) {
1167 PassManagerBuilder *Builder = unwrap(PMB);
1168 Builder->SizeLevel = SizeLevel;
1169 }
1170
1171 void
LLVMPassManagerBuilderSetDisableUnitAtATime(LLVMPassManagerBuilderRef PMB,LLVMBool Value)1172 LLVMPassManagerBuilderSetDisableUnitAtATime(LLVMPassManagerBuilderRef PMB,
1173 LLVMBool Value) {
1174 // NOTE: The DisableUnitAtATime switch has been removed.
1175 }
1176
1177 void
LLVMPassManagerBuilderSetDisableUnrollLoops(LLVMPassManagerBuilderRef PMB,LLVMBool Value)1178 LLVMPassManagerBuilderSetDisableUnrollLoops(LLVMPassManagerBuilderRef PMB,
1179 LLVMBool Value) {
1180 PassManagerBuilder *Builder = unwrap(PMB);
1181 Builder->DisableUnrollLoops = Value;
1182 }
1183
1184 void
LLVMPassManagerBuilderSetDisableSimplifyLibCalls(LLVMPassManagerBuilderRef PMB,LLVMBool Value)1185 LLVMPassManagerBuilderSetDisableSimplifyLibCalls(LLVMPassManagerBuilderRef PMB,
1186 LLVMBool Value) {
1187 // NOTE: The simplify-libcalls pass has been removed.
1188 }
1189
1190 void
LLVMPassManagerBuilderUseInlinerWithThreshold(LLVMPassManagerBuilderRef PMB,unsigned Threshold)1191 LLVMPassManagerBuilderUseInlinerWithThreshold(LLVMPassManagerBuilderRef PMB,
1192 unsigned Threshold) {
1193 PassManagerBuilder *Builder = unwrap(PMB);
1194 Builder->Inliner = createFunctionInliningPass(Threshold);
1195 }
1196
1197 void
LLVMPassManagerBuilderPopulateFunctionPassManager(LLVMPassManagerBuilderRef PMB,LLVMPassManagerRef PM)1198 LLVMPassManagerBuilderPopulateFunctionPassManager(LLVMPassManagerBuilderRef PMB,
1199 LLVMPassManagerRef PM) {
1200 PassManagerBuilder *Builder = unwrap(PMB);
1201 legacy::FunctionPassManager *FPM = unwrap<legacy::FunctionPassManager>(PM);
1202 Builder->populateFunctionPassManager(*FPM);
1203 }
1204
1205 void
LLVMPassManagerBuilderPopulateModulePassManager(LLVMPassManagerBuilderRef PMB,LLVMPassManagerRef PM)1206 LLVMPassManagerBuilderPopulateModulePassManager(LLVMPassManagerBuilderRef PMB,
1207 LLVMPassManagerRef PM) {
1208 PassManagerBuilder *Builder = unwrap(PMB);
1209 legacy::PassManagerBase *MPM = unwrap(PM);
1210 Builder->populateModulePassManager(*MPM);
1211 }
1212
LLVMPassManagerBuilderPopulateLTOPassManager(LLVMPassManagerBuilderRef PMB,LLVMPassManagerRef PM,LLVMBool Internalize,LLVMBool RunInliner)1213 void LLVMPassManagerBuilderPopulateLTOPassManager(LLVMPassManagerBuilderRef PMB,
1214 LLVMPassManagerRef PM,
1215 LLVMBool Internalize,
1216 LLVMBool RunInliner) {
1217 PassManagerBuilder *Builder = unwrap(PMB);
1218 legacy::PassManagerBase *LPM = unwrap(PM);
1219
1220 // A small backwards compatibility hack. populateLTOPassManager used to take
1221 // an RunInliner option.
1222 if (RunInliner && !Builder->Inliner)
1223 Builder->Inliner = createFunctionInliningPass();
1224
1225 Builder->populateLTOPassManager(*LPM);
1226 }
1227