1 //===- Construction of codegen pass pipelines ------------------*- C++ -*--===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// Interfaces for registering analysis passes, producing common pass manager
11 /// configurations, and parsing of pass pipelines.
12 ///
13 //===----------------------------------------------------------------------===//
14
15 #ifndef LLVM_CODEGEN_CODEGENPASSBUILDER_H
16 #define LLVM_CODEGEN_CODEGENPASSBUILDER_H
17
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/Analysis/AliasAnalysis.h"
21 #include "llvm/Analysis/BasicAliasAnalysis.h"
22 #include "llvm/Analysis/ProfileSummaryInfo.h"
23 #include "llvm/Analysis/ScopedNoAliasAA.h"
24 #include "llvm/Analysis/TargetTransformInfo.h"
25 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
26 #include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
27 #include "llvm/CodeGen/CallBrPrepare.h"
28 #include "llvm/CodeGen/CodeGenPrepare.h"
29 #include "llvm/CodeGen/DwarfEHPrepare.h"
30 #include "llvm/CodeGen/ExpandMemCmp.h"
31 #include "llvm/CodeGen/ExpandReductions.h"
32 #include "llvm/CodeGen/GCMetadata.h"
33 #include "llvm/CodeGen/GlobalMerge.h"
34 #include "llvm/CodeGen/IndirectBrExpand.h"
35 #include "llvm/CodeGen/InterleavedAccess.h"
36 #include "llvm/CodeGen/InterleavedLoadCombine.h"
37 #include "llvm/CodeGen/JMCInstrumenter.h"
38 #include "llvm/CodeGen/LowerEmuTLS.h"
39 #include "llvm/CodeGen/MachinePassManager.h"
40 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
41 #include "llvm/CodeGen/ReplaceWithVeclib.h"
42 #include "llvm/CodeGen/SafeStack.h"
43 #include "llvm/CodeGen/SelectOptimize.h"
44 #include "llvm/CodeGen/ShadowStackGCLowering.h"
45 #include "llvm/CodeGen/SjLjEHPrepare.h"
46 #include "llvm/CodeGen/StackProtector.h"
47 #include "llvm/CodeGen/TargetPassConfig.h"
48 #include "llvm/CodeGen/UnreachableBlockElim.h"
49 #include "llvm/CodeGen/WasmEHPrepare.h"
50 #include "llvm/CodeGen/WinEHPrepare.h"
51 #include "llvm/IR/PassManager.h"
52 #include "llvm/IR/Verifier.h"
53 #include "llvm/IRPrinter/IRPrintingPasses.h"
54 #include "llvm/MC/MCAsmInfo.h"
55 #include "llvm/MC/MCTargetOptions.h"
56 #include "llvm/Support/CodeGen.h"
57 #include "llvm/Support/Debug.h"
58 #include "llvm/Support/Error.h"
59 #include "llvm/Support/ErrorHandling.h"
60 #include "llvm/Target/CGPassBuilderOption.h"
61 #include "llvm/Target/TargetMachine.h"
62 #include "llvm/Transforms/CFGuard.h"
63 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
64 #include "llvm/Transforms/Scalar/LoopPassManager.h"
65 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
66 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
67 #include "llvm/Transforms/Scalar/MergeICmps.h"
68 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
69 #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
70 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
71 #include "llvm/Transforms/Utils/LowerInvoke.h"
72 #include <cassert>
73 #include <type_traits>
74 #include <utility>
75
76 namespace llvm {
77
78 // FIXME: Dummy target independent passes definitions that have not yet been
79 // ported to new pass manager. Once they do, remove these.
80 #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
81 struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
82 template <typename... Ts> PASS_NAME(Ts &&...) {} \
83 PreservedAnalyses run(Function &, FunctionAnalysisManager &) { \
84 return PreservedAnalyses::all(); \
85 } \
86 };
87 #define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
88 struct PASS_NAME : public MachinePassInfoMixin<PASS_NAME> { \
89 template <typename... Ts> PASS_NAME(Ts &&...) {} \
90 Error run(Module &, MachineFunctionAnalysisManager &) { \
91 return Error::success(); \
92 } \
93 PreservedAnalyses run(MachineFunction &, \
94 MachineFunctionAnalysisManager &) { \
95 llvm_unreachable("this api is to make new PM api happy"); \
96 } \
97 static MachinePassKey Key; \
98 };
99 #define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
100 struct PASS_NAME : public MachinePassInfoMixin<PASS_NAME> { \
101 template <typename... Ts> PASS_NAME(Ts &&...) {} \
102 PreservedAnalyses run(MachineFunction &, \
103 MachineFunctionAnalysisManager &) { \
104 return PreservedAnalyses::all(); \
105 } \
106 static MachinePassKey Key; \
107 };
108 #define DUMMY_MACHINE_FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
109 struct PASS_NAME : public AnalysisInfoMixin<PASS_NAME> { \
110 template <typename... Ts> PASS_NAME(Ts &&...) {} \
111 using Result = struct {}; \
112 template <typename IRUnitT, typename AnalysisManagerT, \
113 typename... ExtraArgTs> \
114 Result run(IRUnitT &, AnalysisManagerT &, ExtraArgTs &&...) { \
115 return {}; \
116 } \
117 static AnalysisKey Key; \
118 };
119 #include "llvm/CodeGen/MachinePassRegistry.def"
120
121 /// This class provides access to building LLVM's passes.
122 ///
123 /// Its members provide the baseline state available to passes during their
124 /// construction. The \c MachinePassRegistry.def file specifies how to construct
125 /// all of the built-in passes, and those may reference these members during
126 /// construction.
127 template <typename DerivedT> class CodeGenPassBuilder {
128 public:
CodeGenPassBuilder(LLVMTargetMachine & TM,CGPassBuilderOption Opts,PassInstrumentationCallbacks * PIC)129 explicit CodeGenPassBuilder(LLVMTargetMachine &TM, CGPassBuilderOption Opts,
130 PassInstrumentationCallbacks *PIC)
131 : TM(TM), Opt(Opts), PIC(PIC) {
132 // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve
133 // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID)
134
135 // Target should override TM.Options.EnableIPRA in their target-specific
136 // LLVMTM ctor. See TargetMachine::setGlobalISel for example.
137 if (Opt.EnableIPRA)
138 TM.Options.EnableIPRA = *Opt.EnableIPRA;
139
140 if (Opt.EnableGlobalISelAbort)
141 TM.Options.GlobalISelAbort = *Opt.EnableGlobalISelAbort;
142
143 if (!Opt.OptimizeRegAlloc)
144 Opt.OptimizeRegAlloc = getOptLevel() != CodeGenOptLevel::None;
145 }
146
147 Error buildPipeline(ModulePassManager &MPM, MachineFunctionPassManager &MFPM,
148 raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
149 CodeGenFileType FileType) const;
150
151 void registerModuleAnalyses(ModuleAnalysisManager &) const;
152 void registerFunctionAnalyses(FunctionAnalysisManager &) const;
153 void registerMachineFunctionAnalyses(MachineFunctionAnalysisManager &) const;
154 std::pair<StringRef, bool> getPassNameFromLegacyName(StringRef) const;
155
registerAnalyses(MachineFunctionAnalysisManager & MFAM)156 void registerAnalyses(MachineFunctionAnalysisManager &MFAM) const {
157 registerModuleAnalyses(*MFAM.MAM);
158 registerFunctionAnalyses(*MFAM.FAM);
159 registerMachineFunctionAnalyses(MFAM);
160 }
161
getPassInstrumentationCallbacks()162 PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const {
163 return PIC;
164 }
165
166 protected:
167 template <typename PassT> using has_key_t = decltype(PassT::Key);
168
169 template <typename PassT>
170 using is_module_pass_t = decltype(std::declval<PassT &>().run(
171 std::declval<Module &>(), std::declval<ModuleAnalysisManager &>()));
172
173 template <typename PassT>
174 using is_function_pass_t = decltype(std::declval<PassT &>().run(
175 std::declval<Function &>(), std::declval<FunctionAnalysisManager &>()));
176
177 // Function object to maintain state while adding codegen IR passes.
178 class AddIRPass {
179 public:
AddIRPass(ModulePassManager & MPM,const DerivedT & PB)180 AddIRPass(ModulePassManager &MPM, const DerivedT &PB) : MPM(MPM), PB(PB) {}
~AddIRPass()181 ~AddIRPass() {
182 if (!FPM.isEmpty())
183 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
184 }
185
186 template <typename PassT>
operator()187 void operator()(PassT &&Pass, StringRef Name = PassT::name()) {
188 static_assert((is_detected<is_function_pass_t, PassT>::value ||
189 is_detected<is_module_pass_t, PassT>::value) &&
190 "Only module pass and function pass are supported.");
191
192 if (!PB.runBeforeAdding(Name))
193 return;
194
195 // Add Function Pass
196 if constexpr (is_detected<is_function_pass_t, PassT>::value) {
197 FPM.addPass(std::forward<PassT>(Pass));
198
199 for (auto &C : PB.AfterCallbacks)
200 C(Name);
201 } else {
202 // Add Module Pass
203 if (!FPM.isEmpty()) {
204 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
205 FPM = FunctionPassManager();
206 }
207
208 MPM.addPass(std::forward<PassT>(Pass));
209
210 for (auto &C : PB.AfterCallbacks)
211 C(Name);
212 }
213 }
214
215 private:
216 ModulePassManager &MPM;
217 FunctionPassManager FPM;
218 const DerivedT &PB;
219 };
220
221 // Function object to maintain state while adding codegen machine passes.
222 class AddMachinePass {
223 public:
AddMachinePass(MachineFunctionPassManager & PM,const DerivedT & PB)224 AddMachinePass(MachineFunctionPassManager &PM, const DerivedT &PB)
225 : PM(PM), PB(PB) {}
226
operator()227 template <typename PassT> void operator()(PassT &&Pass) {
228 static_assert(
229 is_detected<has_key_t, PassT>::value,
230 "Machine function pass must define a static member variable `Key`.");
231
232 if (!PB.runBeforeAdding(PassT::name()))
233 return;
234
235 PM.addPass(std::forward<PassT>(Pass));
236
237 for (auto &C : PB.AfterCallbacks)
238 C(PassT::name());
239 }
240
insertPass(MachinePassKey * ID,PassT Pass)241 template <typename PassT> void insertPass(MachinePassKey *ID, PassT Pass) {
242 PB.AfterCallbacks.emplace_back(
243 [this, ID, Pass = std::move(Pass)](MachinePassKey *PassID) {
244 if (PassID == ID)
245 this->PM.addPass(std::move(Pass));
246 });
247 }
248
releasePM()249 MachineFunctionPassManager releasePM() { return std::move(PM); }
250
251 private:
252 MachineFunctionPassManager &PM;
253 const DerivedT &PB;
254 };
255
256 LLVMTargetMachine &TM;
257 CGPassBuilderOption Opt;
258 PassInstrumentationCallbacks *PIC;
259
260 /// Target override these hooks to parse target-specific analyses.
registerTargetAnalysis(ModuleAnalysisManager &)261 void registerTargetAnalysis(ModuleAnalysisManager &) const {}
registerTargetAnalysis(FunctionAnalysisManager &)262 void registerTargetAnalysis(FunctionAnalysisManager &) const {}
registerTargetAnalysis(MachineFunctionAnalysisManager &)263 void registerTargetAnalysis(MachineFunctionAnalysisManager &) const {}
getTargetPassNameFromLegacyName(StringRef)264 std::pair<StringRef, bool> getTargetPassNameFromLegacyName(StringRef) const {
265 return {"", false};
266 }
267
getTM()268 template <typename TMC> TMC &getTM() const { return static_cast<TMC &>(TM); }
getOptLevel()269 CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); }
270
271 /// Check whether or not GlobalISel should abort on error.
272 /// When this is disabled, GlobalISel will fall back on SDISel instead of
273 /// erroring out.
isGlobalISelAbortEnabled()274 bool isGlobalISelAbortEnabled() const {
275 return TM.Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
276 }
277
278 /// Check whether or not a diagnostic should be emitted when GlobalISel
279 /// uses the fallback path. In other words, it will emit a diagnostic
280 /// when GlobalISel failed and isGlobalISelAbortEnabled is false.
reportDiagnosticWhenGlobalISelFallback()281 bool reportDiagnosticWhenGlobalISelFallback() const {
282 return TM.Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
283 }
284
285 /// addInstSelector - This method should install an instruction selector pass,
286 /// which converts from LLVM code to machine instructions.
addInstSelector(AddMachinePass &)287 Error addInstSelector(AddMachinePass &) const {
288 return make_error<StringError>("addInstSelector is not overridden",
289 inconvertibleErrorCode());
290 }
291
292 /// Target can override this to add GlobalMergePass before all IR passes.
addGlobalMergePass(AddIRPass &)293 void addGlobalMergePass(AddIRPass &) const {}
294
295 /// Add passes that optimize instruction level parallelism for out-of-order
296 /// targets. These passes are run while the machine code is still in SSA
297 /// form, so they can use MachineTraceMetrics to control their heuristics.
298 ///
299 /// All passes added here should preserve the MachineDominatorTree,
300 /// MachineLoopInfo, and MachineTraceMetrics analyses.
addILPOpts(AddMachinePass &)301 void addILPOpts(AddMachinePass &) const {}
302
303 /// This method may be implemented by targets that want to run passes
304 /// immediately before register allocation.
addPreRegAlloc(AddMachinePass &)305 void addPreRegAlloc(AddMachinePass &) const {}
306
307 /// addPreRewrite - Add passes to the optimized register allocation pipeline
308 /// after register allocation is complete, but before virtual registers are
309 /// rewritten to physical registers.
310 ///
311 /// These passes must preserve VirtRegMap and LiveIntervals, and when running
312 /// after RABasic or RAGreedy, they should take advantage of LiveRegMatrix.
313 /// When these passes run, VirtRegMap contains legal physreg assignments for
314 /// all virtual registers.
315 ///
316 /// Note if the target overloads addRegAssignAndRewriteOptimized, this may not
317 /// be honored. This is also not generally used for the fast variant,
318 /// where the allocation and rewriting are done in one pass.
addPreRewrite(AddMachinePass &)319 void addPreRewrite(AddMachinePass &) const {}
320
321 /// Add passes to be run immediately after virtual registers are rewritten
322 /// to physical registers.
addPostRewrite(AddMachinePass &)323 void addPostRewrite(AddMachinePass &) const {}
324
325 /// This method may be implemented by targets that want to run passes after
326 /// register allocation pass pipeline but before prolog-epilog insertion.
addPostRegAlloc(AddMachinePass &)327 void addPostRegAlloc(AddMachinePass &) const {}
328
329 /// This method may be implemented by targets that want to run passes after
330 /// prolog-epilog insertion and before the second instruction scheduling pass.
addPreSched2(AddMachinePass &)331 void addPreSched2(AddMachinePass &) const {}
332
333 /// This pass may be implemented by targets that want to run passes
334 /// immediately before machine code is emitted.
addPreEmitPass(AddMachinePass &)335 void addPreEmitPass(AddMachinePass &) const {}
336
337 /// Targets may add passes immediately before machine code is emitted in this
338 /// callback. This is called even later than `addPreEmitPass`.
339 // FIXME: Rename `addPreEmitPass` to something more sensible given its actual
340 // position and remove the `2` suffix here as this callback is what
341 // `addPreEmitPass` *should* be but in reality isn't.
addPreEmitPass2(AddMachinePass &)342 void addPreEmitPass2(AddMachinePass &) const {}
343
344 /// {{@ For GlobalISel
345 ///
346
347 /// addPreISel - This method should add any "last minute" LLVM->LLVM
348 /// passes (which are run just before instruction selector).
addPreISel(AddIRPass &)349 void addPreISel(AddIRPass &) const {
350 llvm_unreachable("addPreISel is not overridden");
351 }
352
353 /// This method should install an IR translator pass, which converts from
354 /// LLVM code to machine instructions with possibly generic opcodes.
addIRTranslator(AddMachinePass &)355 Error addIRTranslator(AddMachinePass &) const {
356 return make_error<StringError>("addIRTranslator is not overridden",
357 inconvertibleErrorCode());
358 }
359
360 /// This method may be implemented by targets that want to run passes
361 /// immediately before legalization.
addPreLegalizeMachineIR(AddMachinePass &)362 void addPreLegalizeMachineIR(AddMachinePass &) const {}
363
364 /// This method should install a legalize pass, which converts the instruction
365 /// sequence into one that can be selected by the target.
addLegalizeMachineIR(AddMachinePass &)366 Error addLegalizeMachineIR(AddMachinePass &) const {
367 return make_error<StringError>("addLegalizeMachineIR is not overridden",
368 inconvertibleErrorCode());
369 }
370
371 /// This method may be implemented by targets that want to run passes
372 /// immediately before the register bank selection.
addPreRegBankSelect(AddMachinePass &)373 void addPreRegBankSelect(AddMachinePass &) const {}
374
375 /// This method should install a register bank selector pass, which
376 /// assigns register banks to virtual registers without a register
377 /// class or register banks.
addRegBankSelect(AddMachinePass &)378 Error addRegBankSelect(AddMachinePass &) const {
379 return make_error<StringError>("addRegBankSelect is not overridden",
380 inconvertibleErrorCode());
381 }
382
383 /// This method may be implemented by targets that want to run passes
384 /// immediately before the (global) instruction selection.
addPreGlobalInstructionSelect(AddMachinePass &)385 void addPreGlobalInstructionSelect(AddMachinePass &) const {}
386
387 /// This method should install a (global) instruction selector pass, which
388 /// converts possibly generic instructions to fully target-specific
389 /// instructions, thereby constraining all generic virtual registers to
390 /// register classes.
addGlobalInstructionSelect(AddMachinePass &)391 Error addGlobalInstructionSelect(AddMachinePass &) const {
392 return make_error<StringError>(
393 "addGlobalInstructionSelect is not overridden",
394 inconvertibleErrorCode());
395 }
396 /// @}}
397
398 /// High level function that adds all passes necessary to go from llvm IR
399 /// representation to the MI representation.
400 /// Adds IR based lowering and target specific optimization passes and finally
401 /// the core instruction selection passes.
402 void addISelPasses(AddIRPass &) const;
403
404 /// Add the actual instruction selection passes. This does not include
405 /// preparation passes on IR.
406 Error addCoreISelPasses(AddMachinePass &) const;
407
408 /// Add the complete, standard set of LLVM CodeGen passes.
409 /// Fully developed targets will not generally override this.
410 Error addMachinePasses(AddMachinePass &) const;
411
412 /// Add passes to lower exception handling for the code generator.
413 void addPassesToHandleExceptions(AddIRPass &) const;
414
415 /// Add common target configurable passes that perform LLVM IR to IR
416 /// transforms following machine independent optimization.
417 void addIRPasses(AddIRPass &) const;
418
419 /// Add pass to prepare the LLVM IR for code generation. This should be done
420 /// before exception handling preparation passes.
421 void addCodeGenPrepare(AddIRPass &) const;
422
423 /// Add common passes that perform LLVM IR to IR transforms in preparation for
424 /// instruction selection.
425 void addISelPrepare(AddIRPass &) const;
426
427 /// Methods with trivial inline returns are convenient points in the common
428 /// codegen pass pipeline where targets may insert passes. Methods with
429 /// out-of-line standard implementations are major CodeGen stages called by
430 /// addMachinePasses. Some targets may override major stages when inserting
431 /// passes is insufficient, but maintaining overriden stages is more work.
432 ///
433
434 /// addMachineSSAOptimization - Add standard passes that optimize machine
435 /// instructions in SSA form.
436 void addMachineSSAOptimization(AddMachinePass &) const;
437
438 /// addFastRegAlloc - Add the minimum set of target-independent passes that
439 /// are required for fast register allocation.
440 Error addFastRegAlloc(AddMachinePass &) const;
441
442 /// addOptimizedRegAlloc - Add passes related to register allocation.
443 /// LLVMTargetMachine provides standard regalloc passes for most targets.
444 void addOptimizedRegAlloc(AddMachinePass &) const;
445
446 /// Add passes that optimize machine instructions after register allocation.
447 void addMachineLateOptimization(AddMachinePass &) const;
448
449 /// addGCPasses - Add late codegen passes that analyze code for garbage
450 /// collection. This should return true if GC info should be printed after
451 /// these passes.
addGCPasses(AddMachinePass &)452 void addGCPasses(AddMachinePass &) const {}
453
454 /// Add standard basic block placement passes.
455 void addBlockPlacement(AddMachinePass &) const;
456
457 using CreateMCStreamer =
458 std::function<Expected<std::unique_ptr<MCStreamer>>(MCContext &)>;
addAsmPrinter(AddMachinePass &,CreateMCStreamer)459 void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const {
460 llvm_unreachable("addAsmPrinter is not overridden");
461 }
462
463 /// Utilities for targets to add passes to the pass manager.
464 ///
465
466 /// createTargetRegisterAllocator - Create the register allocator pass for
467 /// this target at the current optimization level.
468 void addTargetRegisterAllocator(AddMachinePass &, bool Optimized) const;
469
470 /// addMachinePasses helper to create the target-selected or overriden
471 /// regalloc pass.
472 void addRegAllocPass(AddMachinePass &, bool Optimized) const;
473
474 /// Add core register alloator passes which do the actual register assignment
475 /// and rewriting. \returns true if any passes were added.
476 Error addRegAssignmentFast(AddMachinePass &) const;
477 Error addRegAssignmentOptimized(AddMachinePass &) const;
478
479 private:
derived()480 DerivedT &derived() { return static_cast<DerivedT &>(*this); }
derived()481 const DerivedT &derived() const {
482 return static_cast<const DerivedT &>(*this);
483 }
484
runBeforeAdding(StringRef Name)485 bool runBeforeAdding(StringRef Name) const {
486 bool ShouldAdd = true;
487 for (auto &C : BeforeCallbacks)
488 ShouldAdd &= C(Name);
489 return ShouldAdd;
490 }
491
492 void setStartStopPasses(const TargetPassConfig::StartStopInfo &Info) const;
493
494 Error verifyStartStop(const TargetPassConfig::StartStopInfo &Info) const;
495
496 mutable SmallVector<llvm::unique_function<bool(StringRef)>, 4>
497 BeforeCallbacks;
498 mutable SmallVector<llvm::unique_function<void(StringRef)>, 4> AfterCallbacks;
499
500 /// Helper variable for `-start-before/-start-after/-stop-before/-stop-after`
501 mutable bool Started = true;
502 mutable bool Stopped = true;
503 };
504
505 template <typename Derived>
buildPipeline(ModulePassManager & MPM,MachineFunctionPassManager & MFPM,raw_pwrite_stream & Out,raw_pwrite_stream * DwoOut,CodeGenFileType FileType)506 Error CodeGenPassBuilder<Derived>::buildPipeline(
507 ModulePassManager &MPM, MachineFunctionPassManager &MFPM,
508 raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
509 CodeGenFileType FileType) const {
510 auto StartStopInfo = TargetPassConfig::getStartStopInfo(*PIC);
511 if (!StartStopInfo)
512 return StartStopInfo.takeError();
513 setStartStopPasses(*StartStopInfo);
514 AddIRPass addIRPass(MPM, derived());
515 // `ProfileSummaryInfo` is always valid.
516 addIRPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
517 addIRPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>());
518 addISelPasses(addIRPass);
519
520 AddMachinePass addPass(MFPM, derived());
521 if (auto Err = addCoreISelPasses(addPass))
522 return std::move(Err);
523
524 if (auto Err = derived().addMachinePasses(addPass))
525 return std::move(Err);
526
527 derived().addAsmPrinter(
528 addPass, [this, &Out, DwoOut, FileType](MCContext &Ctx) {
529 return this->TM.createMCStreamer(Out, DwoOut, FileType, Ctx);
530 });
531
532 addPass(FreeMachineFunctionPass());
533 return verifyStartStop(*StartStopInfo);
534 }
535
536 template <typename Derived>
setStartStopPasses(const TargetPassConfig::StartStopInfo & Info)537 void CodeGenPassBuilder<Derived>::setStartStopPasses(
538 const TargetPassConfig::StartStopInfo &Info) const {
539 if (!Info.StartPass.empty()) {
540 Started = false;
541 BeforeCallbacks.emplace_back([this, &Info, AfterFlag = Info.StartAfter,
542 Count = 0u](StringRef ClassName) mutable {
543 if (Count == Info.StartInstanceNum) {
544 if (AfterFlag) {
545 AfterFlag = false;
546 Started = true;
547 }
548 return Started;
549 }
550
551 auto PassName = PIC->getPassNameForClassName(ClassName);
552 if (Info.StartPass == PassName && ++Count == Info.StartInstanceNum)
553 Started = !Info.StartAfter;
554
555 return Started;
556 });
557 }
558
559 if (!Info.StopPass.empty()) {
560 Stopped = false;
561 BeforeCallbacks.emplace_back([this, &Info, AfterFlag = Info.StopAfter,
562 Count = 0u](StringRef ClassName) mutable {
563 if (Count == Info.StopInstanceNum) {
564 if (AfterFlag) {
565 AfterFlag = false;
566 Stopped = true;
567 }
568 return !Stopped;
569 }
570
571 auto PassName = PIC->getPassNameForClassName(ClassName);
572 if (Info.StopPass == PassName && ++Count == Info.StopInstanceNum)
573 Stopped = !Info.StopAfter;
574 return !Stopped;
575 });
576 }
577 }
578
579 template <typename Derived>
verifyStartStop(const TargetPassConfig::StartStopInfo & Info)580 Error CodeGenPassBuilder<Derived>::verifyStartStop(
581 const TargetPassConfig::StartStopInfo &Info) const {
582 if (Started && Stopped)
583 return Error::success();
584
585 if (!Started)
586 return make_error<StringError>(
587 "Can't find start pass \"" +
588 PIC->getPassNameForClassName(Info.StartPass) + "\".",
589 std::make_error_code(std::errc::invalid_argument));
590 if (!Stopped)
591 return make_error<StringError>(
592 "Can't find stop pass \"" +
593 PIC->getPassNameForClassName(Info.StopPass) + "\".",
594 std::make_error_code(std::errc::invalid_argument));
595 return Error::success();
596 }
597
registerAAAnalyses()598 static inline AAManager registerAAAnalyses() {
599 AAManager AA;
600
601 // The order in which these are registered determines their priority when
602 // being queried.
603
604 // Basic AliasAnalysis support.
605 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
606 // BasicAliasAnalysis wins if they disagree. This is intended to help
607 // support "obvious" type-punning idioms.
608 AA.registerFunctionAnalysis<TypeBasedAA>();
609 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
610 AA.registerFunctionAnalysis<BasicAA>();
611
612 return AA;
613 }
614
615 template <typename Derived>
registerModuleAnalyses(ModuleAnalysisManager & MAM)616 void CodeGenPassBuilder<Derived>::registerModuleAnalyses(
617 ModuleAnalysisManager &MAM) const {
618 #define MODULE_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
619 MAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
620 #include "MachinePassRegistry.def"
621 derived().registerTargetAnalysis(MAM);
622 }
623
624 template <typename Derived>
registerFunctionAnalyses(FunctionAnalysisManager & FAM)625 void CodeGenPassBuilder<Derived>::registerFunctionAnalyses(
626 FunctionAnalysisManager &FAM) const {
627 FAM.registerPass([this] { return registerAAAnalyses(); });
628
629 #define FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
630 FAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
631 #include "MachinePassRegistry.def"
632 derived().registerTargetAnalysis(FAM);
633 }
634
635 template <typename Derived>
registerMachineFunctionAnalyses(MachineFunctionAnalysisManager & MFAM)636 void CodeGenPassBuilder<Derived>::registerMachineFunctionAnalyses(
637 MachineFunctionAnalysisManager &MFAM) const {
638 #define MACHINE_FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
639 MFAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
640 #include "MachinePassRegistry.def"
641 derived().registerTargetAnalysis(MFAM);
642 }
643
644 // FIXME: For new PM, use pass name directly in commandline seems good.
645 // Translate stringfied pass name to its old commandline name. Returns the
646 // matching legacy name and a boolean value indicating if the pass is a machine
647 // pass.
648 template <typename Derived>
649 std::pair<StringRef, bool>
getPassNameFromLegacyName(StringRef Name)650 CodeGenPassBuilder<Derived>::getPassNameFromLegacyName(StringRef Name) const {
651 std::pair<StringRef, bool> Ret;
652 if (Name.empty())
653 return Ret;
654
655 #define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
656 if (Name == NAME) \
657 Ret = {#PASS_NAME, false};
658 #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
659 if (Name == NAME) \
660 Ret = {#PASS_NAME, false};
661 #define MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
662 if (Name == NAME) \
663 Ret = {#PASS_NAME, false};
664 #define DUMMY_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
665 if (Name == NAME) \
666 Ret = {#PASS_NAME, false};
667 #define MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
668 if (Name == NAME) \
669 Ret = {#PASS_NAME, true};
670 #define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
671 if (Name == NAME) \
672 Ret = {#PASS_NAME, true};
673 #define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
674 if (Name == NAME) \
675 Ret = {#PASS_NAME, true};
676 #define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
677 if (Name == NAME) \
678 Ret = {#PASS_NAME, true};
679 #include "llvm/CodeGen/MachinePassRegistry.def"
680
681 if (Ret.first.empty())
682 Ret = derived().getTargetPassNameFromLegacyName(Name);
683
684 if (Ret.first.empty())
685 report_fatal_error(Twine('\"') + Twine(Name) +
686 Twine("\" pass could not be found."));
687
688 return Ret;
689 }
690
691 template <typename Derived>
addISelPasses(AddIRPass & addPass)692 void CodeGenPassBuilder<Derived>::addISelPasses(AddIRPass &addPass) const {
693 derived().addGlobalMergePass(addPass);
694 if (TM.useEmulatedTLS())
695 addPass(LowerEmuTLSPass());
696
697 addPass(PreISelIntrinsicLoweringPass(TM));
698
699 derived().addIRPasses(addPass);
700 derived().addCodeGenPrepare(addPass);
701 addPassesToHandleExceptions(addPass);
702 derived().addISelPrepare(addPass);
703 }
704
705 /// Add common target configurable passes that perform LLVM IR to IR transforms
706 /// following machine independent optimization.
707 template <typename Derived>
addIRPasses(AddIRPass & addPass)708 void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
709 // Before running any passes, run the verifier to determine if the input
710 // coming from the front-end and/or optimizer is valid.
711 if (!Opt.DisableVerify)
712 addPass(VerifierPass());
713
714 // Run loop strength reduction before anything else.
715 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) {
716 addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(),
717 /*UseMemorySSA=*/true));
718 // FIXME: use -stop-after so we could remove PrintLSR
719 if (Opt.PrintLSR)
720 addPass(PrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n"));
721 }
722
723 if (getOptLevel() != CodeGenOptLevel::None) {
724 // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
725 // loads and compares. ExpandMemCmpPass then tries to expand those calls
726 // into optimally-sized loads and compares. The transforms are enabled by a
727 // target lowering hook.
728 if (!Opt.DisableMergeICmps)
729 addPass(MergeICmpsPass());
730 addPass(ExpandMemCmpPass(&TM));
731 }
732
733 // Run GC lowering passes for builtin collectors
734 // TODO: add a pass insertion point here
735 addPass(GCLoweringPass());
736 addPass(ShadowStackGCLoweringPass());
737 addPass(LowerConstantIntrinsicsPass());
738
739 // Make sure that no unreachable blocks are instruction selected.
740 addPass(UnreachableBlockElimPass());
741
742 // Prepare expensive constants for SelectionDAG.
743 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableConstantHoisting)
744 addPass(ConstantHoistingPass());
745
746 // Replace calls to LLVM intrinsics (e.g., exp, log) operating on vector
747 // operands with calls to the corresponding functions in a vector library.
748 if (getOptLevel() != CodeGenOptLevel::None)
749 addPass(ReplaceWithVeclib());
750
751 if (getOptLevel() != CodeGenOptLevel::None &&
752 !Opt.DisablePartialLibcallInlining)
753 addPass(PartiallyInlineLibCallsPass());
754
755 // Instrument function entry and exit, e.g. with calls to mcount().
756 addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
757
758 // Add scalarization of target's unsupported masked memory intrinsics pass.
759 // the unsupported intrinsic will be replaced with a chain of basic blocks,
760 // that stores/loads element one-by-one if the appropriate mask bit is set.
761 addPass(ScalarizeMaskedMemIntrinPass());
762
763 // Expand reduction intrinsics into shuffle sequences if the target wants to.
764 addPass(ExpandReductionsPass());
765
766 // Convert conditional moves to conditional jumps when profitable.
767 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize)
768 addPass(SelectOptimizePass(&TM));
769 }
770
771 /// Turn exception handling constructs into something the code generators can
772 /// handle.
773 template <typename Derived>
addPassesToHandleExceptions(AddIRPass & addPass)774 void CodeGenPassBuilder<Derived>::addPassesToHandleExceptions(
775 AddIRPass &addPass) const {
776 const MCAsmInfo *MCAI = TM.getMCAsmInfo();
777 assert(MCAI && "No MCAsmInfo");
778 switch (MCAI->getExceptionHandlingType()) {
779 case ExceptionHandling::SjLj:
780 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
781 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
782 // catch info can get misplaced when a selector ends up more than one block
783 // removed from the parent invoke(s). This could happen when a landing
784 // pad is shared by multiple invokes and is also a target of a normal
785 // edge from elsewhere.
786 addPass(SjLjEHPreparePass(&TM));
787 [[fallthrough]];
788 case ExceptionHandling::DwarfCFI:
789 case ExceptionHandling::ARM:
790 case ExceptionHandling::AIX:
791 case ExceptionHandling::ZOS:
792 addPass(DwarfEHPreparePass(&TM));
793 break;
794 case ExceptionHandling::WinEH:
795 // We support using both GCC-style and MSVC-style exceptions on Windows, so
796 // add both preparation passes. Each pass will only actually run if it
797 // recognizes the personality function.
798 addPass(WinEHPreparePass());
799 addPass(DwarfEHPreparePass(&TM));
800 break;
801 case ExceptionHandling::Wasm:
802 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
803 // on catchpads and cleanuppads because it does not outline them into
804 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
805 // should remove PHIs there.
806 addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/false));
807 addPass(WasmEHPreparePass());
808 break;
809 case ExceptionHandling::None:
810 addPass(LowerInvokePass());
811
812 // The lower invoke pass may create unreachable code. Remove it.
813 addPass(UnreachableBlockElimPass());
814 break;
815 }
816 }
817
818 /// Add pass to prepare the LLVM IR for code generation. This should be done
819 /// before exception handling preparation passes.
820 template <typename Derived>
addCodeGenPrepare(AddIRPass & addPass)821 void CodeGenPassBuilder<Derived>::addCodeGenPrepare(AddIRPass &addPass) const {
822 if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
823 addPass(CodeGenPreparePass(&TM));
824 // TODO: Default ctor'd RewriteSymbolPass is no-op.
825 // addPass(RewriteSymbolPass());
826 }
827
828 /// Add common passes that perform LLVM IR to IR transforms in preparation for
829 /// instruction selection.
830 template <typename Derived>
addISelPrepare(AddIRPass & addPass)831 void CodeGenPassBuilder<Derived>::addISelPrepare(AddIRPass &addPass) const {
832 derived().addPreISel(addPass);
833
834 addPass(CallBrPreparePass());
835 // Add both the safe stack and the stack protection passes: each of them will
836 // only protect functions that have corresponding attributes.
837 addPass(SafeStackPass(&TM));
838 addPass(StackProtectorPass(&TM));
839
840 if (Opt.PrintISelInput)
841 addPass(PrintFunctionPass(dbgs(),
842 "\n\n*** Final LLVM Code input to ISel ***\n"));
843
844 // All passes which modify the LLVM IR are now complete; run the verifier
845 // to ensure that the IR is valid.
846 if (!Opt.DisableVerify)
847 addPass(VerifierPass());
848 }
849
850 template <typename Derived>
addCoreISelPasses(AddMachinePass & addPass)851 Error CodeGenPassBuilder<Derived>::addCoreISelPasses(
852 AddMachinePass &addPass) const {
853 // Enable FastISel with -fast-isel, but allow that to be overridden.
854 TM.setO0WantsFastISel(Opt.EnableFastISelOption.value_or(true));
855
856 // Determine an instruction selector.
857 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
858 SelectorType Selector;
859
860 if (Opt.EnableFastISelOption && *Opt.EnableFastISelOption == true)
861 Selector = SelectorType::FastISel;
862 else if ((Opt.EnableGlobalISelOption &&
863 *Opt.EnableGlobalISelOption == true) ||
864 (TM.Options.EnableGlobalISel &&
865 (!Opt.EnableGlobalISelOption ||
866 *Opt.EnableGlobalISelOption == false)))
867 Selector = SelectorType::GlobalISel;
868 else if (TM.getOptLevel() == CodeGenOptLevel::None && TM.getO0WantsFastISel())
869 Selector = SelectorType::FastISel;
870 else
871 Selector = SelectorType::SelectionDAG;
872
873 // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
874 if (Selector == SelectorType::FastISel) {
875 TM.setFastISel(true);
876 TM.setGlobalISel(false);
877 } else if (Selector == SelectorType::GlobalISel) {
878 TM.setFastISel(false);
879 TM.setGlobalISel(true);
880 }
881
882 // Add instruction selector passes.
883 if (Selector == SelectorType::GlobalISel) {
884 if (auto Err = derived().addIRTranslator(addPass))
885 return std::move(Err);
886
887 derived().addPreLegalizeMachineIR(addPass);
888
889 if (auto Err = derived().addLegalizeMachineIR(addPass))
890 return std::move(Err);
891
892 // Before running the register bank selector, ask the target if it
893 // wants to run some passes.
894 derived().addPreRegBankSelect(addPass);
895
896 if (auto Err = derived().addRegBankSelect(addPass))
897 return std::move(Err);
898
899 derived().addPreGlobalInstructionSelect(addPass);
900
901 if (auto Err = derived().addGlobalInstructionSelect(addPass))
902 return std::move(Err);
903
904 // Pass to reset the MachineFunction if the ISel failed.
905 addPass(ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(),
906 isGlobalISelAbortEnabled()));
907
908 // Provide a fallback path when we do not want to abort on
909 // not-yet-supported input.
910 if (!isGlobalISelAbortEnabled())
911 if (auto Err = derived().addInstSelector(addPass))
912 return std::move(Err);
913
914 } else if (auto Err = derived().addInstSelector(addPass))
915 return std::move(Err);
916
917 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
918 // FinalizeISel.
919 addPass(FinalizeISelPass());
920
921 // // Print the instruction selected machine code...
922 // printAndVerify("After Instruction Selection");
923
924 return Error::success();
925 }
926
927 /// Add the complete set of target-independent postISel code generator passes.
928 ///
929 /// This can be read as the standard order of major LLVM CodeGen stages. Stages
930 /// with nontrivial configuration or multiple passes are broken out below in
931 /// add%Stage routines.
932 ///
933 /// Any CodeGenPassBuilder<Derived>::addXX routine may be overriden by the
934 /// Target. The addPre/Post methods with empty header implementations allow
935 /// injecting target-specific fixups just before or after major stages.
936 /// Additionally, targets have the flexibility to change pass order within a
937 /// stage by overriding default implementation of add%Stage routines below. Each
938 /// technique has maintainability tradeoffs because alternate pass orders are
939 /// not well supported. addPre/Post works better if the target pass is easily
940 /// tied to a common pass. But if it has subtle dependencies on multiple passes,
941 /// the target should override the stage instead.
942 template <typename Derived>
addMachinePasses(AddMachinePass & addPass)943 Error CodeGenPassBuilder<Derived>::addMachinePasses(
944 AddMachinePass &addPass) const {
945 // Add passes that optimize machine instructions in SSA form.
946 if (getOptLevel() != CodeGenOptLevel::None) {
947 derived().addMachineSSAOptimization(addPass);
948 } else {
949 // If the target requests it, assign local variables to stack slots relative
950 // to one another and simplify frame index references where possible.
951 addPass(LocalStackSlotPass());
952 }
953
954 if (TM.Options.EnableIPRA)
955 addPass(RegUsageInfoPropagationPass());
956
957 // Run pre-ra passes.
958 derived().addPreRegAlloc(addPass);
959
960 // Run register allocation and passes that are tightly coupled with it,
961 // including phi elimination and scheduling.
962 if (*Opt.OptimizeRegAlloc) {
963 derived().addOptimizedRegAlloc(addPass);
964 } else {
965 if (auto Err = derived().addFastRegAlloc(addPass))
966 return Err;
967 }
968
969 // Run post-ra passes.
970 derived().addPostRegAlloc(addPass);
971
972 addPass(RemoveRedundantDebugValuesPass());
973
974 // Insert prolog/epilog code. Eliminate abstract frame index references...
975 if (getOptLevel() != CodeGenOptLevel::None) {
976 addPass(PostRAMachineSinkingPass());
977 addPass(ShrinkWrapPass());
978 }
979
980 addPass(PrologEpilogInserterPass());
981
982 /// Add passes that optimize machine instructions after register allocation.
983 if (getOptLevel() != CodeGenOptLevel::None)
984 derived().addMachineLateOptimization(addPass);
985
986 // Expand pseudo instructions before second scheduling pass.
987 addPass(ExpandPostRAPseudosPass());
988
989 // Run pre-sched2 passes.
990 derived().addPreSched2(addPass);
991
992 if (Opt.EnableImplicitNullChecks)
993 addPass(ImplicitNullChecksPass());
994
995 // Second pass scheduler.
996 // Let Target optionally insert this pass by itself at some other
997 // point.
998 if (getOptLevel() != CodeGenOptLevel::None &&
999 !TM.targetSchedulesPostRAScheduling()) {
1000 if (Opt.MISchedPostRA)
1001 addPass(PostMachineSchedulerPass());
1002 else
1003 addPass(PostRASchedulerPass());
1004 }
1005
1006 // GC
1007 derived().addGCPasses(addPass);
1008
1009 // Basic block placement.
1010 if (getOptLevel() != CodeGenOptLevel::None)
1011 derived().addBlockPlacement(addPass);
1012
1013 // Insert before XRay Instrumentation.
1014 addPass(FEntryInserterPass());
1015
1016 addPass(XRayInstrumentationPass());
1017 addPass(PatchableFunctionPass());
1018
1019 derived().addPreEmitPass(addPass);
1020
1021 if (TM.Options.EnableIPRA)
1022 // Collect register usage information and produce a register mask of
1023 // clobbered registers, to be used to optimize call sites.
1024 addPass(RegUsageInfoCollectorPass());
1025
1026 addPass(FuncletLayoutPass());
1027
1028 addPass(StackMapLivenessPass());
1029 addPass(LiveDebugValuesPass());
1030 addPass(MachineSanitizerBinaryMetadata());
1031
1032 if (TM.Options.EnableMachineOutliner &&
1033 getOptLevel() != CodeGenOptLevel::None &&
1034 Opt.EnableMachineOutliner != RunOutliner::NeverOutline) {
1035 bool RunOnAllFunctions =
1036 (Opt.EnableMachineOutliner == RunOutliner::AlwaysOutline);
1037 bool AddOutliner = RunOnAllFunctions || TM.Options.SupportsDefaultOutlining;
1038 if (AddOutliner)
1039 addPass(MachineOutlinerPass(RunOnAllFunctions));
1040 }
1041
1042 // Add passes that directly emit MI after all other MI passes.
1043 derived().addPreEmitPass2(addPass);
1044
1045 return Error::success();
1046 }
1047
1048 /// Add passes that optimize machine instructions in SSA form.
1049 template <typename Derived>
addMachineSSAOptimization(AddMachinePass & addPass)1050 void CodeGenPassBuilder<Derived>::addMachineSSAOptimization(
1051 AddMachinePass &addPass) const {
1052 // Pre-ra tail duplication.
1053 addPass(EarlyTailDuplicatePass());
1054
1055 // Optimize PHIs before DCE: removing dead PHI cycles may make more
1056 // instructions dead.
1057 addPass(OptimizePHIsPass());
1058
1059 // This pass merges large allocas. StackSlotColoring is a different pass
1060 // which merges spill slots.
1061 addPass(StackColoringPass());
1062
1063 // If the target requests it, assign local variables to stack slots relative
1064 // to one another and simplify frame index references where possible.
1065 addPass(LocalStackSlotPass());
1066
1067 // With optimization, dead code should already be eliminated. However
1068 // there is one known exception: lowered code for arguments that are only
1069 // used by tail calls, where the tail calls reuse the incoming stack
1070 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1071 addPass(DeadMachineInstructionElimPass());
1072
1073 // Allow targets to insert passes that improve instruction level parallelism,
1074 // like if-conversion. Such passes will typically need dominator trees and
1075 // loop info, just like LICM and CSE below.
1076 derived().addILPOpts(addPass);
1077
1078 addPass(EarlyMachineLICMPass());
1079 addPass(MachineCSEPass());
1080
1081 addPass(MachineSinkingPass());
1082
1083 addPass(PeepholeOptimizerPass());
1084 // Clean-up the dead code that may have been generated by peephole
1085 // rewriting.
1086 addPass(DeadMachineInstructionElimPass());
1087 }
1088
1089 //===---------------------------------------------------------------------===//
1090 /// Register Allocation Pass Configuration
1091 //===---------------------------------------------------------------------===//
1092
1093 /// Instantiate the default register allocator pass for this target for either
1094 /// the optimized or unoptimized allocation path. This will be added to the pass
1095 /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1096 /// in the optimized case.
1097 ///
1098 /// A target that uses the standard regalloc pass order for fast or optimized
1099 /// allocation may still override this for per-target regalloc
1100 /// selection. But -regalloc=... always takes precedence.
1101 template <typename Derived>
addTargetRegisterAllocator(AddMachinePass & addPass,bool Optimized)1102 void CodeGenPassBuilder<Derived>::addTargetRegisterAllocator(
1103 AddMachinePass &addPass, bool Optimized) const {
1104 if (Optimized)
1105 addPass(RAGreedyPass());
1106 else
1107 addPass(RAFastPass());
1108 }
1109
1110 /// Find and instantiate the register allocation pass requested by this target
1111 /// at the current optimization level. Different register allocators are
1112 /// defined as separate passes because they may require different analysis.
1113 template <typename Derived>
addRegAllocPass(AddMachinePass & addPass,bool Optimized)1114 void CodeGenPassBuilder<Derived>::addRegAllocPass(AddMachinePass &addPass,
1115 bool Optimized) const {
1116 // TODO: Parse Opt.RegAlloc to add register allocator.
1117 }
1118
1119 template <typename Derived>
addRegAssignmentFast(AddMachinePass & addPass)1120 Error CodeGenPassBuilder<Derived>::addRegAssignmentFast(
1121 AddMachinePass &addPass) const {
1122 // TODO: Ensure allocator is default or fast.
1123 addRegAllocPass(addPass, false);
1124 return Error::success();
1125 }
1126
1127 template <typename Derived>
addRegAssignmentOptimized(AddMachinePass & addPass)1128 Error CodeGenPassBuilder<Derived>::addRegAssignmentOptimized(
1129 AddMachinePass &addPass) const {
1130 // Add the selected register allocation pass.
1131 addRegAllocPass(addPass, true);
1132
1133 // Allow targets to change the register assignments before rewriting.
1134 derived().addPreRewrite(addPass);
1135
1136 // Finally rewrite virtual registers.
1137 addPass(VirtRegRewriterPass());
1138 // Perform stack slot coloring and post-ra machine LICM.
1139 //
1140 // FIXME: Re-enable coloring with register when it's capable of adding
1141 // kill markers.
1142 addPass(StackSlotColoringPass());
1143
1144 return Error::success();
1145 }
1146
1147 /// Add the minimum set of target-independent passes that are required for
1148 /// register allocation. No coalescing or scheduling.
1149 template <typename Derived>
addFastRegAlloc(AddMachinePass & addPass)1150 Error CodeGenPassBuilder<Derived>::addFastRegAlloc(
1151 AddMachinePass &addPass) const {
1152 addPass(PHIEliminationPass());
1153 addPass(TwoAddressInstructionPass());
1154 return derived().addRegAssignmentFast(addPass);
1155 }
1156
1157 /// Add standard target-independent passes that are tightly coupled with
1158 /// optimized register allocation, including coalescing, machine instruction
1159 /// scheduling, and register allocation itself.
1160 template <typename Derived>
addOptimizedRegAlloc(AddMachinePass & addPass)1161 void CodeGenPassBuilder<Derived>::addOptimizedRegAlloc(
1162 AddMachinePass &addPass) const {
1163 addPass(DetectDeadLanesPass());
1164
1165 addPass(ProcessImplicitDefsPass());
1166
1167 // Edge splitting is smarter with machine loop info.
1168 addPass(PHIEliminationPass());
1169
1170 // Eventually, we want to run LiveIntervals before PHI elimination.
1171 if (Opt.EarlyLiveIntervals)
1172 addPass(LiveIntervalsPass());
1173
1174 addPass(TwoAddressInstructionPass());
1175 addPass(RegisterCoalescerPass());
1176
1177 // The machine scheduler may accidentally create disconnected components
1178 // when moving subregister definitions around, avoid this by splitting them to
1179 // separate vregs before. Splitting can also improve reg. allocation quality.
1180 addPass(RenameIndependentSubregsPass());
1181
1182 // PreRA instruction scheduling.
1183 addPass(MachineSchedulerPass());
1184
1185 if (derived().addRegAssignmentOptimized(addPass)) {
1186 // Allow targets to expand pseudo instructions depending on the choice of
1187 // registers before MachineCopyPropagation.
1188 derived().addPostRewrite(addPass);
1189
1190 // Copy propagate to forward register uses and try to eliminate COPYs that
1191 // were not coalesced.
1192 addPass(MachineCopyPropagationPass());
1193
1194 // Run post-ra machine LICM to hoist reloads / remats.
1195 //
1196 // FIXME: can this move into MachineLateOptimization?
1197 addPass(MachineLICMPass());
1198 }
1199 }
1200
1201 //===---------------------------------------------------------------------===//
1202 /// Post RegAlloc Pass Configuration
1203 //===---------------------------------------------------------------------===//
1204
1205 /// Add passes that optimize machine instructions after register allocation.
1206 template <typename Derived>
addMachineLateOptimization(AddMachinePass & addPass)1207 void CodeGenPassBuilder<Derived>::addMachineLateOptimization(
1208 AddMachinePass &addPass) const {
1209 // Branch folding must be run after regalloc and prolog/epilog insertion.
1210 addPass(BranchFolderPass());
1211
1212 // Tail duplication.
1213 // Note that duplicating tail just increases code size and degrades
1214 // performance for targets that require Structured Control Flow.
1215 // In addition it can also make CFG irreducible. Thus we disable it.
1216 if (!TM.requiresStructuredCFG())
1217 addPass(TailDuplicatePass());
1218
1219 // Cleanup of redundant (identical) address/immediate loads.
1220 addPass(MachineLateInstrsCleanupPass());
1221
1222 // Copy propagation.
1223 addPass(MachineCopyPropagationPass());
1224 }
1225
1226 /// Add standard basic block placement passes.
1227 template <typename Derived>
addBlockPlacement(AddMachinePass & addPass)1228 void CodeGenPassBuilder<Derived>::addBlockPlacement(
1229 AddMachinePass &addPass) const {
1230 addPass(MachineBlockPlacementPass());
1231 // Run a separate pass to collect block placement statistics.
1232 if (Opt.EnableBlockPlacementStats)
1233 addPass(MachineBlockPlacementStatsPass());
1234 }
1235
1236 } // namespace llvm
1237
1238 #endif // LLVM_CODEGEN_CODEGENPASSBUILDER_H
1239