10b57cec5SDimitry Andric //===- TargetPassConfig.cpp - Target independent code generation passes ---===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // This file defines interfaces to access the target independent code
100b57cec5SDimitry Andric // generation passes provided by the LLVM backend.
110b57cec5SDimitry Andric //
120b57cec5SDimitry Andric //===---------------------------------------------------------------------===//
130b57cec5SDimitry Andric
140b57cec5SDimitry Andric #include "llvm/CodeGen/TargetPassConfig.h"
150b57cec5SDimitry Andric #include "llvm/ADT/DenseMap.h"
160b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h"
170b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
180b57cec5SDimitry Andric #include "llvm/Analysis/BasicAliasAnalysis.h"
190b57cec5SDimitry Andric #include "llvm/Analysis/CallGraphSCCPass.h"
200b57cec5SDimitry Andric #include "llvm/Analysis/ScopedNoAliasAA.h"
210b57cec5SDimitry Andric #include "llvm/Analysis/TargetTransformInfo.h"
220b57cec5SDimitry Andric #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
2381ad6265SDimitry Andric #include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
240b57cec5SDimitry Andric #include "llvm/CodeGen/CSEConfigBase.h"
250b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunctionPass.h"
260b57cec5SDimitry Andric #include "llvm/CodeGen/MachinePassRegistry.h"
270b57cec5SDimitry Andric #include "llvm/CodeGen/Passes.h"
280b57cec5SDimitry Andric #include "llvm/CodeGen/RegAllocRegistry.h"
290b57cec5SDimitry Andric #include "llvm/IR/IRPrintingPasses.h"
300b57cec5SDimitry Andric #include "llvm/IR/LegacyPassManager.h"
31e8d8bef9SDimitry Andric #include "llvm/IR/PassInstrumentation.h"
320b57cec5SDimitry Andric #include "llvm/IR/Verifier.h"
33480093f4SDimitry Andric #include "llvm/InitializePasses.h"
340b57cec5SDimitry Andric #include "llvm/MC/MCAsmInfo.h"
350b57cec5SDimitry Andric #include "llvm/MC/MCTargetOptions.h"
360b57cec5SDimitry Andric #include "llvm/Pass.h"
370b57cec5SDimitry Andric #include "llvm/Support/CodeGen.h"
380b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h"
390b57cec5SDimitry Andric #include "llvm/Support/Compiler.h"
400b57cec5SDimitry Andric #include "llvm/Support/Debug.h"
41fe6060f1SDimitry Andric #include "llvm/Support/Discriminator.h"
420b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h"
430b57cec5SDimitry Andric #include "llvm/Support/SaveAndRestore.h"
44480093f4SDimitry Andric #include "llvm/Support/Threading.h"
4506c3fb27SDimitry Andric #include "llvm/Support/VirtualFileSystem.h"
4606c3fb27SDimitry Andric #include "llvm/Support/WithColor.h"
47e8d8bef9SDimitry Andric #include "llvm/Target/CGPassBuilderOption.h"
480b57cec5SDimitry Andric #include "llvm/Target/TargetMachine.h"
490b57cec5SDimitry Andric #include "llvm/Transforms/Scalar.h"
500b57cec5SDimitry Andric #include "llvm/Transforms/Utils.h"
510b57cec5SDimitry Andric #include <cassert>
52bdd1243dSDimitry Andric #include <optional>
530b57cec5SDimitry Andric #include <string>
540b57cec5SDimitry Andric
550b57cec5SDimitry Andric using namespace llvm;
560b57cec5SDimitry Andric
578bcb0991SDimitry Andric static cl::opt<bool>
588bcb0991SDimitry Andric EnableIPRA("enable-ipra", cl::init(false), cl::Hidden,
590b57cec5SDimitry Andric cl::desc("Enable interprocedural register allocation "
600b57cec5SDimitry Andric "to reduce load/store at procedure calls."));
610b57cec5SDimitry Andric static cl::opt<bool> DisablePostRASched("disable-post-ra", cl::Hidden,
620b57cec5SDimitry Andric cl::desc("Disable Post Regalloc Scheduler"));
630b57cec5SDimitry Andric static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden,
640b57cec5SDimitry Andric cl::desc("Disable branch folding"));
650b57cec5SDimitry Andric static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden,
660b57cec5SDimitry Andric cl::desc("Disable tail duplication"));
670b57cec5SDimitry Andric static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden,
680b57cec5SDimitry Andric cl::desc("Disable pre-register allocation tail duplication"));
690b57cec5SDimitry Andric static cl::opt<bool> DisableBlockPlacement("disable-block-placement",
700b57cec5SDimitry Andric cl::Hidden, cl::desc("Disable probability-driven block placement"));
710b57cec5SDimitry Andric static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats",
720b57cec5SDimitry Andric cl::Hidden, cl::desc("Collect probability-driven block placement stats"));
730b57cec5SDimitry Andric static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,
740b57cec5SDimitry Andric cl::desc("Disable Stack Slot Coloring"));
750b57cec5SDimitry Andric static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,
760b57cec5SDimitry Andric cl::desc("Disable Machine Dead Code Elimination"));
770b57cec5SDimitry Andric static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden,
780b57cec5SDimitry Andric cl::desc("Disable Early If-conversion"));
790b57cec5SDimitry Andric static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
800b57cec5SDimitry Andric cl::desc("Disable Machine LICM"));
810b57cec5SDimitry Andric static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
820b57cec5SDimitry Andric cl::desc("Disable Machine Common Subexpression Elimination"));
830b57cec5SDimitry Andric static cl::opt<cl::boolOrDefault> OptimizeRegAlloc(
840b57cec5SDimitry Andric "optimize-regalloc", cl::Hidden,
850b57cec5SDimitry Andric cl::desc("Enable optimized register allocation compilation path."));
860b57cec5SDimitry Andric static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",
870b57cec5SDimitry Andric cl::Hidden,
880b57cec5SDimitry Andric cl::desc("Disable Machine LICM"));
890b57cec5SDimitry Andric static cl::opt<bool> DisableMachineSink("disable-machine-sink", cl::Hidden,
900b57cec5SDimitry Andric cl::desc("Disable Machine Sinking"));
910b57cec5SDimitry Andric static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink",
920b57cec5SDimitry Andric cl::Hidden,
930b57cec5SDimitry Andric cl::desc("Disable PostRA Machine Sinking"));
940b57cec5SDimitry Andric static cl::opt<bool> DisableLSR("disable-lsr", cl::Hidden,
950b57cec5SDimitry Andric cl::desc("Disable Loop Strength Reduction Pass"));
960b57cec5SDimitry Andric static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting",
970b57cec5SDimitry Andric cl::Hidden, cl::desc("Disable ConstantHoisting"));
980b57cec5SDimitry Andric static cl::opt<bool> DisableCGP("disable-cgp", cl::Hidden,
990b57cec5SDimitry Andric cl::desc("Disable Codegen Prepare"));
1000b57cec5SDimitry Andric static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,
1010b57cec5SDimitry Andric cl::desc("Disable Copy Propagation pass"));
1020b57cec5SDimitry Andric static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",
1030b57cec5SDimitry Andric cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));
10406c3fb27SDimitry Andric static cl::opt<bool> DisableAtExitBasedGlobalDtorLowering(
10506c3fb27SDimitry Andric "disable-atexit-based-global-dtor-lowering", cl::Hidden,
10606c3fb27SDimitry Andric cl::desc("For MachO, disable atexit()-based global destructor lowering"));
1070b57cec5SDimitry Andric static cl::opt<bool> EnableImplicitNullChecks(
1080b57cec5SDimitry Andric "enable-implicit-null-checks",
1090b57cec5SDimitry Andric cl::desc("Fold null checks into faulting memory operations"),
1100b57cec5SDimitry Andric cl::init(false), cl::Hidden);
1110b57cec5SDimitry Andric static cl::opt<bool> DisableMergeICmps("disable-mergeicmps",
1120b57cec5SDimitry Andric cl::desc("Disable MergeICmps Pass"),
1130b57cec5SDimitry Andric cl::init(false), cl::Hidden);
1140b57cec5SDimitry Andric static cl::opt<bool> PrintLSR("print-lsr-output", cl::Hidden,
1150b57cec5SDimitry Andric cl::desc("Print LLVM IR produced by the loop-reduce pass"));
1165f757f3fSDimitry Andric static cl::opt<bool>
1175f757f3fSDimitry Andric PrintISelInput("print-isel-input", cl::Hidden,
1180b57cec5SDimitry Andric cl::desc("Print LLVM IR input to isel pass"));
1190b57cec5SDimitry Andric static cl::opt<cl::boolOrDefault>
1200b57cec5SDimitry Andric VerifyMachineCode("verify-machineinstrs", cl::Hidden,
12181ad6265SDimitry Andric cl::desc("Verify generated machine code"));
12281ad6265SDimitry Andric static cl::opt<cl::boolOrDefault>
12381ad6265SDimitry Andric DebugifyAndStripAll("debugify-and-strip-all-safe", cl::Hidden,
12481ad6265SDimitry Andric cl::desc("Debugify MIR before and Strip debug after "
12581ad6265SDimitry Andric "each pass except those known to be unsafe "
12681ad6265SDimitry Andric "when debug info is present"));
127e8d8bef9SDimitry Andric static cl::opt<cl::boolOrDefault> DebugifyCheckAndStripAll(
128e8d8bef9SDimitry Andric "debugify-check-and-strip-all-safe", cl::Hidden,
129e8d8bef9SDimitry Andric cl::desc(
130e8d8bef9SDimitry Andric "Debugify MIR before, by checking and stripping the debug info after, "
13181ad6265SDimitry Andric "each pass except those known to be unsafe when debug info is "
13281ad6265SDimitry Andric "present"));
1330b57cec5SDimitry Andric // Enable or disable the MachineOutliner.
1340b57cec5SDimitry Andric static cl::opt<RunOutliner> EnableMachineOutliner(
1350b57cec5SDimitry Andric "enable-machine-outliner", cl::desc("Enable the machine outliner"),
136e8d8bef9SDimitry Andric cl::Hidden, cl::ValueOptional, cl::init(RunOutliner::TargetDefault),
137e8d8bef9SDimitry Andric cl::values(clEnumValN(RunOutliner::AlwaysOutline, "always",
1380b57cec5SDimitry Andric "Run on all functions guaranteed to be beneficial"),
139e8d8bef9SDimitry Andric clEnumValN(RunOutliner::NeverOutline, "never",
140e8d8bef9SDimitry Andric "Disable all outlining"),
1410b57cec5SDimitry Andric // Sentinel value for unspecified option.
142e8d8bef9SDimitry Andric clEnumValN(RunOutliner::AlwaysOutline, "", "")));
14381ad6265SDimitry Andric // Disable the pass to fix unwind information. Whether the pass is included in
14481ad6265SDimitry Andric // the pipeline is controlled via the target options, this option serves as
14581ad6265SDimitry Andric // manual override.
14681ad6265SDimitry Andric static cl::opt<bool> DisableCFIFixup("disable-cfi-fixup", cl::Hidden,
14781ad6265SDimitry Andric cl::desc("Disable the CFI fixup pass"));
1480b57cec5SDimitry Andric // Enable or disable FastISel. Both options are needed, because
1490b57cec5SDimitry Andric // FastISel is enabled by default with -fast, and we wish to be
1500b57cec5SDimitry Andric // able to enable or disable fast-isel independently from -O0.
1510b57cec5SDimitry Andric static cl::opt<cl::boolOrDefault>
1520b57cec5SDimitry Andric EnableFastISelOption("fast-isel", cl::Hidden,
1530b57cec5SDimitry Andric cl::desc("Enable the \"fast\" instruction selector"));
1540b57cec5SDimitry Andric
1550b57cec5SDimitry Andric static cl::opt<cl::boolOrDefault> EnableGlobalISelOption(
1560b57cec5SDimitry Andric "global-isel", cl::Hidden,
1570b57cec5SDimitry Andric cl::desc("Enable the \"global\" instruction selector"));
1580b57cec5SDimitry Andric
159e8d8bef9SDimitry Andric // FIXME: remove this after switching to NPM or GlobalISel, whichever gets there
160e8d8bef9SDimitry Andric // first...
161e8d8bef9SDimitry Andric static cl::opt<bool>
162e8d8bef9SDimitry Andric PrintAfterISel("print-after-isel", cl::init(false), cl::Hidden,
163e8d8bef9SDimitry Andric cl::desc("Print machine instrs after ISel"));
1640b57cec5SDimitry Andric
1650b57cec5SDimitry Andric static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort(
1660b57cec5SDimitry Andric "global-isel-abort", cl::Hidden,
1670b57cec5SDimitry Andric cl::desc("Enable abort calls when \"global\" instruction selection "
1680b57cec5SDimitry Andric "fails to lower/select an instruction"),
1690b57cec5SDimitry Andric cl::values(
1700b57cec5SDimitry Andric clEnumValN(GlobalISelAbortMode::Disable, "0", "Disable the abort"),
1710b57cec5SDimitry Andric clEnumValN(GlobalISelAbortMode::Enable, "1", "Enable the abort"),
1720b57cec5SDimitry Andric clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2",
1730b57cec5SDimitry Andric "Disable the abort but emit a diagnostic on failure")));
1740b57cec5SDimitry Andric
175349cc55cSDimitry Andric // Disable MIRProfileLoader before RegAlloc. This is for for debugging and
176349cc55cSDimitry Andric // tuning purpose.
177349cc55cSDimitry Andric static cl::opt<bool> DisableRAFSProfileLoader(
17881ad6265SDimitry Andric "disable-ra-fsprofile-loader", cl::init(false), cl::Hidden,
179349cc55cSDimitry Andric cl::desc("Disable MIRProfileLoader before RegAlloc"));
180349cc55cSDimitry Andric // Disable MIRProfileLoader before BloackPlacement. This is for for debugging
181349cc55cSDimitry Andric // and tuning purpose.
182349cc55cSDimitry Andric static cl::opt<bool> DisableLayoutFSProfileLoader(
18381ad6265SDimitry Andric "disable-layout-fsprofile-loader", cl::init(false), cl::Hidden,
184349cc55cSDimitry Andric cl::desc("Disable MIRProfileLoader before BlockPlacement"));
185349cc55cSDimitry Andric // Specify FSProfile file name.
186349cc55cSDimitry Andric static cl::opt<std::string>
187349cc55cSDimitry Andric FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"),
188349cc55cSDimitry Andric cl::desc("Flow Sensitive profile file name."), cl::Hidden);
189349cc55cSDimitry Andric // Specify Remapping file for FSProfile.
190349cc55cSDimitry Andric static cl::opt<std::string> FSRemappingFile(
191349cc55cSDimitry Andric "fs-remapping-file", cl::init(""), cl::value_desc("filename"),
192349cc55cSDimitry Andric cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);
193fe6060f1SDimitry Andric
1940b57cec5SDimitry Andric // Temporary option to allow experimenting with MachineScheduler as a post-RA
1950b57cec5SDimitry Andric // scheduler. Targets can "properly" enable this with
1960b57cec5SDimitry Andric // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID).
1970b57cec5SDimitry Andric // Targets can return true in targetSchedulesPostRAScheduling() and
1980b57cec5SDimitry Andric // insert a PostRA scheduling pass wherever it wants.
1998bcb0991SDimitry Andric static cl::opt<bool> MISchedPostRA(
2008bcb0991SDimitry Andric "misched-postra", cl::Hidden,
2018bcb0991SDimitry Andric cl::desc(
2028bcb0991SDimitry Andric "Run MachineScheduler post regalloc (independent of preRA sched)"));
2030b57cec5SDimitry Andric
2040b57cec5SDimitry Andric // Experimental option to run live interval analysis early.
2050b57cec5SDimitry Andric static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden,
2060b57cec5SDimitry Andric cl::desc("Run live interval analysis earlier in the pipeline"));
2070b57cec5SDimitry Andric
2080b57cec5SDimitry Andric /// Option names for limiting the codegen pipeline.
2090b57cec5SDimitry Andric /// Those are used in error reporting and we didn't want
2100b57cec5SDimitry Andric /// to duplicate their names all over the place.
211480093f4SDimitry Andric static const char StartAfterOptName[] = "start-after";
212480093f4SDimitry Andric static const char StartBeforeOptName[] = "start-before";
213480093f4SDimitry Andric static const char StopAfterOptName[] = "stop-after";
214480093f4SDimitry Andric static const char StopBeforeOptName[] = "stop-before";
2150b57cec5SDimitry Andric
2160b57cec5SDimitry Andric static cl::opt<std::string>
2170b57cec5SDimitry Andric StartAfterOpt(StringRef(StartAfterOptName),
2180b57cec5SDimitry Andric cl::desc("Resume compilation after a specific pass"),
2190b57cec5SDimitry Andric cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
2200b57cec5SDimitry Andric
2210b57cec5SDimitry Andric static cl::opt<std::string>
2220b57cec5SDimitry Andric StartBeforeOpt(StringRef(StartBeforeOptName),
2230b57cec5SDimitry Andric cl::desc("Resume compilation before a specific pass"),
2240b57cec5SDimitry Andric cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
2250b57cec5SDimitry Andric
2260b57cec5SDimitry Andric static cl::opt<std::string>
2270b57cec5SDimitry Andric StopAfterOpt(StringRef(StopAfterOptName),
2280b57cec5SDimitry Andric cl::desc("Stop compilation after a specific pass"),
2290b57cec5SDimitry Andric cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
2300b57cec5SDimitry Andric
2310b57cec5SDimitry Andric static cl::opt<std::string>
2320b57cec5SDimitry Andric StopBeforeOpt(StringRef(StopBeforeOptName),
2330b57cec5SDimitry Andric cl::desc("Stop compilation before a specific pass"),
2340b57cec5SDimitry Andric cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
2350b57cec5SDimitry Andric
236e8d8bef9SDimitry Andric /// Enable the machine function splitter pass.
237e8d8bef9SDimitry Andric static cl::opt<bool> EnableMachineFunctionSplitter(
238e8d8bef9SDimitry Andric "enable-split-machine-functions", cl::Hidden,
239e8d8bef9SDimitry Andric cl::desc("Split out cold blocks from machine functions based on profile "
240e8d8bef9SDimitry Andric "information."));
241e8d8bef9SDimitry Andric
242e8d8bef9SDimitry Andric /// Disable the expand reductions pass for testing.
243e8d8bef9SDimitry Andric static cl::opt<bool> DisableExpandReductions(
244e8d8bef9SDimitry Andric "disable-expand-reductions", cl::init(false), cl::Hidden,
245e8d8bef9SDimitry Andric cl::desc("Disable the expand reduction intrinsics pass from running"));
246e8d8bef9SDimitry Andric
24781ad6265SDimitry Andric /// Disable the select optimization pass.
24881ad6265SDimitry Andric static cl::opt<bool> DisableSelectOptimize(
24981ad6265SDimitry Andric "disable-select-optimize", cl::init(true), cl::Hidden,
25081ad6265SDimitry Andric cl::desc("Disable the select-optimization pass from running"));
25181ad6265SDimitry Andric
2525f757f3fSDimitry Andric /// Enable garbage-collecting empty basic blocks.
2535f757f3fSDimitry Andric static cl::opt<bool>
2545f757f3fSDimitry Andric GCEmptyBlocks("gc-empty-basic-blocks", cl::init(false), cl::Hidden,
2555f757f3fSDimitry Andric cl::desc("Enable garbage-collecting empty basic blocks"));
2565f757f3fSDimitry Andric
2570b57cec5SDimitry Andric /// Allow standard passes to be disabled by command line options. This supports
2580b57cec5SDimitry Andric /// simple binary flags that either suppress the pass or do nothing.
2590b57cec5SDimitry Andric /// i.e. -disable-mypass=false has no effect.
2600b57cec5SDimitry Andric /// These should be converted to boolOrDefault in order to use applyOverride.
applyDisable(IdentifyingPassPtr PassID,bool Override)2610b57cec5SDimitry Andric static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,
2620b57cec5SDimitry Andric bool Override) {
2630b57cec5SDimitry Andric if (Override)
2640b57cec5SDimitry Andric return IdentifyingPassPtr();
2650b57cec5SDimitry Andric return PassID;
2660b57cec5SDimitry Andric }
2670b57cec5SDimitry Andric
2680b57cec5SDimitry Andric /// Allow standard passes to be disabled by the command line, regardless of who
2690b57cec5SDimitry Andric /// is adding the pass.
2700b57cec5SDimitry Andric ///
2710b57cec5SDimitry Andric /// StandardID is the pass identified in the standard pass pipeline and provided
2720b57cec5SDimitry Andric /// to addPass(). It may be a target-specific ID in the case that the target
2730b57cec5SDimitry Andric /// directly adds its own pass, but in that case we harmlessly fall through.
2740b57cec5SDimitry Andric ///
2750b57cec5SDimitry Andric /// TargetID is the pass that the target has configured to override StandardID.
2760b57cec5SDimitry Andric ///
2770b57cec5SDimitry Andric /// StandardID may be a pseudo ID. In that case TargetID is the name of the real
2780b57cec5SDimitry Andric /// pass to run. This allows multiple options to control a single pass depending
2790b57cec5SDimitry Andric /// on where in the pipeline that pass is added.
overridePass(AnalysisID StandardID,IdentifyingPassPtr TargetID)2800b57cec5SDimitry Andric static IdentifyingPassPtr overridePass(AnalysisID StandardID,
2810b57cec5SDimitry Andric IdentifyingPassPtr TargetID) {
2820b57cec5SDimitry Andric if (StandardID == &PostRASchedulerID)
2830b57cec5SDimitry Andric return applyDisable(TargetID, DisablePostRASched);
2840b57cec5SDimitry Andric
2850b57cec5SDimitry Andric if (StandardID == &BranchFolderPassID)
2860b57cec5SDimitry Andric return applyDisable(TargetID, DisableBranchFold);
2870b57cec5SDimitry Andric
2880b57cec5SDimitry Andric if (StandardID == &TailDuplicateID)
2890b57cec5SDimitry Andric return applyDisable(TargetID, DisableTailDuplicate);
2900b57cec5SDimitry Andric
2910b57cec5SDimitry Andric if (StandardID == &EarlyTailDuplicateID)
2920b57cec5SDimitry Andric return applyDisable(TargetID, DisableEarlyTailDup);
2930b57cec5SDimitry Andric
2940b57cec5SDimitry Andric if (StandardID == &MachineBlockPlacementID)
2950b57cec5SDimitry Andric return applyDisable(TargetID, DisableBlockPlacement);
2960b57cec5SDimitry Andric
2970b57cec5SDimitry Andric if (StandardID == &StackSlotColoringID)
2980b57cec5SDimitry Andric return applyDisable(TargetID, DisableSSC);
2990b57cec5SDimitry Andric
3000b57cec5SDimitry Andric if (StandardID == &DeadMachineInstructionElimID)
3010b57cec5SDimitry Andric return applyDisable(TargetID, DisableMachineDCE);
3020b57cec5SDimitry Andric
3030b57cec5SDimitry Andric if (StandardID == &EarlyIfConverterID)
3040b57cec5SDimitry Andric return applyDisable(TargetID, DisableEarlyIfConversion);
3050b57cec5SDimitry Andric
3060b57cec5SDimitry Andric if (StandardID == &EarlyMachineLICMID)
3070b57cec5SDimitry Andric return applyDisable(TargetID, DisableMachineLICM);
3080b57cec5SDimitry Andric
3090b57cec5SDimitry Andric if (StandardID == &MachineCSEID)
3100b57cec5SDimitry Andric return applyDisable(TargetID, DisableMachineCSE);
3110b57cec5SDimitry Andric
3120b57cec5SDimitry Andric if (StandardID == &MachineLICMID)
3130b57cec5SDimitry Andric return applyDisable(TargetID, DisablePostRAMachineLICM);
3140b57cec5SDimitry Andric
3150b57cec5SDimitry Andric if (StandardID == &MachineSinkingID)
3160b57cec5SDimitry Andric return applyDisable(TargetID, DisableMachineSink);
3170b57cec5SDimitry Andric
3180b57cec5SDimitry Andric if (StandardID == &PostRAMachineSinkingID)
3190b57cec5SDimitry Andric return applyDisable(TargetID, DisablePostRAMachineSink);
3200b57cec5SDimitry Andric
3210b57cec5SDimitry Andric if (StandardID == &MachineCopyPropagationID)
3220b57cec5SDimitry Andric return applyDisable(TargetID, DisableCopyProp);
3230b57cec5SDimitry Andric
3240b57cec5SDimitry Andric return TargetID;
3250b57cec5SDimitry Andric }
3260b57cec5SDimitry Andric
327349cc55cSDimitry Andric // Find the FSProfile file name. The internal option takes the precedence
328349cc55cSDimitry Andric // before getting from TargetMachine.
getFSProfileFile(const TargetMachine * TM)32904eeddc0SDimitry Andric static std::string getFSProfileFile(const TargetMachine *TM) {
330349cc55cSDimitry Andric if (!FSProfileFile.empty())
331349cc55cSDimitry Andric return FSProfileFile.getValue();
332bdd1243dSDimitry Andric const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
333bdd1243dSDimitry Andric if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
334349cc55cSDimitry Andric return std::string();
335349cc55cSDimitry Andric return PGOOpt->ProfileFile;
336349cc55cSDimitry Andric }
337349cc55cSDimitry Andric
338349cc55cSDimitry Andric // Find the Profile remapping file name. The internal option takes the
339349cc55cSDimitry Andric // precedence before getting from TargetMachine.
getFSRemappingFile(const TargetMachine * TM)34004eeddc0SDimitry Andric static std::string getFSRemappingFile(const TargetMachine *TM) {
341349cc55cSDimitry Andric if (!FSRemappingFile.empty())
342349cc55cSDimitry Andric return FSRemappingFile.getValue();
343bdd1243dSDimitry Andric const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
344bdd1243dSDimitry Andric if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
345349cc55cSDimitry Andric return std::string();
346349cc55cSDimitry Andric return PGOOpt->ProfileRemappingFile;
347349cc55cSDimitry Andric }
348349cc55cSDimitry Andric
3490b57cec5SDimitry Andric //===---------------------------------------------------------------------===//
3500b57cec5SDimitry Andric /// TargetPassConfig
3510b57cec5SDimitry Andric //===---------------------------------------------------------------------===//
3520b57cec5SDimitry Andric
3530b57cec5SDimitry Andric INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",
3540b57cec5SDimitry Andric "Target Pass Configuration", false, false)
3550b57cec5SDimitry Andric char TargetPassConfig::ID = 0;
3560b57cec5SDimitry Andric
3570b57cec5SDimitry Andric namespace {
3580b57cec5SDimitry Andric
3590b57cec5SDimitry Andric struct InsertedPass {
3600b57cec5SDimitry Andric AnalysisID TargetPassID;
3610b57cec5SDimitry Andric IdentifyingPassPtr InsertedPassID;
3620b57cec5SDimitry Andric
InsertedPass__anond9cf71500111::InsertedPass363349cc55cSDimitry Andric InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID)
364349cc55cSDimitry Andric : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {}
3650b57cec5SDimitry Andric
getInsertedPass__anond9cf71500111::InsertedPass3660b57cec5SDimitry Andric Pass *getInsertedPass() const {
3670b57cec5SDimitry Andric assert(InsertedPassID.isValid() && "Illegal Pass ID!");
3680b57cec5SDimitry Andric if (InsertedPassID.isInstance())
3690b57cec5SDimitry Andric return InsertedPassID.getInstance();
3700b57cec5SDimitry Andric Pass *NP = Pass::createPass(InsertedPassID.getID());
3710b57cec5SDimitry Andric assert(NP && "Pass ID not registered");
3720b57cec5SDimitry Andric return NP;
3730b57cec5SDimitry Andric }
3740b57cec5SDimitry Andric };
3750b57cec5SDimitry Andric
3760b57cec5SDimitry Andric } // end anonymous namespace
3770b57cec5SDimitry Andric
3780b57cec5SDimitry Andric namespace llvm {
3790b57cec5SDimitry Andric
380fe6060f1SDimitry Andric extern cl::opt<bool> EnableFSDiscriminator;
381fe6060f1SDimitry Andric
3820b57cec5SDimitry Andric class PassConfigImpl {
3830b57cec5SDimitry Andric public:
3840b57cec5SDimitry Andric // List of passes explicitly substituted by this target. Normally this is
3850b57cec5SDimitry Andric // empty, but it is a convenient way to suppress or replace specific passes
3860b57cec5SDimitry Andric // that are part of a standard pass pipeline without overridding the entire
3870b57cec5SDimitry Andric // pipeline. This mechanism allows target options to inherit a standard pass's
3880b57cec5SDimitry Andric // user interface. For example, a target may disable a standard pass by
3890b57cec5SDimitry Andric // default by substituting a pass ID of zero, and the user may still enable
3900b57cec5SDimitry Andric // that standard pass with an explicit command line option.
3910b57cec5SDimitry Andric DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;
3920b57cec5SDimitry Andric
3930b57cec5SDimitry Andric /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass
3940b57cec5SDimitry Andric /// is inserted after each instance of the first one.
3950b57cec5SDimitry Andric SmallVector<InsertedPass, 4> InsertedPasses;
3960b57cec5SDimitry Andric };
3970b57cec5SDimitry Andric
3980b57cec5SDimitry Andric } // end namespace llvm
3990b57cec5SDimitry Andric
4000b57cec5SDimitry Andric // Out of line virtual method.
~TargetPassConfig()4010b57cec5SDimitry Andric TargetPassConfig::~TargetPassConfig() {
4020b57cec5SDimitry Andric delete Impl;
4030b57cec5SDimitry Andric }
4040b57cec5SDimitry Andric
getPassInfo(StringRef PassName)4050b57cec5SDimitry Andric static const PassInfo *getPassInfo(StringRef PassName) {
4060b57cec5SDimitry Andric if (PassName.empty())
4070b57cec5SDimitry Andric return nullptr;
4080b57cec5SDimitry Andric
4090b57cec5SDimitry Andric const PassRegistry &PR = *PassRegistry::getPassRegistry();
4100b57cec5SDimitry Andric const PassInfo *PI = PR.getPassInfo(PassName);
4110b57cec5SDimitry Andric if (!PI)
4120b57cec5SDimitry Andric report_fatal_error(Twine('\"') + Twine(PassName) +
4130b57cec5SDimitry Andric Twine("\" pass is not registered."));
4140b57cec5SDimitry Andric return PI;
4150b57cec5SDimitry Andric }
4160b57cec5SDimitry Andric
getPassIDFromName(StringRef PassName)4170b57cec5SDimitry Andric static AnalysisID getPassIDFromName(StringRef PassName) {
4180b57cec5SDimitry Andric const PassInfo *PI = getPassInfo(PassName);
4190b57cec5SDimitry Andric return PI ? PI->getTypeInfo() : nullptr;
4200b57cec5SDimitry Andric }
4210b57cec5SDimitry Andric
4220b57cec5SDimitry Andric static std::pair<StringRef, unsigned>
getPassNameAndInstanceNum(StringRef PassName)4230b57cec5SDimitry Andric getPassNameAndInstanceNum(StringRef PassName) {
4240b57cec5SDimitry Andric StringRef Name, InstanceNumStr;
4250b57cec5SDimitry Andric std::tie(Name, InstanceNumStr) = PassName.split(',');
4260b57cec5SDimitry Andric
4270b57cec5SDimitry Andric unsigned InstanceNum = 0;
4280b57cec5SDimitry Andric if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(10, InstanceNum))
4290b57cec5SDimitry Andric report_fatal_error("invalid pass instance specifier " + PassName);
4300b57cec5SDimitry Andric
4310b57cec5SDimitry Andric return std::make_pair(Name, InstanceNum);
4320b57cec5SDimitry Andric }
4330b57cec5SDimitry Andric
setStartStopPasses()4340b57cec5SDimitry Andric void TargetPassConfig::setStartStopPasses() {
4350b57cec5SDimitry Andric StringRef StartBeforeName;
4360b57cec5SDimitry Andric std::tie(StartBeforeName, StartBeforeInstanceNum) =
4370b57cec5SDimitry Andric getPassNameAndInstanceNum(StartBeforeOpt);
4380b57cec5SDimitry Andric
4390b57cec5SDimitry Andric StringRef StartAfterName;
4400b57cec5SDimitry Andric std::tie(StartAfterName, StartAfterInstanceNum) =
4410b57cec5SDimitry Andric getPassNameAndInstanceNum(StartAfterOpt);
4420b57cec5SDimitry Andric
4430b57cec5SDimitry Andric StringRef StopBeforeName;
4440b57cec5SDimitry Andric std::tie(StopBeforeName, StopBeforeInstanceNum)
4450b57cec5SDimitry Andric = getPassNameAndInstanceNum(StopBeforeOpt);
4460b57cec5SDimitry Andric
4470b57cec5SDimitry Andric StringRef StopAfterName;
4480b57cec5SDimitry Andric std::tie(StopAfterName, StopAfterInstanceNum)
4490b57cec5SDimitry Andric = getPassNameAndInstanceNum(StopAfterOpt);
4500b57cec5SDimitry Andric
4510b57cec5SDimitry Andric StartBefore = getPassIDFromName(StartBeforeName);
4520b57cec5SDimitry Andric StartAfter = getPassIDFromName(StartAfterName);
4530b57cec5SDimitry Andric StopBefore = getPassIDFromName(StopBeforeName);
4540b57cec5SDimitry Andric StopAfter = getPassIDFromName(StopAfterName);
4550b57cec5SDimitry Andric if (StartBefore && StartAfter)
4560b57cec5SDimitry Andric report_fatal_error(Twine(StartBeforeOptName) + Twine(" and ") +
4570b57cec5SDimitry Andric Twine(StartAfterOptName) + Twine(" specified!"));
4580b57cec5SDimitry Andric if (StopBefore && StopAfter)
4590b57cec5SDimitry Andric report_fatal_error(Twine(StopBeforeOptName) + Twine(" and ") +
4600b57cec5SDimitry Andric Twine(StopAfterOptName) + Twine(" specified!"));
4610b57cec5SDimitry Andric Started = (StartAfter == nullptr) && (StartBefore == nullptr);
4620b57cec5SDimitry Andric }
4630b57cec5SDimitry Andric
getCGPassBuilderOption()464e8d8bef9SDimitry Andric CGPassBuilderOption llvm::getCGPassBuilderOption() {
465e8d8bef9SDimitry Andric CGPassBuilderOption Opt;
466e8d8bef9SDimitry Andric
467e8d8bef9SDimitry Andric #define SET_OPTION(Option) \
468e8d8bef9SDimitry Andric if (Option.getNumOccurrences()) \
469e8d8bef9SDimitry Andric Opt.Option = Option;
470e8d8bef9SDimitry Andric
471e8d8bef9SDimitry Andric SET_OPTION(EnableFastISelOption)
472e8d8bef9SDimitry Andric SET_OPTION(EnableGlobalISelAbort)
473e8d8bef9SDimitry Andric SET_OPTION(EnableGlobalISelOption)
474e8d8bef9SDimitry Andric SET_OPTION(EnableIPRA)
475e8d8bef9SDimitry Andric SET_OPTION(OptimizeRegAlloc)
476e8d8bef9SDimitry Andric SET_OPTION(VerifyMachineCode)
4775f757f3fSDimitry Andric SET_OPTION(DisableAtExitBasedGlobalDtorLowering)
4785f757f3fSDimitry Andric SET_OPTION(DisableExpandReductions)
4795f757f3fSDimitry Andric SET_OPTION(PrintAfterISel)
4805f757f3fSDimitry Andric SET_OPTION(FSProfileFile)
4815f757f3fSDimitry Andric SET_OPTION(GCEmptyBlocks)
482e8d8bef9SDimitry Andric
483e8d8bef9SDimitry Andric #define SET_BOOLEAN_OPTION(Option) Opt.Option = Option;
484e8d8bef9SDimitry Andric
485e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(EarlyLiveIntervals)
486e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(EnableBlockPlacementStats)
487e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(EnableImplicitNullChecks)
488e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(EnableMachineOutliner)
489e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(MISchedPostRA)
490e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(DisableMergeICmps)
491e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(DisableLSR)
492e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(DisableConstantHoisting)
493e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(DisableCGP)
494e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)
49581ad6265SDimitry Andric SET_BOOLEAN_OPTION(DisableSelectOptimize)
496e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(PrintLSR)
497e8d8bef9SDimitry Andric SET_BOOLEAN_OPTION(PrintISelInput)
4985f757f3fSDimitry Andric SET_BOOLEAN_OPTION(DebugifyAndStripAll)
4995f757f3fSDimitry Andric SET_BOOLEAN_OPTION(DebugifyCheckAndStripAll)
5005f757f3fSDimitry Andric SET_BOOLEAN_OPTION(DisableRAFSProfileLoader)
5015f757f3fSDimitry Andric SET_BOOLEAN_OPTION(DisableCFIFixup)
5025f757f3fSDimitry Andric SET_BOOLEAN_OPTION(EnableMachineFunctionSplitter)
503e8d8bef9SDimitry Andric
504e8d8bef9SDimitry Andric return Opt;
505e8d8bef9SDimitry Andric }
506e8d8bef9SDimitry Andric
registerCodeGenCallback(PassInstrumentationCallbacks & PIC,LLVMTargetMachine & LLVMTM)507e8d8bef9SDimitry Andric void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
508e8d8bef9SDimitry Andric LLVMTargetMachine &LLVMTM) {
509e8d8bef9SDimitry Andric
510e8d8bef9SDimitry Andric // Register a callback for disabling passes.
511e8d8bef9SDimitry Andric PIC.registerShouldRunOptionalPassCallback([](StringRef P, Any) {
512e8d8bef9SDimitry Andric
513e8d8bef9SDimitry Andric #define DISABLE_PASS(Option, Name) \
514e8d8bef9SDimitry Andric if (Option && P.contains(#Name)) \
515e8d8bef9SDimitry Andric return false;
516e8d8bef9SDimitry Andric DISABLE_PASS(DisableBlockPlacement, MachineBlockPlacementPass)
517e8d8bef9SDimitry Andric DISABLE_PASS(DisableBranchFold, BranchFolderPass)
518e8d8bef9SDimitry Andric DISABLE_PASS(DisableCopyProp, MachineCopyPropagationPass)
519e8d8bef9SDimitry Andric DISABLE_PASS(DisableEarlyIfConversion, EarlyIfConverterPass)
520e8d8bef9SDimitry Andric DISABLE_PASS(DisableEarlyTailDup, EarlyTailDuplicatePass)
521e8d8bef9SDimitry Andric DISABLE_PASS(DisableMachineCSE, MachineCSEPass)
522e8d8bef9SDimitry Andric DISABLE_PASS(DisableMachineDCE, DeadMachineInstructionElimPass)
523e8d8bef9SDimitry Andric DISABLE_PASS(DisableMachineLICM, EarlyMachineLICMPass)
524e8d8bef9SDimitry Andric DISABLE_PASS(DisableMachineSink, MachineSinkingPass)
525e8d8bef9SDimitry Andric DISABLE_PASS(DisablePostRAMachineLICM, MachineLICMPass)
526e8d8bef9SDimitry Andric DISABLE_PASS(DisablePostRAMachineSink, PostRAMachineSinkingPass)
527e8d8bef9SDimitry Andric DISABLE_PASS(DisablePostRASched, PostRASchedulerPass)
528e8d8bef9SDimitry Andric DISABLE_PASS(DisableSSC, StackSlotColoringPass)
529e8d8bef9SDimitry Andric DISABLE_PASS(DisableTailDuplicate, TailDuplicatePass)
530e8d8bef9SDimitry Andric
531e8d8bef9SDimitry Andric return true;
532e8d8bef9SDimitry Andric });
5337a6dacacSDimitry Andric }
534e8d8bef9SDimitry Andric
5357a6dacacSDimitry Andric Expected<TargetPassConfig::StartStopInfo>
getStartStopInfo(PassInstrumentationCallbacks & PIC)5367a6dacacSDimitry Andric TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) {
5377a6dacacSDimitry Andric auto [StartBefore, StartBeforeInstanceNum] =
5387a6dacacSDimitry Andric getPassNameAndInstanceNum(StartBeforeOpt);
5397a6dacacSDimitry Andric auto [StartAfter, StartAfterInstanceNum] =
5407a6dacacSDimitry Andric getPassNameAndInstanceNum(StartAfterOpt);
5417a6dacacSDimitry Andric auto [StopBefore, StopBeforeInstanceNum] =
5427a6dacacSDimitry Andric getPassNameAndInstanceNum(StopBeforeOpt);
5437a6dacacSDimitry Andric auto [StopAfter, StopAfterInstanceNum] =
5447a6dacacSDimitry Andric getPassNameAndInstanceNum(StopAfterOpt);
5457a6dacacSDimitry Andric
5467a6dacacSDimitry Andric if (!StartBefore.empty() && !StartAfter.empty())
5477a6dacacSDimitry Andric return make_error<StringError>(
5487a6dacacSDimitry Andric Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!",
5497a6dacacSDimitry Andric std::make_error_code(std::errc::invalid_argument));
5507a6dacacSDimitry Andric if (!StopBefore.empty() && !StopAfter.empty())
5517a6dacacSDimitry Andric return make_error<StringError>(
5527a6dacacSDimitry Andric Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!",
5537a6dacacSDimitry Andric std::make_error_code(std::errc::invalid_argument));
5547a6dacacSDimitry Andric
5557a6dacacSDimitry Andric StartStopInfo Result;
5567a6dacacSDimitry Andric Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore;
5577a6dacacSDimitry Andric Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore;
5587a6dacacSDimitry Andric Result.StartInstanceNum =
5597a6dacacSDimitry Andric StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum;
5607a6dacacSDimitry Andric Result.StopInstanceNum =
5617a6dacacSDimitry Andric StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum;
5627a6dacacSDimitry Andric Result.StartAfter = !StartAfter.empty();
5637a6dacacSDimitry Andric Result.StopAfter = !StopAfter.empty();
5647a6dacacSDimitry Andric Result.StartInstanceNum += Result.StartInstanceNum == 0;
5657a6dacacSDimitry Andric Result.StopInstanceNum += Result.StopInstanceNum == 0;
5667a6dacacSDimitry Andric return Result;
567e8d8bef9SDimitry Andric }
568e8d8bef9SDimitry Andric
5690b57cec5SDimitry Andric // Out of line constructor provides default values for pass options and
5700b57cec5SDimitry Andric // registers all common codegen passes.
TargetPassConfig(LLVMTargetMachine & TM,PassManagerBase & pm)5710b57cec5SDimitry Andric TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm)
5720b57cec5SDimitry Andric : ImmutablePass(ID), PM(&pm), TM(&TM) {
5730b57cec5SDimitry Andric Impl = new PassConfigImpl();
5740b57cec5SDimitry Andric
5750b57cec5SDimitry Andric // Register all target independent codegen passes to activate their PassIDs,
5760b57cec5SDimitry Andric // including this pass itself.
5770b57cec5SDimitry Andric initializeCodeGen(*PassRegistry::getPassRegistry());
5780b57cec5SDimitry Andric
5790b57cec5SDimitry Andric // Also register alias analysis passes required by codegen passes.
5800b57cec5SDimitry Andric initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry());
5810b57cec5SDimitry Andric initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
5820b57cec5SDimitry Andric
5830b57cec5SDimitry Andric if (EnableIPRA.getNumOccurrences())
5840b57cec5SDimitry Andric TM.Options.EnableIPRA = EnableIPRA;
5850b57cec5SDimitry Andric else {
5860b57cec5SDimitry Andric // If not explicitly specified, use target default.
5870b57cec5SDimitry Andric TM.Options.EnableIPRA |= TM.useIPRA();
5880b57cec5SDimitry Andric }
5890b57cec5SDimitry Andric
5900b57cec5SDimitry Andric if (TM.Options.EnableIPRA)
5910b57cec5SDimitry Andric setRequiresCodeGenSCCOrder();
5920b57cec5SDimitry Andric
5930b57cec5SDimitry Andric if (EnableGlobalISelAbort.getNumOccurrences())
5940b57cec5SDimitry Andric TM.Options.GlobalISelAbort = EnableGlobalISelAbort;
5950b57cec5SDimitry Andric
5960b57cec5SDimitry Andric setStartStopPasses();
5970b57cec5SDimitry Andric }
5980b57cec5SDimitry Andric
getOptLevel() const5995f757f3fSDimitry Andric CodeGenOptLevel TargetPassConfig::getOptLevel() const {
6000b57cec5SDimitry Andric return TM->getOptLevel();
6010b57cec5SDimitry Andric }
6020b57cec5SDimitry Andric
6030b57cec5SDimitry Andric /// Insert InsertedPassID pass after TargetPassID.
insertPass(AnalysisID TargetPassID,IdentifyingPassPtr InsertedPassID)6040b57cec5SDimitry Andric void TargetPassConfig::insertPass(AnalysisID TargetPassID,
605349cc55cSDimitry Andric IdentifyingPassPtr InsertedPassID) {
6060b57cec5SDimitry Andric assert(((!InsertedPassID.isInstance() &&
6070b57cec5SDimitry Andric TargetPassID != InsertedPassID.getID()) ||
6080b57cec5SDimitry Andric (InsertedPassID.isInstance() &&
6090b57cec5SDimitry Andric TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
6100b57cec5SDimitry Andric "Insert a pass after itself!");
611349cc55cSDimitry Andric Impl->InsertedPasses.emplace_back(TargetPassID, InsertedPassID);
6120b57cec5SDimitry Andric }
6130b57cec5SDimitry Andric
6140b57cec5SDimitry Andric /// createPassConfig - Create a pass configuration object to be used by
6150b57cec5SDimitry Andric /// addPassToEmitX methods for generating a pipeline of CodeGen passes.
6160b57cec5SDimitry Andric ///
6170b57cec5SDimitry Andric /// Targets may override this to extend TargetPassConfig.
createPassConfig(PassManagerBase & PM)6180b57cec5SDimitry Andric TargetPassConfig *LLVMTargetMachine::createPassConfig(PassManagerBase &PM) {
6190b57cec5SDimitry Andric return new TargetPassConfig(*this, PM);
6200b57cec5SDimitry Andric }
6210b57cec5SDimitry Andric
TargetPassConfig()6220b57cec5SDimitry Andric TargetPassConfig::TargetPassConfig()
6230b57cec5SDimitry Andric : ImmutablePass(ID) {
6240b57cec5SDimitry Andric report_fatal_error("Trying to construct TargetPassConfig without a target "
6250b57cec5SDimitry Andric "machine. Scheduling a CodeGen pass without a target "
6260b57cec5SDimitry Andric "triple set?");
6270b57cec5SDimitry Andric }
6280b57cec5SDimitry Andric
willCompleteCodeGenPipeline()6290b57cec5SDimitry Andric bool TargetPassConfig::willCompleteCodeGenPipeline() {
6300b57cec5SDimitry Andric return StopBeforeOpt.empty() && StopAfterOpt.empty();
6310b57cec5SDimitry Andric }
6320b57cec5SDimitry Andric
hasLimitedCodeGenPipeline()6330b57cec5SDimitry Andric bool TargetPassConfig::hasLimitedCodeGenPipeline() {
6340b57cec5SDimitry Andric return !StartBeforeOpt.empty() || !StartAfterOpt.empty() ||
6350b57cec5SDimitry Andric !willCompleteCodeGenPipeline();
6360b57cec5SDimitry Andric }
6370b57cec5SDimitry Andric
getLimitedCodeGenPipelineReason()6387a6dacacSDimitry Andric std::string TargetPassConfig::getLimitedCodeGenPipelineReason() {
6390b57cec5SDimitry Andric if (!hasLimitedCodeGenPipeline())
6400b57cec5SDimitry Andric return std::string();
6410b57cec5SDimitry Andric std::string Res;
6420b57cec5SDimitry Andric static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt,
6430b57cec5SDimitry Andric &StopAfterOpt, &StopBeforeOpt};
6440b57cec5SDimitry Andric static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,
6450b57cec5SDimitry Andric StopAfterOptName, StopBeforeOptName};
6460b57cec5SDimitry Andric bool IsFirst = true;
6470b57cec5SDimitry Andric for (int Idx = 0; Idx < 4; ++Idx)
6480b57cec5SDimitry Andric if (!PassNames[Idx]->empty()) {
6490b57cec5SDimitry Andric if (!IsFirst)
6507a6dacacSDimitry Andric Res += " and ";
6510b57cec5SDimitry Andric IsFirst = false;
6520b57cec5SDimitry Andric Res += OptNames[Idx];
6530b57cec5SDimitry Andric }
6540b57cec5SDimitry Andric return Res;
6550b57cec5SDimitry Andric }
6560b57cec5SDimitry Andric
6570b57cec5SDimitry Andric // Helper to verify the analysis is really immutable.
setOpt(bool & Opt,bool Val)6580b57cec5SDimitry Andric void TargetPassConfig::setOpt(bool &Opt, bool Val) {
6590b57cec5SDimitry Andric assert(!Initialized && "PassConfig is immutable");
6600b57cec5SDimitry Andric Opt = Val;
6610b57cec5SDimitry Andric }
6620b57cec5SDimitry Andric
substitutePass(AnalysisID StandardID,IdentifyingPassPtr TargetID)6630b57cec5SDimitry Andric void TargetPassConfig::substitutePass(AnalysisID StandardID,
6640b57cec5SDimitry Andric IdentifyingPassPtr TargetID) {
6650b57cec5SDimitry Andric Impl->TargetPasses[StandardID] = TargetID;
6660b57cec5SDimitry Andric }
6670b57cec5SDimitry Andric
getPassSubstitution(AnalysisID ID) const6680b57cec5SDimitry Andric IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {
6690b57cec5SDimitry Andric DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator
6700b57cec5SDimitry Andric I = Impl->TargetPasses.find(ID);
6710b57cec5SDimitry Andric if (I == Impl->TargetPasses.end())
6720b57cec5SDimitry Andric return ID;
6730b57cec5SDimitry Andric return I->second;
6740b57cec5SDimitry Andric }
6750b57cec5SDimitry Andric
isPassSubstitutedOrOverridden(AnalysisID ID) const6760b57cec5SDimitry Andric bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
6770b57cec5SDimitry Andric IdentifyingPassPtr TargetID = getPassSubstitution(ID);
6780b57cec5SDimitry Andric IdentifyingPassPtr FinalPtr = overridePass(ID, TargetID);
6790b57cec5SDimitry Andric return !FinalPtr.isValid() || FinalPtr.isInstance() ||
6800b57cec5SDimitry Andric FinalPtr.getID() != ID;
6810b57cec5SDimitry Andric }
6820b57cec5SDimitry Andric
6830b57cec5SDimitry Andric /// Add a pass to the PassManager if that pass is supposed to be run. If the
6840b57cec5SDimitry Andric /// Started/Stopped flags indicate either that the compilation should start at
6850b57cec5SDimitry Andric /// a later pass or that it should stop after an earlier pass, then do not add
6860b57cec5SDimitry Andric /// the pass. Finally, compare the current pass against the StartAfter
6870b57cec5SDimitry Andric /// and StopAfter options and change the Started/Stopped flags accordingly.
addPass(Pass * P)688349cc55cSDimitry Andric void TargetPassConfig::addPass(Pass *P) {
6890b57cec5SDimitry Andric assert(!Initialized && "PassConfig is immutable");
6900b57cec5SDimitry Andric
6910b57cec5SDimitry Andric // Cache the Pass ID here in case the pass manager finds this pass is
6920b57cec5SDimitry Andric // redundant with ones already scheduled / available, and deletes it.
6930b57cec5SDimitry Andric // Fundamentally, once we add the pass to the manager, we no longer own it
6940b57cec5SDimitry Andric // and shouldn't reference it.
6950b57cec5SDimitry Andric AnalysisID PassID = P->getPassID();
6960b57cec5SDimitry Andric
6970b57cec5SDimitry Andric if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum)
6980b57cec5SDimitry Andric Started = true;
6990b57cec5SDimitry Andric if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum)
7000b57cec5SDimitry Andric Stopped = true;
7010b57cec5SDimitry Andric if (Started && !Stopped) {
70281ad6265SDimitry Andric if (AddingMachinePasses) {
7030b57cec5SDimitry Andric // Construct banner message before PM->add() as that may delete the pass.
70481ad6265SDimitry Andric std::string Banner =
70581ad6265SDimitry Andric std::string("After ") + std::string(P->getPassName());
70681ad6265SDimitry Andric addMachinePrePasses();
7070b57cec5SDimitry Andric PM->add(P);
708349cc55cSDimitry Andric addMachinePostPasses(Banner);
70981ad6265SDimitry Andric } else {
71081ad6265SDimitry Andric PM->add(P);
71181ad6265SDimitry Andric }
7120b57cec5SDimitry Andric
7130b57cec5SDimitry Andric // Add the passes after the pass P if there is any.
71481ad6265SDimitry Andric for (const auto &IP : Impl->InsertedPasses)
7150b57cec5SDimitry Andric if (IP.TargetPassID == PassID)
716349cc55cSDimitry Andric addPass(IP.getInsertedPass());
7170b57cec5SDimitry Andric } else {
7180b57cec5SDimitry Andric delete P;
7190b57cec5SDimitry Andric }
7200b57cec5SDimitry Andric
7210b57cec5SDimitry Andric if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum)
7220b57cec5SDimitry Andric Stopped = true;
7230b57cec5SDimitry Andric
7240b57cec5SDimitry Andric if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum)
7250b57cec5SDimitry Andric Started = true;
7260b57cec5SDimitry Andric if (Stopped && !Started)
7270b57cec5SDimitry Andric report_fatal_error("Cannot stop compilation after pass that is not run");
7280b57cec5SDimitry Andric }
7290b57cec5SDimitry Andric
7300b57cec5SDimitry Andric /// Add a CodeGen pass at this point in the pipeline after checking for target
7310b57cec5SDimitry Andric /// and command line overrides.
7320b57cec5SDimitry Andric ///
7330b57cec5SDimitry Andric /// addPass cannot return a pointer to the pass instance because is internal the
7340b57cec5SDimitry Andric /// PassManager and the instance we create here may already be freed.
addPass(AnalysisID PassID)735349cc55cSDimitry Andric AnalysisID TargetPassConfig::addPass(AnalysisID PassID) {
7360b57cec5SDimitry Andric IdentifyingPassPtr TargetID = getPassSubstitution(PassID);
7370b57cec5SDimitry Andric IdentifyingPassPtr FinalPtr = overridePass(PassID, TargetID);
7380b57cec5SDimitry Andric if (!FinalPtr.isValid())
7390b57cec5SDimitry Andric return nullptr;
7400b57cec5SDimitry Andric
7410b57cec5SDimitry Andric Pass *P;
7420b57cec5SDimitry Andric if (FinalPtr.isInstance())
7430b57cec5SDimitry Andric P = FinalPtr.getInstance();
7440b57cec5SDimitry Andric else {
7450b57cec5SDimitry Andric P = Pass::createPass(FinalPtr.getID());
7460b57cec5SDimitry Andric if (!P)
7470b57cec5SDimitry Andric llvm_unreachable("Pass ID not registered");
7480b57cec5SDimitry Andric }
7490b57cec5SDimitry Andric AnalysisID FinalID = P->getPassID();
750349cc55cSDimitry Andric addPass(P); // Ends the lifetime of P.
7510b57cec5SDimitry Andric
7520b57cec5SDimitry Andric return FinalID;
7530b57cec5SDimitry Andric }
7540b57cec5SDimitry Andric
printAndVerify(const std::string & Banner)7550b57cec5SDimitry Andric void TargetPassConfig::printAndVerify(const std::string &Banner) {
7560b57cec5SDimitry Andric addPrintPass(Banner);
7570b57cec5SDimitry Andric addVerifyPass(Banner);
7580b57cec5SDimitry Andric }
7590b57cec5SDimitry Andric
addPrintPass(const std::string & Banner)7600b57cec5SDimitry Andric void TargetPassConfig::addPrintPass(const std::string &Banner) {
761e8d8bef9SDimitry Andric if (PrintAfterISel)
7620b57cec5SDimitry Andric PM->add(createMachineFunctionPrinterPass(dbgs(), Banner));
7630b57cec5SDimitry Andric }
7640b57cec5SDimitry Andric
addVerifyPass(const std::string & Banner)7650b57cec5SDimitry Andric void TargetPassConfig::addVerifyPass(const std::string &Banner) {
7660b57cec5SDimitry Andric bool Verify = VerifyMachineCode == cl::BOU_TRUE;
7670b57cec5SDimitry Andric #ifdef EXPENSIVE_CHECKS
7680b57cec5SDimitry Andric if (VerifyMachineCode == cl::BOU_UNSET)
7690b57cec5SDimitry Andric Verify = TM->isMachineVerifierClean();
7700b57cec5SDimitry Andric #endif
7710b57cec5SDimitry Andric if (Verify)
7720b57cec5SDimitry Andric PM->add(createMachineVerifierPass(Banner));
7730b57cec5SDimitry Andric }
7740b57cec5SDimitry Andric
addDebugifyPass()7755ffd83dbSDimitry Andric void TargetPassConfig::addDebugifyPass() {
7765ffd83dbSDimitry Andric PM->add(createDebugifyMachineModulePass());
7775ffd83dbSDimitry Andric }
7785ffd83dbSDimitry Andric
addStripDebugPass()7795ffd83dbSDimitry Andric void TargetPassConfig::addStripDebugPass() {
7805ffd83dbSDimitry Andric PM->add(createStripDebugMachineModulePass(/*OnlyDebugified=*/true));
7815ffd83dbSDimitry Andric }
7825ffd83dbSDimitry Andric
addCheckDebugPass()783e8d8bef9SDimitry Andric void TargetPassConfig::addCheckDebugPass() {
784e8d8bef9SDimitry Andric PM->add(createCheckDebugMachineModulePass());
785e8d8bef9SDimitry Andric }
786e8d8bef9SDimitry Andric
addMachinePrePasses(bool AllowDebugify)7875ffd83dbSDimitry Andric void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) {
788e8d8bef9SDimitry Andric if (AllowDebugify && DebugifyIsSafe &&
789e8d8bef9SDimitry Andric (DebugifyAndStripAll == cl::BOU_TRUE ||
790e8d8bef9SDimitry Andric DebugifyCheckAndStripAll == cl::BOU_TRUE))
7915ffd83dbSDimitry Andric addDebugifyPass();
7925ffd83dbSDimitry Andric }
7935ffd83dbSDimitry Andric
addMachinePostPasses(const std::string & Banner)794349cc55cSDimitry Andric void TargetPassConfig::addMachinePostPasses(const std::string &Banner) {
795e8d8bef9SDimitry Andric if (DebugifyIsSafe) {
796e8d8bef9SDimitry Andric if (DebugifyCheckAndStripAll == cl::BOU_TRUE) {
797e8d8bef9SDimitry Andric addCheckDebugPass();
7985ffd83dbSDimitry Andric addStripDebugPass();
799e8d8bef9SDimitry Andric } else if (DebugifyAndStripAll == cl::BOU_TRUE)
800e8d8bef9SDimitry Andric addStripDebugPass();
801e8d8bef9SDimitry Andric }
8025ffd83dbSDimitry Andric addVerifyPass(Banner);
8035ffd83dbSDimitry Andric }
8045ffd83dbSDimitry Andric
8050b57cec5SDimitry Andric /// Add common target configurable passes that perform LLVM IR to IR transforms
8060b57cec5SDimitry Andric /// following machine independent optimization.
addIRPasses()8070b57cec5SDimitry Andric void TargetPassConfig::addIRPasses() {
8085ffd83dbSDimitry Andric // Before running any passes, run the verifier to determine if the input
8095ffd83dbSDimitry Andric // coming from the front-end and/or optimizer is valid.
8105ffd83dbSDimitry Andric if (!DisableVerify)
8115ffd83dbSDimitry Andric addPass(createVerifierPass());
8125ffd83dbSDimitry Andric
8135f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None) {
8140b57cec5SDimitry Andric // Basic AliasAnalysis support.
8150b57cec5SDimitry Andric // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
8160b57cec5SDimitry Andric // BasicAliasAnalysis wins if they disagree. This is intended to help
8170b57cec5SDimitry Andric // support "obvious" type-punning idioms.
8180b57cec5SDimitry Andric addPass(createTypeBasedAAWrapperPass());
8190b57cec5SDimitry Andric addPass(createScopedNoAliasAAWrapperPass());
8200b57cec5SDimitry Andric addPass(createBasicAAWrapperPass());
8210b57cec5SDimitry Andric
8220b57cec5SDimitry Andric // Run loop strength reduction before anything else.
8235ffd83dbSDimitry Andric if (!DisableLSR) {
8245ffd83dbSDimitry Andric addPass(createCanonicalizeFreezeInLoopsPass());
8250b57cec5SDimitry Andric addPass(createLoopStrengthReducePass());
8260b57cec5SDimitry Andric if (PrintLSR)
8275ffd83dbSDimitry Andric addPass(createPrintFunctionPass(dbgs(),
8285ffd83dbSDimitry Andric "\n\n*** Code after LSR ***\n"));
8290b57cec5SDimitry Andric }
8300b57cec5SDimitry Andric
8310b57cec5SDimitry Andric // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
8320b57cec5SDimitry Andric // loads and compares. ExpandMemCmpPass then tries to expand those calls
8330b57cec5SDimitry Andric // into optimally-sized loads and compares. The transforms are enabled by a
8340b57cec5SDimitry Andric // target lowering hook.
8350b57cec5SDimitry Andric if (!DisableMergeICmps)
8360b57cec5SDimitry Andric addPass(createMergeICmpsLegacyPass());
8375f757f3fSDimitry Andric addPass(createExpandMemCmpLegacyPass());
8380b57cec5SDimitry Andric }
8390b57cec5SDimitry Andric
8400b57cec5SDimitry Andric // Run GC lowering passes for builtin collectors
8410b57cec5SDimitry Andric // TODO: add a pass insertion point here
842fe6060f1SDimitry Andric addPass(&GCLoweringID);
843fe6060f1SDimitry Andric addPass(&ShadowStackGCLoweringID);
8448bcb0991SDimitry Andric addPass(createLowerConstantIntrinsicsPass());
8450b57cec5SDimitry Andric
846bdd1243dSDimitry Andric // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
84781ad6265SDimitry Andric // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
84881ad6265SDimitry Andric if (TM->getTargetTriple().isOSBinFormatMachO() &&
84906c3fb27SDimitry Andric !DisableAtExitBasedGlobalDtorLowering)
85081ad6265SDimitry Andric addPass(createLowerGlobalDtorsLegacyPass());
85181ad6265SDimitry Andric
8520b57cec5SDimitry Andric // Make sure that no unreachable blocks are instruction selected.
8530b57cec5SDimitry Andric addPass(createUnreachableBlockEliminationPass());
8540b57cec5SDimitry Andric
8550b57cec5SDimitry Andric // Prepare expensive constants for SelectionDAG.
8565f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None && !DisableConstantHoisting)
8570b57cec5SDimitry Andric addPass(createConstantHoistingPass());
8580b57cec5SDimitry Andric
8595f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None)
860fe6060f1SDimitry Andric addPass(createReplaceWithVeclibLegacyPass());
861fe6060f1SDimitry Andric
8625f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining)
8630b57cec5SDimitry Andric addPass(createPartiallyInlineLibCallsPass());
8640b57cec5SDimitry Andric
865fe6060f1SDimitry Andric // Expand vector predication intrinsics into standard IR instructions.
866fe6060f1SDimitry Andric // This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction
867fe6060f1SDimitry Andric // passes since it emits those kinds of intrinsics.
868fe6060f1SDimitry Andric addPass(createExpandVectorPredicationPass());
8690b57cec5SDimitry Andric
8700b57cec5SDimitry Andric // Add scalarization of target's unsupported masked memory intrinsics pass.
8710b57cec5SDimitry Andric // the unsupported intrinsic will be replaced with a chain of basic blocks,
8720b57cec5SDimitry Andric // that stores/loads element one-by-one if the appropriate mask bit is set.
873e8d8bef9SDimitry Andric addPass(createScalarizeMaskedMemIntrinLegacyPass());
8740b57cec5SDimitry Andric
8750b57cec5SDimitry Andric // Expand reduction intrinsics into shuffle sequences if the target wants to.
876e8d8bef9SDimitry Andric // Allow disabling it for testing purposes.
877e8d8bef9SDimitry Andric if (!DisableExpandReductions)
8780b57cec5SDimitry Andric addPass(createExpandReductionsPass());
87981ad6265SDimitry Andric
8805f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None)
88181ad6265SDimitry Andric addPass(createTLSVariableHoistPass());
88281ad6265SDimitry Andric
88381ad6265SDimitry Andric // Convert conditional moves to conditional jumps when profitable.
8845f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None && !DisableSelectOptimize)
88581ad6265SDimitry Andric addPass(createSelectOptimizePass());
8860b57cec5SDimitry Andric }
8870b57cec5SDimitry Andric
8880b57cec5SDimitry Andric /// Turn exception handling constructs into something the code generators can
8890b57cec5SDimitry Andric /// handle.
addPassesToHandleExceptions()8900b57cec5SDimitry Andric void TargetPassConfig::addPassesToHandleExceptions() {
8910b57cec5SDimitry Andric const MCAsmInfo *MCAI = TM->getMCAsmInfo();
8920b57cec5SDimitry Andric assert(MCAI && "No MCAsmInfo");
8930b57cec5SDimitry Andric switch (MCAI->getExceptionHandlingType()) {
8940b57cec5SDimitry Andric case ExceptionHandling::SjLj:
8950b57cec5SDimitry Andric // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
8960b57cec5SDimitry Andric // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
8970b57cec5SDimitry Andric // catch info can get misplaced when a selector ends up more than one block
8980b57cec5SDimitry Andric // removed from the parent invoke(s). This could happen when a landing
8990b57cec5SDimitry Andric // pad is shared by multiple invokes and is also a target of a normal
9000b57cec5SDimitry Andric // edge from elsewhere.
9015ffd83dbSDimitry Andric addPass(createSjLjEHPreparePass(TM));
902bdd1243dSDimitry Andric [[fallthrough]];
9030b57cec5SDimitry Andric case ExceptionHandling::DwarfCFI:
9040b57cec5SDimitry Andric case ExceptionHandling::ARM:
905e8d8bef9SDimitry Andric case ExceptionHandling::AIX:
906cb14a3feSDimitry Andric case ExceptionHandling::ZOS:
9075ffd83dbSDimitry Andric addPass(createDwarfEHPass(getOptLevel()));
9080b57cec5SDimitry Andric break;
9090b57cec5SDimitry Andric case ExceptionHandling::WinEH:
9100b57cec5SDimitry Andric // We support using both GCC-style and MSVC-style exceptions on Windows, so
9110b57cec5SDimitry Andric // add both preparation passes. Each pass will only actually run if it
9120b57cec5SDimitry Andric // recognizes the personality function.
9130b57cec5SDimitry Andric addPass(createWinEHPass());
9145ffd83dbSDimitry Andric addPass(createDwarfEHPass(getOptLevel()));
9150b57cec5SDimitry Andric break;
9160b57cec5SDimitry Andric case ExceptionHandling::Wasm:
9170b57cec5SDimitry Andric // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
9180b57cec5SDimitry Andric // on catchpads and cleanuppads because it does not outline them into
9190b57cec5SDimitry Andric // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
9200b57cec5SDimitry Andric // should remove PHIs there.
9210b57cec5SDimitry Andric addPass(createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/false));
9220b57cec5SDimitry Andric addPass(createWasmEHPass());
9230b57cec5SDimitry Andric break;
9240b57cec5SDimitry Andric case ExceptionHandling::None:
9250b57cec5SDimitry Andric addPass(createLowerInvokePass());
9260b57cec5SDimitry Andric
9270b57cec5SDimitry Andric // The lower invoke pass may create unreachable code. Remove it.
9280b57cec5SDimitry Andric addPass(createUnreachableBlockEliminationPass());
9290b57cec5SDimitry Andric break;
9300b57cec5SDimitry Andric }
9310b57cec5SDimitry Andric }
9320b57cec5SDimitry Andric
9330b57cec5SDimitry Andric /// Add pass to prepare the LLVM IR for code generation. This should be done
9340b57cec5SDimitry Andric /// before exception handling preparation passes.
addCodeGenPrepare()9350b57cec5SDimitry Andric void TargetPassConfig::addCodeGenPrepare() {
9365f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None && !DisableCGP)
9371db9f3b2SDimitry Andric addPass(createCodeGenPrepareLegacyPass());
9380b57cec5SDimitry Andric }
9390b57cec5SDimitry Andric
9400b57cec5SDimitry Andric /// Add common passes that perform LLVM IR to IR transforms in preparation for
9410b57cec5SDimitry Andric /// instruction selection.
addISelPrepare()9420b57cec5SDimitry Andric void TargetPassConfig::addISelPrepare() {
9430b57cec5SDimitry Andric addPreISel();
9440b57cec5SDimitry Andric
9450b57cec5SDimitry Andric // Force codegen to run according to the callgraph.
9460b57cec5SDimitry Andric if (requiresCodeGenSCCOrder())
9470b57cec5SDimitry Andric addPass(new DummyCGSCCPass);
9480b57cec5SDimitry Andric
94906c3fb27SDimitry Andric addPass(createCallBrPass());
95006c3fb27SDimitry Andric
9510b57cec5SDimitry Andric // Add both the safe stack and the stack protection passes: each of them will
9520b57cec5SDimitry Andric // only protect functions that have corresponding attributes.
9530b57cec5SDimitry Andric addPass(createSafeStackPass());
9540b57cec5SDimitry Andric addPass(createStackProtectorPass());
9550b57cec5SDimitry Andric
9560b57cec5SDimitry Andric if (PrintISelInput)
9570b57cec5SDimitry Andric addPass(createPrintFunctionPass(
9580b57cec5SDimitry Andric dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"));
9590b57cec5SDimitry Andric
9600b57cec5SDimitry Andric // All passes which modify the LLVM IR are now complete; run the verifier
9610b57cec5SDimitry Andric // to ensure that the IR is valid.
9620b57cec5SDimitry Andric if (!DisableVerify)
9630b57cec5SDimitry Andric addPass(createVerifierPass());
9640b57cec5SDimitry Andric }
9650b57cec5SDimitry Andric
addCoreISelPasses()9660b57cec5SDimitry Andric bool TargetPassConfig::addCoreISelPasses() {
9670b57cec5SDimitry Andric // Enable FastISel with -fast-isel, but allow that to be overridden.
9680b57cec5SDimitry Andric TM->setO0WantsFastISel(EnableFastISelOption != cl::BOU_FALSE);
9690b57cec5SDimitry Andric
9700b57cec5SDimitry Andric // Determine an instruction selector.
9710b57cec5SDimitry Andric enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
9720b57cec5SDimitry Andric SelectorType Selector;
9730b57cec5SDimitry Andric
9740b57cec5SDimitry Andric if (EnableFastISelOption == cl::BOU_TRUE)
9750b57cec5SDimitry Andric Selector = SelectorType::FastISel;
9760b57cec5SDimitry Andric else if (EnableGlobalISelOption == cl::BOU_TRUE ||
9770b57cec5SDimitry Andric (TM->Options.EnableGlobalISel &&
9780b57cec5SDimitry Andric EnableGlobalISelOption != cl::BOU_FALSE))
9790b57cec5SDimitry Andric Selector = SelectorType::GlobalISel;
9805f757f3fSDimitry Andric else if (TM->getOptLevel() == CodeGenOptLevel::None &&
9815f757f3fSDimitry Andric TM->getO0WantsFastISel())
9820b57cec5SDimitry Andric Selector = SelectorType::FastISel;
9830b57cec5SDimitry Andric else
9840b57cec5SDimitry Andric Selector = SelectorType::SelectionDAG;
9850b57cec5SDimitry Andric
9860b57cec5SDimitry Andric // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
9870b57cec5SDimitry Andric if (Selector == SelectorType::FastISel) {
9880b57cec5SDimitry Andric TM->setFastISel(true);
9890b57cec5SDimitry Andric TM->setGlobalISel(false);
9900b57cec5SDimitry Andric } else if (Selector == SelectorType::GlobalISel) {
9910b57cec5SDimitry Andric TM->setFastISel(false);
9920b57cec5SDimitry Andric TM->setGlobalISel(true);
9930b57cec5SDimitry Andric }
9940b57cec5SDimitry Andric
9955ffd83dbSDimitry Andric // FIXME: Injecting into the DAGISel pipeline seems to cause issues with
9965ffd83dbSDimitry Andric // analyses needing to be re-run. This can result in being unable to
9975ffd83dbSDimitry Andric // schedule passes (particularly with 'Function Alias Analysis
9985ffd83dbSDimitry Andric // Results'). It's not entirely clear why but AFAICT this seems to be
9995ffd83dbSDimitry Andric // due to one FunctionPassManager not being able to use analyses from a
10005ffd83dbSDimitry Andric // previous one. As we're injecting a ModulePass we break the usual
10015ffd83dbSDimitry Andric // pass manager into two. GlobalISel with the fallback path disabled
10025ffd83dbSDimitry Andric // and -run-pass seem to be unaffected. The majority of GlobalISel
10035ffd83dbSDimitry Andric // testing uses -run-pass so this probably isn't too bad.
1004bdd1243dSDimitry Andric SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe);
10055ffd83dbSDimitry Andric if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
10065ffd83dbSDimitry Andric DebugifyIsSafe = false;
10075ffd83dbSDimitry Andric
10080b57cec5SDimitry Andric // Add instruction selector passes.
10090b57cec5SDimitry Andric if (Selector == SelectorType::GlobalISel) {
1010bdd1243dSDimitry Andric SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true);
10110b57cec5SDimitry Andric if (addIRTranslator())
10120b57cec5SDimitry Andric return true;
10130b57cec5SDimitry Andric
10140b57cec5SDimitry Andric addPreLegalizeMachineIR();
10150b57cec5SDimitry Andric
10160b57cec5SDimitry Andric if (addLegalizeMachineIR())
10170b57cec5SDimitry Andric return true;
10180b57cec5SDimitry Andric
10190b57cec5SDimitry Andric // Before running the register bank selector, ask the target if it
10200b57cec5SDimitry Andric // wants to run some passes.
10210b57cec5SDimitry Andric addPreRegBankSelect();
10220b57cec5SDimitry Andric
10230b57cec5SDimitry Andric if (addRegBankSelect())
10240b57cec5SDimitry Andric return true;
10250b57cec5SDimitry Andric
10260b57cec5SDimitry Andric addPreGlobalInstructionSelect();
10270b57cec5SDimitry Andric
10280b57cec5SDimitry Andric if (addGlobalInstructionSelect())
10290b57cec5SDimitry Andric return true;
10300b57cec5SDimitry Andric
10310b57cec5SDimitry Andric // Pass to reset the MachineFunction if the ISel failed.
10320b57cec5SDimitry Andric addPass(createResetMachineFunctionPass(
10330b57cec5SDimitry Andric reportDiagnosticWhenGlobalISelFallback(), isGlobalISelAbortEnabled()));
10340b57cec5SDimitry Andric
10350b57cec5SDimitry Andric // Provide a fallback path when we do not want to abort on
10360b57cec5SDimitry Andric // not-yet-supported input.
10370b57cec5SDimitry Andric if (!isGlobalISelAbortEnabled() && addInstSelector())
10380b57cec5SDimitry Andric return true;
10390b57cec5SDimitry Andric
10400b57cec5SDimitry Andric } else if (addInstSelector())
10410b57cec5SDimitry Andric return true;
10420b57cec5SDimitry Andric
10430b57cec5SDimitry Andric // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
10440b57cec5SDimitry Andric // FinalizeISel.
10450b57cec5SDimitry Andric addPass(&FinalizeISelID);
10460b57cec5SDimitry Andric
10470b57cec5SDimitry Andric // Print the instruction selected machine code...
10480b57cec5SDimitry Andric printAndVerify("After Instruction Selection");
10490b57cec5SDimitry Andric
10500b57cec5SDimitry Andric return false;
10510b57cec5SDimitry Andric }
10520b57cec5SDimitry Andric
addISelPasses()10530b57cec5SDimitry Andric bool TargetPassConfig::addISelPasses() {
10540b57cec5SDimitry Andric if (TM->useEmulatedTLS())
10550b57cec5SDimitry Andric addPass(createLowerEmuTLSPass());
10560b57cec5SDimitry Andric
1057e8d8bef9SDimitry Andric PM->add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis()));
105806c3fb27SDimitry Andric addPass(createPreISelIntrinsicLoweringPass());
1059bdd1243dSDimitry Andric addPass(createExpandLargeDivRemPass());
1060bdd1243dSDimitry Andric addPass(createExpandLargeFpConvertPass());
10610b57cec5SDimitry Andric addIRPasses();
10620b57cec5SDimitry Andric addCodeGenPrepare();
10630b57cec5SDimitry Andric addPassesToHandleExceptions();
10640b57cec5SDimitry Andric addISelPrepare();
10650b57cec5SDimitry Andric
10660b57cec5SDimitry Andric return addCoreISelPasses();
10670b57cec5SDimitry Andric }
10680b57cec5SDimitry Andric
10690b57cec5SDimitry Andric /// -regalloc=... command line option.
useDefaultRegisterAllocator()10700b57cec5SDimitry Andric static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
10710b57cec5SDimitry Andric static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
10720b57cec5SDimitry Andric RegisterPassParser<RegisterRegAlloc>>
10730b57cec5SDimitry Andric RegAlloc("regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),
10740b57cec5SDimitry Andric cl::desc("Register allocator to use"));
10750b57cec5SDimitry Andric
10760b57cec5SDimitry Andric /// Add the complete set of target-independent postISel code generator passes.
10770b57cec5SDimitry Andric ///
10780b57cec5SDimitry Andric /// This can be read as the standard order of major LLVM CodeGen stages. Stages
10790b57cec5SDimitry Andric /// with nontrivial configuration or multiple passes are broken out below in
10800b57cec5SDimitry Andric /// add%Stage routines.
10810b57cec5SDimitry Andric ///
10820b57cec5SDimitry Andric /// Any TargetPassConfig::addXX routine may be overriden by the Target. The
10830b57cec5SDimitry Andric /// addPre/Post methods with empty header implementations allow injecting
10840b57cec5SDimitry Andric /// target-specific fixups just before or after major stages. Additionally,
10850b57cec5SDimitry Andric /// targets have the flexibility to change pass order within a stage by
10860b57cec5SDimitry Andric /// overriding default implementation of add%Stage routines below. Each
10870b57cec5SDimitry Andric /// technique has maintainability tradeoffs because alternate pass orders are
10880b57cec5SDimitry Andric /// not well supported. addPre/Post works better if the target pass is easily
10890b57cec5SDimitry Andric /// tied to a common pass. But if it has subtle dependencies on multiple passes,
10900b57cec5SDimitry Andric /// the target should override the stage instead.
10910b57cec5SDimitry Andric ///
10920b57cec5SDimitry Andric /// TODO: We could use a single addPre/Post(ID) hook to allow pass injection
10930b57cec5SDimitry Andric /// before/after any target-independent pass. But it's currently overkill.
addMachinePasses()10940b57cec5SDimitry Andric void TargetPassConfig::addMachinePasses() {
10950b57cec5SDimitry Andric AddingMachinePasses = true;
10960b57cec5SDimitry Andric
10970b57cec5SDimitry Andric // Add passes that optimize machine instructions in SSA form.
10985f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None) {
10990b57cec5SDimitry Andric addMachineSSAOptimization();
11000b57cec5SDimitry Andric } else {
11010b57cec5SDimitry Andric // If the target requests it, assign local variables to stack slots relative
11020b57cec5SDimitry Andric // to one another and simplify frame index references where possible.
11035ffd83dbSDimitry Andric addPass(&LocalStackSlotAllocationID);
11040b57cec5SDimitry Andric }
11050b57cec5SDimitry Andric
11060b57cec5SDimitry Andric if (TM->Options.EnableIPRA)
11070b57cec5SDimitry Andric addPass(createRegUsageInfoPropPass());
11080b57cec5SDimitry Andric
11090b57cec5SDimitry Andric // Run pre-ra passes.
11100b57cec5SDimitry Andric addPreRegAlloc();
11110b57cec5SDimitry Andric
11125ffd83dbSDimitry Andric // Debugifying the register allocator passes seems to provoke some
11135ffd83dbSDimitry Andric // non-determinism that affects CodeGen and there doesn't seem to be a point
11145ffd83dbSDimitry Andric // where it becomes safe again so stop debugifying here.
11155ffd83dbSDimitry Andric DebugifyIsSafe = false;
11165ffd83dbSDimitry Andric
1117349cc55cSDimitry Andric // Add a FSDiscriminator pass right before RA, so that we could get
1118349cc55cSDimitry Andric // more precise SampleFDO profile for RA.
1119349cc55cSDimitry Andric if (EnableFSDiscriminator) {
1120349cc55cSDimitry Andric addPass(createMIRAddFSDiscriminatorsPass(
1121349cc55cSDimitry Andric sampleprof::FSDiscriminatorPass::Pass1));
1122349cc55cSDimitry Andric const std::string ProfileFile = getFSProfileFile(TM);
1123349cc55cSDimitry Andric if (!ProfileFile.empty() && !DisableRAFSProfileLoader)
112406c3fb27SDimitry Andric addPass(createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM),
112506c3fb27SDimitry Andric sampleprof::FSDiscriminatorPass::Pass1,
112606c3fb27SDimitry Andric nullptr));
1127349cc55cSDimitry Andric }
1128349cc55cSDimitry Andric
11290b57cec5SDimitry Andric // Run register allocation and passes that are tightly coupled with it,
11300b57cec5SDimitry Andric // including phi elimination and scheduling.
11310b57cec5SDimitry Andric if (getOptimizeRegAlloc())
11320b57cec5SDimitry Andric addOptimizedRegAlloc();
11330b57cec5SDimitry Andric else
11340b57cec5SDimitry Andric addFastRegAlloc();
11350b57cec5SDimitry Andric
11360b57cec5SDimitry Andric // Run post-ra passes.
11370b57cec5SDimitry Andric addPostRegAlloc();
11380b57cec5SDimitry Andric
1139349cc55cSDimitry Andric addPass(&RemoveRedundantDebugValuesID);
1140fe6060f1SDimitry Andric
11415ffd83dbSDimitry Andric addPass(&FixupStatepointCallerSavedID);
11425ffd83dbSDimitry Andric
11430b57cec5SDimitry Andric // Insert prolog/epilog code. Eliminate abstract frame index references...
11445f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None) {
11450b57cec5SDimitry Andric addPass(&PostRAMachineSinkingID);
11460b57cec5SDimitry Andric addPass(&ShrinkWrapID);
11470b57cec5SDimitry Andric }
11480b57cec5SDimitry Andric
11490b57cec5SDimitry Andric // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
11500b57cec5SDimitry Andric // do so if it hasn't been disabled, substituted, or overridden.
11510b57cec5SDimitry Andric if (!isPassSubstitutedOrOverridden(&PrologEpilogCodeInserterID))
11520b57cec5SDimitry Andric addPass(createPrologEpilogInserterPass());
11530b57cec5SDimitry Andric
11540b57cec5SDimitry Andric /// Add passes that optimize machine instructions after register allocation.
11555f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None)
11560b57cec5SDimitry Andric addMachineLateOptimization();
11570b57cec5SDimitry Andric
11580b57cec5SDimitry Andric // Expand pseudo instructions before second scheduling pass.
11590b57cec5SDimitry Andric addPass(&ExpandPostRAPseudosID);
11600b57cec5SDimitry Andric
11610b57cec5SDimitry Andric // Run pre-sched2 passes.
11620b57cec5SDimitry Andric addPreSched2();
11630b57cec5SDimitry Andric
11640b57cec5SDimitry Andric if (EnableImplicitNullChecks)
11650b57cec5SDimitry Andric addPass(&ImplicitNullChecksID);
11660b57cec5SDimitry Andric
11670b57cec5SDimitry Andric // Second pass scheduler.
11680b57cec5SDimitry Andric // Let Target optionally insert this pass by itself at some other
11690b57cec5SDimitry Andric // point.
11705f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None &&
11710b57cec5SDimitry Andric !TM->targetSchedulesPostRAScheduling()) {
11720b57cec5SDimitry Andric if (MISchedPostRA)
11730b57cec5SDimitry Andric addPass(&PostMachineSchedulerID);
11740b57cec5SDimitry Andric else
11750b57cec5SDimitry Andric addPass(&PostRASchedulerID);
11760b57cec5SDimitry Andric }
11770b57cec5SDimitry Andric
11780b57cec5SDimitry Andric // GC
11795f757f3fSDimitry Andric addGCPasses();
11800b57cec5SDimitry Andric
11810b57cec5SDimitry Andric // Basic block placement.
11825f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None)
11830b57cec5SDimitry Andric addBlockPlacement();
11840b57cec5SDimitry Andric
118555e4f9d5SDimitry Andric // Insert before XRay Instrumentation.
11865ffd83dbSDimitry Andric addPass(&FEntryInserterID);
118755e4f9d5SDimitry Andric
11885ffd83dbSDimitry Andric addPass(&XRayInstrumentationID);
11895ffd83dbSDimitry Andric addPass(&PatchableFunctionID);
119055e4f9d5SDimitry Andric
11910b57cec5SDimitry Andric addPreEmitPass();
11920b57cec5SDimitry Andric
11930b57cec5SDimitry Andric if (TM->Options.EnableIPRA)
11940b57cec5SDimitry Andric // Collect register usage information and produce a register mask of
11950b57cec5SDimitry Andric // clobbered registers, to be used to optimize call sites.
11960b57cec5SDimitry Andric addPass(createRegUsageInfoCollector());
11970b57cec5SDimitry Andric
11985ffd83dbSDimitry Andric // FIXME: Some backends are incompatible with running the verifier after
11995ffd83dbSDimitry Andric // addPreEmitPass. Maybe only pass "false" here for those targets?
1200349cc55cSDimitry Andric addPass(&FuncletLayoutID);
12010b57cec5SDimitry Andric
1202349cc55cSDimitry Andric addPass(&StackMapLivenessID);
1203349cc55cSDimitry Andric addPass(&LiveDebugValuesID);
1204bdd1243dSDimitry Andric addPass(&MachineSanitizerBinaryMetadataID);
12050b57cec5SDimitry Andric
12065f757f3fSDimitry Andric if (TM->Options.EnableMachineOutliner &&
12075f757f3fSDimitry Andric getOptLevel() != CodeGenOptLevel::None &&
1208e8d8bef9SDimitry Andric EnableMachineOutliner != RunOutliner::NeverOutline) {
1209e8d8bef9SDimitry Andric bool RunOnAllFunctions =
1210e8d8bef9SDimitry Andric (EnableMachineOutliner == RunOutliner::AlwaysOutline);
1211e8d8bef9SDimitry Andric bool AddOutliner =
1212e8d8bef9SDimitry Andric RunOnAllFunctions || TM->Options.SupportsDefaultOutlining;
12130b57cec5SDimitry Andric if (AddOutliner)
12140b57cec5SDimitry Andric addPass(createMachineOutlinerPass(RunOnAllFunctions));
12150b57cec5SDimitry Andric }
12160b57cec5SDimitry Andric
12175f757f3fSDimitry Andric if (GCEmptyBlocks)
12185f757f3fSDimitry Andric addPass(llvm::createGCEmptyBasicBlocksPass());
12195f757f3fSDimitry Andric
122006c3fb27SDimitry Andric if (EnableFSDiscriminator)
122106c3fb27SDimitry Andric addPass(createMIRAddFSDiscriminatorsPass(
122206c3fb27SDimitry Andric sampleprof::FSDiscriminatorPass::PassLast));
122306c3fb27SDimitry Andric
1224e8d8bef9SDimitry Andric // Machine function splitter uses the basic block sections feature. Both
1225fe6060f1SDimitry Andric // cannot be enabled at the same time. Basic block sections takes precedence.
1226fe6060f1SDimitry Andric // FIXME: In principle, BasicBlockSection::Labels and splitting can used
1227fe6060f1SDimitry Andric // together. Update this check once we have addressed any issues.
1228fe6060f1SDimitry Andric if (TM->getBBSectionsType() != llvm::BasicBlockSection::None) {
122981ad6265SDimitry Andric if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
12301db9f3b2SDimitry Andric addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass(
123181ad6265SDimitry Andric TM->getBBSectionsFuncListBuf()));
12325f757f3fSDimitry Andric addPass(llvm::createBasicBlockPathCloningPass());
123381ad6265SDimitry Andric }
123481ad6265SDimitry Andric addPass(llvm::createBasicBlockSectionsPass());
1235fe6060f1SDimitry Andric } else if (TM->Options.EnableMachineFunctionSplitter ||
1236e8d8bef9SDimitry Andric EnableMachineFunctionSplitter) {
123706c3fb27SDimitry Andric const std::string ProfileFile = getFSProfileFile(TM);
123806c3fb27SDimitry Andric if (!ProfileFile.empty()) {
123906c3fb27SDimitry Andric if (EnableFSDiscriminator) {
124006c3fb27SDimitry Andric addPass(createMIRProfileLoaderPass(
124106c3fb27SDimitry Andric ProfileFile, getFSRemappingFile(TM),
124206c3fb27SDimitry Andric sampleprof::FSDiscriminatorPass::PassLast, nullptr));
124306c3fb27SDimitry Andric } else {
124406c3fb27SDimitry Andric // Sample profile is given, but FSDiscriminator is not
124506c3fb27SDimitry Andric // enabled, this may result in performance regression.
124606c3fb27SDimitry Andric WithColor::warning()
124706c3fb27SDimitry Andric << "Using AutoFDO without FSDiscriminator for MFS may regress "
12487a6dacacSDimitry Andric "performance.\n";
124906c3fb27SDimitry Andric }
125006c3fb27SDimitry Andric }
1251e8d8bef9SDimitry Andric addPass(createMachineFunctionSplitterPass());
1252e8d8bef9SDimitry Andric }
12535ffd83dbSDimitry Andric
125406c3fb27SDimitry Andric addPostBBSections();
125506c3fb27SDimitry Andric
125681ad6265SDimitry Andric if (!DisableCFIFixup && TM->Options.EnableCFIFixup)
125781ad6265SDimitry Andric addPass(createCFIFixup());
125881ad6265SDimitry Andric
1259bdd1243dSDimitry Andric PM->add(createStackFrameLayoutAnalysisPass());
1260bdd1243dSDimitry Andric
12610b57cec5SDimitry Andric // Add passes that directly emit MI after all other MI passes.
12620b57cec5SDimitry Andric addPreEmitPass2();
12630b57cec5SDimitry Andric
12640b57cec5SDimitry Andric AddingMachinePasses = false;
12650b57cec5SDimitry Andric }
12660b57cec5SDimitry Andric
12670b57cec5SDimitry Andric /// Add passes that optimize machine instructions in SSA form.
addMachineSSAOptimization()12680b57cec5SDimitry Andric void TargetPassConfig::addMachineSSAOptimization() {
12690b57cec5SDimitry Andric // Pre-ra tail duplication.
12700b57cec5SDimitry Andric addPass(&EarlyTailDuplicateID);
12710b57cec5SDimitry Andric
12720b57cec5SDimitry Andric // Optimize PHIs before DCE: removing dead PHI cycles may make more
12730b57cec5SDimitry Andric // instructions dead.
12745ffd83dbSDimitry Andric addPass(&OptimizePHIsID);
12750b57cec5SDimitry Andric
12760b57cec5SDimitry Andric // This pass merges large allocas. StackSlotColoring is a different pass
12770b57cec5SDimitry Andric // which merges spill slots.
12785ffd83dbSDimitry Andric addPass(&StackColoringID);
12790b57cec5SDimitry Andric
12800b57cec5SDimitry Andric // If the target requests it, assign local variables to stack slots relative
12810b57cec5SDimitry Andric // to one another and simplify frame index references where possible.
12825ffd83dbSDimitry Andric addPass(&LocalStackSlotAllocationID);
12830b57cec5SDimitry Andric
12840b57cec5SDimitry Andric // With optimization, dead code should already be eliminated. However
12850b57cec5SDimitry Andric // there is one known exception: lowered code for arguments that are only
12860b57cec5SDimitry Andric // used by tail calls, where the tail calls reuse the incoming stack
12870b57cec5SDimitry Andric // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
12880b57cec5SDimitry Andric addPass(&DeadMachineInstructionElimID);
12890b57cec5SDimitry Andric
12900b57cec5SDimitry Andric // Allow targets to insert passes that improve instruction level parallelism,
12910b57cec5SDimitry Andric // like if-conversion. Such passes will typically need dominator trees and
12920b57cec5SDimitry Andric // loop info, just like LICM and CSE below.
12930b57cec5SDimitry Andric addILPOpts();
12940b57cec5SDimitry Andric
12955ffd83dbSDimitry Andric addPass(&EarlyMachineLICMID);
12965ffd83dbSDimitry Andric addPass(&MachineCSEID);
12970b57cec5SDimitry Andric
12980b57cec5SDimitry Andric addPass(&MachineSinkingID);
12990b57cec5SDimitry Andric
13000b57cec5SDimitry Andric addPass(&PeepholeOptimizerID);
13010b57cec5SDimitry Andric // Clean-up the dead code that may have been generated by peephole
13020b57cec5SDimitry Andric // rewriting.
13030b57cec5SDimitry Andric addPass(&DeadMachineInstructionElimID);
13040b57cec5SDimitry Andric }
13050b57cec5SDimitry Andric
13060b57cec5SDimitry Andric //===---------------------------------------------------------------------===//
13070b57cec5SDimitry Andric /// Register Allocation Pass Configuration
13080b57cec5SDimitry Andric //===---------------------------------------------------------------------===//
13090b57cec5SDimitry Andric
getOptimizeRegAlloc() const13100b57cec5SDimitry Andric bool TargetPassConfig::getOptimizeRegAlloc() const {
13110b57cec5SDimitry Andric switch (OptimizeRegAlloc) {
13125f757f3fSDimitry Andric case cl::BOU_UNSET:
13135f757f3fSDimitry Andric return getOptLevel() != CodeGenOptLevel::None;
13140b57cec5SDimitry Andric case cl::BOU_TRUE: return true;
13150b57cec5SDimitry Andric case cl::BOU_FALSE: return false;
13160b57cec5SDimitry Andric }
13170b57cec5SDimitry Andric llvm_unreachable("Invalid optimize-regalloc state");
13180b57cec5SDimitry Andric }
13190b57cec5SDimitry Andric
13200b57cec5SDimitry Andric /// A dummy default pass factory indicates whether the register allocator is
13210b57cec5SDimitry Andric /// overridden on the command line.
13220b57cec5SDimitry Andric static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;
13230b57cec5SDimitry Andric
13240b57cec5SDimitry Andric static RegisterRegAlloc
13250b57cec5SDimitry Andric defaultRegAlloc("default",
13260b57cec5SDimitry Andric "pick register allocator based on -O option",
13270b57cec5SDimitry Andric useDefaultRegisterAllocator);
13280b57cec5SDimitry Andric
initializeDefaultRegisterAllocatorOnce()13290b57cec5SDimitry Andric static void initializeDefaultRegisterAllocatorOnce() {
13300b57cec5SDimitry Andric if (!RegisterRegAlloc::getDefault())
13310b57cec5SDimitry Andric RegisterRegAlloc::setDefault(RegAlloc);
13320b57cec5SDimitry Andric }
13330b57cec5SDimitry Andric
13340b57cec5SDimitry Andric /// Instantiate the default register allocator pass for this target for either
13350b57cec5SDimitry Andric /// the optimized or unoptimized allocation path. This will be added to the pass
13360b57cec5SDimitry Andric /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
13370b57cec5SDimitry Andric /// in the optimized case.
13380b57cec5SDimitry Andric ///
13390b57cec5SDimitry Andric /// A target that uses the standard regalloc pass order for fast or optimized
13400b57cec5SDimitry Andric /// allocation may still override this for per-target regalloc
13410b57cec5SDimitry Andric /// selection. But -regalloc=... always takes precedence.
createTargetRegisterAllocator(bool Optimized)13420b57cec5SDimitry Andric FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {
13430b57cec5SDimitry Andric if (Optimized)
13440b57cec5SDimitry Andric return createGreedyRegisterAllocator();
13450b57cec5SDimitry Andric else
13460b57cec5SDimitry Andric return createFastRegisterAllocator();
13470b57cec5SDimitry Andric }
13480b57cec5SDimitry Andric
13490b57cec5SDimitry Andric /// Find and instantiate the register allocation pass requested by this target
13500b57cec5SDimitry Andric /// at the current optimization level. Different register allocators are
13510b57cec5SDimitry Andric /// defined as separate passes because they may require different analysis.
13520b57cec5SDimitry Andric ///
13530b57cec5SDimitry Andric /// This helper ensures that the regalloc= option is always available,
13540b57cec5SDimitry Andric /// even for targets that override the default allocator.
13550b57cec5SDimitry Andric ///
13560b57cec5SDimitry Andric /// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,
13570b57cec5SDimitry Andric /// this can be folded into addPass.
createRegAllocPass(bool Optimized)13580b57cec5SDimitry Andric FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {
13590b57cec5SDimitry Andric // Initialize the global default.
13600b57cec5SDimitry Andric llvm::call_once(InitializeDefaultRegisterAllocatorFlag,
13610b57cec5SDimitry Andric initializeDefaultRegisterAllocatorOnce);
13620b57cec5SDimitry Andric
13630b57cec5SDimitry Andric RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
13640b57cec5SDimitry Andric if (Ctor != useDefaultRegisterAllocator)
13650b57cec5SDimitry Andric return Ctor();
13660b57cec5SDimitry Andric
13670b57cec5SDimitry Andric // With no -regalloc= override, ask the target for a regalloc pass.
13680b57cec5SDimitry Andric return createTargetRegisterAllocator(Optimized);
13690b57cec5SDimitry Andric }
13700b57cec5SDimitry Andric
isCustomizedRegAlloc()137181ad6265SDimitry Andric bool TargetPassConfig::isCustomizedRegAlloc() {
137281ad6265SDimitry Andric return RegAlloc !=
137381ad6265SDimitry Andric (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator;
137481ad6265SDimitry Andric }
137581ad6265SDimitry Andric
addRegAssignAndRewriteFast()1376e8d8bef9SDimitry Andric bool TargetPassConfig::addRegAssignAndRewriteFast() {
1377fe6060f1SDimitry Andric if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator &&
1378fe6060f1SDimitry Andric RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator)
13790b57cec5SDimitry Andric report_fatal_error("Must use fast (default) register allocator for unoptimized regalloc.");
13800b57cec5SDimitry Andric
13810b57cec5SDimitry Andric addPass(createRegAllocPass(false));
1382fe6060f1SDimitry Andric
1383fe6060f1SDimitry Andric // Allow targets to change the register assignments after
1384fe6060f1SDimitry Andric // fast register allocation.
1385fe6060f1SDimitry Andric addPostFastRegAllocRewrite();
13860b57cec5SDimitry Andric return true;
13870b57cec5SDimitry Andric }
13880b57cec5SDimitry Andric
addRegAssignAndRewriteOptimized()1389e8d8bef9SDimitry Andric bool TargetPassConfig::addRegAssignAndRewriteOptimized() {
13900b57cec5SDimitry Andric // Add the selected register allocation pass.
13910b57cec5SDimitry Andric addPass(createRegAllocPass(true));
13920b57cec5SDimitry Andric
13930b57cec5SDimitry Andric // Allow targets to change the register assignments before rewriting.
13940b57cec5SDimitry Andric addPreRewrite();
13950b57cec5SDimitry Andric
13960b57cec5SDimitry Andric // Finally rewrite virtual registers.
13970b57cec5SDimitry Andric addPass(&VirtRegRewriterID);
13985ffd83dbSDimitry Andric
139904eeddc0SDimitry Andric // Regalloc scoring for ML-driven eviction - noop except when learning a new
140004eeddc0SDimitry Andric // eviction policy.
140104eeddc0SDimitry Andric addPass(createRegAllocScoringPass());
14020b57cec5SDimitry Andric return true;
14030b57cec5SDimitry Andric }
14040b57cec5SDimitry Andric
14050b57cec5SDimitry Andric /// Return true if the default global register allocator is in use and
14060b57cec5SDimitry Andric /// has not be overriden on the command line with '-regalloc=...'
usingDefaultRegAlloc() const14070b57cec5SDimitry Andric bool TargetPassConfig::usingDefaultRegAlloc() const {
14080b57cec5SDimitry Andric return RegAlloc.getNumOccurrences() == 0;
14090b57cec5SDimitry Andric }
14100b57cec5SDimitry Andric
14110b57cec5SDimitry Andric /// Add the minimum set of target-independent passes that are required for
14120b57cec5SDimitry Andric /// register allocation. No coalescing or scheduling.
addFastRegAlloc()14130b57cec5SDimitry Andric void TargetPassConfig::addFastRegAlloc() {
1414349cc55cSDimitry Andric addPass(&PHIEliminationID);
1415349cc55cSDimitry Andric addPass(&TwoAddressInstructionPassID);
14160b57cec5SDimitry Andric
1417e8d8bef9SDimitry Andric addRegAssignAndRewriteFast();
14180b57cec5SDimitry Andric }
14190b57cec5SDimitry Andric
14200b57cec5SDimitry Andric /// Add standard target-independent passes that are tightly coupled with
14210b57cec5SDimitry Andric /// optimized register allocation, including coalescing, machine instruction
14220b57cec5SDimitry Andric /// scheduling, and register allocation itself.
addOptimizedRegAlloc()14230b57cec5SDimitry Andric void TargetPassConfig::addOptimizedRegAlloc() {
1424349cc55cSDimitry Andric addPass(&DetectDeadLanesID);
14250b57cec5SDimitry Andric
1426349cc55cSDimitry Andric addPass(&ProcessImplicitDefsID);
14270b57cec5SDimitry Andric
14280b57cec5SDimitry Andric // LiveVariables currently requires pure SSA form.
14290b57cec5SDimitry Andric //
14300b57cec5SDimitry Andric // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
14310b57cec5SDimitry Andric // LiveVariables can be removed completely, and LiveIntervals can be directly
14320b57cec5SDimitry Andric // computed. (We still either need to regenerate kill flags after regalloc, or
14330b57cec5SDimitry Andric // preferably fix the scavenger to not depend on them).
1434e8d8bef9SDimitry Andric // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
1435e8d8bef9SDimitry Andric // When LiveVariables is removed this has to be removed/moved either.
1436e8d8bef9SDimitry Andric // Explicit addition of UnreachableMachineBlockElim allows stopping before or
1437e8d8bef9SDimitry Andric // after it with -stop-before/-stop-after.
1438349cc55cSDimitry Andric addPass(&UnreachableMachineBlockElimID);
1439349cc55cSDimitry Andric addPass(&LiveVariablesID);
14400b57cec5SDimitry Andric
14410b57cec5SDimitry Andric // Edge splitting is smarter with machine loop info.
1442349cc55cSDimitry Andric addPass(&MachineLoopInfoID);
1443349cc55cSDimitry Andric addPass(&PHIEliminationID);
14440b57cec5SDimitry Andric
14450b57cec5SDimitry Andric // Eventually, we want to run LiveIntervals before PHI elimination.
14460b57cec5SDimitry Andric if (EarlyLiveIntervals)
1447349cc55cSDimitry Andric addPass(&LiveIntervalsID);
14480b57cec5SDimitry Andric
1449349cc55cSDimitry Andric addPass(&TwoAddressInstructionPassID);
14500b57cec5SDimitry Andric addPass(&RegisterCoalescerID);
14510b57cec5SDimitry Andric
14520b57cec5SDimitry Andric // The machine scheduler may accidentally create disconnected components
14530b57cec5SDimitry Andric // when moving subregister definitions around, avoid this by splitting them to
14540b57cec5SDimitry Andric // separate vregs before. Splitting can also improve reg. allocation quality.
14550b57cec5SDimitry Andric addPass(&RenameIndependentSubregsID);
14560b57cec5SDimitry Andric
14570b57cec5SDimitry Andric // PreRA instruction scheduling.
14580b57cec5SDimitry Andric addPass(&MachineSchedulerID);
14590b57cec5SDimitry Andric
1460e8d8bef9SDimitry Andric if (addRegAssignAndRewriteOptimized()) {
1461e8d8bef9SDimitry Andric // Perform stack slot coloring and post-ra machine LICM.
1462e8d8bef9SDimitry Andric addPass(&StackSlotColoringID);
1463e8d8bef9SDimitry Andric
14640b57cec5SDimitry Andric // Allow targets to expand pseudo instructions depending on the choice of
14650b57cec5SDimitry Andric // registers before MachineCopyPropagation.
14660b57cec5SDimitry Andric addPostRewrite();
14670b57cec5SDimitry Andric
14680b57cec5SDimitry Andric // Copy propagate to forward register uses and try to eliminate COPYs that
14690b57cec5SDimitry Andric // were not coalesced.
14700b57cec5SDimitry Andric addPass(&MachineCopyPropagationID);
14710b57cec5SDimitry Andric
14720b57cec5SDimitry Andric // Run post-ra machine LICM to hoist reloads / remats.
14730b57cec5SDimitry Andric //
14740b57cec5SDimitry Andric // FIXME: can this move into MachineLateOptimization?
14750b57cec5SDimitry Andric addPass(&MachineLICMID);
14760b57cec5SDimitry Andric }
14770b57cec5SDimitry Andric }
14780b57cec5SDimitry Andric
14790b57cec5SDimitry Andric //===---------------------------------------------------------------------===//
14800b57cec5SDimitry Andric /// Post RegAlloc Pass Configuration
14810b57cec5SDimitry Andric //===---------------------------------------------------------------------===//
14820b57cec5SDimitry Andric
14830b57cec5SDimitry Andric /// Add passes that optimize machine instructions after register allocation.
addMachineLateOptimization()14840b57cec5SDimitry Andric void TargetPassConfig::addMachineLateOptimization() {
1485bdd1243dSDimitry Andric // Cleanup of redundant immediate/address loads.
1486bdd1243dSDimitry Andric addPass(&MachineLateInstrsCleanupID);
1487bdd1243dSDimitry Andric
14880b57cec5SDimitry Andric // Branch folding must be run after regalloc and prolog/epilog insertion.
14890b57cec5SDimitry Andric addPass(&BranchFolderPassID);
14900b57cec5SDimitry Andric
14910b57cec5SDimitry Andric // Tail duplication.
14920b57cec5SDimitry Andric // Note that duplicating tail just increases code size and degrades
14930b57cec5SDimitry Andric // performance for targets that require Structured Control Flow.
14940b57cec5SDimitry Andric // In addition it can also make CFG irreducible. Thus we disable it.
14950b57cec5SDimitry Andric if (!TM->requiresStructuredCFG())
14960b57cec5SDimitry Andric addPass(&TailDuplicateID);
14970b57cec5SDimitry Andric
14980b57cec5SDimitry Andric // Copy propagation.
14990b57cec5SDimitry Andric addPass(&MachineCopyPropagationID);
15000b57cec5SDimitry Andric }
15010b57cec5SDimitry Andric
15020b57cec5SDimitry Andric /// Add standard GC passes.
addGCPasses()15030b57cec5SDimitry Andric bool TargetPassConfig::addGCPasses() {
1504349cc55cSDimitry Andric addPass(&GCMachineCodeAnalysisID);
15050b57cec5SDimitry Andric return true;
15060b57cec5SDimitry Andric }
15070b57cec5SDimitry Andric
15080b57cec5SDimitry Andric /// Add standard basic block placement passes.
addBlockPlacement()15090b57cec5SDimitry Andric void TargetPassConfig::addBlockPlacement() {
1510349cc55cSDimitry Andric if (EnableFSDiscriminator) {
1511349cc55cSDimitry Andric addPass(createMIRAddFSDiscriminatorsPass(
1512349cc55cSDimitry Andric sampleprof::FSDiscriminatorPass::Pass2));
1513349cc55cSDimitry Andric const std::string ProfileFile = getFSProfileFile(TM);
1514349cc55cSDimitry Andric if (!ProfileFile.empty() && !DisableLayoutFSProfileLoader)
151506c3fb27SDimitry Andric addPass(createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM),
151606c3fb27SDimitry Andric sampleprof::FSDiscriminatorPass::Pass2,
151706c3fb27SDimitry Andric nullptr));
1518349cc55cSDimitry Andric }
15190b57cec5SDimitry Andric if (addPass(&MachineBlockPlacementID)) {
15200b57cec5SDimitry Andric // Run a separate pass to collect block placement statistics.
15210b57cec5SDimitry Andric if (EnableBlockPlacementStats)
15220b57cec5SDimitry Andric addPass(&MachineBlockPlacementStatsID);
15230b57cec5SDimitry Andric }
15240b57cec5SDimitry Andric }
15250b57cec5SDimitry Andric
15260b57cec5SDimitry Andric //===---------------------------------------------------------------------===//
15270b57cec5SDimitry Andric /// GlobalISel Configuration
15280b57cec5SDimitry Andric //===---------------------------------------------------------------------===//
isGlobalISelAbortEnabled() const15290b57cec5SDimitry Andric bool TargetPassConfig::isGlobalISelAbortEnabled() const {
15300b57cec5SDimitry Andric return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
15310b57cec5SDimitry Andric }
15320b57cec5SDimitry Andric
reportDiagnosticWhenGlobalISelFallback() const15330b57cec5SDimitry Andric bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
15340b57cec5SDimitry Andric return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
15350b57cec5SDimitry Andric }
15360b57cec5SDimitry Andric
isGISelCSEEnabled() const15370b57cec5SDimitry Andric bool TargetPassConfig::isGISelCSEEnabled() const {
15380b57cec5SDimitry Andric return true;
15390b57cec5SDimitry Andric }
15400b57cec5SDimitry Andric
getCSEConfig() const15410b57cec5SDimitry Andric std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {
15428bcb0991SDimitry Andric return std::make_unique<CSEConfigBase>();
15430b57cec5SDimitry Andric }
1544