1 //===--- AArch64StorePairSuppress.cpp --- Suppress store pair formation ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass identifies floating point stores that should not be combined into
10 // store pairs. Later we may do the same for floating point loads.
11 // ===---------------------------------------------------------------------===//
12
13 #include "AArch64InstrInfo.h"
14 #include "llvm/CodeGen/MachineFunction.h"
15 #include "llvm/CodeGen/MachineFunctionPass.h"
16 #include "llvm/CodeGen/MachineInstr.h"
17 #include "llvm/CodeGen/MachineTraceMetrics.h"
18 #include "llvm/CodeGen/TargetInstrInfo.h"
19 #include "llvm/CodeGen/TargetSchedule.h"
20 #include "llvm/Support/Debug.h"
21 #include "llvm/Support/raw_ostream.h"
22
23 using namespace llvm;
24
25 #define DEBUG_TYPE "aarch64-stp-suppress"
26
27 #define STPSUPPRESS_PASS_NAME "AArch64 Store Pair Suppression"
28
29 namespace {
30 class AArch64StorePairSuppress : public MachineFunctionPass {
31 const AArch64InstrInfo *TII;
32 const TargetRegisterInfo *TRI;
33 const MachineRegisterInfo *MRI;
34 TargetSchedModel SchedModel;
35 MachineTraceMetrics *Traces;
36 MachineTraceMetrics::Ensemble *MinInstr;
37
38 public:
39 static char ID;
AArch64StorePairSuppress()40 AArch64StorePairSuppress() : MachineFunctionPass(ID) {
41 initializeAArch64StorePairSuppressPass(*PassRegistry::getPassRegistry());
42 }
43
getPassName() const44 StringRef getPassName() const override { return STPSUPPRESS_PASS_NAME; }
45
46 bool runOnMachineFunction(MachineFunction &F) override;
47
48 private:
49 bool shouldAddSTPToBlock(const MachineBasicBlock *BB);
50
51 bool isNarrowFPStore(const MachineInstr &MI);
52
getAnalysisUsage(AnalysisUsage & AU) const53 void getAnalysisUsage(AnalysisUsage &AU) const override {
54 AU.setPreservesCFG();
55 AU.addRequired<MachineTraceMetrics>();
56 AU.addPreserved<MachineTraceMetrics>();
57 MachineFunctionPass::getAnalysisUsage(AU);
58 }
59 };
60 char AArch64StorePairSuppress::ID = 0;
61 } // anonymous
62
63 INITIALIZE_PASS(AArch64StorePairSuppress, "aarch64-stp-suppress",
64 STPSUPPRESS_PASS_NAME, false, false)
65
createAArch64StorePairSuppressPass()66 FunctionPass *llvm::createAArch64StorePairSuppressPass() {
67 return new AArch64StorePairSuppress();
68 }
69
70 /// Return true if an STP can be added to this block without increasing the
71 /// critical resource height. STP is good to form in Ld/St limited blocks and
72 /// bad to form in float-point limited blocks. This is true independent of the
73 /// critical path. If the critical path is longer than the resource height, the
74 /// extra vector ops can limit physreg renaming. Otherwise, it could simply
75 /// oversaturate the vector units.
shouldAddSTPToBlock(const MachineBasicBlock * BB)76 bool AArch64StorePairSuppress::shouldAddSTPToBlock(const MachineBasicBlock *BB) {
77 if (!MinInstr)
78 MinInstr = Traces->getEnsemble(MachineTraceMetrics::TS_MinInstrCount);
79
80 MachineTraceMetrics::Trace BBTrace = MinInstr->getTrace(BB);
81 unsigned ResLength = BBTrace.getResourceLength();
82
83 // Get the machine model's scheduling class for STPQi.
84 // Bypass TargetSchedule's SchedClass resolution since we only have an opcode.
85 unsigned SCIdx = TII->get(AArch64::STPDi).getSchedClass();
86 const MCSchedClassDesc *SCDesc =
87 SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx);
88
89 // If a subtarget does not define resources for STPQi, bail here.
90 if (SCDesc->isValid() && !SCDesc->isVariant()) {
91 unsigned ResLenWithSTP = BBTrace.getResourceLength(None, SCDesc);
92 if (ResLenWithSTP > ResLength) {
93 LLVM_DEBUG(dbgs() << " Suppress STP in BB: " << BB->getNumber()
94 << " resources " << ResLength << " -> " << ResLenWithSTP
95 << "\n");
96 return false;
97 }
98 }
99 return true;
100 }
101
102 /// Return true if this is a floating-point store smaller than the V reg. On
103 /// cyclone, these require a vector shuffle before storing a pair.
104 /// Ideally we would call getMatchingPairOpcode() and have the machine model
105 /// tell us if it's profitable with no cpu knowledge here.
106 ///
107 /// FIXME: We plan to develop a decent Target abstraction for simple loads and
108 /// stores. Until then use a nasty switch similar to AArch64LoadStoreOptimizer.
isNarrowFPStore(const MachineInstr & MI)109 bool AArch64StorePairSuppress::isNarrowFPStore(const MachineInstr &MI) {
110 switch (MI.getOpcode()) {
111 default:
112 return false;
113 case AArch64::STRSui:
114 case AArch64::STRDui:
115 case AArch64::STURSi:
116 case AArch64::STURDi:
117 return true;
118 }
119 }
120
runOnMachineFunction(MachineFunction & MF)121 bool AArch64StorePairSuppress::runOnMachineFunction(MachineFunction &MF) {
122 if (skipFunction(MF.getFunction()) || MF.getFunction().hasOptSize())
123 return false;
124
125 const TargetSubtargetInfo &ST = MF.getSubtarget();
126 TII = static_cast<const AArch64InstrInfo *>(ST.getInstrInfo());
127 TRI = ST.getRegisterInfo();
128 MRI = &MF.getRegInfo();
129 SchedModel.init(&ST);
130 Traces = &getAnalysis<MachineTraceMetrics>();
131 MinInstr = nullptr;
132
133 LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << MF.getName() << '\n');
134
135 if (!SchedModel.hasInstrSchedModel()) {
136 LLVM_DEBUG(dbgs() << " Skipping pass: no machine model present.\n");
137 return false;
138 }
139
140 // Check for a sequence of stores to the same base address. We don't need to
141 // precisely determine whether a store pair can be formed. But we do want to
142 // filter out most situations where we can't form store pairs to avoid
143 // computing trace metrics in those cases.
144 for (auto &MBB : MF) {
145 bool SuppressSTP = false;
146 unsigned PrevBaseReg = 0;
147 for (auto &MI : MBB) {
148 if (!isNarrowFPStore(MI))
149 continue;
150 const MachineOperand *BaseOp;
151 int64_t Offset;
152 bool OffsetIsScalable;
153 if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable,
154 TRI) &&
155 BaseOp->isReg()) {
156 Register BaseReg = BaseOp->getReg();
157 if (PrevBaseReg == BaseReg) {
158 // If this block can take STPs, skip ahead to the next block.
159 if (!SuppressSTP && shouldAddSTPToBlock(MI.getParent()))
160 break;
161 // Otherwise, continue unpairing the stores in this block.
162 LLVM_DEBUG(dbgs() << "Unpairing store " << MI << "\n");
163 SuppressSTP = true;
164 TII->suppressLdStPair(MI);
165 }
166 PrevBaseReg = BaseReg;
167 } else
168 PrevBaseReg = 0;
169 }
170 }
171 // This pass just sets some internal MachineMemOperand flags. It can't really
172 // invalidate anything.
173 return false;
174 }
175