1 //===-- ARMHazardRecognizer.cpp - ARM postra hazard recognizer ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "ARMHazardRecognizer.h"
10 #include "ARMBaseInstrInfo.h"
11 #include "ARMBaseRegisterInfo.h"
12 #include "ARMSubtarget.h"
13 #include "llvm/Analysis/ValueTracking.h"
14 #include "llvm/CodeGen/MachineFrameInfo.h"
15 #include "llvm/CodeGen/MachineFunctionPass.h"
16 #include "llvm/CodeGen/MachineInstr.h"
17 #include "llvm/CodeGen/ScheduleDAG.h"
18 #include "llvm/CodeGen/TargetRegisterInfo.h"
19 #include "llvm/Support/CommandLine.h"
20 
21 using namespace llvm;
22 
23 static cl::opt<int> DataBankMask("arm-data-bank-mask", cl::init(-1),
24                                  cl::Hidden);
25 static cl::opt<bool> AssumeITCMConflict("arm-assume-itcm-bankconflict",
26                                         cl::init(false), cl::Hidden);
27 
28 static bool hasRAWHazard(MachineInstr *DefMI, MachineInstr *MI,
29                          const TargetRegisterInfo &TRI) {
30   // FIXME: Detect integer instructions properly.
31   const MCInstrDesc &MCID = MI->getDesc();
32   unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
33   if (MI->mayStore())
34     return false;
35   unsigned Opcode = MCID.getOpcode();
36   if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
37     return false;
38   if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
39     return MI->readsRegister(DefMI->getOperand(0).getReg(), &TRI);
40   return false;
41 }
42 
43 ScheduleHazardRecognizer::HazardType
44 ARMHazardRecognizerFPMLx::getHazardType(SUnit *SU, int Stalls) {
45   assert(Stalls == 0 && "ARM hazards don't support scoreboard lookahead");
46 
47   MachineInstr *MI = SU->getInstr();
48 
49   if (!MI->isDebugInstr()) {
50     // Look for special VMLA / VMLS hazards. A VMUL / VADD / VSUB following
51     // a VMLA / VMLS will cause 4 cycle stall.
52     const MCInstrDesc &MCID = MI->getDesc();
53     if (LastMI && (MCID.TSFlags & ARMII::DomainMask) != ARMII::DomainGeneral) {
54       MachineInstr *DefMI = LastMI;
55       const MCInstrDesc &LastMCID = LastMI->getDesc();
56       const MachineFunction *MF = MI->getParent()->getParent();
57       const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
58                                         MF->getSubtarget().getInstrInfo());
59 
60       // Skip over one non-VFP / NEON instruction.
61       if (!LastMI->isBarrier() &&
62           !(TII.getSubtarget().hasMuxedUnits() && LastMI->mayLoadOrStore()) &&
63           (LastMCID.TSFlags & ARMII::DomainMask) == ARMII::DomainGeneral) {
64         MachineBasicBlock::iterator I = LastMI;
65         if (I != LastMI->getParent()->begin()) {
66           I = std::prev(I);
67           DefMI = &*I;
68         }
69       }
70 
71       if (TII.isFpMLxInstruction(DefMI->getOpcode()) &&
72           (TII.canCauseFpMLxStall(MI->getOpcode()) ||
73            hasRAWHazard(DefMI, MI, TII.getRegisterInfo()))) {
74         // Try to schedule another instruction for the next 4 cycles.
75         if (FpMLxStalls == 0)
76           FpMLxStalls = 4;
77         return Hazard;
78       }
79     }
80   }
81   return NoHazard;
82 }
83 
84 void ARMHazardRecognizerFPMLx::Reset() {
85   LastMI = nullptr;
86   FpMLxStalls = 0;
87 }
88 
89 void ARMHazardRecognizerFPMLx::EmitInstruction(SUnit *SU) {
90   MachineInstr *MI = SU->getInstr();
91   if (!MI->isDebugInstr()) {
92     LastMI = MI;
93     FpMLxStalls = 0;
94   }
95 }
96 
97 void ARMHazardRecognizerFPMLx::AdvanceCycle() {
98   if (FpMLxStalls && --FpMLxStalls == 0)
99     // Stalled for 4 cycles but still can't schedule any other instructions.
100     LastMI = nullptr;
101 }
102 
103 void ARMHazardRecognizerFPMLx::RecedeCycle() {
104   llvm_unreachable("reverse ARM hazard checking unsupported");
105 }
106 
107 ///////// Bank conflicts handled as hazards //////////////
108 
109 static bool getBaseOffset(const MachineInstr &MI, const MachineOperand *&BaseOp,
110                           int64_t &Offset) {
111 
112   uint64_t TSFlags = MI.getDesc().TSFlags;
113   unsigned AddrMode = (TSFlags & ARMII::AddrModeMask);
114   unsigned IndexMode =
115       (TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift;
116 
117   // Address mode tells us what we want to know about operands for T2
118   // instructions (but not size).  It tells us size (but not about operands)
119   // for T1 instructions.
120   switch (AddrMode) {
121   default:
122     return false;
123   case ARMII::AddrModeT2_i8:
124     // t2LDRBT, t2LDRB_POST, t2LDRB_PRE, t2LDRBi8,
125     // t2LDRHT, t2LDRH_POST, t2LDRH_PRE, t2LDRHi8,
126     // t2LDRSBT, t2LDRSB_POST, t2LDRSB_PRE, t2LDRSBi8,
127     // t2LDRSHT, t2LDRSH_POST, t2LDRSH_PRE, t2LDRSHi8,
128     // t2LDRT, t2LDR_POST, t2LDR_PRE, t2LDRi8
129     BaseOp = &MI.getOperand(1);
130     Offset = (IndexMode == ARMII::IndexModePost)
131                  ? 0
132                  : (IndexMode == ARMII::IndexModePre ||
133                     IndexMode == ARMII::IndexModeUpd)
134                        ? MI.getOperand(3).getImm()
135                        : MI.getOperand(2).getImm();
136     return true;
137   case ARMII::AddrModeT2_i12:
138     // t2LDRBi12, t2LDRHi12
139     // t2LDRSBi12, t2LDRSHi12
140     // t2LDRi12
141     BaseOp = &MI.getOperand(1);
142     Offset = MI.getOperand(2).getImm();
143     return true;
144   case ARMII::AddrModeT2_i8s4:
145     // t2LDRD_POST, t2LDRD_PRE, t2LDRDi8
146     BaseOp = &MI.getOperand(2);
147     Offset = (IndexMode == ARMII::IndexModePost)
148                  ? 0
149                  : (IndexMode == ARMII::IndexModePre ||
150                     IndexMode == ARMII::IndexModeUpd)
151                        ? MI.getOperand(4).getImm()
152                        : MI.getOperand(3).getImm();
153     return true;
154   case ARMII::AddrModeT1_1:
155     // tLDRBi, tLDRBr (watch out!), TLDRSB
156   case ARMII::AddrModeT1_2:
157     // tLDRHi, tLDRHr (watch out!), TLDRSH
158   case ARMII::AddrModeT1_4:
159     // tLDRi, tLDRr (watch out!)
160     BaseOp = &MI.getOperand(1);
161     Offset = MI.getOperand(2).isImm() ? MI.getOperand(2).getImm() : 0;
162     return MI.getOperand(2).isImm();
163   }
164   return false;
165 }
166 
167 ARMBankConflictHazardRecognizer::ARMBankConflictHazardRecognizer(
168     const ScheduleDAG *DAG, int64_t CPUBankMask, bool CPUAssumeITCMConflict)
169     : MF(DAG->MF), DL(DAG->MF.getDataLayout()),
170       DataMask(DataBankMask.getNumOccurrences() ? int64_t(DataBankMask)
171                                                 : CPUBankMask),
172       AssumeITCMBankConflict(AssumeITCMConflict.getNumOccurrences()
173                                  ? AssumeITCMConflict
174                                  : CPUAssumeITCMConflict) {
175   MaxLookAhead = 1;
176 }
177 
178 ScheduleHazardRecognizer::HazardType
179 ARMBankConflictHazardRecognizer::CheckOffsets(unsigned O0, unsigned O1) {
180   return (((O0 ^ O1) & DataMask) != 0) ? NoHazard : Hazard;
181 }
182 
183 ScheduleHazardRecognizer::HazardType
184 ARMBankConflictHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
185   MachineInstr &L0 = *SU->getInstr();
186   if (!L0.mayLoad() || L0.mayStore() || L0.getNumMemOperands() != 1)
187     return NoHazard;
188 
189   auto MO0 = *L0.memoperands().begin();
190   auto BaseVal0 = MO0->getValue();
191   auto BasePseudoVal0 = MO0->getPseudoValue();
192   int64_t Offset0 = 0;
193 
194   if (MO0->getSize() > 4)
195     return NoHazard;
196 
197   bool SPvalid = false;
198   const MachineOperand *SP = nullptr;
199   int64_t SPOffset0 = 0;
200 
201   for (auto L1 : Accesses) {
202     auto MO1 = *L1->memoperands().begin();
203     auto BaseVal1 = MO1->getValue();
204     auto BasePseudoVal1 = MO1->getPseudoValue();
205     int64_t Offset1 = 0;
206 
207     // Pointers to the same object
208     if (BaseVal0 && BaseVal1) {
209       const Value *Ptr0, *Ptr1;
210       Ptr0 = GetPointerBaseWithConstantOffset(BaseVal0, Offset0, DL, true);
211       Ptr1 = GetPointerBaseWithConstantOffset(BaseVal1, Offset1, DL, true);
212       if (Ptr0 == Ptr1 && Ptr0)
213         return CheckOffsets(Offset0, Offset1);
214     }
215 
216     if (BasePseudoVal0 && BasePseudoVal1 &&
217         BasePseudoVal0->kind() == BasePseudoVal1->kind() &&
218         BasePseudoVal0->kind() == PseudoSourceValue::FixedStack) {
219       // Spills/fills
220       auto FS0 = cast<FixedStackPseudoSourceValue>(BasePseudoVal0);
221       auto FS1 = cast<FixedStackPseudoSourceValue>(BasePseudoVal1);
222       Offset0 = MF.getFrameInfo().getObjectOffset(FS0->getFrameIndex());
223       Offset1 = MF.getFrameInfo().getObjectOffset(FS1->getFrameIndex());
224       return CheckOffsets(Offset0, Offset1);
225     }
226 
227     // Constant pools (likely in ITCM)
228     if (BasePseudoVal0 && BasePseudoVal1 &&
229         BasePseudoVal0->kind() == BasePseudoVal1->kind() &&
230         BasePseudoVal0->isConstantPool() && AssumeITCMBankConflict)
231       return Hazard;
232 
233     // Is this a stack pointer-relative access?  We could in general try to
234     // use "is this the same register and is it unchanged?", but the
235     // memory operand tracking is highly likely to have already found that.
236     // What we're after here is bank conflicts between different objects in
237     // the stack frame.
238     if (!SPvalid) { // set up SP
239       if (!getBaseOffset(L0, SP, SPOffset0) || SP->getReg().id() != ARM::SP)
240         SP = nullptr;
241       SPvalid = true;
242     }
243     if (SP) {
244       int64_t SPOffset1;
245       const MachineOperand *SP1;
246       if (getBaseOffset(*L1, SP1, SPOffset1) && SP1->getReg().id() == ARM::SP)
247         return CheckOffsets(SPOffset0, SPOffset1);
248     }
249   }
250 
251   return NoHazard;
252 }
253 
254 void ARMBankConflictHazardRecognizer::Reset() { Accesses.clear(); }
255 
256 void ARMBankConflictHazardRecognizer::EmitInstruction(SUnit *SU) {
257   MachineInstr &MI = *SU->getInstr();
258   if (!MI.mayLoad() || MI.mayStore() || MI.getNumMemOperands() != 1)
259     return;
260 
261   auto MO = *MI.memoperands().begin();
262   uint64_t Size1 = MO->getSize();
263   if (Size1 > 4)
264     return;
265   Accesses.push_back(&MI);
266 }
267 
268 void ARMBankConflictHazardRecognizer::AdvanceCycle() { Accesses.clear(); }
269 
270 void ARMBankConflictHazardRecognizer::RecedeCycle() { Accesses.clear(); }
271