10b57cec5SDimitry Andric //===-- ARMHazardRecognizer.cpp - ARM postra hazard recognizer ------------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric
90b57cec5SDimitry Andric #include "ARMHazardRecognizer.h"
100b57cec5SDimitry Andric #include "ARMBaseInstrInfo.h"
110b57cec5SDimitry Andric #include "ARMBaseRegisterInfo.h"
120b57cec5SDimitry Andric #include "ARMSubtarget.h"
13e8d8bef9SDimitry Andric #include "llvm/Analysis/ValueTracking.h"
1481ad6265SDimitry Andric #include "llvm/CodeGen/MachineFrameInfo.h"
1581ad6265SDimitry Andric #include "llvm/CodeGen/MachineFunctionPass.h"
160b57cec5SDimitry Andric #include "llvm/CodeGen/MachineInstr.h"
170b57cec5SDimitry Andric #include "llvm/CodeGen/ScheduleDAG.h"
180b57cec5SDimitry Andric #include "llvm/CodeGen/TargetRegisterInfo.h"
19e8d8bef9SDimitry Andric #include "llvm/Support/CommandLine.h"
20e8d8bef9SDimitry Andric
210b57cec5SDimitry Andric using namespace llvm;
220b57cec5SDimitry Andric
23e8d8bef9SDimitry Andric static cl::opt<int> DataBankMask("arm-data-bank-mask", cl::init(-1),
24e8d8bef9SDimitry Andric cl::Hidden);
25e8d8bef9SDimitry Andric static cl::opt<bool> AssumeITCMConflict("arm-assume-itcm-bankconflict",
26e8d8bef9SDimitry Andric cl::init(false), cl::Hidden);
27e8d8bef9SDimitry Andric
hasRAWHazard(MachineInstr * DefMI,MachineInstr * MI,const TargetRegisterInfo & TRI)280b57cec5SDimitry Andric static bool hasRAWHazard(MachineInstr *DefMI, MachineInstr *MI,
290b57cec5SDimitry Andric const TargetRegisterInfo &TRI) {
300b57cec5SDimitry Andric // FIXME: Detect integer instructions properly.
310b57cec5SDimitry Andric const MCInstrDesc &MCID = MI->getDesc();
320b57cec5SDimitry Andric unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
330b57cec5SDimitry Andric if (MI->mayStore())
340b57cec5SDimitry Andric return false;
350b57cec5SDimitry Andric unsigned Opcode = MCID.getOpcode();
360b57cec5SDimitry Andric if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
370b57cec5SDimitry Andric return false;
380b57cec5SDimitry Andric if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
390b57cec5SDimitry Andric return MI->readsRegister(DefMI->getOperand(0).getReg(), &TRI);
400b57cec5SDimitry Andric return false;
410b57cec5SDimitry Andric }
420b57cec5SDimitry Andric
430b57cec5SDimitry Andric ScheduleHazardRecognizer::HazardType
getHazardType(SUnit * SU,int Stalls)44e8d8bef9SDimitry Andric ARMHazardRecognizerFPMLx::getHazardType(SUnit *SU, int Stalls) {
450b57cec5SDimitry Andric assert(Stalls == 0 && "ARM hazards don't support scoreboard lookahead");
460b57cec5SDimitry Andric
470b57cec5SDimitry Andric MachineInstr *MI = SU->getInstr();
480b57cec5SDimitry Andric
490b57cec5SDimitry Andric if (!MI->isDebugInstr()) {
500b57cec5SDimitry Andric // Look for special VMLA / VMLS hazards. A VMUL / VADD / VSUB following
510b57cec5SDimitry Andric // a VMLA / VMLS will cause 4 cycle stall.
520b57cec5SDimitry Andric const MCInstrDesc &MCID = MI->getDesc();
530b57cec5SDimitry Andric if (LastMI && (MCID.TSFlags & ARMII::DomainMask) != ARMII::DomainGeneral) {
540b57cec5SDimitry Andric MachineInstr *DefMI = LastMI;
550b57cec5SDimitry Andric const MCInstrDesc &LastMCID = LastMI->getDesc();
560b57cec5SDimitry Andric const MachineFunction *MF = MI->getParent()->getParent();
570b57cec5SDimitry Andric const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
580b57cec5SDimitry Andric MF->getSubtarget().getInstrInfo());
590b57cec5SDimitry Andric
600b57cec5SDimitry Andric // Skip over one non-VFP / NEON instruction.
610b57cec5SDimitry Andric if (!LastMI->isBarrier() &&
620b57cec5SDimitry Andric !(TII.getSubtarget().hasMuxedUnits() && LastMI->mayLoadOrStore()) &&
630b57cec5SDimitry Andric (LastMCID.TSFlags & ARMII::DomainMask) == ARMII::DomainGeneral) {
640b57cec5SDimitry Andric MachineBasicBlock::iterator I = LastMI;
650b57cec5SDimitry Andric if (I != LastMI->getParent()->begin()) {
660b57cec5SDimitry Andric I = std::prev(I);
670b57cec5SDimitry Andric DefMI = &*I;
680b57cec5SDimitry Andric }
690b57cec5SDimitry Andric }
700b57cec5SDimitry Andric
710b57cec5SDimitry Andric if (TII.isFpMLxInstruction(DefMI->getOpcode()) &&
720b57cec5SDimitry Andric (TII.canCauseFpMLxStall(MI->getOpcode()) ||
730b57cec5SDimitry Andric hasRAWHazard(DefMI, MI, TII.getRegisterInfo()))) {
740b57cec5SDimitry Andric // Try to schedule another instruction for the next 4 cycles.
750b57cec5SDimitry Andric if (FpMLxStalls == 0)
760b57cec5SDimitry Andric FpMLxStalls = 4;
770b57cec5SDimitry Andric return Hazard;
780b57cec5SDimitry Andric }
790b57cec5SDimitry Andric }
800b57cec5SDimitry Andric }
81e8d8bef9SDimitry Andric return NoHazard;
820b57cec5SDimitry Andric }
830b57cec5SDimitry Andric
Reset()84e8d8bef9SDimitry Andric void ARMHazardRecognizerFPMLx::Reset() {
850b57cec5SDimitry Andric LastMI = nullptr;
860b57cec5SDimitry Andric FpMLxStalls = 0;
870b57cec5SDimitry Andric }
880b57cec5SDimitry Andric
EmitInstruction(SUnit * SU)89e8d8bef9SDimitry Andric void ARMHazardRecognizerFPMLx::EmitInstruction(SUnit *SU) {
900b57cec5SDimitry Andric MachineInstr *MI = SU->getInstr();
910b57cec5SDimitry Andric if (!MI->isDebugInstr()) {
920b57cec5SDimitry Andric LastMI = MI;
930b57cec5SDimitry Andric FpMLxStalls = 0;
940b57cec5SDimitry Andric }
950b57cec5SDimitry Andric }
960b57cec5SDimitry Andric
AdvanceCycle()97e8d8bef9SDimitry Andric void ARMHazardRecognizerFPMLx::AdvanceCycle() {
980b57cec5SDimitry Andric if (FpMLxStalls && --FpMLxStalls == 0)
990b57cec5SDimitry Andric // Stalled for 4 cycles but still can't schedule any other instructions.
1000b57cec5SDimitry Andric LastMI = nullptr;
1010b57cec5SDimitry Andric }
1020b57cec5SDimitry Andric
RecedeCycle()103e8d8bef9SDimitry Andric void ARMHazardRecognizerFPMLx::RecedeCycle() {
1040b57cec5SDimitry Andric llvm_unreachable("reverse ARM hazard checking unsupported");
1050b57cec5SDimitry Andric }
106e8d8bef9SDimitry Andric
107e8d8bef9SDimitry Andric ///////// Bank conflicts handled as hazards //////////////
108e8d8bef9SDimitry Andric
getBaseOffset(const MachineInstr & MI,const MachineOperand * & BaseOp,int64_t & Offset)109e8d8bef9SDimitry Andric static bool getBaseOffset(const MachineInstr &MI, const MachineOperand *&BaseOp,
110e8d8bef9SDimitry Andric int64_t &Offset) {
111e8d8bef9SDimitry Andric
112e8d8bef9SDimitry Andric uint64_t TSFlags = MI.getDesc().TSFlags;
113e8d8bef9SDimitry Andric unsigned AddrMode = (TSFlags & ARMII::AddrModeMask);
114e8d8bef9SDimitry Andric unsigned IndexMode =
115e8d8bef9SDimitry Andric (TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift;
116e8d8bef9SDimitry Andric
117e8d8bef9SDimitry Andric // Address mode tells us what we want to know about operands for T2
118e8d8bef9SDimitry Andric // instructions (but not size). It tells us size (but not about operands)
119e8d8bef9SDimitry Andric // for T1 instructions.
120e8d8bef9SDimitry Andric switch (AddrMode) {
121e8d8bef9SDimitry Andric default:
122e8d8bef9SDimitry Andric return false;
123e8d8bef9SDimitry Andric case ARMII::AddrModeT2_i8:
124e8d8bef9SDimitry Andric // t2LDRBT, t2LDRB_POST, t2LDRB_PRE, t2LDRBi8,
125e8d8bef9SDimitry Andric // t2LDRHT, t2LDRH_POST, t2LDRH_PRE, t2LDRHi8,
126e8d8bef9SDimitry Andric // t2LDRSBT, t2LDRSB_POST, t2LDRSB_PRE, t2LDRSBi8,
127e8d8bef9SDimitry Andric // t2LDRSHT, t2LDRSH_POST, t2LDRSH_PRE, t2LDRSHi8,
128e8d8bef9SDimitry Andric // t2LDRT, t2LDR_POST, t2LDR_PRE, t2LDRi8
129e8d8bef9SDimitry Andric BaseOp = &MI.getOperand(1);
130e8d8bef9SDimitry Andric Offset = (IndexMode == ARMII::IndexModePost)
131e8d8bef9SDimitry Andric ? 0
132e8d8bef9SDimitry Andric : (IndexMode == ARMII::IndexModePre ||
133e8d8bef9SDimitry Andric IndexMode == ARMII::IndexModeUpd)
134e8d8bef9SDimitry Andric ? MI.getOperand(3).getImm()
135e8d8bef9SDimitry Andric : MI.getOperand(2).getImm();
136e8d8bef9SDimitry Andric return true;
137e8d8bef9SDimitry Andric case ARMII::AddrModeT2_i12:
138e8d8bef9SDimitry Andric // t2LDRBi12, t2LDRHi12
139e8d8bef9SDimitry Andric // t2LDRSBi12, t2LDRSHi12
140e8d8bef9SDimitry Andric // t2LDRi12
141e8d8bef9SDimitry Andric BaseOp = &MI.getOperand(1);
142e8d8bef9SDimitry Andric Offset = MI.getOperand(2).getImm();
143e8d8bef9SDimitry Andric return true;
144e8d8bef9SDimitry Andric case ARMII::AddrModeT2_i8s4:
145e8d8bef9SDimitry Andric // t2LDRD_POST, t2LDRD_PRE, t2LDRDi8
146e8d8bef9SDimitry Andric BaseOp = &MI.getOperand(2);
147e8d8bef9SDimitry Andric Offset = (IndexMode == ARMII::IndexModePost)
148e8d8bef9SDimitry Andric ? 0
149e8d8bef9SDimitry Andric : (IndexMode == ARMII::IndexModePre ||
150e8d8bef9SDimitry Andric IndexMode == ARMII::IndexModeUpd)
151e8d8bef9SDimitry Andric ? MI.getOperand(4).getImm()
152e8d8bef9SDimitry Andric : MI.getOperand(3).getImm();
153e8d8bef9SDimitry Andric return true;
154e8d8bef9SDimitry Andric case ARMII::AddrModeT1_1:
155e8d8bef9SDimitry Andric // tLDRBi, tLDRBr (watch out!), TLDRSB
156e8d8bef9SDimitry Andric case ARMII::AddrModeT1_2:
157e8d8bef9SDimitry Andric // tLDRHi, tLDRHr (watch out!), TLDRSH
158e8d8bef9SDimitry Andric case ARMII::AddrModeT1_4:
159e8d8bef9SDimitry Andric // tLDRi, tLDRr (watch out!)
160e8d8bef9SDimitry Andric BaseOp = &MI.getOperand(1);
161e8d8bef9SDimitry Andric Offset = MI.getOperand(2).isImm() ? MI.getOperand(2).getImm() : 0;
162e8d8bef9SDimitry Andric return MI.getOperand(2).isImm();
163e8d8bef9SDimitry Andric }
164e8d8bef9SDimitry Andric return false;
165e8d8bef9SDimitry Andric }
166e8d8bef9SDimitry Andric
ARMBankConflictHazardRecognizer(const ScheduleDAG * DAG,int64_t CPUBankMask,bool CPUAssumeITCMConflict)167e8d8bef9SDimitry Andric ARMBankConflictHazardRecognizer::ARMBankConflictHazardRecognizer(
168e8d8bef9SDimitry Andric const ScheduleDAG *DAG, int64_t CPUBankMask, bool CPUAssumeITCMConflict)
16904eeddc0SDimitry Andric : MF(DAG->MF), DL(DAG->MF.getDataLayout()),
170e8d8bef9SDimitry Andric DataMask(DataBankMask.getNumOccurrences() ? int64_t(DataBankMask)
171e8d8bef9SDimitry Andric : CPUBankMask),
172e8d8bef9SDimitry Andric AssumeITCMBankConflict(AssumeITCMConflict.getNumOccurrences()
173e8d8bef9SDimitry Andric ? AssumeITCMConflict
174e8d8bef9SDimitry Andric : CPUAssumeITCMConflict) {
175e8d8bef9SDimitry Andric MaxLookAhead = 1;
176e8d8bef9SDimitry Andric }
177e8d8bef9SDimitry Andric
178e8d8bef9SDimitry Andric ScheduleHazardRecognizer::HazardType
CheckOffsets(unsigned O0,unsigned O1)179e8d8bef9SDimitry Andric ARMBankConflictHazardRecognizer::CheckOffsets(unsigned O0, unsigned O1) {
180e8d8bef9SDimitry Andric return (((O0 ^ O1) & DataMask) != 0) ? NoHazard : Hazard;
181e8d8bef9SDimitry Andric }
182e8d8bef9SDimitry Andric
183e8d8bef9SDimitry Andric ScheduleHazardRecognizer::HazardType
getHazardType(SUnit * SU,int Stalls)184e8d8bef9SDimitry Andric ARMBankConflictHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
185e8d8bef9SDimitry Andric MachineInstr &L0 = *SU->getInstr();
186e8d8bef9SDimitry Andric if (!L0.mayLoad() || L0.mayStore() || L0.getNumMemOperands() != 1)
187e8d8bef9SDimitry Andric return NoHazard;
188e8d8bef9SDimitry Andric
189e8d8bef9SDimitry Andric auto MO0 = *L0.memoperands().begin();
190e8d8bef9SDimitry Andric auto BaseVal0 = MO0->getValue();
191e8d8bef9SDimitry Andric auto BasePseudoVal0 = MO0->getPseudoValue();
192e8d8bef9SDimitry Andric int64_t Offset0 = 0;
193e8d8bef9SDimitry Andric
194e8d8bef9SDimitry Andric if (MO0->getSize() > 4)
195e8d8bef9SDimitry Andric return NoHazard;
196e8d8bef9SDimitry Andric
197e8d8bef9SDimitry Andric bool SPvalid = false;
198e8d8bef9SDimitry Andric const MachineOperand *SP = nullptr;
199e8d8bef9SDimitry Andric int64_t SPOffset0 = 0;
200e8d8bef9SDimitry Andric
201e8d8bef9SDimitry Andric for (auto L1 : Accesses) {
202e8d8bef9SDimitry Andric auto MO1 = *L1->memoperands().begin();
203e8d8bef9SDimitry Andric auto BaseVal1 = MO1->getValue();
204e8d8bef9SDimitry Andric auto BasePseudoVal1 = MO1->getPseudoValue();
205e8d8bef9SDimitry Andric int64_t Offset1 = 0;
206e8d8bef9SDimitry Andric
207e8d8bef9SDimitry Andric // Pointers to the same object
208e8d8bef9SDimitry Andric if (BaseVal0 && BaseVal1) {
209e8d8bef9SDimitry Andric const Value *Ptr0, *Ptr1;
210e8d8bef9SDimitry Andric Ptr0 = GetPointerBaseWithConstantOffset(BaseVal0, Offset0, DL, true);
211e8d8bef9SDimitry Andric Ptr1 = GetPointerBaseWithConstantOffset(BaseVal1, Offset1, DL, true);
212e8d8bef9SDimitry Andric if (Ptr0 == Ptr1 && Ptr0)
213e8d8bef9SDimitry Andric return CheckOffsets(Offset0, Offset1);
214e8d8bef9SDimitry Andric }
215e8d8bef9SDimitry Andric
216e8d8bef9SDimitry Andric if (BasePseudoVal0 && BasePseudoVal1 &&
217e8d8bef9SDimitry Andric BasePseudoVal0->kind() == BasePseudoVal1->kind() &&
218e8d8bef9SDimitry Andric BasePseudoVal0->kind() == PseudoSourceValue::FixedStack) {
219e8d8bef9SDimitry Andric // Spills/fills
220e8d8bef9SDimitry Andric auto FS0 = cast<FixedStackPseudoSourceValue>(BasePseudoVal0);
221e8d8bef9SDimitry Andric auto FS1 = cast<FixedStackPseudoSourceValue>(BasePseudoVal1);
222e8d8bef9SDimitry Andric Offset0 = MF.getFrameInfo().getObjectOffset(FS0->getFrameIndex());
223e8d8bef9SDimitry Andric Offset1 = MF.getFrameInfo().getObjectOffset(FS1->getFrameIndex());
224e8d8bef9SDimitry Andric return CheckOffsets(Offset0, Offset1);
225e8d8bef9SDimitry Andric }
226e8d8bef9SDimitry Andric
227e8d8bef9SDimitry Andric // Constant pools (likely in ITCM)
228e8d8bef9SDimitry Andric if (BasePseudoVal0 && BasePseudoVal1 &&
229e8d8bef9SDimitry Andric BasePseudoVal0->kind() == BasePseudoVal1->kind() &&
230e8d8bef9SDimitry Andric BasePseudoVal0->isConstantPool() && AssumeITCMBankConflict)
231e8d8bef9SDimitry Andric return Hazard;
232e8d8bef9SDimitry Andric
233e8d8bef9SDimitry Andric // Is this a stack pointer-relative access? We could in general try to
234e8d8bef9SDimitry Andric // use "is this the same register and is it unchanged?", but the
235e8d8bef9SDimitry Andric // memory operand tracking is highly likely to have already found that.
236e8d8bef9SDimitry Andric // What we're after here is bank conflicts between different objects in
237e8d8bef9SDimitry Andric // the stack frame.
238e8d8bef9SDimitry Andric if (!SPvalid) { // set up SP
239e8d8bef9SDimitry Andric if (!getBaseOffset(L0, SP, SPOffset0) || SP->getReg().id() != ARM::SP)
240e8d8bef9SDimitry Andric SP = nullptr;
241e8d8bef9SDimitry Andric SPvalid = true;
242e8d8bef9SDimitry Andric }
243e8d8bef9SDimitry Andric if (SP) {
244e8d8bef9SDimitry Andric int64_t SPOffset1;
245e8d8bef9SDimitry Andric const MachineOperand *SP1;
246e8d8bef9SDimitry Andric if (getBaseOffset(*L1, SP1, SPOffset1) && SP1->getReg().id() == ARM::SP)
247e8d8bef9SDimitry Andric return CheckOffsets(SPOffset0, SPOffset1);
248e8d8bef9SDimitry Andric }
249e8d8bef9SDimitry Andric }
250e8d8bef9SDimitry Andric
251e8d8bef9SDimitry Andric return NoHazard;
252e8d8bef9SDimitry Andric }
253e8d8bef9SDimitry Andric
Reset()254e8d8bef9SDimitry Andric void ARMBankConflictHazardRecognizer::Reset() { Accesses.clear(); }
255e8d8bef9SDimitry Andric
EmitInstruction(SUnit * SU)256e8d8bef9SDimitry Andric void ARMBankConflictHazardRecognizer::EmitInstruction(SUnit *SU) {
257e8d8bef9SDimitry Andric MachineInstr &MI = *SU->getInstr();
258e8d8bef9SDimitry Andric if (!MI.mayLoad() || MI.mayStore() || MI.getNumMemOperands() != 1)
259e8d8bef9SDimitry Andric return;
260e8d8bef9SDimitry Andric
261e8d8bef9SDimitry Andric auto MO = *MI.memoperands().begin();
262e8d8bef9SDimitry Andric uint64_t Size1 = MO->getSize();
263e8d8bef9SDimitry Andric if (Size1 > 4)
264e8d8bef9SDimitry Andric return;
265e8d8bef9SDimitry Andric Accesses.push_back(&MI);
266e8d8bef9SDimitry Andric }
267e8d8bef9SDimitry Andric
AdvanceCycle()268e8d8bef9SDimitry Andric void ARMBankConflictHazardRecognizer::AdvanceCycle() { Accesses.clear(); }
269e8d8bef9SDimitry Andric
RecedeCycle()270e8d8bef9SDimitry Andric void ARMBankConflictHazardRecognizer::RecedeCycle() { Accesses.clear(); }
271