1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "GCNSubtarget.h"
15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
16 #include "llvm/CodeGen/MachineFunction.h"
17 #include "llvm/CodeGen/ScheduleDAG.h"
18 #include "llvm/Support/TargetParser.h"
19 
20 using namespace llvm;
21 
22 //===----------------------------------------------------------------------===//
23 // Hazard Recoginizer Implementation
24 //===----------------------------------------------------------------------===//
25 
26 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
27                                                  const GCNSubtarget &ST);
28 
29 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) :
30   IsHazardRecognizerMode(false),
31   CurrCycleInstr(nullptr),
32   MF(MF),
33   ST(MF.getSubtarget<GCNSubtarget>()),
34   TII(*ST.getInstrInfo()),
35   TRI(TII.getRegisterInfo()),
36   ClauseUses(TRI.getNumRegUnits()),
37   ClauseDefs(TRI.getNumRegUnits()) {
38   MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
39   TSchedModel.init(&ST);
40   RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
41 }
42 
43 void GCNHazardRecognizer::Reset() {
44   EmittedInstrs.clear();
45 }
46 
47 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
48   EmitInstruction(SU->getInstr());
49 }
50 
51 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
52   CurrCycleInstr = MI;
53 }
54 
55 static bool isDivFMas(unsigned Opcode) {
56   return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
57 }
58 
59 static bool isSGetReg(unsigned Opcode) {
60   return Opcode == AMDGPU::S_GETREG_B32;
61 }
62 
63 static bool isSSetReg(unsigned Opcode) {
64   switch (Opcode) {
65   case AMDGPU::S_SETREG_B32:
66   case AMDGPU::S_SETREG_B32_mode:
67   case AMDGPU::S_SETREG_IMM32_B32:
68   case AMDGPU::S_SETREG_IMM32_B32_mode:
69     return true;
70   }
71   return false;
72 }
73 
74 static bool isRWLane(unsigned Opcode) {
75   return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
76 }
77 
78 static bool isRFE(unsigned Opcode) {
79   return Opcode == AMDGPU::S_RFE_B64;
80 }
81 
82 static bool isSMovRel(unsigned Opcode) {
83   switch (Opcode) {
84   case AMDGPU::S_MOVRELS_B32:
85   case AMDGPU::S_MOVRELS_B64:
86   case AMDGPU::S_MOVRELD_B32:
87   case AMDGPU::S_MOVRELD_B64:
88     return true;
89   default:
90     return false;
91   }
92 }
93 
94 static bool isDGEMM(unsigned Opcode) {
95   return Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
96          Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64 ||
97          Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_e64 ||
98          Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64 ||
99          Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64 ||
100          Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64;
101 }
102 
103 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
104   unsigned Opcode = MI.getOpcode();
105 
106   if (!SIInstrInfo::isMAI(MI) ||
107       isDGEMM(Opcode) ||
108       Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
109       Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
110     return false;
111 
112   return true;
113 }
114 
115 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
116                                     const MachineInstr &MI) {
117   if (TII.isAlwaysGDS(MI.getOpcode()))
118     return true;
119 
120   switch (MI.getOpcode()) {
121   case AMDGPU::S_SENDMSG:
122   case AMDGPU::S_SENDMSGHALT:
123   case AMDGPU::S_TTRACEDATA:
124     return true;
125   // These DS opcodes don't support GDS.
126   case AMDGPU::DS_NOP:
127   case AMDGPU::DS_PERMUTE_B32:
128   case AMDGPU::DS_BPERMUTE_B32:
129     return false;
130   default:
131     if (TII.isDS(MI.getOpcode())) {
132       int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
133                                            AMDGPU::OpName::gds);
134       if (MI.getOperand(GDS).getImm())
135         return true;
136     }
137     return false;
138   }
139 }
140 
141 static bool isPermlane(const MachineInstr &MI) {
142   unsigned Opcode = MI.getOpcode();
143   return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
144          Opcode == AMDGPU::V_PERMLANEX16_B32_e64;
145 }
146 
147 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
148   const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
149                                                      AMDGPU::OpName::simm16);
150   return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
151 }
152 
153 ScheduleHazardRecognizer::HazardType
154 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
155   MachineInstr *MI = SU->getInstr();
156   // If we are not in "HazardRecognizerMode" and therefore not being run from
157   // the scheduler, track possible stalls from hazards but don't insert noops.
158   auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard;
159 
160   if (MI->isBundle())
161    return NoHazard;
162 
163   if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
164     return HazardType;
165 
166   if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
167     return HazardType;
168 
169   if (checkFPAtomicToDenormModeHazard(MI) > 0)
170     return HazardType;
171 
172   if (ST.hasNoDataDepHazard())
173     return NoHazard;
174 
175   // FIXME: Should flat be considered vmem?
176   if ((SIInstrInfo::isVMEM(*MI) ||
177        SIInstrInfo::isFLAT(*MI))
178       && checkVMEMHazards(MI) > 0)
179     return HazardType;
180 
181   if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
182     return HazardType;
183 
184   if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
185     return HazardType;
186 
187   if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
188     return HazardType;
189 
190   if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
191     return HazardType;
192 
193   if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
194        SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
195        SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
196     return HazardType;
197 
198   if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
199     return HazardType;
200 
201   if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
202     return HazardType;
203 
204   if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
205     return HazardType;
206 
207   if (ST.hasReadM0MovRelInterpHazard() &&
208       (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) &&
209       checkReadM0Hazards(MI) > 0)
210     return HazardType;
211 
212   if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI) &&
213       checkReadM0Hazards(MI) > 0)
214     return HazardType;
215 
216   if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
217     return HazardType;
218 
219   if ((SIInstrInfo::isVMEM(*MI) ||
220        SIInstrInfo::isFLAT(*MI) ||
221        SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0)
222     return HazardType;
223 
224   if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
225     return HazardType;
226 
227   return NoHazard;
228 }
229 
230 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
231                                 unsigned Quantity) {
232   while (Quantity > 0) {
233     unsigned Arg = std::min(Quantity, 8u);
234     Quantity -= Arg;
235     BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
236         .addImm(Arg - 1);
237   }
238 }
239 
240 void GCNHazardRecognizer::processBundle() {
241   MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
242   MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
243   // Check bundled MachineInstr's for hazards.
244   for (; MI != E && MI->isInsideBundle(); ++MI) {
245     CurrCycleInstr = &*MI;
246     unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
247 
248     if (IsHazardRecognizerMode) {
249       fixHazards(CurrCycleInstr);
250 
251       insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
252     }
253 
254     // It’s unnecessary to track more than MaxLookAhead instructions. Since we
255     // include the bundled MI directly after, only add a maximum of
256     // (MaxLookAhead - 1) noops to EmittedInstrs.
257     for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
258       EmittedInstrs.push_front(nullptr);
259 
260     EmittedInstrs.push_front(CurrCycleInstr);
261     EmittedInstrs.resize(MaxLookAhead);
262   }
263   CurrCycleInstr = nullptr;
264 }
265 
266 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
267   IsHazardRecognizerMode = true;
268   CurrCycleInstr = MI;
269   unsigned W = PreEmitNoopsCommon(MI);
270   fixHazards(MI);
271   CurrCycleInstr = nullptr;
272   return W;
273 }
274 
275 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
276   if (MI->isBundle())
277     return 0;
278 
279   int WaitStates = 0;
280 
281   if (SIInstrInfo::isSMRD(*MI))
282     return std::max(WaitStates, checkSMRDHazards(MI));
283 
284   if (ST.hasNSAtoVMEMBug())
285     WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
286 
287   WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI));
288 
289   if (ST.hasNoDataDepHazard())
290     return WaitStates;
291 
292   if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
293     WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
294 
295   if (SIInstrInfo::isVALU(*MI))
296     WaitStates = std::max(WaitStates, checkVALUHazards(MI));
297 
298   if (SIInstrInfo::isDPP(*MI))
299     WaitStates = std::max(WaitStates, checkDPPHazards(MI));
300 
301   if (isDivFMas(MI->getOpcode()))
302     WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
303 
304   if (isRWLane(MI->getOpcode()))
305     WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
306 
307   if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
308        SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
309        SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
310     WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
311 
312   if (MI->isInlineAsm())
313     return std::max(WaitStates, checkInlineAsmHazards(MI));
314 
315   if (isSGetReg(MI->getOpcode()))
316     return std::max(WaitStates, checkGetRegHazards(MI));
317 
318   if (isSSetReg(MI->getOpcode()))
319     return std::max(WaitStates, checkSetRegHazards(MI));
320 
321   if (isRFE(MI->getOpcode()))
322     return std::max(WaitStates, checkRFEHazards(MI));
323 
324   if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) ||
325                                            isSMovRel(MI->getOpcode())))
326     return std::max(WaitStates, checkReadM0Hazards(MI));
327 
328   if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI))
329     return std::max(WaitStates, checkReadM0Hazards(MI));
330 
331   if (SIInstrInfo::isMAI(*MI))
332     return std::max(WaitStates, checkMAIHazards(MI));
333 
334   if (SIInstrInfo::isVMEM(*MI) ||
335       SIInstrInfo::isFLAT(*MI) ||
336       SIInstrInfo::isDS(*MI))
337     return std::max(WaitStates, checkMAILdStHazards(MI));
338 
339   return WaitStates;
340 }
341 
342 void GCNHazardRecognizer::EmitNoop() {
343   EmittedInstrs.push_front(nullptr);
344 }
345 
346 void GCNHazardRecognizer::AdvanceCycle() {
347   // When the scheduler detects a stall, it will call AdvanceCycle() without
348   // emitting any instructions.
349   if (!CurrCycleInstr) {
350     EmittedInstrs.push_front(nullptr);
351     return;
352   }
353 
354   if (CurrCycleInstr->isBundle()) {
355     processBundle();
356     return;
357   }
358 
359   unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
360   if (!NumWaitStates) {
361     CurrCycleInstr = nullptr;
362     return;
363   }
364 
365   // Keep track of emitted instructions
366   EmittedInstrs.push_front(CurrCycleInstr);
367 
368   // Add a nullptr for each additional wait state after the first.  Make sure
369   // not to add more than getMaxLookAhead() items to the list, since we
370   // truncate the list to that size right after this loop.
371   for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
372        i < e; ++i) {
373     EmittedInstrs.push_front(nullptr);
374   }
375 
376   // getMaxLookahead() is the largest number of wait states we will ever need
377   // to insert, so there is no point in keeping track of more than that many
378   // wait states.
379   EmittedInstrs.resize(getMaxLookAhead());
380 
381   CurrCycleInstr = nullptr;
382 }
383 
384 void GCNHazardRecognizer::RecedeCycle() {
385   llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
386 }
387 
388 //===----------------------------------------------------------------------===//
389 // Helper Functions
390 //===----------------------------------------------------------------------===//
391 
392 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn;
393 
394 // Returns a minimum wait states since \p I walking all predecessors.
395 // Only scans until \p IsExpired does not return true.
396 // Can only be run in a hazard recognizer mode.
397 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
398                               const MachineBasicBlock *MBB,
399                               MachineBasicBlock::const_reverse_instr_iterator I,
400                               int WaitStates, IsExpiredFn IsExpired,
401                               DenseSet<const MachineBasicBlock *> &Visited) {
402   for (auto E = MBB->instr_rend(); I != E; ++I) {
403     // Don't add WaitStates for parent BUNDLE instructions.
404     if (I->isBundle())
405       continue;
406 
407     if (IsHazard(*I))
408       return WaitStates;
409 
410     if (I->isInlineAsm())
411       continue;
412 
413     WaitStates += SIInstrInfo::getNumWaitStates(*I);
414 
415     if (IsExpired(*I, WaitStates))
416       return std::numeric_limits<int>::max();
417   }
418 
419   int MinWaitStates = std::numeric_limits<int>::max();
420   for (MachineBasicBlock *Pred : MBB->predecessors()) {
421     if (!Visited.insert(Pred).second)
422       continue;
423 
424     int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(),
425                                WaitStates, IsExpired, Visited);
426 
427     MinWaitStates = std::min(MinWaitStates, W);
428   }
429 
430   return MinWaitStates;
431 }
432 
433 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
434                               const MachineInstr *MI, IsExpiredFn IsExpired) {
435   DenseSet<const MachineBasicBlock *> Visited;
436   return getWaitStatesSince(IsHazard, MI->getParent(),
437                             std::next(MI->getReverseIterator()),
438                             0, IsExpired, Visited);
439 }
440 
441 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
442   if (IsHazardRecognizerMode) {
443     auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
444       return WaitStates >= Limit;
445     };
446     return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
447   }
448 
449   int WaitStates = 0;
450   for (MachineInstr *MI : EmittedInstrs) {
451     if (MI) {
452       if (IsHazard(*MI))
453         return WaitStates;
454 
455       if (MI->isInlineAsm())
456         continue;
457     }
458     ++WaitStates;
459 
460     if (WaitStates >= Limit)
461       break;
462   }
463   return std::numeric_limits<int>::max();
464 }
465 
466 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
467                                                IsHazardFn IsHazardDef,
468                                                int Limit) {
469   const SIRegisterInfo *TRI = ST.getRegisterInfo();
470 
471   auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
472     return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
473   };
474 
475   return getWaitStatesSince(IsHazardFn, Limit);
476 }
477 
478 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
479                                                   int Limit) {
480   auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
481     return isSSetReg(MI.getOpcode()) && IsHazard(MI);
482   };
483 
484   return getWaitStatesSince(IsHazardFn, Limit);
485 }
486 
487 //===----------------------------------------------------------------------===//
488 // No-op Hazard Detection
489 //===----------------------------------------------------------------------===//
490 
491 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
492                         MCRegister Reg) {
493   for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
494     BV.set(*RUI);
495 }
496 
497 static void addRegsToSet(const SIRegisterInfo &TRI,
498                          iterator_range<MachineInstr::const_mop_iterator> Ops,
499                          BitVector &Set) {
500   for (const MachineOperand &Op : Ops) {
501     if (Op.isReg())
502       addRegUnits(TRI, Set, Op.getReg().asMCReg());
503   }
504 }
505 
506 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
507   // XXX: Do we need to worry about implicit operands
508   addRegsToSet(TRI, MI.defs(), ClauseDefs);
509   addRegsToSet(TRI, MI.uses(), ClauseUses);
510 }
511 
512 static bool breaksSMEMSoftClause(MachineInstr *MI) {
513   return !SIInstrInfo::isSMRD(*MI);
514 }
515 
516 static bool breaksVMEMSoftClause(MachineInstr *MI) {
517   return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI);
518 }
519 
520 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
521   // SMEM soft clause are only present on VI+, and only matter if xnack is
522   // enabled.
523   if (!ST.isXNACKEnabled())
524     return 0;
525 
526   bool IsSMRD = TII.isSMRD(*MEM);
527 
528   resetClause();
529 
530   // A soft-clause is any group of consecutive SMEM instructions.  The
531   // instructions in this group may return out of order and/or may be
532   // replayed (i.e. the same instruction issued more than once).
533   //
534   // In order to handle these situations correctly we need to make sure that
535   // when a clause has more than one instruction, no instruction in the clause
536   // writes to a register that is read by another instruction in the clause
537   // (including itself). If we encounter this situaion, we need to break the
538   // clause by inserting a non SMEM instruction.
539 
540   for (MachineInstr *MI : EmittedInstrs) {
541     // When we hit a non-SMEM instruction then we have passed the start of the
542     // clause and we can stop.
543     if (!MI)
544       break;
545 
546     if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
547       break;
548 
549     addClauseInst(*MI);
550   }
551 
552   if (ClauseDefs.none())
553     return 0;
554 
555   // We need to make sure not to put loads and stores in the same clause if they
556   // use the same address. For now, just start a new clause whenever we see a
557   // store.
558   if (MEM->mayStore())
559     return 1;
560 
561   addClauseInst(*MEM);
562 
563   // If the set of defs and uses intersect then we cannot add this instruction
564   // to the clause, so we have a hazard.
565   return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
566 }
567 
568 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
569   int WaitStatesNeeded = 0;
570 
571   WaitStatesNeeded = checkSoftClauseHazards(SMRD);
572 
573   // This SMRD hazard only affects SI.
574   if (!ST.hasSMRDReadVALUDefHazard())
575     return WaitStatesNeeded;
576 
577   // A read of an SGPR by SMRD instruction requires 4 wait states when the
578   // SGPR was written by a VALU instruction.
579   int SmrdSgprWaitStates = 4;
580   auto IsHazardDefFn = [this](const MachineInstr &MI) {
581     return TII.isVALU(MI);
582   };
583   auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
584     return TII.isSALU(MI);
585   };
586 
587   bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
588 
589   for (const MachineOperand &Use : SMRD->uses()) {
590     if (!Use.isReg())
591       continue;
592     int WaitStatesNeededForUse =
593         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
594                                                    SmrdSgprWaitStates);
595     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
596 
597     // This fixes what appears to be undocumented hardware behavior in SI where
598     // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
599     // needs some number of nops in between. We don't know how many we need, but
600     // let's use 4. This wasn't discovered before probably because the only
601     // case when this happens is when we expand a 64-bit pointer into a full
602     // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
603     // probably never encountered in the closed-source land.
604     if (IsBufferSMRD) {
605       int WaitStatesNeededForUse =
606         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
607                                                    IsBufferHazardDefFn,
608                                                    SmrdSgprWaitStates);
609       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
610     }
611   }
612 
613   return WaitStatesNeeded;
614 }
615 
616 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
617   if (!ST.hasVMEMReadSGPRVALUDefHazard())
618     return 0;
619 
620   int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
621 
622   // A read of an SGPR by a VMEM instruction requires 5 wait states when the
623   // SGPR was written by a VALU Instruction.
624   const int VmemSgprWaitStates = 5;
625   auto IsHazardDefFn = [this](const MachineInstr &MI) {
626     return TII.isVALU(MI);
627   };
628   for (const MachineOperand &Use : VMEM->uses()) {
629     if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
630       continue;
631 
632     int WaitStatesNeededForUse =
633         VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
634                                                    VmemSgprWaitStates);
635     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
636   }
637   return WaitStatesNeeded;
638 }
639 
640 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
641   const SIRegisterInfo *TRI = ST.getRegisterInfo();
642   const SIInstrInfo *TII = ST.getInstrInfo();
643 
644   // Check for DPP VGPR read after VALU VGPR write and EXEC write.
645   int DppVgprWaitStates = 2;
646   int DppExecWaitStates = 5;
647   int WaitStatesNeeded = 0;
648   auto IsHazardDefFn = [TII](const MachineInstr &MI) {
649     return TII->isVALU(MI);
650   };
651 
652   for (const MachineOperand &Use : DPP->uses()) {
653     if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
654       continue;
655     int WaitStatesNeededForUse =
656         DppVgprWaitStates - getWaitStatesSinceDef(
657                                 Use.getReg(),
658                                 [](const MachineInstr &) { return true; },
659                                 DppVgprWaitStates);
660     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
661   }
662 
663   WaitStatesNeeded = std::max(
664       WaitStatesNeeded,
665       DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
666                                                 DppExecWaitStates));
667 
668   return WaitStatesNeeded;
669 }
670 
671 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
672   const SIInstrInfo *TII = ST.getInstrInfo();
673 
674   // v_div_fmas requires 4 wait states after a write to vcc from a VALU
675   // instruction.
676   const int DivFMasWaitStates = 4;
677   auto IsHazardDefFn = [TII](const MachineInstr &MI) {
678     return TII->isVALU(MI);
679   };
680   int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
681                                                DivFMasWaitStates);
682 
683   return DivFMasWaitStates - WaitStatesNeeded;
684 }
685 
686 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
687   const SIInstrInfo *TII = ST.getInstrInfo();
688   unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
689 
690   const int GetRegWaitStates = 2;
691   auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
692     return GetRegHWReg == getHWReg(TII, MI);
693   };
694   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
695 
696   return GetRegWaitStates - WaitStatesNeeded;
697 }
698 
699 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
700   const SIInstrInfo *TII = ST.getInstrInfo();
701   unsigned HWReg = getHWReg(TII, *SetRegInstr);
702 
703   const int SetRegWaitStates = ST.getSetRegWaitStates();
704   auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
705     return HWReg == getHWReg(TII, MI);
706   };
707   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
708   return SetRegWaitStates - WaitStatesNeeded;
709 }
710 
711 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
712   if (!MI.mayStore())
713     return -1;
714 
715   const SIInstrInfo *TII = ST.getInstrInfo();
716   unsigned Opcode = MI.getOpcode();
717   const MCInstrDesc &Desc = MI.getDesc();
718 
719   int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
720   int VDataRCID = -1;
721   if (VDataIdx != -1)
722     VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
723 
724   if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
725     // There is no hazard if the instruction does not use vector regs
726     // (like wbinvl1)
727     if (VDataIdx == -1)
728       return -1;
729     // For MUBUF/MTBUF instructions this hazard only exists if the
730     // instruction is not using a register in the soffset field.
731     const MachineOperand *SOffset =
732         TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
733     // If we have no soffset operand, then assume this field has been
734     // hardcoded to zero.
735     if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
736         (!SOffset || !SOffset->isReg()))
737       return VDataIdx;
738   }
739 
740   // MIMG instructions create a hazard if they don't use a 256-bit T# and
741   // the store size is greater than 8 bytes and they have more than two bits
742   // of their dmask set.
743   // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
744   if (TII->isMIMG(MI)) {
745     int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
746     assert(SRsrcIdx != -1 &&
747            AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
748     (void)SRsrcIdx;
749   }
750 
751   if (TII->isFLAT(MI)) {
752     int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
753     if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
754       return DataIdx;
755   }
756 
757   return -1;
758 }
759 
760 int
761 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
762                                             const MachineRegisterInfo &MRI) {
763   // Helper to check for the hazard where VMEM instructions that store more than
764   // 8 bytes can have there store data over written by the next instruction.
765   const SIRegisterInfo *TRI = ST.getRegisterInfo();
766 
767   const int VALUWaitStates = 1;
768   int WaitStatesNeeded = 0;
769 
770   if (!TRI->isVectorRegister(MRI, Def.getReg()))
771     return WaitStatesNeeded;
772   Register Reg = Def.getReg();
773   auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) {
774     int DataIdx = createsVALUHazard(MI);
775     return DataIdx >= 0 &&
776            TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg);
777   };
778   int WaitStatesNeededForDef =
779     VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
780   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
781 
782   return WaitStatesNeeded;
783 }
784 
785 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
786   // This checks for the hazard where VMEM instructions that store more than
787   // 8 bytes can have there store data over written by the next instruction.
788   if (!ST.has12DWordStoreHazard())
789     return 0;
790 
791   const MachineRegisterInfo &MRI = MF.getRegInfo();
792   int WaitStatesNeeded = 0;
793 
794   for (const MachineOperand &Def : VALU->defs()) {
795     WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
796   }
797 
798   return WaitStatesNeeded;
799 }
800 
801 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
802   // This checks for hazards associated with inline asm statements.
803   // Since inline asms can contain just about anything, we use this
804   // to call/leverage other check*Hazard routines. Note that
805   // this function doesn't attempt to address all possible inline asm
806   // hazards (good luck), but is a collection of what has been
807   // problematic thus far.
808 
809   // see checkVALUHazards()
810   if (!ST.has12DWordStoreHazard())
811     return 0;
812 
813   const MachineRegisterInfo &MRI = MF.getRegInfo();
814   int WaitStatesNeeded = 0;
815 
816   for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
817        I != E; ++I) {
818     const MachineOperand &Op = IA->getOperand(I);
819     if (Op.isReg() && Op.isDef()) {
820       WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
821     }
822   }
823 
824   return WaitStatesNeeded;
825 }
826 
827 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
828   const SIInstrInfo *TII = ST.getInstrInfo();
829   const SIRegisterInfo *TRI = ST.getRegisterInfo();
830   const MachineRegisterInfo &MRI = MF.getRegInfo();
831 
832   const MachineOperand *LaneSelectOp =
833       TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
834 
835   if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
836     return 0;
837 
838   Register LaneSelectReg = LaneSelectOp->getReg();
839   auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); };
840 
841   const int RWLaneWaitStates = 4;
842   int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
843                                               RWLaneWaitStates);
844   return RWLaneWaitStates - WaitStatesSince;
845 }
846 
847 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
848   if (!ST.hasRFEHazards())
849     return 0;
850 
851   const SIInstrInfo *TII = ST.getInstrInfo();
852 
853   const int RFEWaitStates = 1;
854 
855   auto IsHazardFn = [TII](const MachineInstr &MI) {
856     return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS;
857   };
858   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
859   return RFEWaitStates - WaitStatesNeeded;
860 }
861 
862 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
863   const SIInstrInfo *TII = ST.getInstrInfo();
864   const int SMovRelWaitStates = 1;
865   auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
866   return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn,
867                                                    SMovRelWaitStates);
868 }
869 
870 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
871   fixVMEMtoScalarWriteHazards(MI);
872   fixVcmpxPermlaneHazards(MI);
873   fixSMEMtoVectorWriteHazards(MI);
874   fixVcmpxExecWARHazard(MI);
875   fixLdsBranchVmemWARHazard(MI);
876 }
877 
878 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
879   if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI))
880     return false;
881 
882   const SIInstrInfo *TII = ST.getInstrInfo();
883   auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVOPC(MI); };
884 
885   auto IsExpiredFn = [](const MachineInstr &MI, int) {
886     unsigned Opc = MI.getOpcode();
887     return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 &&
888            Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa;
889   };
890 
891   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
892       std::numeric_limits<int>::max())
893     return false;
894 
895   // V_NOP will be discarded by SQ.
896   // Use V_MOB_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
897   // which is always a VGPR and available.
898   auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
899   Register Reg = Src0->getReg();
900   bool IsUndef = Src0->isUndef();
901   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
902           TII->get(AMDGPU::V_MOV_B32_e32))
903     .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0))
904     .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill);
905 
906   return true;
907 }
908 
909 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
910   if (!ST.hasVMEMtoScalarWriteHazard())
911     return false;
912 
913   if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI))
914     return false;
915 
916   if (MI->getNumDefs() == 0)
917     return false;
918 
919   const SIRegisterInfo *TRI = ST.getRegisterInfo();
920 
921   auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
922     if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I) &&
923         !SIInstrInfo::isFLAT(I))
924       return false;
925 
926     for (const MachineOperand &Def : MI->defs()) {
927       const MachineOperand *Op =
928           I.findRegisterUseOperand(Def.getReg(), false, TRI);
929       if (!Op)
930         continue;
931       return true;
932     }
933     return false;
934   };
935 
936   auto IsExpiredFn = [](const MachineInstr &MI, int) {
937     return SIInstrInfo::isVALU(MI) ||
938            (MI.getOpcode() == AMDGPU::S_WAITCNT &&
939             !MI.getOperand(0).getImm()) ||
940            (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
941             MI.getOperand(0).getImm() == 0xffe3);
942   };
943 
944   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
945       std::numeric_limits<int>::max())
946     return false;
947 
948   const SIInstrInfo *TII = ST.getInstrInfo();
949   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
950           TII->get(AMDGPU::S_WAITCNT_DEPCTR))
951       .addImm(0xffe3);
952   return true;
953 }
954 
955 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
956   if (!ST.hasSMEMtoVectorWriteHazard())
957     return false;
958 
959   if (!SIInstrInfo::isVALU(*MI))
960     return false;
961 
962   unsigned SDSTName;
963   switch (MI->getOpcode()) {
964   case AMDGPU::V_READLANE_B32:
965   case AMDGPU::V_READFIRSTLANE_B32:
966     SDSTName = AMDGPU::OpName::vdst;
967     break;
968   default:
969     SDSTName = AMDGPU::OpName::sdst;
970     break;
971   }
972 
973   const SIInstrInfo *TII = ST.getInstrInfo();
974   const SIRegisterInfo *TRI = ST.getRegisterInfo();
975   const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
976   const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
977   if (!SDST) {
978     for (const auto &MO : MI->implicit_operands()) {
979       if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) {
980         SDST = &MO;
981         break;
982       }
983     }
984   }
985 
986   if (!SDST)
987     return false;
988 
989   const Register SDSTReg = SDST->getReg();
990   auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
991     return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI);
992   };
993 
994   auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
995     if (TII->isSALU(MI)) {
996       switch (MI.getOpcode()) {
997       case AMDGPU::S_SETVSKIP:
998       case AMDGPU::S_VERSION:
999       case AMDGPU::S_WAITCNT_VSCNT:
1000       case AMDGPU::S_WAITCNT_VMCNT:
1001       case AMDGPU::S_WAITCNT_EXPCNT:
1002         // These instructions cannot not mitigate the hazard.
1003         return false;
1004       case AMDGPU::S_WAITCNT_LGKMCNT:
1005         // Reducing lgkmcnt count to 0 always mitigates the hazard.
1006         return (MI.getOperand(1).getImm() == 0) &&
1007                (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1008       case AMDGPU::S_WAITCNT: {
1009         const int64_t Imm = MI.getOperand(0).getImm();
1010         AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm);
1011         return (Decoded.LgkmCnt == 0);
1012       }
1013       default:
1014         // SOPP instructions cannot mitigate the hazard.
1015         if (TII->isSOPP(MI))
1016           return false;
1017         // At this point the SALU can be assumed to mitigate the hazard
1018         // because either:
1019         // (a) it is independent of the at risk SMEM (breaking chain),
1020         // or
1021         // (b) it is dependent on the SMEM, in which case an appropriate
1022         //     s_waitcnt lgkmcnt _must_ exist between it and the at risk
1023         //     SMEM instruction.
1024         return true;
1025       }
1026     }
1027     return false;
1028   };
1029 
1030   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1031       std::numeric_limits<int>::max())
1032     return false;
1033 
1034   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1035           TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
1036       .addImm(0);
1037   return true;
1038 }
1039 
1040 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
1041   if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI))
1042     return false;
1043 
1044   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1045   if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
1046     return false;
1047 
1048   auto IsHazardFn = [TRI](const MachineInstr &I) {
1049     if (SIInstrInfo::isVALU(I))
1050       return false;
1051     return I.readsRegister(AMDGPU::EXEC, TRI);
1052   };
1053 
1054   const SIInstrInfo *TII = ST.getInstrInfo();
1055   auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
1056     if (SIInstrInfo::isVALU(MI)) {
1057       if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst))
1058         return true;
1059       for (auto MO : MI.implicit_operands())
1060         if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg())))
1061           return true;
1062     }
1063     if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1064         (MI.getOperand(0).getImm() & 0xfffe) == 0xfffe)
1065       return true;
1066     return false;
1067   };
1068 
1069   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1070       std::numeric_limits<int>::max())
1071     return false;
1072 
1073   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1074           TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1075     .addImm(0xfffe);
1076   return true;
1077 }
1078 
1079 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
1080                                                  const GCNSubtarget &ST) {
1081   if (!ST.hasLdsBranchVmemWARHazard())
1082     return false;
1083 
1084   // Check if the necessary condition for the hazard is met: both LDS and VMEM
1085   // instructions need to appear in the same function.
1086   bool HasLds = false;
1087   bool HasVmem = false;
1088   for (auto &MBB : MF) {
1089     for (auto &MI : MBB) {
1090       HasLds |= SIInstrInfo::isDS(MI);
1091       HasVmem |=
1092           SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI);
1093       if (HasLds && HasVmem)
1094         return true;
1095     }
1096   }
1097   return false;
1098 }
1099 
1100 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
1101   if (!RunLdsBranchVmemWARHazardFixup)
1102     return false;
1103 
1104   assert(ST.hasLdsBranchVmemWARHazard());
1105 
1106   auto IsHazardInst = [](const MachineInstr &MI) {
1107     if (SIInstrInfo::isDS(MI))
1108       return 1;
1109     if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI))
1110       return 2;
1111     return 0;
1112   };
1113 
1114   auto InstType = IsHazardInst(*MI);
1115   if (!InstType)
1116     return false;
1117 
1118   auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
1119     return IsHazardInst(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1120                                I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1121                                !I.getOperand(1).getImm());
1122   };
1123 
1124   auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
1125     if (!I.isBranch())
1126       return false;
1127 
1128     auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
1129       auto InstType2 = IsHazardInst(I);
1130       return InstType2 && InstType != InstType2;
1131     };
1132 
1133     auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
1134       auto InstType2 = IsHazardInst(I);
1135       if (InstType == InstType2)
1136         return true;
1137 
1138       return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1139              I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1140              !I.getOperand(1).getImm();
1141     };
1142 
1143     return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) !=
1144            std::numeric_limits<int>::max();
1145   };
1146 
1147   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1148       std::numeric_limits<int>::max())
1149     return false;
1150 
1151   const SIInstrInfo *TII = ST.getInstrInfo();
1152   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1153           TII->get(AMDGPU::S_WAITCNT_VSCNT))
1154     .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1155     .addImm(0);
1156 
1157   return true;
1158 }
1159 
1160 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) {
1161   int NSAtoVMEMWaitStates = 1;
1162 
1163   if (!ST.hasNSAtoVMEMBug())
1164     return 0;
1165 
1166   if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI))
1167     return 0;
1168 
1169   const SIInstrInfo *TII = ST.getInstrInfo();
1170   const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
1171   if (!Offset || (Offset->getImm() & 6) == 0)
1172     return 0;
1173 
1174   auto IsHazardFn = [TII](const MachineInstr &I) {
1175     if (!SIInstrInfo::isMIMG(I))
1176       return false;
1177     const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode());
1178     return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
1179            TII->getInstSizeInBytes(I) >= 16;
1180   };
1181 
1182   return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
1183 }
1184 
1185 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) {
1186   int FPAtomicToDenormModeWaitStates = 3;
1187 
1188   if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
1189     return 0;
1190 
1191   auto IsHazardFn = [](const MachineInstr &I) {
1192     if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I))
1193       return false;
1194     return SIInstrInfo::isFPAtomic(I);
1195   };
1196 
1197   auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
1198     if (WaitStates >= 3 || SIInstrInfo::isVALU(MI))
1199       return true;
1200 
1201     switch (MI.getOpcode()) {
1202     case AMDGPU::S_WAITCNT:
1203     case AMDGPU::S_WAITCNT_VSCNT:
1204     case AMDGPU::S_WAITCNT_VMCNT:
1205     case AMDGPU::S_WAITCNT_EXPCNT:
1206     case AMDGPU::S_WAITCNT_LGKMCNT:
1207     case AMDGPU::S_WAIT_IDLE:
1208       return true;
1209     default:
1210       break;
1211     }
1212 
1213     return false;
1214   };
1215 
1216   return FPAtomicToDenormModeWaitStates -
1217          ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn);
1218 }
1219 
1220 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) {
1221   assert(SIInstrInfo::isMAI(*MI));
1222 
1223   return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
1224 }
1225 
1226 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) {
1227   int WaitStatesNeeded = 0;
1228   unsigned Opc = MI->getOpcode();
1229 
1230   auto IsVALUFn = [](const MachineInstr &MI) {
1231     return SIInstrInfo::isVALU(MI);
1232   };
1233 
1234   if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
1235     const int LegacyVALUWritesVGPRWaitStates = 2;
1236     const int VALUWritesExecWaitStates = 4;
1237     const int MaxWaitStates = 4;
1238 
1239     int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1240       getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
1241     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1242 
1243     if (WaitStatesNeeded < MaxWaitStates) {
1244       for (const MachineOperand &Use : MI->explicit_uses()) {
1245         const int MaxWaitStates = 2;
1246 
1247         if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1248           continue;
1249 
1250         int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
1251           getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates);
1252         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1253 
1254         if (WaitStatesNeeded == MaxWaitStates)
1255           break;
1256       }
1257     }
1258   }
1259 
1260   auto IsMFMAFn = [](const MachineInstr &MI) {
1261     return SIInstrInfo::isMAI(MI) &&
1262            MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1263            MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1264   };
1265 
1266   for (const MachineOperand &Op : MI->explicit_operands()) {
1267     if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg()))
1268       continue;
1269 
1270     if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1271       continue;
1272 
1273     const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
1274     const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
1275     const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
1276     const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
1277     const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
1278     const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
1279     const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
1280     const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
1281     const int MaxWaitStates = 18;
1282     Register Reg = Op.getReg();
1283     unsigned HazardDefLatency = 0;
1284 
1285     auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &HazardDefLatency,
1286                                this](const MachineInstr &MI) {
1287       if (!IsMFMAFn(MI))
1288         return false;
1289       Register DstReg = MI.getOperand(0).getReg();
1290       if (DstReg == Reg)
1291         return false;
1292       HazardDefLatency =
1293           std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
1294       return TRI.regsOverlap(DstReg, Reg);
1295     };
1296 
1297     int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn,
1298                                                    MaxWaitStates);
1299     int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
1300     int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1301     int OpNo = MI->getOperandNo(&Op);
1302     if (OpNo == SrcCIdx) {
1303       NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
1304     } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
1305       switch (HazardDefLatency) {
1306       case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
1307                break;
1308       case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
1309                break;
1310       case 16: LLVM_FALLTHROUGH;
1311       default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
1312                break;
1313       }
1314     } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1315       switch (HazardDefLatency) {
1316       case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
1317                break;
1318       case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
1319                break;
1320       case 16: LLVM_FALLTHROUGH;
1321       default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
1322                break;
1323       }
1324     }
1325 
1326     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1327     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1328 
1329     if (WaitStatesNeeded == MaxWaitStates)
1330       return WaitStatesNeeded; // Early exit.
1331 
1332     auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
1333       if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1334         return false;
1335       Register DstReg = MI.getOperand(0).getReg();
1336       return TRI.regsOverlap(Reg, DstReg);
1337     };
1338 
1339     const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
1340     const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
1341     const int AccVGPRWriteAccVgprReadWaitStates = 3;
1342     NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
1343     if (OpNo == SrcCIdx)
1344       NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
1345     else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
1346       NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
1347 
1348     WaitStatesNeededForUse = NeedWaitStates -
1349       getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates);
1350     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1351 
1352     if (WaitStatesNeeded == MaxWaitStates)
1353       return WaitStatesNeeded; // Early exit.
1354   }
1355 
1356   if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1357     const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
1358     const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
1359     const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
1360     const int MaxWaitStates = 13;
1361     Register DstReg = MI->getOperand(0).getReg();
1362     unsigned HazardDefLatency = 0;
1363 
1364     auto IsSrcCMFMAFn = [DstReg, &IsMFMAFn, &HazardDefLatency,
1365                          this](const MachineInstr &MI) {
1366       if (!IsMFMAFn(MI))
1367         return false;
1368       Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
1369       HazardDefLatency =
1370           std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
1371       return TRI.regsOverlap(Reg, DstReg);
1372     };
1373 
1374     int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
1375     int NeedWaitStates;
1376     switch (HazardDefLatency) {
1377     case 2:  NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
1378              break;
1379     case 8:  NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
1380              break;
1381     case 16: LLVM_FALLTHROUGH;
1382     default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
1383              break;
1384     }
1385 
1386     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
1387     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1388   }
1389 
1390   return WaitStatesNeeded;
1391 }
1392 
1393 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
1394   int WaitStatesNeeded = 0;
1395   unsigned Opc = MI->getOpcode();
1396 
1397   auto IsMFMAFn = [](const MachineInstr &MI) {
1398     return SIInstrInfo::isMAI(MI) &&
1399            MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1400            MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1401   };
1402 
1403   auto IsLegacyVALUFn = [&IsMFMAFn](const MachineInstr &MI) {
1404     return SIInstrInfo::isVALU(MI) && !IsMFMAFn(MI);
1405   };
1406 
1407   auto IsLegacyVALUNotDotFn = [&IsMFMAFn](const MachineInstr &MI) {
1408     return SIInstrInfo::isVALU(MI) && !IsMFMAFn(MI) && !SIInstrInfo::isDOT(MI);
1409   };
1410 
1411   if (!IsMFMAFn(*MI))
1412     return WaitStatesNeeded;
1413 
1414   const int VALUWritesExecWaitStates = 4;
1415   int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1416     getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
1417                           VALUWritesExecWaitStates);
1418   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1419 
1420   int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1421 
1422   // Loop for both DGEMM and S/HGEMM 2nd instruction.
1423   for (const MachineOperand &Use : MI->explicit_uses()) {
1424     const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
1425     const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
1426     const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
1427     const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
1428     const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
1429     const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
1430     const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
1431     const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
1432     const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
1433     const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
1434     const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
1435     const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
1436     const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
1437     const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
1438     const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
1439     const int MaxWaitStates = 19;
1440 
1441     if (!Use.isReg())
1442       continue;
1443     Register Reg = Use.getReg();
1444     bool FullReg;
1445     const MachineInstr *MI1;
1446 
1447     auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &FullReg, &MI1,
1448                                this](const MachineInstr &MI) {
1449       if (!IsMFMAFn(MI))
1450         return false;
1451       Register DstReg = MI.getOperand(0).getReg();
1452       FullReg = (DstReg == Reg);
1453       MI1 = &MI;
1454       return TRI.regsOverlap(DstReg, Reg);
1455     };
1456 
1457     WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
1458       getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
1459     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1460 
1461     int NumWaitStates =
1462         getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates);
1463     if (NumWaitStates == std::numeric_limits<int>::max())
1464       continue;
1465 
1466     int OpNo = MI->getOperandNo(&Use);
1467     unsigned Opc1 = MI1->getOpcode();
1468     int NeedWaitStates = 0;
1469     if (OpNo == SrcCIdx) {
1470       if (!isDGEMM(Opc) && isDGEMM(Opc1)) {
1471         NeedWaitStates = 0;
1472       } else if (FullReg) {
1473         if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
1474              Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
1475             (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
1476              Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
1477           NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
1478       } else {
1479         switch (Opc1) {
1480         case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
1481         case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
1482         case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
1483         case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
1484           if (!isXDL(ST, *MI))
1485             NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
1486           break;
1487         case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
1488         case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
1489           if (!isXDL(ST, *MI))
1490             NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
1491           break;
1492         default:
1493           switch (TSchedModel.computeInstrLatency(MI1)) {
1494           case 2:
1495             NeedWaitStates = isDGEMM(Opc)
1496               ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
1497               : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
1498             break;
1499           case 8:
1500             NeedWaitStates = isDGEMM(Opc)
1501               ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
1502               : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
1503             break;
1504           case 16: LLVM_FALLTHROUGH;
1505           default:
1506             NeedWaitStates = isDGEMM(Opc)
1507               ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
1508               : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
1509           }
1510         }
1511       }
1512     } else {
1513       switch (Opc1) {
1514       case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
1515       case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
1516       case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
1517       case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
1518         NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
1519         break;
1520       case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
1521       case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
1522         NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
1523         break;
1524       default:
1525         switch (TSchedModel.computeInstrLatency(MI1)) {
1526         case 2:
1527           NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
1528           break;
1529         case 8:
1530           NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
1531           break;
1532         case 16: LLVM_FALLTHROUGH;
1533         default:
1534           NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
1535         }
1536       }
1537     }
1538     if (WaitStatesNeeded >= NeedWaitStates)
1539       continue;
1540 
1541     WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
1542     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1543 
1544     if (WaitStatesNeeded == MaxWaitStates)
1545       break;
1546   }
1547 
1548   return WaitStatesNeeded;
1549 }
1550 
1551 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) {
1552   // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
1553   if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
1554     return 0;
1555 
1556   int WaitStatesNeeded = 0;
1557 
1558   auto IsAccVgprReadFn = [](const MachineInstr &MI) {
1559     return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
1560   };
1561 
1562   for (const MachineOperand &Op : MI->explicit_uses()) {
1563     if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg()))
1564       continue;
1565 
1566     Register Reg = Op.getReg();
1567 
1568     const int AccVgprReadLdStWaitStates = 2;
1569     const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
1570     const int MaxWaitStates = 2;
1571 
1572     int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
1573       getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates);
1574     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1575 
1576     if (WaitStatesNeeded == MaxWaitStates)
1577       return WaitStatesNeeded; // Early exit.
1578 
1579     auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
1580       if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
1581           MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1582         return false;
1583       auto IsVALUFn = [](const MachineInstr &MI) {
1584         return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI);
1585       };
1586       return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
1587              std::numeric_limits<int>::max();
1588     };
1589 
1590     WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
1591       getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
1592     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1593   }
1594 
1595   return WaitStatesNeeded;
1596 }
1597 
1598 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
1599   if (!ST.hasGFX90AInsts())
1600     return 0;
1601 
1602   auto IsMFMAFn = [](const MachineInstr &MI) -> bool {
1603     return SIInstrInfo::isMAI(MI) &&
1604            MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1605            MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1606   };
1607 
1608   auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
1609     return isDGEMM(MI.getOpcode());
1610   };
1611 
1612   // This is checked in checkMAIHazards90A()
1613   if (IsMFMAFn(*MI))
1614     return 0;
1615 
1616   int WaitStatesNeeded = 0;
1617 
1618   bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) ||
1619                        SIInstrInfo::isFLAT(*MI) ||
1620                        SIInstrInfo::isDS(*MI) ||
1621                        SIInstrInfo::isEXP(*MI);
1622   bool IsVALU = SIInstrInfo::isVALU(*MI);
1623 
1624   const MachineInstr *MFMA = nullptr;
1625   unsigned Reg;
1626   auto IsMFMAWriteFn = [&Reg, &IsMFMAFn, &MFMA, this](const MachineInstr &MI) {
1627     if (!IsMFMAFn(MI) || !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
1628       return false;
1629     MFMA = &MI;
1630     return true;
1631   };
1632 
1633   const MachineInstr *DOT = nullptr;
1634   auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
1635     if (!SIInstrInfo::isDOT(MI) ||
1636         !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
1637       return false;
1638     DOT = &MI;
1639     return true;
1640   };
1641 
1642   int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
1643                                            AMDGPU::OpName::src2);
1644 
1645   if (IsMemOrExport || IsVALU) {
1646     const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
1647     const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
1648     const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
1649     const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
1650     const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
1651     const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
1652     const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
1653     const int DotWriteSameDotReadSrcAB = 3;
1654     const int DotWriteDifferentVALURead = 3;
1655     const int MaxWaitStates = 19;
1656 
1657     for (const MachineOperand &Use : MI->explicit_uses()) {
1658       if (!Use.isReg())
1659         continue;
1660       Reg = Use.getReg();
1661 
1662       DOT = nullptr;
1663       int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
1664                                                      MaxWaitStates);
1665       if (DOT) {
1666         int NeedWaitStates = 0;
1667         if (DOT->getOpcode() == MI->getOpcode()) {
1668           if (&Use - &MI->getOperand(0) != SrcCIdx)
1669             NeedWaitStates = DotWriteSameDotReadSrcAB;
1670         } else {
1671           NeedWaitStates = DotWriteDifferentVALURead;
1672         }
1673 
1674         int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1675         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1676       }
1677 
1678       MFMA = nullptr;
1679       WaitStatesSinceDef =
1680           getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
1681       if (!MFMA)
1682         continue;
1683 
1684       unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
1685       int NeedWaitStates = MaxWaitStates;
1686       switch (HazardDefLatency) {
1687       case 2:
1688         NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
1689         break;
1690       case 4:
1691         assert(isDGEMM(MFMA->getOpcode()));
1692         NeedWaitStates =
1693             IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
1694                           : DMFMA4x4WriteVgprVALUReadWaitStates;
1695         break;
1696       case 8:
1697         NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
1698         break;
1699       case 16: LLVM_FALLTHROUGH;
1700       default:
1701         NeedWaitStates =
1702           isDGEMM(MFMA->getOpcode())
1703             ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates
1704                             : DMFMA16x16WriteVgprVALUReadWaitStates
1705             : SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
1706         break;
1707       }
1708 
1709       int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1710       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1711 
1712       if (WaitStatesNeeded == MaxWaitStates)
1713         break;
1714     }
1715   }
1716 
1717   unsigned Opc = MI->getOpcode();
1718   const int DMFMAToFMA64WaitStates = 2;
1719   if ((Opc == AMDGPU::V_FMA_F64_e64 ||
1720        Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
1721        Opc == AMDGPU::V_FMAC_F64_dpp) &&
1722       WaitStatesNeeded < DMFMAToFMA64WaitStates) {
1723     int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
1724       getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
1725     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1726   }
1727 
1728   if (!IsVALU && !IsMemOrExport)
1729     return WaitStatesNeeded;
1730 
1731   for (const MachineOperand &Def : MI->defs()) {
1732     const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
1733     const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
1734     const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
1735     const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
1736     const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
1737     const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
1738     const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
1739     const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
1740     const int DotWriteDifferentVALUWrite = 3;
1741     const int MaxWaitStates = 19;
1742     const int MaxWarWaitStates = 15;
1743 
1744     Reg = Def.getReg();
1745 
1746     DOT = nullptr;
1747     int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
1748                                                    MaxWaitStates);
1749     if (DOT && DOT->getOpcode() != MI->getOpcode())
1750       WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
1751                                                     WaitStatesSinceDef);
1752 
1753     MFMA = nullptr;
1754     WaitStatesSinceDef =
1755         getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
1756     if (MFMA) {
1757       int NeedWaitStates = MaxWaitStates;
1758       switch (TSchedModel.computeInstrLatency(MFMA)) {
1759       case 2:
1760         NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
1761         break;
1762       case 4:
1763         assert(isDGEMM(MFMA->getOpcode()));
1764         NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
1765         break;
1766       case 8:
1767         NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
1768         break;
1769       case 16: LLVM_FALLTHROUGH;
1770       default:
1771         NeedWaitStates = isDGEMM(MFMA->getOpcode())
1772                    ? DMFMA16x16WriteVgprVALUWriteWaitStates
1773                    : SMFMA32x32WriteVgprVALUWawWaitStates;
1774         break;
1775       }
1776 
1777       int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1778       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1779 
1780       if (WaitStatesNeeded == MaxWaitStates)
1781         break;
1782     }
1783 
1784     auto IsSMFMAReadAsCFn = [&Reg, &IsMFMAFn, &MFMA,
1785                              this](const MachineInstr &MI) {
1786       if (!IsMFMAFn(MI) || isDGEMM(MI.getOpcode()) ||
1787           !MI.readsRegister(Reg, &TRI))
1788         return false;
1789 
1790       const MachineOperand *SrcC =
1791           TII.getNamedOperand(MI, AMDGPU::OpName::src2);
1792       assert(SrcC);
1793       if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
1794         return false;
1795 
1796       MFMA = &MI;
1797       return true;
1798     };
1799 
1800     MFMA = nullptr;
1801     int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn,
1802                                                 MaxWarWaitStates);
1803     if (!MFMA)
1804       continue;
1805 
1806     unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
1807     int NeedWaitStates = MaxWaitStates;
1808     switch (HazardDefLatency) {
1809     case 2:  NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
1810              break;
1811     case 8:  NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
1812              break;
1813     case 16: LLVM_FALLTHROUGH;
1814     default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
1815              break;
1816     }
1817 
1818     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
1819     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1820   }
1821 
1822   return WaitStatesNeeded;
1823 }
1824 
1825 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) {
1826   if (!SU->isInstr())
1827     return false;
1828 
1829   const MachineInstr *MAI = nullptr;
1830   auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
1831     MAI = nullptr;
1832     if (SIInstrInfo::isMAI(MI) &&
1833         MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1834         MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64)
1835       MAI = &MI;
1836     return MAI != nullptr;
1837   };
1838 
1839   MachineInstr *MI = SU->getInstr();
1840   if (IsMFMAFn(*MI)) {
1841     int W = getWaitStatesSince(IsMFMAFn, 16);
1842     if (MAI)
1843       return W < (int)TSchedModel.computeInstrLatency(MAI);
1844   }
1845 
1846   return false;
1847 }
1848