1 //===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
14 #define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
15 
16 #include "llvm/ADT/BitVector.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/CodeGen/ScheduleHazardRecognizer.h"
19 #include "llvm/CodeGen/TargetSchedule.h"
20 #include <list>
21 
22 namespace llvm {
23 
24 class MachineFunction;
25 class MachineInstr;
26 class MachineOperand;
27 class MachineRegisterInfo;
28 class SIInstrInfo;
29 class SIRegisterInfo;
30 class GCNSubtarget;
31 
32 class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
33 public:
34   typedef function_ref<bool(const MachineInstr &)> IsHazardFn;
35 
36 private:
37   // Distinguish if we are called from scheduler or hazard recognizer
38   bool IsHazardRecognizerMode;
39 
40   // This variable stores the instruction that has been emitted this cycle. It
41   // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is
42   // called.
43   MachineInstr *CurrCycleInstr;
44   std::list<MachineInstr*> EmittedInstrs;
45   const MachineFunction &MF;
46   const GCNSubtarget &ST;
47   const SIInstrInfo &TII;
48   const SIRegisterInfo &TRI;
49   TargetSchedModel TSchedModel;
50   bool RunLdsBranchVmemWARHazardFixup;
51 
52   /// RegUnits of uses in the current soft memory clause.
53   BitVector ClauseUses;
54 
55   /// RegUnits of defs in the current soft memory clause.
56   BitVector ClauseDefs;
57 
58   void resetClause() {
59     ClauseUses.reset();
60     ClauseDefs.reset();
61   }
62 
63   void addClauseInst(const MachineInstr &MI);
64 
65   /// \returns the number of wait states before another MFMA instruction can be
66   /// issued after \p MI.
67   unsigned getMFMAPipelineWaitStates(const MachineInstr &MI) const;
68 
69   // Advance over a MachineInstr bundle. Look for hazards in the bundled
70   // instructions.
71   void processBundle();
72 
73   int getWaitStatesSince(IsHazardFn IsHazard, int Limit);
74   int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit);
75   int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit);
76 
77   int checkSoftClauseHazards(MachineInstr *SMEM);
78   int checkSMRDHazards(MachineInstr *SMRD);
79   int checkVMEMHazards(MachineInstr* VMEM);
80   int checkDPPHazards(MachineInstr *DPP);
81   int checkDivFMasHazards(MachineInstr *DivFMas);
82   int checkGetRegHazards(MachineInstr *GetRegInstr);
83   int checkSetRegHazards(MachineInstr *SetRegInstr);
84   int createsVALUHazard(const MachineInstr &MI);
85   int checkVALUHazards(MachineInstr *VALU);
86   int checkVALUHazardsHelper(const MachineOperand &Def, const MachineRegisterInfo &MRI);
87   int checkRWLaneHazards(MachineInstr *RWLane);
88   int checkRFEHazards(MachineInstr *RFE);
89   int checkInlineAsmHazards(MachineInstr *IA);
90   int checkReadM0Hazards(MachineInstr *SMovRel);
91   int checkNSAtoVMEMHazard(MachineInstr *MI);
92   int checkFPAtomicToDenormModeHazard(MachineInstr *MI);
93   void fixHazards(MachineInstr *MI);
94   bool fixVcmpxPermlaneHazards(MachineInstr *MI);
95   bool fixVMEMtoScalarWriteHazards(MachineInstr *MI);
96   bool fixSMEMtoVectorWriteHazards(MachineInstr *MI);
97   bool fixVcmpxExecWARHazard(MachineInstr *MI);
98   bool fixLdsBranchVmemWARHazard(MachineInstr *MI);
99   bool fixLdsDirectVALUHazard(MachineInstr *MI);
100   bool fixLdsDirectVMEMHazard(MachineInstr *MI);
101   bool fixVALUPartialForwardingHazard(MachineInstr *MI);
102   bool fixVALUTransUseHazard(MachineInstr *MI);
103   bool fixWMMAHazards(MachineInstr *MI);
104 
105   int checkMAIHazards(MachineInstr *MI);
106   int checkMAIHazards908(MachineInstr *MI);
107   int checkMAIHazards90A(MachineInstr *MI);
108   /// Pad the latency between neighboring MFMA instructions with s_nops. The
109   /// percentage of wait states to fill with s_nops is specified by the command
110   /// line option '-amdgpu-mfma-padding-ratio'.
111   ///
112   /// For example, with '-amdgpu-mfma-padding-ratio=100':
113   ///
114   /// 2 pass MFMA instructions have a latency of 2 wait states. Therefore, a
115   /// 'S_NOP 1' will be added between sequential MFMA instructions.
116   ///
117   /// V_MFMA_F32_4X4X1F32
118   /// V_MFMA_F32_4X4X1F32
119   ///-->
120   /// V_MFMA_F32_4X4X1F32
121   /// S_NOP 1
122   /// V_MFMA_F32_4X4X1F32
123   int checkMFMAPadding(MachineInstr *MI);
124   int checkMAIVALUHazards(MachineInstr *MI);
125   int checkMAILdStHazards(MachineInstr *MI);
126 
127 public:
128   GCNHazardRecognizer(const MachineFunction &MF);
129   // We can only issue one instruction per cycle.
130   bool atIssueLimit() const override { return true; }
131   void EmitInstruction(SUnit *SU) override;
132   void EmitInstruction(MachineInstr *MI) override;
133   HazardType getHazardType(SUnit *SU, int Stalls) override;
134   void EmitNoop() override;
135   unsigned PreEmitNoops(MachineInstr *) override;
136   unsigned PreEmitNoopsCommon(MachineInstr *);
137   void AdvanceCycle() override;
138   void RecedeCycle() override;
139   bool ShouldPreferAnother(SUnit *SU) override;
140   void Reset() override;
141 };
142 
143 } // end namespace llvm
144 
145 #endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
146