10b57cec5SDimitry Andric //===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // This file defines hazard recognizers for scheduling on GCN processors. 100b57cec5SDimitry Andric // 110b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 120b57cec5SDimitry Andric 130b57cec5SDimitry Andric #ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H 140b57cec5SDimitry Andric #define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H 150b57cec5SDimitry Andric 160b57cec5SDimitry Andric #include "llvm/ADT/BitVector.h" 170b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h" 180b57cec5SDimitry Andric #include "llvm/CodeGen/ScheduleHazardRecognizer.h" 190b57cec5SDimitry Andric #include "llvm/CodeGen/TargetSchedule.h" 200b57cec5SDimitry Andric #include <list> 210b57cec5SDimitry Andric 220b57cec5SDimitry Andric namespace llvm { 230b57cec5SDimitry Andric 240b57cec5SDimitry Andric class MachineFunction; 250b57cec5SDimitry Andric class MachineInstr; 260b57cec5SDimitry Andric class MachineOperand; 270b57cec5SDimitry Andric class MachineRegisterInfo; 280b57cec5SDimitry Andric class SIInstrInfo; 290b57cec5SDimitry Andric class SIRegisterInfo; 300b57cec5SDimitry Andric class GCNSubtarget; 310b57cec5SDimitry Andric 320b57cec5SDimitry Andric class GCNHazardRecognizer final : public ScheduleHazardRecognizer { 330b57cec5SDimitry Andric public: 34fe6060f1SDimitry Andric typedef function_ref<bool(const MachineInstr &)> IsHazardFn; 350b57cec5SDimitry Andric 360b57cec5SDimitry Andric private: 370b57cec5SDimitry Andric // Distinguish if we are called from scheduler or hazard recognizer 380b57cec5SDimitry Andric bool IsHazardRecognizerMode; 390b57cec5SDimitry Andric 400b57cec5SDimitry Andric // This variable stores the instruction that has been emitted this cycle. It 410b57cec5SDimitry Andric // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is 420b57cec5SDimitry Andric // called. 430b57cec5SDimitry Andric MachineInstr *CurrCycleInstr; 440b57cec5SDimitry Andric std::list<MachineInstr*> EmittedInstrs; 450b57cec5SDimitry Andric const MachineFunction &MF; 460b57cec5SDimitry Andric const GCNSubtarget &ST; 470b57cec5SDimitry Andric const SIInstrInfo &TII; 480b57cec5SDimitry Andric const SIRegisterInfo &TRI; 490b57cec5SDimitry Andric TargetSchedModel TSchedModel; 50fe6060f1SDimitry Andric bool RunLdsBranchVmemWARHazardFixup; 510b57cec5SDimitry Andric 520b57cec5SDimitry Andric /// RegUnits of uses in the current soft memory clause. 530b57cec5SDimitry Andric BitVector ClauseUses; 540b57cec5SDimitry Andric 550b57cec5SDimitry Andric /// RegUnits of defs in the current soft memory clause. 560b57cec5SDimitry Andric BitVector ClauseDefs; 570b57cec5SDimitry Andric resetClause()580b57cec5SDimitry Andric void resetClause() { 590b57cec5SDimitry Andric ClauseUses.reset(); 600b57cec5SDimitry Andric ClauseDefs.reset(); 610b57cec5SDimitry Andric } 620b57cec5SDimitry Andric 630b57cec5SDimitry Andric void addClauseInst(const MachineInstr &MI); 640b57cec5SDimitry Andric 6581ad6265SDimitry Andric /// \returns the number of wait states before another MFMA instruction can be 6681ad6265SDimitry Andric /// issued after \p MI. 6781ad6265SDimitry Andric unsigned getMFMAPipelineWaitStates(const MachineInstr &MI) const; 6881ad6265SDimitry Andric 690b57cec5SDimitry Andric // Advance over a MachineInstr bundle. Look for hazards in the bundled 700b57cec5SDimitry Andric // instructions. 710b57cec5SDimitry Andric void processBundle(); 720b57cec5SDimitry Andric 73bdd1243dSDimitry Andric // Run on an individual instruction in hazard recognizer mode. This can be 74bdd1243dSDimitry Andric // used on a newly inserted instruction before returning from PreEmitNoops. 75bdd1243dSDimitry Andric void runOnInstruction(MachineInstr *MI); 76bdd1243dSDimitry Andric 770b57cec5SDimitry Andric int getWaitStatesSince(IsHazardFn IsHazard, int Limit); 780b57cec5SDimitry Andric int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit); 790b57cec5SDimitry Andric int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit); 800b57cec5SDimitry Andric 810b57cec5SDimitry Andric int checkSoftClauseHazards(MachineInstr *SMEM); 820b57cec5SDimitry Andric int checkSMRDHazards(MachineInstr *SMRD); 830b57cec5SDimitry Andric int checkVMEMHazards(MachineInstr* VMEM); 840b57cec5SDimitry Andric int checkDPPHazards(MachineInstr *DPP); 850b57cec5SDimitry Andric int checkDivFMasHazards(MachineInstr *DivFMas); 860b57cec5SDimitry Andric int checkGetRegHazards(MachineInstr *GetRegInstr); 870b57cec5SDimitry Andric int checkSetRegHazards(MachineInstr *SetRegInstr); 880b57cec5SDimitry Andric int createsVALUHazard(const MachineInstr &MI); 890b57cec5SDimitry Andric int checkVALUHazards(MachineInstr *VALU); 900b57cec5SDimitry Andric int checkVALUHazardsHelper(const MachineOperand &Def, const MachineRegisterInfo &MRI); 910b57cec5SDimitry Andric int checkRWLaneHazards(MachineInstr *RWLane); 920b57cec5SDimitry Andric int checkRFEHazards(MachineInstr *RFE); 930b57cec5SDimitry Andric int checkInlineAsmHazards(MachineInstr *IA); 940b57cec5SDimitry Andric int checkReadM0Hazards(MachineInstr *SMovRel); 950b57cec5SDimitry Andric int checkNSAtoVMEMHazard(MachineInstr *MI); 960b57cec5SDimitry Andric int checkFPAtomicToDenormModeHazard(MachineInstr *MI); 970b57cec5SDimitry Andric void fixHazards(MachineInstr *MI); 980b57cec5SDimitry Andric bool fixVcmpxPermlaneHazards(MachineInstr *MI); 990b57cec5SDimitry Andric bool fixVMEMtoScalarWriteHazards(MachineInstr *MI); 1000b57cec5SDimitry Andric bool fixSMEMtoVectorWriteHazards(MachineInstr *MI); 1010b57cec5SDimitry Andric bool fixVcmpxExecWARHazard(MachineInstr *MI); 1020b57cec5SDimitry Andric bool fixLdsBranchVmemWARHazard(MachineInstr *MI); 10381ad6265SDimitry Andric bool fixLdsDirectVALUHazard(MachineInstr *MI); 10481ad6265SDimitry Andric bool fixLdsDirectVMEMHazard(MachineInstr *MI); 10581ad6265SDimitry Andric bool fixVALUPartialForwardingHazard(MachineInstr *MI); 10681ad6265SDimitry Andric bool fixVALUTransUseHazard(MachineInstr *MI); 10781ad6265SDimitry Andric bool fixWMMAHazards(MachineInstr *MI); 108bdd1243dSDimitry Andric bool fixShift64HighRegBug(MachineInstr *MI); 109bdd1243dSDimitry Andric bool fixVALUMaskWriteHazard(MachineInstr *MI); 1100b57cec5SDimitry Andric 1110b57cec5SDimitry Andric int checkMAIHazards(MachineInstr *MI); 112fe6060f1SDimitry Andric int checkMAIHazards908(MachineInstr *MI); 113fe6060f1SDimitry Andric int checkMAIHazards90A(MachineInstr *MI); 11481ad6265SDimitry Andric /// Pad the latency between neighboring MFMA instructions with s_nops. The 11581ad6265SDimitry Andric /// percentage of wait states to fill with s_nops is specified by the command 11681ad6265SDimitry Andric /// line option '-amdgpu-mfma-padding-ratio'. 11781ad6265SDimitry Andric /// 11881ad6265SDimitry Andric /// For example, with '-amdgpu-mfma-padding-ratio=100': 11981ad6265SDimitry Andric /// 12081ad6265SDimitry Andric /// 2 pass MFMA instructions have a latency of 2 wait states. Therefore, a 12181ad6265SDimitry Andric /// 'S_NOP 1' will be added between sequential MFMA instructions. 12281ad6265SDimitry Andric /// 12381ad6265SDimitry Andric /// V_MFMA_F32_4X4X1F32 12481ad6265SDimitry Andric /// V_MFMA_F32_4X4X1F32 12581ad6265SDimitry Andric ///--> 12681ad6265SDimitry Andric /// V_MFMA_F32_4X4X1F32 12781ad6265SDimitry Andric /// S_NOP 1 12881ad6265SDimitry Andric /// V_MFMA_F32_4X4X1F32 12981ad6265SDimitry Andric int checkMFMAPadding(MachineInstr *MI); 130fe6060f1SDimitry Andric int checkMAIVALUHazards(MachineInstr *MI); 1310b57cec5SDimitry Andric int checkMAILdStHazards(MachineInstr *MI); 1320b57cec5SDimitry Andric 1330b57cec5SDimitry Andric public: 1340b57cec5SDimitry Andric GCNHazardRecognizer(const MachineFunction &MF); 1350b57cec5SDimitry Andric // We can only issue one instruction per cycle. atIssueLimit()1360b57cec5SDimitry Andric bool atIssueLimit() const override { return true; } 1370b57cec5SDimitry Andric void EmitInstruction(SUnit *SU) override; 1380b57cec5SDimitry Andric void EmitInstruction(MachineInstr *MI) override; 1390b57cec5SDimitry Andric HazardType getHazardType(SUnit *SU, int Stalls) override; 1400b57cec5SDimitry Andric void EmitNoop() override; 1410b57cec5SDimitry Andric unsigned PreEmitNoops(MachineInstr *) override; 1420b57cec5SDimitry Andric unsigned PreEmitNoopsCommon(MachineInstr *); 1430b57cec5SDimitry Andric void AdvanceCycle() override; 1440b57cec5SDimitry Andric void RecedeCycle() override; 145e8d8bef9SDimitry Andric bool ShouldPreferAnother(SUnit *SU) override; 146e8d8bef9SDimitry Andric void Reset() override; 1470b57cec5SDimitry Andric }; 1480b57cec5SDimitry Andric 1490b57cec5SDimitry Andric } // end namespace llvm 1500b57cec5SDimitry Andric 1510b57cec5SDimitry Andric #endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H 152