1 //===- GCNVOPDUtils.cpp - GCN VOPD Utils ------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file contains the AMDGPU DAG scheduling
10 /// mutation to pair VOPD instructions back to back. It also contains
11 // subroutines useful in the creation of VOPD instructions
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "GCNVOPDUtils.h"
16 #include "AMDGPUSubtarget.h"
17 #include "GCNSubtarget.h"
18 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19 #include "SIInstrInfo.h"
20 #include "Utils/AMDGPUBaseInfo.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/CodeGen/MachineBasicBlock.h"
24 #include "llvm/CodeGen/MachineInstr.h"
25 #include "llvm/CodeGen/MachineOperand.h"
26 #include "llvm/CodeGen/MachineRegisterInfo.h"
27 #include "llvm/CodeGen/MacroFusion.h"
28 #include "llvm/CodeGen/ScheduleDAG.h"
29 #include "llvm/CodeGen/ScheduleDAGMutation.h"
30 #include "llvm/CodeGen/TargetInstrInfo.h"
31 #include "llvm/MC/MCInst.h"
32
33 using namespace llvm;
34
35 #define DEBUG_TYPE "gcn-vopd-utils"
36
checkVOPDRegConstraints(const SIInstrInfo & TII,const MachineInstr & FirstMI,const MachineInstr & SecondMI)37 bool llvm::checkVOPDRegConstraints(const SIInstrInfo &TII,
38 const MachineInstr &FirstMI,
39 const MachineInstr &SecondMI) {
40 namespace VOPD = AMDGPU::VOPD;
41
42 const MachineFunction *MF = FirstMI.getMF();
43 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
44 const SIRegisterInfo *TRI = dyn_cast<SIRegisterInfo>(ST.getRegisterInfo());
45 const MachineRegisterInfo &MRI = MF->getRegInfo();
46 // Literals also count against scalar bus limit
47 SmallVector<const MachineOperand *> UniqueLiterals;
48 auto addLiteral = [&](const MachineOperand &Op) {
49 for (auto &Literal : UniqueLiterals) {
50 if (Literal->isIdenticalTo(Op))
51 return;
52 }
53 UniqueLiterals.push_back(&Op);
54 };
55 SmallVector<Register> UniqueScalarRegs;
56 assert([&]() -> bool {
57 for (auto MII = MachineBasicBlock::const_iterator(&FirstMI);
58 MII != FirstMI.getParent()->instr_end(); ++MII) {
59 if (&*MII == &SecondMI)
60 return true;
61 }
62 return false;
63 }() && "Expected FirstMI to precede SecondMI");
64 // Cannot pair dependent instructions
65 for (const auto &Use : SecondMI.uses())
66 if (Use.isReg() && FirstMI.modifiesRegister(Use.getReg()))
67 return false;
68
69 auto getVRegIdx = [&](unsigned OpcodeIdx, unsigned OperandIdx) {
70 const MachineInstr &MI = (OpcodeIdx == VOPD::X) ? FirstMI : SecondMI;
71 const MachineOperand &Operand = MI.getOperand(OperandIdx);
72 if (Operand.isReg() && TRI->isVectorRegister(MRI, Operand.getReg()))
73 return Operand.getReg();
74 return Register();
75 };
76
77 auto InstInfo =
78 AMDGPU::getVOPDInstInfo(FirstMI.getDesc(), SecondMI.getDesc());
79
80 for (auto CompIdx : VOPD::COMPONENTS) {
81 const MachineInstr &MI = (CompIdx == VOPD::X) ? FirstMI : SecondMI;
82
83 const MachineOperand &Src0 = MI.getOperand(VOPD::Component::SRC0);
84 if (Src0.isReg()) {
85 if (!TRI->isVectorRegister(MRI, Src0.getReg())) {
86 if (!is_contained(UniqueScalarRegs, Src0.getReg()))
87 UniqueScalarRegs.push_back(Src0.getReg());
88 }
89 } else {
90 if (!TII.isInlineConstant(MI, VOPD::Component::SRC0))
91 addLiteral(Src0);
92 }
93
94 if (InstInfo[CompIdx].hasMandatoryLiteral()) {
95 auto CompOprIdx = InstInfo[CompIdx].getMandatoryLiteralCompOperandIndex();
96 addLiteral(MI.getOperand(CompOprIdx));
97 }
98 if (MI.getDesc().hasImplicitUseOfPhysReg(AMDGPU::VCC))
99 UniqueScalarRegs.push_back(AMDGPU::VCC_LO);
100 }
101
102 if (UniqueLiterals.size() > 1)
103 return false;
104 if ((UniqueLiterals.size() + UniqueScalarRegs.size()) > 2)
105 return false;
106 if (InstInfo.hasInvalidOperand(getVRegIdx))
107 return false;
108
109 LLVM_DEBUG(dbgs() << "VOPD Reg Constraints Passed\n\tX: " << FirstMI
110 << "\n\tY: " << SecondMI << "\n");
111 return true;
112 }
113
114 /// Check if the instr pair, FirstMI and SecondMI, should be scheduled
115 /// together. Given SecondMI, when FirstMI is unspecified, then check if
116 /// SecondMI may be part of a fused pair at all.
shouldScheduleVOPDAdjacent(const TargetInstrInfo & TII,const TargetSubtargetInfo & TSI,const MachineInstr * FirstMI,const MachineInstr & SecondMI)117 static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII,
118 const TargetSubtargetInfo &TSI,
119 const MachineInstr *FirstMI,
120 const MachineInstr &SecondMI) {
121 const SIInstrInfo &STII = static_cast<const SIInstrInfo &>(TII);
122 unsigned Opc2 = SecondMI.getOpcode();
123 auto SecondCanBeVOPD = AMDGPU::getCanBeVOPD(Opc2);
124
125 // One instruction case
126 if (!FirstMI)
127 return SecondCanBeVOPD.Y;
128
129 unsigned Opc = FirstMI->getOpcode();
130 auto FirstCanBeVOPD = AMDGPU::getCanBeVOPD(Opc);
131
132 if (!((FirstCanBeVOPD.X && SecondCanBeVOPD.Y) ||
133 (FirstCanBeVOPD.Y && SecondCanBeVOPD.X)))
134 return false;
135
136 return checkVOPDRegConstraints(STII, *FirstMI, SecondMI);
137 }
138
139 namespace {
140 /// Adapts design from MacroFusion
141 /// Puts valid candidate instructions back-to-back so they can easily
142 /// be turned into VOPD instructions
143 /// Greedily pairs instruction candidates. O(n^2) algorithm.
144 struct VOPDPairingMutation : ScheduleDAGMutation {
145 ShouldSchedulePredTy shouldScheduleAdjacent; // NOLINT: function pointer
146
VOPDPairingMutation__anon3623d8820411::VOPDPairingMutation147 VOPDPairingMutation(
148 ShouldSchedulePredTy shouldScheduleAdjacent) // NOLINT: function pointer
149 : shouldScheduleAdjacent(shouldScheduleAdjacent) {}
150
apply__anon3623d8820411::VOPDPairingMutation151 void apply(ScheduleDAGInstrs *DAG) override {
152 const TargetInstrInfo &TII = *DAG->TII;
153 const GCNSubtarget &ST = DAG->MF.getSubtarget<GCNSubtarget>();
154 if (!AMDGPU::hasVOPD(ST) || !ST.isWave32()) {
155 LLVM_DEBUG(dbgs() << "Target does not support VOPDPairingMutation\n");
156 return;
157 }
158
159 std::vector<SUnit>::iterator ISUI, JSUI;
160 for (ISUI = DAG->SUnits.begin(); ISUI != DAG->SUnits.end(); ++ISUI) {
161 const MachineInstr *IMI = ISUI->getInstr();
162 if (!shouldScheduleAdjacent(TII, ST, nullptr, *IMI))
163 continue;
164 if (!hasLessThanNumFused(*ISUI, 2))
165 continue;
166
167 for (JSUI = ISUI + 1; JSUI != DAG->SUnits.end(); ++JSUI) {
168 if (JSUI->isBoundaryNode())
169 continue;
170 const MachineInstr *JMI = JSUI->getInstr();
171 if (!hasLessThanNumFused(*JSUI, 2) ||
172 !shouldScheduleAdjacent(TII, ST, IMI, *JMI))
173 continue;
174 if (fuseInstructionPair(*DAG, *ISUI, *JSUI))
175 break;
176 }
177 }
178 LLVM_DEBUG(dbgs() << "Completed VOPDPairingMutation\n");
179 }
180 };
181 } // namespace
182
createVOPDPairingMutation()183 std::unique_ptr<ScheduleDAGMutation> llvm::createVOPDPairingMutation() {
184 return std::make_unique<VOPDPairingMutation>(shouldScheduleVOPDAdjacent);
185 }
186