1 //===- GCNRegPressure.h -----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file defines the GCNRegPressure class, which tracks registry pressure
11 /// by bookkeeping number of SGPR/VGPRs used, weights for large SGPR/VGPRs. It
12 /// also implements a compare function, which compares different register
13 /// pressures, and declares one with max occupance as winner.
14 ///
15 //===----------------------------------------------------------------------===//
16
17 #ifndef LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
18 #define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
19
20 #include "AMDGPUSubtarget.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/CodeGen/LiveIntervals.h"
23 #include "llvm/CodeGen/MachineBasicBlock.h"
24 #include "llvm/CodeGen/MachineInstr.h"
25 #include "llvm/CodeGen/SlotIndexes.h"
26 #include "llvm/MC/LaneBitmask.h"
27 #include "llvm/Support/Debug.h"
28 #include <algorithm>
29 #include <limits>
30
31 namespace llvm {
32
33 class MachineRegisterInfo;
34 class raw_ostream;
35
36 struct GCNRegPressure {
37 enum RegKind {
38 SGPR32,
39 SGPR_TUPLE,
40 VGPR32,
41 VGPR_TUPLE,
42 AGPR32,
43 AGPR_TUPLE,
44 TOTAL_KINDS
45 };
46
GCNRegPressureGCNRegPressure47 GCNRegPressure() {
48 clear();
49 }
50
emptyGCNRegPressure51 bool empty() const { return getSGPRNum() == 0 && getVGPRNum() == 0; }
52
clearGCNRegPressure53 void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); }
54
getSGPRNumGCNRegPressure55 unsigned getSGPRNum() const { return Value[SGPR32]; }
getVGPRNumGCNRegPressure56 unsigned getVGPRNum() const { return std::max(Value[VGPR32], Value[AGPR32]); }
57
getVGPRTuplesWeightGCNRegPressure58 unsigned getVGPRTuplesWeight() const { return std::max(Value[VGPR_TUPLE],
59 Value[AGPR_TUPLE]); }
getSGPRTuplesWeightGCNRegPressure60 unsigned getSGPRTuplesWeight() const { return Value[SGPR_TUPLE]; }
61
getOccupancyGCNRegPressure62 unsigned getOccupancy(const GCNSubtarget &ST) const {
63 return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()),
64 ST.getOccupancyWithNumVGPRs(getVGPRNum()));
65 }
66
67 void inc(unsigned Reg,
68 LaneBitmask PrevMask,
69 LaneBitmask NewMask,
70 const MachineRegisterInfo &MRI);
71
higherOccupancyGCNRegPressure72 bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure& O) const {
73 return getOccupancy(ST) > O.getOccupancy(ST);
74 }
75
76 bool less(const GCNSubtarget &ST, const GCNRegPressure& O,
77 unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const;
78
79 bool operator==(const GCNRegPressure &O) const {
80 return std::equal(&Value[0], &Value[TOTAL_KINDS], O.Value);
81 }
82
83 bool operator!=(const GCNRegPressure &O) const {
84 return !(*this == O);
85 }
86
87 void print(raw_ostream &OS, const GCNSubtarget *ST = nullptr) const;
dumpGCNRegPressure88 void dump() const { print(dbgs()); }
89
90 private:
91 unsigned Value[TOTAL_KINDS];
92
93 static unsigned getRegKind(unsigned Reg, const MachineRegisterInfo &MRI);
94
95 friend GCNRegPressure max(const GCNRegPressure &P1,
96 const GCNRegPressure &P2);
97 };
98
max(const GCNRegPressure & P1,const GCNRegPressure & P2)99 inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) {
100 GCNRegPressure Res;
101 for (unsigned I = 0; I < GCNRegPressure::TOTAL_KINDS; ++I)
102 Res.Value[I] = std::max(P1.Value[I], P2.Value[I]);
103 return Res;
104 }
105
106 class GCNRPTracker {
107 public:
108 using LiveRegSet = DenseMap<unsigned, LaneBitmask>;
109
110 protected:
111 const LiveIntervals &LIS;
112 LiveRegSet LiveRegs;
113 GCNRegPressure CurPressure, MaxPressure;
114 const MachineInstr *LastTrackedMI = nullptr;
115 mutable const MachineRegisterInfo *MRI = nullptr;
116
GCNRPTracker(const LiveIntervals & LIS_)117 GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {}
118
119 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy,
120 bool After);
121
122 public:
123 // live regs for the current state
decltype(LiveRegs)124 const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; }
getLastTrackedMI()125 const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; }
126
clearMaxPressure()127 void clearMaxPressure() { MaxPressure.clear(); }
128
129 // returns MaxPressure, resetting it
moveMaxPressure()130 decltype(MaxPressure) moveMaxPressure() {
131 auto Res = MaxPressure;
132 MaxPressure.clear();
133 return Res;
134 }
135
moveLiveRegs()136 decltype(LiveRegs) moveLiveRegs() {
137 return std::move(LiveRegs);
138 }
139
140 static void printLiveRegs(raw_ostream &OS, const LiveRegSet& LiveRegs,
141 const MachineRegisterInfo &MRI);
142 };
143
144 class GCNUpwardRPTracker : public GCNRPTracker {
145 public:
GCNUpwardRPTracker(const LiveIntervals & LIS_)146 GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
147
148 // reset tracker to the point just below MI
149 // filling live regs upon this point using LIS
150 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
151
152 // move to the state just above the MI
153 void recede(const MachineInstr &MI);
154
155 // checks whether the tracker's state after receding MI corresponds
156 // to reported by LIS
157 bool isValid() const;
158 };
159
160 class GCNDownwardRPTracker : public GCNRPTracker {
161 // Last position of reset or advanceBeforeNext
162 MachineBasicBlock::const_iterator NextMI;
163
164 MachineBasicBlock::const_iterator MBBEnd;
165
166 public:
GCNDownwardRPTracker(const LiveIntervals & LIS_)167 GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
168
getNext()169 const MachineBasicBlock::const_iterator getNext() const { return NextMI; }
170
171 // Reset tracker to the point before the MI
172 // filling live regs upon this point using LIS.
173 // Returns false if block is empty except debug values.
174 bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
175
176 // Move to the state right before the next MI. Returns false if reached
177 // end of the block.
178 bool advanceBeforeNext();
179
180 // Move to the state at the MI, advanceBeforeNext has to be called first.
181 void advanceToNext();
182
183 // Move to the state at the next MI. Returns false if reached end of block.
184 bool advance();
185
186 // Advance instructions until before End.
187 bool advance(MachineBasicBlock::const_iterator End);
188
189 // Reset to Begin and advance to End.
190 bool advance(MachineBasicBlock::const_iterator Begin,
191 MachineBasicBlock::const_iterator End,
192 const LiveRegSet *LiveRegsCopy = nullptr);
193 };
194
195 LaneBitmask getLiveLaneMask(unsigned Reg,
196 SlotIndex SI,
197 const LiveIntervals &LIS,
198 const MachineRegisterInfo &MRI);
199
200 GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI,
201 const LiveIntervals &LIS,
202 const MachineRegisterInfo &MRI);
203
204 /// creates a map MachineInstr -> LiveRegSet
205 /// R - range of iterators on instructions
206 /// After - upon entry or exit of every instruction
207 /// Note: there is no entry in the map for instructions with empty live reg set
208 /// Complexity = O(NumVirtRegs * averageLiveRangeSegmentsPerReg * lg(R))
209 template <typename Range>
210 DenseMap<MachineInstr*, GCNRPTracker::LiveRegSet>
getLiveRegMap(Range && R,bool After,LiveIntervals & LIS)211 getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) {
212 std::vector<SlotIndex> Indexes;
213 Indexes.reserve(std::distance(R.begin(), R.end()));
214 auto &SII = *LIS.getSlotIndexes();
215 for (MachineInstr *I : R) {
216 auto SI = SII.getInstructionIndex(*I);
217 Indexes.push_back(After ? SI.getDeadSlot() : SI.getBaseIndex());
218 }
219 llvm::sort(Indexes);
220
221 auto &MRI = (*R.begin())->getParent()->getParent()->getRegInfo();
222 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap;
223 SmallVector<SlotIndex, 32> LiveIdxs, SRLiveIdxs;
224 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
225 auto Reg = Register::index2VirtReg(I);
226 if (!LIS.hasInterval(Reg))
227 continue;
228 auto &LI = LIS.getInterval(Reg);
229 LiveIdxs.clear();
230 if (!LI.findIndexesLiveAt(Indexes, std::back_inserter(LiveIdxs)))
231 continue;
232 if (!LI.hasSubRanges()) {
233 for (auto SI : LiveIdxs)
234 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] =
235 MRI.getMaxLaneMaskForVReg(Reg);
236 } else
237 for (const auto &S : LI.subranges()) {
238 // constrain search for subranges by indexes live at main range
239 SRLiveIdxs.clear();
240 S.findIndexesLiveAt(LiveIdxs, std::back_inserter(SRLiveIdxs));
241 for (auto SI : SRLiveIdxs)
242 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] |= S.LaneMask;
243 }
244 }
245 return LiveRegMap;
246 }
247
getLiveRegsAfter(const MachineInstr & MI,const LiveIntervals & LIS)248 inline GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI,
249 const LiveIntervals &LIS) {
250 return getLiveRegs(LIS.getInstructionIndex(MI).getDeadSlot(), LIS,
251 MI.getParent()->getParent()->getRegInfo());
252 }
253
getLiveRegsBefore(const MachineInstr & MI,const LiveIntervals & LIS)254 inline GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI,
255 const LiveIntervals &LIS) {
256 return getLiveRegs(LIS.getInstructionIndex(MI).getBaseIndex(), LIS,
257 MI.getParent()->getParent()->getRegInfo());
258 }
259
260 template <typename Range>
getRegPressure(const MachineRegisterInfo & MRI,Range && LiveRegs)261 GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI,
262 Range &&LiveRegs) {
263 GCNRegPressure Res;
264 for (const auto &RM : LiveRegs)
265 Res.inc(RM.first, LaneBitmask::getNone(), RM.second, MRI);
266 return Res;
267 }
268
269 bool isEqual(const GCNRPTracker::LiveRegSet &S1,
270 const GCNRPTracker::LiveRegSet &S2);
271
272 void printLivesAt(SlotIndex SI,
273 const LiveIntervals &LIS,
274 const MachineRegisterInfo &MRI);
275
276 } // end namespace llvm
277
278 #endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
279