1 //===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief Insert wait instructions for memory reads and writes.
12 ///
13 /// Memory reads and writes are issued asynchronously, so we need to insert
14 /// S_WAITCNT instructions when we want to access any of their results or
15 /// overwrite any register that's used asynchronously.
16 //
17 //===----------------------------------------------------------------------===//
18
19 #include "AMDGPU.h"
20 #include "AMDGPUSubtarget.h"
21 #include "SIDefines.h"
22 #include "SIInstrInfo.h"
23 #include "SIMachineFunctionInfo.h"
24 #include "llvm/CodeGen/MachineFunction.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineRegisterInfo.h"
28
29 using namespace llvm;
30
31 namespace {
32
33 /// \brief One variable for each of the hardware counters
34 typedef union {
35 struct {
36 unsigned VM;
37 unsigned EXP;
38 unsigned LGKM;
39 } Named;
40 unsigned Array[3];
41
42 } Counters;
43
44 typedef enum {
45 OTHER,
46 SMEM,
47 VMEM
48 } InstType;
49
50 typedef Counters RegCounters[512];
51 typedef std::pair<unsigned, unsigned> RegInterval;
52
53 class SIInsertWaits : public MachineFunctionPass {
54
55 private:
56 static char ID;
57 const SIInstrInfo *TII;
58 const SIRegisterInfo *TRI;
59 const MachineRegisterInfo *MRI;
60
61 /// \brief Constant hardware limits
62 static const Counters WaitCounts;
63
64 /// \brief Constant zero value
65 static const Counters ZeroCounts;
66
67 /// \brief Counter values we have already waited on.
68 Counters WaitedOn;
69
70 /// \brief Counter values for last instruction issued.
71 Counters LastIssued;
72
73 /// \brief Registers used by async instructions.
74 RegCounters UsedRegs;
75
76 /// \brief Registers defined by async instructions.
77 RegCounters DefinedRegs;
78
79 /// \brief Different export instruction types seen since last wait.
80 unsigned ExpInstrTypesSeen;
81
82 /// \brief Type of the last opcode.
83 InstType LastOpcodeType;
84
85 bool LastInstWritesM0;
86
87 /// \brief Get increment/decrement amount for this instruction.
88 Counters getHwCounts(MachineInstr &MI);
89
90 /// \brief Is operand relevant for async execution?
91 bool isOpRelevant(MachineOperand &Op);
92
93 /// \brief Get register interval an operand affects.
94 RegInterval getRegInterval(MachineOperand &Op);
95
96 /// \brief Handle instructions async components
97 void pushInstruction(MachineBasicBlock &MBB,
98 MachineBasicBlock::iterator I);
99
100 /// \brief Insert the actual wait instruction
101 bool insertWait(MachineBasicBlock &MBB,
102 MachineBasicBlock::iterator I,
103 const Counters &Counts);
104
105 /// \brief Do we need def2def checks?
106 bool unorderedDefines(MachineInstr &MI);
107
108 /// \brief Resolve all operand dependencies to counter requirements
109 Counters handleOperands(MachineInstr &MI);
110
111 /// \brief Insert S_NOP between an instruction writing M0 and S_SENDMSG.
112 void handleSendMsg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I);
113
114 public:
SIInsertWaits(TargetMachine & tm)115 SIInsertWaits(TargetMachine &tm) :
116 MachineFunctionPass(ID),
117 TII(nullptr),
118 TRI(nullptr),
119 ExpInstrTypesSeen(0) { }
120
121 bool runOnMachineFunction(MachineFunction &MF) override;
122
getPassName() const123 const char *getPassName() const override {
124 return "SI insert wait instructions";
125 }
126
127 };
128
129 } // End anonymous namespace
130
131 char SIInsertWaits::ID = 0;
132
133 const Counters SIInsertWaits::WaitCounts = { { 15, 7, 7 } };
134 const Counters SIInsertWaits::ZeroCounts = { { 0, 0, 0 } };
135
createSIInsertWaits(TargetMachine & tm)136 FunctionPass *llvm::createSIInsertWaits(TargetMachine &tm) {
137 return new SIInsertWaits(tm);
138 }
139
getHwCounts(MachineInstr & MI)140 Counters SIInsertWaits::getHwCounts(MachineInstr &MI) {
141
142 uint64_t TSFlags = TII->get(MI.getOpcode()).TSFlags;
143 Counters Result;
144
145 Result.Named.VM = !!(TSFlags & SIInstrFlags::VM_CNT);
146
147 // Only consider stores or EXP for EXP_CNT
148 Result.Named.EXP = !!(TSFlags & SIInstrFlags::EXP_CNT &&
149 (MI.getOpcode() == AMDGPU::EXP || MI.getDesc().mayStore()));
150
151 // LGKM may uses larger values
152 if (TSFlags & SIInstrFlags::LGKM_CNT) {
153
154 if (TII->isSMRD(MI.getOpcode())) {
155
156 MachineOperand &Op = MI.getOperand(0);
157 assert(Op.isReg() && "First LGKM operand must be a register!");
158
159 unsigned Reg = Op.getReg();
160 unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize();
161 Result.Named.LGKM = Size > 4 ? 2 : 1;
162
163 } else {
164 // DS
165 Result.Named.LGKM = 1;
166 }
167
168 } else {
169 Result.Named.LGKM = 0;
170 }
171
172 return Result;
173 }
174
isOpRelevant(MachineOperand & Op)175 bool SIInsertWaits::isOpRelevant(MachineOperand &Op) {
176
177 // Constants are always irrelevant
178 if (!Op.isReg())
179 return false;
180
181 // Defines are always relevant
182 if (Op.isDef())
183 return true;
184
185 // For exports all registers are relevant
186 MachineInstr &MI = *Op.getParent();
187 if (MI.getOpcode() == AMDGPU::EXP)
188 return true;
189
190 // For stores the stored value is also relevant
191 if (!MI.getDesc().mayStore())
192 return false;
193
194 // Check if this operand is the value being stored.
195 // Special case for DS instructions, since the address
196 // operand comes before the value operand and it may have
197 // multiple data operands.
198
199 if (TII->isDS(MI.getOpcode())) {
200 MachineOperand *Data = TII->getNamedOperand(MI, AMDGPU::OpName::data);
201 if (Data && Op.isIdenticalTo(*Data))
202 return true;
203
204 MachineOperand *Data0 = TII->getNamedOperand(MI, AMDGPU::OpName::data0);
205 if (Data0 && Op.isIdenticalTo(*Data0))
206 return true;
207
208 MachineOperand *Data1 = TII->getNamedOperand(MI, AMDGPU::OpName::data1);
209 if (Data1 && Op.isIdenticalTo(*Data1))
210 return true;
211
212 return false;
213 }
214
215 // NOTE: This assumes that the value operand is before the
216 // address operand, and that there is only one value operand.
217 for (MachineInstr::mop_iterator I = MI.operands_begin(),
218 E = MI.operands_end(); I != E; ++I) {
219
220 if (I->isReg() && I->isUse())
221 return Op.isIdenticalTo(*I);
222 }
223
224 return false;
225 }
226
getRegInterval(MachineOperand & Op)227 RegInterval SIInsertWaits::getRegInterval(MachineOperand &Op) {
228
229 if (!Op.isReg() || !TRI->isInAllocatableClass(Op.getReg()))
230 return std::make_pair(0, 0);
231
232 unsigned Reg = Op.getReg();
233 unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize();
234
235 assert(Size >= 4);
236
237 RegInterval Result;
238 Result.first = TRI->getEncodingValue(Reg);
239 Result.second = Result.first + Size / 4;
240
241 return Result;
242 }
243
pushInstruction(MachineBasicBlock & MBB,MachineBasicBlock::iterator I)244 void SIInsertWaits::pushInstruction(MachineBasicBlock &MBB,
245 MachineBasicBlock::iterator I) {
246
247 // Get the hardware counter increments and sum them up
248 Counters Increment = getHwCounts(*I);
249 unsigned Sum = 0;
250
251 for (unsigned i = 0; i < 3; ++i) {
252 LastIssued.Array[i] += Increment.Array[i];
253 Sum += Increment.Array[i];
254 }
255
256 // If we don't increase anything then that's it
257 if (Sum == 0) {
258 LastOpcodeType = OTHER;
259 return;
260 }
261
262 if (TRI->ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
263 // Any occurence of consecutive VMEM or SMEM instructions forms a VMEM
264 // or SMEM clause, respectively.
265 //
266 // The temporary workaround is to break the clauses with S_NOP.
267 //
268 // The proper solution would be to allocate registers such that all source
269 // and destination registers don't overlap, e.g. this is illegal:
270 // r0 = load r2
271 // r2 = load r0
272 if ((LastOpcodeType == SMEM && TII->isSMRD(I->getOpcode())) ||
273 (LastOpcodeType == VMEM && Increment.Named.VM)) {
274 // Insert a NOP to break the clause.
275 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_NOP))
276 .addImm(0);
277 LastInstWritesM0 = false;
278 }
279
280 if (TII->isSMRD(I->getOpcode()))
281 LastOpcodeType = SMEM;
282 else if (Increment.Named.VM)
283 LastOpcodeType = VMEM;
284 }
285
286 // Remember which export instructions we have seen
287 if (Increment.Named.EXP) {
288 ExpInstrTypesSeen |= I->getOpcode() == AMDGPU::EXP ? 1 : 2;
289 }
290
291 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
292
293 MachineOperand &Op = I->getOperand(i);
294 if (!isOpRelevant(Op))
295 continue;
296
297 RegInterval Interval = getRegInterval(Op);
298 for (unsigned j = Interval.first; j < Interval.second; ++j) {
299
300 // Remember which registers we define
301 if (Op.isDef())
302 DefinedRegs[j] = LastIssued;
303
304 // and which one we are using
305 if (Op.isUse())
306 UsedRegs[j] = LastIssued;
307 }
308 }
309 }
310
insertWait(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const Counters & Required)311 bool SIInsertWaits::insertWait(MachineBasicBlock &MBB,
312 MachineBasicBlock::iterator I,
313 const Counters &Required) {
314
315 // End of program? No need to wait on anything
316 if (I != MBB.end() && I->getOpcode() == AMDGPU::S_ENDPGM)
317 return false;
318
319 // Figure out if the async instructions execute in order
320 bool Ordered[3];
321
322 // VM_CNT is always ordered
323 Ordered[0] = true;
324
325 // EXP_CNT is unordered if we have both EXP & VM-writes
326 Ordered[1] = ExpInstrTypesSeen == 3;
327
328 // LGKM_CNT is handled as always unordered. TODO: Handle LDS and GDS
329 Ordered[2] = false;
330
331 // The values we are going to put into the S_WAITCNT instruction
332 Counters Counts = WaitCounts;
333
334 // Do we really need to wait?
335 bool NeedWait = false;
336
337 for (unsigned i = 0; i < 3; ++i) {
338
339 if (Required.Array[i] <= WaitedOn.Array[i])
340 continue;
341
342 NeedWait = true;
343
344 if (Ordered[i]) {
345 unsigned Value = LastIssued.Array[i] - Required.Array[i];
346
347 // Adjust the value to the real hardware possibilities.
348 Counts.Array[i] = std::min(Value, WaitCounts.Array[i]);
349
350 } else
351 Counts.Array[i] = 0;
352
353 // Remember on what we have waited on.
354 WaitedOn.Array[i] = LastIssued.Array[i] - Counts.Array[i];
355 }
356
357 if (!NeedWait)
358 return false;
359
360 // Reset EXP_CNT instruction types
361 if (Counts.Named.EXP == 0)
362 ExpInstrTypesSeen = 0;
363
364 // Build the wait instruction
365 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
366 .addImm((Counts.Named.VM & 0xF) |
367 ((Counts.Named.EXP & 0x7) << 4) |
368 ((Counts.Named.LGKM & 0x7) << 8));
369
370 LastOpcodeType = OTHER;
371 LastInstWritesM0 = false;
372 return true;
373 }
374
375 /// \brief helper function for handleOperands
increaseCounters(Counters & Dst,const Counters & Src)376 static void increaseCounters(Counters &Dst, const Counters &Src) {
377
378 for (unsigned i = 0; i < 3; ++i)
379 Dst.Array[i] = std::max(Dst.Array[i], Src.Array[i]);
380 }
381
handleOperands(MachineInstr & MI)382 Counters SIInsertWaits::handleOperands(MachineInstr &MI) {
383
384 Counters Result = ZeroCounts;
385
386 // S_SENDMSG implicitly waits for all outstanding LGKM transfers to finish,
387 // but we also want to wait for any other outstanding transfers before
388 // signalling other hardware blocks
389 if (MI.getOpcode() == AMDGPU::S_SENDMSG)
390 return LastIssued;
391
392 // For each register affected by this
393 // instruction increase the result sequence
394 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
395
396 MachineOperand &Op = MI.getOperand(i);
397 RegInterval Interval = getRegInterval(Op);
398 for (unsigned j = Interval.first; j < Interval.second; ++j) {
399
400 if (Op.isDef()) {
401 increaseCounters(Result, UsedRegs[j]);
402 increaseCounters(Result, DefinedRegs[j]);
403 }
404
405 if (Op.isUse())
406 increaseCounters(Result, DefinedRegs[j]);
407 }
408 }
409
410 return Result;
411 }
412
handleSendMsg(MachineBasicBlock & MBB,MachineBasicBlock::iterator I)413 void SIInsertWaits::handleSendMsg(MachineBasicBlock &MBB,
414 MachineBasicBlock::iterator I) {
415 if (TRI->ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
416 return;
417
418 // There must be "S_NOP 0" between an instruction writing M0 and S_SENDMSG.
419 if (LastInstWritesM0 && I->getOpcode() == AMDGPU::S_SENDMSG) {
420 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_NOP)).addImm(0);
421 LastInstWritesM0 = false;
422 return;
423 }
424
425 // Set whether this instruction sets M0
426 LastInstWritesM0 = false;
427
428 unsigned NumOperands = I->getNumOperands();
429 for (unsigned i = 0; i < NumOperands; i++) {
430 const MachineOperand &Op = I->getOperand(i);
431
432 if (Op.isReg() && Op.isDef() && Op.getReg() == AMDGPU::M0)
433 LastInstWritesM0 = true;
434 }
435 }
436
437 // FIXME: Insert waits listed in Table 4.2 "Required User-Inserted Wait States"
438 // around other non-memory instructions.
runOnMachineFunction(MachineFunction & MF)439 bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
440 bool Changes = false;
441
442 TII = static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo());
443 TRI =
444 static_cast<const SIRegisterInfo *>(MF.getSubtarget().getRegisterInfo());
445
446 MRI = &MF.getRegInfo();
447
448 WaitedOn = ZeroCounts;
449 LastIssued = ZeroCounts;
450 LastOpcodeType = OTHER;
451 LastInstWritesM0 = false;
452
453 memset(&UsedRegs, 0, sizeof(UsedRegs));
454 memset(&DefinedRegs, 0, sizeof(DefinedRegs));
455
456 for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
457 BI != BE; ++BI) {
458
459 MachineBasicBlock &MBB = *BI;
460 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
461 I != E; ++I) {
462
463 // Wait for everything before a barrier.
464 if (I->getOpcode() == AMDGPU::S_BARRIER)
465 Changes |= insertWait(MBB, I, LastIssued);
466 else
467 Changes |= insertWait(MBB, I, handleOperands(*I));
468
469 pushInstruction(MBB, I);
470 handleSendMsg(MBB, I);
471 }
472
473 // Wait for everything at the end of the MBB
474 Changes |= insertWait(MBB, MBB.getFirstTerminator(), LastIssued);
475 }
476
477 return Changes;
478 }
479