1 //===- SplitKit.cpp - Toolkit for splitting live ranges -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the SplitAnalysis class as well as mutator functions for
10 // live range splitting.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "SplitKit.h"
15 #include "llvm/ADT/None.h"
16 #include "llvm/ADT/STLExtras.h"
17 #include "llvm/ADT/Statistic.h"
18 #include "llvm/Analysis/AliasAnalysis.h"
19 #include "llvm/CodeGen/LiveRangeEdit.h"
20 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
21 #include "llvm/CodeGen/MachineDominators.h"
22 #include "llvm/CodeGen/MachineInstr.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineLoopInfo.h"
25 #include "llvm/CodeGen/MachineOperand.h"
26 #include "llvm/CodeGen/MachineRegisterInfo.h"
27 #include "llvm/CodeGen/TargetInstrInfo.h"
28 #include "llvm/CodeGen/TargetOpcodes.h"
29 #include "llvm/CodeGen/TargetRegisterInfo.h"
30 #include "llvm/CodeGen/TargetSubtargetInfo.h"
31 #include "llvm/CodeGen/VirtRegMap.h"
32 #include "llvm/Config/llvm-config.h"
33 #include "llvm/IR/DebugLoc.h"
34 #include "llvm/Support/Allocator.h"
35 #include "llvm/Support/BlockFrequency.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/raw_ostream.h"
39 #include <algorithm>
40 #include <cassert>
41 #include <iterator>
42 #include <limits>
43 #include <tuple>
44
45 using namespace llvm;
46
47 #define DEBUG_TYPE "regalloc"
48
49 STATISTIC(NumFinished, "Number of splits finished");
50 STATISTIC(NumSimple, "Number of splits that were simple");
51 STATISTIC(NumCopies, "Number of copies inserted for splitting");
52 STATISTIC(NumRemats, "Number of rematerialized defs for splitting");
53 STATISTIC(NumRepairs, "Number of invalid live ranges repaired");
54
55 //===----------------------------------------------------------------------===//
56 // Last Insert Point Analysis
57 //===----------------------------------------------------------------------===//
58
InsertPointAnalysis(const LiveIntervals & lis,unsigned BBNum)59 InsertPointAnalysis::InsertPointAnalysis(const LiveIntervals &lis,
60 unsigned BBNum)
61 : LIS(lis), LastInsertPoint(BBNum) {}
62
63 SlotIndex
computeLastInsertPoint(const LiveInterval & CurLI,const MachineBasicBlock & MBB)64 InsertPointAnalysis::computeLastInsertPoint(const LiveInterval &CurLI,
65 const MachineBasicBlock &MBB) {
66 unsigned Num = MBB.getNumber();
67 std::pair<SlotIndex, SlotIndex> &LIP = LastInsertPoint[Num];
68 SlotIndex MBBEnd = LIS.getMBBEndIdx(&MBB);
69
70 SmallVector<const MachineBasicBlock *, 1> ExceptionalSuccessors;
71 bool EHPadSuccessor = false;
72 for (const MachineBasicBlock *SMBB : MBB.successors()) {
73 if (SMBB->isEHPad()) {
74 ExceptionalSuccessors.push_back(SMBB);
75 EHPadSuccessor = true;
76 } else if (SMBB->isInlineAsmBrIndirectTarget())
77 ExceptionalSuccessors.push_back(SMBB);
78 }
79
80 // Compute insert points on the first call. The pair is independent of the
81 // current live interval.
82 if (!LIP.first.isValid()) {
83 MachineBasicBlock::const_iterator FirstTerm = MBB.getFirstTerminator();
84 if (FirstTerm == MBB.end())
85 LIP.first = MBBEnd;
86 else
87 LIP.first = LIS.getInstructionIndex(*FirstTerm);
88
89 // If there is a landing pad or inlineasm_br successor, also find the
90 // instruction. If there is no such instruction, we don't need to do
91 // anything special. We assume there cannot be multiple instructions that
92 // are Calls with EHPad successors or INLINEASM_BR in a block. Further, we
93 // assume that if there are any, they will be after any other call
94 // instructions in the block.
95 if (ExceptionalSuccessors.empty())
96 return LIP.first;
97 for (auto I = MBB.rbegin(), E = MBB.rend(); I != E; ++I) {
98 if ((EHPadSuccessor && I->isCall()) ||
99 I->getOpcode() == TargetOpcode::INLINEASM_BR) {
100 LIP.second = LIS.getInstructionIndex(*I);
101 break;
102 }
103 }
104 }
105
106 // If CurLI is live into a landing pad successor, move the last insert point
107 // back to the call that may throw.
108 if (!LIP.second)
109 return LIP.first;
110
111 if (none_of(ExceptionalSuccessors, [&](const MachineBasicBlock *EHPad) {
112 return LIS.isLiveInToMBB(CurLI, EHPad);
113 }))
114 return LIP.first;
115
116 // Find the value leaving MBB.
117 const VNInfo *VNI = CurLI.getVNInfoBefore(MBBEnd);
118 if (!VNI)
119 return LIP.first;
120
121 // If the value leaving MBB was defined after the call in MBB, it can't
122 // really be live-in to the landing pad. This can happen if the landing pad
123 // has a PHI, and this register is undef on the exceptional edge.
124 // <rdar://problem/10664933>
125 if (!SlotIndex::isEarlierInstr(VNI->def, LIP.second) && VNI->def < MBBEnd)
126 return LIP.first;
127
128 // Value is properly live-in to the landing pad.
129 // Only allow inserts before the call.
130 return LIP.second;
131 }
132
133 MachineBasicBlock::iterator
getLastInsertPointIter(const LiveInterval & CurLI,MachineBasicBlock & MBB)134 InsertPointAnalysis::getLastInsertPointIter(const LiveInterval &CurLI,
135 MachineBasicBlock &MBB) {
136 SlotIndex LIP = getLastInsertPoint(CurLI, MBB);
137 if (LIP == LIS.getMBBEndIdx(&MBB))
138 return MBB.end();
139 return LIS.getInstructionFromIndex(LIP);
140 }
141
142 //===----------------------------------------------------------------------===//
143 // Split Analysis
144 //===----------------------------------------------------------------------===//
145
SplitAnalysis(const VirtRegMap & vrm,const LiveIntervals & lis,const MachineLoopInfo & mli)146 SplitAnalysis::SplitAnalysis(const VirtRegMap &vrm, const LiveIntervals &lis,
147 const MachineLoopInfo &mli)
148 : MF(vrm.getMachineFunction()), VRM(vrm), LIS(lis), Loops(mli),
149 TII(*MF.getSubtarget().getInstrInfo()), IPA(lis, MF.getNumBlockIDs()) {}
150
clear()151 void SplitAnalysis::clear() {
152 UseSlots.clear();
153 UseBlocks.clear();
154 ThroughBlocks.clear();
155 CurLI = nullptr;
156 DidRepairRange = false;
157 }
158
159 /// analyzeUses - Count instructions, basic blocks, and loops using CurLI.
analyzeUses()160 void SplitAnalysis::analyzeUses() {
161 assert(UseSlots.empty() && "Call clear first");
162
163 // First get all the defs from the interval values. This provides the correct
164 // slots for early clobbers.
165 for (const VNInfo *VNI : CurLI->valnos)
166 if (!VNI->isPHIDef() && !VNI->isUnused())
167 UseSlots.push_back(VNI->def);
168
169 // Get use slots form the use-def chain.
170 const MachineRegisterInfo &MRI = MF.getRegInfo();
171 for (MachineOperand &MO : MRI.use_nodbg_operands(CurLI->reg()))
172 if (!MO.isUndef())
173 UseSlots.push_back(LIS.getInstructionIndex(*MO.getParent()).getRegSlot());
174
175 array_pod_sort(UseSlots.begin(), UseSlots.end());
176
177 // Remove duplicates, keeping the smaller slot for each instruction.
178 // That is what we want for early clobbers.
179 UseSlots.erase(std::unique(UseSlots.begin(), UseSlots.end(),
180 SlotIndex::isSameInstr),
181 UseSlots.end());
182
183 // Compute per-live block info.
184 if (!calcLiveBlockInfo()) {
185 // FIXME: calcLiveBlockInfo found inconsistencies in the live range.
186 // I am looking at you, RegisterCoalescer!
187 DidRepairRange = true;
188 ++NumRepairs;
189 LLVM_DEBUG(dbgs() << "*** Fixing inconsistent live interval! ***\n");
190 const_cast<LiveIntervals&>(LIS)
191 .shrinkToUses(const_cast<LiveInterval*>(CurLI));
192 UseBlocks.clear();
193 ThroughBlocks.clear();
194 bool fixed = calcLiveBlockInfo();
195 (void)fixed;
196 assert(fixed && "Couldn't fix broken live interval");
197 }
198
199 LLVM_DEBUG(dbgs() << "Analyze counted " << UseSlots.size() << " instrs in "
200 << UseBlocks.size() << " blocks, through "
201 << NumThroughBlocks << " blocks.\n");
202 }
203
204 /// calcLiveBlockInfo - Fill the LiveBlocks array with information about blocks
205 /// where CurLI is live.
calcLiveBlockInfo()206 bool SplitAnalysis::calcLiveBlockInfo() {
207 ThroughBlocks.resize(MF.getNumBlockIDs());
208 NumThroughBlocks = NumGapBlocks = 0;
209 if (CurLI->empty())
210 return true;
211
212 LiveInterval::const_iterator LVI = CurLI->begin();
213 LiveInterval::const_iterator LVE = CurLI->end();
214
215 SmallVectorImpl<SlotIndex>::const_iterator UseI, UseE;
216 UseI = UseSlots.begin();
217 UseE = UseSlots.end();
218
219 // Loop over basic blocks where CurLI is live.
220 MachineFunction::iterator MFI =
221 LIS.getMBBFromIndex(LVI->start)->getIterator();
222 while (true) {
223 BlockInfo BI;
224 BI.MBB = &*MFI;
225 SlotIndex Start, Stop;
226 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB);
227
228 // If the block contains no uses, the range must be live through. At one
229 // point, RegisterCoalescer could create dangling ranges that ended
230 // mid-block.
231 if (UseI == UseE || *UseI >= Stop) {
232 ++NumThroughBlocks;
233 ThroughBlocks.set(BI.MBB->getNumber());
234 // The range shouldn't end mid-block if there are no uses. This shouldn't
235 // happen.
236 if (LVI->end < Stop)
237 return false;
238 } else {
239 // This block has uses. Find the first and last uses in the block.
240 BI.FirstInstr = *UseI;
241 assert(BI.FirstInstr >= Start);
242 do ++UseI;
243 while (UseI != UseE && *UseI < Stop);
244 BI.LastInstr = UseI[-1];
245 assert(BI.LastInstr < Stop);
246
247 // LVI is the first live segment overlapping MBB.
248 BI.LiveIn = LVI->start <= Start;
249
250 // When not live in, the first use should be a def.
251 if (!BI.LiveIn) {
252 assert(LVI->start == LVI->valno->def && "Dangling Segment start");
253 assert(LVI->start == BI.FirstInstr && "First instr should be a def");
254 BI.FirstDef = BI.FirstInstr;
255 }
256
257 // Look for gaps in the live range.
258 BI.LiveOut = true;
259 while (LVI->end < Stop) {
260 SlotIndex LastStop = LVI->end;
261 if (++LVI == LVE || LVI->start >= Stop) {
262 BI.LiveOut = false;
263 BI.LastInstr = LastStop;
264 break;
265 }
266
267 if (LastStop < LVI->start) {
268 // There is a gap in the live range. Create duplicate entries for the
269 // live-in snippet and the live-out snippet.
270 ++NumGapBlocks;
271
272 // Push the Live-in part.
273 BI.LiveOut = false;
274 UseBlocks.push_back(BI);
275 UseBlocks.back().LastInstr = LastStop;
276
277 // Set up BI for the live-out part.
278 BI.LiveIn = false;
279 BI.LiveOut = true;
280 BI.FirstInstr = BI.FirstDef = LVI->start;
281 }
282
283 // A Segment that starts in the middle of the block must be a def.
284 assert(LVI->start == LVI->valno->def && "Dangling Segment start");
285 if (!BI.FirstDef)
286 BI.FirstDef = LVI->start;
287 }
288
289 UseBlocks.push_back(BI);
290
291 // LVI is now at LVE or LVI->end >= Stop.
292 if (LVI == LVE)
293 break;
294 }
295
296 // Live segment ends exactly at Stop. Move to the next segment.
297 if (LVI->end == Stop && ++LVI == LVE)
298 break;
299
300 // Pick the next basic block.
301 if (LVI->start < Stop)
302 ++MFI;
303 else
304 MFI = LIS.getMBBFromIndex(LVI->start)->getIterator();
305 }
306
307 assert(getNumLiveBlocks() == countLiveBlocks(CurLI) && "Bad block count");
308 return true;
309 }
310
countLiveBlocks(const LiveInterval * cli) const311 unsigned SplitAnalysis::countLiveBlocks(const LiveInterval *cli) const {
312 if (cli->empty())
313 return 0;
314 LiveInterval *li = const_cast<LiveInterval*>(cli);
315 LiveInterval::iterator LVI = li->begin();
316 LiveInterval::iterator LVE = li->end();
317 unsigned Count = 0;
318
319 // Loop over basic blocks where li is live.
320 MachineFunction::const_iterator MFI =
321 LIS.getMBBFromIndex(LVI->start)->getIterator();
322 SlotIndex Stop = LIS.getMBBEndIdx(&*MFI);
323 while (true) {
324 ++Count;
325 LVI = li->advanceTo(LVI, Stop);
326 if (LVI == LVE)
327 return Count;
328 do {
329 ++MFI;
330 Stop = LIS.getMBBEndIdx(&*MFI);
331 } while (Stop <= LVI->start);
332 }
333 }
334
isOriginalEndpoint(SlotIndex Idx) const335 bool SplitAnalysis::isOriginalEndpoint(SlotIndex Idx) const {
336 unsigned OrigReg = VRM.getOriginal(CurLI->reg());
337 const LiveInterval &Orig = LIS.getInterval(OrigReg);
338 assert(!Orig.empty() && "Splitting empty interval?");
339 LiveInterval::const_iterator I = Orig.find(Idx);
340
341 // Range containing Idx should begin at Idx.
342 if (I != Orig.end() && I->start <= Idx)
343 return I->start == Idx;
344
345 // Range does not contain Idx, previous must end at Idx.
346 return I != Orig.begin() && (--I)->end == Idx;
347 }
348
analyze(const LiveInterval * li)349 void SplitAnalysis::analyze(const LiveInterval *li) {
350 clear();
351 CurLI = li;
352 analyzeUses();
353 }
354
355 //===----------------------------------------------------------------------===//
356 // Split Editor
357 //===----------------------------------------------------------------------===//
358
359 /// Create a new SplitEditor for editing the LiveInterval analyzed by SA.
SplitEditor(SplitAnalysis & sa,AliasAnalysis & aa,LiveIntervals & lis,VirtRegMap & vrm,MachineDominatorTree & mdt,MachineBlockFrequencyInfo & mbfi)360 SplitEditor::SplitEditor(SplitAnalysis &sa, AliasAnalysis &aa,
361 LiveIntervals &lis, VirtRegMap &vrm,
362 MachineDominatorTree &mdt,
363 MachineBlockFrequencyInfo &mbfi)
364 : SA(sa), AA(aa), LIS(lis), VRM(vrm),
365 MRI(vrm.getMachineFunction().getRegInfo()), MDT(mdt),
366 TII(*vrm.getMachineFunction().getSubtarget().getInstrInfo()),
367 TRI(*vrm.getMachineFunction().getSubtarget().getRegisterInfo()),
368 MBFI(mbfi), RegAssign(Allocator) {}
369
reset(LiveRangeEdit & LRE,ComplementSpillMode SM)370 void SplitEditor::reset(LiveRangeEdit &LRE, ComplementSpillMode SM) {
371 Edit = &LRE;
372 SpillMode = SM;
373 OpenIdx = 0;
374 RegAssign.clear();
375 Values.clear();
376
377 // Reset the LiveIntervalCalc instances needed for this spill mode.
378 LICalc[0].reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
379 &LIS.getVNInfoAllocator());
380 if (SpillMode)
381 LICalc[1].reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
382 &LIS.getVNInfoAllocator());
383
384 // We don't need an AliasAnalysis since we will only be performing
385 // cheap-as-a-copy remats anyway.
386 Edit->anyRematerializable(nullptr);
387 }
388
389 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
dump() const390 LLVM_DUMP_METHOD void SplitEditor::dump() const {
391 if (RegAssign.empty()) {
392 dbgs() << " empty\n";
393 return;
394 }
395
396 for (RegAssignMap::const_iterator I = RegAssign.begin(); I.valid(); ++I)
397 dbgs() << " [" << I.start() << ';' << I.stop() << "):" << I.value();
398 dbgs() << '\n';
399 }
400 #endif
401
getSubRangeForMaskExact(LaneBitmask LM,LiveInterval & LI)402 LiveInterval::SubRange &SplitEditor::getSubRangeForMaskExact(LaneBitmask LM,
403 LiveInterval &LI) {
404 for (LiveInterval::SubRange &S : LI.subranges())
405 if (S.LaneMask == LM)
406 return S;
407 llvm_unreachable("SubRange for this mask not found");
408 }
409
getSubRangeForMask(LaneBitmask LM,LiveInterval & LI)410 LiveInterval::SubRange &SplitEditor::getSubRangeForMask(LaneBitmask LM,
411 LiveInterval &LI) {
412 for (LiveInterval::SubRange &S : LI.subranges())
413 if ((S.LaneMask & LM) == LM)
414 return S;
415 llvm_unreachable("SubRange for this mask not found");
416 }
417
addDeadDef(LiveInterval & LI,VNInfo * VNI,bool Original)418 void SplitEditor::addDeadDef(LiveInterval &LI, VNInfo *VNI, bool Original) {
419 if (!LI.hasSubRanges()) {
420 LI.createDeadDef(VNI);
421 return;
422 }
423
424 SlotIndex Def = VNI->def;
425 if (Original) {
426 // If we are transferring a def from the original interval, make sure
427 // to only update the subranges for which the original subranges had
428 // a def at this location.
429 for (LiveInterval::SubRange &S : LI.subranges()) {
430 auto &PS = getSubRangeForMask(S.LaneMask, Edit->getParent());
431 VNInfo *PV = PS.getVNInfoAt(Def);
432 if (PV != nullptr && PV->def == Def)
433 S.createDeadDef(Def, LIS.getVNInfoAllocator());
434 }
435 } else {
436 // This is a new def: either from rematerialization, or from an inserted
437 // copy. Since rematerialization can regenerate a definition of a sub-
438 // register, we need to check which subranges need to be updated.
439 const MachineInstr *DefMI = LIS.getInstructionFromIndex(Def);
440 assert(DefMI != nullptr);
441 LaneBitmask LM;
442 for (const MachineOperand &DefOp : DefMI->defs()) {
443 Register R = DefOp.getReg();
444 if (R != LI.reg())
445 continue;
446 if (unsigned SR = DefOp.getSubReg())
447 LM |= TRI.getSubRegIndexLaneMask(SR);
448 else {
449 LM = MRI.getMaxLaneMaskForVReg(R);
450 break;
451 }
452 }
453 for (LiveInterval::SubRange &S : LI.subranges())
454 if ((S.LaneMask & LM).any())
455 S.createDeadDef(Def, LIS.getVNInfoAllocator());
456 }
457 }
458
defValue(unsigned RegIdx,const VNInfo * ParentVNI,SlotIndex Idx,bool Original)459 VNInfo *SplitEditor::defValue(unsigned RegIdx,
460 const VNInfo *ParentVNI,
461 SlotIndex Idx,
462 bool Original) {
463 assert(ParentVNI && "Mapping NULL value");
464 assert(Idx.isValid() && "Invalid SlotIndex");
465 assert(Edit->getParent().getVNInfoAt(Idx) == ParentVNI && "Bad Parent VNI");
466 LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx));
467
468 // Create a new value.
469 VNInfo *VNI = LI->getNextValue(Idx, LIS.getVNInfoAllocator());
470
471 bool Force = LI->hasSubRanges();
472 ValueForcePair FP(Force ? nullptr : VNI, Force);
473 // Use insert for lookup, so we can add missing values with a second lookup.
474 std::pair<ValueMap::iterator, bool> InsP =
475 Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id), FP));
476
477 // This was the first time (RegIdx, ParentVNI) was mapped, and it is not
478 // forced. Keep it as a simple def without any liveness.
479 if (!Force && InsP.second)
480 return VNI;
481
482 // If the previous value was a simple mapping, add liveness for it now.
483 if (VNInfo *OldVNI = InsP.first->second.getPointer()) {
484 addDeadDef(*LI, OldVNI, Original);
485
486 // No longer a simple mapping. Switch to a complex mapping. If the
487 // interval has subranges, make it a forced mapping.
488 InsP.first->second = ValueForcePair(nullptr, Force);
489 }
490
491 // This is a complex mapping, add liveness for VNI
492 addDeadDef(*LI, VNI, Original);
493 return VNI;
494 }
495
forceRecompute(unsigned RegIdx,const VNInfo & ParentVNI)496 void SplitEditor::forceRecompute(unsigned RegIdx, const VNInfo &ParentVNI) {
497 ValueForcePair &VFP = Values[std::make_pair(RegIdx, ParentVNI.id)];
498 VNInfo *VNI = VFP.getPointer();
499
500 // ParentVNI was either unmapped or already complex mapped. Either way, just
501 // set the force bit.
502 if (!VNI) {
503 VFP.setInt(true);
504 return;
505 }
506
507 // This was previously a single mapping. Make sure the old def is represented
508 // by a trivial live range.
509 addDeadDef(LIS.getInterval(Edit->get(RegIdx)), VNI, false);
510
511 // Mark as complex mapped, forced.
512 VFP = ValueForcePair(nullptr, true);
513 }
514
buildSingleSubRegCopy(Register FromReg,Register ToReg,MachineBasicBlock & MBB,MachineBasicBlock::iterator InsertBefore,unsigned SubIdx,LiveInterval & DestLI,bool Late,SlotIndex Def)515 SlotIndex SplitEditor::buildSingleSubRegCopy(Register FromReg, Register ToReg,
516 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
517 unsigned SubIdx, LiveInterval &DestLI, bool Late, SlotIndex Def) {
518 const MCInstrDesc &Desc = TII.get(TargetOpcode::COPY);
519 bool FirstCopy = !Def.isValid();
520 MachineInstr *CopyMI = BuildMI(MBB, InsertBefore, DebugLoc(), Desc)
521 .addReg(ToReg, RegState::Define | getUndefRegState(FirstCopy)
522 | getInternalReadRegState(!FirstCopy), SubIdx)
523 .addReg(FromReg, 0, SubIdx);
524
525 BumpPtrAllocator &Allocator = LIS.getVNInfoAllocator();
526 SlotIndexes &Indexes = *LIS.getSlotIndexes();
527 if (FirstCopy) {
528 Def = Indexes.insertMachineInstrInMaps(*CopyMI, Late).getRegSlot();
529 } else {
530 CopyMI->bundleWithPred();
531 }
532 LaneBitmask LaneMask = TRI.getSubRegIndexLaneMask(SubIdx);
533 DestLI.refineSubRanges(Allocator, LaneMask,
534 [Def, &Allocator](LiveInterval::SubRange &SR) {
535 SR.createDeadDef(Def, Allocator);
536 },
537 Indexes, TRI);
538 return Def;
539 }
540
buildCopy(Register FromReg,Register ToReg,LaneBitmask LaneMask,MachineBasicBlock & MBB,MachineBasicBlock::iterator InsertBefore,bool Late,unsigned RegIdx)541 SlotIndex SplitEditor::buildCopy(Register FromReg, Register ToReg,
542 LaneBitmask LaneMask, MachineBasicBlock &MBB,
543 MachineBasicBlock::iterator InsertBefore, bool Late, unsigned RegIdx) {
544 const MCInstrDesc &Desc = TII.get(TargetOpcode::COPY);
545 if (LaneMask.all() || LaneMask == MRI.getMaxLaneMaskForVReg(FromReg)) {
546 // The full vreg is copied.
547 MachineInstr *CopyMI =
548 BuildMI(MBB, InsertBefore, DebugLoc(), Desc, ToReg).addReg(FromReg);
549 SlotIndexes &Indexes = *LIS.getSlotIndexes();
550 return Indexes.insertMachineInstrInMaps(*CopyMI, Late).getRegSlot();
551 }
552
553 // Only a subset of lanes needs to be copied. The following is a simple
554 // heuristic to construct a sequence of COPYs. We could add a target
555 // specific callback if this turns out to be suboptimal.
556 LiveInterval &DestLI = LIS.getInterval(Edit->get(RegIdx));
557
558 // First pass: Try to find a perfectly matching subregister index. If none
559 // exists find the one covering the most lanemask bits.
560 SmallVector<unsigned, 8> PossibleIndexes;
561 unsigned BestIdx = 0;
562 unsigned BestCover = 0;
563 const TargetRegisterClass *RC = MRI.getRegClass(FromReg);
564 assert(RC == MRI.getRegClass(ToReg) && "Should have same reg class");
565 for (unsigned Idx = 1, E = TRI.getNumSubRegIndices(); Idx < E; ++Idx) {
566 // Is this index even compatible with the given class?
567 if (TRI.getSubClassWithSubReg(RC, Idx) != RC)
568 continue;
569 LaneBitmask SubRegMask = TRI.getSubRegIndexLaneMask(Idx);
570 // Early exit if we found a perfect match.
571 if (SubRegMask == LaneMask) {
572 BestIdx = Idx;
573 break;
574 }
575
576 // The index must not cover any lanes outside \p LaneMask.
577 if ((SubRegMask & ~LaneMask).any())
578 continue;
579
580 unsigned PopCount = SubRegMask.getNumLanes();
581 PossibleIndexes.push_back(Idx);
582 if (PopCount > BestCover) {
583 BestCover = PopCount;
584 BestIdx = Idx;
585 }
586 }
587
588 // Abort if we cannot possibly implement the COPY with the given indexes.
589 if (BestIdx == 0)
590 report_fatal_error("Impossible to implement partial COPY");
591
592 SlotIndex Def = buildSingleSubRegCopy(FromReg, ToReg, MBB, InsertBefore,
593 BestIdx, DestLI, Late, SlotIndex());
594
595 // Greedy heuristic: Keep iterating keeping the best covering subreg index
596 // each time.
597 LaneBitmask LanesLeft = LaneMask & ~(TRI.getSubRegIndexLaneMask(BestIdx));
598 while (LanesLeft.any()) {
599 unsigned BestIdx = 0;
600 int BestCover = std::numeric_limits<int>::min();
601 for (unsigned Idx : PossibleIndexes) {
602 LaneBitmask SubRegMask = TRI.getSubRegIndexLaneMask(Idx);
603 // Early exit if we found a perfect match.
604 if (SubRegMask == LanesLeft) {
605 BestIdx = Idx;
606 break;
607 }
608
609 // Try to cover as much of the remaining lanes as possible but
610 // as few of the already covered lanes as possible.
611 int Cover = (SubRegMask & LanesLeft).getNumLanes()
612 - (SubRegMask & ~LanesLeft).getNumLanes();
613 if (Cover > BestCover) {
614 BestCover = Cover;
615 BestIdx = Idx;
616 }
617 }
618
619 if (BestIdx == 0)
620 report_fatal_error("Impossible to implement partial COPY");
621
622 buildSingleSubRegCopy(FromReg, ToReg, MBB, InsertBefore, BestIdx,
623 DestLI, Late, Def);
624 LanesLeft &= ~TRI.getSubRegIndexLaneMask(BestIdx);
625 }
626
627 return Def;
628 }
629
defFromParent(unsigned RegIdx,VNInfo * ParentVNI,SlotIndex UseIdx,MachineBasicBlock & MBB,MachineBasicBlock::iterator I)630 VNInfo *SplitEditor::defFromParent(unsigned RegIdx,
631 VNInfo *ParentVNI,
632 SlotIndex UseIdx,
633 MachineBasicBlock &MBB,
634 MachineBasicBlock::iterator I) {
635 SlotIndex Def;
636 LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx));
637
638 // We may be trying to avoid interference that ends at a deleted instruction,
639 // so always begin RegIdx 0 early and all others late.
640 bool Late = RegIdx != 0;
641
642 // Attempt cheap-as-a-copy rematerialization.
643 unsigned Original = VRM.getOriginal(Edit->get(RegIdx));
644 LiveInterval &OrigLI = LIS.getInterval(Original);
645 VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx);
646
647 Register Reg = LI->reg();
648 bool DidRemat = false;
649 if (OrigVNI) {
650 LiveRangeEdit::Remat RM(ParentVNI);
651 RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def);
652 if (Edit->canRematerializeAt(RM, OrigVNI, UseIdx, true)) {
653 Def = Edit->rematerializeAt(MBB, I, Reg, RM, TRI, Late);
654 ++NumRemats;
655 DidRemat = true;
656 }
657 }
658 if (!DidRemat) {
659 LaneBitmask LaneMask;
660 if (OrigLI.hasSubRanges()) {
661 LaneMask = LaneBitmask::getNone();
662 for (LiveInterval::SubRange &S : OrigLI.subranges()) {
663 if (S.liveAt(UseIdx))
664 LaneMask |= S.LaneMask;
665 }
666 } else {
667 LaneMask = LaneBitmask::getAll();
668 }
669
670 if (LaneMask.none()) {
671 const MCInstrDesc &Desc = TII.get(TargetOpcode::IMPLICIT_DEF);
672 MachineInstr *ImplicitDef = BuildMI(MBB, I, DebugLoc(), Desc, Reg);
673 SlotIndexes &Indexes = *LIS.getSlotIndexes();
674 Def = Indexes.insertMachineInstrInMaps(*ImplicitDef, Late).getRegSlot();
675 } else {
676 ++NumCopies;
677 Def = buildCopy(Edit->getReg(), Reg, LaneMask, MBB, I, Late, RegIdx);
678 }
679 }
680
681 // Define the value in Reg.
682 return defValue(RegIdx, ParentVNI, Def, false);
683 }
684
685 /// Create a new virtual register and live interval.
openIntv()686 unsigned SplitEditor::openIntv() {
687 // Create the complement as index 0.
688 if (Edit->empty())
689 Edit->createEmptyInterval();
690
691 // Create the open interval.
692 OpenIdx = Edit->size();
693 Edit->createEmptyInterval();
694 return OpenIdx;
695 }
696
selectIntv(unsigned Idx)697 void SplitEditor::selectIntv(unsigned Idx) {
698 assert(Idx != 0 && "Cannot select the complement interval");
699 assert(Idx < Edit->size() && "Can only select previously opened interval");
700 LLVM_DEBUG(dbgs() << " selectIntv " << OpenIdx << " -> " << Idx << '\n');
701 OpenIdx = Idx;
702 }
703
enterIntvBefore(SlotIndex Idx)704 SlotIndex SplitEditor::enterIntvBefore(SlotIndex Idx) {
705 assert(OpenIdx && "openIntv not called before enterIntvBefore");
706 LLVM_DEBUG(dbgs() << " enterIntvBefore " << Idx);
707 Idx = Idx.getBaseIndex();
708 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx);
709 if (!ParentVNI) {
710 LLVM_DEBUG(dbgs() << ": not live\n");
711 return Idx;
712 }
713 LLVM_DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
714 MachineInstr *MI = LIS.getInstructionFromIndex(Idx);
715 assert(MI && "enterIntvBefore called with invalid index");
716
717 VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Idx, *MI->getParent(), MI);
718 return VNI->def;
719 }
720
enterIntvAfter(SlotIndex Idx)721 SlotIndex SplitEditor::enterIntvAfter(SlotIndex Idx) {
722 assert(OpenIdx && "openIntv not called before enterIntvAfter");
723 LLVM_DEBUG(dbgs() << " enterIntvAfter " << Idx);
724 Idx = Idx.getBoundaryIndex();
725 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx);
726 if (!ParentVNI) {
727 LLVM_DEBUG(dbgs() << ": not live\n");
728 return Idx;
729 }
730 LLVM_DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
731 MachineInstr *MI = LIS.getInstructionFromIndex(Idx);
732 assert(MI && "enterIntvAfter called with invalid index");
733
734 VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Idx, *MI->getParent(),
735 std::next(MachineBasicBlock::iterator(MI)));
736 return VNI->def;
737 }
738
enterIntvAtEnd(MachineBasicBlock & MBB)739 SlotIndex SplitEditor::enterIntvAtEnd(MachineBasicBlock &MBB) {
740 assert(OpenIdx && "openIntv not called before enterIntvAtEnd");
741 SlotIndex End = LIS.getMBBEndIdx(&MBB);
742 SlotIndex Last = End.getPrevSlot();
743 LLVM_DEBUG(dbgs() << " enterIntvAtEnd " << printMBBReference(MBB) << ", "
744 << Last);
745 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Last);
746 if (!ParentVNI) {
747 LLVM_DEBUG(dbgs() << ": not live\n");
748 return End;
749 }
750 LLVM_DEBUG(dbgs() << ": valno " << ParentVNI->id);
751 VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Last, MBB,
752 SA.getLastSplitPointIter(&MBB));
753 RegAssign.insert(VNI->def, End, OpenIdx);
754 LLVM_DEBUG(dump());
755 return VNI->def;
756 }
757
758 /// useIntv - indicate that all instructions in MBB should use OpenLI.
useIntv(const MachineBasicBlock & MBB)759 void SplitEditor::useIntv(const MachineBasicBlock &MBB) {
760 useIntv(LIS.getMBBStartIdx(&MBB), LIS.getMBBEndIdx(&MBB));
761 }
762
useIntv(SlotIndex Start,SlotIndex End)763 void SplitEditor::useIntv(SlotIndex Start, SlotIndex End) {
764 assert(OpenIdx && "openIntv not called before useIntv");
765 LLVM_DEBUG(dbgs() << " useIntv [" << Start << ';' << End << "):");
766 RegAssign.insert(Start, End, OpenIdx);
767 LLVM_DEBUG(dump());
768 }
769
leaveIntvAfter(SlotIndex Idx)770 SlotIndex SplitEditor::leaveIntvAfter(SlotIndex Idx) {
771 assert(OpenIdx && "openIntv not called before leaveIntvAfter");
772 LLVM_DEBUG(dbgs() << " leaveIntvAfter " << Idx);
773
774 // The interval must be live beyond the instruction at Idx.
775 SlotIndex Boundary = Idx.getBoundaryIndex();
776 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Boundary);
777 if (!ParentVNI) {
778 LLVM_DEBUG(dbgs() << ": not live\n");
779 return Boundary.getNextSlot();
780 }
781 LLVM_DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
782 MachineInstr *MI = LIS.getInstructionFromIndex(Boundary);
783 assert(MI && "No instruction at index");
784
785 // In spill mode, make live ranges as short as possible by inserting the copy
786 // before MI. This is only possible if that instruction doesn't redefine the
787 // value. The inserted COPY is not a kill, and we don't need to recompute
788 // the source live range. The spiller also won't try to hoist this copy.
789 if (SpillMode && !SlotIndex::isSameInstr(ParentVNI->def, Idx) &&
790 MI->readsVirtualRegister(Edit->getReg())) {
791 forceRecompute(0, *ParentVNI);
792 defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI);
793 return Idx;
794 }
795
796 VNInfo *VNI = defFromParent(0, ParentVNI, Boundary, *MI->getParent(),
797 std::next(MachineBasicBlock::iterator(MI)));
798 return VNI->def;
799 }
800
leaveIntvBefore(SlotIndex Idx)801 SlotIndex SplitEditor::leaveIntvBefore(SlotIndex Idx) {
802 assert(OpenIdx && "openIntv not called before leaveIntvBefore");
803 LLVM_DEBUG(dbgs() << " leaveIntvBefore " << Idx);
804
805 // The interval must be live into the instruction at Idx.
806 Idx = Idx.getBaseIndex();
807 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx);
808 if (!ParentVNI) {
809 LLVM_DEBUG(dbgs() << ": not live\n");
810 return Idx.getNextSlot();
811 }
812 LLVM_DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
813
814 MachineInstr *MI = LIS.getInstructionFromIndex(Idx);
815 assert(MI && "No instruction at index");
816 VNInfo *VNI = defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI);
817 return VNI->def;
818 }
819
leaveIntvAtTop(MachineBasicBlock & MBB)820 SlotIndex SplitEditor::leaveIntvAtTop(MachineBasicBlock &MBB) {
821 assert(OpenIdx && "openIntv not called before leaveIntvAtTop");
822 SlotIndex Start = LIS.getMBBStartIdx(&MBB);
823 LLVM_DEBUG(dbgs() << " leaveIntvAtTop " << printMBBReference(MBB) << ", "
824 << Start);
825
826 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Start);
827 if (!ParentVNI) {
828 LLVM_DEBUG(dbgs() << ": not live\n");
829 return Start;
830 }
831
832 VNInfo *VNI = defFromParent(0, ParentVNI, Start, MBB,
833 MBB.SkipPHIsLabelsAndDebug(MBB.begin()));
834 RegAssign.insert(Start, VNI->def, OpenIdx);
835 LLVM_DEBUG(dump());
836 return VNI->def;
837 }
838
overlapIntv(SlotIndex Start,SlotIndex End)839 void SplitEditor::overlapIntv(SlotIndex Start, SlotIndex End) {
840 assert(OpenIdx && "openIntv not called before overlapIntv");
841 const VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Start);
842 assert(ParentVNI == Edit->getParent().getVNInfoBefore(End) &&
843 "Parent changes value in extended range");
844 assert(LIS.getMBBFromIndex(Start) == LIS.getMBBFromIndex(End) &&
845 "Range cannot span basic blocks");
846
847 // The complement interval will be extended as needed by LICalc.extend().
848 if (ParentVNI)
849 forceRecompute(0, *ParentVNI);
850 LLVM_DEBUG(dbgs() << " overlapIntv [" << Start << ';' << End << "):");
851 RegAssign.insert(Start, End, OpenIdx);
852 LLVM_DEBUG(dump());
853 }
854
855 //===----------------------------------------------------------------------===//
856 // Spill modes
857 //===----------------------------------------------------------------------===//
858
removeBackCopies(SmallVectorImpl<VNInfo * > & Copies)859 void SplitEditor::removeBackCopies(SmallVectorImpl<VNInfo*> &Copies) {
860 LiveInterval *LI = &LIS.getInterval(Edit->get(0));
861 LLVM_DEBUG(dbgs() << "Removing " << Copies.size() << " back-copies.\n");
862 RegAssignMap::iterator AssignI;
863 AssignI.setMap(RegAssign);
864
865 for (unsigned i = 0, e = Copies.size(); i != e; ++i) {
866 SlotIndex Def = Copies[i]->def;
867 MachineInstr *MI = LIS.getInstructionFromIndex(Def);
868 assert(MI && "No instruction for back-copy");
869
870 MachineBasicBlock *MBB = MI->getParent();
871 MachineBasicBlock::iterator MBBI(MI);
872 bool AtBegin;
873 do AtBegin = MBBI == MBB->begin();
874 while (!AtBegin && (--MBBI)->isDebugInstr());
875
876 LLVM_DEBUG(dbgs() << "Removing " << Def << '\t' << *MI);
877 LIS.removeVRegDefAt(*LI, Def);
878 LIS.RemoveMachineInstrFromMaps(*MI);
879 MI->eraseFromParent();
880
881 // Adjust RegAssign if a register assignment is killed at Def. We want to
882 // avoid calculating the live range of the source register if possible.
883 AssignI.find(Def.getPrevSlot());
884 if (!AssignI.valid() || AssignI.start() >= Def)
885 continue;
886 // If MI doesn't kill the assigned register, just leave it.
887 if (AssignI.stop() != Def)
888 continue;
889 unsigned RegIdx = AssignI.value();
890 if (AtBegin || !MBBI->readsVirtualRegister(Edit->getReg())) {
891 LLVM_DEBUG(dbgs() << " cannot find simple kill of RegIdx " << RegIdx
892 << '\n');
893 forceRecompute(RegIdx, *Edit->getParent().getVNInfoAt(Def));
894 } else {
895 SlotIndex Kill = LIS.getInstructionIndex(*MBBI).getRegSlot();
896 LLVM_DEBUG(dbgs() << " move kill to " << Kill << '\t' << *MBBI);
897 AssignI.setStop(Kill);
898 }
899 }
900 }
901
902 MachineBasicBlock*
findShallowDominator(MachineBasicBlock * MBB,MachineBasicBlock * DefMBB)903 SplitEditor::findShallowDominator(MachineBasicBlock *MBB,
904 MachineBasicBlock *DefMBB) {
905 if (MBB == DefMBB)
906 return MBB;
907 assert(MDT.dominates(DefMBB, MBB) && "MBB must be dominated by the def.");
908
909 const MachineLoopInfo &Loops = SA.Loops;
910 const MachineLoop *DefLoop = Loops.getLoopFor(DefMBB);
911 MachineDomTreeNode *DefDomNode = MDT[DefMBB];
912
913 // Best candidate so far.
914 MachineBasicBlock *BestMBB = MBB;
915 unsigned BestDepth = std::numeric_limits<unsigned>::max();
916
917 while (true) {
918 const MachineLoop *Loop = Loops.getLoopFor(MBB);
919
920 // MBB isn't in a loop, it doesn't get any better. All dominators have a
921 // higher frequency by definition.
922 if (!Loop) {
923 LLVM_DEBUG(dbgs() << "Def in " << printMBBReference(*DefMBB)
924 << " dominates " << printMBBReference(*MBB)
925 << " at depth 0\n");
926 return MBB;
927 }
928
929 // We'll never be able to exit the DefLoop.
930 if (Loop == DefLoop) {
931 LLVM_DEBUG(dbgs() << "Def in " << printMBBReference(*DefMBB)
932 << " dominates " << printMBBReference(*MBB)
933 << " in the same loop\n");
934 return MBB;
935 }
936
937 // Least busy dominator seen so far.
938 unsigned Depth = Loop->getLoopDepth();
939 if (Depth < BestDepth) {
940 BestMBB = MBB;
941 BestDepth = Depth;
942 LLVM_DEBUG(dbgs() << "Def in " << printMBBReference(*DefMBB)
943 << " dominates " << printMBBReference(*MBB)
944 << " at depth " << Depth << '\n');
945 }
946
947 // Leave loop by going to the immediate dominator of the loop header.
948 // This is a bigger stride than simply walking up the dominator tree.
949 MachineDomTreeNode *IDom = MDT[Loop->getHeader()]->getIDom();
950
951 // Too far up the dominator tree?
952 if (!IDom || !MDT.dominates(DefDomNode, IDom))
953 return BestMBB;
954
955 MBB = IDom->getBlock();
956 }
957 }
958
computeRedundantBackCopies(DenseSet<unsigned> & NotToHoistSet,SmallVectorImpl<VNInfo * > & BackCopies)959 void SplitEditor::computeRedundantBackCopies(
960 DenseSet<unsigned> &NotToHoistSet, SmallVectorImpl<VNInfo *> &BackCopies) {
961 LiveInterval *LI = &LIS.getInterval(Edit->get(0));
962 LiveInterval *Parent = &Edit->getParent();
963 SmallVector<SmallPtrSet<VNInfo *, 8>, 8> EqualVNs(Parent->getNumValNums());
964 SmallPtrSet<VNInfo *, 8> DominatedVNIs;
965
966 // Aggregate VNIs having the same value as ParentVNI.
967 for (VNInfo *VNI : LI->valnos) {
968 if (VNI->isUnused())
969 continue;
970 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def);
971 EqualVNs[ParentVNI->id].insert(VNI);
972 }
973
974 // For VNI aggregation of each ParentVNI, collect dominated, i.e.,
975 // redundant VNIs to BackCopies.
976 for (unsigned i = 0, e = Parent->getNumValNums(); i != e; ++i) {
977 VNInfo *ParentVNI = Parent->getValNumInfo(i);
978 if (!NotToHoistSet.count(ParentVNI->id))
979 continue;
980 SmallPtrSetIterator<VNInfo *> It1 = EqualVNs[ParentVNI->id].begin();
981 SmallPtrSetIterator<VNInfo *> It2 = It1;
982 for (; It1 != EqualVNs[ParentVNI->id].end(); ++It1) {
983 It2 = It1;
984 for (++It2; It2 != EqualVNs[ParentVNI->id].end(); ++It2) {
985 if (DominatedVNIs.count(*It1) || DominatedVNIs.count(*It2))
986 continue;
987
988 MachineBasicBlock *MBB1 = LIS.getMBBFromIndex((*It1)->def);
989 MachineBasicBlock *MBB2 = LIS.getMBBFromIndex((*It2)->def);
990 if (MBB1 == MBB2) {
991 DominatedVNIs.insert((*It1)->def < (*It2)->def ? (*It2) : (*It1));
992 } else if (MDT.dominates(MBB1, MBB2)) {
993 DominatedVNIs.insert(*It2);
994 } else if (MDT.dominates(MBB2, MBB1)) {
995 DominatedVNIs.insert(*It1);
996 }
997 }
998 }
999 if (!DominatedVNIs.empty()) {
1000 forceRecompute(0, *ParentVNI);
1001 append_range(BackCopies, DominatedVNIs);
1002 DominatedVNIs.clear();
1003 }
1004 }
1005 }
1006
1007 /// For SM_Size mode, find a common dominator for all the back-copies for
1008 /// the same ParentVNI and hoist the backcopies to the dominator BB.
1009 /// For SM_Speed mode, if the common dominator is hot and it is not beneficial
1010 /// to do the hoisting, simply remove the dominated backcopies for the same
1011 /// ParentVNI.
hoistCopies()1012 void SplitEditor::hoistCopies() {
1013 // Get the complement interval, always RegIdx 0.
1014 LiveInterval *LI = &LIS.getInterval(Edit->get(0));
1015 LiveInterval *Parent = &Edit->getParent();
1016
1017 // Track the nearest common dominator for all back-copies for each ParentVNI,
1018 // indexed by ParentVNI->id.
1019 using DomPair = std::pair<MachineBasicBlock *, SlotIndex>;
1020 SmallVector<DomPair, 8> NearestDom(Parent->getNumValNums());
1021 // The total cost of all the back-copies for each ParentVNI.
1022 SmallVector<BlockFrequency, 8> Costs(Parent->getNumValNums());
1023 // The ParentVNI->id set for which hoisting back-copies are not beneficial
1024 // for Speed.
1025 DenseSet<unsigned> NotToHoistSet;
1026
1027 // Find the nearest common dominator for parent values with multiple
1028 // back-copies. If a single back-copy dominates, put it in DomPair.second.
1029 for (VNInfo *VNI : LI->valnos) {
1030 if (VNI->isUnused())
1031 continue;
1032 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def);
1033 assert(ParentVNI && "Parent not live at complement def");
1034
1035 // Don't hoist remats. The complement is probably going to disappear
1036 // completely anyway.
1037 if (Edit->didRematerialize(ParentVNI))
1038 continue;
1039
1040 MachineBasicBlock *ValMBB = LIS.getMBBFromIndex(VNI->def);
1041
1042 DomPair &Dom = NearestDom[ParentVNI->id];
1043
1044 // Keep directly defined parent values. This is either a PHI or an
1045 // instruction in the complement range. All other copies of ParentVNI
1046 // should be eliminated.
1047 if (VNI->def == ParentVNI->def) {
1048 LLVM_DEBUG(dbgs() << "Direct complement def at " << VNI->def << '\n');
1049 Dom = DomPair(ValMBB, VNI->def);
1050 continue;
1051 }
1052 // Skip the singly mapped values. There is nothing to gain from hoisting a
1053 // single back-copy.
1054 if (Values.lookup(std::make_pair(0, ParentVNI->id)).getPointer()) {
1055 LLVM_DEBUG(dbgs() << "Single complement def at " << VNI->def << '\n');
1056 continue;
1057 }
1058
1059 if (!Dom.first) {
1060 // First time we see ParentVNI. VNI dominates itself.
1061 Dom = DomPair(ValMBB, VNI->def);
1062 } else if (Dom.first == ValMBB) {
1063 // Two defs in the same block. Pick the earlier def.
1064 if (!Dom.second.isValid() || VNI->def < Dom.second)
1065 Dom.second = VNI->def;
1066 } else {
1067 // Different basic blocks. Check if one dominates.
1068 MachineBasicBlock *Near =
1069 MDT.findNearestCommonDominator(Dom.first, ValMBB);
1070 if (Near == ValMBB)
1071 // Def ValMBB dominates.
1072 Dom = DomPair(ValMBB, VNI->def);
1073 else if (Near != Dom.first)
1074 // None dominate. Hoist to common dominator, need new def.
1075 Dom = DomPair(Near, SlotIndex());
1076 Costs[ParentVNI->id] += MBFI.getBlockFreq(ValMBB);
1077 }
1078
1079 LLVM_DEBUG(dbgs() << "Multi-mapped complement " << VNI->id << '@'
1080 << VNI->def << " for parent " << ParentVNI->id << '@'
1081 << ParentVNI->def << " hoist to "
1082 << printMBBReference(*Dom.first) << ' ' << Dom.second
1083 << '\n');
1084 }
1085
1086 // Insert the hoisted copies.
1087 for (unsigned i = 0, e = Parent->getNumValNums(); i != e; ++i) {
1088 DomPair &Dom = NearestDom[i];
1089 if (!Dom.first || Dom.second.isValid())
1090 continue;
1091 // This value needs a hoisted copy inserted at the end of Dom.first.
1092 VNInfo *ParentVNI = Parent->getValNumInfo(i);
1093 MachineBasicBlock *DefMBB = LIS.getMBBFromIndex(ParentVNI->def);
1094 // Get a less loopy dominator than Dom.first.
1095 Dom.first = findShallowDominator(Dom.first, DefMBB);
1096 if (SpillMode == SM_Speed &&
1097 MBFI.getBlockFreq(Dom.first) > Costs[ParentVNI->id]) {
1098 NotToHoistSet.insert(ParentVNI->id);
1099 continue;
1100 }
1101 SlotIndex Last = LIS.getMBBEndIdx(Dom.first).getPrevSlot();
1102 Dom.second =
1103 defFromParent(0, ParentVNI, Last, *Dom.first,
1104 SA.getLastSplitPointIter(Dom.first))->def;
1105 }
1106
1107 // Remove redundant back-copies that are now known to be dominated by another
1108 // def with the same value.
1109 SmallVector<VNInfo*, 8> BackCopies;
1110 for (VNInfo *VNI : LI->valnos) {
1111 if (VNI->isUnused())
1112 continue;
1113 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def);
1114 const DomPair &Dom = NearestDom[ParentVNI->id];
1115 if (!Dom.first || Dom.second == VNI->def ||
1116 NotToHoistSet.count(ParentVNI->id))
1117 continue;
1118 BackCopies.push_back(VNI);
1119 forceRecompute(0, *ParentVNI);
1120 }
1121
1122 // If it is not beneficial to hoist all the BackCopies, simply remove
1123 // redundant BackCopies in speed mode.
1124 if (SpillMode == SM_Speed && !NotToHoistSet.empty())
1125 computeRedundantBackCopies(NotToHoistSet, BackCopies);
1126
1127 removeBackCopies(BackCopies);
1128 }
1129
1130 /// transferValues - Transfer all possible values to the new live ranges.
1131 /// Values that were rematerialized are left alone, they need LICalc.extend().
transferValues()1132 bool SplitEditor::transferValues() {
1133 bool Skipped = false;
1134 RegAssignMap::const_iterator AssignI = RegAssign.begin();
1135 for (const LiveRange::Segment &S : Edit->getParent()) {
1136 LLVM_DEBUG(dbgs() << " blit " << S << ':');
1137 VNInfo *ParentVNI = S.valno;
1138 // RegAssign has holes where RegIdx 0 should be used.
1139 SlotIndex Start = S.start;
1140 AssignI.advanceTo(Start);
1141 do {
1142 unsigned RegIdx;
1143 SlotIndex End = S.end;
1144 if (!AssignI.valid()) {
1145 RegIdx = 0;
1146 } else if (AssignI.start() <= Start) {
1147 RegIdx = AssignI.value();
1148 if (AssignI.stop() < End) {
1149 End = AssignI.stop();
1150 ++AssignI;
1151 }
1152 } else {
1153 RegIdx = 0;
1154 End = std::min(End, AssignI.start());
1155 }
1156
1157 // The interval [Start;End) is continuously mapped to RegIdx, ParentVNI.
1158 LLVM_DEBUG(dbgs() << " [" << Start << ';' << End << ")=" << RegIdx << '('
1159 << printReg(Edit->get(RegIdx)) << ')');
1160 LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
1161
1162 // Check for a simply defined value that can be blitted directly.
1163 ValueForcePair VFP = Values.lookup(std::make_pair(RegIdx, ParentVNI->id));
1164 if (VNInfo *VNI = VFP.getPointer()) {
1165 LLVM_DEBUG(dbgs() << ':' << VNI->id);
1166 LI.addSegment(LiveInterval::Segment(Start, End, VNI));
1167 Start = End;
1168 continue;
1169 }
1170
1171 // Skip values with forced recomputation.
1172 if (VFP.getInt()) {
1173 LLVM_DEBUG(dbgs() << "(recalc)");
1174 Skipped = true;
1175 Start = End;
1176 continue;
1177 }
1178
1179 LiveIntervalCalc &LIC = getLICalc(RegIdx);
1180
1181 // This value has multiple defs in RegIdx, but it wasn't rematerialized,
1182 // so the live range is accurate. Add live-in blocks in [Start;End) to the
1183 // LiveInBlocks.
1184 MachineFunction::iterator MBB = LIS.getMBBFromIndex(Start)->getIterator();
1185 SlotIndex BlockStart, BlockEnd;
1186 std::tie(BlockStart, BlockEnd) = LIS.getSlotIndexes()->getMBBRange(&*MBB);
1187
1188 // The first block may be live-in, or it may have its own def.
1189 if (Start != BlockStart) {
1190 VNInfo *VNI = LI.extendInBlock(BlockStart, std::min(BlockEnd, End));
1191 assert(VNI && "Missing def for complex mapped value");
1192 LLVM_DEBUG(dbgs() << ':' << VNI->id << "*" << printMBBReference(*MBB));
1193 // MBB has its own def. Is it also live-out?
1194 if (BlockEnd <= End)
1195 LIC.setLiveOutValue(&*MBB, VNI);
1196
1197 // Skip to the next block for live-in.
1198 ++MBB;
1199 BlockStart = BlockEnd;
1200 }
1201
1202 // Handle the live-in blocks covered by [Start;End).
1203 assert(Start <= BlockStart && "Expected live-in block");
1204 while (BlockStart < End) {
1205 LLVM_DEBUG(dbgs() << ">" << printMBBReference(*MBB));
1206 BlockEnd = LIS.getMBBEndIdx(&*MBB);
1207 if (BlockStart == ParentVNI->def) {
1208 // This block has the def of a parent PHI, so it isn't live-in.
1209 assert(ParentVNI->isPHIDef() && "Non-phi defined at block start?");
1210 VNInfo *VNI = LI.extendInBlock(BlockStart, std::min(BlockEnd, End));
1211 assert(VNI && "Missing def for complex mapped parent PHI");
1212 if (End >= BlockEnd)
1213 LIC.setLiveOutValue(&*MBB, VNI); // Live-out as well.
1214 } else {
1215 // This block needs a live-in value. The last block covered may not
1216 // be live-out.
1217 if (End < BlockEnd)
1218 LIC.addLiveInBlock(LI, MDT[&*MBB], End);
1219 else {
1220 // Live-through, and we don't know the value.
1221 LIC.addLiveInBlock(LI, MDT[&*MBB]);
1222 LIC.setLiveOutValue(&*MBB, nullptr);
1223 }
1224 }
1225 BlockStart = BlockEnd;
1226 ++MBB;
1227 }
1228 Start = End;
1229 } while (Start != S.end);
1230 LLVM_DEBUG(dbgs() << '\n');
1231 }
1232
1233 LICalc[0].calculateValues();
1234 if (SpillMode)
1235 LICalc[1].calculateValues();
1236
1237 return Skipped;
1238 }
1239
removeDeadSegment(SlotIndex Def,LiveRange & LR)1240 static bool removeDeadSegment(SlotIndex Def, LiveRange &LR) {
1241 const LiveRange::Segment *Seg = LR.getSegmentContaining(Def);
1242 if (Seg == nullptr)
1243 return true;
1244 if (Seg->end != Def.getDeadSlot())
1245 return false;
1246 // This is a dead PHI. Remove it.
1247 LR.removeSegment(*Seg, true);
1248 return true;
1249 }
1250
extendPHIRange(MachineBasicBlock & B,LiveIntervalCalc & LIC,LiveRange & LR,LaneBitmask LM,ArrayRef<SlotIndex> Undefs)1251 void SplitEditor::extendPHIRange(MachineBasicBlock &B, LiveIntervalCalc &LIC,
1252 LiveRange &LR, LaneBitmask LM,
1253 ArrayRef<SlotIndex> Undefs) {
1254 for (MachineBasicBlock *P : B.predecessors()) {
1255 SlotIndex End = LIS.getMBBEndIdx(P);
1256 SlotIndex LastUse = End.getPrevSlot();
1257 // The predecessor may not have a live-out value. That is OK, like an
1258 // undef PHI operand.
1259 LiveInterval &PLI = Edit->getParent();
1260 // Need the cast because the inputs to ?: would otherwise be deemed
1261 // "incompatible": SubRange vs LiveInterval.
1262 LiveRange &PSR = !LM.all() ? getSubRangeForMaskExact(LM, PLI)
1263 : static_cast<LiveRange &>(PLI);
1264 if (PSR.liveAt(LastUse))
1265 LIC.extend(LR, End, /*PhysReg=*/0, Undefs);
1266 }
1267 }
1268
extendPHIKillRanges()1269 void SplitEditor::extendPHIKillRanges() {
1270 // Extend live ranges to be live-out for successor PHI values.
1271
1272 // Visit each PHI def slot in the parent live interval. If the def is dead,
1273 // remove it. Otherwise, extend the live interval to reach the end indexes
1274 // of all predecessor blocks.
1275
1276 LiveInterval &ParentLI = Edit->getParent();
1277 for (const VNInfo *V : ParentLI.valnos) {
1278 if (V->isUnused() || !V->isPHIDef())
1279 continue;
1280
1281 unsigned RegIdx = RegAssign.lookup(V->def);
1282 LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
1283 LiveIntervalCalc &LIC = getLICalc(RegIdx);
1284 MachineBasicBlock &B = *LIS.getMBBFromIndex(V->def);
1285 if (!removeDeadSegment(V->def, LI))
1286 extendPHIRange(B, LIC, LI, LaneBitmask::getAll(), /*Undefs=*/{});
1287 }
1288
1289 SmallVector<SlotIndex, 4> Undefs;
1290 LiveIntervalCalc SubLIC;
1291
1292 for (LiveInterval::SubRange &PS : ParentLI.subranges()) {
1293 for (const VNInfo *V : PS.valnos) {
1294 if (V->isUnused() || !V->isPHIDef())
1295 continue;
1296 unsigned RegIdx = RegAssign.lookup(V->def);
1297 LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
1298 LiveInterval::SubRange &S = getSubRangeForMaskExact(PS.LaneMask, LI);
1299 if (removeDeadSegment(V->def, S))
1300 continue;
1301
1302 MachineBasicBlock &B = *LIS.getMBBFromIndex(V->def);
1303 SubLIC.reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
1304 &LIS.getVNInfoAllocator());
1305 Undefs.clear();
1306 LI.computeSubRangeUndefs(Undefs, PS.LaneMask, MRI, *LIS.getSlotIndexes());
1307 extendPHIRange(B, SubLIC, S, PS.LaneMask, Undefs);
1308 }
1309 }
1310 }
1311
1312 /// rewriteAssigned - Rewrite all uses of Edit->getReg().
rewriteAssigned(bool ExtendRanges)1313 void SplitEditor::rewriteAssigned(bool ExtendRanges) {
1314 struct ExtPoint {
1315 ExtPoint(const MachineOperand &O, unsigned R, SlotIndex N)
1316 : MO(O), RegIdx(R), Next(N) {}
1317
1318 MachineOperand MO;
1319 unsigned RegIdx;
1320 SlotIndex Next;
1321 };
1322
1323 SmallVector<ExtPoint,4> ExtPoints;
1324
1325 for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(Edit->getReg()),
1326 RE = MRI.reg_end(); RI != RE;) {
1327 MachineOperand &MO = *RI;
1328 MachineInstr *MI = MO.getParent();
1329 ++RI;
1330 // LiveDebugVariables should have handled all DBG_VALUE instructions.
1331 if (MI->isDebugValue()) {
1332 LLVM_DEBUG(dbgs() << "Zapping " << *MI);
1333 MO.setReg(0);
1334 continue;
1335 }
1336
1337 // <undef> operands don't really read the register, so it doesn't matter
1338 // which register we choose. When the use operand is tied to a def, we must
1339 // use the same register as the def, so just do that always.
1340 SlotIndex Idx = LIS.getInstructionIndex(*MI);
1341 if (MO.isDef() || MO.isUndef())
1342 Idx = Idx.getRegSlot(MO.isEarlyClobber());
1343
1344 // Rewrite to the mapped register at Idx.
1345 unsigned RegIdx = RegAssign.lookup(Idx);
1346 LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
1347 MO.setReg(LI.reg());
1348 LLVM_DEBUG(dbgs() << " rewr " << printMBBReference(*MI->getParent())
1349 << '\t' << Idx << ':' << RegIdx << '\t' << *MI);
1350
1351 // Extend liveness to Idx if the instruction reads reg.
1352 if (!ExtendRanges || MO.isUndef())
1353 continue;
1354
1355 // Skip instructions that don't read Reg.
1356 if (MO.isDef()) {
1357 if (!MO.getSubReg() && !MO.isEarlyClobber())
1358 continue;
1359 // We may want to extend a live range for a partial redef, or for a use
1360 // tied to an early clobber.
1361 Idx = Idx.getPrevSlot();
1362 if (!Edit->getParent().liveAt(Idx))
1363 continue;
1364 } else
1365 Idx = Idx.getRegSlot(true);
1366
1367 SlotIndex Next = Idx.getNextSlot();
1368 if (LI.hasSubRanges()) {
1369 // We have to delay extending subranges until we have seen all operands
1370 // defining the register. This is because a <def,read-undef> operand
1371 // will create an "undef" point, and we cannot extend any subranges
1372 // until all of them have been accounted for.
1373 if (MO.isUse())
1374 ExtPoints.push_back(ExtPoint(MO, RegIdx, Next));
1375 } else {
1376 LiveIntervalCalc &LIC = getLICalc(RegIdx);
1377 LIC.extend(LI, Next, 0, ArrayRef<SlotIndex>());
1378 }
1379 }
1380
1381 for (ExtPoint &EP : ExtPoints) {
1382 LiveInterval &LI = LIS.getInterval(Edit->get(EP.RegIdx));
1383 assert(LI.hasSubRanges());
1384
1385 LiveIntervalCalc SubLIC;
1386 Register Reg = EP.MO.getReg(), Sub = EP.MO.getSubReg();
1387 LaneBitmask LM = Sub != 0 ? TRI.getSubRegIndexLaneMask(Sub)
1388 : MRI.getMaxLaneMaskForVReg(Reg);
1389 for (LiveInterval::SubRange &S : LI.subranges()) {
1390 if ((S.LaneMask & LM).none())
1391 continue;
1392 // The problem here can be that the new register may have been created
1393 // for a partially defined original register. For example:
1394 // %0:subreg_hireg<def,read-undef> = ...
1395 // ...
1396 // %1 = COPY %0
1397 if (S.empty())
1398 continue;
1399 SubLIC.reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
1400 &LIS.getVNInfoAllocator());
1401 SmallVector<SlotIndex, 4> Undefs;
1402 LI.computeSubRangeUndefs(Undefs, S.LaneMask, MRI, *LIS.getSlotIndexes());
1403 SubLIC.extend(S, EP.Next, 0, Undefs);
1404 }
1405 }
1406
1407 for (Register R : *Edit) {
1408 LiveInterval &LI = LIS.getInterval(R);
1409 if (!LI.hasSubRanges())
1410 continue;
1411 LI.clear();
1412 LI.removeEmptySubRanges();
1413 LIS.constructMainRangeFromSubranges(LI);
1414 }
1415 }
1416
deleteRematVictims()1417 void SplitEditor::deleteRematVictims() {
1418 SmallVector<MachineInstr*, 8> Dead;
1419 for (LiveRangeEdit::iterator I = Edit->begin(), E = Edit->end(); I != E; ++I){
1420 LiveInterval *LI = &LIS.getInterval(*I);
1421 for (const LiveRange::Segment &S : LI->segments) {
1422 // Dead defs end at the dead slot.
1423 if (S.end != S.valno->def.getDeadSlot())
1424 continue;
1425 if (S.valno->isPHIDef())
1426 continue;
1427 MachineInstr *MI = LIS.getInstructionFromIndex(S.valno->def);
1428 assert(MI && "Missing instruction for dead def");
1429 MI->addRegisterDead(LI->reg(), &TRI);
1430
1431 if (!MI->allDefsAreDead())
1432 continue;
1433
1434 LLVM_DEBUG(dbgs() << "All defs dead: " << *MI);
1435 Dead.push_back(MI);
1436 }
1437 }
1438
1439 if (Dead.empty())
1440 return;
1441
1442 Edit->eliminateDeadDefs(Dead, None, &AA);
1443 }
1444
forceRecomputeVNI(const VNInfo & ParentVNI)1445 void SplitEditor::forceRecomputeVNI(const VNInfo &ParentVNI) {
1446 // Fast-path for common case.
1447 if (!ParentVNI.isPHIDef()) {
1448 for (unsigned I = 0, E = Edit->size(); I != E; ++I)
1449 forceRecompute(I, ParentVNI);
1450 return;
1451 }
1452
1453 // Trace value through phis.
1454 SmallPtrSet<const VNInfo *, 8> Visited; ///< whether VNI was/is in worklist.
1455 SmallVector<const VNInfo *, 4> WorkList;
1456 Visited.insert(&ParentVNI);
1457 WorkList.push_back(&ParentVNI);
1458
1459 const LiveInterval &ParentLI = Edit->getParent();
1460 const SlotIndexes &Indexes = *LIS.getSlotIndexes();
1461 do {
1462 const VNInfo &VNI = *WorkList.back();
1463 WorkList.pop_back();
1464 for (unsigned I = 0, E = Edit->size(); I != E; ++I)
1465 forceRecompute(I, VNI);
1466 if (!VNI.isPHIDef())
1467 continue;
1468
1469 MachineBasicBlock &MBB = *Indexes.getMBBFromIndex(VNI.def);
1470 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
1471 SlotIndex PredEnd = Indexes.getMBBEndIdx(Pred);
1472 VNInfo *PredVNI = ParentLI.getVNInfoBefore(PredEnd);
1473 assert(PredVNI && "Value available in PhiVNI predecessor");
1474 if (Visited.insert(PredVNI).second)
1475 WorkList.push_back(PredVNI);
1476 }
1477 } while(!WorkList.empty());
1478 }
1479
finish(SmallVectorImpl<unsigned> * LRMap)1480 void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) {
1481 ++NumFinished;
1482
1483 // At this point, the live intervals in Edit contain VNInfos corresponding to
1484 // the inserted copies.
1485
1486 // Add the original defs from the parent interval.
1487 for (const VNInfo *ParentVNI : Edit->getParent().valnos) {
1488 if (ParentVNI->isUnused())
1489 continue;
1490 unsigned RegIdx = RegAssign.lookup(ParentVNI->def);
1491 defValue(RegIdx, ParentVNI, ParentVNI->def, true);
1492
1493 // Force rematted values to be recomputed everywhere.
1494 // The new live ranges may be truncated.
1495 if (Edit->didRematerialize(ParentVNI))
1496 forceRecomputeVNI(*ParentVNI);
1497 }
1498
1499 // Hoist back-copies to the complement interval when in spill mode.
1500 switch (SpillMode) {
1501 case SM_Partition:
1502 // Leave all back-copies as is.
1503 break;
1504 case SM_Size:
1505 case SM_Speed:
1506 // hoistCopies will behave differently between size and speed.
1507 hoistCopies();
1508 }
1509
1510 // Transfer the simply mapped values, check if any are skipped.
1511 bool Skipped = transferValues();
1512
1513 // Rewrite virtual registers, possibly extending ranges.
1514 rewriteAssigned(Skipped);
1515
1516 if (Skipped)
1517 extendPHIKillRanges();
1518 else
1519 ++NumSimple;
1520
1521 // Delete defs that were rematted everywhere.
1522 if (Skipped)
1523 deleteRematVictims();
1524
1525 // Get rid of unused values and set phi-kill flags.
1526 for (Register Reg : *Edit) {
1527 LiveInterval &LI = LIS.getInterval(Reg);
1528 LI.removeEmptySubRanges();
1529 LI.RenumberValues();
1530 }
1531
1532 // Provide a reverse mapping from original indices to Edit ranges.
1533 if (LRMap) {
1534 LRMap->clear();
1535 for (unsigned i = 0, e = Edit->size(); i != e; ++i)
1536 LRMap->push_back(i);
1537 }
1538
1539 // Now check if any registers were separated into multiple components.
1540 ConnectedVNInfoEqClasses ConEQ(LIS);
1541 for (unsigned i = 0, e = Edit->size(); i != e; ++i) {
1542 // Don't use iterators, they are invalidated by create() below.
1543 Register VReg = Edit->get(i);
1544 LiveInterval &LI = LIS.getInterval(VReg);
1545 SmallVector<LiveInterval*, 8> SplitLIs;
1546 LIS.splitSeparateComponents(LI, SplitLIs);
1547 Register Original = VRM.getOriginal(VReg);
1548 for (LiveInterval *SplitLI : SplitLIs)
1549 VRM.setIsSplitFromReg(SplitLI->reg(), Original);
1550
1551 // The new intervals all map back to i.
1552 if (LRMap)
1553 LRMap->resize(Edit->size(), i);
1554 }
1555
1556 // Calculate spill weight and allocation hints for new intervals.
1557 Edit->calculateRegClassAndHint(VRM.getMachineFunction(), SA.Loops, MBFI);
1558
1559 assert(!LRMap || LRMap->size() == Edit->size());
1560 }
1561
1562 //===----------------------------------------------------------------------===//
1563 // Single Block Splitting
1564 //===----------------------------------------------------------------------===//
1565
shouldSplitSingleBlock(const BlockInfo & BI,bool SingleInstrs) const1566 bool SplitAnalysis::shouldSplitSingleBlock(const BlockInfo &BI,
1567 bool SingleInstrs) const {
1568 // Always split for multiple instructions.
1569 if (!BI.isOneInstr())
1570 return true;
1571 // Don't split for single instructions unless explicitly requested.
1572 if (!SingleInstrs)
1573 return false;
1574 // Splitting a live-through range always makes progress.
1575 if (BI.LiveIn && BI.LiveOut)
1576 return true;
1577 // No point in isolating a copy. It has no register class constraints.
1578 if (LIS.getInstructionFromIndex(BI.FirstInstr)->isCopyLike())
1579 return false;
1580 // Finally, don't isolate an end point that was created by earlier splits.
1581 return isOriginalEndpoint(BI.FirstInstr);
1582 }
1583
splitSingleBlock(const SplitAnalysis::BlockInfo & BI)1584 void SplitEditor::splitSingleBlock(const SplitAnalysis::BlockInfo &BI) {
1585 openIntv();
1586 SlotIndex LastSplitPoint = SA.getLastSplitPoint(BI.MBB->getNumber());
1587 SlotIndex SegStart = enterIntvBefore(std::min(BI.FirstInstr,
1588 LastSplitPoint));
1589 if (!BI.LiveOut || BI.LastInstr < LastSplitPoint) {
1590 useIntv(SegStart, leaveIntvAfter(BI.LastInstr));
1591 } else {
1592 // The last use is after the last valid split point.
1593 SlotIndex SegStop = leaveIntvBefore(LastSplitPoint);
1594 useIntv(SegStart, SegStop);
1595 overlapIntv(SegStop, BI.LastInstr);
1596 }
1597 }
1598
1599 //===----------------------------------------------------------------------===//
1600 // Global Live Range Splitting Support
1601 //===----------------------------------------------------------------------===//
1602
1603 // These methods support a method of global live range splitting that uses a
1604 // global algorithm to decide intervals for CFG edges. They will insert split
1605 // points and color intervals in basic blocks while avoiding interference.
1606 //
1607 // Note that splitSingleBlock is also useful for blocks where both CFG edges
1608 // are on the stack.
1609
splitLiveThroughBlock(unsigned MBBNum,unsigned IntvIn,SlotIndex LeaveBefore,unsigned IntvOut,SlotIndex EnterAfter)1610 void SplitEditor::splitLiveThroughBlock(unsigned MBBNum,
1611 unsigned IntvIn, SlotIndex LeaveBefore,
1612 unsigned IntvOut, SlotIndex EnterAfter){
1613 SlotIndex Start, Stop;
1614 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(MBBNum);
1615
1616 LLVM_DEBUG(dbgs() << "%bb." << MBBNum << " [" << Start << ';' << Stop
1617 << ") intf " << LeaveBefore << '-' << EnterAfter
1618 << ", live-through " << IntvIn << " -> " << IntvOut);
1619
1620 assert((IntvIn || IntvOut) && "Use splitSingleBlock for isolated blocks");
1621
1622 assert((!LeaveBefore || LeaveBefore < Stop) && "Interference after block");
1623 assert((!IntvIn || !LeaveBefore || LeaveBefore > Start) && "Impossible intf");
1624 assert((!EnterAfter || EnterAfter >= Start) && "Interference before block");
1625
1626 MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum);
1627
1628 if (!IntvOut) {
1629 LLVM_DEBUG(dbgs() << ", spill on entry.\n");
1630 //
1631 // <<<<<<<<< Possible LeaveBefore interference.
1632 // |-----------| Live through.
1633 // -____________ Spill on entry.
1634 //
1635 selectIntv(IntvIn);
1636 SlotIndex Idx = leaveIntvAtTop(*MBB);
1637 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
1638 (void)Idx;
1639 return;
1640 }
1641
1642 if (!IntvIn) {
1643 LLVM_DEBUG(dbgs() << ", reload on exit.\n");
1644 //
1645 // >>>>>>> Possible EnterAfter interference.
1646 // |-----------| Live through.
1647 // ___________-- Reload on exit.
1648 //
1649 selectIntv(IntvOut);
1650 SlotIndex Idx = enterIntvAtEnd(*MBB);
1651 assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
1652 (void)Idx;
1653 return;
1654 }
1655
1656 if (IntvIn == IntvOut && !LeaveBefore && !EnterAfter) {
1657 LLVM_DEBUG(dbgs() << ", straight through.\n");
1658 //
1659 // |-----------| Live through.
1660 // ------------- Straight through, same intv, no interference.
1661 //
1662 selectIntv(IntvOut);
1663 useIntv(Start, Stop);
1664 return;
1665 }
1666
1667 // We cannot legally insert splits after LSP.
1668 SlotIndex LSP = SA.getLastSplitPoint(MBBNum);
1669 assert((!IntvOut || !EnterAfter || EnterAfter < LSP) && "Impossible intf");
1670
1671 if (IntvIn != IntvOut && (!LeaveBefore || !EnterAfter ||
1672 LeaveBefore.getBaseIndex() > EnterAfter.getBoundaryIndex())) {
1673 LLVM_DEBUG(dbgs() << ", switch avoiding interference.\n");
1674 //
1675 // >>>> <<<< Non-overlapping EnterAfter/LeaveBefore interference.
1676 // |-----------| Live through.
1677 // ------======= Switch intervals between interference.
1678 //
1679 selectIntv(IntvOut);
1680 SlotIndex Idx;
1681 if (LeaveBefore && LeaveBefore < LSP) {
1682 Idx = enterIntvBefore(LeaveBefore);
1683 useIntv(Idx, Stop);
1684 } else {
1685 Idx = enterIntvAtEnd(*MBB);
1686 }
1687 selectIntv(IntvIn);
1688 useIntv(Start, Idx);
1689 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
1690 assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
1691 return;
1692 }
1693
1694 LLVM_DEBUG(dbgs() << ", create local intv for interference.\n");
1695 //
1696 // >>><><><><<<< Overlapping EnterAfter/LeaveBefore interference.
1697 // |-----------| Live through.
1698 // ==---------== Switch intervals before/after interference.
1699 //
1700 assert(LeaveBefore <= EnterAfter && "Missed case");
1701
1702 selectIntv(IntvOut);
1703 SlotIndex Idx = enterIntvAfter(EnterAfter);
1704 useIntv(Idx, Stop);
1705 assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
1706
1707 selectIntv(IntvIn);
1708 Idx = leaveIntvBefore(LeaveBefore);
1709 useIntv(Start, Idx);
1710 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
1711 }
1712
splitRegInBlock(const SplitAnalysis::BlockInfo & BI,unsigned IntvIn,SlotIndex LeaveBefore)1713 void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI,
1714 unsigned IntvIn, SlotIndex LeaveBefore) {
1715 SlotIndex Start, Stop;
1716 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB);
1717
1718 LLVM_DEBUG(dbgs() << printMBBReference(*BI.MBB) << " [" << Start << ';'
1719 << Stop << "), uses " << BI.FirstInstr << '-'
1720 << BI.LastInstr << ", reg-in " << IntvIn
1721 << ", leave before " << LeaveBefore
1722 << (BI.LiveOut ? ", stack-out" : ", killed in block"));
1723
1724 assert(IntvIn && "Must have register in");
1725 assert(BI.LiveIn && "Must be live-in");
1726 assert((!LeaveBefore || LeaveBefore > Start) && "Bad interference");
1727
1728 if (!BI.LiveOut && (!LeaveBefore || LeaveBefore >= BI.LastInstr)) {
1729 LLVM_DEBUG(dbgs() << " before interference.\n");
1730 //
1731 // <<< Interference after kill.
1732 // |---o---x | Killed in block.
1733 // ========= Use IntvIn everywhere.
1734 //
1735 selectIntv(IntvIn);
1736 useIntv(Start, BI.LastInstr);
1737 return;
1738 }
1739
1740 SlotIndex LSP = SA.getLastSplitPoint(BI.MBB->getNumber());
1741
1742 if (!LeaveBefore || LeaveBefore > BI.LastInstr.getBoundaryIndex()) {
1743 //
1744 // <<< Possible interference after last use.
1745 // |---o---o---| Live-out on stack.
1746 // =========____ Leave IntvIn after last use.
1747 //
1748 // < Interference after last use.
1749 // |---o---o--o| Live-out on stack, late last use.
1750 // ============ Copy to stack after LSP, overlap IntvIn.
1751 // \_____ Stack interval is live-out.
1752 //
1753 if (BI.LastInstr < LSP) {
1754 LLVM_DEBUG(dbgs() << ", spill after last use before interference.\n");
1755 selectIntv(IntvIn);
1756 SlotIndex Idx = leaveIntvAfter(BI.LastInstr);
1757 useIntv(Start, Idx);
1758 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
1759 } else {
1760 LLVM_DEBUG(dbgs() << ", spill before last split point.\n");
1761 selectIntv(IntvIn);
1762 SlotIndex Idx = leaveIntvBefore(LSP);
1763 overlapIntv(Idx, BI.LastInstr);
1764 useIntv(Start, Idx);
1765 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
1766 }
1767 return;
1768 }
1769
1770 // The interference is overlapping somewhere we wanted to use IntvIn. That
1771 // means we need to create a local interval that can be allocated a
1772 // different register.
1773 unsigned LocalIntv = openIntv();
1774 (void)LocalIntv;
1775 LLVM_DEBUG(dbgs() << ", creating local interval " << LocalIntv << ".\n");
1776
1777 if (!BI.LiveOut || BI.LastInstr < LSP) {
1778 //
1779 // <<<<<<< Interference overlapping uses.
1780 // |---o---o---| Live-out on stack.
1781 // =====----____ Leave IntvIn before interference, then spill.
1782 //
1783 SlotIndex To = leaveIntvAfter(BI.LastInstr);
1784 SlotIndex From = enterIntvBefore(LeaveBefore);
1785 useIntv(From, To);
1786 selectIntv(IntvIn);
1787 useIntv(Start, From);
1788 assert((!LeaveBefore || From <= LeaveBefore) && "Interference");
1789 return;
1790 }
1791
1792 // <<<<<<< Interference overlapping uses.
1793 // |---o---o--o| Live-out on stack, late last use.
1794 // =====------- Copy to stack before LSP, overlap LocalIntv.
1795 // \_____ Stack interval is live-out.
1796 //
1797 SlotIndex To = leaveIntvBefore(LSP);
1798 overlapIntv(To, BI.LastInstr);
1799 SlotIndex From = enterIntvBefore(std::min(To, LeaveBefore));
1800 useIntv(From, To);
1801 selectIntv(IntvIn);
1802 useIntv(Start, From);
1803 assert((!LeaveBefore || From <= LeaveBefore) && "Interference");
1804 }
1805
splitRegOutBlock(const SplitAnalysis::BlockInfo & BI,unsigned IntvOut,SlotIndex EnterAfter)1806 void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI,
1807 unsigned IntvOut, SlotIndex EnterAfter) {
1808 SlotIndex Start, Stop;
1809 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB);
1810
1811 LLVM_DEBUG(dbgs() << printMBBReference(*BI.MBB) << " [" << Start << ';'
1812 << Stop << "), uses " << BI.FirstInstr << '-'
1813 << BI.LastInstr << ", reg-out " << IntvOut
1814 << ", enter after " << EnterAfter
1815 << (BI.LiveIn ? ", stack-in" : ", defined in block"));
1816
1817 SlotIndex LSP = SA.getLastSplitPoint(BI.MBB->getNumber());
1818
1819 assert(IntvOut && "Must have register out");
1820 assert(BI.LiveOut && "Must be live-out");
1821 assert((!EnterAfter || EnterAfter < LSP) && "Bad interference");
1822
1823 if (!BI.LiveIn && (!EnterAfter || EnterAfter <= BI.FirstInstr)) {
1824 LLVM_DEBUG(dbgs() << " after interference.\n");
1825 //
1826 // >>>> Interference before def.
1827 // | o---o---| Defined in block.
1828 // ========= Use IntvOut everywhere.
1829 //
1830 selectIntv(IntvOut);
1831 useIntv(BI.FirstInstr, Stop);
1832 return;
1833 }
1834
1835 if (!EnterAfter || EnterAfter < BI.FirstInstr.getBaseIndex()) {
1836 LLVM_DEBUG(dbgs() << ", reload after interference.\n");
1837 //
1838 // >>>> Interference before def.
1839 // |---o---o---| Live-through, stack-in.
1840 // ____========= Enter IntvOut before first use.
1841 //
1842 selectIntv(IntvOut);
1843 SlotIndex Idx = enterIntvBefore(std::min(LSP, BI.FirstInstr));
1844 useIntv(Idx, Stop);
1845 assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
1846 return;
1847 }
1848
1849 // The interference is overlapping somewhere we wanted to use IntvOut. That
1850 // means we need to create a local interval that can be allocated a
1851 // different register.
1852 LLVM_DEBUG(dbgs() << ", interference overlaps uses.\n");
1853 //
1854 // >>>>>>> Interference overlapping uses.
1855 // |---o---o---| Live-through, stack-in.
1856 // ____---====== Create local interval for interference range.
1857 //
1858 selectIntv(IntvOut);
1859 SlotIndex Idx = enterIntvAfter(EnterAfter);
1860 useIntv(Idx, Stop);
1861 assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
1862
1863 openIntv();
1864 SlotIndex From = enterIntvBefore(std::min(Idx, BI.FirstInstr));
1865 useIntv(From, Idx);
1866 }
1867