1 //===- BasicBlockUtils.cpp - BasicBlock Utilities --------------------------==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This family of functions perform manipulations on basic blocks, and
10 // instructions contained within basic blocks.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/SmallPtrSet.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/Twine.h"
19 #include "llvm/Analysis/CFG.h"
20 #include "llvm/Analysis/DomTreeUpdater.h"
21 #include "llvm/Analysis/LoopInfo.h"
22 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
23 #include "llvm/Analysis/MemorySSAUpdater.h"
24 #include "llvm/Analysis/PostDominators.h"
25 #include "llvm/IR/BasicBlock.h"
26 #include "llvm/IR/CFG.h"
27 #include "llvm/IR/Constants.h"
28 #include "llvm/IR/DebugInfoMetadata.h"
29 #include "llvm/IR/Dominators.h"
30 #include "llvm/IR/Function.h"
31 #include "llvm/IR/InstrTypes.h"
32 #include "llvm/IR/Instruction.h"
33 #include "llvm/IR/Instructions.h"
34 #include "llvm/IR/IntrinsicInst.h"
35 #include "llvm/IR/LLVMContext.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/IR/User.h"
38 #include "llvm/IR/Value.h"
39 #include "llvm/IR/ValueHandle.h"
40 #include "llvm/Support/Casting.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/raw_ostream.h"
43 #include "llvm/Transforms/Utils/Local.h"
44 #include <cassert>
45 #include <cstdint>
46 #include <string>
47 #include <utility>
48 #include <vector>
49
50 using namespace llvm;
51
52 #define DEBUG_TYPE "basicblock-utils"
53
DetatchDeadBlocks(ArrayRef<BasicBlock * > BBs,SmallVectorImpl<DominatorTree::UpdateType> * Updates,bool KeepOneInputPHIs)54 void llvm::DetatchDeadBlocks(
55 ArrayRef<BasicBlock *> BBs,
56 SmallVectorImpl<DominatorTree::UpdateType> *Updates,
57 bool KeepOneInputPHIs) {
58 for (auto *BB : BBs) {
59 // Loop through all of our successors and make sure they know that one
60 // of their predecessors is going away.
61 SmallPtrSet<BasicBlock *, 4> UniqueSuccessors;
62 for (BasicBlock *Succ : successors(BB)) {
63 Succ->removePredecessor(BB, KeepOneInputPHIs);
64 if (Updates && UniqueSuccessors.insert(Succ).second)
65 Updates->push_back({DominatorTree::Delete, BB, Succ});
66 }
67
68 // Zap all the instructions in the block.
69 while (!BB->empty()) {
70 Instruction &I = BB->back();
71 // If this instruction is used, replace uses with an arbitrary value.
72 // Because control flow can't get here, we don't care what we replace the
73 // value with. Note that since this block is unreachable, and all values
74 // contained within it must dominate their uses, that all uses will
75 // eventually be removed (they are themselves dead).
76 if (!I.use_empty())
77 I.replaceAllUsesWith(UndefValue::get(I.getType()));
78 BB->getInstList().pop_back();
79 }
80 new UnreachableInst(BB->getContext(), BB);
81 assert(BB->getInstList().size() == 1 &&
82 isa<UnreachableInst>(BB->getTerminator()) &&
83 "The successor list of BB isn't empty before "
84 "applying corresponding DTU updates.");
85 }
86 }
87
DeleteDeadBlock(BasicBlock * BB,DomTreeUpdater * DTU,bool KeepOneInputPHIs)88 void llvm::DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU,
89 bool KeepOneInputPHIs) {
90 DeleteDeadBlocks({BB}, DTU, KeepOneInputPHIs);
91 }
92
DeleteDeadBlocks(ArrayRef<BasicBlock * > BBs,DomTreeUpdater * DTU,bool KeepOneInputPHIs)93 void llvm::DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs, DomTreeUpdater *DTU,
94 bool KeepOneInputPHIs) {
95 #ifndef NDEBUG
96 // Make sure that all predecessors of each dead block is also dead.
97 SmallPtrSet<BasicBlock *, 4> Dead(BBs.begin(), BBs.end());
98 assert(Dead.size() == BBs.size() && "Duplicating blocks?");
99 for (auto *BB : Dead)
100 for (BasicBlock *Pred : predecessors(BB))
101 assert(Dead.count(Pred) && "All predecessors must be dead!");
102 #endif
103
104 SmallVector<DominatorTree::UpdateType, 4> Updates;
105 DetatchDeadBlocks(BBs, DTU ? &Updates : nullptr, KeepOneInputPHIs);
106
107 if (DTU)
108 DTU->applyUpdates(Updates);
109
110 for (BasicBlock *BB : BBs)
111 if (DTU)
112 DTU->deleteBB(BB);
113 else
114 BB->eraseFromParent();
115 }
116
EliminateUnreachableBlocks(Function & F,DomTreeUpdater * DTU,bool KeepOneInputPHIs)117 bool llvm::EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
118 bool KeepOneInputPHIs) {
119 df_iterator_default_set<BasicBlock*> Reachable;
120
121 // Mark all reachable blocks.
122 for (BasicBlock *BB : depth_first_ext(&F, Reachable))
123 (void)BB/* Mark all reachable blocks */;
124
125 // Collect all dead blocks.
126 std::vector<BasicBlock*> DeadBlocks;
127 for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I)
128 if (!Reachable.count(&*I)) {
129 BasicBlock *BB = &*I;
130 DeadBlocks.push_back(BB);
131 }
132
133 // Delete the dead blocks.
134 DeleteDeadBlocks(DeadBlocks, DTU, KeepOneInputPHIs);
135
136 return !DeadBlocks.empty();
137 }
138
FoldSingleEntryPHINodes(BasicBlock * BB,MemoryDependenceResults * MemDep)139 bool llvm::FoldSingleEntryPHINodes(BasicBlock *BB,
140 MemoryDependenceResults *MemDep) {
141 if (!isa<PHINode>(BB->begin()))
142 return false;
143
144 while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
145 if (PN->getIncomingValue(0) != PN)
146 PN->replaceAllUsesWith(PN->getIncomingValue(0));
147 else
148 PN->replaceAllUsesWith(UndefValue::get(PN->getType()));
149
150 if (MemDep)
151 MemDep->removeInstruction(PN); // Memdep updates AA itself.
152
153 PN->eraseFromParent();
154 }
155 return true;
156 }
157
DeleteDeadPHIs(BasicBlock * BB,const TargetLibraryInfo * TLI,MemorySSAUpdater * MSSAU)158 bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI,
159 MemorySSAUpdater *MSSAU) {
160 // Recursively deleting a PHI may cause multiple PHIs to be deleted
161 // or RAUW'd undef, so use an array of WeakTrackingVH for the PHIs to delete.
162 SmallVector<WeakTrackingVH, 8> PHIs;
163 for (PHINode &PN : BB->phis())
164 PHIs.push_back(&PN);
165
166 bool Changed = false;
167 for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
168 if (PHINode *PN = dyn_cast_or_null<PHINode>(PHIs[i].operator Value*()))
169 Changed |= RecursivelyDeleteDeadPHINode(PN, TLI, MSSAU);
170
171 return Changed;
172 }
173
MergeBlockIntoPredecessor(BasicBlock * BB,DomTreeUpdater * DTU,LoopInfo * LI,MemorySSAUpdater * MSSAU,MemoryDependenceResults * MemDep,bool PredecessorWithTwoSuccessors)174 bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
175 LoopInfo *LI, MemorySSAUpdater *MSSAU,
176 MemoryDependenceResults *MemDep,
177 bool PredecessorWithTwoSuccessors) {
178 if (BB->hasAddressTaken())
179 return false;
180
181 // Can't merge if there are multiple predecessors, or no predecessors.
182 BasicBlock *PredBB = BB->getUniquePredecessor();
183 if (!PredBB) return false;
184
185 // Don't break self-loops.
186 if (PredBB == BB) return false;
187 // Don't break unwinding instructions.
188 if (PredBB->getTerminator()->isExceptionalTerminator())
189 return false;
190
191 // Can't merge if there are multiple distinct successors.
192 if (!PredecessorWithTwoSuccessors && PredBB->getUniqueSuccessor() != BB)
193 return false;
194
195 // Currently only allow PredBB to have two predecessors, one being BB.
196 // Update BI to branch to BB's only successor instead of BB.
197 BranchInst *PredBB_BI;
198 BasicBlock *NewSucc = nullptr;
199 unsigned FallThruPath;
200 if (PredecessorWithTwoSuccessors) {
201 if (!(PredBB_BI = dyn_cast<BranchInst>(PredBB->getTerminator())))
202 return false;
203 BranchInst *BB_JmpI = dyn_cast<BranchInst>(BB->getTerminator());
204 if (!BB_JmpI || !BB_JmpI->isUnconditional())
205 return false;
206 NewSucc = BB_JmpI->getSuccessor(0);
207 FallThruPath = PredBB_BI->getSuccessor(0) == BB ? 0 : 1;
208 }
209
210 // Can't merge if there is PHI loop.
211 for (PHINode &PN : BB->phis())
212 for (Value *IncValue : PN.incoming_values())
213 if (IncValue == &PN)
214 return false;
215
216 LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
217 << PredBB->getName() << "\n");
218
219 // Begin by getting rid of unneeded PHIs.
220 SmallVector<AssertingVH<Value>, 4> IncomingValues;
221 if (isa<PHINode>(BB->front())) {
222 for (PHINode &PN : BB->phis())
223 if (!isa<PHINode>(PN.getIncomingValue(0)) ||
224 cast<PHINode>(PN.getIncomingValue(0))->getParent() != BB)
225 IncomingValues.push_back(PN.getIncomingValue(0));
226 FoldSingleEntryPHINodes(BB, MemDep);
227 }
228
229 // DTU update: Collect all the edges that exit BB.
230 // These dominator edges will be redirected from Pred.
231 std::vector<DominatorTree::UpdateType> Updates;
232 if (DTU) {
233 SmallSetVector<BasicBlock *, 2> UniqueSuccessors(succ_begin(BB),
234 succ_end(BB));
235 Updates.reserve(1 + (2 * UniqueSuccessors.size()));
236 // Add insert edges first. Experimentally, for the particular case of two
237 // blocks that can be merged, with a single successor and single predecessor
238 // respectively, it is beneficial to have all insert updates first. Deleting
239 // edges first may lead to unreachable blocks, followed by inserting edges
240 // making the blocks reachable again. Such DT updates lead to high compile
241 // times. We add inserts before deletes here to reduce compile time.
242 for (BasicBlock *UniqueSuccessor : UniqueSuccessors)
243 // This successor of BB may already have PredBB as a predecessor.
244 if (!llvm::is_contained(successors(PredBB), UniqueSuccessor))
245 Updates.push_back({DominatorTree::Insert, PredBB, UniqueSuccessor});
246 for (BasicBlock *UniqueSuccessor : UniqueSuccessors)
247 Updates.push_back({DominatorTree::Delete, BB, UniqueSuccessor});
248 Updates.push_back({DominatorTree::Delete, PredBB, BB});
249 }
250
251 Instruction *PTI = PredBB->getTerminator();
252 Instruction *STI = BB->getTerminator();
253 Instruction *Start = &*BB->begin();
254 // If there's nothing to move, mark the starting instruction as the last
255 // instruction in the block. Terminator instruction is handled separately.
256 if (Start == STI)
257 Start = PTI;
258
259 // Move all definitions in the successor to the predecessor...
260 PredBB->getInstList().splice(PTI->getIterator(), BB->getInstList(),
261 BB->begin(), STI->getIterator());
262
263 if (MSSAU)
264 MSSAU->moveAllAfterMergeBlocks(BB, PredBB, Start);
265
266 // Make all PHI nodes that referred to BB now refer to Pred as their
267 // source...
268 BB->replaceAllUsesWith(PredBB);
269
270 if (PredecessorWithTwoSuccessors) {
271 // Delete the unconditional branch from BB.
272 BB->getInstList().pop_back();
273
274 // Update branch in the predecessor.
275 PredBB_BI->setSuccessor(FallThruPath, NewSucc);
276 } else {
277 // Delete the unconditional branch from the predecessor.
278 PredBB->getInstList().pop_back();
279
280 // Move terminator instruction.
281 PredBB->getInstList().splice(PredBB->end(), BB->getInstList());
282
283 // Terminator may be a memory accessing instruction too.
284 if (MSSAU)
285 if (MemoryUseOrDef *MUD = cast_or_null<MemoryUseOrDef>(
286 MSSAU->getMemorySSA()->getMemoryAccess(PredBB->getTerminator())))
287 MSSAU->moveToPlace(MUD, PredBB, MemorySSA::End);
288 }
289 // Add unreachable to now empty BB.
290 new UnreachableInst(BB->getContext(), BB);
291
292 // Inherit predecessors name if it exists.
293 if (!PredBB->hasName())
294 PredBB->takeName(BB);
295
296 if (LI)
297 LI->removeBlock(BB);
298
299 if (MemDep)
300 MemDep->invalidateCachedPredecessors();
301
302 // Finally, erase the old block and update dominator info.
303 if (DTU) {
304 assert(BB->getInstList().size() == 1 &&
305 isa<UnreachableInst>(BB->getTerminator()) &&
306 "The successor list of BB isn't empty before "
307 "applying corresponding DTU updates.");
308 DTU->applyUpdates(Updates);
309 DTU->deleteBB(BB);
310 } else {
311 BB->eraseFromParent(); // Nuke BB if DTU is nullptr.
312 }
313
314 return true;
315 }
316
MergeBlockSuccessorsIntoGivenBlocks(SmallPtrSetImpl<BasicBlock * > & MergeBlocks,Loop * L,DomTreeUpdater * DTU,LoopInfo * LI)317 bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
318 SmallPtrSetImpl<BasicBlock *> &MergeBlocks, Loop *L, DomTreeUpdater *DTU,
319 LoopInfo *LI) {
320 assert(!MergeBlocks.empty() && "MergeBlocks should not be empty");
321
322 bool BlocksHaveBeenMerged = false;
323 while (!MergeBlocks.empty()) {
324 BasicBlock *BB = *MergeBlocks.begin();
325 BasicBlock *Dest = BB->getSingleSuccessor();
326 if (Dest && (!L || L->contains(Dest))) {
327 BasicBlock *Fold = Dest->getUniquePredecessor();
328 (void)Fold;
329 if (MergeBlockIntoPredecessor(Dest, DTU, LI)) {
330 assert(Fold == BB &&
331 "Expecting BB to be unique predecessor of the Dest block");
332 MergeBlocks.erase(Dest);
333 BlocksHaveBeenMerged = true;
334 } else
335 MergeBlocks.erase(BB);
336 } else
337 MergeBlocks.erase(BB);
338 }
339 return BlocksHaveBeenMerged;
340 }
341
342 /// Remove redundant instructions within sequences of consecutive dbg.value
343 /// instructions. This is done using a backward scan to keep the last dbg.value
344 /// describing a specific variable/fragment.
345 ///
346 /// BackwardScan strategy:
347 /// ----------------------
348 /// Given a sequence of consecutive DbgValueInst like this
349 ///
350 /// dbg.value ..., "x", FragmentX1 (*)
351 /// dbg.value ..., "y", FragmentY1
352 /// dbg.value ..., "x", FragmentX2
353 /// dbg.value ..., "x", FragmentX1 (**)
354 ///
355 /// then the instruction marked with (*) can be removed (it is guaranteed to be
356 /// obsoleted by the instruction marked with (**) as the latter instruction is
357 /// describing the same variable using the same fragment info).
358 ///
359 /// Possible improvements:
360 /// - Check fully overlapping fragments and not only identical fragments.
361 /// - Support dbg.addr, dbg.declare. dbg.label, and possibly other meta
362 /// instructions being part of the sequence of consecutive instructions.
removeRedundantDbgInstrsUsingBackwardScan(BasicBlock * BB)363 static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
364 SmallVector<DbgValueInst *, 8> ToBeRemoved;
365 SmallDenseSet<DebugVariable> VariableSet;
366 for (auto &I : reverse(*BB)) {
367 if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
368 DebugVariable Key(DVI->getVariable(),
369 DVI->getExpression(),
370 DVI->getDebugLoc()->getInlinedAt());
371 auto R = VariableSet.insert(Key);
372 // If the same variable fragment is described more than once it is enough
373 // to keep the last one (i.e. the first found since we for reverse
374 // iteration).
375 if (!R.second)
376 ToBeRemoved.push_back(DVI);
377 continue;
378 }
379 // Sequence with consecutive dbg.value instrs ended. Clear the map to
380 // restart identifying redundant instructions if case we find another
381 // dbg.value sequence.
382 VariableSet.clear();
383 }
384
385 for (auto &Instr : ToBeRemoved)
386 Instr->eraseFromParent();
387
388 return !ToBeRemoved.empty();
389 }
390
391 /// Remove redundant dbg.value instructions using a forward scan. This can
392 /// remove a dbg.value instruction that is redundant due to indicating that a
393 /// variable has the same value as already being indicated by an earlier
394 /// dbg.value.
395 ///
396 /// ForwardScan strategy:
397 /// ---------------------
398 /// Given two identical dbg.value instructions, separated by a block of
399 /// instructions that isn't describing the same variable, like this
400 ///
401 /// dbg.value X1, "x", FragmentX1 (**)
402 /// <block of instructions, none being "dbg.value ..., "x", ...">
403 /// dbg.value X1, "x", FragmentX1 (*)
404 ///
405 /// then the instruction marked with (*) can be removed. Variable "x" is already
406 /// described as being mapped to the SSA value X1.
407 ///
408 /// Possible improvements:
409 /// - Keep track of non-overlapping fragments.
removeRedundantDbgInstrsUsingForwardScan(BasicBlock * BB)410 static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
411 SmallVector<DbgValueInst *, 8> ToBeRemoved;
412 DenseMap<DebugVariable, std::pair<Value *, DIExpression *> > VariableMap;
413 for (auto &I : *BB) {
414 if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
415 DebugVariable Key(DVI->getVariable(),
416 NoneType(),
417 DVI->getDebugLoc()->getInlinedAt());
418 auto VMI = VariableMap.find(Key);
419 // Update the map if we found a new value/expression describing the
420 // variable, or if the variable wasn't mapped already.
421 if (VMI == VariableMap.end() ||
422 VMI->second.first != DVI->getValue() ||
423 VMI->second.second != DVI->getExpression()) {
424 VariableMap[Key] = { DVI->getValue(), DVI->getExpression() };
425 continue;
426 }
427 // Found an identical mapping. Remember the instruction for later removal.
428 ToBeRemoved.push_back(DVI);
429 }
430 }
431
432 for (auto &Instr : ToBeRemoved)
433 Instr->eraseFromParent();
434
435 return !ToBeRemoved.empty();
436 }
437
RemoveRedundantDbgInstrs(BasicBlock * BB)438 bool llvm::RemoveRedundantDbgInstrs(BasicBlock *BB) {
439 bool MadeChanges = false;
440 // By using the "backward scan" strategy before the "forward scan" strategy we
441 // can remove both dbg.value (2) and (3) in a situation like this:
442 //
443 // (1) dbg.value V1, "x", DIExpression()
444 // ...
445 // (2) dbg.value V2, "x", DIExpression()
446 // (3) dbg.value V1, "x", DIExpression()
447 //
448 // The backward scan will remove (2), it is made obsolete by (3). After
449 // getting (2) out of the way, the foward scan will remove (3) since "x"
450 // already is described as having the value V1 at (1).
451 MadeChanges |= removeRedundantDbgInstrsUsingBackwardScan(BB);
452 MadeChanges |= removeRedundantDbgInstrsUsingForwardScan(BB);
453
454 if (MadeChanges)
455 LLVM_DEBUG(dbgs() << "Removed redundant dbg instrs from: "
456 << BB->getName() << "\n");
457 return MadeChanges;
458 }
459
ReplaceInstWithValue(BasicBlock::InstListType & BIL,BasicBlock::iterator & BI,Value * V)460 void llvm::ReplaceInstWithValue(BasicBlock::InstListType &BIL,
461 BasicBlock::iterator &BI, Value *V) {
462 Instruction &I = *BI;
463 // Replaces all of the uses of the instruction with uses of the value
464 I.replaceAllUsesWith(V);
465
466 // Make sure to propagate a name if there is one already.
467 if (I.hasName() && !V->hasName())
468 V->takeName(&I);
469
470 // Delete the unnecessary instruction now...
471 BI = BIL.erase(BI);
472 }
473
ReplaceInstWithInst(BasicBlock::InstListType & BIL,BasicBlock::iterator & BI,Instruction * I)474 void llvm::ReplaceInstWithInst(BasicBlock::InstListType &BIL,
475 BasicBlock::iterator &BI, Instruction *I) {
476 assert(I->getParent() == nullptr &&
477 "ReplaceInstWithInst: Instruction already inserted into basic block!");
478
479 // Copy debug location to newly added instruction, if it wasn't already set
480 // by the caller.
481 if (!I->getDebugLoc())
482 I->setDebugLoc(BI->getDebugLoc());
483
484 // Insert the new instruction into the basic block...
485 BasicBlock::iterator New = BIL.insert(BI, I);
486
487 // Replace all uses of the old instruction, and delete it.
488 ReplaceInstWithValue(BIL, BI, I);
489
490 // Move BI back to point to the newly inserted instruction
491 BI = New;
492 }
493
ReplaceInstWithInst(Instruction * From,Instruction * To)494 void llvm::ReplaceInstWithInst(Instruction *From, Instruction *To) {
495 BasicBlock::iterator BI(From);
496 ReplaceInstWithInst(From->getParent()->getInstList(), BI, To);
497 }
498
SplitEdge(BasicBlock * BB,BasicBlock * Succ,DominatorTree * DT,LoopInfo * LI,MemorySSAUpdater * MSSAU,const Twine & BBName)499 BasicBlock *llvm::SplitEdge(BasicBlock *BB, BasicBlock *Succ, DominatorTree *DT,
500 LoopInfo *LI, MemorySSAUpdater *MSSAU,
501 const Twine &BBName) {
502 unsigned SuccNum = GetSuccessorNumber(BB, Succ);
503
504 // If this is a critical edge, let SplitCriticalEdge do it.
505 Instruction *LatchTerm = BB->getTerminator();
506 if (SplitCriticalEdge(
507 LatchTerm, SuccNum,
508 CriticalEdgeSplittingOptions(DT, LI, MSSAU).setPreserveLCSSA(),
509 BBName))
510 return LatchTerm->getSuccessor(SuccNum);
511
512 // If the edge isn't critical, then BB has a single successor or Succ has a
513 // single pred. Split the block.
514 if (BasicBlock *SP = Succ->getSinglePredecessor()) {
515 // If the successor only has a single pred, split the top of the successor
516 // block.
517 assert(SP == BB && "CFG broken");
518 SP = nullptr;
519 return SplitBlock(Succ, &Succ->front(), DT, LI, MSSAU, BBName,
520 /*Before=*/true);
521 }
522
523 // Otherwise, if BB has a single successor, split it at the bottom of the
524 // block.
525 assert(BB->getTerminator()->getNumSuccessors() == 1 &&
526 "Should have a single succ!");
527 return SplitBlock(BB, BB->getTerminator(), DT, LI, MSSAU, BBName);
528 }
529
530 unsigned
SplitAllCriticalEdges(Function & F,const CriticalEdgeSplittingOptions & Options)531 llvm::SplitAllCriticalEdges(Function &F,
532 const CriticalEdgeSplittingOptions &Options) {
533 unsigned NumBroken = 0;
534 for (BasicBlock &BB : F) {
535 Instruction *TI = BB.getTerminator();
536 if (TI->getNumSuccessors() > 1 && !isa<IndirectBrInst>(TI) &&
537 !isa<CallBrInst>(TI))
538 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
539 if (SplitCriticalEdge(TI, i, Options))
540 ++NumBroken;
541 }
542 return NumBroken;
543 }
544
SplitBlockImpl(BasicBlock * Old,Instruction * SplitPt,DomTreeUpdater * DTU,DominatorTree * DT,LoopInfo * LI,MemorySSAUpdater * MSSAU,const Twine & BBName,bool Before)545 static BasicBlock *SplitBlockImpl(BasicBlock *Old, Instruction *SplitPt,
546 DomTreeUpdater *DTU, DominatorTree *DT,
547 LoopInfo *LI, MemorySSAUpdater *MSSAU,
548 const Twine &BBName, bool Before) {
549 if (Before) {
550 DomTreeUpdater LocalDTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
551 return splitBlockBefore(Old, SplitPt,
552 DTU ? DTU : (DT ? &LocalDTU : nullptr), LI, MSSAU,
553 BBName);
554 }
555 BasicBlock::iterator SplitIt = SplitPt->getIterator();
556 while (isa<PHINode>(SplitIt) || SplitIt->isEHPad())
557 ++SplitIt;
558 std::string Name = BBName.str();
559 BasicBlock *New = Old->splitBasicBlock(
560 SplitIt, Name.empty() ? Old->getName() + ".split" : Name);
561
562 // The new block lives in whichever loop the old one did. This preserves
563 // LCSSA as well, because we force the split point to be after any PHI nodes.
564 if (LI)
565 if (Loop *L = LI->getLoopFor(Old))
566 L->addBasicBlockToLoop(New, *LI);
567
568 if (DTU) {
569 SmallVector<DominatorTree::UpdateType, 8> Updates;
570 // Old dominates New. New node dominates all other nodes dominated by Old.
571 SmallSetVector<BasicBlock *, 8> UniqueSuccessorsOfOld(succ_begin(New),
572 succ_end(New));
573 Updates.push_back({DominatorTree::Insert, Old, New});
574 Updates.reserve(Updates.size() + 2 * UniqueSuccessorsOfOld.size());
575 for (BasicBlock *UniqueSuccessorOfOld : UniqueSuccessorsOfOld) {
576 Updates.push_back({DominatorTree::Insert, New, UniqueSuccessorOfOld});
577 Updates.push_back({DominatorTree::Delete, Old, UniqueSuccessorOfOld});
578 }
579
580 DTU->applyUpdates(Updates);
581 } else if (DT)
582 // Old dominates New. New node dominates all other nodes dominated by Old.
583 if (DomTreeNode *OldNode = DT->getNode(Old)) {
584 std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
585
586 DomTreeNode *NewNode = DT->addNewBlock(New, Old);
587 for (DomTreeNode *I : Children)
588 DT->changeImmediateDominator(I, NewNode);
589 }
590
591 // Move MemoryAccesses still tracked in Old, but part of New now.
592 // Update accesses in successor blocks accordingly.
593 if (MSSAU)
594 MSSAU->moveAllAfterSpliceBlocks(Old, New, &*(New->begin()));
595
596 return New;
597 }
598
SplitBlock(BasicBlock * Old,Instruction * SplitPt,DominatorTree * DT,LoopInfo * LI,MemorySSAUpdater * MSSAU,const Twine & BBName,bool Before)599 BasicBlock *llvm::SplitBlock(BasicBlock *Old, Instruction *SplitPt,
600 DominatorTree *DT, LoopInfo *LI,
601 MemorySSAUpdater *MSSAU, const Twine &BBName,
602 bool Before) {
603 return SplitBlockImpl(Old, SplitPt, /*DTU=*/nullptr, DT, LI, MSSAU, BBName,
604 Before);
605 }
SplitBlock(BasicBlock * Old,Instruction * SplitPt,DomTreeUpdater * DTU,LoopInfo * LI,MemorySSAUpdater * MSSAU,const Twine & BBName,bool Before)606 BasicBlock *llvm::SplitBlock(BasicBlock *Old, Instruction *SplitPt,
607 DomTreeUpdater *DTU, LoopInfo *LI,
608 MemorySSAUpdater *MSSAU, const Twine &BBName,
609 bool Before) {
610 return SplitBlockImpl(Old, SplitPt, DTU, /*DT=*/nullptr, LI, MSSAU, BBName,
611 Before);
612 }
613
splitBlockBefore(BasicBlock * Old,Instruction * SplitPt,DomTreeUpdater * DTU,LoopInfo * LI,MemorySSAUpdater * MSSAU,const Twine & BBName)614 BasicBlock *llvm::splitBlockBefore(BasicBlock *Old, Instruction *SplitPt,
615 DomTreeUpdater *DTU, LoopInfo *LI,
616 MemorySSAUpdater *MSSAU,
617 const Twine &BBName) {
618
619 BasicBlock::iterator SplitIt = SplitPt->getIterator();
620 while (isa<PHINode>(SplitIt) || SplitIt->isEHPad())
621 ++SplitIt;
622 std::string Name = BBName.str();
623 BasicBlock *New = Old->splitBasicBlock(
624 SplitIt, Name.empty() ? Old->getName() + ".split" : Name,
625 /* Before=*/true);
626
627 // The new block lives in whichever loop the old one did. This preserves
628 // LCSSA as well, because we force the split point to be after any PHI nodes.
629 if (LI)
630 if (Loop *L = LI->getLoopFor(Old))
631 L->addBasicBlockToLoop(New, *LI);
632
633 if (DTU) {
634 SmallVector<DominatorTree::UpdateType, 8> DTUpdates;
635 // New dominates Old. The predecessor nodes of the Old node dominate
636 // New node.
637 SmallSetVector<BasicBlock *, 8> UniquePredecessorsOfOld(pred_begin(New),
638 pred_end(New));
639 DTUpdates.push_back({DominatorTree::Insert, New, Old});
640 DTUpdates.reserve(DTUpdates.size() + 2 * UniquePredecessorsOfOld.size());
641 for (BasicBlock *UniquePredecessorOfOld : UniquePredecessorsOfOld) {
642 DTUpdates.push_back({DominatorTree::Insert, UniquePredecessorOfOld, New});
643 DTUpdates.push_back({DominatorTree::Delete, UniquePredecessorOfOld, Old});
644 }
645
646 DTU->applyUpdates(DTUpdates);
647
648 // Move MemoryAccesses still tracked in Old, but part of New now.
649 // Update accesses in successor blocks accordingly.
650 if (MSSAU) {
651 MSSAU->applyUpdates(DTUpdates, DTU->getDomTree());
652 if (VerifyMemorySSA)
653 MSSAU->getMemorySSA()->verifyMemorySSA();
654 }
655 }
656 return New;
657 }
658
659 /// Update DominatorTree, LoopInfo, and LCCSA analysis information.
UpdateAnalysisInformation(BasicBlock * OldBB,BasicBlock * NewBB,ArrayRef<BasicBlock * > Preds,DomTreeUpdater * DTU,DominatorTree * DT,LoopInfo * LI,MemorySSAUpdater * MSSAU,bool PreserveLCSSA,bool & HasLoopExit)660 static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
661 ArrayRef<BasicBlock *> Preds,
662 DomTreeUpdater *DTU, DominatorTree *DT,
663 LoopInfo *LI, MemorySSAUpdater *MSSAU,
664 bool PreserveLCSSA, bool &HasLoopExit) {
665 // Update dominator tree if available.
666 if (DTU) {
667 // Recalculation of DomTree is needed when updating a forward DomTree and
668 // the Entry BB is replaced.
669 if (NewBB == &NewBB->getParent()->getEntryBlock() && DTU->hasDomTree()) {
670 // The entry block was removed and there is no external interface for
671 // the dominator tree to be notified of this change. In this corner-case
672 // we recalculate the entire tree.
673 DTU->recalculate(*NewBB->getParent());
674 } else {
675 // Split block expects NewBB to have a non-empty set of predecessors.
676 SmallVector<DominatorTree::UpdateType, 8> Updates;
677 SmallSetVector<BasicBlock *, 8> UniquePreds(Preds.begin(), Preds.end());
678 Updates.push_back({DominatorTree::Insert, NewBB, OldBB});
679 Updates.reserve(Updates.size() + 2 * UniquePreds.size());
680 for (auto *UniquePred : UniquePreds) {
681 Updates.push_back({DominatorTree::Insert, UniquePred, NewBB});
682 Updates.push_back({DominatorTree::Delete, UniquePred, OldBB});
683 }
684 DTU->applyUpdates(Updates);
685 }
686 } else if (DT) {
687 if (OldBB == DT->getRootNode()->getBlock()) {
688 assert(NewBB == &NewBB->getParent()->getEntryBlock());
689 DT->setNewRoot(NewBB);
690 } else {
691 // Split block expects NewBB to have a non-empty set of predecessors.
692 DT->splitBlock(NewBB);
693 }
694 }
695
696 // Update MemoryPhis after split if MemorySSA is available
697 if (MSSAU)
698 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(OldBB, NewBB, Preds);
699
700 // The rest of the logic is only relevant for updating the loop structures.
701 if (!LI)
702 return;
703
704 if (DTU && DTU->hasDomTree())
705 DT = &DTU->getDomTree();
706 assert(DT && "DT should be available to update LoopInfo!");
707 Loop *L = LI->getLoopFor(OldBB);
708
709 // If we need to preserve loop analyses, collect some information about how
710 // this split will affect loops.
711 bool IsLoopEntry = !!L;
712 bool SplitMakesNewLoopHeader = false;
713 for (BasicBlock *Pred : Preds) {
714 // Preds that are not reachable from entry should not be used to identify if
715 // OldBB is a loop entry or if SplitMakesNewLoopHeader. Unreachable blocks
716 // are not within any loops, so we incorrectly mark SplitMakesNewLoopHeader
717 // as true and make the NewBB the header of some loop. This breaks LI.
718 if (!DT->isReachableFromEntry(Pred))
719 continue;
720 // If we need to preserve LCSSA, determine if any of the preds is a loop
721 // exit.
722 if (PreserveLCSSA)
723 if (Loop *PL = LI->getLoopFor(Pred))
724 if (!PL->contains(OldBB))
725 HasLoopExit = true;
726
727 // If we need to preserve LoopInfo, note whether any of the preds crosses
728 // an interesting loop boundary.
729 if (!L)
730 continue;
731 if (L->contains(Pred))
732 IsLoopEntry = false;
733 else
734 SplitMakesNewLoopHeader = true;
735 }
736
737 // Unless we have a loop for OldBB, nothing else to do here.
738 if (!L)
739 return;
740
741 if (IsLoopEntry) {
742 // Add the new block to the nearest enclosing loop (and not an adjacent
743 // loop). To find this, examine each of the predecessors and determine which
744 // loops enclose them, and select the most-nested loop which contains the
745 // loop containing the block being split.
746 Loop *InnermostPredLoop = nullptr;
747 for (BasicBlock *Pred : Preds) {
748 if (Loop *PredLoop = LI->getLoopFor(Pred)) {
749 // Seek a loop which actually contains the block being split (to avoid
750 // adjacent loops).
751 while (PredLoop && !PredLoop->contains(OldBB))
752 PredLoop = PredLoop->getParentLoop();
753
754 // Select the most-nested of these loops which contains the block.
755 if (PredLoop && PredLoop->contains(OldBB) &&
756 (!InnermostPredLoop ||
757 InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
758 InnermostPredLoop = PredLoop;
759 }
760 }
761
762 if (InnermostPredLoop)
763 InnermostPredLoop->addBasicBlockToLoop(NewBB, *LI);
764 } else {
765 L->addBasicBlockToLoop(NewBB, *LI);
766 if (SplitMakesNewLoopHeader)
767 L->moveToHeader(NewBB);
768 }
769 }
770
771 /// Update the PHI nodes in OrigBB to include the values coming from NewBB.
772 /// This also updates AliasAnalysis, if available.
UpdatePHINodes(BasicBlock * OrigBB,BasicBlock * NewBB,ArrayRef<BasicBlock * > Preds,BranchInst * BI,bool HasLoopExit)773 static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
774 ArrayRef<BasicBlock *> Preds, BranchInst *BI,
775 bool HasLoopExit) {
776 // Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
777 SmallPtrSet<BasicBlock *, 16> PredSet(Preds.begin(), Preds.end());
778 for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(I); ) {
779 PHINode *PN = cast<PHINode>(I++);
780
781 // Check to see if all of the values coming in are the same. If so, we
782 // don't need to create a new PHI node, unless it's needed for LCSSA.
783 Value *InVal = nullptr;
784 if (!HasLoopExit) {
785 InVal = PN->getIncomingValueForBlock(Preds[0]);
786 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
787 if (!PredSet.count(PN->getIncomingBlock(i)))
788 continue;
789 if (!InVal)
790 InVal = PN->getIncomingValue(i);
791 else if (InVal != PN->getIncomingValue(i)) {
792 InVal = nullptr;
793 break;
794 }
795 }
796 }
797
798 if (InVal) {
799 // If all incoming values for the new PHI would be the same, just don't
800 // make a new PHI. Instead, just remove the incoming values from the old
801 // PHI.
802
803 // NOTE! This loop walks backwards for a reason! First off, this minimizes
804 // the cost of removal if we end up removing a large number of values, and
805 // second off, this ensures that the indices for the incoming values
806 // aren't invalidated when we remove one.
807 for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i)
808 if (PredSet.count(PN->getIncomingBlock(i)))
809 PN->removeIncomingValue(i, false);
810
811 // Add an incoming value to the PHI node in the loop for the preheader
812 // edge.
813 PN->addIncoming(InVal, NewBB);
814 continue;
815 }
816
817 // If the values coming into the block are not the same, we need a new
818 // PHI.
819 // Create the new PHI node, insert it into NewBB at the end of the block
820 PHINode *NewPHI =
821 PHINode::Create(PN->getType(), Preds.size(), PN->getName() + ".ph", BI);
822
823 // NOTE! This loop walks backwards for a reason! First off, this minimizes
824 // the cost of removal if we end up removing a large number of values, and
825 // second off, this ensures that the indices for the incoming values aren't
826 // invalidated when we remove one.
827 for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i) {
828 BasicBlock *IncomingBB = PN->getIncomingBlock(i);
829 if (PredSet.count(IncomingBB)) {
830 Value *V = PN->removeIncomingValue(i, false);
831 NewPHI->addIncoming(V, IncomingBB);
832 }
833 }
834
835 PN->addIncoming(NewPHI, NewBB);
836 }
837 }
838
839 static void SplitLandingPadPredecessorsImpl(
840 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
841 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
842 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
843 MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
844
845 static BasicBlock *
SplitBlockPredecessorsImpl(BasicBlock * BB,ArrayRef<BasicBlock * > Preds,const char * Suffix,DomTreeUpdater * DTU,DominatorTree * DT,LoopInfo * LI,MemorySSAUpdater * MSSAU,bool PreserveLCSSA)846 SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
847 const char *Suffix, DomTreeUpdater *DTU,
848 DominatorTree *DT, LoopInfo *LI,
849 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
850 // Do not attempt to split that which cannot be split.
851 if (!BB->canSplitPredecessors())
852 return nullptr;
853
854 // For the landingpads we need to act a bit differently.
855 // Delegate this work to the SplitLandingPadPredecessors.
856 if (BB->isLandingPad()) {
857 SmallVector<BasicBlock*, 2> NewBBs;
858 std::string NewName = std::string(Suffix) + ".split-lp";
859
860 SplitLandingPadPredecessorsImpl(BB, Preds, Suffix, NewName.c_str(), NewBBs,
861 DTU, DT, LI, MSSAU, PreserveLCSSA);
862 return NewBBs[0];
863 }
864
865 // Create new basic block, insert right before the original block.
866 BasicBlock *NewBB = BasicBlock::Create(
867 BB->getContext(), BB->getName() + Suffix, BB->getParent(), BB);
868
869 // The new block unconditionally branches to the old block.
870 BranchInst *BI = BranchInst::Create(BB, NewBB);
871
872 Loop *L = nullptr;
873 BasicBlock *OldLatch = nullptr;
874 // Splitting the predecessors of a loop header creates a preheader block.
875 if (LI && LI->isLoopHeader(BB)) {
876 L = LI->getLoopFor(BB);
877 // Using the loop start line number prevents debuggers stepping into the
878 // loop body for this instruction.
879 BI->setDebugLoc(L->getStartLoc());
880
881 // If BB is the header of the Loop, it is possible that the loop is
882 // modified, such that the current latch does not remain the latch of the
883 // loop. If that is the case, the loop metadata from the current latch needs
884 // to be applied to the new latch.
885 OldLatch = L->getLoopLatch();
886 } else
887 BI->setDebugLoc(BB->getFirstNonPHIOrDbg()->getDebugLoc());
888
889 // Move the edges from Preds to point to NewBB instead of BB.
890 for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
891 // This is slightly more strict than necessary; the minimum requirement
892 // is that there be no more than one indirectbr branching to BB. And
893 // all BlockAddress uses would need to be updated.
894 assert(!isa<IndirectBrInst>(Preds[i]->getTerminator()) &&
895 "Cannot split an edge from an IndirectBrInst");
896 assert(!isa<CallBrInst>(Preds[i]->getTerminator()) &&
897 "Cannot split an edge from a CallBrInst");
898 Preds[i]->getTerminator()->replaceUsesOfWith(BB, NewBB);
899 }
900
901 // Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
902 // node becomes an incoming value for BB's phi node. However, if the Preds
903 // list is empty, we need to insert dummy entries into the PHI nodes in BB to
904 // account for the newly created predecessor.
905 if (Preds.empty()) {
906 // Insert dummy values as the incoming value.
907 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++I)
908 cast<PHINode>(I)->addIncoming(UndefValue::get(I->getType()), NewBB);
909 }
910
911 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
912 bool HasLoopExit = false;
913 UpdateAnalysisInformation(BB, NewBB, Preds, DTU, DT, LI, MSSAU, PreserveLCSSA,
914 HasLoopExit);
915
916 if (!Preds.empty()) {
917 // Update the PHI nodes in BB with the values coming from NewBB.
918 UpdatePHINodes(BB, NewBB, Preds, BI, HasLoopExit);
919 }
920
921 if (OldLatch) {
922 BasicBlock *NewLatch = L->getLoopLatch();
923 if (NewLatch != OldLatch) {
924 MDNode *MD = OldLatch->getTerminator()->getMetadata("llvm.loop");
925 NewLatch->getTerminator()->setMetadata("llvm.loop", MD);
926 OldLatch->getTerminator()->setMetadata("llvm.loop", nullptr);
927 }
928 }
929
930 return NewBB;
931 }
932
SplitBlockPredecessors(BasicBlock * BB,ArrayRef<BasicBlock * > Preds,const char * Suffix,DominatorTree * DT,LoopInfo * LI,MemorySSAUpdater * MSSAU,bool PreserveLCSSA)933 BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
934 ArrayRef<BasicBlock *> Preds,
935 const char *Suffix, DominatorTree *DT,
936 LoopInfo *LI, MemorySSAUpdater *MSSAU,
937 bool PreserveLCSSA) {
938 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, /*DTU=*/nullptr, DT, LI,
939 MSSAU, PreserveLCSSA);
940 }
SplitBlockPredecessors(BasicBlock * BB,ArrayRef<BasicBlock * > Preds,const char * Suffix,DomTreeUpdater * DTU,LoopInfo * LI,MemorySSAUpdater * MSSAU,bool PreserveLCSSA)941 BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
942 ArrayRef<BasicBlock *> Preds,
943 const char *Suffix,
944 DomTreeUpdater *DTU, LoopInfo *LI,
945 MemorySSAUpdater *MSSAU,
946 bool PreserveLCSSA) {
947 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, DTU,
948 /*DT=*/nullptr, LI, MSSAU, PreserveLCSSA);
949 }
950
SplitLandingPadPredecessorsImpl(BasicBlock * OrigBB,ArrayRef<BasicBlock * > Preds,const char * Suffix1,const char * Suffix2,SmallVectorImpl<BasicBlock * > & NewBBs,DomTreeUpdater * DTU,DominatorTree * DT,LoopInfo * LI,MemorySSAUpdater * MSSAU,bool PreserveLCSSA)951 static void SplitLandingPadPredecessorsImpl(
952 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
953 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
954 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
955 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
956 assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
957
958 // Create a new basic block for OrigBB's predecessors listed in Preds. Insert
959 // it right before the original block.
960 BasicBlock *NewBB1 = BasicBlock::Create(OrigBB->getContext(),
961 OrigBB->getName() + Suffix1,
962 OrigBB->getParent(), OrigBB);
963 NewBBs.push_back(NewBB1);
964
965 // The new block unconditionally branches to the old block.
966 BranchInst *BI1 = BranchInst::Create(OrigBB, NewBB1);
967 BI1->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
968
969 // Move the edges from Preds to point to NewBB1 instead of OrigBB.
970 for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
971 // This is slightly more strict than necessary; the minimum requirement
972 // is that there be no more than one indirectbr branching to BB. And
973 // all BlockAddress uses would need to be updated.
974 assert(!isa<IndirectBrInst>(Preds[i]->getTerminator()) &&
975 "Cannot split an edge from an IndirectBrInst");
976 Preds[i]->getTerminator()->replaceUsesOfWith(OrigBB, NewBB1);
977 }
978
979 bool HasLoopExit = false;
980 UpdateAnalysisInformation(OrigBB, NewBB1, Preds, DTU, DT, LI, MSSAU,
981 PreserveLCSSA, HasLoopExit);
982
983 // Update the PHI nodes in OrigBB with the values coming from NewBB1.
984 UpdatePHINodes(OrigBB, NewBB1, Preds, BI1, HasLoopExit);
985
986 // Move the remaining edges from OrigBB to point to NewBB2.
987 SmallVector<BasicBlock*, 8> NewBB2Preds;
988 for (pred_iterator i = pred_begin(OrigBB), e = pred_end(OrigBB);
989 i != e; ) {
990 BasicBlock *Pred = *i++;
991 if (Pred == NewBB1) continue;
992 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
993 "Cannot split an edge from an IndirectBrInst");
994 NewBB2Preds.push_back(Pred);
995 e = pred_end(OrigBB);
996 }
997
998 BasicBlock *NewBB2 = nullptr;
999 if (!NewBB2Preds.empty()) {
1000 // Create another basic block for the rest of OrigBB's predecessors.
1001 NewBB2 = BasicBlock::Create(OrigBB->getContext(),
1002 OrigBB->getName() + Suffix2,
1003 OrigBB->getParent(), OrigBB);
1004 NewBBs.push_back(NewBB2);
1005
1006 // The new block unconditionally branches to the old block.
1007 BranchInst *BI2 = BranchInst::Create(OrigBB, NewBB2);
1008 BI2->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
1009
1010 // Move the remaining edges from OrigBB to point to NewBB2.
1011 for (BasicBlock *NewBB2Pred : NewBB2Preds)
1012 NewBB2Pred->getTerminator()->replaceUsesOfWith(OrigBB, NewBB2);
1013
1014 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1015 HasLoopExit = false;
1016 UpdateAnalysisInformation(OrigBB, NewBB2, NewBB2Preds, DTU, DT, LI, MSSAU,
1017 PreserveLCSSA, HasLoopExit);
1018
1019 // Update the PHI nodes in OrigBB with the values coming from NewBB2.
1020 UpdatePHINodes(OrigBB, NewBB2, NewBB2Preds, BI2, HasLoopExit);
1021 }
1022
1023 LandingPadInst *LPad = OrigBB->getLandingPadInst();
1024 Instruction *Clone1 = LPad->clone();
1025 Clone1->setName(Twine("lpad") + Suffix1);
1026 NewBB1->getInstList().insert(NewBB1->getFirstInsertionPt(), Clone1);
1027
1028 if (NewBB2) {
1029 Instruction *Clone2 = LPad->clone();
1030 Clone2->setName(Twine("lpad") + Suffix2);
1031 NewBB2->getInstList().insert(NewBB2->getFirstInsertionPt(), Clone2);
1032
1033 // Create a PHI node for the two cloned landingpad instructions only
1034 // if the original landingpad instruction has some uses.
1035 if (!LPad->use_empty()) {
1036 assert(!LPad->getType()->isTokenTy() &&
1037 "Split cannot be applied if LPad is token type. Otherwise an "
1038 "invalid PHINode of token type would be created.");
1039 PHINode *PN = PHINode::Create(LPad->getType(), 2, "lpad.phi", LPad);
1040 PN->addIncoming(Clone1, NewBB1);
1041 PN->addIncoming(Clone2, NewBB2);
1042 LPad->replaceAllUsesWith(PN);
1043 }
1044 LPad->eraseFromParent();
1045 } else {
1046 // There is no second clone. Just replace the landing pad with the first
1047 // clone.
1048 LPad->replaceAllUsesWith(Clone1);
1049 LPad->eraseFromParent();
1050 }
1051 }
1052
SplitLandingPadPredecessors(BasicBlock * OrigBB,ArrayRef<BasicBlock * > Preds,const char * Suffix1,const char * Suffix2,SmallVectorImpl<BasicBlock * > & NewBBs,DominatorTree * DT,LoopInfo * LI,MemorySSAUpdater * MSSAU,bool PreserveLCSSA)1053 void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
1054 ArrayRef<BasicBlock *> Preds,
1055 const char *Suffix1, const char *Suffix2,
1056 SmallVectorImpl<BasicBlock *> &NewBBs,
1057 DominatorTree *DT, LoopInfo *LI,
1058 MemorySSAUpdater *MSSAU,
1059 bool PreserveLCSSA) {
1060 return SplitLandingPadPredecessorsImpl(
1061 OrigBB, Preds, Suffix1, Suffix2, NewBBs,
1062 /*DTU=*/nullptr, DT, LI, MSSAU, PreserveLCSSA);
1063 }
SplitLandingPadPredecessors(BasicBlock * OrigBB,ArrayRef<BasicBlock * > Preds,const char * Suffix1,const char * Suffix2,SmallVectorImpl<BasicBlock * > & NewBBs,DomTreeUpdater * DTU,LoopInfo * LI,MemorySSAUpdater * MSSAU,bool PreserveLCSSA)1064 void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
1065 ArrayRef<BasicBlock *> Preds,
1066 const char *Suffix1, const char *Suffix2,
1067 SmallVectorImpl<BasicBlock *> &NewBBs,
1068 DomTreeUpdater *DTU, LoopInfo *LI,
1069 MemorySSAUpdater *MSSAU,
1070 bool PreserveLCSSA) {
1071 return SplitLandingPadPredecessorsImpl(OrigBB, Preds, Suffix1, Suffix2,
1072 NewBBs, DTU, /*DT=*/nullptr, LI, MSSAU,
1073 PreserveLCSSA);
1074 }
1075
FoldReturnIntoUncondBranch(ReturnInst * RI,BasicBlock * BB,BasicBlock * Pred,DomTreeUpdater * DTU)1076 ReturnInst *llvm::FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
1077 BasicBlock *Pred,
1078 DomTreeUpdater *DTU) {
1079 Instruction *UncondBranch = Pred->getTerminator();
1080 // Clone the return and add it to the end of the predecessor.
1081 Instruction *NewRet = RI->clone();
1082 Pred->getInstList().push_back(NewRet);
1083
1084 // If the return instruction returns a value, and if the value was a
1085 // PHI node in "BB", propagate the right value into the return.
1086 for (User::op_iterator i = NewRet->op_begin(), e = NewRet->op_end();
1087 i != e; ++i) {
1088 Value *V = *i;
1089 Instruction *NewBC = nullptr;
1090 if (BitCastInst *BCI = dyn_cast<BitCastInst>(V)) {
1091 // Return value might be bitcasted. Clone and insert it before the
1092 // return instruction.
1093 V = BCI->getOperand(0);
1094 NewBC = BCI->clone();
1095 Pred->getInstList().insert(NewRet->getIterator(), NewBC);
1096 *i = NewBC;
1097 }
1098
1099 Instruction *NewEV = nullptr;
1100 if (ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(V)) {
1101 V = EVI->getOperand(0);
1102 NewEV = EVI->clone();
1103 if (NewBC) {
1104 NewBC->setOperand(0, NewEV);
1105 Pred->getInstList().insert(NewBC->getIterator(), NewEV);
1106 } else {
1107 Pred->getInstList().insert(NewRet->getIterator(), NewEV);
1108 *i = NewEV;
1109 }
1110 }
1111
1112 if (PHINode *PN = dyn_cast<PHINode>(V)) {
1113 if (PN->getParent() == BB) {
1114 if (NewEV) {
1115 NewEV->setOperand(0, PN->getIncomingValueForBlock(Pred));
1116 } else if (NewBC)
1117 NewBC->setOperand(0, PN->getIncomingValueForBlock(Pred));
1118 else
1119 *i = PN->getIncomingValueForBlock(Pred);
1120 }
1121 }
1122 }
1123
1124 // Update any PHI nodes in the returning block to realize that we no
1125 // longer branch to them.
1126 BB->removePredecessor(Pred);
1127 UncondBranch->eraseFromParent();
1128
1129 if (DTU)
1130 DTU->applyUpdates({{DominatorTree::Delete, Pred, BB}});
1131
1132 return cast<ReturnInst>(NewRet);
1133 }
1134
1135 static Instruction *
SplitBlockAndInsertIfThenImpl(Value * Cond,Instruction * SplitBefore,bool Unreachable,MDNode * BranchWeights,DomTreeUpdater * DTU,DominatorTree * DT,LoopInfo * LI,BasicBlock * ThenBlock)1136 SplitBlockAndInsertIfThenImpl(Value *Cond, Instruction *SplitBefore,
1137 bool Unreachable, MDNode *BranchWeights,
1138 DomTreeUpdater *DTU, DominatorTree *DT,
1139 LoopInfo *LI, BasicBlock *ThenBlock) {
1140 SmallVector<DominatorTree::UpdateType, 8> Updates;
1141 BasicBlock *Head = SplitBefore->getParent();
1142 BasicBlock *Tail = Head->splitBasicBlock(SplitBefore->getIterator());
1143 if (DTU) {
1144 SmallSetVector<BasicBlock *, 8> UniqueSuccessorsOfHead(succ_begin(Tail),
1145 succ_end(Tail));
1146 Updates.push_back({DominatorTree::Insert, Head, Tail});
1147 Updates.reserve(Updates.size() + 2 * UniqueSuccessorsOfHead.size());
1148 for (BasicBlock *UniqueSuccessorOfHead : UniqueSuccessorsOfHead) {
1149 Updates.push_back({DominatorTree::Insert, Tail, UniqueSuccessorOfHead});
1150 Updates.push_back({DominatorTree::Delete, Head, UniqueSuccessorOfHead});
1151 }
1152 }
1153 Instruction *HeadOldTerm = Head->getTerminator();
1154 LLVMContext &C = Head->getContext();
1155 Instruction *CheckTerm;
1156 bool CreateThenBlock = (ThenBlock == nullptr);
1157 if (CreateThenBlock) {
1158 ThenBlock = BasicBlock::Create(C, "", Head->getParent(), Tail);
1159 if (Unreachable)
1160 CheckTerm = new UnreachableInst(C, ThenBlock);
1161 else {
1162 CheckTerm = BranchInst::Create(Tail, ThenBlock);
1163 if (DTU)
1164 Updates.push_back({DominatorTree::Insert, ThenBlock, Tail});
1165 }
1166 CheckTerm->setDebugLoc(SplitBefore->getDebugLoc());
1167 } else
1168 CheckTerm = ThenBlock->getTerminator();
1169 BranchInst *HeadNewTerm =
1170 BranchInst::Create(/*ifTrue*/ ThenBlock, /*ifFalse*/ Tail, Cond);
1171 if (DTU)
1172 Updates.push_back({DominatorTree::Insert, Head, ThenBlock});
1173 HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
1174 ReplaceInstWithInst(HeadOldTerm, HeadNewTerm);
1175
1176 if (DTU)
1177 DTU->applyUpdates(Updates);
1178 else if (DT) {
1179 if (DomTreeNode *OldNode = DT->getNode(Head)) {
1180 std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
1181
1182 DomTreeNode *NewNode = DT->addNewBlock(Tail, Head);
1183 for (DomTreeNode *Child : Children)
1184 DT->changeImmediateDominator(Child, NewNode);
1185
1186 // Head dominates ThenBlock.
1187 if (CreateThenBlock)
1188 DT->addNewBlock(ThenBlock, Head);
1189 else
1190 DT->changeImmediateDominator(ThenBlock, Head);
1191 }
1192 }
1193
1194 if (LI) {
1195 if (Loop *L = LI->getLoopFor(Head)) {
1196 L->addBasicBlockToLoop(ThenBlock, *LI);
1197 L->addBasicBlockToLoop(Tail, *LI);
1198 }
1199 }
1200
1201 return CheckTerm;
1202 }
1203
SplitBlockAndInsertIfThen(Value * Cond,Instruction * SplitBefore,bool Unreachable,MDNode * BranchWeights,DominatorTree * DT,LoopInfo * LI,BasicBlock * ThenBlock)1204 Instruction *llvm::SplitBlockAndInsertIfThen(Value *Cond,
1205 Instruction *SplitBefore,
1206 bool Unreachable,
1207 MDNode *BranchWeights,
1208 DominatorTree *DT, LoopInfo *LI,
1209 BasicBlock *ThenBlock) {
1210 return SplitBlockAndInsertIfThenImpl(Cond, SplitBefore, Unreachable,
1211 BranchWeights,
1212 /*DTU=*/nullptr, DT, LI, ThenBlock);
1213 }
SplitBlockAndInsertIfThen(Value * Cond,Instruction * SplitBefore,bool Unreachable,MDNode * BranchWeights,DomTreeUpdater * DTU,LoopInfo * LI,BasicBlock * ThenBlock)1214 Instruction *llvm::SplitBlockAndInsertIfThen(Value *Cond,
1215 Instruction *SplitBefore,
1216 bool Unreachable,
1217 MDNode *BranchWeights,
1218 DomTreeUpdater *DTU, LoopInfo *LI,
1219 BasicBlock *ThenBlock) {
1220 return SplitBlockAndInsertIfThenImpl(Cond, SplitBefore, Unreachable,
1221 BranchWeights, DTU, /*DT=*/nullptr, LI,
1222 ThenBlock);
1223 }
1224
SplitBlockAndInsertIfThenElse(Value * Cond,Instruction * SplitBefore,Instruction ** ThenTerm,Instruction ** ElseTerm,MDNode * BranchWeights)1225 void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore,
1226 Instruction **ThenTerm,
1227 Instruction **ElseTerm,
1228 MDNode *BranchWeights) {
1229 BasicBlock *Head = SplitBefore->getParent();
1230 BasicBlock *Tail = Head->splitBasicBlock(SplitBefore->getIterator());
1231 Instruction *HeadOldTerm = Head->getTerminator();
1232 LLVMContext &C = Head->getContext();
1233 BasicBlock *ThenBlock = BasicBlock::Create(C, "", Head->getParent(), Tail);
1234 BasicBlock *ElseBlock = BasicBlock::Create(C, "", Head->getParent(), Tail);
1235 *ThenTerm = BranchInst::Create(Tail, ThenBlock);
1236 (*ThenTerm)->setDebugLoc(SplitBefore->getDebugLoc());
1237 *ElseTerm = BranchInst::Create(Tail, ElseBlock);
1238 (*ElseTerm)->setDebugLoc(SplitBefore->getDebugLoc());
1239 BranchInst *HeadNewTerm =
1240 BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/ElseBlock, Cond);
1241 HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
1242 ReplaceInstWithInst(HeadOldTerm, HeadNewTerm);
1243 }
1244
GetIfCondition(BasicBlock * BB,BasicBlock * & IfTrue,BasicBlock * & IfFalse)1245 Value *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
1246 BasicBlock *&IfFalse) {
1247 PHINode *SomePHI = dyn_cast<PHINode>(BB->begin());
1248 BasicBlock *Pred1 = nullptr;
1249 BasicBlock *Pred2 = nullptr;
1250
1251 if (SomePHI) {
1252 if (SomePHI->getNumIncomingValues() != 2)
1253 return nullptr;
1254 Pred1 = SomePHI->getIncomingBlock(0);
1255 Pred2 = SomePHI->getIncomingBlock(1);
1256 } else {
1257 pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
1258 if (PI == PE) // No predecessor
1259 return nullptr;
1260 Pred1 = *PI++;
1261 if (PI == PE) // Only one predecessor
1262 return nullptr;
1263 Pred2 = *PI++;
1264 if (PI != PE) // More than two predecessors
1265 return nullptr;
1266 }
1267
1268 // We can only handle branches. Other control flow will be lowered to
1269 // branches if possible anyway.
1270 BranchInst *Pred1Br = dyn_cast<BranchInst>(Pred1->getTerminator());
1271 BranchInst *Pred2Br = dyn_cast<BranchInst>(Pred2->getTerminator());
1272 if (!Pred1Br || !Pred2Br)
1273 return nullptr;
1274
1275 // Eliminate code duplication by ensuring that Pred1Br is conditional if
1276 // either are.
1277 if (Pred2Br->isConditional()) {
1278 // If both branches are conditional, we don't have an "if statement". In
1279 // reality, we could transform this case, but since the condition will be
1280 // required anyway, we stand no chance of eliminating it, so the xform is
1281 // probably not profitable.
1282 if (Pred1Br->isConditional())
1283 return nullptr;
1284
1285 std::swap(Pred1, Pred2);
1286 std::swap(Pred1Br, Pred2Br);
1287 }
1288
1289 if (Pred1Br->isConditional()) {
1290 // The only thing we have to watch out for here is to make sure that Pred2
1291 // doesn't have incoming edges from other blocks. If it does, the condition
1292 // doesn't dominate BB.
1293 if (!Pred2->getSinglePredecessor())
1294 return nullptr;
1295
1296 // If we found a conditional branch predecessor, make sure that it branches
1297 // to BB and Pred2Br. If it doesn't, this isn't an "if statement".
1298 if (Pred1Br->getSuccessor(0) == BB &&
1299 Pred1Br->getSuccessor(1) == Pred2) {
1300 IfTrue = Pred1;
1301 IfFalse = Pred2;
1302 } else if (Pred1Br->getSuccessor(0) == Pred2 &&
1303 Pred1Br->getSuccessor(1) == BB) {
1304 IfTrue = Pred2;
1305 IfFalse = Pred1;
1306 } else {
1307 // We know that one arm of the conditional goes to BB, so the other must
1308 // go somewhere unrelated, and this must not be an "if statement".
1309 return nullptr;
1310 }
1311
1312 return Pred1Br->getCondition();
1313 }
1314
1315 // Ok, if we got here, both predecessors end with an unconditional branch to
1316 // BB. Don't panic! If both blocks only have a single (identical)
1317 // predecessor, and THAT is a conditional branch, then we're all ok!
1318 BasicBlock *CommonPred = Pred1->getSinglePredecessor();
1319 if (CommonPred == nullptr || CommonPred != Pred2->getSinglePredecessor())
1320 return nullptr;
1321
1322 // Otherwise, if this is a conditional branch, then we can use it!
1323 BranchInst *BI = dyn_cast<BranchInst>(CommonPred->getTerminator());
1324 if (!BI) return nullptr;
1325
1326 assert(BI->isConditional() && "Two successors but not conditional?");
1327 if (BI->getSuccessor(0) == Pred1) {
1328 IfTrue = Pred1;
1329 IfFalse = Pred2;
1330 } else {
1331 IfTrue = Pred2;
1332 IfFalse = Pred1;
1333 }
1334 return BI->getCondition();
1335 }
1336
1337 // After creating a control flow hub, the operands of PHINodes in an outgoing
1338 // block Out no longer match the predecessors of that block. Predecessors of Out
1339 // that are incoming blocks to the hub are now replaced by just one edge from
1340 // the hub. To match this new control flow, the corresponding values from each
1341 // PHINode must now be moved a new PHINode in the first guard block of the hub.
1342 //
1343 // This operation cannot be performed with SSAUpdater, because it involves one
1344 // new use: If the block Out is in the list of Incoming blocks, then the newly
1345 // created PHI in the Hub will use itself along that edge from Out to Hub.
reconnectPhis(BasicBlock * Out,BasicBlock * GuardBlock,const SetVector<BasicBlock * > & Incoming,BasicBlock * FirstGuardBlock)1346 static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock,
1347 const SetVector<BasicBlock *> &Incoming,
1348 BasicBlock *FirstGuardBlock) {
1349 auto I = Out->begin();
1350 while (I != Out->end() && isa<PHINode>(I)) {
1351 auto Phi = cast<PHINode>(I);
1352 auto NewPhi =
1353 PHINode::Create(Phi->getType(), Incoming.size(),
1354 Phi->getName() + ".moved", &FirstGuardBlock->back());
1355 for (auto In : Incoming) {
1356 Value *V = UndefValue::get(Phi->getType());
1357 if (In == Out) {
1358 V = NewPhi;
1359 } else if (Phi->getBasicBlockIndex(In) != -1) {
1360 V = Phi->removeIncomingValue(In, false);
1361 }
1362 NewPhi->addIncoming(V, In);
1363 }
1364 assert(NewPhi->getNumIncomingValues() == Incoming.size());
1365 if (Phi->getNumOperands() == 0) {
1366 Phi->replaceAllUsesWith(NewPhi);
1367 I = Phi->eraseFromParent();
1368 continue;
1369 }
1370 Phi->addIncoming(NewPhi, GuardBlock);
1371 ++I;
1372 }
1373 }
1374
1375 using BBPredicates = DenseMap<BasicBlock *, PHINode *>;
1376 using BBSetVector = SetVector<BasicBlock *>;
1377
1378 // Redirects the terminator of the incoming block to the first guard
1379 // block in the hub. The condition of the original terminator (if it
1380 // was conditional) and its original successors are returned as a
1381 // tuple <condition, succ0, succ1>. The function additionally filters
1382 // out successors that are not in the set of outgoing blocks.
1383 //
1384 // - condition is non-null iff the branch is conditional.
1385 // - Succ1 is non-null iff the sole/taken target is an outgoing block.
1386 // - Succ2 is non-null iff condition is non-null and the fallthrough
1387 // target is an outgoing block.
1388 static std::tuple<Value *, BasicBlock *, BasicBlock *>
redirectToHub(BasicBlock * BB,BasicBlock * FirstGuardBlock,const BBSetVector & Outgoing)1389 redirectToHub(BasicBlock *BB, BasicBlock *FirstGuardBlock,
1390 const BBSetVector &Outgoing) {
1391 auto Branch = cast<BranchInst>(BB->getTerminator());
1392 auto Condition = Branch->isConditional() ? Branch->getCondition() : nullptr;
1393
1394 BasicBlock *Succ0 = Branch->getSuccessor(0);
1395 BasicBlock *Succ1 = nullptr;
1396 Succ0 = Outgoing.count(Succ0) ? Succ0 : nullptr;
1397
1398 if (Branch->isUnconditional()) {
1399 Branch->setSuccessor(0, FirstGuardBlock);
1400 assert(Succ0);
1401 } else {
1402 Succ1 = Branch->getSuccessor(1);
1403 Succ1 = Outgoing.count(Succ1) ? Succ1 : nullptr;
1404 assert(Succ0 || Succ1);
1405 if (Succ0 && !Succ1) {
1406 Branch->setSuccessor(0, FirstGuardBlock);
1407 } else if (Succ1 && !Succ0) {
1408 Branch->setSuccessor(1, FirstGuardBlock);
1409 } else {
1410 Branch->eraseFromParent();
1411 BranchInst::Create(FirstGuardBlock, BB);
1412 }
1413 }
1414
1415 assert(Succ0 || Succ1);
1416 return std::make_tuple(Condition, Succ0, Succ1);
1417 }
1418
1419 // Capture the existing control flow as guard predicates, and redirect
1420 // control flow from every incoming block to the first guard block in
1421 // the hub.
1422 //
1423 // There is one guard predicate for each outgoing block OutBB. The
1424 // predicate is a PHINode with one input for each InBB which
1425 // represents whether the hub should transfer control flow to OutBB if
1426 // it arrived from InBB. These predicates are NOT ORTHOGONAL. The Hub
1427 // evaluates them in the same order as the Outgoing set-vector, and
1428 // control branches to the first outgoing block whose predicate
1429 // evaluates to true.
convertToGuardPredicates(BasicBlock * FirstGuardBlock,BBPredicates & GuardPredicates,SmallVectorImpl<WeakVH> & DeletionCandidates,const BBSetVector & Incoming,const BBSetVector & Outgoing)1430 static void convertToGuardPredicates(
1431 BasicBlock *FirstGuardBlock, BBPredicates &GuardPredicates,
1432 SmallVectorImpl<WeakVH> &DeletionCandidates, const BBSetVector &Incoming,
1433 const BBSetVector &Outgoing) {
1434 auto &Context = Incoming.front()->getContext();
1435 auto BoolTrue = ConstantInt::getTrue(Context);
1436 auto BoolFalse = ConstantInt::getFalse(Context);
1437
1438 // The predicate for the last outgoing is trivially true, and so we
1439 // process only the first N-1 successors.
1440 for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
1441 auto Out = Outgoing[i];
1442 LLVM_DEBUG(dbgs() << "Creating guard for " << Out->getName() << "\n");
1443 auto Phi =
1444 PHINode::Create(Type::getInt1Ty(Context), Incoming.size(),
1445 StringRef("Guard.") + Out->getName(), FirstGuardBlock);
1446 GuardPredicates[Out] = Phi;
1447 }
1448
1449 for (auto In : Incoming) {
1450 Value *Condition;
1451 BasicBlock *Succ0;
1452 BasicBlock *Succ1;
1453 std::tie(Condition, Succ0, Succ1) =
1454 redirectToHub(In, FirstGuardBlock, Outgoing);
1455
1456 // Optimization: Consider an incoming block A with both successors
1457 // Succ0 and Succ1 in the set of outgoing blocks. The predicates
1458 // for Succ0 and Succ1 complement each other. If Succ0 is visited
1459 // first in the loop below, control will branch to Succ0 using the
1460 // corresponding predicate. But if that branch is not taken, then
1461 // control must reach Succ1, which means that the predicate for
1462 // Succ1 is always true.
1463 bool OneSuccessorDone = false;
1464 for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
1465 auto Out = Outgoing[i];
1466 auto Phi = GuardPredicates[Out];
1467 if (Out != Succ0 && Out != Succ1) {
1468 Phi->addIncoming(BoolFalse, In);
1469 continue;
1470 }
1471 // Optimization: When only one successor is an outgoing block,
1472 // the predicate is always true.
1473 if (!Succ0 || !Succ1 || OneSuccessorDone) {
1474 Phi->addIncoming(BoolTrue, In);
1475 continue;
1476 }
1477 assert(Succ0 && Succ1);
1478 OneSuccessorDone = true;
1479 if (Out == Succ0) {
1480 Phi->addIncoming(Condition, In);
1481 continue;
1482 }
1483 auto Inverted = invertCondition(Condition);
1484 DeletionCandidates.push_back(Condition);
1485 Phi->addIncoming(Inverted, In);
1486 }
1487 }
1488 }
1489
1490 // For each outgoing block OutBB, create a guard block in the Hub. The
1491 // first guard block was already created outside, and available as the
1492 // first element in the vector of guard blocks.
1493 //
1494 // Each guard block terminates in a conditional branch that transfers
1495 // control to the corresponding outgoing block or the next guard
1496 // block. The last guard block has two outgoing blocks as successors
1497 // since the condition for the final outgoing block is trivially
1498 // true. So we create one less block (including the first guard block)
1499 // than the number of outgoing blocks.
createGuardBlocks(SmallVectorImpl<BasicBlock * > & GuardBlocks,Function * F,const BBSetVector & Outgoing,BBPredicates & GuardPredicates,StringRef Prefix)1500 static void createGuardBlocks(SmallVectorImpl<BasicBlock *> &GuardBlocks,
1501 Function *F, const BBSetVector &Outgoing,
1502 BBPredicates &GuardPredicates, StringRef Prefix) {
1503 for (int i = 0, e = Outgoing.size() - 2; i != e; ++i) {
1504 GuardBlocks.push_back(
1505 BasicBlock::Create(F->getContext(), Prefix + ".guard", F));
1506 }
1507 assert(GuardBlocks.size() == GuardPredicates.size());
1508
1509 // To help keep the loop simple, temporarily append the last
1510 // outgoing block to the list of guard blocks.
1511 GuardBlocks.push_back(Outgoing.back());
1512
1513 for (int i = 0, e = GuardBlocks.size() - 1; i != e; ++i) {
1514 auto Out = Outgoing[i];
1515 assert(GuardPredicates.count(Out));
1516 BranchInst::Create(Out, GuardBlocks[i + 1], GuardPredicates[Out],
1517 GuardBlocks[i]);
1518 }
1519
1520 // Remove the last block from the guard list.
1521 GuardBlocks.pop_back();
1522 }
1523
CreateControlFlowHub(DomTreeUpdater * DTU,SmallVectorImpl<BasicBlock * > & GuardBlocks,const BBSetVector & Incoming,const BBSetVector & Outgoing,const StringRef Prefix)1524 BasicBlock *llvm::CreateControlFlowHub(
1525 DomTreeUpdater *DTU, SmallVectorImpl<BasicBlock *> &GuardBlocks,
1526 const BBSetVector &Incoming, const BBSetVector &Outgoing,
1527 const StringRef Prefix) {
1528 auto F = Incoming.front()->getParent();
1529 auto FirstGuardBlock =
1530 BasicBlock::Create(F->getContext(), Prefix + ".guard", F);
1531
1532 SmallVector<DominatorTree::UpdateType, 16> Updates;
1533 if (DTU) {
1534 for (auto In : Incoming) {
1535 Updates.push_back({DominatorTree::Insert, In, FirstGuardBlock});
1536 for (auto Succ : successors(In)) {
1537 if (Outgoing.count(Succ))
1538 Updates.push_back({DominatorTree::Delete, In, Succ});
1539 }
1540 }
1541 }
1542
1543 BBPredicates GuardPredicates;
1544 SmallVector<WeakVH, 8> DeletionCandidates;
1545 convertToGuardPredicates(FirstGuardBlock, GuardPredicates, DeletionCandidates,
1546 Incoming, Outgoing);
1547
1548 GuardBlocks.push_back(FirstGuardBlock);
1549 createGuardBlocks(GuardBlocks, F, Outgoing, GuardPredicates, Prefix);
1550
1551 // Update the PHINodes in each outgoing block to match the new control flow.
1552 for (int i = 0, e = GuardBlocks.size(); i != e; ++i) {
1553 reconnectPhis(Outgoing[i], GuardBlocks[i], Incoming, FirstGuardBlock);
1554 }
1555 reconnectPhis(Outgoing.back(), GuardBlocks.back(), Incoming, FirstGuardBlock);
1556
1557 if (DTU) {
1558 int NumGuards = GuardBlocks.size();
1559 assert((int)Outgoing.size() == NumGuards + 1);
1560 for (int i = 0; i != NumGuards - 1; ++i) {
1561 Updates.push_back({DominatorTree::Insert, GuardBlocks[i], Outgoing[i]});
1562 Updates.push_back(
1563 {DominatorTree::Insert, GuardBlocks[i], GuardBlocks[i + 1]});
1564 }
1565 Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
1566 Outgoing[NumGuards - 1]});
1567 Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
1568 Outgoing[NumGuards]});
1569 DTU->applyUpdates(Updates);
1570 }
1571
1572 for (auto I : DeletionCandidates) {
1573 if (I->use_empty())
1574 if (auto Inst = dyn_cast_or_null<Instruction>(I))
1575 Inst->eraseFromParent();
1576 }
1577
1578 return FirstGuardBlock;
1579 }
1580