1 //===- BasicBlockUtils.cpp - BasicBlock Utilities --------------------------==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This family of functions perform manipulations on basic blocks, and
10 // instructions contained within basic blocks.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/SmallPtrSet.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/Twine.h"
19 #include "llvm/Analysis/CFG.h"
20 #include "llvm/Analysis/DomTreeUpdater.h"
21 #include "llvm/Analysis/LoopInfo.h"
22 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
23 #include "llvm/Analysis/MemorySSAUpdater.h"
24 #include "llvm/IR/BasicBlock.h"
25 #include "llvm/IR/CFG.h"
26 #include "llvm/IR/Constants.h"
27 #include "llvm/IR/DebugInfo.h"
28 #include "llvm/IR/DebugInfoMetadata.h"
29 #include "llvm/IR/Dominators.h"
30 #include "llvm/IR/Function.h"
31 #include "llvm/IR/InstrTypes.h"
32 #include "llvm/IR/Instruction.h"
33 #include "llvm/IR/Instructions.h"
34 #include "llvm/IR/IntrinsicInst.h"
35 #include "llvm/IR/IRBuilder.h"
36 #include "llvm/IR/LLVMContext.h"
37 #include "llvm/IR/Type.h"
38 #include "llvm/IR/User.h"
39 #include "llvm/IR/Value.h"
40 #include "llvm/IR/ValueHandle.h"
41 #include "llvm/Support/Casting.h"
42 #include "llvm/Support/CommandLine.h"
43 #include "llvm/Support/Debug.h"
44 #include "llvm/Support/raw_ostream.h"
45 #include "llvm/Transforms/Utils/Local.h"
46 #include <cassert>
47 #include <cstdint>
48 #include <string>
49 #include <utility>
50 #include <vector>
51 
52 using namespace llvm;
53 
54 #define DEBUG_TYPE "basicblock-utils"
55 
56 static cl::opt<unsigned> MaxDeoptOrUnreachableSuccessorCheckDepth(
57     "max-deopt-or-unreachable-succ-check-depth", cl::init(8), cl::Hidden,
58     cl::desc("Set the maximum path length when checking whether a basic block "
59              "is followed by a block that either has a terminating "
60              "deoptimizing call or is terminated with an unreachable"));
61 
62 void llvm::detachDeadBlocks(
63     ArrayRef<BasicBlock *> BBs,
64     SmallVectorImpl<DominatorTree::UpdateType> *Updates,
65     bool KeepOneInputPHIs) {
66   for (auto *BB : BBs) {
67     // Loop through all of our successors and make sure they know that one
68     // of their predecessors is going away.
69     SmallPtrSet<BasicBlock *, 4> UniqueSuccessors;
70     for (BasicBlock *Succ : successors(BB)) {
71       Succ->removePredecessor(BB, KeepOneInputPHIs);
72       if (Updates && UniqueSuccessors.insert(Succ).second)
73         Updates->push_back({DominatorTree::Delete, BB, Succ});
74     }
75 
76     // Zap all the instructions in the block.
77     while (!BB->empty()) {
78       Instruction &I = BB->back();
79       // If this instruction is used, replace uses with an arbitrary value.
80       // Because control flow can't get here, we don't care what we replace the
81       // value with.  Note that since this block is unreachable, and all values
82       // contained within it must dominate their uses, that all uses will
83       // eventually be removed (they are themselves dead).
84       if (!I.use_empty())
85         I.replaceAllUsesWith(PoisonValue::get(I.getType()));
86       BB->back().eraseFromParent();
87     }
88     new UnreachableInst(BB->getContext(), BB);
89     assert(BB->size() == 1 &&
90            isa<UnreachableInst>(BB->getTerminator()) &&
91            "The successor list of BB isn't empty before "
92            "applying corresponding DTU updates.");
93   }
94 }
95 
96 void llvm::DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU,
97                            bool KeepOneInputPHIs) {
98   DeleteDeadBlocks({BB}, DTU, KeepOneInputPHIs);
99 }
100 
101 void llvm::DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs, DomTreeUpdater *DTU,
102                             bool KeepOneInputPHIs) {
103 #ifndef NDEBUG
104   // Make sure that all predecessors of each dead block is also dead.
105   SmallPtrSet<BasicBlock *, 4> Dead(BBs.begin(), BBs.end());
106   assert(Dead.size() == BBs.size() && "Duplicating blocks?");
107   for (auto *BB : Dead)
108     for (BasicBlock *Pred : predecessors(BB))
109       assert(Dead.count(Pred) && "All predecessors must be dead!");
110 #endif
111 
112   SmallVector<DominatorTree::UpdateType, 4> Updates;
113   detachDeadBlocks(BBs, DTU ? &Updates : nullptr, KeepOneInputPHIs);
114 
115   if (DTU)
116     DTU->applyUpdates(Updates);
117 
118   for (BasicBlock *BB : BBs)
119     if (DTU)
120       DTU->deleteBB(BB);
121     else
122       BB->eraseFromParent();
123 }
124 
125 bool llvm::EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
126                                       bool KeepOneInputPHIs) {
127   df_iterator_default_set<BasicBlock*> Reachable;
128 
129   // Mark all reachable blocks.
130   for (BasicBlock *BB : depth_first_ext(&F, Reachable))
131     (void)BB/* Mark all reachable blocks */;
132 
133   // Collect all dead blocks.
134   std::vector<BasicBlock*> DeadBlocks;
135   for (BasicBlock &BB : F)
136     if (!Reachable.count(&BB))
137       DeadBlocks.push_back(&BB);
138 
139   // Delete the dead blocks.
140   DeleteDeadBlocks(DeadBlocks, DTU, KeepOneInputPHIs);
141 
142   return !DeadBlocks.empty();
143 }
144 
145 bool llvm::FoldSingleEntryPHINodes(BasicBlock *BB,
146                                    MemoryDependenceResults *MemDep) {
147   if (!isa<PHINode>(BB->begin()))
148     return false;
149 
150   while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
151     if (PN->getIncomingValue(0) != PN)
152       PN->replaceAllUsesWith(PN->getIncomingValue(0));
153     else
154       PN->replaceAllUsesWith(PoisonValue::get(PN->getType()));
155 
156     if (MemDep)
157       MemDep->removeInstruction(PN);  // Memdep updates AA itself.
158 
159     PN->eraseFromParent();
160   }
161   return true;
162 }
163 
164 bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI,
165                           MemorySSAUpdater *MSSAU) {
166   // Recursively deleting a PHI may cause multiple PHIs to be deleted
167   // or RAUW'd undef, so use an array of WeakTrackingVH for the PHIs to delete.
168   SmallVector<WeakTrackingVH, 8> PHIs;
169   for (PHINode &PN : BB->phis())
170     PHIs.push_back(&PN);
171 
172   bool Changed = false;
173   for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
174     if (PHINode *PN = dyn_cast_or_null<PHINode>(PHIs[i].operator Value*()))
175       Changed |= RecursivelyDeleteDeadPHINode(PN, TLI, MSSAU);
176 
177   return Changed;
178 }
179 
180 bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
181                                      LoopInfo *LI, MemorySSAUpdater *MSSAU,
182                                      MemoryDependenceResults *MemDep,
183                                      bool PredecessorWithTwoSuccessors,
184                                      DominatorTree *DT) {
185   if (BB->hasAddressTaken())
186     return false;
187 
188   // Can't merge if there are multiple predecessors, or no predecessors.
189   BasicBlock *PredBB = BB->getUniquePredecessor();
190   if (!PredBB) return false;
191 
192   // Don't break self-loops.
193   if (PredBB == BB) return false;
194 
195   // Don't break unwinding instructions or terminators with other side-effects.
196   Instruction *PTI = PredBB->getTerminator();
197   if (PTI->isExceptionalTerminator() || PTI->mayHaveSideEffects())
198     return false;
199 
200   // Can't merge if there are multiple distinct successors.
201   if (!PredecessorWithTwoSuccessors && PredBB->getUniqueSuccessor() != BB)
202     return false;
203 
204   // Currently only allow PredBB to have two predecessors, one being BB.
205   // Update BI to branch to BB's only successor instead of BB.
206   BranchInst *PredBB_BI;
207   BasicBlock *NewSucc = nullptr;
208   unsigned FallThruPath;
209   if (PredecessorWithTwoSuccessors) {
210     if (!(PredBB_BI = dyn_cast<BranchInst>(PTI)))
211       return false;
212     BranchInst *BB_JmpI = dyn_cast<BranchInst>(BB->getTerminator());
213     if (!BB_JmpI || !BB_JmpI->isUnconditional())
214       return false;
215     NewSucc = BB_JmpI->getSuccessor(0);
216     FallThruPath = PredBB_BI->getSuccessor(0) == BB ? 0 : 1;
217   }
218 
219   // Can't merge if there is PHI loop.
220   for (PHINode &PN : BB->phis())
221     if (llvm::is_contained(PN.incoming_values(), &PN))
222       return false;
223 
224   LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
225                     << PredBB->getName() << "\n");
226 
227   // Begin by getting rid of unneeded PHIs.
228   SmallVector<AssertingVH<Value>, 4> IncomingValues;
229   if (isa<PHINode>(BB->front())) {
230     for (PHINode &PN : BB->phis())
231       if (!isa<PHINode>(PN.getIncomingValue(0)) ||
232           cast<PHINode>(PN.getIncomingValue(0))->getParent() != BB)
233         IncomingValues.push_back(PN.getIncomingValue(0));
234     FoldSingleEntryPHINodes(BB, MemDep);
235   }
236 
237   if (DT) {
238     assert(!DTU && "cannot use both DT and DTU for updates");
239     DomTreeNode *PredNode = DT->getNode(PredBB);
240     DomTreeNode *BBNode = DT->getNode(BB);
241     if (PredNode) {
242       assert(BBNode && "PredNode unreachable but BBNode reachable?");
243       for (DomTreeNode *C : to_vector(BBNode->children()))
244         C->setIDom(PredNode);
245     }
246   }
247   // DTU update: Collect all the edges that exit BB.
248   // These dominator edges will be redirected from Pred.
249   std::vector<DominatorTree::UpdateType> Updates;
250   if (DTU) {
251     assert(!DT && "cannot use both DT and DTU for updates");
252     // To avoid processing the same predecessor more than once.
253     SmallPtrSet<BasicBlock *, 8> SeenSuccs;
254     SmallPtrSet<BasicBlock *, 2> SuccsOfPredBB(succ_begin(PredBB),
255                                                succ_end(PredBB));
256     Updates.reserve(Updates.size() + 2 * succ_size(BB) + 1);
257     // Add insert edges first. Experimentally, for the particular case of two
258     // blocks that can be merged, with a single successor and single predecessor
259     // respectively, it is beneficial to have all insert updates first. Deleting
260     // edges first may lead to unreachable blocks, followed by inserting edges
261     // making the blocks reachable again. Such DT updates lead to high compile
262     // times. We add inserts before deletes here to reduce compile time.
263     for (BasicBlock *SuccOfBB : successors(BB))
264       // This successor of BB may already be a PredBB's successor.
265       if (!SuccsOfPredBB.contains(SuccOfBB))
266         if (SeenSuccs.insert(SuccOfBB).second)
267           Updates.push_back({DominatorTree::Insert, PredBB, SuccOfBB});
268     SeenSuccs.clear();
269     for (BasicBlock *SuccOfBB : successors(BB))
270       if (SeenSuccs.insert(SuccOfBB).second)
271         Updates.push_back({DominatorTree::Delete, BB, SuccOfBB});
272     Updates.push_back({DominatorTree::Delete, PredBB, BB});
273   }
274 
275   Instruction *STI = BB->getTerminator();
276   Instruction *Start = &*BB->begin();
277   // If there's nothing to move, mark the starting instruction as the last
278   // instruction in the block. Terminator instruction is handled separately.
279   if (Start == STI)
280     Start = PTI;
281 
282   // Move all definitions in the successor to the predecessor...
283   PredBB->splice(PTI->getIterator(), BB, BB->begin(), STI->getIterator());
284 
285   if (MSSAU)
286     MSSAU->moveAllAfterMergeBlocks(BB, PredBB, Start);
287 
288   // Make all PHI nodes that referred to BB now refer to Pred as their
289   // source...
290   BB->replaceAllUsesWith(PredBB);
291 
292   if (PredecessorWithTwoSuccessors) {
293     // Delete the unconditional branch from BB.
294     BB->back().eraseFromParent();
295 
296     // Update branch in the predecessor.
297     PredBB_BI->setSuccessor(FallThruPath, NewSucc);
298   } else {
299     // Delete the unconditional branch from the predecessor.
300     PredBB->back().eraseFromParent();
301 
302     // Move terminator instruction.
303     PredBB->splice(PredBB->end(), BB);
304 
305     // Terminator may be a memory accessing instruction too.
306     if (MSSAU)
307       if (MemoryUseOrDef *MUD = cast_or_null<MemoryUseOrDef>(
308               MSSAU->getMemorySSA()->getMemoryAccess(PredBB->getTerminator())))
309         MSSAU->moveToPlace(MUD, PredBB, MemorySSA::End);
310   }
311   // Add unreachable to now empty BB.
312   new UnreachableInst(BB->getContext(), BB);
313 
314   // Inherit predecessors name if it exists.
315   if (!PredBB->hasName())
316     PredBB->takeName(BB);
317 
318   if (LI)
319     LI->removeBlock(BB);
320 
321   if (MemDep)
322     MemDep->invalidateCachedPredecessors();
323 
324   if (DTU)
325     DTU->applyUpdates(Updates);
326 
327   if (DT) {
328     assert(succ_empty(BB) &&
329            "successors should have been transferred to PredBB");
330     DT->eraseNode(BB);
331   }
332 
333   // Finally, erase the old block and update dominator info.
334   DeleteDeadBlock(BB, DTU);
335 
336   return true;
337 }
338 
339 bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
340     SmallPtrSetImpl<BasicBlock *> &MergeBlocks, Loop *L, DomTreeUpdater *DTU,
341     LoopInfo *LI) {
342   assert(!MergeBlocks.empty() && "MergeBlocks should not be empty");
343 
344   bool BlocksHaveBeenMerged = false;
345   while (!MergeBlocks.empty()) {
346     BasicBlock *BB = *MergeBlocks.begin();
347     BasicBlock *Dest = BB->getSingleSuccessor();
348     if (Dest && (!L || L->contains(Dest))) {
349       BasicBlock *Fold = Dest->getUniquePredecessor();
350       (void)Fold;
351       if (MergeBlockIntoPredecessor(Dest, DTU, LI)) {
352         assert(Fold == BB &&
353                "Expecting BB to be unique predecessor of the Dest block");
354         MergeBlocks.erase(Dest);
355         BlocksHaveBeenMerged = true;
356       } else
357         MergeBlocks.erase(BB);
358     } else
359       MergeBlocks.erase(BB);
360   }
361   return BlocksHaveBeenMerged;
362 }
363 
364 /// Remove redundant instructions within sequences of consecutive dbg.value
365 /// instructions. This is done using a backward scan to keep the last dbg.value
366 /// describing a specific variable/fragment.
367 ///
368 /// BackwardScan strategy:
369 /// ----------------------
370 /// Given a sequence of consecutive DbgValueInst like this
371 ///
372 ///   dbg.value ..., "x", FragmentX1  (*)
373 ///   dbg.value ..., "y", FragmentY1
374 ///   dbg.value ..., "x", FragmentX2
375 ///   dbg.value ..., "x", FragmentX1  (**)
376 ///
377 /// then the instruction marked with (*) can be removed (it is guaranteed to be
378 /// obsoleted by the instruction marked with (**) as the latter instruction is
379 /// describing the same variable using the same fragment info).
380 ///
381 /// Possible improvements:
382 /// - Check fully overlapping fragments and not only identical fragments.
383 /// - Support dbg.declare. dbg.label, and possibly other meta instructions being
384 ///   part of the sequence of consecutive instructions.
385 static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
386   SmallVector<DbgValueInst *, 8> ToBeRemoved;
387   SmallDenseSet<DebugVariable> VariableSet;
388   for (auto &I : reverse(*BB)) {
389     if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
390       DebugVariable Key(DVI->getVariable(),
391                         DVI->getExpression(),
392                         DVI->getDebugLoc()->getInlinedAt());
393       auto R = VariableSet.insert(Key);
394       // If the variable fragment hasn't been seen before then we don't want
395       // to remove this dbg intrinsic.
396       if (R.second)
397         continue;
398 
399       if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI)) {
400         // Don't delete dbg.assign intrinsics that are linked to instructions.
401         if (!at::getAssignmentInsts(DAI).empty())
402           continue;
403         // Unlinked dbg.assign intrinsics can be treated like dbg.values.
404       }
405 
406       // If the same variable fragment is described more than once it is enough
407       // to keep the last one (i.e. the first found since we for reverse
408       // iteration).
409       ToBeRemoved.push_back(DVI);
410       continue;
411     }
412     // Sequence with consecutive dbg.value instrs ended. Clear the map to
413     // restart identifying redundant instructions if case we find another
414     // dbg.value sequence.
415     VariableSet.clear();
416   }
417 
418   for (auto &Instr : ToBeRemoved)
419     Instr->eraseFromParent();
420 
421   return !ToBeRemoved.empty();
422 }
423 
424 /// Remove redundant dbg.value instructions using a forward scan. This can
425 /// remove a dbg.value instruction that is redundant due to indicating that a
426 /// variable has the same value as already being indicated by an earlier
427 /// dbg.value.
428 ///
429 /// ForwardScan strategy:
430 /// ---------------------
431 /// Given two identical dbg.value instructions, separated by a block of
432 /// instructions that isn't describing the same variable, like this
433 ///
434 ///   dbg.value X1, "x", FragmentX1  (**)
435 ///   <block of instructions, none being "dbg.value ..., "x", ...">
436 ///   dbg.value X1, "x", FragmentX1  (*)
437 ///
438 /// then the instruction marked with (*) can be removed. Variable "x" is already
439 /// described as being mapped to the SSA value X1.
440 ///
441 /// Possible improvements:
442 /// - Keep track of non-overlapping fragments.
443 static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
444   SmallVector<DbgValueInst *, 8> ToBeRemoved;
445   DenseMap<DebugVariable, std::pair<SmallVector<Value *, 4>, DIExpression *>>
446       VariableMap;
447   for (auto &I : *BB) {
448     if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
449       DebugVariable Key(DVI->getVariable(), std::nullopt,
450                         DVI->getDebugLoc()->getInlinedAt());
451       auto VMI = VariableMap.find(Key);
452       auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
453       // A dbg.assign with no linked instructions can be treated like a
454       // dbg.value (i.e. can be deleted).
455       bool IsDbgValueKind = (!DAI || at::getAssignmentInsts(DAI).empty());
456 
457       // Update the map if we found a new value/expression describing the
458       // variable, or if the variable wasn't mapped already.
459       SmallVector<Value *, 4> Values(DVI->getValues());
460       if (VMI == VariableMap.end() || VMI->second.first != Values ||
461           VMI->second.second != DVI->getExpression()) {
462         // Use a sentinal value (nullptr) for the DIExpression when we see a
463         // linked dbg.assign so that the next debug intrinsic will never match
464         // it (i.e. always treat linked dbg.assigns as if they're unique).
465         if (IsDbgValueKind)
466           VariableMap[Key] = {Values, DVI->getExpression()};
467         else
468           VariableMap[Key] = {Values, nullptr};
469         continue;
470       }
471 
472       // Don't delete dbg.assign intrinsics that are linked to instructions.
473       if (!IsDbgValueKind)
474         continue;
475       ToBeRemoved.push_back(DVI);
476     }
477   }
478 
479   for (auto &Instr : ToBeRemoved)
480     Instr->eraseFromParent();
481 
482   return !ToBeRemoved.empty();
483 }
484 
485 /// Remove redundant undef dbg.assign intrinsic from an entry block using a
486 /// forward scan.
487 /// Strategy:
488 /// ---------------------
489 /// Scanning forward, delete dbg.assign intrinsics iff they are undef, not
490 /// linked to an intrinsic, and don't share an aggregate variable with a debug
491 /// intrinsic that didn't meet the criteria. In other words, undef dbg.assigns
492 /// that come before non-undef debug intrinsics for the variable are
493 /// deleted. Given:
494 ///
495 ///   dbg.assign undef, "x", FragmentX1 (*)
496 ///   <block of instructions, none being "dbg.value ..., "x", ...">
497 ///   dbg.value %V, "x", FragmentX2
498 ///   <block of instructions, none being "dbg.value ..., "x", ...">
499 ///   dbg.assign undef, "x", FragmentX1
500 ///
501 /// then (only) the instruction marked with (*) can be removed.
502 /// Possible improvements:
503 /// - Keep track of non-overlapping fragments.
504 static bool remomveUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
505   assert(BB->isEntryBlock() && "expected entry block");
506   SmallVector<DbgAssignIntrinsic *, 8> ToBeRemoved;
507   DenseSet<DebugVariable> SeenDefForAggregate;
508   // Returns the DebugVariable for DVI with no fragment info.
509   auto GetAggregateVariable = [](DbgValueInst *DVI) {
510     return DebugVariable(DVI->getVariable(), std::nullopt,
511                          DVI->getDebugLoc()->getInlinedAt());
512   };
513 
514   // Remove undef dbg.assign intrinsics that are encountered before
515   // any non-undef intrinsics from the entry block.
516   for (auto &I : *BB) {
517     DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I);
518     if (!DVI)
519       continue;
520     auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
521     bool IsDbgValueKind = (!DAI || at::getAssignmentInsts(DAI).empty());
522     DebugVariable Aggregate = GetAggregateVariable(DVI);
523     if (!SeenDefForAggregate.contains(Aggregate)) {
524       bool IsKill = DVI->isKillLocation() && IsDbgValueKind;
525       if (!IsKill) {
526         SeenDefForAggregate.insert(Aggregate);
527       } else if (DAI) {
528         ToBeRemoved.push_back(DAI);
529       }
530     }
531   }
532 
533   for (DbgAssignIntrinsic *DAI : ToBeRemoved)
534     DAI->eraseFromParent();
535 
536   return !ToBeRemoved.empty();
537 }
538 
539 bool llvm::RemoveRedundantDbgInstrs(BasicBlock *BB) {
540   bool MadeChanges = false;
541   // By using the "backward scan" strategy before the "forward scan" strategy we
542   // can remove both dbg.value (2) and (3) in a situation like this:
543   //
544   //   (1) dbg.value V1, "x", DIExpression()
545   //       ...
546   //   (2) dbg.value V2, "x", DIExpression()
547   //   (3) dbg.value V1, "x", DIExpression()
548   //
549   // The backward scan will remove (2), it is made obsolete by (3). After
550   // getting (2) out of the way, the foward scan will remove (3) since "x"
551   // already is described as having the value V1 at (1).
552   MadeChanges |= removeRedundantDbgInstrsUsingBackwardScan(BB);
553   if (BB->isEntryBlock() &&
554       isAssignmentTrackingEnabled(*BB->getParent()->getParent()))
555     MadeChanges |= remomveUndefDbgAssignsFromEntryBlock(BB);
556   MadeChanges |= removeRedundantDbgInstrsUsingForwardScan(BB);
557 
558   if (MadeChanges)
559     LLVM_DEBUG(dbgs() << "Removed redundant dbg instrs from: "
560                       << BB->getName() << "\n");
561   return MadeChanges;
562 }
563 
564 void llvm::ReplaceInstWithValue(BasicBlock::iterator &BI, Value *V) {
565   Instruction &I = *BI;
566   // Replaces all of the uses of the instruction with uses of the value
567   I.replaceAllUsesWith(V);
568 
569   // Make sure to propagate a name if there is one already.
570   if (I.hasName() && !V->hasName())
571     V->takeName(&I);
572 
573   // Delete the unnecessary instruction now...
574   BI = BI->eraseFromParent();
575 }
576 
577 void llvm::ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI,
578                                Instruction *I) {
579   assert(I->getParent() == nullptr &&
580          "ReplaceInstWithInst: Instruction already inserted into basic block!");
581 
582   // Copy debug location to newly added instruction, if it wasn't already set
583   // by the caller.
584   if (!I->getDebugLoc())
585     I->setDebugLoc(BI->getDebugLoc());
586 
587   // Insert the new instruction into the basic block...
588   BasicBlock::iterator New = I->insertInto(BB, BI);
589 
590   // Replace all uses of the old instruction, and delete it.
591   ReplaceInstWithValue(BI, I);
592 
593   // Move BI back to point to the newly inserted instruction
594   BI = New;
595 }
596 
597 bool llvm::IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB) {
598   // Remember visited blocks to avoid infinite loop
599   SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
600   unsigned Depth = 0;
601   while (BB && Depth++ < MaxDeoptOrUnreachableSuccessorCheckDepth &&
602          VisitedBlocks.insert(BB).second) {
603     if (isa<UnreachableInst>(BB->getTerminator()) ||
604         BB->getTerminatingDeoptimizeCall())
605       return true;
606     BB = BB->getUniqueSuccessor();
607   }
608   return false;
609 }
610 
611 void llvm::ReplaceInstWithInst(Instruction *From, Instruction *To) {
612   BasicBlock::iterator BI(From);
613   ReplaceInstWithInst(From->getParent(), BI, To);
614 }
615 
616 BasicBlock *llvm::SplitEdge(BasicBlock *BB, BasicBlock *Succ, DominatorTree *DT,
617                             LoopInfo *LI, MemorySSAUpdater *MSSAU,
618                             const Twine &BBName) {
619   unsigned SuccNum = GetSuccessorNumber(BB, Succ);
620 
621   Instruction *LatchTerm = BB->getTerminator();
622 
623   CriticalEdgeSplittingOptions Options =
624       CriticalEdgeSplittingOptions(DT, LI, MSSAU).setPreserveLCSSA();
625 
626   if ((isCriticalEdge(LatchTerm, SuccNum, Options.MergeIdenticalEdges))) {
627     // If it is a critical edge, and the succesor is an exception block, handle
628     // the split edge logic in this specific function
629     if (Succ->isEHPad())
630       return ehAwareSplitEdge(BB, Succ, nullptr, nullptr, Options, BBName);
631 
632     // If this is a critical edge, let SplitKnownCriticalEdge do it.
633     return SplitKnownCriticalEdge(LatchTerm, SuccNum, Options, BBName);
634   }
635 
636   // If the edge isn't critical, then BB has a single successor or Succ has a
637   // single pred.  Split the block.
638   if (BasicBlock *SP = Succ->getSinglePredecessor()) {
639     // If the successor only has a single pred, split the top of the successor
640     // block.
641     assert(SP == BB && "CFG broken");
642     SP = nullptr;
643     return SplitBlock(Succ, &Succ->front(), DT, LI, MSSAU, BBName,
644                       /*Before=*/true);
645   }
646 
647   // Otherwise, if BB has a single successor, split it at the bottom of the
648   // block.
649   assert(BB->getTerminator()->getNumSuccessors() == 1 &&
650          "Should have a single succ!");
651   return SplitBlock(BB, BB->getTerminator(), DT, LI, MSSAU, BBName);
652 }
653 
654 void llvm::setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ) {
655   if (auto *II = dyn_cast<InvokeInst>(TI))
656     II->setUnwindDest(Succ);
657   else if (auto *CS = dyn_cast<CatchSwitchInst>(TI))
658     CS->setUnwindDest(Succ);
659   else if (auto *CR = dyn_cast<CleanupReturnInst>(TI))
660     CR->setUnwindDest(Succ);
661   else
662     llvm_unreachable("unexpected terminator instruction");
663 }
664 
665 void llvm::updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred,
666                           BasicBlock *NewPred, PHINode *Until) {
667   int BBIdx = 0;
668   for (PHINode &PN : DestBB->phis()) {
669     // We manually update the LandingPadReplacement PHINode and it is the last
670     // PHI Node. So, if we find it, we are done.
671     if (Until == &PN)
672       break;
673 
674     // Reuse the previous value of BBIdx if it lines up.  In cases where we
675     // have multiple phi nodes with *lots* of predecessors, this is a speed
676     // win because we don't have to scan the PHI looking for TIBB.  This
677     // happens because the BB list of PHI nodes are usually in the same
678     // order.
679     if (PN.getIncomingBlock(BBIdx) != OldPred)
680       BBIdx = PN.getBasicBlockIndex(OldPred);
681 
682     assert(BBIdx != -1 && "Invalid PHI Index!");
683     PN.setIncomingBlock(BBIdx, NewPred);
684   }
685 }
686 
687 BasicBlock *llvm::ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
688                                    LandingPadInst *OriginalPad,
689                                    PHINode *LandingPadReplacement,
690                                    const CriticalEdgeSplittingOptions &Options,
691                                    const Twine &BBName) {
692 
693   auto *PadInst = Succ->getFirstNonPHI();
694   if (!LandingPadReplacement && !PadInst->isEHPad())
695     return SplitEdge(BB, Succ, Options.DT, Options.LI, Options.MSSAU, BBName);
696 
697   auto *LI = Options.LI;
698   SmallVector<BasicBlock *, 4> LoopPreds;
699   // Check if extra modifications will be required to preserve loop-simplify
700   // form after splitting. If it would require splitting blocks with IndirectBr
701   // terminators, bail out if preserving loop-simplify form is requested.
702   if (Options.PreserveLoopSimplify && LI) {
703     if (Loop *BBLoop = LI->getLoopFor(BB)) {
704 
705       // The only way that we can break LoopSimplify form by splitting a
706       // critical edge is when there exists some edge from BBLoop to Succ *and*
707       // the only edge into Succ from outside of BBLoop is that of NewBB after
708       // the split. If the first isn't true, then LoopSimplify still holds,
709       // NewBB is the new exit block and it has no non-loop predecessors. If the
710       // second isn't true, then Succ was not in LoopSimplify form prior to
711       // the split as it had a non-loop predecessor. In both of these cases,
712       // the predecessor must be directly in BBLoop, not in a subloop, or again
713       // LoopSimplify doesn't hold.
714       for (BasicBlock *P : predecessors(Succ)) {
715         if (P == BB)
716           continue; // The new block is known.
717         if (LI->getLoopFor(P) != BBLoop) {
718           // Loop is not in LoopSimplify form, no need to re simplify after
719           // splitting edge.
720           LoopPreds.clear();
721           break;
722         }
723         LoopPreds.push_back(P);
724       }
725       // Loop-simplify form can be preserved, if we can split all in-loop
726       // predecessors.
727       if (any_of(LoopPreds, [](BasicBlock *Pred) {
728             return isa<IndirectBrInst>(Pred->getTerminator());
729           })) {
730         return nullptr;
731       }
732     }
733   }
734 
735   auto *NewBB =
736       BasicBlock::Create(BB->getContext(), BBName, BB->getParent(), Succ);
737   setUnwindEdgeTo(BB->getTerminator(), NewBB);
738   updatePhiNodes(Succ, BB, NewBB, LandingPadReplacement);
739 
740   if (LandingPadReplacement) {
741     auto *NewLP = OriginalPad->clone();
742     auto *Terminator = BranchInst::Create(Succ, NewBB);
743     NewLP->insertBefore(Terminator);
744     LandingPadReplacement->addIncoming(NewLP, NewBB);
745   } else {
746     Value *ParentPad = nullptr;
747     if (auto *FuncletPad = dyn_cast<FuncletPadInst>(PadInst))
748       ParentPad = FuncletPad->getParentPad();
749     else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(PadInst))
750       ParentPad = CatchSwitch->getParentPad();
751     else if (auto *CleanupPad = dyn_cast<CleanupPadInst>(PadInst))
752       ParentPad = CleanupPad->getParentPad();
753     else if (auto *LandingPad = dyn_cast<LandingPadInst>(PadInst))
754       ParentPad = LandingPad->getParent();
755     else
756       llvm_unreachable("handling for other EHPads not implemented yet");
757 
758     auto *NewCleanupPad = CleanupPadInst::Create(ParentPad, {}, BBName, NewBB);
759     CleanupReturnInst::Create(NewCleanupPad, Succ, NewBB);
760   }
761 
762   auto *DT = Options.DT;
763   auto *MSSAU = Options.MSSAU;
764   if (!DT && !LI)
765     return NewBB;
766 
767   if (DT) {
768     DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
769     SmallVector<DominatorTree::UpdateType, 3> Updates;
770 
771     Updates.push_back({DominatorTree::Insert, BB, NewBB});
772     Updates.push_back({DominatorTree::Insert, NewBB, Succ});
773     Updates.push_back({DominatorTree::Delete, BB, Succ});
774 
775     DTU.applyUpdates(Updates);
776     DTU.flush();
777 
778     if (MSSAU) {
779       MSSAU->applyUpdates(Updates, *DT);
780       if (VerifyMemorySSA)
781         MSSAU->getMemorySSA()->verifyMemorySSA();
782     }
783   }
784 
785   if (LI) {
786     if (Loop *BBLoop = LI->getLoopFor(BB)) {
787       // If one or the other blocks were not in a loop, the new block is not
788       // either, and thus LI doesn't need to be updated.
789       if (Loop *SuccLoop = LI->getLoopFor(Succ)) {
790         if (BBLoop == SuccLoop) {
791           // Both in the same loop, the NewBB joins loop.
792           SuccLoop->addBasicBlockToLoop(NewBB, *LI);
793         } else if (BBLoop->contains(SuccLoop)) {
794           // Edge from an outer loop to an inner loop.  Add to the outer loop.
795           BBLoop->addBasicBlockToLoop(NewBB, *LI);
796         } else if (SuccLoop->contains(BBLoop)) {
797           // Edge from an inner loop to an outer loop.  Add to the outer loop.
798           SuccLoop->addBasicBlockToLoop(NewBB, *LI);
799         } else {
800           // Edge from two loops with no containment relation.  Because these
801           // are natural loops, we know that the destination block must be the
802           // header of its loop (adding a branch into a loop elsewhere would
803           // create an irreducible loop).
804           assert(SuccLoop->getHeader() == Succ &&
805                  "Should not create irreducible loops!");
806           if (Loop *P = SuccLoop->getParentLoop())
807             P->addBasicBlockToLoop(NewBB, *LI);
808         }
809       }
810 
811       // If BB is in a loop and Succ is outside of that loop, we may need to
812       // update LoopSimplify form and LCSSA form.
813       if (!BBLoop->contains(Succ)) {
814         assert(!BBLoop->contains(NewBB) &&
815                "Split point for loop exit is contained in loop!");
816 
817         // Update LCSSA form in the newly created exit block.
818         if (Options.PreserveLCSSA) {
819           createPHIsForSplitLoopExit(BB, NewBB, Succ);
820         }
821 
822         if (!LoopPreds.empty()) {
823           BasicBlock *NewExitBB = SplitBlockPredecessors(
824               Succ, LoopPreds, "split", DT, LI, MSSAU, Options.PreserveLCSSA);
825           if (Options.PreserveLCSSA)
826             createPHIsForSplitLoopExit(LoopPreds, NewExitBB, Succ);
827         }
828       }
829     }
830   }
831 
832   return NewBB;
833 }
834 
835 void llvm::createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
836                                       BasicBlock *SplitBB, BasicBlock *DestBB) {
837   // SplitBB shouldn't have anything non-trivial in it yet.
838   assert((SplitBB->getFirstNonPHI() == SplitBB->getTerminator() ||
839           SplitBB->isLandingPad()) &&
840          "SplitBB has non-PHI nodes!");
841 
842   // For each PHI in the destination block.
843   for (PHINode &PN : DestBB->phis()) {
844     int Idx = PN.getBasicBlockIndex(SplitBB);
845     assert(Idx >= 0 && "Invalid Block Index");
846     Value *V = PN.getIncomingValue(Idx);
847 
848     // If the input is a PHI which already satisfies LCSSA, don't create
849     // a new one.
850     if (const PHINode *VP = dyn_cast<PHINode>(V))
851       if (VP->getParent() == SplitBB)
852         continue;
853 
854     // Otherwise a new PHI is needed. Create one and populate it.
855     PHINode *NewPN = PHINode::Create(
856         PN.getType(), Preds.size(), "split",
857         SplitBB->isLandingPad() ? &SplitBB->front() : SplitBB->getTerminator());
858     for (BasicBlock *BB : Preds)
859       NewPN->addIncoming(V, BB);
860 
861     // Update the original PHI.
862     PN.setIncomingValue(Idx, NewPN);
863   }
864 }
865 
866 unsigned
867 llvm::SplitAllCriticalEdges(Function &F,
868                             const CriticalEdgeSplittingOptions &Options) {
869   unsigned NumBroken = 0;
870   for (BasicBlock &BB : F) {
871     Instruction *TI = BB.getTerminator();
872     if (TI->getNumSuccessors() > 1 && !isa<IndirectBrInst>(TI))
873       for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
874         if (SplitCriticalEdge(TI, i, Options))
875           ++NumBroken;
876   }
877   return NumBroken;
878 }
879 
880 static BasicBlock *SplitBlockImpl(BasicBlock *Old, Instruction *SplitPt,
881                                   DomTreeUpdater *DTU, DominatorTree *DT,
882                                   LoopInfo *LI, MemorySSAUpdater *MSSAU,
883                                   const Twine &BBName, bool Before) {
884   if (Before) {
885     DomTreeUpdater LocalDTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
886     return splitBlockBefore(Old, SplitPt,
887                             DTU ? DTU : (DT ? &LocalDTU : nullptr), LI, MSSAU,
888                             BBName);
889   }
890   BasicBlock::iterator SplitIt = SplitPt->getIterator();
891   while (isa<PHINode>(SplitIt) || SplitIt->isEHPad()) {
892     ++SplitIt;
893     assert(SplitIt != SplitPt->getParent()->end());
894   }
895   std::string Name = BBName.str();
896   BasicBlock *New = Old->splitBasicBlock(
897       SplitIt, Name.empty() ? Old->getName() + ".split" : Name);
898 
899   // The new block lives in whichever loop the old one did. This preserves
900   // LCSSA as well, because we force the split point to be after any PHI nodes.
901   if (LI)
902     if (Loop *L = LI->getLoopFor(Old))
903       L->addBasicBlockToLoop(New, *LI);
904 
905   if (DTU) {
906     SmallVector<DominatorTree::UpdateType, 8> Updates;
907     // Old dominates New. New node dominates all other nodes dominated by Old.
908     SmallPtrSet<BasicBlock *, 8> UniqueSuccessorsOfOld;
909     Updates.push_back({DominatorTree::Insert, Old, New});
910     Updates.reserve(Updates.size() + 2 * succ_size(New));
911     for (BasicBlock *SuccessorOfOld : successors(New))
912       if (UniqueSuccessorsOfOld.insert(SuccessorOfOld).second) {
913         Updates.push_back({DominatorTree::Insert, New, SuccessorOfOld});
914         Updates.push_back({DominatorTree::Delete, Old, SuccessorOfOld});
915       }
916 
917     DTU->applyUpdates(Updates);
918   } else if (DT)
919     // Old dominates New. New node dominates all other nodes dominated by Old.
920     if (DomTreeNode *OldNode = DT->getNode(Old)) {
921       std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
922 
923       DomTreeNode *NewNode = DT->addNewBlock(New, Old);
924       for (DomTreeNode *I : Children)
925         DT->changeImmediateDominator(I, NewNode);
926     }
927 
928   // Move MemoryAccesses still tracked in Old, but part of New now.
929   // Update accesses in successor blocks accordingly.
930   if (MSSAU)
931     MSSAU->moveAllAfterSpliceBlocks(Old, New, &*(New->begin()));
932 
933   return New;
934 }
935 
936 BasicBlock *llvm::SplitBlock(BasicBlock *Old, Instruction *SplitPt,
937                              DominatorTree *DT, LoopInfo *LI,
938                              MemorySSAUpdater *MSSAU, const Twine &BBName,
939                              bool Before) {
940   return SplitBlockImpl(Old, SplitPt, /*DTU=*/nullptr, DT, LI, MSSAU, BBName,
941                         Before);
942 }
943 BasicBlock *llvm::SplitBlock(BasicBlock *Old, Instruction *SplitPt,
944                              DomTreeUpdater *DTU, LoopInfo *LI,
945                              MemorySSAUpdater *MSSAU, const Twine &BBName,
946                              bool Before) {
947   return SplitBlockImpl(Old, SplitPt, DTU, /*DT=*/nullptr, LI, MSSAU, BBName,
948                         Before);
949 }
950 
951 BasicBlock *llvm::splitBlockBefore(BasicBlock *Old, Instruction *SplitPt,
952                                    DomTreeUpdater *DTU, LoopInfo *LI,
953                                    MemorySSAUpdater *MSSAU,
954                                    const Twine &BBName) {
955 
956   BasicBlock::iterator SplitIt = SplitPt->getIterator();
957   while (isa<PHINode>(SplitIt) || SplitIt->isEHPad())
958     ++SplitIt;
959   std::string Name = BBName.str();
960   BasicBlock *New = Old->splitBasicBlock(
961       SplitIt, Name.empty() ? Old->getName() + ".split" : Name,
962       /* Before=*/true);
963 
964   // The new block lives in whichever loop the old one did. This preserves
965   // LCSSA as well, because we force the split point to be after any PHI nodes.
966   if (LI)
967     if (Loop *L = LI->getLoopFor(Old))
968       L->addBasicBlockToLoop(New, *LI);
969 
970   if (DTU) {
971     SmallVector<DominatorTree::UpdateType, 8> DTUpdates;
972     // New dominates Old. The predecessor nodes of the Old node dominate
973     // New node.
974     SmallPtrSet<BasicBlock *, 8> UniquePredecessorsOfOld;
975     DTUpdates.push_back({DominatorTree::Insert, New, Old});
976     DTUpdates.reserve(DTUpdates.size() + 2 * pred_size(New));
977     for (BasicBlock *PredecessorOfOld : predecessors(New))
978       if (UniquePredecessorsOfOld.insert(PredecessorOfOld).second) {
979         DTUpdates.push_back({DominatorTree::Insert, PredecessorOfOld, New});
980         DTUpdates.push_back({DominatorTree::Delete, PredecessorOfOld, Old});
981       }
982 
983     DTU->applyUpdates(DTUpdates);
984 
985     // Move MemoryAccesses still tracked in Old, but part of New now.
986     // Update accesses in successor blocks accordingly.
987     if (MSSAU) {
988       MSSAU->applyUpdates(DTUpdates, DTU->getDomTree());
989       if (VerifyMemorySSA)
990         MSSAU->getMemorySSA()->verifyMemorySSA();
991     }
992   }
993   return New;
994 }
995 
996 /// Update DominatorTree, LoopInfo, and LCCSA analysis information.
997 static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
998                                       ArrayRef<BasicBlock *> Preds,
999                                       DomTreeUpdater *DTU, DominatorTree *DT,
1000                                       LoopInfo *LI, MemorySSAUpdater *MSSAU,
1001                                       bool PreserveLCSSA, bool &HasLoopExit) {
1002   // Update dominator tree if available.
1003   if (DTU) {
1004     // Recalculation of DomTree is needed when updating a forward DomTree and
1005     // the Entry BB is replaced.
1006     if (NewBB->isEntryBlock() && DTU->hasDomTree()) {
1007       // The entry block was removed and there is no external interface for
1008       // the dominator tree to be notified of this change. In this corner-case
1009       // we recalculate the entire tree.
1010       DTU->recalculate(*NewBB->getParent());
1011     } else {
1012       // Split block expects NewBB to have a non-empty set of predecessors.
1013       SmallVector<DominatorTree::UpdateType, 8> Updates;
1014       SmallPtrSet<BasicBlock *, 8> UniquePreds;
1015       Updates.push_back({DominatorTree::Insert, NewBB, OldBB});
1016       Updates.reserve(Updates.size() + 2 * Preds.size());
1017       for (auto *Pred : Preds)
1018         if (UniquePreds.insert(Pred).second) {
1019           Updates.push_back({DominatorTree::Insert, Pred, NewBB});
1020           Updates.push_back({DominatorTree::Delete, Pred, OldBB});
1021         }
1022       DTU->applyUpdates(Updates);
1023     }
1024   } else if (DT) {
1025     if (OldBB == DT->getRootNode()->getBlock()) {
1026       assert(NewBB->isEntryBlock());
1027       DT->setNewRoot(NewBB);
1028     } else {
1029       // Split block expects NewBB to have a non-empty set of predecessors.
1030       DT->splitBlock(NewBB);
1031     }
1032   }
1033 
1034   // Update MemoryPhis after split if MemorySSA is available
1035   if (MSSAU)
1036     MSSAU->wireOldPredecessorsToNewImmediatePredecessor(OldBB, NewBB, Preds);
1037 
1038   // The rest of the logic is only relevant for updating the loop structures.
1039   if (!LI)
1040     return;
1041 
1042   if (DTU && DTU->hasDomTree())
1043     DT = &DTU->getDomTree();
1044   assert(DT && "DT should be available to update LoopInfo!");
1045   Loop *L = LI->getLoopFor(OldBB);
1046 
1047   // If we need to preserve loop analyses, collect some information about how
1048   // this split will affect loops.
1049   bool IsLoopEntry = !!L;
1050   bool SplitMakesNewLoopHeader = false;
1051   for (BasicBlock *Pred : Preds) {
1052     // Preds that are not reachable from entry should not be used to identify if
1053     // OldBB is a loop entry or if SplitMakesNewLoopHeader. Unreachable blocks
1054     // are not within any loops, so we incorrectly mark SplitMakesNewLoopHeader
1055     // as true and make the NewBB the header of some loop. This breaks LI.
1056     if (!DT->isReachableFromEntry(Pred))
1057       continue;
1058     // If we need to preserve LCSSA, determine if any of the preds is a loop
1059     // exit.
1060     if (PreserveLCSSA)
1061       if (Loop *PL = LI->getLoopFor(Pred))
1062         if (!PL->contains(OldBB))
1063           HasLoopExit = true;
1064 
1065     // If we need to preserve LoopInfo, note whether any of the preds crosses
1066     // an interesting loop boundary.
1067     if (!L)
1068       continue;
1069     if (L->contains(Pred))
1070       IsLoopEntry = false;
1071     else
1072       SplitMakesNewLoopHeader = true;
1073   }
1074 
1075   // Unless we have a loop for OldBB, nothing else to do here.
1076   if (!L)
1077     return;
1078 
1079   if (IsLoopEntry) {
1080     // Add the new block to the nearest enclosing loop (and not an adjacent
1081     // loop). To find this, examine each of the predecessors and determine which
1082     // loops enclose them, and select the most-nested loop which contains the
1083     // loop containing the block being split.
1084     Loop *InnermostPredLoop = nullptr;
1085     for (BasicBlock *Pred : Preds) {
1086       if (Loop *PredLoop = LI->getLoopFor(Pred)) {
1087         // Seek a loop which actually contains the block being split (to avoid
1088         // adjacent loops).
1089         while (PredLoop && !PredLoop->contains(OldBB))
1090           PredLoop = PredLoop->getParentLoop();
1091 
1092         // Select the most-nested of these loops which contains the block.
1093         if (PredLoop && PredLoop->contains(OldBB) &&
1094             (!InnermostPredLoop ||
1095              InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
1096           InnermostPredLoop = PredLoop;
1097       }
1098     }
1099 
1100     if (InnermostPredLoop)
1101       InnermostPredLoop->addBasicBlockToLoop(NewBB, *LI);
1102   } else {
1103     L->addBasicBlockToLoop(NewBB, *LI);
1104     if (SplitMakesNewLoopHeader)
1105       L->moveToHeader(NewBB);
1106   }
1107 }
1108 
1109 /// Update the PHI nodes in OrigBB to include the values coming from NewBB.
1110 /// This also updates AliasAnalysis, if available.
1111 static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
1112                            ArrayRef<BasicBlock *> Preds, BranchInst *BI,
1113                            bool HasLoopExit) {
1114   // Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
1115   SmallPtrSet<BasicBlock *, 16> PredSet(Preds.begin(), Preds.end());
1116   for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(I); ) {
1117     PHINode *PN = cast<PHINode>(I++);
1118 
1119     // Check to see if all of the values coming in are the same.  If so, we
1120     // don't need to create a new PHI node, unless it's needed for LCSSA.
1121     Value *InVal = nullptr;
1122     if (!HasLoopExit) {
1123       InVal = PN->getIncomingValueForBlock(Preds[0]);
1124       for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
1125         if (!PredSet.count(PN->getIncomingBlock(i)))
1126           continue;
1127         if (!InVal)
1128           InVal = PN->getIncomingValue(i);
1129         else if (InVal != PN->getIncomingValue(i)) {
1130           InVal = nullptr;
1131           break;
1132         }
1133       }
1134     }
1135 
1136     if (InVal) {
1137       // If all incoming values for the new PHI would be the same, just don't
1138       // make a new PHI.  Instead, just remove the incoming values from the old
1139       // PHI.
1140 
1141       // NOTE! This loop walks backwards for a reason! First off, this minimizes
1142       // the cost of removal if we end up removing a large number of values, and
1143       // second off, this ensures that the indices for the incoming values
1144       // aren't invalidated when we remove one.
1145       for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i)
1146         if (PredSet.count(PN->getIncomingBlock(i)))
1147           PN->removeIncomingValue(i, false);
1148 
1149       // Add an incoming value to the PHI node in the loop for the preheader
1150       // edge.
1151       PN->addIncoming(InVal, NewBB);
1152       continue;
1153     }
1154 
1155     // If the values coming into the block are not the same, we need a new
1156     // PHI.
1157     // Create the new PHI node, insert it into NewBB at the end of the block
1158     PHINode *NewPHI =
1159         PHINode::Create(PN->getType(), Preds.size(), PN->getName() + ".ph", BI);
1160 
1161     // NOTE! This loop walks backwards for a reason! First off, this minimizes
1162     // the cost of removal if we end up removing a large number of values, and
1163     // second off, this ensures that the indices for the incoming values aren't
1164     // invalidated when we remove one.
1165     for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i) {
1166       BasicBlock *IncomingBB = PN->getIncomingBlock(i);
1167       if (PredSet.count(IncomingBB)) {
1168         Value *V = PN->removeIncomingValue(i, false);
1169         NewPHI->addIncoming(V, IncomingBB);
1170       }
1171     }
1172 
1173     PN->addIncoming(NewPHI, NewBB);
1174   }
1175 }
1176 
1177 static void SplitLandingPadPredecessorsImpl(
1178     BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1179     const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1180     DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1181     MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
1182 
1183 static BasicBlock *
1184 SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
1185                            const char *Suffix, DomTreeUpdater *DTU,
1186                            DominatorTree *DT, LoopInfo *LI,
1187                            MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1188   // Do not attempt to split that which cannot be split.
1189   if (!BB->canSplitPredecessors())
1190     return nullptr;
1191 
1192   // For the landingpads we need to act a bit differently.
1193   // Delegate this work to the SplitLandingPadPredecessors.
1194   if (BB->isLandingPad()) {
1195     SmallVector<BasicBlock*, 2> NewBBs;
1196     std::string NewName = std::string(Suffix) + ".split-lp";
1197 
1198     SplitLandingPadPredecessorsImpl(BB, Preds, Suffix, NewName.c_str(), NewBBs,
1199                                     DTU, DT, LI, MSSAU, PreserveLCSSA);
1200     return NewBBs[0];
1201   }
1202 
1203   // Create new basic block, insert right before the original block.
1204   BasicBlock *NewBB = BasicBlock::Create(
1205       BB->getContext(), BB->getName() + Suffix, BB->getParent(), BB);
1206 
1207   // The new block unconditionally branches to the old block.
1208   BranchInst *BI = BranchInst::Create(BB, NewBB);
1209 
1210   Loop *L = nullptr;
1211   BasicBlock *OldLatch = nullptr;
1212   // Splitting the predecessors of a loop header creates a preheader block.
1213   if (LI && LI->isLoopHeader(BB)) {
1214     L = LI->getLoopFor(BB);
1215     // Using the loop start line number prevents debuggers stepping into the
1216     // loop body for this instruction.
1217     BI->setDebugLoc(L->getStartLoc());
1218 
1219     // If BB is the header of the Loop, it is possible that the loop is
1220     // modified, such that the current latch does not remain the latch of the
1221     // loop. If that is the case, the loop metadata from the current latch needs
1222     // to be applied to the new latch.
1223     OldLatch = L->getLoopLatch();
1224   } else
1225     BI->setDebugLoc(BB->getFirstNonPHIOrDbg()->getDebugLoc());
1226 
1227   // Move the edges from Preds to point to NewBB instead of BB.
1228   for (BasicBlock *Pred : Preds) {
1229     // This is slightly more strict than necessary; the minimum requirement
1230     // is that there be no more than one indirectbr branching to BB. And
1231     // all BlockAddress uses would need to be updated.
1232     assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1233            "Cannot split an edge from an IndirectBrInst");
1234     Pred->getTerminator()->replaceSuccessorWith(BB, NewBB);
1235   }
1236 
1237   // Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
1238   // node becomes an incoming value for BB's phi node.  However, if the Preds
1239   // list is empty, we need to insert dummy entries into the PHI nodes in BB to
1240   // account for the newly created predecessor.
1241   if (Preds.empty()) {
1242     // Insert dummy values as the incoming value.
1243     for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++I)
1244       cast<PHINode>(I)->addIncoming(PoisonValue::get(I->getType()), NewBB);
1245   }
1246 
1247   // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1248   bool HasLoopExit = false;
1249   UpdateAnalysisInformation(BB, NewBB, Preds, DTU, DT, LI, MSSAU, PreserveLCSSA,
1250                             HasLoopExit);
1251 
1252   if (!Preds.empty()) {
1253     // Update the PHI nodes in BB with the values coming from NewBB.
1254     UpdatePHINodes(BB, NewBB, Preds, BI, HasLoopExit);
1255   }
1256 
1257   if (OldLatch) {
1258     BasicBlock *NewLatch = L->getLoopLatch();
1259     if (NewLatch != OldLatch) {
1260       MDNode *MD = OldLatch->getTerminator()->getMetadata("llvm.loop");
1261       NewLatch->getTerminator()->setMetadata("llvm.loop", MD);
1262       // It's still possible that OldLatch is the latch of another inner loop,
1263       // in which case we do not remove the metadata.
1264       Loop *IL = LI->getLoopFor(OldLatch);
1265       if (IL && IL->getLoopLatch() != OldLatch)
1266         OldLatch->getTerminator()->setMetadata("llvm.loop", nullptr);
1267     }
1268   }
1269 
1270   return NewBB;
1271 }
1272 
1273 BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1274                                          ArrayRef<BasicBlock *> Preds,
1275                                          const char *Suffix, DominatorTree *DT,
1276                                          LoopInfo *LI, MemorySSAUpdater *MSSAU,
1277                                          bool PreserveLCSSA) {
1278   return SplitBlockPredecessorsImpl(BB, Preds, Suffix, /*DTU=*/nullptr, DT, LI,
1279                                     MSSAU, PreserveLCSSA);
1280 }
1281 BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1282                                          ArrayRef<BasicBlock *> Preds,
1283                                          const char *Suffix,
1284                                          DomTreeUpdater *DTU, LoopInfo *LI,
1285                                          MemorySSAUpdater *MSSAU,
1286                                          bool PreserveLCSSA) {
1287   return SplitBlockPredecessorsImpl(BB, Preds, Suffix, DTU,
1288                                     /*DT=*/nullptr, LI, MSSAU, PreserveLCSSA);
1289 }
1290 
1291 static void SplitLandingPadPredecessorsImpl(
1292     BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1293     const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1294     DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1295     MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1296   assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
1297 
1298   // Create a new basic block for OrigBB's predecessors listed in Preds. Insert
1299   // it right before the original block.
1300   BasicBlock *NewBB1 = BasicBlock::Create(OrigBB->getContext(),
1301                                           OrigBB->getName() + Suffix1,
1302                                           OrigBB->getParent(), OrigBB);
1303   NewBBs.push_back(NewBB1);
1304 
1305   // The new block unconditionally branches to the old block.
1306   BranchInst *BI1 = BranchInst::Create(OrigBB, NewBB1);
1307   BI1->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
1308 
1309   // Move the edges from Preds to point to NewBB1 instead of OrigBB.
1310   for (BasicBlock *Pred : Preds) {
1311     // This is slightly more strict than necessary; the minimum requirement
1312     // is that there be no more than one indirectbr branching to BB. And
1313     // all BlockAddress uses would need to be updated.
1314     assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1315            "Cannot split an edge from an IndirectBrInst");
1316     Pred->getTerminator()->replaceUsesOfWith(OrigBB, NewBB1);
1317   }
1318 
1319   bool HasLoopExit = false;
1320   UpdateAnalysisInformation(OrigBB, NewBB1, Preds, DTU, DT, LI, MSSAU,
1321                             PreserveLCSSA, HasLoopExit);
1322 
1323   // Update the PHI nodes in OrigBB with the values coming from NewBB1.
1324   UpdatePHINodes(OrigBB, NewBB1, Preds, BI1, HasLoopExit);
1325 
1326   // Move the remaining edges from OrigBB to point to NewBB2.
1327   SmallVector<BasicBlock*, 8> NewBB2Preds;
1328   for (pred_iterator i = pred_begin(OrigBB), e = pred_end(OrigBB);
1329        i != e; ) {
1330     BasicBlock *Pred = *i++;
1331     if (Pred == NewBB1) continue;
1332     assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1333            "Cannot split an edge from an IndirectBrInst");
1334     NewBB2Preds.push_back(Pred);
1335     e = pred_end(OrigBB);
1336   }
1337 
1338   BasicBlock *NewBB2 = nullptr;
1339   if (!NewBB2Preds.empty()) {
1340     // Create another basic block for the rest of OrigBB's predecessors.
1341     NewBB2 = BasicBlock::Create(OrigBB->getContext(),
1342                                 OrigBB->getName() + Suffix2,
1343                                 OrigBB->getParent(), OrigBB);
1344     NewBBs.push_back(NewBB2);
1345 
1346     // The new block unconditionally branches to the old block.
1347     BranchInst *BI2 = BranchInst::Create(OrigBB, NewBB2);
1348     BI2->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
1349 
1350     // Move the remaining edges from OrigBB to point to NewBB2.
1351     for (BasicBlock *NewBB2Pred : NewBB2Preds)
1352       NewBB2Pred->getTerminator()->replaceUsesOfWith(OrigBB, NewBB2);
1353 
1354     // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1355     HasLoopExit = false;
1356     UpdateAnalysisInformation(OrigBB, NewBB2, NewBB2Preds, DTU, DT, LI, MSSAU,
1357                               PreserveLCSSA, HasLoopExit);
1358 
1359     // Update the PHI nodes in OrigBB with the values coming from NewBB2.
1360     UpdatePHINodes(OrigBB, NewBB2, NewBB2Preds, BI2, HasLoopExit);
1361   }
1362 
1363   LandingPadInst *LPad = OrigBB->getLandingPadInst();
1364   Instruction *Clone1 = LPad->clone();
1365   Clone1->setName(Twine("lpad") + Suffix1);
1366   Clone1->insertInto(NewBB1, NewBB1->getFirstInsertionPt());
1367 
1368   if (NewBB2) {
1369     Instruction *Clone2 = LPad->clone();
1370     Clone2->setName(Twine("lpad") + Suffix2);
1371     Clone2->insertInto(NewBB2, NewBB2->getFirstInsertionPt());
1372 
1373     // Create a PHI node for the two cloned landingpad instructions only
1374     // if the original landingpad instruction has some uses.
1375     if (!LPad->use_empty()) {
1376       assert(!LPad->getType()->isTokenTy() &&
1377              "Split cannot be applied if LPad is token type. Otherwise an "
1378              "invalid PHINode of token type would be created.");
1379       PHINode *PN = PHINode::Create(LPad->getType(), 2, "lpad.phi", LPad);
1380       PN->addIncoming(Clone1, NewBB1);
1381       PN->addIncoming(Clone2, NewBB2);
1382       LPad->replaceAllUsesWith(PN);
1383     }
1384     LPad->eraseFromParent();
1385   } else {
1386     // There is no second clone. Just replace the landing pad with the first
1387     // clone.
1388     LPad->replaceAllUsesWith(Clone1);
1389     LPad->eraseFromParent();
1390   }
1391 }
1392 
1393 void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
1394                                        ArrayRef<BasicBlock *> Preds,
1395                                        const char *Suffix1, const char *Suffix2,
1396                                        SmallVectorImpl<BasicBlock *> &NewBBs,
1397                                        DominatorTree *DT, LoopInfo *LI,
1398                                        MemorySSAUpdater *MSSAU,
1399                                        bool PreserveLCSSA) {
1400   return SplitLandingPadPredecessorsImpl(
1401       OrigBB, Preds, Suffix1, Suffix2, NewBBs,
1402       /*DTU=*/nullptr, DT, LI, MSSAU, PreserveLCSSA);
1403 }
1404 void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
1405                                        ArrayRef<BasicBlock *> Preds,
1406                                        const char *Suffix1, const char *Suffix2,
1407                                        SmallVectorImpl<BasicBlock *> &NewBBs,
1408                                        DomTreeUpdater *DTU, LoopInfo *LI,
1409                                        MemorySSAUpdater *MSSAU,
1410                                        bool PreserveLCSSA) {
1411   return SplitLandingPadPredecessorsImpl(OrigBB, Preds, Suffix1, Suffix2,
1412                                          NewBBs, DTU, /*DT=*/nullptr, LI, MSSAU,
1413                                          PreserveLCSSA);
1414 }
1415 
1416 ReturnInst *llvm::FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
1417                                              BasicBlock *Pred,
1418                                              DomTreeUpdater *DTU) {
1419   Instruction *UncondBranch = Pred->getTerminator();
1420   // Clone the return and add it to the end of the predecessor.
1421   Instruction *NewRet = RI->clone();
1422   NewRet->insertInto(Pred, Pred->end());
1423 
1424   // If the return instruction returns a value, and if the value was a
1425   // PHI node in "BB", propagate the right value into the return.
1426   for (Use &Op : NewRet->operands()) {
1427     Value *V = Op;
1428     Instruction *NewBC = nullptr;
1429     if (BitCastInst *BCI = dyn_cast<BitCastInst>(V)) {
1430       // Return value might be bitcasted. Clone and insert it before the
1431       // return instruction.
1432       V = BCI->getOperand(0);
1433       NewBC = BCI->clone();
1434       NewBC->insertInto(Pred, NewRet->getIterator());
1435       Op = NewBC;
1436     }
1437 
1438     Instruction *NewEV = nullptr;
1439     if (ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(V)) {
1440       V = EVI->getOperand(0);
1441       NewEV = EVI->clone();
1442       if (NewBC) {
1443         NewBC->setOperand(0, NewEV);
1444         NewEV->insertInto(Pred, NewBC->getIterator());
1445       } else {
1446         NewEV->insertInto(Pred, NewRet->getIterator());
1447         Op = NewEV;
1448       }
1449     }
1450 
1451     if (PHINode *PN = dyn_cast<PHINode>(V)) {
1452       if (PN->getParent() == BB) {
1453         if (NewEV) {
1454           NewEV->setOperand(0, PN->getIncomingValueForBlock(Pred));
1455         } else if (NewBC)
1456           NewBC->setOperand(0, PN->getIncomingValueForBlock(Pred));
1457         else
1458           Op = PN->getIncomingValueForBlock(Pred);
1459       }
1460     }
1461   }
1462 
1463   // Update any PHI nodes in the returning block to realize that we no
1464   // longer branch to them.
1465   BB->removePredecessor(Pred);
1466   UncondBranch->eraseFromParent();
1467 
1468   if (DTU)
1469     DTU->applyUpdates({{DominatorTree::Delete, Pred, BB}});
1470 
1471   return cast<ReturnInst>(NewRet);
1472 }
1473 
1474 Instruction *llvm::SplitBlockAndInsertIfThen(Value *Cond,
1475                                              Instruction *SplitBefore,
1476                                              bool Unreachable,
1477                                              MDNode *BranchWeights,
1478                                              DomTreeUpdater *DTU, LoopInfo *LI,
1479                                              BasicBlock *ThenBlock) {
1480   SplitBlockAndInsertIfThenElse(
1481       Cond, SplitBefore, &ThenBlock, /* ElseBlock */ nullptr,
1482       /* UnreachableThen */ Unreachable,
1483       /* UnreachableElse */ false, BranchWeights, DTU, LI);
1484   return ThenBlock->getTerminator();
1485 }
1486 
1487 Instruction *llvm::SplitBlockAndInsertIfElse(Value *Cond,
1488                                              Instruction *SplitBefore,
1489                                              bool Unreachable,
1490                                              MDNode *BranchWeights,
1491                                              DomTreeUpdater *DTU, LoopInfo *LI,
1492                                              BasicBlock *ElseBlock) {
1493   SplitBlockAndInsertIfThenElse(
1494       Cond, SplitBefore, /* ThenBlock */ nullptr, &ElseBlock,
1495       /* UnreachableThen */ false,
1496       /* UnreachableElse */ Unreachable, BranchWeights, DTU, LI);
1497   return ElseBlock->getTerminator();
1498 }
1499 
1500 void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore,
1501                                          Instruction **ThenTerm,
1502                                          Instruction **ElseTerm,
1503                                          MDNode *BranchWeights,
1504                                          DomTreeUpdater *DTU, LoopInfo *LI) {
1505   BasicBlock *ThenBlock = nullptr;
1506   BasicBlock *ElseBlock = nullptr;
1507   SplitBlockAndInsertIfThenElse(
1508       Cond, SplitBefore, &ThenBlock, &ElseBlock, /* UnreachableThen */ false,
1509       /* UnreachableElse */ false, BranchWeights, DTU, LI);
1510 
1511   *ThenTerm = ThenBlock->getTerminator();
1512   *ElseTerm = ElseBlock->getTerminator();
1513 }
1514 
1515 void llvm::SplitBlockAndInsertIfThenElse(
1516     Value *Cond, Instruction *SplitBefore, BasicBlock **ThenBlock,
1517     BasicBlock **ElseBlock, bool UnreachableThen, bool UnreachableElse,
1518     MDNode *BranchWeights, DomTreeUpdater *DTU, LoopInfo *LI) {
1519   assert((ThenBlock || ElseBlock) &&
1520          "At least one branch block must be created");
1521   assert((!UnreachableThen || !UnreachableElse) &&
1522          "Split block tail must be reachable");
1523 
1524   SmallVector<DominatorTree::UpdateType, 8> Updates;
1525   SmallPtrSet<BasicBlock *, 8> UniqueOrigSuccessors;
1526   BasicBlock *Head = SplitBefore->getParent();
1527   if (DTU) {
1528     UniqueOrigSuccessors.insert(succ_begin(Head), succ_end(Head));
1529     Updates.reserve(4 + 2 * UniqueOrigSuccessors.size());
1530   }
1531 
1532   LLVMContext &C = Head->getContext();
1533   BasicBlock *Tail = Head->splitBasicBlock(SplitBefore->getIterator());
1534   BasicBlock *TrueBlock = Tail;
1535   BasicBlock *FalseBlock = Tail;
1536   bool ThenToTailEdge = false;
1537   bool ElseToTailEdge = false;
1538 
1539   // Encapsulate the logic around creation/insertion/etc of a new block.
1540   auto handleBlock = [&](BasicBlock **PBB, bool Unreachable, BasicBlock *&BB,
1541                          bool &ToTailEdge) {
1542     if (PBB == nullptr)
1543       return; // Do not create/insert a block.
1544 
1545     if (*PBB)
1546       BB = *PBB; // Caller supplied block, use it.
1547     else {
1548       // Create a new block.
1549       BB = BasicBlock::Create(C, "", Head->getParent(), Tail);
1550       if (Unreachable)
1551         (void)new UnreachableInst(C, BB);
1552       else {
1553         (void)BranchInst::Create(Tail, BB);
1554         ToTailEdge = true;
1555       }
1556       BB->getTerminator()->setDebugLoc(SplitBefore->getDebugLoc());
1557       // Pass the new block back to the caller.
1558       *PBB = BB;
1559     }
1560   };
1561 
1562   handleBlock(ThenBlock, UnreachableThen, TrueBlock, ThenToTailEdge);
1563   handleBlock(ElseBlock, UnreachableElse, FalseBlock, ElseToTailEdge);
1564 
1565   Instruction *HeadOldTerm = Head->getTerminator();
1566   BranchInst *HeadNewTerm =
1567       BranchInst::Create(/*ifTrue*/ TrueBlock, /*ifFalse*/ FalseBlock, Cond);
1568   HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
1569   ReplaceInstWithInst(HeadOldTerm, HeadNewTerm);
1570 
1571   if (DTU) {
1572     Updates.emplace_back(DominatorTree::Insert, Head, TrueBlock);
1573     Updates.emplace_back(DominatorTree::Insert, Head, FalseBlock);
1574     if (ThenToTailEdge)
1575       Updates.emplace_back(DominatorTree::Insert, TrueBlock, Tail);
1576     if (ElseToTailEdge)
1577       Updates.emplace_back(DominatorTree::Insert, FalseBlock, Tail);
1578     for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1579       Updates.emplace_back(DominatorTree::Insert, Tail, UniqueOrigSuccessor);
1580     for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1581       Updates.emplace_back(DominatorTree::Delete, Head, UniqueOrigSuccessor);
1582     DTU->applyUpdates(Updates);
1583   }
1584 
1585   if (LI) {
1586     if (Loop *L = LI->getLoopFor(Head); L) {
1587       if (ThenToTailEdge)
1588         L->addBasicBlockToLoop(TrueBlock, *LI);
1589       if (ElseToTailEdge)
1590         L->addBasicBlockToLoop(FalseBlock, *LI);
1591       L->addBasicBlockToLoop(Tail, *LI);
1592     }
1593   }
1594 }
1595 
1596 std::pair<Instruction*, Value*>
1597 llvm::SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore) {
1598   BasicBlock *LoopPred = SplitBefore->getParent();
1599   BasicBlock *LoopBody = SplitBlock(SplitBefore->getParent(), SplitBefore);
1600   BasicBlock *LoopExit = SplitBlock(SplitBefore->getParent(), SplitBefore);
1601 
1602   auto *Ty = End->getType();
1603   auto &DL = SplitBefore->getModule()->getDataLayout();
1604   const unsigned Bitwidth = DL.getTypeSizeInBits(Ty);
1605 
1606   IRBuilder<> Builder(LoopBody->getTerminator());
1607   auto *IV = Builder.CreatePHI(Ty, 2, "iv");
1608   auto *IVNext =
1609     Builder.CreateAdd(IV, ConstantInt::get(Ty, 1), IV->getName() + ".next",
1610                       /*HasNUW=*/true, /*HasNSW=*/Bitwidth != 2);
1611   auto *IVCheck = Builder.CreateICmpEQ(IVNext, End,
1612                                        IV->getName() + ".check");
1613   Builder.CreateCondBr(IVCheck, LoopExit, LoopBody);
1614   LoopBody->getTerminator()->eraseFromParent();
1615 
1616   // Populate the IV PHI.
1617   IV->addIncoming(ConstantInt::get(Ty, 0), LoopPred);
1618   IV->addIncoming(IVNext, LoopBody);
1619 
1620   return std::make_pair(LoopBody->getFirstNonPHI(), IV);
1621 }
1622 
1623 void llvm::SplitBlockAndInsertForEachLane(ElementCount EC,
1624      Type *IndexTy, Instruction *InsertBefore,
1625      std::function<void(IRBuilderBase&, Value*)> Func) {
1626 
1627   IRBuilder<> IRB(InsertBefore);
1628 
1629   if (EC.isScalable()) {
1630     Value *NumElements = IRB.CreateElementCount(IndexTy, EC);
1631 
1632     auto [BodyIP, Index] =
1633       SplitBlockAndInsertSimpleForLoop(NumElements, InsertBefore);
1634 
1635     IRB.SetInsertPoint(BodyIP);
1636     Func(IRB, Index);
1637     return;
1638   }
1639 
1640   unsigned Num = EC.getFixedValue();
1641   for (unsigned Idx = 0; Idx < Num; ++Idx) {
1642     IRB.SetInsertPoint(InsertBefore);
1643     Func(IRB, ConstantInt::get(IndexTy, Idx));
1644   }
1645 }
1646 
1647 void llvm::SplitBlockAndInsertForEachLane(
1648     Value *EVL, Instruction *InsertBefore,
1649     std::function<void(IRBuilderBase &, Value *)> Func) {
1650 
1651   IRBuilder<> IRB(InsertBefore);
1652   Type *Ty = EVL->getType();
1653 
1654   if (!isa<ConstantInt>(EVL)) {
1655     auto [BodyIP, Index] = SplitBlockAndInsertSimpleForLoop(EVL, InsertBefore);
1656     IRB.SetInsertPoint(BodyIP);
1657     Func(IRB, Index);
1658     return;
1659   }
1660 
1661   unsigned Num = cast<ConstantInt>(EVL)->getZExtValue();
1662   for (unsigned Idx = 0; Idx < Num; ++Idx) {
1663     IRB.SetInsertPoint(InsertBefore);
1664     Func(IRB, ConstantInt::get(Ty, Idx));
1665   }
1666 }
1667 
1668 BranchInst *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
1669                                  BasicBlock *&IfFalse) {
1670   PHINode *SomePHI = dyn_cast<PHINode>(BB->begin());
1671   BasicBlock *Pred1 = nullptr;
1672   BasicBlock *Pred2 = nullptr;
1673 
1674   if (SomePHI) {
1675     if (SomePHI->getNumIncomingValues() != 2)
1676       return nullptr;
1677     Pred1 = SomePHI->getIncomingBlock(0);
1678     Pred2 = SomePHI->getIncomingBlock(1);
1679   } else {
1680     pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
1681     if (PI == PE) // No predecessor
1682       return nullptr;
1683     Pred1 = *PI++;
1684     if (PI == PE) // Only one predecessor
1685       return nullptr;
1686     Pred2 = *PI++;
1687     if (PI != PE) // More than two predecessors
1688       return nullptr;
1689   }
1690 
1691   // We can only handle branches.  Other control flow will be lowered to
1692   // branches if possible anyway.
1693   BranchInst *Pred1Br = dyn_cast<BranchInst>(Pred1->getTerminator());
1694   BranchInst *Pred2Br = dyn_cast<BranchInst>(Pred2->getTerminator());
1695   if (!Pred1Br || !Pred2Br)
1696     return nullptr;
1697 
1698   // Eliminate code duplication by ensuring that Pred1Br is conditional if
1699   // either are.
1700   if (Pred2Br->isConditional()) {
1701     // If both branches are conditional, we don't have an "if statement".  In
1702     // reality, we could transform this case, but since the condition will be
1703     // required anyway, we stand no chance of eliminating it, so the xform is
1704     // probably not profitable.
1705     if (Pred1Br->isConditional())
1706       return nullptr;
1707 
1708     std::swap(Pred1, Pred2);
1709     std::swap(Pred1Br, Pred2Br);
1710   }
1711 
1712   if (Pred1Br->isConditional()) {
1713     // The only thing we have to watch out for here is to make sure that Pred2
1714     // doesn't have incoming edges from other blocks.  If it does, the condition
1715     // doesn't dominate BB.
1716     if (!Pred2->getSinglePredecessor())
1717       return nullptr;
1718 
1719     // If we found a conditional branch predecessor, make sure that it branches
1720     // to BB and Pred2Br.  If it doesn't, this isn't an "if statement".
1721     if (Pred1Br->getSuccessor(0) == BB &&
1722         Pred1Br->getSuccessor(1) == Pred2) {
1723       IfTrue = Pred1;
1724       IfFalse = Pred2;
1725     } else if (Pred1Br->getSuccessor(0) == Pred2 &&
1726                Pred1Br->getSuccessor(1) == BB) {
1727       IfTrue = Pred2;
1728       IfFalse = Pred1;
1729     } else {
1730       // We know that one arm of the conditional goes to BB, so the other must
1731       // go somewhere unrelated, and this must not be an "if statement".
1732       return nullptr;
1733     }
1734 
1735     return Pred1Br;
1736   }
1737 
1738   // Ok, if we got here, both predecessors end with an unconditional branch to
1739   // BB.  Don't panic!  If both blocks only have a single (identical)
1740   // predecessor, and THAT is a conditional branch, then we're all ok!
1741   BasicBlock *CommonPred = Pred1->getSinglePredecessor();
1742   if (CommonPred == nullptr || CommonPred != Pred2->getSinglePredecessor())
1743     return nullptr;
1744 
1745   // Otherwise, if this is a conditional branch, then we can use it!
1746   BranchInst *BI = dyn_cast<BranchInst>(CommonPred->getTerminator());
1747   if (!BI) return nullptr;
1748 
1749   assert(BI->isConditional() && "Two successors but not conditional?");
1750   if (BI->getSuccessor(0) == Pred1) {
1751     IfTrue = Pred1;
1752     IfFalse = Pred2;
1753   } else {
1754     IfTrue = Pred2;
1755     IfFalse = Pred1;
1756   }
1757   return BI;
1758 }
1759 
1760 // After creating a control flow hub, the operands of PHINodes in an outgoing
1761 // block Out no longer match the predecessors of that block. Predecessors of Out
1762 // that are incoming blocks to the hub are now replaced by just one edge from
1763 // the hub. To match this new control flow, the corresponding values from each
1764 // PHINode must now be moved a new PHINode in the first guard block of the hub.
1765 //
1766 // This operation cannot be performed with SSAUpdater, because it involves one
1767 // new use: If the block Out is in the list of Incoming blocks, then the newly
1768 // created PHI in the Hub will use itself along that edge from Out to Hub.
1769 static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock,
1770                           const SetVector<BasicBlock *> &Incoming,
1771                           BasicBlock *FirstGuardBlock) {
1772   auto I = Out->begin();
1773   while (I != Out->end() && isa<PHINode>(I)) {
1774     auto Phi = cast<PHINode>(I);
1775     auto NewPhi =
1776         PHINode::Create(Phi->getType(), Incoming.size(),
1777                         Phi->getName() + ".moved", &FirstGuardBlock->front());
1778     for (auto *In : Incoming) {
1779       Value *V = UndefValue::get(Phi->getType());
1780       if (In == Out) {
1781         V = NewPhi;
1782       } else if (Phi->getBasicBlockIndex(In) != -1) {
1783         V = Phi->removeIncomingValue(In, false);
1784       }
1785       NewPhi->addIncoming(V, In);
1786     }
1787     assert(NewPhi->getNumIncomingValues() == Incoming.size());
1788     if (Phi->getNumOperands() == 0) {
1789       Phi->replaceAllUsesWith(NewPhi);
1790       I = Phi->eraseFromParent();
1791       continue;
1792     }
1793     Phi->addIncoming(NewPhi, GuardBlock);
1794     ++I;
1795   }
1796 }
1797 
1798 using BBPredicates = DenseMap<BasicBlock *, Instruction *>;
1799 using BBSetVector = SetVector<BasicBlock *>;
1800 
1801 // Redirects the terminator of the incoming block to the first guard
1802 // block in the hub. The condition of the original terminator (if it
1803 // was conditional) and its original successors are returned as a
1804 // tuple <condition, succ0, succ1>. The function additionally filters
1805 // out successors that are not in the set of outgoing blocks.
1806 //
1807 // - condition is non-null iff the branch is conditional.
1808 // - Succ1 is non-null iff the sole/taken target is an outgoing block.
1809 // - Succ2 is non-null iff condition is non-null and the fallthrough
1810 //         target is an outgoing block.
1811 static std::tuple<Value *, BasicBlock *, BasicBlock *>
1812 redirectToHub(BasicBlock *BB, BasicBlock *FirstGuardBlock,
1813               const BBSetVector &Outgoing) {
1814   assert(isa<BranchInst>(BB->getTerminator()) &&
1815          "Only support branch terminator.");
1816   auto Branch = cast<BranchInst>(BB->getTerminator());
1817   auto Condition = Branch->isConditional() ? Branch->getCondition() : nullptr;
1818 
1819   BasicBlock *Succ0 = Branch->getSuccessor(0);
1820   BasicBlock *Succ1 = nullptr;
1821   Succ0 = Outgoing.count(Succ0) ? Succ0 : nullptr;
1822 
1823   if (Branch->isUnconditional()) {
1824     Branch->setSuccessor(0, FirstGuardBlock);
1825     assert(Succ0);
1826   } else {
1827     Succ1 = Branch->getSuccessor(1);
1828     Succ1 = Outgoing.count(Succ1) ? Succ1 : nullptr;
1829     assert(Succ0 || Succ1);
1830     if (Succ0 && !Succ1) {
1831       Branch->setSuccessor(0, FirstGuardBlock);
1832     } else if (Succ1 && !Succ0) {
1833       Branch->setSuccessor(1, FirstGuardBlock);
1834     } else {
1835       Branch->eraseFromParent();
1836       BranchInst::Create(FirstGuardBlock, BB);
1837     }
1838   }
1839 
1840   assert(Succ0 || Succ1);
1841   return std::make_tuple(Condition, Succ0, Succ1);
1842 }
1843 // Setup the branch instructions for guard blocks.
1844 //
1845 // Each guard block terminates in a conditional branch that transfers
1846 // control to the corresponding outgoing block or the next guard
1847 // block. The last guard block has two outgoing blocks as successors
1848 // since the condition for the final outgoing block is trivially
1849 // true. So we create one less block (including the first guard block)
1850 // than the number of outgoing blocks.
1851 static void setupBranchForGuard(SmallVectorImpl<BasicBlock *> &GuardBlocks,
1852                                 const BBSetVector &Outgoing,
1853                                 BBPredicates &GuardPredicates) {
1854   // To help keep the loop simple, temporarily append the last
1855   // outgoing block to the list of guard blocks.
1856   GuardBlocks.push_back(Outgoing.back());
1857 
1858   for (int i = 0, e = GuardBlocks.size() - 1; i != e; ++i) {
1859     auto Out = Outgoing[i];
1860     assert(GuardPredicates.count(Out));
1861     BranchInst::Create(Out, GuardBlocks[i + 1], GuardPredicates[Out],
1862                        GuardBlocks[i]);
1863   }
1864 
1865   // Remove the last block from the guard list.
1866   GuardBlocks.pop_back();
1867 }
1868 
1869 /// We are using one integer to represent the block we are branching to. Then at
1870 /// each guard block, the predicate was calcuated using a simple `icmp eq`.
1871 static void calcPredicateUsingInteger(
1872     const BBSetVector &Incoming, const BBSetVector &Outgoing,
1873     SmallVectorImpl<BasicBlock *> &GuardBlocks, BBPredicates &GuardPredicates) {
1874   auto &Context = Incoming.front()->getContext();
1875   auto FirstGuardBlock = GuardBlocks.front();
1876 
1877   auto Phi = PHINode::Create(Type::getInt32Ty(Context), Incoming.size(),
1878                              "merged.bb.idx", FirstGuardBlock);
1879 
1880   for (auto In : Incoming) {
1881     Value *Condition;
1882     BasicBlock *Succ0;
1883     BasicBlock *Succ1;
1884     std::tie(Condition, Succ0, Succ1) =
1885         redirectToHub(In, FirstGuardBlock, Outgoing);
1886     Value *IncomingId = nullptr;
1887     if (Succ0 && Succ1) {
1888       // target_bb_index = Condition ? index_of_succ0 : index_of_succ1.
1889       auto Succ0Iter = find(Outgoing, Succ0);
1890       auto Succ1Iter = find(Outgoing, Succ1);
1891       Value *Id0 = ConstantInt::get(Type::getInt32Ty(Context),
1892                                     std::distance(Outgoing.begin(), Succ0Iter));
1893       Value *Id1 = ConstantInt::get(Type::getInt32Ty(Context),
1894                                     std::distance(Outgoing.begin(), Succ1Iter));
1895       IncomingId = SelectInst::Create(Condition, Id0, Id1, "target.bb.idx",
1896                                       In->getTerminator());
1897     } else {
1898       // Get the index of the non-null successor.
1899       auto SuccIter = Succ0 ? find(Outgoing, Succ0) : find(Outgoing, Succ1);
1900       IncomingId = ConstantInt::get(Type::getInt32Ty(Context),
1901                                     std::distance(Outgoing.begin(), SuccIter));
1902     }
1903     Phi->addIncoming(IncomingId, In);
1904   }
1905 
1906   for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
1907     auto Out = Outgoing[i];
1908     auto Cmp = ICmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, Phi,
1909                                 ConstantInt::get(Type::getInt32Ty(Context), i),
1910                                 Out->getName() + ".predicate", GuardBlocks[i]);
1911     GuardPredicates[Out] = Cmp;
1912   }
1913 }
1914 
1915 /// We record the predicate of each outgoing block using a phi of boolean.
1916 static void calcPredicateUsingBooleans(
1917     const BBSetVector &Incoming, const BBSetVector &Outgoing,
1918     SmallVectorImpl<BasicBlock *> &GuardBlocks, BBPredicates &GuardPredicates,
1919     SmallVectorImpl<WeakVH> &DeletionCandidates) {
1920   auto &Context = Incoming.front()->getContext();
1921   auto BoolTrue = ConstantInt::getTrue(Context);
1922   auto BoolFalse = ConstantInt::getFalse(Context);
1923   auto FirstGuardBlock = GuardBlocks.front();
1924 
1925   // The predicate for the last outgoing is trivially true, and so we
1926   // process only the first N-1 successors.
1927   for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
1928     auto Out = Outgoing[i];
1929     LLVM_DEBUG(dbgs() << "Creating guard for " << Out->getName() << "\n");
1930 
1931     auto Phi =
1932         PHINode::Create(Type::getInt1Ty(Context), Incoming.size(),
1933                         StringRef("Guard.") + Out->getName(), FirstGuardBlock);
1934     GuardPredicates[Out] = Phi;
1935   }
1936 
1937   for (auto *In : Incoming) {
1938     Value *Condition;
1939     BasicBlock *Succ0;
1940     BasicBlock *Succ1;
1941     std::tie(Condition, Succ0, Succ1) =
1942         redirectToHub(In, FirstGuardBlock, Outgoing);
1943 
1944     // Optimization: Consider an incoming block A with both successors
1945     // Succ0 and Succ1 in the set of outgoing blocks. The predicates
1946     // for Succ0 and Succ1 complement each other. If Succ0 is visited
1947     // first in the loop below, control will branch to Succ0 using the
1948     // corresponding predicate. But if that branch is not taken, then
1949     // control must reach Succ1, which means that the incoming value of
1950     // the predicate from `In` is true for Succ1.
1951     bool OneSuccessorDone = false;
1952     for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
1953       auto Out = Outgoing[i];
1954       PHINode *Phi = cast<PHINode>(GuardPredicates[Out]);
1955       if (Out != Succ0 && Out != Succ1) {
1956         Phi->addIncoming(BoolFalse, In);
1957       } else if (!Succ0 || !Succ1 || OneSuccessorDone) {
1958         // Optimization: When only one successor is an outgoing block,
1959         // the incoming predicate from `In` is always true.
1960         Phi->addIncoming(BoolTrue, In);
1961       } else {
1962         assert(Succ0 && Succ1);
1963         if (Out == Succ0) {
1964           Phi->addIncoming(Condition, In);
1965         } else {
1966           auto Inverted = invertCondition(Condition);
1967           DeletionCandidates.push_back(Condition);
1968           Phi->addIncoming(Inverted, In);
1969         }
1970         OneSuccessorDone = true;
1971       }
1972     }
1973   }
1974 }
1975 
1976 // Capture the existing control flow as guard predicates, and redirect
1977 // control flow from \p Incoming block through the \p GuardBlocks to the
1978 // \p Outgoing blocks.
1979 //
1980 // There is one guard predicate for each outgoing block OutBB. The
1981 // predicate represents whether the hub should transfer control flow
1982 // to OutBB. These predicates are NOT ORTHOGONAL. The Hub evaluates
1983 // them in the same order as the Outgoing set-vector, and control
1984 // branches to the first outgoing block whose predicate evaluates to true.
1985 static void
1986 convertToGuardPredicates(SmallVectorImpl<BasicBlock *> &GuardBlocks,
1987                          SmallVectorImpl<WeakVH> &DeletionCandidates,
1988                          const BBSetVector &Incoming,
1989                          const BBSetVector &Outgoing, const StringRef Prefix,
1990                          std::optional<unsigned> MaxControlFlowBooleans) {
1991   BBPredicates GuardPredicates;
1992   auto F = Incoming.front()->getParent();
1993 
1994   for (int i = 0, e = Outgoing.size() - 1; i != e; ++i)
1995     GuardBlocks.push_back(
1996         BasicBlock::Create(F->getContext(), Prefix + ".guard", F));
1997 
1998   // When we are using an integer to record which target block to jump to, we
1999   // are creating less live values, actually we are using one single integer to
2000   // store the index of the target block. When we are using booleans to store
2001   // the branching information, we need (N-1) boolean values, where N is the
2002   // number of outgoing block.
2003   if (!MaxControlFlowBooleans || Outgoing.size() <= *MaxControlFlowBooleans)
2004     calcPredicateUsingBooleans(Incoming, Outgoing, GuardBlocks, GuardPredicates,
2005                                DeletionCandidates);
2006   else
2007     calcPredicateUsingInteger(Incoming, Outgoing, GuardBlocks, GuardPredicates);
2008 
2009   setupBranchForGuard(GuardBlocks, Outgoing, GuardPredicates);
2010 }
2011 
2012 BasicBlock *llvm::CreateControlFlowHub(
2013     DomTreeUpdater *DTU, SmallVectorImpl<BasicBlock *> &GuardBlocks,
2014     const BBSetVector &Incoming, const BBSetVector &Outgoing,
2015     const StringRef Prefix, std::optional<unsigned> MaxControlFlowBooleans) {
2016   if (Outgoing.size() < 2)
2017     return Outgoing.front();
2018 
2019   SmallVector<DominatorTree::UpdateType, 16> Updates;
2020   if (DTU) {
2021     for (auto *In : Incoming) {
2022       for (auto Succ : successors(In))
2023         if (Outgoing.count(Succ))
2024           Updates.push_back({DominatorTree::Delete, In, Succ});
2025     }
2026   }
2027 
2028   SmallVector<WeakVH, 8> DeletionCandidates;
2029   convertToGuardPredicates(GuardBlocks, DeletionCandidates, Incoming, Outgoing,
2030                            Prefix, MaxControlFlowBooleans);
2031   auto FirstGuardBlock = GuardBlocks.front();
2032 
2033   // Update the PHINodes in each outgoing block to match the new control flow.
2034   for (int i = 0, e = GuardBlocks.size(); i != e; ++i)
2035     reconnectPhis(Outgoing[i], GuardBlocks[i], Incoming, FirstGuardBlock);
2036 
2037   reconnectPhis(Outgoing.back(), GuardBlocks.back(), Incoming, FirstGuardBlock);
2038 
2039   if (DTU) {
2040     int NumGuards = GuardBlocks.size();
2041     assert((int)Outgoing.size() == NumGuards + 1);
2042 
2043     for (auto In : Incoming)
2044       Updates.push_back({DominatorTree::Insert, In, FirstGuardBlock});
2045 
2046     for (int i = 0; i != NumGuards - 1; ++i) {
2047       Updates.push_back({DominatorTree::Insert, GuardBlocks[i], Outgoing[i]});
2048       Updates.push_back(
2049           {DominatorTree::Insert, GuardBlocks[i], GuardBlocks[i + 1]});
2050     }
2051     Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
2052                        Outgoing[NumGuards - 1]});
2053     Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
2054                        Outgoing[NumGuards]});
2055     DTU->applyUpdates(Updates);
2056   }
2057 
2058   for (auto I : DeletionCandidates) {
2059     if (I->use_empty())
2060       if (auto Inst = dyn_cast_or_null<Instruction>(I))
2061         Inst->eraseFromParent();
2062   }
2063 
2064   return FirstGuardBlock;
2065 }
2066 
2067 void llvm::InvertBranch(BranchInst *PBI, IRBuilderBase &Builder) {
2068   Value *NewCond = PBI->getCondition();
2069   // If this is a "cmp" instruction, only used for branching (and nowhere
2070   // else), then we can simply invert the predicate.
2071   if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) {
2072     CmpInst *CI = cast<CmpInst>(NewCond);
2073     CI->setPredicate(CI->getInversePredicate());
2074   } else
2075     NewCond = Builder.CreateNot(NewCond, NewCond->getName() + ".not");
2076 
2077   PBI->setCondition(NewCond);
2078   PBI->swapSuccessors();
2079 }
2080