1 //===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Peephole optimize the CFG. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ADT/APInt.h" 14 #include "llvm/ADT/ArrayRef.h" 15 #include "llvm/ADT/DenseMap.h" 16 #include "llvm/ADT/MapVector.h" 17 #include "llvm/ADT/Optional.h" 18 #include "llvm/ADT/STLExtras.h" 19 #include "llvm/ADT/ScopeExit.h" 20 #include "llvm/ADT/Sequence.h" 21 #include "llvm/ADT/SetOperations.h" 22 #include "llvm/ADT/SetVector.h" 23 #include "llvm/ADT/SmallPtrSet.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/Statistic.h" 26 #include "llvm/ADT/StringRef.h" 27 #include "llvm/Analysis/AssumptionCache.h" 28 #include "llvm/Analysis/ConstantFolding.h" 29 #include "llvm/Analysis/EHPersonalities.h" 30 #include "llvm/Analysis/GuardUtils.h" 31 #include "llvm/Analysis/InstructionSimplify.h" 32 #include "llvm/Analysis/MemorySSA.h" 33 #include "llvm/Analysis/MemorySSAUpdater.h" 34 #include "llvm/Analysis/TargetTransformInfo.h" 35 #include "llvm/Analysis/ValueTracking.h" 36 #include "llvm/IR/Attributes.h" 37 #include "llvm/IR/BasicBlock.h" 38 #include "llvm/IR/CFG.h" 39 #include "llvm/IR/Constant.h" 40 #include "llvm/IR/ConstantRange.h" 41 #include "llvm/IR/Constants.h" 42 #include "llvm/IR/DataLayout.h" 43 #include "llvm/IR/DerivedTypes.h" 44 #include "llvm/IR/Function.h" 45 #include "llvm/IR/GlobalValue.h" 46 #include "llvm/IR/GlobalVariable.h" 47 #include "llvm/IR/IRBuilder.h" 48 #include "llvm/IR/InstrTypes.h" 49 #include "llvm/IR/Instruction.h" 50 #include "llvm/IR/Instructions.h" 51 #include "llvm/IR/IntrinsicInst.h" 52 #include "llvm/IR/Intrinsics.h" 53 #include "llvm/IR/LLVMContext.h" 54 #include "llvm/IR/MDBuilder.h" 55 #include "llvm/IR/Metadata.h" 56 #include "llvm/IR/Module.h" 57 #include "llvm/IR/NoFolder.h" 58 #include "llvm/IR/Operator.h" 59 #include "llvm/IR/PatternMatch.h" 60 #include "llvm/IR/PseudoProbe.h" 61 #include "llvm/IR/Type.h" 62 #include "llvm/IR/Use.h" 63 #include "llvm/IR/User.h" 64 #include "llvm/IR/Value.h" 65 #include "llvm/IR/ValueHandle.h" 66 #include "llvm/Support/BranchProbability.h" 67 #include "llvm/Support/Casting.h" 68 #include "llvm/Support/CommandLine.h" 69 #include "llvm/Support/Debug.h" 70 #include "llvm/Support/ErrorHandling.h" 71 #include "llvm/Support/KnownBits.h" 72 #include "llvm/Support/MathExtras.h" 73 #include "llvm/Support/raw_ostream.h" 74 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 75 #include "llvm/Transforms/Utils/Local.h" 76 #include "llvm/Transforms/Utils/SSAUpdater.h" 77 #include "llvm/Transforms/Utils/ValueMapper.h" 78 #include <algorithm> 79 #include <cassert> 80 #include <climits> 81 #include <cstddef> 82 #include <cstdint> 83 #include <iterator> 84 #include <map> 85 #include <set> 86 #include <tuple> 87 #include <utility> 88 #include <vector> 89 90 using namespace llvm; 91 using namespace PatternMatch; 92 93 #define DEBUG_TYPE "simplifycfg" 94 95 cl::opt<bool> llvm::RequireAndPreserveDomTree( 96 "simplifycfg-require-and-preserve-domtree", cl::Hidden, cl::ZeroOrMore, 97 cl::init(false), 98 cl::desc("Temorary development switch used to gradually uplift SimplifyCFG " 99 "into preserving DomTree,")); 100 101 // Chosen as 2 so as to be cheap, but still to have enough power to fold 102 // a select, so the "clamp" idiom (of a min followed by a max) will be caught. 103 // To catch this, we need to fold a compare and a select, hence '2' being the 104 // minimum reasonable default. 105 static cl::opt<unsigned> PHINodeFoldingThreshold( 106 "phi-node-folding-threshold", cl::Hidden, cl::init(2), 107 cl::desc( 108 "Control the amount of phi node folding to perform (default = 2)")); 109 110 static cl::opt<unsigned> TwoEntryPHINodeFoldingThreshold( 111 "two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4), 112 cl::desc("Control the maximal total instruction cost that we are willing " 113 "to speculatively execute to fold a 2-entry PHI node into a " 114 "select (default = 4)")); 115 116 static cl::opt<bool> 117 HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true), 118 cl::desc("Hoist common instructions up to the parent block")); 119 120 static cl::opt<bool> 121 SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true), 122 cl::desc("Sink common instructions down to the end block")); 123 124 static cl::opt<bool> HoistCondStores( 125 "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true), 126 cl::desc("Hoist conditional stores if an unconditional store precedes")); 127 128 static cl::opt<bool> MergeCondStores( 129 "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true), 130 cl::desc("Hoist conditional stores even if an unconditional store does not " 131 "precede - hoist multiple conditional stores into a single " 132 "predicated store")); 133 134 static cl::opt<bool> MergeCondStoresAggressively( 135 "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false), 136 cl::desc("When merging conditional stores, do so even if the resultant " 137 "basic blocks are unlikely to be if-converted as a result")); 138 139 static cl::opt<bool> SpeculateOneExpensiveInst( 140 "speculate-one-expensive-inst", cl::Hidden, cl::init(true), 141 cl::desc("Allow exactly one expensive instruction to be speculatively " 142 "executed")); 143 144 static cl::opt<unsigned> MaxSpeculationDepth( 145 "max-speculation-depth", cl::Hidden, cl::init(10), 146 cl::desc("Limit maximum recursion depth when calculating costs of " 147 "speculatively executed instructions")); 148 149 static cl::opt<int> 150 MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden, 151 cl::init(10), 152 cl::desc("Max size of a block which is still considered " 153 "small enough to thread through")); 154 155 // Two is chosen to allow one negation and a logical combine. 156 static cl::opt<unsigned> 157 BranchFoldThreshold("simplifycfg-branch-fold-threshold", cl::Hidden, 158 cl::init(2), 159 cl::desc("Maximum cost of combining conditions when " 160 "folding branches")); 161 162 STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps"); 163 STATISTIC(NumLinearMaps, 164 "Number of switch instructions turned into linear mapping"); 165 STATISTIC(NumLookupTables, 166 "Number of switch instructions turned into lookup tables"); 167 STATISTIC( 168 NumLookupTablesHoles, 169 "Number of switch instructions turned into lookup tables (holes checked)"); 170 STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares"); 171 STATISTIC(NumFoldValueComparisonIntoPredecessors, 172 "Number of value comparisons folded into predecessor basic blocks"); 173 STATISTIC(NumFoldBranchToCommonDest, 174 "Number of branches folded into predecessor basic block"); 175 STATISTIC( 176 NumHoistCommonCode, 177 "Number of common instruction 'blocks' hoisted up to the begin block"); 178 STATISTIC(NumHoistCommonInstrs, 179 "Number of common instructions hoisted up to the begin block"); 180 STATISTIC(NumSinkCommonCode, 181 "Number of common instruction 'blocks' sunk down to the end block"); 182 STATISTIC(NumSinkCommonInstrs, 183 "Number of common instructions sunk down to the end block"); 184 STATISTIC(NumSpeculations, "Number of speculative executed instructions"); 185 STATISTIC(NumInvokes, 186 "Number of invokes with empty resume blocks simplified into calls"); 187 188 namespace { 189 190 // The first field contains the value that the switch produces when a certain 191 // case group is selected, and the second field is a vector containing the 192 // cases composing the case group. 193 using SwitchCaseResultVectorTy = 194 SmallVector<std::pair<Constant *, SmallVector<ConstantInt *, 4>>, 2>; 195 196 // The first field contains the phi node that generates a result of the switch 197 // and the second field contains the value generated for a certain case in the 198 // switch for that PHI. 199 using SwitchCaseResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>; 200 201 /// ValueEqualityComparisonCase - Represents a case of a switch. 202 struct ValueEqualityComparisonCase { 203 ConstantInt *Value; 204 BasicBlock *Dest; 205 206 ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest) 207 : Value(Value), Dest(Dest) {} 208 209 bool operator<(ValueEqualityComparisonCase RHS) const { 210 // Comparing pointers is ok as we only rely on the order for uniquing. 211 return Value < RHS.Value; 212 } 213 214 bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; } 215 }; 216 217 class SimplifyCFGOpt { 218 const TargetTransformInfo &TTI; 219 DomTreeUpdater *DTU; 220 const DataLayout &DL; 221 ArrayRef<WeakVH> LoopHeaders; 222 const SimplifyCFGOptions &Options; 223 bool Resimplify; 224 225 Value *isValueEqualityComparison(Instruction *TI); 226 BasicBlock *GetValueEqualityComparisonCases( 227 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases); 228 bool SimplifyEqualityComparisonWithOnlyPredecessor(Instruction *TI, 229 BasicBlock *Pred, 230 IRBuilder<> &Builder); 231 bool PerformValueComparisonIntoPredecessorFolding(Instruction *TI, Value *&CV, 232 Instruction *PTI, 233 IRBuilder<> &Builder); 234 bool FoldValueComparisonIntoPredecessors(Instruction *TI, 235 IRBuilder<> &Builder); 236 237 bool simplifyResume(ResumeInst *RI, IRBuilder<> &Builder); 238 bool simplifySingleResume(ResumeInst *RI); 239 bool simplifyCommonResume(ResumeInst *RI); 240 bool simplifyCleanupReturn(CleanupReturnInst *RI); 241 bool simplifyUnreachable(UnreachableInst *UI); 242 bool simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder); 243 bool simplifyIndirectBr(IndirectBrInst *IBI); 244 bool simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder); 245 bool simplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder); 246 bool simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder); 247 248 bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI, 249 IRBuilder<> &Builder); 250 251 bool HoistThenElseCodeToIf(BranchInst *BI, const TargetTransformInfo &TTI, 252 bool EqTermsOnly); 253 bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB, 254 const TargetTransformInfo &TTI); 255 bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond, 256 BasicBlock *TrueBB, BasicBlock *FalseBB, 257 uint32_t TrueWeight, uint32_t FalseWeight); 258 bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder, 259 const DataLayout &DL); 260 bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select); 261 bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI); 262 bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder); 263 264 public: 265 SimplifyCFGOpt(const TargetTransformInfo &TTI, DomTreeUpdater *DTU, 266 const DataLayout &DL, ArrayRef<WeakVH> LoopHeaders, 267 const SimplifyCFGOptions &Opts) 268 : TTI(TTI), DTU(DTU), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) { 269 assert((!DTU || !DTU->hasPostDomTree()) && 270 "SimplifyCFG is not yet capable of maintaining validity of a " 271 "PostDomTree, so don't ask for it."); 272 } 273 274 bool simplifyOnce(BasicBlock *BB); 275 bool simplifyOnceImpl(BasicBlock *BB); 276 bool run(BasicBlock *BB); 277 278 // Helper to set Resimplify and return change indication. 279 bool requestResimplify() { 280 Resimplify = true; 281 return true; 282 } 283 }; 284 285 } // end anonymous namespace 286 287 /// Return true if it is safe to merge these two 288 /// terminator instructions together. 289 static bool 290 SafeToMergeTerminators(Instruction *SI1, Instruction *SI2, 291 SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) { 292 if (SI1 == SI2) 293 return false; // Can't merge with self! 294 295 // It is not safe to merge these two switch instructions if they have a common 296 // successor, and if that successor has a PHI node, and if *that* PHI node has 297 // conflicting incoming values from the two switch blocks. 298 BasicBlock *SI1BB = SI1->getParent(); 299 BasicBlock *SI2BB = SI2->getParent(); 300 301 SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB)); 302 bool Fail = false; 303 for (BasicBlock *Succ : successors(SI2BB)) 304 if (SI1Succs.count(Succ)) 305 for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) { 306 PHINode *PN = cast<PHINode>(BBI); 307 if (PN->getIncomingValueForBlock(SI1BB) != 308 PN->getIncomingValueForBlock(SI2BB)) { 309 if (FailBlocks) 310 FailBlocks->insert(Succ); 311 Fail = true; 312 } 313 } 314 315 return !Fail; 316 } 317 318 /// Update PHI nodes in Succ to indicate that there will now be entries in it 319 /// from the 'NewPred' block. The values that will be flowing into the PHI nodes 320 /// will be the same as those coming in from ExistPred, an existing predecessor 321 /// of Succ. 322 static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred, 323 BasicBlock *ExistPred, 324 MemorySSAUpdater *MSSAU = nullptr) { 325 for (PHINode &PN : Succ->phis()) 326 PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred); 327 if (MSSAU) 328 if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ)) 329 MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred); 330 } 331 332 /// Compute an abstract "cost" of speculating the given instruction, 333 /// which is assumed to be safe to speculate. TCC_Free means cheap, 334 /// TCC_Basic means less cheap, and TCC_Expensive means prohibitively 335 /// expensive. 336 static InstructionCost computeSpeculationCost(const User *I, 337 const TargetTransformInfo &TTI) { 338 assert(isSafeToSpeculativelyExecute(I) && 339 "Instruction is not safe to speculatively execute!"); 340 return TTI.getUserCost(I, TargetTransformInfo::TCK_SizeAndLatency); 341 } 342 343 /// If we have a merge point of an "if condition" as accepted above, 344 /// return true if the specified value dominates the block. We 345 /// don't handle the true generality of domination here, just a special case 346 /// which works well enough for us. 347 /// 348 /// If AggressiveInsts is non-null, and if V does not dominate BB, we check to 349 /// see if V (which must be an instruction) and its recursive operands 350 /// that do not dominate BB have a combined cost lower than Budget and 351 /// are non-trapping. If both are true, the instruction is inserted into the 352 /// set and true is returned. 353 /// 354 /// The cost for most non-trapping instructions is defined as 1 except for 355 /// Select whose cost is 2. 356 /// 357 /// After this function returns, Cost is increased by the cost of 358 /// V plus its non-dominating operands. If that cost is greater than 359 /// Budget, false is returned and Cost is undefined. 360 static bool dominatesMergePoint(Value *V, BasicBlock *BB, 361 SmallPtrSetImpl<Instruction *> &AggressiveInsts, 362 InstructionCost &Cost, 363 InstructionCost Budget, 364 const TargetTransformInfo &TTI, 365 unsigned Depth = 0) { 366 // It is possible to hit a zero-cost cycle (phi/gep instructions for example), 367 // so limit the recursion depth. 368 // TODO: While this recursion limit does prevent pathological behavior, it 369 // would be better to track visited instructions to avoid cycles. 370 if (Depth == MaxSpeculationDepth) 371 return false; 372 373 Instruction *I = dyn_cast<Instruction>(V); 374 if (!I) { 375 // Non-instructions all dominate instructions, but not all constantexprs 376 // can be executed unconditionally. 377 if (ConstantExpr *C = dyn_cast<ConstantExpr>(V)) 378 if (C->canTrap()) 379 return false; 380 return true; 381 } 382 BasicBlock *PBB = I->getParent(); 383 384 // We don't want to allow weird loops that might have the "if condition" in 385 // the bottom of this block. 386 if (PBB == BB) 387 return false; 388 389 // If this instruction is defined in a block that contains an unconditional 390 // branch to BB, then it must be in the 'conditional' part of the "if 391 // statement". If not, it definitely dominates the region. 392 BranchInst *BI = dyn_cast<BranchInst>(PBB->getTerminator()); 393 if (!BI || BI->isConditional() || BI->getSuccessor(0) != BB) 394 return true; 395 396 // If we have seen this instruction before, don't count it again. 397 if (AggressiveInsts.count(I)) 398 return true; 399 400 // Okay, it looks like the instruction IS in the "condition". Check to 401 // see if it's a cheap instruction to unconditionally compute, and if it 402 // only uses stuff defined outside of the condition. If so, hoist it out. 403 if (!isSafeToSpeculativelyExecute(I)) 404 return false; 405 406 Cost += computeSpeculationCost(I, TTI); 407 408 // Allow exactly one instruction to be speculated regardless of its cost 409 // (as long as it is safe to do so). 410 // This is intended to flatten the CFG even if the instruction is a division 411 // or other expensive operation. The speculation of an expensive instruction 412 // is expected to be undone in CodeGenPrepare if the speculation has not 413 // enabled further IR optimizations. 414 if (Cost > Budget && 415 (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0 || 416 !Cost.isValid())) 417 return false; 418 419 // Okay, we can only really hoist these out if their operands do 420 // not take us over the cost threshold. 421 for (Use &Op : I->operands()) 422 if (!dominatesMergePoint(Op, BB, AggressiveInsts, Cost, Budget, TTI, 423 Depth + 1)) 424 return false; 425 // Okay, it's safe to do this! Remember this instruction. 426 AggressiveInsts.insert(I); 427 return true; 428 } 429 430 /// Extract ConstantInt from value, looking through IntToPtr 431 /// and PointerNullValue. Return NULL if value is not a constant int. 432 static ConstantInt *GetConstantInt(Value *V, const DataLayout &DL) { 433 // Normal constant int. 434 ConstantInt *CI = dyn_cast<ConstantInt>(V); 435 if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy()) 436 return CI; 437 438 // This is some kind of pointer constant. Turn it into a pointer-sized 439 // ConstantInt if possible. 440 IntegerType *PtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType())); 441 442 // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*). 443 if (isa<ConstantPointerNull>(V)) 444 return ConstantInt::get(PtrTy, 0); 445 446 // IntToPtr const int. 447 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) 448 if (CE->getOpcode() == Instruction::IntToPtr) 449 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) { 450 // The constant is very likely to have the right type already. 451 if (CI->getType() == PtrTy) 452 return CI; 453 else 454 return cast<ConstantInt>( 455 ConstantExpr::getIntegerCast(CI, PtrTy, /*isSigned=*/false)); 456 } 457 return nullptr; 458 } 459 460 namespace { 461 462 /// Given a chain of or (||) or and (&&) comparison of a value against a 463 /// constant, this will try to recover the information required for a switch 464 /// structure. 465 /// It will depth-first traverse the chain of comparison, seeking for patterns 466 /// like %a == 12 or %a < 4 and combine them to produce a set of integer 467 /// representing the different cases for the switch. 468 /// Note that if the chain is composed of '||' it will build the set of elements 469 /// that matches the comparisons (i.e. any of this value validate the chain) 470 /// while for a chain of '&&' it will build the set elements that make the test 471 /// fail. 472 struct ConstantComparesGatherer { 473 const DataLayout &DL; 474 475 /// Value found for the switch comparison 476 Value *CompValue = nullptr; 477 478 /// Extra clause to be checked before the switch 479 Value *Extra = nullptr; 480 481 /// Set of integers to match in switch 482 SmallVector<ConstantInt *, 8> Vals; 483 484 /// Number of comparisons matched in the and/or chain 485 unsigned UsedICmps = 0; 486 487 /// Construct and compute the result for the comparison instruction Cond 488 ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) { 489 gather(Cond); 490 } 491 492 ConstantComparesGatherer(const ConstantComparesGatherer &) = delete; 493 ConstantComparesGatherer & 494 operator=(const ConstantComparesGatherer &) = delete; 495 496 private: 497 /// Try to set the current value used for the comparison, it succeeds only if 498 /// it wasn't set before or if the new value is the same as the old one 499 bool setValueOnce(Value *NewVal) { 500 if (CompValue && CompValue != NewVal) 501 return false; 502 CompValue = NewVal; 503 return (CompValue != nullptr); 504 } 505 506 /// Try to match Instruction "I" as a comparison against a constant and 507 /// populates the array Vals with the set of values that match (or do not 508 /// match depending on isEQ). 509 /// Return false on failure. On success, the Value the comparison matched 510 /// against is placed in CompValue. 511 /// If CompValue is already set, the function is expected to fail if a match 512 /// is found but the value compared to is different. 513 bool matchInstruction(Instruction *I, bool isEQ) { 514 // If this is an icmp against a constant, handle this as one of the cases. 515 ICmpInst *ICI; 516 ConstantInt *C; 517 if (!((ICI = dyn_cast<ICmpInst>(I)) && 518 (C = GetConstantInt(I->getOperand(1), DL)))) { 519 return false; 520 } 521 522 Value *RHSVal; 523 const APInt *RHSC; 524 525 // Pattern match a special case 526 // (x & ~2^z) == y --> x == y || x == y|2^z 527 // This undoes a transformation done by instcombine to fuse 2 compares. 528 if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) { 529 // It's a little bit hard to see why the following transformations are 530 // correct. Here is a CVC3 program to verify them for 64-bit values: 531 532 /* 533 ONE : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63); 534 x : BITVECTOR(64); 535 y : BITVECTOR(64); 536 z : BITVECTOR(64); 537 mask : BITVECTOR(64) = BVSHL(ONE, z); 538 QUERY( (y & ~mask = y) => 539 ((x & ~mask = y) <=> (x = y OR x = (y | mask))) 540 ); 541 QUERY( (y | mask = y) => 542 ((x | mask = y) <=> (x = y OR x = (y & ~mask))) 543 ); 544 */ 545 546 // Please note that each pattern must be a dual implication (<--> or 547 // iff). One directional implication can create spurious matches. If the 548 // implication is only one-way, an unsatisfiable condition on the left 549 // side can imply a satisfiable condition on the right side. Dual 550 // implication ensures that satisfiable conditions are transformed to 551 // other satisfiable conditions and unsatisfiable conditions are 552 // transformed to other unsatisfiable conditions. 553 554 // Here is a concrete example of a unsatisfiable condition on the left 555 // implying a satisfiable condition on the right: 556 // 557 // mask = (1 << z) 558 // (x & ~mask) == y --> (x == y || x == (y | mask)) 559 // 560 // Substituting y = 3, z = 0 yields: 561 // (x & -2) == 3 --> (x == 3 || x == 2) 562 563 // Pattern match a special case: 564 /* 565 QUERY( (y & ~mask = y) => 566 ((x & ~mask = y) <=> (x = y OR x = (y | mask))) 567 ); 568 */ 569 if (match(ICI->getOperand(0), 570 m_And(m_Value(RHSVal), m_APInt(RHSC)))) { 571 APInt Mask = ~*RHSC; 572 if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) { 573 // If we already have a value for the switch, it has to match! 574 if (!setValueOnce(RHSVal)) 575 return false; 576 577 Vals.push_back(C); 578 Vals.push_back( 579 ConstantInt::get(C->getContext(), 580 C->getValue() | Mask)); 581 UsedICmps++; 582 return true; 583 } 584 } 585 586 // Pattern match a special case: 587 /* 588 QUERY( (y | mask = y) => 589 ((x | mask = y) <=> (x = y OR x = (y & ~mask))) 590 ); 591 */ 592 if (match(ICI->getOperand(0), 593 m_Or(m_Value(RHSVal), m_APInt(RHSC)))) { 594 APInt Mask = *RHSC; 595 if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) { 596 // If we already have a value for the switch, it has to match! 597 if (!setValueOnce(RHSVal)) 598 return false; 599 600 Vals.push_back(C); 601 Vals.push_back(ConstantInt::get(C->getContext(), 602 C->getValue() & ~Mask)); 603 UsedICmps++; 604 return true; 605 } 606 } 607 608 // If we already have a value for the switch, it has to match! 609 if (!setValueOnce(ICI->getOperand(0))) 610 return false; 611 612 UsedICmps++; 613 Vals.push_back(C); 614 return ICI->getOperand(0); 615 } 616 617 // If we have "x ult 3", for example, then we can add 0,1,2 to the set. 618 ConstantRange Span = 619 ConstantRange::makeExactICmpRegion(ICI->getPredicate(), C->getValue()); 620 621 // Shift the range if the compare is fed by an add. This is the range 622 // compare idiom as emitted by instcombine. 623 Value *CandidateVal = I->getOperand(0); 624 if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) { 625 Span = Span.subtract(*RHSC); 626 CandidateVal = RHSVal; 627 } 628 629 // If this is an and/!= check, then we are looking to build the set of 630 // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into 631 // x != 0 && x != 1. 632 if (!isEQ) 633 Span = Span.inverse(); 634 635 // If there are a ton of values, we don't want to make a ginormous switch. 636 if (Span.isSizeLargerThan(8) || Span.isEmptySet()) { 637 return false; 638 } 639 640 // If we already have a value for the switch, it has to match! 641 if (!setValueOnce(CandidateVal)) 642 return false; 643 644 // Add all values from the range to the set 645 for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp) 646 Vals.push_back(ConstantInt::get(I->getContext(), Tmp)); 647 648 UsedICmps++; 649 return true; 650 } 651 652 /// Given a potentially 'or'd or 'and'd together collection of icmp 653 /// eq/ne/lt/gt instructions that compare a value against a constant, extract 654 /// the value being compared, and stick the list constants into the Vals 655 /// vector. 656 /// One "Extra" case is allowed to differ from the other. 657 void gather(Value *V) { 658 bool isEQ = match(V, m_LogicalOr(m_Value(), m_Value())); 659 660 // Keep a stack (SmallVector for efficiency) for depth-first traversal 661 SmallVector<Value *, 8> DFT; 662 SmallPtrSet<Value *, 8> Visited; 663 664 // Initialize 665 Visited.insert(V); 666 DFT.push_back(V); 667 668 while (!DFT.empty()) { 669 V = DFT.pop_back_val(); 670 671 if (Instruction *I = dyn_cast<Instruction>(V)) { 672 // If it is a || (or && depending on isEQ), process the operands. 673 Value *Op0, *Op1; 674 if (isEQ ? match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1))) 675 : match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) { 676 if (Visited.insert(Op1).second) 677 DFT.push_back(Op1); 678 if (Visited.insert(Op0).second) 679 DFT.push_back(Op0); 680 681 continue; 682 } 683 684 // Try to match the current instruction 685 if (matchInstruction(I, isEQ)) 686 // Match succeed, continue the loop 687 continue; 688 } 689 690 // One element of the sequence of || (or &&) could not be match as a 691 // comparison against the same value as the others. 692 // We allow only one "Extra" case to be checked before the switch 693 if (!Extra) { 694 Extra = V; 695 continue; 696 } 697 // Failed to parse a proper sequence, abort now 698 CompValue = nullptr; 699 break; 700 } 701 } 702 }; 703 704 } // end anonymous namespace 705 706 static void EraseTerminatorAndDCECond(Instruction *TI, 707 MemorySSAUpdater *MSSAU = nullptr) { 708 Instruction *Cond = nullptr; 709 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { 710 Cond = dyn_cast<Instruction>(SI->getCondition()); 711 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { 712 if (BI->isConditional()) 713 Cond = dyn_cast<Instruction>(BI->getCondition()); 714 } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) { 715 Cond = dyn_cast<Instruction>(IBI->getAddress()); 716 } 717 718 TI->eraseFromParent(); 719 if (Cond) 720 RecursivelyDeleteTriviallyDeadInstructions(Cond, nullptr, MSSAU); 721 } 722 723 /// Return true if the specified terminator checks 724 /// to see if a value is equal to constant integer value. 725 Value *SimplifyCFGOpt::isValueEqualityComparison(Instruction *TI) { 726 Value *CV = nullptr; 727 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { 728 // Do not permit merging of large switch instructions into their 729 // predecessors unless there is only one predecessor. 730 if (!SI->getParent()->hasNPredecessorsOrMore(128 / SI->getNumSuccessors())) 731 CV = SI->getCondition(); 732 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) 733 if (BI->isConditional() && BI->getCondition()->hasOneUse()) 734 if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) { 735 if (ICI->isEquality() && GetConstantInt(ICI->getOperand(1), DL)) 736 CV = ICI->getOperand(0); 737 } 738 739 // Unwrap any lossless ptrtoint cast. 740 if (CV) { 741 if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) { 742 Value *Ptr = PTII->getPointerOperand(); 743 if (PTII->getType() == DL.getIntPtrType(Ptr->getType())) 744 CV = Ptr; 745 } 746 } 747 return CV; 748 } 749 750 /// Given a value comparison instruction, 751 /// decode all of the 'cases' that it represents and return the 'default' block. 752 BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases( 753 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases) { 754 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { 755 Cases.reserve(SI->getNumCases()); 756 for (auto Case : SI->cases()) 757 Cases.push_back(ValueEqualityComparisonCase(Case.getCaseValue(), 758 Case.getCaseSuccessor())); 759 return SI->getDefaultDest(); 760 } 761 762 BranchInst *BI = cast<BranchInst>(TI); 763 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition()); 764 BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE); 765 Cases.push_back(ValueEqualityComparisonCase( 766 GetConstantInt(ICI->getOperand(1), DL), Succ)); 767 return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ); 768 } 769 770 /// Given a vector of bb/value pairs, remove any entries 771 /// in the list that match the specified block. 772 static void 773 EliminateBlockCases(BasicBlock *BB, 774 std::vector<ValueEqualityComparisonCase> &Cases) { 775 llvm::erase_value(Cases, BB); 776 } 777 778 /// Return true if there are any keys in C1 that exist in C2 as well. 779 static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1, 780 std::vector<ValueEqualityComparisonCase> &C2) { 781 std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2; 782 783 // Make V1 be smaller than V2. 784 if (V1->size() > V2->size()) 785 std::swap(V1, V2); 786 787 if (V1->empty()) 788 return false; 789 if (V1->size() == 1) { 790 // Just scan V2. 791 ConstantInt *TheVal = (*V1)[0].Value; 792 for (unsigned i = 0, e = V2->size(); i != e; ++i) 793 if (TheVal == (*V2)[i].Value) 794 return true; 795 } 796 797 // Otherwise, just sort both lists and compare element by element. 798 array_pod_sort(V1->begin(), V1->end()); 799 array_pod_sort(V2->begin(), V2->end()); 800 unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size(); 801 while (i1 != e1 && i2 != e2) { 802 if ((*V1)[i1].Value == (*V2)[i2].Value) 803 return true; 804 if ((*V1)[i1].Value < (*V2)[i2].Value) 805 ++i1; 806 else 807 ++i2; 808 } 809 return false; 810 } 811 812 // Set branch weights on SwitchInst. This sets the metadata if there is at 813 // least one non-zero weight. 814 static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights) { 815 // Check that there is at least one non-zero weight. Otherwise, pass 816 // nullptr to setMetadata which will erase the existing metadata. 817 MDNode *N = nullptr; 818 if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; })) 819 N = MDBuilder(SI->getParent()->getContext()).createBranchWeights(Weights); 820 SI->setMetadata(LLVMContext::MD_prof, N); 821 } 822 823 // Similar to the above, but for branch and select instructions that take 824 // exactly 2 weights. 825 static void setBranchWeights(Instruction *I, uint32_t TrueWeight, 826 uint32_t FalseWeight) { 827 assert(isa<BranchInst>(I) || isa<SelectInst>(I)); 828 // Check that there is at least one non-zero weight. Otherwise, pass 829 // nullptr to setMetadata which will erase the existing metadata. 830 MDNode *N = nullptr; 831 if (TrueWeight || FalseWeight) 832 N = MDBuilder(I->getParent()->getContext()) 833 .createBranchWeights(TrueWeight, FalseWeight); 834 I->setMetadata(LLVMContext::MD_prof, N); 835 } 836 837 /// If TI is known to be a terminator instruction and its block is known to 838 /// only have a single predecessor block, check to see if that predecessor is 839 /// also a value comparison with the same value, and if that comparison 840 /// determines the outcome of this comparison. If so, simplify TI. This does a 841 /// very limited form of jump threading. 842 bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor( 843 Instruction *TI, BasicBlock *Pred, IRBuilder<> &Builder) { 844 Value *PredVal = isValueEqualityComparison(Pred->getTerminator()); 845 if (!PredVal) 846 return false; // Not a value comparison in predecessor. 847 848 Value *ThisVal = isValueEqualityComparison(TI); 849 assert(ThisVal && "This isn't a value comparison!!"); 850 if (ThisVal != PredVal) 851 return false; // Different predicates. 852 853 // TODO: Preserve branch weight metadata, similarly to how 854 // FoldValueComparisonIntoPredecessors preserves it. 855 856 // Find out information about when control will move from Pred to TI's block. 857 std::vector<ValueEqualityComparisonCase> PredCases; 858 BasicBlock *PredDef = 859 GetValueEqualityComparisonCases(Pred->getTerminator(), PredCases); 860 EliminateBlockCases(PredDef, PredCases); // Remove default from cases. 861 862 // Find information about how control leaves this block. 863 std::vector<ValueEqualityComparisonCase> ThisCases; 864 BasicBlock *ThisDef = GetValueEqualityComparisonCases(TI, ThisCases); 865 EliminateBlockCases(ThisDef, ThisCases); // Remove default from cases. 866 867 // If TI's block is the default block from Pred's comparison, potentially 868 // simplify TI based on this knowledge. 869 if (PredDef == TI->getParent()) { 870 // If we are here, we know that the value is none of those cases listed in 871 // PredCases. If there are any cases in ThisCases that are in PredCases, we 872 // can simplify TI. 873 if (!ValuesOverlap(PredCases, ThisCases)) 874 return false; 875 876 if (isa<BranchInst>(TI)) { 877 // Okay, one of the successors of this condbr is dead. Convert it to a 878 // uncond br. 879 assert(ThisCases.size() == 1 && "Branch can only have one case!"); 880 // Insert the new branch. 881 Instruction *NI = Builder.CreateBr(ThisDef); 882 (void)NI; 883 884 // Remove PHI node entries for the dead edge. 885 ThisCases[0].Dest->removePredecessor(PredDef); 886 887 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator() 888 << "Through successor TI: " << *TI << "Leaving: " << *NI 889 << "\n"); 890 891 EraseTerminatorAndDCECond(TI); 892 893 if (DTU) 894 DTU->applyUpdates( 895 {{DominatorTree::Delete, PredDef, ThisCases[0].Dest}}); 896 897 return true; 898 } 899 900 SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI); 901 // Okay, TI has cases that are statically dead, prune them away. 902 SmallPtrSet<Constant *, 16> DeadCases; 903 for (unsigned i = 0, e = PredCases.size(); i != e; ++i) 904 DeadCases.insert(PredCases[i].Value); 905 906 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator() 907 << "Through successor TI: " << *TI); 908 909 SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases; 910 for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) { 911 --i; 912 auto *Successor = i->getCaseSuccessor(); 913 if (DTU) 914 ++NumPerSuccessorCases[Successor]; 915 if (DeadCases.count(i->getCaseValue())) { 916 Successor->removePredecessor(PredDef); 917 SI.removeCase(i); 918 if (DTU) 919 --NumPerSuccessorCases[Successor]; 920 } 921 } 922 923 if (DTU) { 924 std::vector<DominatorTree::UpdateType> Updates; 925 for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases) 926 if (I.second == 0) 927 Updates.push_back({DominatorTree::Delete, PredDef, I.first}); 928 DTU->applyUpdates(Updates); 929 } 930 931 LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n"); 932 return true; 933 } 934 935 // Otherwise, TI's block must correspond to some matched value. Find out 936 // which value (or set of values) this is. 937 ConstantInt *TIV = nullptr; 938 BasicBlock *TIBB = TI->getParent(); 939 for (unsigned i = 0, e = PredCases.size(); i != e; ++i) 940 if (PredCases[i].Dest == TIBB) { 941 if (TIV) 942 return false; // Cannot handle multiple values coming to this block. 943 TIV = PredCases[i].Value; 944 } 945 assert(TIV && "No edge from pred to succ?"); 946 947 // Okay, we found the one constant that our value can be if we get into TI's 948 // BB. Find out which successor will unconditionally be branched to. 949 BasicBlock *TheRealDest = nullptr; 950 for (unsigned i = 0, e = ThisCases.size(); i != e; ++i) 951 if (ThisCases[i].Value == TIV) { 952 TheRealDest = ThisCases[i].Dest; 953 break; 954 } 955 956 // If not handled by any explicit cases, it is handled by the default case. 957 if (!TheRealDest) 958 TheRealDest = ThisDef; 959 960 SmallPtrSet<BasicBlock *, 2> RemovedSuccs; 961 962 // Remove PHI node entries for dead edges. 963 BasicBlock *CheckEdge = TheRealDest; 964 for (BasicBlock *Succ : successors(TIBB)) 965 if (Succ != CheckEdge) { 966 if (Succ != TheRealDest) 967 RemovedSuccs.insert(Succ); 968 Succ->removePredecessor(TIBB); 969 } else 970 CheckEdge = nullptr; 971 972 // Insert the new branch. 973 Instruction *NI = Builder.CreateBr(TheRealDest); 974 (void)NI; 975 976 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator() 977 << "Through successor TI: " << *TI << "Leaving: " << *NI 978 << "\n"); 979 980 EraseTerminatorAndDCECond(TI); 981 if (DTU) { 982 SmallVector<DominatorTree::UpdateType, 2> Updates; 983 Updates.reserve(RemovedSuccs.size()); 984 for (auto *RemovedSucc : RemovedSuccs) 985 Updates.push_back({DominatorTree::Delete, TIBB, RemovedSucc}); 986 DTU->applyUpdates(Updates); 987 } 988 return true; 989 } 990 991 namespace { 992 993 /// This class implements a stable ordering of constant 994 /// integers that does not depend on their address. This is important for 995 /// applications that sort ConstantInt's to ensure uniqueness. 996 struct ConstantIntOrdering { 997 bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const { 998 return LHS->getValue().ult(RHS->getValue()); 999 } 1000 }; 1001 1002 } // end anonymous namespace 1003 1004 static int ConstantIntSortPredicate(ConstantInt *const *P1, 1005 ConstantInt *const *P2) { 1006 const ConstantInt *LHS = *P1; 1007 const ConstantInt *RHS = *P2; 1008 if (LHS == RHS) 1009 return 0; 1010 return LHS->getValue().ult(RHS->getValue()) ? 1 : -1; 1011 } 1012 1013 static inline bool HasBranchWeights(const Instruction *I) { 1014 MDNode *ProfMD = I->getMetadata(LLVMContext::MD_prof); 1015 if (ProfMD && ProfMD->getOperand(0)) 1016 if (MDString *MDS = dyn_cast<MDString>(ProfMD->getOperand(0))) 1017 return MDS->getString().equals("branch_weights"); 1018 1019 return false; 1020 } 1021 1022 /// Get Weights of a given terminator, the default weight is at the front 1023 /// of the vector. If TI is a conditional eq, we need to swap the branch-weight 1024 /// metadata. 1025 static void GetBranchWeights(Instruction *TI, 1026 SmallVectorImpl<uint64_t> &Weights) { 1027 MDNode *MD = TI->getMetadata(LLVMContext::MD_prof); 1028 assert(MD); 1029 for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) { 1030 ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(i)); 1031 Weights.push_back(CI->getValue().getZExtValue()); 1032 } 1033 1034 // If TI is a conditional eq, the default case is the false case, 1035 // and the corresponding branch-weight data is at index 2. We swap the 1036 // default weight to be the first entry. 1037 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { 1038 assert(Weights.size() == 2); 1039 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition()); 1040 if (ICI->getPredicate() == ICmpInst::ICMP_EQ) 1041 std::swap(Weights.front(), Weights.back()); 1042 } 1043 } 1044 1045 /// Keep halving the weights until all can fit in uint32_t. 1046 static void FitWeights(MutableArrayRef<uint64_t> Weights) { 1047 uint64_t Max = *std::max_element(Weights.begin(), Weights.end()); 1048 if (Max > UINT_MAX) { 1049 unsigned Offset = 32 - countLeadingZeros(Max); 1050 for (uint64_t &I : Weights) 1051 I >>= Offset; 1052 } 1053 } 1054 1055 static void CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses( 1056 BasicBlock *BB, BasicBlock *PredBlock, ValueToValueMapTy &VMap) { 1057 Instruction *PTI = PredBlock->getTerminator(); 1058 1059 // If we have bonus instructions, clone them into the predecessor block. 1060 // Note that there may be multiple predecessor blocks, so we cannot move 1061 // bonus instructions to a predecessor block. 1062 for (Instruction &BonusInst : *BB) { 1063 if (isa<DbgInfoIntrinsic>(BonusInst) || BonusInst.isTerminator()) 1064 continue; 1065 1066 Instruction *NewBonusInst = BonusInst.clone(); 1067 1068 if (PTI->getDebugLoc() != NewBonusInst->getDebugLoc()) { 1069 // Unless the instruction has the same !dbg location as the original 1070 // branch, drop it. When we fold the bonus instructions we want to make 1071 // sure we reset their debug locations in order to avoid stepping on 1072 // dead code caused by folding dead branches. 1073 NewBonusInst->setDebugLoc(DebugLoc()); 1074 } 1075 1076 RemapInstruction(NewBonusInst, VMap, 1077 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals); 1078 VMap[&BonusInst] = NewBonusInst; 1079 1080 // If we moved a load, we cannot any longer claim any knowledge about 1081 // its potential value. The previous information might have been valid 1082 // only given the branch precondition. 1083 // For an analogous reason, we must also drop all the metadata whose 1084 // semantics we don't understand. We *can* preserve !annotation, because 1085 // it is tied to the instruction itself, not the value or position. 1086 // Similarly strip attributes on call parameters that may cause UB in 1087 // location the call is moved to. 1088 NewBonusInst->dropUndefImplyingAttrsAndUnknownMetadata( 1089 LLVMContext::MD_annotation); 1090 1091 PredBlock->getInstList().insert(PTI->getIterator(), NewBonusInst); 1092 NewBonusInst->takeName(&BonusInst); 1093 BonusInst.setName(NewBonusInst->getName() + ".old"); 1094 1095 // Update (liveout) uses of bonus instructions, 1096 // now that the bonus instruction has been cloned into predecessor. 1097 // Note that we expect to be in a block-closed SSA form for this to work! 1098 for (Use &U : make_early_inc_range(BonusInst.uses())) { 1099 auto *UI = cast<Instruction>(U.getUser()); 1100 auto *PN = dyn_cast<PHINode>(UI); 1101 if (!PN) { 1102 assert(UI->getParent() == BB && BonusInst.comesBefore(UI) && 1103 "If the user is not a PHI node, then it should be in the same " 1104 "block as, and come after, the original bonus instruction."); 1105 continue; // Keep using the original bonus instruction. 1106 } 1107 // Is this the block-closed SSA form PHI node? 1108 if (PN->getIncomingBlock(U) == BB) 1109 continue; // Great, keep using the original bonus instruction. 1110 // The only other alternative is an "use" when coming from 1111 // the predecessor block - here we should refer to the cloned bonus instr. 1112 assert(PN->getIncomingBlock(U) == PredBlock && 1113 "Not in block-closed SSA form?"); 1114 U.set(NewBonusInst); 1115 } 1116 } 1117 } 1118 1119 bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding( 1120 Instruction *TI, Value *&CV, Instruction *PTI, IRBuilder<> &Builder) { 1121 BasicBlock *BB = TI->getParent(); 1122 BasicBlock *Pred = PTI->getParent(); 1123 1124 SmallVector<DominatorTree::UpdateType, 32> Updates; 1125 1126 // Figure out which 'cases' to copy from SI to PSI. 1127 std::vector<ValueEqualityComparisonCase> BBCases; 1128 BasicBlock *BBDefault = GetValueEqualityComparisonCases(TI, BBCases); 1129 1130 std::vector<ValueEqualityComparisonCase> PredCases; 1131 BasicBlock *PredDefault = GetValueEqualityComparisonCases(PTI, PredCases); 1132 1133 // Based on whether the default edge from PTI goes to BB or not, fill in 1134 // PredCases and PredDefault with the new switch cases we would like to 1135 // build. 1136 SmallMapVector<BasicBlock *, int, 8> NewSuccessors; 1137 1138 // Update the branch weight metadata along the way 1139 SmallVector<uint64_t, 8> Weights; 1140 bool PredHasWeights = HasBranchWeights(PTI); 1141 bool SuccHasWeights = HasBranchWeights(TI); 1142 1143 if (PredHasWeights) { 1144 GetBranchWeights(PTI, Weights); 1145 // branch-weight metadata is inconsistent here. 1146 if (Weights.size() != 1 + PredCases.size()) 1147 PredHasWeights = SuccHasWeights = false; 1148 } else if (SuccHasWeights) 1149 // If there are no predecessor weights but there are successor weights, 1150 // populate Weights with 1, which will later be scaled to the sum of 1151 // successor's weights 1152 Weights.assign(1 + PredCases.size(), 1); 1153 1154 SmallVector<uint64_t, 8> SuccWeights; 1155 if (SuccHasWeights) { 1156 GetBranchWeights(TI, SuccWeights); 1157 // branch-weight metadata is inconsistent here. 1158 if (SuccWeights.size() != 1 + BBCases.size()) 1159 PredHasWeights = SuccHasWeights = false; 1160 } else if (PredHasWeights) 1161 SuccWeights.assign(1 + BBCases.size(), 1); 1162 1163 if (PredDefault == BB) { 1164 // If this is the default destination from PTI, only the edges in TI 1165 // that don't occur in PTI, or that branch to BB will be activated. 1166 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled; 1167 for (unsigned i = 0, e = PredCases.size(); i != e; ++i) 1168 if (PredCases[i].Dest != BB) 1169 PTIHandled.insert(PredCases[i].Value); 1170 else { 1171 // The default destination is BB, we don't need explicit targets. 1172 std::swap(PredCases[i], PredCases.back()); 1173 1174 if (PredHasWeights || SuccHasWeights) { 1175 // Increase weight for the default case. 1176 Weights[0] += Weights[i + 1]; 1177 std::swap(Weights[i + 1], Weights.back()); 1178 Weights.pop_back(); 1179 } 1180 1181 PredCases.pop_back(); 1182 --i; 1183 --e; 1184 } 1185 1186 // Reconstruct the new switch statement we will be building. 1187 if (PredDefault != BBDefault) { 1188 PredDefault->removePredecessor(Pred); 1189 if (DTU && PredDefault != BB) 1190 Updates.push_back({DominatorTree::Delete, Pred, PredDefault}); 1191 PredDefault = BBDefault; 1192 ++NewSuccessors[BBDefault]; 1193 } 1194 1195 unsigned CasesFromPred = Weights.size(); 1196 uint64_t ValidTotalSuccWeight = 0; 1197 for (unsigned i = 0, e = BBCases.size(); i != e; ++i) 1198 if (!PTIHandled.count(BBCases[i].Value) && BBCases[i].Dest != BBDefault) { 1199 PredCases.push_back(BBCases[i]); 1200 ++NewSuccessors[BBCases[i].Dest]; 1201 if (SuccHasWeights || PredHasWeights) { 1202 // The default weight is at index 0, so weight for the ith case 1203 // should be at index i+1. Scale the cases from successor by 1204 // PredDefaultWeight (Weights[0]). 1205 Weights.push_back(Weights[0] * SuccWeights[i + 1]); 1206 ValidTotalSuccWeight += SuccWeights[i + 1]; 1207 } 1208 } 1209 1210 if (SuccHasWeights || PredHasWeights) { 1211 ValidTotalSuccWeight += SuccWeights[0]; 1212 // Scale the cases from predecessor by ValidTotalSuccWeight. 1213 for (unsigned i = 1; i < CasesFromPred; ++i) 1214 Weights[i] *= ValidTotalSuccWeight; 1215 // Scale the default weight by SuccDefaultWeight (SuccWeights[0]). 1216 Weights[0] *= SuccWeights[0]; 1217 } 1218 } else { 1219 // If this is not the default destination from PSI, only the edges 1220 // in SI that occur in PSI with a destination of BB will be 1221 // activated. 1222 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled; 1223 std::map<ConstantInt *, uint64_t> WeightsForHandled; 1224 for (unsigned i = 0, e = PredCases.size(); i != e; ++i) 1225 if (PredCases[i].Dest == BB) { 1226 PTIHandled.insert(PredCases[i].Value); 1227 1228 if (PredHasWeights || SuccHasWeights) { 1229 WeightsForHandled[PredCases[i].Value] = Weights[i + 1]; 1230 std::swap(Weights[i + 1], Weights.back()); 1231 Weights.pop_back(); 1232 } 1233 1234 std::swap(PredCases[i], PredCases.back()); 1235 PredCases.pop_back(); 1236 --i; 1237 --e; 1238 } 1239 1240 // Okay, now we know which constants were sent to BB from the 1241 // predecessor. Figure out where they will all go now. 1242 for (unsigned i = 0, e = BBCases.size(); i != e; ++i) 1243 if (PTIHandled.count(BBCases[i].Value)) { 1244 // If this is one we are capable of getting... 1245 if (PredHasWeights || SuccHasWeights) 1246 Weights.push_back(WeightsForHandled[BBCases[i].Value]); 1247 PredCases.push_back(BBCases[i]); 1248 ++NewSuccessors[BBCases[i].Dest]; 1249 PTIHandled.erase(BBCases[i].Value); // This constant is taken care of 1250 } 1251 1252 // If there are any constants vectored to BB that TI doesn't handle, 1253 // they must go to the default destination of TI. 1254 for (ConstantInt *I : PTIHandled) { 1255 if (PredHasWeights || SuccHasWeights) 1256 Weights.push_back(WeightsForHandled[I]); 1257 PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault)); 1258 ++NewSuccessors[BBDefault]; 1259 } 1260 } 1261 1262 // Okay, at this point, we know which new successor Pred will get. Make 1263 // sure we update the number of entries in the PHI nodes for these 1264 // successors. 1265 SmallPtrSet<BasicBlock *, 2> SuccsOfPred; 1266 if (DTU) { 1267 SuccsOfPred = {succ_begin(Pred), succ_end(Pred)}; 1268 Updates.reserve(Updates.size() + NewSuccessors.size()); 1269 } 1270 for (const std::pair<BasicBlock *, int /*Num*/> &NewSuccessor : 1271 NewSuccessors) { 1272 for (auto I : seq(0, NewSuccessor.second)) { 1273 (void)I; 1274 AddPredecessorToBlock(NewSuccessor.first, Pred, BB); 1275 } 1276 if (DTU && !SuccsOfPred.contains(NewSuccessor.first)) 1277 Updates.push_back({DominatorTree::Insert, Pred, NewSuccessor.first}); 1278 } 1279 1280 Builder.SetInsertPoint(PTI); 1281 // Convert pointer to int before we switch. 1282 if (CV->getType()->isPointerTy()) { 1283 CV = 1284 Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()), "magicptr"); 1285 } 1286 1287 // Now that the successors are updated, create the new Switch instruction. 1288 SwitchInst *NewSI = Builder.CreateSwitch(CV, PredDefault, PredCases.size()); 1289 NewSI->setDebugLoc(PTI->getDebugLoc()); 1290 for (ValueEqualityComparisonCase &V : PredCases) 1291 NewSI->addCase(V.Value, V.Dest); 1292 1293 if (PredHasWeights || SuccHasWeights) { 1294 // Halve the weights if any of them cannot fit in an uint32_t 1295 FitWeights(Weights); 1296 1297 SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end()); 1298 1299 setBranchWeights(NewSI, MDWeights); 1300 } 1301 1302 EraseTerminatorAndDCECond(PTI); 1303 1304 // Okay, last check. If BB is still a successor of PSI, then we must 1305 // have an infinite loop case. If so, add an infinitely looping block 1306 // to handle the case to preserve the behavior of the code. 1307 BasicBlock *InfLoopBlock = nullptr; 1308 for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i) 1309 if (NewSI->getSuccessor(i) == BB) { 1310 if (!InfLoopBlock) { 1311 // Insert it at the end of the function, because it's either code, 1312 // or it won't matter if it's hot. :) 1313 InfLoopBlock = 1314 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent()); 1315 BranchInst::Create(InfLoopBlock, InfLoopBlock); 1316 if (DTU) 1317 Updates.push_back( 1318 {DominatorTree::Insert, InfLoopBlock, InfLoopBlock}); 1319 } 1320 NewSI->setSuccessor(i, InfLoopBlock); 1321 } 1322 1323 if (DTU) { 1324 if (InfLoopBlock) 1325 Updates.push_back({DominatorTree::Insert, Pred, InfLoopBlock}); 1326 1327 Updates.push_back({DominatorTree::Delete, Pred, BB}); 1328 1329 DTU->applyUpdates(Updates); 1330 } 1331 1332 ++NumFoldValueComparisonIntoPredecessors; 1333 return true; 1334 } 1335 1336 /// The specified terminator is a value equality comparison instruction 1337 /// (either a switch or a branch on "X == c"). 1338 /// See if any of the predecessors of the terminator block are value comparisons 1339 /// on the same value. If so, and if safe to do so, fold them together. 1340 bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(Instruction *TI, 1341 IRBuilder<> &Builder) { 1342 BasicBlock *BB = TI->getParent(); 1343 Value *CV = isValueEqualityComparison(TI); // CondVal 1344 assert(CV && "Not a comparison?"); 1345 1346 bool Changed = false; 1347 1348 SmallSetVector<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB)); 1349 while (!Preds.empty()) { 1350 BasicBlock *Pred = Preds.pop_back_val(); 1351 Instruction *PTI = Pred->getTerminator(); 1352 1353 // Don't try to fold into itself. 1354 if (Pred == BB) 1355 continue; 1356 1357 // See if the predecessor is a comparison with the same value. 1358 Value *PCV = isValueEqualityComparison(PTI); // PredCondVal 1359 if (PCV != CV) 1360 continue; 1361 1362 SmallSetVector<BasicBlock *, 4> FailBlocks; 1363 if (!SafeToMergeTerminators(TI, PTI, &FailBlocks)) { 1364 for (auto *Succ : FailBlocks) { 1365 if (!SplitBlockPredecessors(Succ, TI->getParent(), ".fold.split", DTU)) 1366 return false; 1367 } 1368 } 1369 1370 PerformValueComparisonIntoPredecessorFolding(TI, CV, PTI, Builder); 1371 Changed = true; 1372 } 1373 return Changed; 1374 } 1375 1376 // If we would need to insert a select that uses the value of this invoke 1377 // (comments in HoistThenElseCodeToIf explain why we would need to do this), we 1378 // can't hoist the invoke, as there is nowhere to put the select in this case. 1379 static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2, 1380 Instruction *I1, Instruction *I2) { 1381 for (BasicBlock *Succ : successors(BB1)) { 1382 for (const PHINode &PN : Succ->phis()) { 1383 Value *BB1V = PN.getIncomingValueForBlock(BB1); 1384 Value *BB2V = PN.getIncomingValueForBlock(BB2); 1385 if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) { 1386 return false; 1387 } 1388 } 1389 } 1390 return true; 1391 } 1392 1393 static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified = false); 1394 1395 /// Given a conditional branch that goes to BB1 and BB2, hoist any common code 1396 /// in the two blocks up into the branch block. The caller of this function 1397 /// guarantees that BI's block dominates BB1 and BB2. If EqTermsOnly is given, 1398 /// only perform hoisting in case both blocks only contain a terminator. In that 1399 /// case, only the original BI will be replaced and selects for PHIs are added. 1400 bool SimplifyCFGOpt::HoistThenElseCodeToIf(BranchInst *BI, 1401 const TargetTransformInfo &TTI, 1402 bool EqTermsOnly) { 1403 // This does very trivial matching, with limited scanning, to find identical 1404 // instructions in the two blocks. In particular, we don't want to get into 1405 // O(M*N) situations here where M and N are the sizes of BB1 and BB2. As 1406 // such, we currently just scan for obviously identical instructions in an 1407 // identical order. 1408 BasicBlock *BB1 = BI->getSuccessor(0); // The true destination. 1409 BasicBlock *BB2 = BI->getSuccessor(1); // The false destination 1410 1411 // If either of the blocks has it's address taken, then we can't do this fold, 1412 // because the code we'd hoist would no longer run when we jump into the block 1413 // by it's address. 1414 if (BB1->hasAddressTaken() || BB2->hasAddressTaken()) 1415 return false; 1416 1417 BasicBlock::iterator BB1_Itr = BB1->begin(); 1418 BasicBlock::iterator BB2_Itr = BB2->begin(); 1419 1420 Instruction *I1 = &*BB1_Itr++, *I2 = &*BB2_Itr++; 1421 // Skip debug info if it is not identical. 1422 DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1); 1423 DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2); 1424 if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) { 1425 while (isa<DbgInfoIntrinsic>(I1)) 1426 I1 = &*BB1_Itr++; 1427 while (isa<DbgInfoIntrinsic>(I2)) 1428 I2 = &*BB2_Itr++; 1429 } 1430 // FIXME: Can we define a safety predicate for CallBr? 1431 if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) || 1432 (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) || 1433 isa<CallBrInst>(I1)) 1434 return false; 1435 1436 BasicBlock *BIParent = BI->getParent(); 1437 1438 bool Changed = false; 1439 1440 auto _ = make_scope_exit([&]() { 1441 if (Changed) 1442 ++NumHoistCommonCode; 1443 }); 1444 1445 // Check if only hoisting terminators is allowed. This does not add new 1446 // instructions to the hoist location. 1447 if (EqTermsOnly) { 1448 // Skip any debug intrinsics, as they are free to hoist. 1449 auto *I1NonDbg = &*skipDebugIntrinsics(I1->getIterator()); 1450 auto *I2NonDbg = &*skipDebugIntrinsics(I2->getIterator()); 1451 if (!I1NonDbg->isIdenticalToWhenDefined(I2NonDbg)) 1452 return false; 1453 if (!I1NonDbg->isTerminator()) 1454 return false; 1455 // Now we know that we only need to hoist debug instrinsics and the 1456 // terminator. Let the loop below handle those 2 cases. 1457 } 1458 1459 do { 1460 // If we are hoisting the terminator instruction, don't move one (making a 1461 // broken BB), instead clone it, and remove BI. 1462 if (I1->isTerminator()) 1463 goto HoistTerminator; 1464 1465 // If we're going to hoist a call, make sure that the two instructions we're 1466 // commoning/hoisting are both marked with musttail, or neither of them is 1467 // marked as such. Otherwise, we might end up in a situation where we hoist 1468 // from a block where the terminator is a `ret` to a block where the terminator 1469 // is a `br`, and `musttail` calls expect to be followed by a return. 1470 auto *C1 = dyn_cast<CallInst>(I1); 1471 auto *C2 = dyn_cast<CallInst>(I2); 1472 if (C1 && C2) 1473 if (C1->isMustTailCall() != C2->isMustTailCall()) 1474 return Changed; 1475 1476 if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2)) 1477 return Changed; 1478 1479 // If any of the two call sites has nomerge attribute, stop hoisting. 1480 if (const auto *CB1 = dyn_cast<CallBase>(I1)) 1481 if (CB1->cannotMerge()) 1482 return Changed; 1483 if (const auto *CB2 = dyn_cast<CallBase>(I2)) 1484 if (CB2->cannotMerge()) 1485 return Changed; 1486 1487 if (isa<DbgInfoIntrinsic>(I1) || isa<DbgInfoIntrinsic>(I2)) { 1488 assert (isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic>(I2)); 1489 // The debug location is an integral part of a debug info intrinsic 1490 // and can't be separated from it or replaced. Instead of attempting 1491 // to merge locations, simply hoist both copies of the intrinsic. 1492 BIParent->getInstList().splice(BI->getIterator(), 1493 BB1->getInstList(), I1); 1494 BIParent->getInstList().splice(BI->getIterator(), 1495 BB2->getInstList(), I2); 1496 Changed = true; 1497 } else { 1498 // For a normal instruction, we just move one to right before the branch, 1499 // then replace all uses of the other with the first. Finally, we remove 1500 // the now redundant second instruction. 1501 BIParent->getInstList().splice(BI->getIterator(), 1502 BB1->getInstList(), I1); 1503 if (!I2->use_empty()) 1504 I2->replaceAllUsesWith(I1); 1505 I1->andIRFlags(I2); 1506 unsigned KnownIDs[] = {LLVMContext::MD_tbaa, 1507 LLVMContext::MD_range, 1508 LLVMContext::MD_fpmath, 1509 LLVMContext::MD_invariant_load, 1510 LLVMContext::MD_nonnull, 1511 LLVMContext::MD_invariant_group, 1512 LLVMContext::MD_align, 1513 LLVMContext::MD_dereferenceable, 1514 LLVMContext::MD_dereferenceable_or_null, 1515 LLVMContext::MD_mem_parallel_loop_access, 1516 LLVMContext::MD_access_group, 1517 LLVMContext::MD_preserve_access_index}; 1518 combineMetadata(I1, I2, KnownIDs, true); 1519 1520 // I1 and I2 are being combined into a single instruction. Its debug 1521 // location is the merged locations of the original instructions. 1522 I1->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc()); 1523 1524 I2->eraseFromParent(); 1525 Changed = true; 1526 } 1527 ++NumHoistCommonInstrs; 1528 1529 I1 = &*BB1_Itr++; 1530 I2 = &*BB2_Itr++; 1531 // Skip debug info if it is not identical. 1532 DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1); 1533 DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2); 1534 if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) { 1535 while (isa<DbgInfoIntrinsic>(I1)) 1536 I1 = &*BB1_Itr++; 1537 while (isa<DbgInfoIntrinsic>(I2)) 1538 I2 = &*BB2_Itr++; 1539 } 1540 } while (I1->isIdenticalToWhenDefined(I2)); 1541 1542 return true; 1543 1544 HoistTerminator: 1545 // It may not be possible to hoist an invoke. 1546 // FIXME: Can we define a safety predicate for CallBr? 1547 if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) 1548 return Changed; 1549 1550 // TODO: callbr hoisting currently disabled pending further study. 1551 if (isa<CallBrInst>(I1)) 1552 return Changed; 1553 1554 for (BasicBlock *Succ : successors(BB1)) { 1555 for (PHINode &PN : Succ->phis()) { 1556 Value *BB1V = PN.getIncomingValueForBlock(BB1); 1557 Value *BB2V = PN.getIncomingValueForBlock(BB2); 1558 if (BB1V == BB2V) 1559 continue; 1560 1561 // Check for passingValueIsAlwaysUndefined here because we would rather 1562 // eliminate undefined control flow then converting it to a select. 1563 if (passingValueIsAlwaysUndefined(BB1V, &PN) || 1564 passingValueIsAlwaysUndefined(BB2V, &PN)) 1565 return Changed; 1566 1567 if (isa<ConstantExpr>(BB1V) && !isSafeToSpeculativelyExecute(BB1V)) 1568 return Changed; 1569 if (isa<ConstantExpr>(BB2V) && !isSafeToSpeculativelyExecute(BB2V)) 1570 return Changed; 1571 } 1572 } 1573 1574 // Okay, it is safe to hoist the terminator. 1575 Instruction *NT = I1->clone(); 1576 BIParent->getInstList().insert(BI->getIterator(), NT); 1577 if (!NT->getType()->isVoidTy()) { 1578 I1->replaceAllUsesWith(NT); 1579 I2->replaceAllUsesWith(NT); 1580 NT->takeName(I1); 1581 } 1582 Changed = true; 1583 ++NumHoistCommonInstrs; 1584 1585 // Ensure terminator gets a debug location, even an unknown one, in case 1586 // it involves inlinable calls. 1587 NT->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc()); 1588 1589 // PHIs created below will adopt NT's merged DebugLoc. 1590 IRBuilder<NoFolder> Builder(NT); 1591 1592 // Hoisting one of the terminators from our successor is a great thing. 1593 // Unfortunately, the successors of the if/else blocks may have PHI nodes in 1594 // them. If they do, all PHI entries for BB1/BB2 must agree for all PHI 1595 // nodes, so we insert select instruction to compute the final result. 1596 std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects; 1597 for (BasicBlock *Succ : successors(BB1)) { 1598 for (PHINode &PN : Succ->phis()) { 1599 Value *BB1V = PN.getIncomingValueForBlock(BB1); 1600 Value *BB2V = PN.getIncomingValueForBlock(BB2); 1601 if (BB1V == BB2V) 1602 continue; 1603 1604 // These values do not agree. Insert a select instruction before NT 1605 // that determines the right value. 1606 SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)]; 1607 if (!SI) { 1608 // Propagate fast-math-flags from phi node to its replacement select. 1609 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); 1610 if (isa<FPMathOperator>(PN)) 1611 Builder.setFastMathFlags(PN.getFastMathFlags()); 1612 1613 SI = cast<SelectInst>( 1614 Builder.CreateSelect(BI->getCondition(), BB1V, BB2V, 1615 BB1V->getName() + "." + BB2V->getName(), BI)); 1616 } 1617 1618 // Make the PHI node use the select for all incoming values for BB1/BB2 1619 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) 1620 if (PN.getIncomingBlock(i) == BB1 || PN.getIncomingBlock(i) == BB2) 1621 PN.setIncomingValue(i, SI); 1622 } 1623 } 1624 1625 SmallVector<DominatorTree::UpdateType, 4> Updates; 1626 1627 // Update any PHI nodes in our new successors. 1628 for (BasicBlock *Succ : successors(BB1)) { 1629 AddPredecessorToBlock(Succ, BIParent, BB1); 1630 if (DTU) 1631 Updates.push_back({DominatorTree::Insert, BIParent, Succ}); 1632 } 1633 1634 if (DTU) 1635 for (BasicBlock *Succ : successors(BI)) 1636 Updates.push_back({DominatorTree::Delete, BIParent, Succ}); 1637 1638 EraseTerminatorAndDCECond(BI); 1639 if (DTU) 1640 DTU->applyUpdates(Updates); 1641 return Changed; 1642 } 1643 1644 // Check lifetime markers. 1645 static bool isLifeTimeMarker(const Instruction *I) { 1646 if (auto II = dyn_cast<IntrinsicInst>(I)) { 1647 switch (II->getIntrinsicID()) { 1648 default: 1649 break; 1650 case Intrinsic::lifetime_start: 1651 case Intrinsic::lifetime_end: 1652 return true; 1653 } 1654 } 1655 return false; 1656 } 1657 1658 // TODO: Refine this. This should avoid cases like turning constant memcpy sizes 1659 // into variables. 1660 static bool replacingOperandWithVariableIsCheap(const Instruction *I, 1661 int OpIdx) { 1662 return !isa<IntrinsicInst>(I); 1663 } 1664 1665 // All instructions in Insts belong to different blocks that all unconditionally 1666 // branch to a common successor. Analyze each instruction and return true if it 1667 // would be possible to sink them into their successor, creating one common 1668 // instruction instead. For every value that would be required to be provided by 1669 // PHI node (because an operand varies in each input block), add to PHIOperands. 1670 static bool canSinkInstructions( 1671 ArrayRef<Instruction *> Insts, 1672 DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) { 1673 // Prune out obviously bad instructions to move. Any non-store instruction 1674 // must have exactly one use, and we check later that use is by a single, 1675 // common PHI instruction in the successor. 1676 for (auto *I : Insts) { 1677 // These instructions may change or break semantics if moved. 1678 if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) || 1679 I->getType()->isTokenTy()) 1680 return false; 1681 1682 // Do not try to sink an instruction in an infinite loop - it can cause 1683 // this algorithm to infinite loop. 1684 if (I->getParent()->getSingleSuccessor() == I->getParent()) 1685 return false; 1686 1687 // Conservatively return false if I is an inline-asm instruction. Sinking 1688 // and merging inline-asm instructions can potentially create arguments 1689 // that cannot satisfy the inline-asm constraints. 1690 // If the instruction has nomerge attribute, return false. 1691 if (const auto *C = dyn_cast<CallBase>(I)) 1692 if (C->isInlineAsm() || C->cannotMerge()) 1693 return false; 1694 1695 // Everything must have only one use too, apart from stores which 1696 // have no uses. 1697 if (!isa<StoreInst>(I) && !I->hasOneUse()) 1698 return false; 1699 } 1700 1701 const Instruction *I0 = Insts.front(); 1702 for (auto *I : Insts) 1703 if (!I->isSameOperationAs(I0)) 1704 return false; 1705 1706 // All instructions in Insts are known to be the same opcode. If they aren't 1707 // stores, check the only user of each is a PHI or in the same block as the 1708 // instruction, because if a user is in the same block as an instruction 1709 // we're contemplating sinking, it must already be determined to be sinkable. 1710 if (!isa<StoreInst>(I0)) { 1711 auto *PNUse = dyn_cast<PHINode>(*I0->user_begin()); 1712 auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0); 1713 if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool { 1714 auto *U = cast<Instruction>(*I->user_begin()); 1715 return (PNUse && 1716 PNUse->getParent() == Succ && 1717 PNUse->getIncomingValueForBlock(I->getParent()) == I) || 1718 U->getParent() == I->getParent(); 1719 })) 1720 return false; 1721 } 1722 1723 // Because SROA can't handle speculating stores of selects, try not to sink 1724 // loads, stores or lifetime markers of allocas when we'd have to create a 1725 // PHI for the address operand. Also, because it is likely that loads or 1726 // stores of allocas will disappear when Mem2Reg/SROA is run, don't sink 1727 // them. 1728 // This can cause code churn which can have unintended consequences down 1729 // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244. 1730 // FIXME: This is a workaround for a deficiency in SROA - see 1731 // https://llvm.org/bugs/show_bug.cgi?id=30188 1732 if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) { 1733 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts()); 1734 })) 1735 return false; 1736 if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) { 1737 return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts()); 1738 })) 1739 return false; 1740 if (isLifeTimeMarker(I0) && any_of(Insts, [](const Instruction *I) { 1741 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts()); 1742 })) 1743 return false; 1744 1745 // For calls to be sinkable, they must all be indirect, or have same callee. 1746 // I.e. if we have two direct calls to different callees, we don't want to 1747 // turn that into an indirect call. Likewise, if we have an indirect call, 1748 // and a direct call, we don't actually want to have a single indirect call. 1749 if (isa<CallBase>(I0)) { 1750 auto IsIndirectCall = [](const Instruction *I) { 1751 return cast<CallBase>(I)->isIndirectCall(); 1752 }; 1753 bool HaveIndirectCalls = any_of(Insts, IsIndirectCall); 1754 bool AllCallsAreIndirect = all_of(Insts, IsIndirectCall); 1755 if (HaveIndirectCalls) { 1756 if (!AllCallsAreIndirect) 1757 return false; 1758 } else { 1759 // All callees must be identical. 1760 Value *Callee = nullptr; 1761 for (const Instruction *I : Insts) { 1762 Value *CurrCallee = cast<CallBase>(I)->getCalledOperand(); 1763 if (!Callee) 1764 Callee = CurrCallee; 1765 else if (Callee != CurrCallee) 1766 return false; 1767 } 1768 } 1769 } 1770 1771 for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) { 1772 Value *Op = I0->getOperand(OI); 1773 if (Op->getType()->isTokenTy()) 1774 // Don't touch any operand of token type. 1775 return false; 1776 1777 auto SameAsI0 = [&I0, OI](const Instruction *I) { 1778 assert(I->getNumOperands() == I0->getNumOperands()); 1779 return I->getOperand(OI) == I0->getOperand(OI); 1780 }; 1781 if (!all_of(Insts, SameAsI0)) { 1782 if ((isa<Constant>(Op) && !replacingOperandWithVariableIsCheap(I0, OI)) || 1783 !canReplaceOperandWithVariable(I0, OI)) 1784 // We can't create a PHI from this GEP. 1785 return false; 1786 for (auto *I : Insts) 1787 PHIOperands[I].push_back(I->getOperand(OI)); 1788 } 1789 } 1790 return true; 1791 } 1792 1793 // Assuming canSinkInstructions(Blocks) has returned true, sink the last 1794 // instruction of every block in Blocks to their common successor, commoning 1795 // into one instruction. 1796 static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) { 1797 auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0); 1798 1799 // canSinkInstructions returning true guarantees that every block has at 1800 // least one non-terminator instruction. 1801 SmallVector<Instruction*,4> Insts; 1802 for (auto *BB : Blocks) { 1803 Instruction *I = BB->getTerminator(); 1804 do { 1805 I = I->getPrevNode(); 1806 } while (isa<DbgInfoIntrinsic>(I) && I != &BB->front()); 1807 if (!isa<DbgInfoIntrinsic>(I)) 1808 Insts.push_back(I); 1809 } 1810 1811 // The only checking we need to do now is that all users of all instructions 1812 // are the same PHI node. canSinkInstructions should have checked this but 1813 // it is slightly over-aggressive - it gets confused by commutative 1814 // instructions so double-check it here. 1815 Instruction *I0 = Insts.front(); 1816 if (!isa<StoreInst>(I0)) { 1817 auto *PNUse = dyn_cast<PHINode>(*I0->user_begin()); 1818 if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool { 1819 auto *U = cast<Instruction>(*I->user_begin()); 1820 return U == PNUse; 1821 })) 1822 return false; 1823 } 1824 1825 // We don't need to do any more checking here; canSinkInstructions should 1826 // have done it all for us. 1827 SmallVector<Value*, 4> NewOperands; 1828 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) { 1829 // This check is different to that in canSinkInstructions. There, we 1830 // cared about the global view once simplifycfg (and instcombine) have 1831 // completed - it takes into account PHIs that become trivially 1832 // simplifiable. However here we need a more local view; if an operand 1833 // differs we create a PHI and rely on instcombine to clean up the very 1834 // small mess we may make. 1835 bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) { 1836 return I->getOperand(O) != I0->getOperand(O); 1837 }); 1838 if (!NeedPHI) { 1839 NewOperands.push_back(I0->getOperand(O)); 1840 continue; 1841 } 1842 1843 // Create a new PHI in the successor block and populate it. 1844 auto *Op = I0->getOperand(O); 1845 assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!"); 1846 auto *PN = PHINode::Create(Op->getType(), Insts.size(), 1847 Op->getName() + ".sink", &BBEnd->front()); 1848 for (auto *I : Insts) 1849 PN->addIncoming(I->getOperand(O), I->getParent()); 1850 NewOperands.push_back(PN); 1851 } 1852 1853 // Arbitrarily use I0 as the new "common" instruction; remap its operands 1854 // and move it to the start of the successor block. 1855 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) 1856 I0->getOperandUse(O).set(NewOperands[O]); 1857 I0->moveBefore(&*BBEnd->getFirstInsertionPt()); 1858 1859 // Update metadata and IR flags, and merge debug locations. 1860 for (auto *I : Insts) 1861 if (I != I0) { 1862 // The debug location for the "common" instruction is the merged locations 1863 // of all the commoned instructions. We start with the original location 1864 // of the "common" instruction and iteratively merge each location in the 1865 // loop below. 1866 // This is an N-way merge, which will be inefficient if I0 is a CallInst. 1867 // However, as N-way merge for CallInst is rare, so we use simplified API 1868 // instead of using complex API for N-way merge. 1869 I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc()); 1870 combineMetadataForCSE(I0, I, true); 1871 I0->andIRFlags(I); 1872 } 1873 1874 if (!isa<StoreInst>(I0)) { 1875 // canSinkLastInstruction checked that all instructions were used by 1876 // one and only one PHI node. Find that now, RAUW it to our common 1877 // instruction and nuke it. 1878 assert(I0->hasOneUse()); 1879 auto *PN = cast<PHINode>(*I0->user_begin()); 1880 PN->replaceAllUsesWith(I0); 1881 PN->eraseFromParent(); 1882 } 1883 1884 // Finally nuke all instructions apart from the common instruction. 1885 for (auto *I : Insts) 1886 if (I != I0) 1887 I->eraseFromParent(); 1888 1889 return true; 1890 } 1891 1892 namespace { 1893 1894 // LockstepReverseIterator - Iterates through instructions 1895 // in a set of blocks in reverse order from the first non-terminator. 1896 // For example (assume all blocks have size n): 1897 // LockstepReverseIterator I([B1, B2, B3]); 1898 // *I-- = [B1[n], B2[n], B3[n]]; 1899 // *I-- = [B1[n-1], B2[n-1], B3[n-1]]; 1900 // *I-- = [B1[n-2], B2[n-2], B3[n-2]]; 1901 // ... 1902 class LockstepReverseIterator { 1903 ArrayRef<BasicBlock*> Blocks; 1904 SmallVector<Instruction*,4> Insts; 1905 bool Fail; 1906 1907 public: 1908 LockstepReverseIterator(ArrayRef<BasicBlock*> Blocks) : Blocks(Blocks) { 1909 reset(); 1910 } 1911 1912 void reset() { 1913 Fail = false; 1914 Insts.clear(); 1915 for (auto *BB : Blocks) { 1916 Instruction *Inst = BB->getTerminator(); 1917 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);) 1918 Inst = Inst->getPrevNode(); 1919 if (!Inst) { 1920 // Block wasn't big enough. 1921 Fail = true; 1922 return; 1923 } 1924 Insts.push_back(Inst); 1925 } 1926 } 1927 1928 bool isValid() const { 1929 return !Fail; 1930 } 1931 1932 void operator--() { 1933 if (Fail) 1934 return; 1935 for (auto *&Inst : Insts) { 1936 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);) 1937 Inst = Inst->getPrevNode(); 1938 // Already at beginning of block. 1939 if (!Inst) { 1940 Fail = true; 1941 return; 1942 } 1943 } 1944 } 1945 1946 void operator++() { 1947 if (Fail) 1948 return; 1949 for (auto *&Inst : Insts) { 1950 for (Inst = Inst->getNextNode(); Inst && isa<DbgInfoIntrinsic>(Inst);) 1951 Inst = Inst->getNextNode(); 1952 // Already at end of block. 1953 if (!Inst) { 1954 Fail = true; 1955 return; 1956 } 1957 } 1958 } 1959 1960 ArrayRef<Instruction*> operator * () const { 1961 return Insts; 1962 } 1963 }; 1964 1965 } // end anonymous namespace 1966 1967 /// Check whether BB's predecessors end with unconditional branches. If it is 1968 /// true, sink any common code from the predecessors to BB. 1969 static bool SinkCommonCodeFromPredecessors(BasicBlock *BB, 1970 DomTreeUpdater *DTU) { 1971 // We support two situations: 1972 // (1) all incoming arcs are unconditional 1973 // (2) there are non-unconditional incoming arcs 1974 // 1975 // (2) is very common in switch defaults and 1976 // else-if patterns; 1977 // 1978 // if (a) f(1); 1979 // else if (b) f(2); 1980 // 1981 // produces: 1982 // 1983 // [if] 1984 // / \ 1985 // [f(1)] [if] 1986 // | | \ 1987 // | | | 1988 // | [f(2)]| 1989 // \ | / 1990 // [ end ] 1991 // 1992 // [end] has two unconditional predecessor arcs and one conditional. The 1993 // conditional refers to the implicit empty 'else' arc. This conditional 1994 // arc can also be caused by an empty default block in a switch. 1995 // 1996 // In this case, we attempt to sink code from all *unconditional* arcs. 1997 // If we can sink instructions from these arcs (determined during the scan 1998 // phase below) we insert a common successor for all unconditional arcs and 1999 // connect that to [end], to enable sinking: 2000 // 2001 // [if] 2002 // / \ 2003 // [x(1)] [if] 2004 // | | \ 2005 // | | \ 2006 // | [x(2)] | 2007 // \ / | 2008 // [sink.split] | 2009 // \ / 2010 // [ end ] 2011 // 2012 SmallVector<BasicBlock*,4> UnconditionalPreds; 2013 bool HaveNonUnconditionalPredecessors = false; 2014 for (auto *PredBB : predecessors(BB)) { 2015 auto *PredBr = dyn_cast<BranchInst>(PredBB->getTerminator()); 2016 if (PredBr && PredBr->isUnconditional()) 2017 UnconditionalPreds.push_back(PredBB); 2018 else 2019 HaveNonUnconditionalPredecessors = true; 2020 } 2021 if (UnconditionalPreds.size() < 2) 2022 return false; 2023 2024 // We take a two-step approach to tail sinking. First we scan from the end of 2025 // each block upwards in lockstep. If the n'th instruction from the end of each 2026 // block can be sunk, those instructions are added to ValuesToSink and we 2027 // carry on. If we can sink an instruction but need to PHI-merge some operands 2028 // (because they're not identical in each instruction) we add these to 2029 // PHIOperands. 2030 int ScanIdx = 0; 2031 SmallPtrSet<Value*,4> InstructionsToSink; 2032 DenseMap<Instruction*, SmallVector<Value*,4>> PHIOperands; 2033 LockstepReverseIterator LRI(UnconditionalPreds); 2034 while (LRI.isValid() && 2035 canSinkInstructions(*LRI, PHIOperands)) { 2036 LLVM_DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0] 2037 << "\n"); 2038 InstructionsToSink.insert((*LRI).begin(), (*LRI).end()); 2039 ++ScanIdx; 2040 --LRI; 2041 } 2042 2043 // If no instructions can be sunk, early-return. 2044 if (ScanIdx == 0) 2045 return false; 2046 2047 // Okay, we *could* sink last ScanIdx instructions. But how many can we 2048 // actually sink before encountering instruction that is unprofitable to sink? 2049 auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) { 2050 unsigned NumPHIdValues = 0; 2051 for (auto *I : *LRI) 2052 for (auto *V : PHIOperands[I]) { 2053 if (InstructionsToSink.count(V) == 0) 2054 ++NumPHIdValues; 2055 // FIXME: this check is overly optimistic. We may end up not sinking 2056 // said instruction, due to the very same profitability check. 2057 // See @creating_too_many_phis in sink-common-code.ll. 2058 } 2059 LLVM_DEBUG(dbgs() << "SINK: #phid values: " << NumPHIdValues << "\n"); 2060 unsigned NumPHIInsts = NumPHIdValues / UnconditionalPreds.size(); 2061 if ((NumPHIdValues % UnconditionalPreds.size()) != 0) 2062 NumPHIInsts++; 2063 2064 return NumPHIInsts <= 1; 2065 }; 2066 2067 // We've determined that we are going to sink last ScanIdx instructions, 2068 // and recorded them in InstructionsToSink. Now, some instructions may be 2069 // unprofitable to sink. But that determination depends on the instructions 2070 // that we are going to sink. 2071 2072 // First, forward scan: find the first instruction unprofitable to sink, 2073 // recording all the ones that are profitable to sink. 2074 // FIXME: would it be better, after we detect that not all are profitable. 2075 // to either record the profitable ones, or erase the unprofitable ones? 2076 // Maybe we need to choose (at runtime) the one that will touch least instrs? 2077 LRI.reset(); 2078 int Idx = 0; 2079 SmallPtrSet<Value *, 4> InstructionsProfitableToSink; 2080 while (Idx < ScanIdx) { 2081 if (!ProfitableToSinkInstruction(LRI)) { 2082 // Too many PHIs would be created. 2083 LLVM_DEBUG( 2084 dbgs() << "SINK: stopping here, too many PHIs would be created!\n"); 2085 break; 2086 } 2087 InstructionsProfitableToSink.insert((*LRI).begin(), (*LRI).end()); 2088 --LRI; 2089 ++Idx; 2090 } 2091 2092 // If no instructions can be sunk, early-return. 2093 if (Idx == 0) 2094 return false; 2095 2096 // Did we determine that (only) some instructions are unprofitable to sink? 2097 if (Idx < ScanIdx) { 2098 // Okay, some instructions are unprofitable. 2099 ScanIdx = Idx; 2100 InstructionsToSink = InstructionsProfitableToSink; 2101 2102 // But, that may make other instructions unprofitable, too. 2103 // So, do a backward scan, do any earlier instructions become unprofitable? 2104 assert(!ProfitableToSinkInstruction(LRI) && 2105 "We already know that the last instruction is unprofitable to sink"); 2106 ++LRI; 2107 --Idx; 2108 while (Idx >= 0) { 2109 // If we detect that an instruction becomes unprofitable to sink, 2110 // all earlier instructions won't be sunk either, 2111 // so preemptively keep InstructionsProfitableToSink in sync. 2112 // FIXME: is this the most performant approach? 2113 for (auto *I : *LRI) 2114 InstructionsProfitableToSink.erase(I); 2115 if (!ProfitableToSinkInstruction(LRI)) { 2116 // Everything starting with this instruction won't be sunk. 2117 ScanIdx = Idx; 2118 InstructionsToSink = InstructionsProfitableToSink; 2119 } 2120 ++LRI; 2121 --Idx; 2122 } 2123 } 2124 2125 // If no instructions can be sunk, early-return. 2126 if (ScanIdx == 0) 2127 return false; 2128 2129 bool Changed = false; 2130 2131 if (HaveNonUnconditionalPredecessors) { 2132 // It is always legal to sink common instructions from unconditional 2133 // predecessors. However, if not all predecessors are unconditional, 2134 // this transformation might be pessimizing. So as a rule of thumb, 2135 // don't do it unless we'd sink at least one non-speculatable instruction. 2136 // See https://bugs.llvm.org/show_bug.cgi?id=30244 2137 LRI.reset(); 2138 int Idx = 0; 2139 bool Profitable = false; 2140 while (Idx < ScanIdx) { 2141 if (!isSafeToSpeculativelyExecute((*LRI)[0])) { 2142 Profitable = true; 2143 break; 2144 } 2145 --LRI; 2146 ++Idx; 2147 } 2148 if (!Profitable) 2149 return false; 2150 2151 LLVM_DEBUG(dbgs() << "SINK: Splitting edge\n"); 2152 // We have a conditional edge and we're going to sink some instructions. 2153 // Insert a new block postdominating all blocks we're going to sink from. 2154 if (!SplitBlockPredecessors(BB, UnconditionalPreds, ".sink.split", DTU)) 2155 // Edges couldn't be split. 2156 return false; 2157 Changed = true; 2158 } 2159 2160 // Now that we've analyzed all potential sinking candidates, perform the 2161 // actual sink. We iteratively sink the last non-terminator of the source 2162 // blocks into their common successor unless doing so would require too 2163 // many PHI instructions to be generated (currently only one PHI is allowed 2164 // per sunk instruction). 2165 // 2166 // We can use InstructionsToSink to discount values needing PHI-merging that will 2167 // actually be sunk in a later iteration. This allows us to be more 2168 // aggressive in what we sink. This does allow a false positive where we 2169 // sink presuming a later value will also be sunk, but stop half way through 2170 // and never actually sink it which means we produce more PHIs than intended. 2171 // This is unlikely in practice though. 2172 int SinkIdx = 0; 2173 for (; SinkIdx != ScanIdx; ++SinkIdx) { 2174 LLVM_DEBUG(dbgs() << "SINK: Sink: " 2175 << *UnconditionalPreds[0]->getTerminator()->getPrevNode() 2176 << "\n"); 2177 2178 // Because we've sunk every instruction in turn, the current instruction to 2179 // sink is always at index 0. 2180 LRI.reset(); 2181 2182 if (!sinkLastInstruction(UnconditionalPreds)) { 2183 LLVM_DEBUG( 2184 dbgs() 2185 << "SINK: stopping here, failed to actually sink instruction!\n"); 2186 break; 2187 } 2188 2189 NumSinkCommonInstrs++; 2190 Changed = true; 2191 } 2192 if (SinkIdx != 0) 2193 ++NumSinkCommonCode; 2194 return Changed; 2195 } 2196 2197 /// Determine if we can hoist sink a sole store instruction out of a 2198 /// conditional block. 2199 /// 2200 /// We are looking for code like the following: 2201 /// BrBB: 2202 /// store i32 %add, i32* %arrayidx2 2203 /// ... // No other stores or function calls (we could be calling a memory 2204 /// ... // function). 2205 /// %cmp = icmp ult %x, %y 2206 /// br i1 %cmp, label %EndBB, label %ThenBB 2207 /// ThenBB: 2208 /// store i32 %add5, i32* %arrayidx2 2209 /// br label EndBB 2210 /// EndBB: 2211 /// ... 2212 /// We are going to transform this into: 2213 /// BrBB: 2214 /// store i32 %add, i32* %arrayidx2 2215 /// ... // 2216 /// %cmp = icmp ult %x, %y 2217 /// %add.add5 = select i1 %cmp, i32 %add, %add5 2218 /// store i32 %add.add5, i32* %arrayidx2 2219 /// ... 2220 /// 2221 /// \return The pointer to the value of the previous store if the store can be 2222 /// hoisted into the predecessor block. 0 otherwise. 2223 static Value *isSafeToSpeculateStore(Instruction *I, BasicBlock *BrBB, 2224 BasicBlock *StoreBB, BasicBlock *EndBB) { 2225 StoreInst *StoreToHoist = dyn_cast<StoreInst>(I); 2226 if (!StoreToHoist) 2227 return nullptr; 2228 2229 // Volatile or atomic. 2230 if (!StoreToHoist->isSimple()) 2231 return nullptr; 2232 2233 Value *StorePtr = StoreToHoist->getPointerOperand(); 2234 Type *StoreTy = StoreToHoist->getValueOperand()->getType(); 2235 2236 // Look for a store to the same pointer in BrBB. 2237 unsigned MaxNumInstToLookAt = 9; 2238 // Skip pseudo probe intrinsic calls which are not really killing any memory 2239 // accesses. 2240 for (Instruction &CurI : reverse(BrBB->instructionsWithoutDebug(true))) { 2241 if (!MaxNumInstToLookAt) 2242 break; 2243 --MaxNumInstToLookAt; 2244 2245 // Could be calling an instruction that affects memory like free(). 2246 if (CurI.mayWriteToMemory() && !isa<StoreInst>(CurI)) 2247 return nullptr; 2248 2249 if (auto *SI = dyn_cast<StoreInst>(&CurI)) { 2250 // Found the previous store to same location and type. Make sure it is 2251 // simple, to avoid introducing a spurious non-atomic write after an 2252 // atomic write. 2253 if (SI->getPointerOperand() == StorePtr && 2254 SI->getValueOperand()->getType() == StoreTy && SI->isSimple()) 2255 // Found the previous store, return its value operand. 2256 return SI->getValueOperand(); 2257 return nullptr; // Unknown store. 2258 } 2259 } 2260 2261 return nullptr; 2262 } 2263 2264 /// Estimate the cost of the insertion(s) and check that the PHI nodes can be 2265 /// converted to selects. 2266 static bool validateAndCostRequiredSelects(BasicBlock *BB, BasicBlock *ThenBB, 2267 BasicBlock *EndBB, 2268 unsigned &SpeculatedInstructions, 2269 InstructionCost &Cost, 2270 const TargetTransformInfo &TTI) { 2271 TargetTransformInfo::TargetCostKind CostKind = 2272 BB->getParent()->hasMinSize() 2273 ? TargetTransformInfo::TCK_CodeSize 2274 : TargetTransformInfo::TCK_SizeAndLatency; 2275 2276 bool HaveRewritablePHIs = false; 2277 for (PHINode &PN : EndBB->phis()) { 2278 Value *OrigV = PN.getIncomingValueForBlock(BB); 2279 Value *ThenV = PN.getIncomingValueForBlock(ThenBB); 2280 2281 // FIXME: Try to remove some of the duplication with HoistThenElseCodeToIf. 2282 // Skip PHIs which are trivial. 2283 if (ThenV == OrigV) 2284 continue; 2285 2286 Cost += TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr, 2287 CmpInst::BAD_ICMP_PREDICATE, CostKind); 2288 2289 // Don't convert to selects if we could remove undefined behavior instead. 2290 if (passingValueIsAlwaysUndefined(OrigV, &PN) || 2291 passingValueIsAlwaysUndefined(ThenV, &PN)) 2292 return false; 2293 2294 HaveRewritablePHIs = true; 2295 ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV); 2296 ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV); 2297 if (!OrigCE && !ThenCE) 2298 continue; // Known safe and cheap. 2299 2300 if ((ThenCE && !isSafeToSpeculativelyExecute(ThenCE)) || 2301 (OrigCE && !isSafeToSpeculativelyExecute(OrigCE))) 2302 return false; 2303 InstructionCost OrigCost = OrigCE ? computeSpeculationCost(OrigCE, TTI) : 0; 2304 InstructionCost ThenCost = ThenCE ? computeSpeculationCost(ThenCE, TTI) : 0; 2305 InstructionCost MaxCost = 2306 2 * PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; 2307 if (OrigCost + ThenCost > MaxCost) 2308 return false; 2309 2310 // Account for the cost of an unfolded ConstantExpr which could end up 2311 // getting expanded into Instructions. 2312 // FIXME: This doesn't account for how many operations are combined in the 2313 // constant expression. 2314 ++SpeculatedInstructions; 2315 if (SpeculatedInstructions > 1) 2316 return false; 2317 } 2318 2319 return HaveRewritablePHIs; 2320 } 2321 2322 /// Speculate a conditional basic block flattening the CFG. 2323 /// 2324 /// Note that this is a very risky transform currently. Speculating 2325 /// instructions like this is most often not desirable. Instead, there is an MI 2326 /// pass which can do it with full awareness of the resource constraints. 2327 /// However, some cases are "obvious" and we should do directly. An example of 2328 /// this is speculating a single, reasonably cheap instruction. 2329 /// 2330 /// There is only one distinct advantage to flattening the CFG at the IR level: 2331 /// it makes very common but simplistic optimizations such as are common in 2332 /// instcombine and the DAG combiner more powerful by removing CFG edges and 2333 /// modeling their effects with easier to reason about SSA value graphs. 2334 /// 2335 /// 2336 /// An illustration of this transform is turning this IR: 2337 /// \code 2338 /// BB: 2339 /// %cmp = icmp ult %x, %y 2340 /// br i1 %cmp, label %EndBB, label %ThenBB 2341 /// ThenBB: 2342 /// %sub = sub %x, %y 2343 /// br label BB2 2344 /// EndBB: 2345 /// %phi = phi [ %sub, %ThenBB ], [ 0, %EndBB ] 2346 /// ... 2347 /// \endcode 2348 /// 2349 /// Into this IR: 2350 /// \code 2351 /// BB: 2352 /// %cmp = icmp ult %x, %y 2353 /// %sub = sub %x, %y 2354 /// %cond = select i1 %cmp, 0, %sub 2355 /// ... 2356 /// \endcode 2357 /// 2358 /// \returns true if the conditional block is removed. 2359 bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB, 2360 const TargetTransformInfo &TTI) { 2361 // Be conservative for now. FP select instruction can often be expensive. 2362 Value *BrCond = BI->getCondition(); 2363 if (isa<FCmpInst>(BrCond)) 2364 return false; 2365 2366 BasicBlock *BB = BI->getParent(); 2367 BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0); 2368 InstructionCost Budget = 2369 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; 2370 2371 // If ThenBB is actually on the false edge of the conditional branch, remember 2372 // to swap the select operands later. 2373 bool Invert = false; 2374 if (ThenBB != BI->getSuccessor(0)) { 2375 assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?"); 2376 Invert = true; 2377 } 2378 assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block"); 2379 2380 // If the branch is non-unpredictable, and is predicted to *not* branch to 2381 // the `then` block, then avoid speculating it. 2382 if (!BI->getMetadata(LLVMContext::MD_unpredictable)) { 2383 uint64_t TWeight, FWeight; 2384 if (BI->extractProfMetadata(TWeight, FWeight) && (TWeight + FWeight) != 0) { 2385 uint64_t EndWeight = Invert ? TWeight : FWeight; 2386 BranchProbability BIEndProb = 2387 BranchProbability::getBranchProbability(EndWeight, TWeight + FWeight); 2388 BranchProbability Likely = TTI.getPredictableBranchThreshold(); 2389 if (BIEndProb >= Likely) 2390 return false; 2391 } 2392 } 2393 2394 // Keep a count of how many times instructions are used within ThenBB when 2395 // they are candidates for sinking into ThenBB. Specifically: 2396 // - They are defined in BB, and 2397 // - They have no side effects, and 2398 // - All of their uses are in ThenBB. 2399 SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts; 2400 2401 SmallVector<Instruction *, 4> SpeculatedDbgIntrinsics; 2402 2403 unsigned SpeculatedInstructions = 0; 2404 Value *SpeculatedStoreValue = nullptr; 2405 StoreInst *SpeculatedStore = nullptr; 2406 for (BasicBlock::iterator BBI = ThenBB->begin(), 2407 BBE = std::prev(ThenBB->end()); 2408 BBI != BBE; ++BBI) { 2409 Instruction *I = &*BBI; 2410 // Skip debug info. 2411 if (isa<DbgInfoIntrinsic>(I)) { 2412 SpeculatedDbgIntrinsics.push_back(I); 2413 continue; 2414 } 2415 2416 // Skip pseudo probes. The consequence is we lose track of the branch 2417 // probability for ThenBB, which is fine since the optimization here takes 2418 // place regardless of the branch probability. 2419 if (isa<PseudoProbeInst>(I)) { 2420 // The probe should be deleted so that it will not be over-counted when 2421 // the samples collected on the non-conditional path are counted towards 2422 // the conditional path. We leave it for the counts inference algorithm to 2423 // figure out a proper count for an unknown probe. 2424 SpeculatedDbgIntrinsics.push_back(I); 2425 continue; 2426 } 2427 2428 // Only speculatively execute a single instruction (not counting the 2429 // terminator) for now. 2430 ++SpeculatedInstructions; 2431 if (SpeculatedInstructions > 1) 2432 return false; 2433 2434 // Don't hoist the instruction if it's unsafe or expensive. 2435 if (!isSafeToSpeculativelyExecute(I) && 2436 !(HoistCondStores && (SpeculatedStoreValue = isSafeToSpeculateStore( 2437 I, BB, ThenBB, EndBB)))) 2438 return false; 2439 if (!SpeculatedStoreValue && 2440 computeSpeculationCost(I, TTI) > 2441 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic) 2442 return false; 2443 2444 // Store the store speculation candidate. 2445 if (SpeculatedStoreValue) 2446 SpeculatedStore = cast<StoreInst>(I); 2447 2448 // Do not hoist the instruction if any of its operands are defined but not 2449 // used in BB. The transformation will prevent the operand from 2450 // being sunk into the use block. 2451 for (Use &Op : I->operands()) { 2452 Instruction *OpI = dyn_cast<Instruction>(Op); 2453 if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects()) 2454 continue; // Not a candidate for sinking. 2455 2456 ++SinkCandidateUseCounts[OpI]; 2457 } 2458 } 2459 2460 // Consider any sink candidates which are only used in ThenBB as costs for 2461 // speculation. Note, while we iterate over a DenseMap here, we are summing 2462 // and so iteration order isn't significant. 2463 for (SmallDenseMap<Instruction *, unsigned, 4>::iterator 2464 I = SinkCandidateUseCounts.begin(), 2465 E = SinkCandidateUseCounts.end(); 2466 I != E; ++I) 2467 if (I->first->hasNUses(I->second)) { 2468 ++SpeculatedInstructions; 2469 if (SpeculatedInstructions > 1) 2470 return false; 2471 } 2472 2473 // Check that we can insert the selects and that it's not too expensive to do 2474 // so. 2475 bool Convert = SpeculatedStore != nullptr; 2476 InstructionCost Cost = 0; 2477 Convert |= validateAndCostRequiredSelects(BB, ThenBB, EndBB, 2478 SpeculatedInstructions, 2479 Cost, TTI); 2480 if (!Convert || Cost > Budget) 2481 return false; 2482 2483 // If we get here, we can hoist the instruction and if-convert. 2484 LLVM_DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";); 2485 2486 // Insert a select of the value of the speculated store. 2487 if (SpeculatedStoreValue) { 2488 IRBuilder<NoFolder> Builder(BI); 2489 Value *TrueV = SpeculatedStore->getValueOperand(); 2490 Value *FalseV = SpeculatedStoreValue; 2491 if (Invert) 2492 std::swap(TrueV, FalseV); 2493 Value *S = Builder.CreateSelect( 2494 BrCond, TrueV, FalseV, "spec.store.select", BI); 2495 SpeculatedStore->setOperand(0, S); 2496 SpeculatedStore->applyMergedLocation(BI->getDebugLoc(), 2497 SpeculatedStore->getDebugLoc()); 2498 } 2499 2500 // Metadata can be dependent on the condition we are hoisting above. 2501 // Conservatively strip all metadata on the instruction. Drop the debug loc 2502 // to avoid making it appear as if the condition is a constant, which would 2503 // be misleading while debugging. 2504 // Similarly strip attributes that maybe dependent on condition we are 2505 // hoisting above. 2506 for (auto &I : *ThenBB) { 2507 if (!SpeculatedStoreValue || &I != SpeculatedStore) 2508 I.setDebugLoc(DebugLoc()); 2509 I.dropUndefImplyingAttrsAndUnknownMetadata(); 2510 } 2511 2512 // Hoist the instructions. 2513 BB->getInstList().splice(BI->getIterator(), ThenBB->getInstList(), 2514 ThenBB->begin(), std::prev(ThenBB->end())); 2515 2516 // Insert selects and rewrite the PHI operands. 2517 IRBuilder<NoFolder> Builder(BI); 2518 for (PHINode &PN : EndBB->phis()) { 2519 unsigned OrigI = PN.getBasicBlockIndex(BB); 2520 unsigned ThenI = PN.getBasicBlockIndex(ThenBB); 2521 Value *OrigV = PN.getIncomingValue(OrigI); 2522 Value *ThenV = PN.getIncomingValue(ThenI); 2523 2524 // Skip PHIs which are trivial. 2525 if (OrigV == ThenV) 2526 continue; 2527 2528 // Create a select whose true value is the speculatively executed value and 2529 // false value is the pre-existing value. Swap them if the branch 2530 // destinations were inverted. 2531 Value *TrueV = ThenV, *FalseV = OrigV; 2532 if (Invert) 2533 std::swap(TrueV, FalseV); 2534 Value *V = Builder.CreateSelect(BrCond, TrueV, FalseV, "spec.select", BI); 2535 PN.setIncomingValue(OrigI, V); 2536 PN.setIncomingValue(ThenI, V); 2537 } 2538 2539 // Remove speculated dbg intrinsics. 2540 // FIXME: Is it possible to do this in a more elegant way? Moving/merging the 2541 // dbg value for the different flows and inserting it after the select. 2542 for (Instruction *I : SpeculatedDbgIntrinsics) 2543 I->eraseFromParent(); 2544 2545 ++NumSpeculations; 2546 return true; 2547 } 2548 2549 /// Return true if we can thread a branch across this block. 2550 static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) { 2551 int Size = 0; 2552 2553 SmallPtrSet<const Value *, 32> EphValues; 2554 auto IsEphemeral = [&](const Value *V) { 2555 if (isa<AssumeInst>(V)) 2556 return true; 2557 return isSafeToSpeculativelyExecute(V) && 2558 all_of(V->users(), 2559 [&](const User *U) { return EphValues.count(U); }); 2560 }; 2561 2562 // Walk the loop in reverse so that we can identify ephemeral values properly 2563 // (values only feeding assumes). 2564 for (Instruction &I : reverse(BB->instructionsWithoutDebug())) { 2565 // Can't fold blocks that contain noduplicate or convergent calls. 2566 if (CallInst *CI = dyn_cast<CallInst>(&I)) 2567 if (CI->cannotDuplicate() || CI->isConvergent()) 2568 return false; 2569 2570 // Ignore ephemeral values which are deleted during codegen. 2571 if (IsEphemeral(&I)) 2572 EphValues.insert(&I); 2573 // We will delete Phis while threading, so Phis should not be accounted in 2574 // block's size. 2575 else if (!isa<PHINode>(I)) { 2576 if (Size++ > MaxSmallBlockSize) 2577 return false; // Don't clone large BB's. 2578 } 2579 2580 // We can only support instructions that do not define values that are 2581 // live outside of the current basic block. 2582 for (User *U : I.users()) { 2583 Instruction *UI = cast<Instruction>(U); 2584 if (UI->getParent() != BB || isa<PHINode>(UI)) 2585 return false; 2586 } 2587 2588 // Looks ok, continue checking. 2589 } 2590 2591 return true; 2592 } 2593 2594 /// If we have a conditional branch on a PHI node value that is defined in the 2595 /// same block as the branch and if any PHI entries are constants, thread edges 2596 /// corresponding to that entry to be branches to their ultimate destination. 2597 static bool FoldCondBranchOnPHI(BranchInst *BI, DomTreeUpdater *DTU, 2598 const DataLayout &DL, AssumptionCache *AC) { 2599 BasicBlock *BB = BI->getParent(); 2600 PHINode *PN = dyn_cast<PHINode>(BI->getCondition()); 2601 // NOTE: we currently cannot transform this case if the PHI node is used 2602 // outside of the block. 2603 if (!PN || PN->getParent() != BB || !PN->hasOneUse()) 2604 return false; 2605 2606 // Degenerate case of a single entry PHI. 2607 if (PN->getNumIncomingValues() == 1) { 2608 FoldSingleEntryPHINodes(PN->getParent()); 2609 return true; 2610 } 2611 2612 // Now we know that this block has multiple preds and two succs. 2613 if (!BlockIsSimpleEnoughToThreadThrough(BB)) 2614 return false; 2615 2616 // Okay, this is a simple enough basic block. See if any phi values are 2617 // constants. 2618 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 2619 ConstantInt *CB = dyn_cast<ConstantInt>(PN->getIncomingValue(i)); 2620 if (!CB || !CB->getType()->isIntegerTy(1)) 2621 continue; 2622 2623 // Okay, we now know that all edges from PredBB should be revectored to 2624 // branch to RealDest. 2625 BasicBlock *PredBB = PN->getIncomingBlock(i); 2626 BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue()); 2627 2628 if (RealDest == BB) 2629 continue; // Skip self loops. 2630 // Skip if the predecessor's terminator is an indirect branch. 2631 if (isa<IndirectBrInst>(PredBB->getTerminator())) 2632 continue; 2633 2634 SmallVector<DominatorTree::UpdateType, 3> Updates; 2635 2636 // The dest block might have PHI nodes, other predecessors and other 2637 // difficult cases. Instead of being smart about this, just insert a new 2638 // block that jumps to the destination block, effectively splitting 2639 // the edge we are about to create. 2640 BasicBlock *EdgeBB = 2641 BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge", 2642 RealDest->getParent(), RealDest); 2643 BranchInst *CritEdgeBranch = BranchInst::Create(RealDest, EdgeBB); 2644 if (DTU) 2645 Updates.push_back({DominatorTree::Insert, EdgeBB, RealDest}); 2646 CritEdgeBranch->setDebugLoc(BI->getDebugLoc()); 2647 2648 // Update PHI nodes. 2649 AddPredecessorToBlock(RealDest, EdgeBB, BB); 2650 2651 // BB may have instructions that are being threaded over. Clone these 2652 // instructions into EdgeBB. We know that there will be no uses of the 2653 // cloned instructions outside of EdgeBB. 2654 BasicBlock::iterator InsertPt = EdgeBB->begin(); 2655 DenseMap<Value *, Value *> TranslateMap; // Track translated values. 2656 for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) { 2657 if (PHINode *PN = dyn_cast<PHINode>(BBI)) { 2658 TranslateMap[PN] = PN->getIncomingValueForBlock(PredBB); 2659 continue; 2660 } 2661 // Clone the instruction. 2662 Instruction *N = BBI->clone(); 2663 if (BBI->hasName()) 2664 N->setName(BBI->getName() + ".c"); 2665 2666 // Update operands due to translation. 2667 for (Use &Op : N->operands()) { 2668 DenseMap<Value *, Value *>::iterator PI = TranslateMap.find(Op); 2669 if (PI != TranslateMap.end()) 2670 Op = PI->second; 2671 } 2672 2673 // Check for trivial simplification. 2674 if (Value *V = SimplifyInstruction(N, {DL, nullptr, nullptr, AC})) { 2675 if (!BBI->use_empty()) 2676 TranslateMap[&*BBI] = V; 2677 if (!N->mayHaveSideEffects()) { 2678 N->deleteValue(); // Instruction folded away, don't need actual inst 2679 N = nullptr; 2680 } 2681 } else { 2682 if (!BBI->use_empty()) 2683 TranslateMap[&*BBI] = N; 2684 } 2685 if (N) { 2686 // Insert the new instruction into its new home. 2687 EdgeBB->getInstList().insert(InsertPt, N); 2688 2689 // Register the new instruction with the assumption cache if necessary. 2690 if (auto *Assume = dyn_cast<AssumeInst>(N)) 2691 if (AC) 2692 AC->registerAssumption(Assume); 2693 } 2694 } 2695 2696 // Loop over all of the edges from PredBB to BB, changing them to branch 2697 // to EdgeBB instead. 2698 Instruction *PredBBTI = PredBB->getTerminator(); 2699 for (unsigned i = 0, e = PredBBTI->getNumSuccessors(); i != e; ++i) 2700 if (PredBBTI->getSuccessor(i) == BB) { 2701 BB->removePredecessor(PredBB); 2702 PredBBTI->setSuccessor(i, EdgeBB); 2703 } 2704 2705 if (DTU) { 2706 Updates.push_back({DominatorTree::Insert, PredBB, EdgeBB}); 2707 Updates.push_back({DominatorTree::Delete, PredBB, BB}); 2708 2709 DTU->applyUpdates(Updates); 2710 } 2711 2712 // Recurse, simplifying any other constants. 2713 return FoldCondBranchOnPHI(BI, DTU, DL, AC) || true; 2714 } 2715 2716 return false; 2717 } 2718 2719 /// Given a BB that starts with the specified two-entry PHI node, 2720 /// see if we can eliminate it. 2721 static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI, 2722 DomTreeUpdater *DTU, const DataLayout &DL) { 2723 // Ok, this is a two entry PHI node. Check to see if this is a simple "if 2724 // statement", which has a very simple dominance structure. Basically, we 2725 // are trying to find the condition that is being branched on, which 2726 // subsequently causes this merge to happen. We really want control 2727 // dependence information for this check, but simplifycfg can't keep it up 2728 // to date, and this catches most of the cases we care about anyway. 2729 BasicBlock *BB = PN->getParent(); 2730 2731 BasicBlock *IfTrue, *IfFalse; 2732 BranchInst *DomBI = GetIfCondition(BB, IfTrue, IfFalse); 2733 if (!DomBI) 2734 return false; 2735 Value *IfCond = DomBI->getCondition(); 2736 // Don't bother if the branch will be constant folded trivially. 2737 if (isa<ConstantInt>(IfCond)) 2738 return false; 2739 2740 BasicBlock *DomBlock = DomBI->getParent(); 2741 SmallVector<BasicBlock *, 2> IfBlocks; 2742 llvm::copy_if( 2743 PN->blocks(), std::back_inserter(IfBlocks), [](BasicBlock *IfBlock) { 2744 return cast<BranchInst>(IfBlock->getTerminator())->isUnconditional(); 2745 }); 2746 assert((IfBlocks.size() == 1 || IfBlocks.size() == 2) && 2747 "Will have either one or two blocks to speculate."); 2748 2749 // If the branch is non-unpredictable, see if we either predictably jump to 2750 // the merge bb (if we have only a single 'then' block), or if we predictably 2751 // jump to one specific 'then' block (if we have two of them). 2752 // It isn't beneficial to speculatively execute the code 2753 // from the block that we know is predictably not entered. 2754 if (!DomBI->getMetadata(LLVMContext::MD_unpredictable)) { 2755 uint64_t TWeight, FWeight; 2756 if (DomBI->extractProfMetadata(TWeight, FWeight) && 2757 (TWeight + FWeight) != 0) { 2758 BranchProbability BITrueProb = 2759 BranchProbability::getBranchProbability(TWeight, TWeight + FWeight); 2760 BranchProbability Likely = TTI.getPredictableBranchThreshold(); 2761 BranchProbability BIFalseProb = BITrueProb.getCompl(); 2762 if (IfBlocks.size() == 1) { 2763 BranchProbability BIBBProb = 2764 DomBI->getSuccessor(0) == BB ? BITrueProb : BIFalseProb; 2765 if (BIBBProb >= Likely) 2766 return false; 2767 } else { 2768 if (BITrueProb >= Likely || BIFalseProb >= Likely) 2769 return false; 2770 } 2771 } 2772 } 2773 2774 // Don't try to fold an unreachable block. For example, the phi node itself 2775 // can't be the candidate if-condition for a select that we want to form. 2776 if (auto *IfCondPhiInst = dyn_cast<PHINode>(IfCond)) 2777 if (IfCondPhiInst->getParent() == BB) 2778 return false; 2779 2780 // Okay, we found that we can merge this two-entry phi node into a select. 2781 // Doing so would require us to fold *all* two entry phi nodes in this block. 2782 // At some point this becomes non-profitable (particularly if the target 2783 // doesn't support cmov's). Only do this transformation if there are two or 2784 // fewer PHI nodes in this block. 2785 unsigned NumPhis = 0; 2786 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I) 2787 if (NumPhis > 2) 2788 return false; 2789 2790 // Loop over the PHI's seeing if we can promote them all to select 2791 // instructions. While we are at it, keep track of the instructions 2792 // that need to be moved to the dominating block. 2793 SmallPtrSet<Instruction *, 4> AggressiveInsts; 2794 InstructionCost Cost = 0; 2795 InstructionCost Budget = 2796 TwoEntryPHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; 2797 2798 bool Changed = false; 2799 for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) { 2800 PHINode *PN = cast<PHINode>(II++); 2801 if (Value *V = SimplifyInstruction(PN, {DL, PN})) { 2802 PN->replaceAllUsesWith(V); 2803 PN->eraseFromParent(); 2804 Changed = true; 2805 continue; 2806 } 2807 2808 if (!dominatesMergePoint(PN->getIncomingValue(0), BB, AggressiveInsts, 2809 Cost, Budget, TTI) || 2810 !dominatesMergePoint(PN->getIncomingValue(1), BB, AggressiveInsts, 2811 Cost, Budget, TTI)) 2812 return Changed; 2813 } 2814 2815 // If we folded the first phi, PN dangles at this point. Refresh it. If 2816 // we ran out of PHIs then we simplified them all. 2817 PN = dyn_cast<PHINode>(BB->begin()); 2818 if (!PN) 2819 return true; 2820 2821 // Return true if at least one of these is a 'not', and another is either 2822 // a 'not' too, or a constant. 2823 auto CanHoistNotFromBothValues = [](Value *V0, Value *V1) { 2824 if (!match(V0, m_Not(m_Value()))) 2825 std::swap(V0, V1); 2826 auto Invertible = m_CombineOr(m_Not(m_Value()), m_AnyIntegralConstant()); 2827 return match(V0, m_Not(m_Value())) && match(V1, Invertible); 2828 }; 2829 2830 // Don't fold i1 branches on PHIs which contain binary operators or 2831 // (possibly inverted) select form of or/ands, unless one of 2832 // the incoming values is an 'not' and another one is freely invertible. 2833 // These can often be turned into switches and other things. 2834 auto IsBinOpOrAnd = [](Value *V) { 2835 return match( 2836 V, m_CombineOr( 2837 m_BinOp(), 2838 m_CombineOr(m_Select(m_Value(), m_ImmConstant(), m_Value()), 2839 m_Select(m_Value(), m_Value(), m_ImmConstant())))); 2840 }; 2841 if (PN->getType()->isIntegerTy(1) && 2842 (IsBinOpOrAnd(PN->getIncomingValue(0)) || 2843 IsBinOpOrAnd(PN->getIncomingValue(1)) || IsBinOpOrAnd(IfCond)) && 2844 !CanHoistNotFromBothValues(PN->getIncomingValue(0), 2845 PN->getIncomingValue(1))) 2846 return Changed; 2847 2848 // If all PHI nodes are promotable, check to make sure that all instructions 2849 // in the predecessor blocks can be promoted as well. If not, we won't be able 2850 // to get rid of the control flow, so it's not worth promoting to select 2851 // instructions. 2852 for (BasicBlock *IfBlock : IfBlocks) 2853 for (BasicBlock::iterator I = IfBlock->begin(); !I->isTerminator(); ++I) 2854 if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I) && 2855 !isa<PseudoProbeInst>(I)) { 2856 // This is not an aggressive instruction that we can promote. 2857 // Because of this, we won't be able to get rid of the control flow, so 2858 // the xform is not worth it. 2859 return Changed; 2860 } 2861 2862 // If either of the blocks has it's address taken, we can't do this fold. 2863 if (any_of(IfBlocks, 2864 [](BasicBlock *IfBlock) { return IfBlock->hasAddressTaken(); })) 2865 return Changed; 2866 2867 LLVM_DEBUG(dbgs() << "FOUND IF CONDITION! " << *IfCond 2868 << " T: " << IfTrue->getName() 2869 << " F: " << IfFalse->getName() << "\n"); 2870 2871 // If we can still promote the PHI nodes after this gauntlet of tests, 2872 // do all of the PHI's now. 2873 2874 // Move all 'aggressive' instructions, which are defined in the 2875 // conditional parts of the if's up to the dominating block. 2876 for (BasicBlock *IfBlock : IfBlocks) 2877 hoistAllInstructionsInto(DomBlock, DomBI, IfBlock); 2878 2879 IRBuilder<NoFolder> Builder(DomBI); 2880 // Propagate fast-math-flags from phi nodes to replacement selects. 2881 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); 2882 while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) { 2883 if (isa<FPMathOperator>(PN)) 2884 Builder.setFastMathFlags(PN->getFastMathFlags()); 2885 2886 // Change the PHI node into a select instruction. 2887 Value *TrueVal = PN->getIncomingValueForBlock(IfTrue); 2888 Value *FalseVal = PN->getIncomingValueForBlock(IfFalse); 2889 2890 Value *Sel = Builder.CreateSelect(IfCond, TrueVal, FalseVal, "", DomBI); 2891 PN->replaceAllUsesWith(Sel); 2892 Sel->takeName(PN); 2893 PN->eraseFromParent(); 2894 } 2895 2896 // At this point, all IfBlocks are empty, so our if statement 2897 // has been flattened. Change DomBlock to jump directly to our new block to 2898 // avoid other simplifycfg's kicking in on the diamond. 2899 Builder.CreateBr(BB); 2900 2901 SmallVector<DominatorTree::UpdateType, 3> Updates; 2902 if (DTU) { 2903 Updates.push_back({DominatorTree::Insert, DomBlock, BB}); 2904 for (auto *Successor : successors(DomBlock)) 2905 Updates.push_back({DominatorTree::Delete, DomBlock, Successor}); 2906 } 2907 2908 DomBI->eraseFromParent(); 2909 if (DTU) 2910 DTU->applyUpdates(Updates); 2911 2912 return true; 2913 } 2914 2915 static Value *createLogicalOp(IRBuilderBase &Builder, 2916 Instruction::BinaryOps Opc, Value *LHS, 2917 Value *RHS, const Twine &Name = "") { 2918 // Try to relax logical op to binary op. 2919 if (impliesPoison(RHS, LHS)) 2920 return Builder.CreateBinOp(Opc, LHS, RHS, Name); 2921 if (Opc == Instruction::And) 2922 return Builder.CreateLogicalAnd(LHS, RHS, Name); 2923 if (Opc == Instruction::Or) 2924 return Builder.CreateLogicalOr(LHS, RHS, Name); 2925 llvm_unreachable("Invalid logical opcode"); 2926 } 2927 2928 /// Return true if either PBI or BI has branch weight available, and store 2929 /// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does 2930 /// not have branch weight, use 1:1 as its weight. 2931 static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI, 2932 uint64_t &PredTrueWeight, 2933 uint64_t &PredFalseWeight, 2934 uint64_t &SuccTrueWeight, 2935 uint64_t &SuccFalseWeight) { 2936 bool PredHasWeights = 2937 PBI->extractProfMetadata(PredTrueWeight, PredFalseWeight); 2938 bool SuccHasWeights = 2939 BI->extractProfMetadata(SuccTrueWeight, SuccFalseWeight); 2940 if (PredHasWeights || SuccHasWeights) { 2941 if (!PredHasWeights) 2942 PredTrueWeight = PredFalseWeight = 1; 2943 if (!SuccHasWeights) 2944 SuccTrueWeight = SuccFalseWeight = 1; 2945 return true; 2946 } else { 2947 return false; 2948 } 2949 } 2950 2951 /// Determine if the two branches share a common destination and deduce a glue 2952 /// that joins the branches' conditions to arrive at the common destination if 2953 /// that would be profitable. 2954 static Optional<std::pair<Instruction::BinaryOps, bool>> 2955 shouldFoldCondBranchesToCommonDestination(BranchInst *BI, BranchInst *PBI, 2956 const TargetTransformInfo *TTI) { 2957 assert(BI && PBI && BI->isConditional() && PBI->isConditional() && 2958 "Both blocks must end with a conditional branches."); 2959 assert(is_contained(predecessors(BI->getParent()), PBI->getParent()) && 2960 "PredBB must be a predecessor of BB."); 2961 2962 // We have the potential to fold the conditions together, but if the 2963 // predecessor branch is predictable, we may not want to merge them. 2964 uint64_t PTWeight, PFWeight; 2965 BranchProbability PBITrueProb, Likely; 2966 if (TTI && !PBI->getMetadata(LLVMContext::MD_unpredictable) && 2967 PBI->extractProfMetadata(PTWeight, PFWeight) && 2968 (PTWeight + PFWeight) != 0) { 2969 PBITrueProb = 2970 BranchProbability::getBranchProbability(PTWeight, PTWeight + PFWeight); 2971 Likely = TTI->getPredictableBranchThreshold(); 2972 } 2973 2974 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) { 2975 // Speculate the 2nd condition unless the 1st is probably true. 2976 if (PBITrueProb.isUnknown() || PBITrueProb < Likely) 2977 return {{Instruction::Or, false}}; 2978 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) { 2979 // Speculate the 2nd condition unless the 1st is probably false. 2980 if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely) 2981 return {{Instruction::And, false}}; 2982 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) { 2983 // Speculate the 2nd condition unless the 1st is probably true. 2984 if (PBITrueProb.isUnknown() || PBITrueProb < Likely) 2985 return {{Instruction::And, true}}; 2986 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) { 2987 // Speculate the 2nd condition unless the 1st is probably false. 2988 if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely) 2989 return {{Instruction::Or, true}}; 2990 } 2991 return None; 2992 } 2993 2994 static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI, 2995 DomTreeUpdater *DTU, 2996 MemorySSAUpdater *MSSAU, 2997 const TargetTransformInfo *TTI) { 2998 BasicBlock *BB = BI->getParent(); 2999 BasicBlock *PredBlock = PBI->getParent(); 3000 3001 // Determine if the two branches share a common destination. 3002 Instruction::BinaryOps Opc; 3003 bool InvertPredCond; 3004 std::tie(Opc, InvertPredCond) = 3005 *shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI); 3006 3007 LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB); 3008 3009 IRBuilder<> Builder(PBI); 3010 // The builder is used to create instructions to eliminate the branch in BB. 3011 // If BB's terminator has !annotation metadata, add it to the new 3012 // instructions. 3013 Builder.CollectMetadataToCopy(BB->getTerminator(), 3014 {LLVMContext::MD_annotation}); 3015 3016 // If we need to invert the condition in the pred block to match, do so now. 3017 if (InvertPredCond) { 3018 Value *NewCond = PBI->getCondition(); 3019 if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) { 3020 CmpInst *CI = cast<CmpInst>(NewCond); 3021 CI->setPredicate(CI->getInversePredicate()); 3022 } else { 3023 NewCond = 3024 Builder.CreateNot(NewCond, PBI->getCondition()->getName() + ".not"); 3025 } 3026 3027 PBI->setCondition(NewCond); 3028 PBI->swapSuccessors(); 3029 } 3030 3031 BasicBlock *UniqueSucc = 3032 PBI->getSuccessor(0) == BB ? BI->getSuccessor(0) : BI->getSuccessor(1); 3033 3034 // Before cloning instructions, notify the successor basic block that it 3035 // is about to have a new predecessor. This will update PHI nodes, 3036 // which will allow us to update live-out uses of bonus instructions. 3037 AddPredecessorToBlock(UniqueSucc, PredBlock, BB, MSSAU); 3038 3039 // Try to update branch weights. 3040 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight; 3041 if (extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight, 3042 SuccTrueWeight, SuccFalseWeight)) { 3043 SmallVector<uint64_t, 8> NewWeights; 3044 3045 if (PBI->getSuccessor(0) == BB) { 3046 // PBI: br i1 %x, BB, FalseDest 3047 // BI: br i1 %y, UniqueSucc, FalseDest 3048 // TrueWeight is TrueWeight for PBI * TrueWeight for BI. 3049 NewWeights.push_back(PredTrueWeight * SuccTrueWeight); 3050 // FalseWeight is FalseWeight for PBI * TotalWeight for BI + 3051 // TrueWeight for PBI * FalseWeight for BI. 3052 // We assume that total weights of a BranchInst can fit into 32 bits. 3053 // Therefore, we will not have overflow using 64-bit arithmetic. 3054 NewWeights.push_back(PredFalseWeight * 3055 (SuccFalseWeight + SuccTrueWeight) + 3056 PredTrueWeight * SuccFalseWeight); 3057 } else { 3058 // PBI: br i1 %x, TrueDest, BB 3059 // BI: br i1 %y, TrueDest, UniqueSucc 3060 // TrueWeight is TrueWeight for PBI * TotalWeight for BI + 3061 // FalseWeight for PBI * TrueWeight for BI. 3062 NewWeights.push_back(PredTrueWeight * (SuccFalseWeight + SuccTrueWeight) + 3063 PredFalseWeight * SuccTrueWeight); 3064 // FalseWeight is FalseWeight for PBI * FalseWeight for BI. 3065 NewWeights.push_back(PredFalseWeight * SuccFalseWeight); 3066 } 3067 3068 // Halve the weights if any of them cannot fit in an uint32_t 3069 FitWeights(NewWeights); 3070 3071 SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(), NewWeights.end()); 3072 setBranchWeights(PBI, MDWeights[0], MDWeights[1]); 3073 3074 // TODO: If BB is reachable from all paths through PredBlock, then we 3075 // could replace PBI's branch probabilities with BI's. 3076 } else 3077 PBI->setMetadata(LLVMContext::MD_prof, nullptr); 3078 3079 // Now, update the CFG. 3080 PBI->setSuccessor(PBI->getSuccessor(0) != BB, UniqueSucc); 3081 3082 if (DTU) 3083 DTU->applyUpdates({{DominatorTree::Insert, PredBlock, UniqueSucc}, 3084 {DominatorTree::Delete, PredBlock, BB}}); 3085 3086 // If BI was a loop latch, it may have had associated loop metadata. 3087 // We need to copy it to the new latch, that is, PBI. 3088 if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop)) 3089 PBI->setMetadata(LLVMContext::MD_loop, LoopMD); 3090 3091 ValueToValueMapTy VMap; // maps original values to cloned values 3092 CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(BB, PredBlock, VMap); 3093 3094 // Now that the Cond was cloned into the predecessor basic block, 3095 // or/and the two conditions together. 3096 Value *BICond = VMap[BI->getCondition()]; 3097 PBI->setCondition( 3098 createLogicalOp(Builder, Opc, PBI->getCondition(), BICond, "or.cond")); 3099 3100 // Copy any debug value intrinsics into the end of PredBlock. 3101 for (Instruction &I : *BB) { 3102 if (isa<DbgInfoIntrinsic>(I)) { 3103 Instruction *NewI = I.clone(); 3104 RemapInstruction(NewI, VMap, 3105 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals); 3106 NewI->insertBefore(PBI); 3107 } 3108 } 3109 3110 ++NumFoldBranchToCommonDest; 3111 return true; 3112 } 3113 3114 /// If this basic block is simple enough, and if a predecessor branches to us 3115 /// and one of our successors, fold the block into the predecessor and use 3116 /// logical operations to pick the right destination. 3117 bool llvm::FoldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU, 3118 MemorySSAUpdater *MSSAU, 3119 const TargetTransformInfo *TTI, 3120 unsigned BonusInstThreshold) { 3121 // If this block ends with an unconditional branch, 3122 // let SpeculativelyExecuteBB() deal with it. 3123 if (!BI->isConditional()) 3124 return false; 3125 3126 BasicBlock *BB = BI->getParent(); 3127 TargetTransformInfo::TargetCostKind CostKind = 3128 BB->getParent()->hasMinSize() ? TargetTransformInfo::TCK_CodeSize 3129 : TargetTransformInfo::TCK_SizeAndLatency; 3130 3131 Instruction *Cond = dyn_cast<Instruction>(BI->getCondition()); 3132 3133 if (!Cond || (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond)) || 3134 Cond->getParent() != BB || !Cond->hasOneUse()) 3135 return false; 3136 3137 // Cond is known to be a compare or binary operator. Check to make sure that 3138 // neither operand is a potentially-trapping constant expression. 3139 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(0))) 3140 if (CE->canTrap()) 3141 return false; 3142 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(1))) 3143 if (CE->canTrap()) 3144 return false; 3145 3146 // Finally, don't infinitely unroll conditional loops. 3147 if (is_contained(successors(BB), BB)) 3148 return false; 3149 3150 // With which predecessors will we want to deal with? 3151 SmallVector<BasicBlock *, 8> Preds; 3152 for (BasicBlock *PredBlock : predecessors(BB)) { 3153 BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator()); 3154 3155 // Check that we have two conditional branches. If there is a PHI node in 3156 // the common successor, verify that the same value flows in from both 3157 // blocks. 3158 if (!PBI || PBI->isUnconditional() || !SafeToMergeTerminators(BI, PBI)) 3159 continue; 3160 3161 // Determine if the two branches share a common destination. 3162 Instruction::BinaryOps Opc; 3163 bool InvertPredCond; 3164 if (auto Recipe = shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI)) 3165 std::tie(Opc, InvertPredCond) = *Recipe; 3166 else 3167 continue; 3168 3169 // Check the cost of inserting the necessary logic before performing the 3170 // transformation. 3171 if (TTI) { 3172 Type *Ty = BI->getCondition()->getType(); 3173 InstructionCost Cost = TTI->getArithmeticInstrCost(Opc, Ty, CostKind); 3174 if (InvertPredCond && (!PBI->getCondition()->hasOneUse() || 3175 !isa<CmpInst>(PBI->getCondition()))) 3176 Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind); 3177 3178 if (Cost > BranchFoldThreshold) 3179 continue; 3180 } 3181 3182 // Ok, we do want to deal with this predecessor. Record it. 3183 Preds.emplace_back(PredBlock); 3184 } 3185 3186 // If there aren't any predecessors into which we can fold, 3187 // don't bother checking the cost. 3188 if (Preds.empty()) 3189 return false; 3190 3191 // Only allow this transformation if computing the condition doesn't involve 3192 // too many instructions and these involved instructions can be executed 3193 // unconditionally. We denote all involved instructions except the condition 3194 // as "bonus instructions", and only allow this transformation when the 3195 // number of the bonus instructions we'll need to create when cloning into 3196 // each predecessor does not exceed a certain threshold. 3197 unsigned NumBonusInsts = 0; 3198 const unsigned PredCount = Preds.size(); 3199 for (Instruction &I : *BB) { 3200 // Don't check the branch condition comparison itself. 3201 if (&I == Cond) 3202 continue; 3203 // Ignore dbg intrinsics, and the terminator. 3204 if (isa<DbgInfoIntrinsic>(I) || isa<BranchInst>(I)) 3205 continue; 3206 // I must be safe to execute unconditionally. 3207 if (!isSafeToSpeculativelyExecute(&I)) 3208 return false; 3209 3210 // Account for the cost of duplicating this instruction into each 3211 // predecessor. 3212 NumBonusInsts += PredCount; 3213 // Early exits once we reach the limit. 3214 if (NumBonusInsts > BonusInstThreshold) 3215 return false; 3216 3217 auto IsBCSSAUse = [BB, &I](Use &U) { 3218 auto *UI = cast<Instruction>(U.getUser()); 3219 if (auto *PN = dyn_cast<PHINode>(UI)) 3220 return PN->getIncomingBlock(U) == BB; 3221 return UI->getParent() == BB && I.comesBefore(UI); 3222 }; 3223 3224 // Does this instruction require rewriting of uses? 3225 if (!all_of(I.uses(), IsBCSSAUse)) 3226 return false; 3227 } 3228 3229 // Ok, we have the budget. Perform the transformation. 3230 for (BasicBlock *PredBlock : Preds) { 3231 auto *PBI = cast<BranchInst>(PredBlock->getTerminator()); 3232 return performBranchToCommonDestFolding(BI, PBI, DTU, MSSAU, TTI); 3233 } 3234 return false; 3235 } 3236 3237 // If there is only one store in BB1 and BB2, return it, otherwise return 3238 // nullptr. 3239 static StoreInst *findUniqueStoreInBlocks(BasicBlock *BB1, BasicBlock *BB2) { 3240 StoreInst *S = nullptr; 3241 for (auto *BB : {BB1, BB2}) { 3242 if (!BB) 3243 continue; 3244 for (auto &I : *BB) 3245 if (auto *SI = dyn_cast<StoreInst>(&I)) { 3246 if (S) 3247 // Multiple stores seen. 3248 return nullptr; 3249 else 3250 S = SI; 3251 } 3252 } 3253 return S; 3254 } 3255 3256 static Value *ensureValueAvailableInSuccessor(Value *V, BasicBlock *BB, 3257 Value *AlternativeV = nullptr) { 3258 // PHI is going to be a PHI node that allows the value V that is defined in 3259 // BB to be referenced in BB's only successor. 3260 // 3261 // If AlternativeV is nullptr, the only value we care about in PHI is V. It 3262 // doesn't matter to us what the other operand is (it'll never get used). We 3263 // could just create a new PHI with an undef incoming value, but that could 3264 // increase register pressure if EarlyCSE/InstCombine can't fold it with some 3265 // other PHI. So here we directly look for some PHI in BB's successor with V 3266 // as an incoming operand. If we find one, we use it, else we create a new 3267 // one. 3268 // 3269 // If AlternativeV is not nullptr, we care about both incoming values in PHI. 3270 // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV] 3271 // where OtherBB is the single other predecessor of BB's only successor. 3272 PHINode *PHI = nullptr; 3273 BasicBlock *Succ = BB->getSingleSuccessor(); 3274 3275 for (auto I = Succ->begin(); isa<PHINode>(I); ++I) 3276 if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) { 3277 PHI = cast<PHINode>(I); 3278 if (!AlternativeV) 3279 break; 3280 3281 assert(Succ->hasNPredecessors(2)); 3282 auto PredI = pred_begin(Succ); 3283 BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI; 3284 if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV) 3285 break; 3286 PHI = nullptr; 3287 } 3288 if (PHI) 3289 return PHI; 3290 3291 // If V is not an instruction defined in BB, just return it. 3292 if (!AlternativeV && 3293 (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB)) 3294 return V; 3295 3296 PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge", &Succ->front()); 3297 PHI->addIncoming(V, BB); 3298 for (BasicBlock *PredBB : predecessors(Succ)) 3299 if (PredBB != BB) 3300 PHI->addIncoming( 3301 AlternativeV ? AlternativeV : UndefValue::get(V->getType()), PredBB); 3302 return PHI; 3303 } 3304 3305 static bool mergeConditionalStoreToAddress( 3306 BasicBlock *PTB, BasicBlock *PFB, BasicBlock *QTB, BasicBlock *QFB, 3307 BasicBlock *PostBB, Value *Address, bool InvertPCond, bool InvertQCond, 3308 DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI) { 3309 // For every pointer, there must be exactly two stores, one coming from 3310 // PTB or PFB, and the other from QTB or QFB. We don't support more than one 3311 // store (to any address) in PTB,PFB or QTB,QFB. 3312 // FIXME: We could relax this restriction with a bit more work and performance 3313 // testing. 3314 StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB); 3315 StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB); 3316 if (!PStore || !QStore) 3317 return false; 3318 3319 // Now check the stores are compatible. 3320 if (!QStore->isUnordered() || !PStore->isUnordered()) 3321 return false; 3322 3323 // Check that sinking the store won't cause program behavior changes. Sinking 3324 // the store out of the Q blocks won't change any behavior as we're sinking 3325 // from a block to its unconditional successor. But we're moving a store from 3326 // the P blocks down through the middle block (QBI) and past both QFB and QTB. 3327 // So we need to check that there are no aliasing loads or stores in 3328 // QBI, QTB and QFB. We also need to check there are no conflicting memory 3329 // operations between PStore and the end of its parent block. 3330 // 3331 // The ideal way to do this is to query AliasAnalysis, but we don't 3332 // preserve AA currently so that is dangerous. Be super safe and just 3333 // check there are no other memory operations at all. 3334 for (auto &I : *QFB->getSinglePredecessor()) 3335 if (I.mayReadOrWriteMemory()) 3336 return false; 3337 for (auto &I : *QFB) 3338 if (&I != QStore && I.mayReadOrWriteMemory()) 3339 return false; 3340 if (QTB) 3341 for (auto &I : *QTB) 3342 if (&I != QStore && I.mayReadOrWriteMemory()) 3343 return false; 3344 for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end(); 3345 I != E; ++I) 3346 if (&*I != PStore && I->mayReadOrWriteMemory()) 3347 return false; 3348 3349 // If we're not in aggressive mode, we only optimize if we have some 3350 // confidence that by optimizing we'll allow P and/or Q to be if-converted. 3351 auto IsWorthwhile = [&](BasicBlock *BB, ArrayRef<StoreInst *> FreeStores) { 3352 if (!BB) 3353 return true; 3354 // Heuristic: if the block can be if-converted/phi-folded and the 3355 // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to 3356 // thread this store. 3357 InstructionCost Cost = 0; 3358 InstructionCost Budget = 3359 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; 3360 for (auto &I : BB->instructionsWithoutDebug()) { 3361 // Consider terminator instruction to be free. 3362 if (I.isTerminator()) 3363 continue; 3364 // If this is one the stores that we want to speculate out of this BB, 3365 // then don't count it's cost, consider it to be free. 3366 if (auto *S = dyn_cast<StoreInst>(&I)) 3367 if (llvm::find(FreeStores, S)) 3368 continue; 3369 // Else, we have a white-list of instructions that we are ak speculating. 3370 if (!isa<BinaryOperator>(I) && !isa<GetElementPtrInst>(I)) 3371 return false; // Not in white-list - not worthwhile folding. 3372 // And finally, if this is a non-free instruction that we are okay 3373 // speculating, ensure that we consider the speculation budget. 3374 Cost += TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency); 3375 if (Cost > Budget) 3376 return false; // Eagerly refuse to fold as soon as we're out of budget. 3377 } 3378 assert(Cost <= Budget && 3379 "When we run out of budget we will eagerly return from within the " 3380 "per-instruction loop."); 3381 return true; 3382 }; 3383 3384 const std::array<StoreInst *, 2> FreeStores = {PStore, QStore}; 3385 if (!MergeCondStoresAggressively && 3386 (!IsWorthwhile(PTB, FreeStores) || !IsWorthwhile(PFB, FreeStores) || 3387 !IsWorthwhile(QTB, FreeStores) || !IsWorthwhile(QFB, FreeStores))) 3388 return false; 3389 3390 // If PostBB has more than two predecessors, we need to split it so we can 3391 // sink the store. 3392 if (std::next(pred_begin(PostBB), 2) != pred_end(PostBB)) { 3393 // We know that QFB's only successor is PostBB. And QFB has a single 3394 // predecessor. If QTB exists, then its only successor is also PostBB. 3395 // If QTB does not exist, then QFB's only predecessor has a conditional 3396 // branch to QFB and PostBB. 3397 BasicBlock *TruePred = QTB ? QTB : QFB->getSinglePredecessor(); 3398 BasicBlock *NewBB = 3399 SplitBlockPredecessors(PostBB, {QFB, TruePred}, "condstore.split", DTU); 3400 if (!NewBB) 3401 return false; 3402 PostBB = NewBB; 3403 } 3404 3405 // OK, we're going to sink the stores to PostBB. The store has to be 3406 // conditional though, so first create the predicate. 3407 Value *PCond = cast<BranchInst>(PFB->getSinglePredecessor()->getTerminator()) 3408 ->getCondition(); 3409 Value *QCond = cast<BranchInst>(QFB->getSinglePredecessor()->getTerminator()) 3410 ->getCondition(); 3411 3412 Value *PPHI = ensureValueAvailableInSuccessor(PStore->getValueOperand(), 3413 PStore->getParent()); 3414 Value *QPHI = ensureValueAvailableInSuccessor(QStore->getValueOperand(), 3415 QStore->getParent(), PPHI); 3416 3417 IRBuilder<> QB(&*PostBB->getFirstInsertionPt()); 3418 3419 Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond); 3420 Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond); 3421 3422 if (InvertPCond) 3423 PPred = QB.CreateNot(PPred); 3424 if (InvertQCond) 3425 QPred = QB.CreateNot(QPred); 3426 Value *CombinedPred = QB.CreateOr(PPred, QPred); 3427 3428 auto *T = SplitBlockAndInsertIfThen(CombinedPred, &*QB.GetInsertPoint(), 3429 /*Unreachable=*/false, 3430 /*BranchWeights=*/nullptr, DTU); 3431 QB.SetInsertPoint(T); 3432 StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address)); 3433 AAMDNodes AAMD; 3434 PStore->getAAMetadata(AAMD, /*Merge=*/false); 3435 PStore->getAAMetadata(AAMD, /*Merge=*/true); 3436 SI->setAAMetadata(AAMD); 3437 // Choose the minimum alignment. If we could prove both stores execute, we 3438 // could use biggest one. In this case, though, we only know that one of the 3439 // stores executes. And we don't know it's safe to take the alignment from a 3440 // store that doesn't execute. 3441 SI->setAlignment(std::min(PStore->getAlign(), QStore->getAlign())); 3442 3443 QStore->eraseFromParent(); 3444 PStore->eraseFromParent(); 3445 3446 return true; 3447 } 3448 3449 static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI, 3450 DomTreeUpdater *DTU, const DataLayout &DL, 3451 const TargetTransformInfo &TTI) { 3452 // The intention here is to find diamonds or triangles (see below) where each 3453 // conditional block contains a store to the same address. Both of these 3454 // stores are conditional, so they can't be unconditionally sunk. But it may 3455 // be profitable to speculatively sink the stores into one merged store at the 3456 // end, and predicate the merged store on the union of the two conditions of 3457 // PBI and QBI. 3458 // 3459 // This can reduce the number of stores executed if both of the conditions are 3460 // true, and can allow the blocks to become small enough to be if-converted. 3461 // This optimization will also chain, so that ladders of test-and-set 3462 // sequences can be if-converted away. 3463 // 3464 // We only deal with simple diamonds or triangles: 3465 // 3466 // PBI or PBI or a combination of the two 3467 // / \ | \ 3468 // PTB PFB | PFB 3469 // \ / | / 3470 // QBI QBI 3471 // / \ | \ 3472 // QTB QFB | QFB 3473 // \ / | / 3474 // PostBB PostBB 3475 // 3476 // We model triangles as a type of diamond with a nullptr "true" block. 3477 // Triangles are canonicalized so that the fallthrough edge is represented by 3478 // a true condition, as in the diagram above. 3479 BasicBlock *PTB = PBI->getSuccessor(0); 3480 BasicBlock *PFB = PBI->getSuccessor(1); 3481 BasicBlock *QTB = QBI->getSuccessor(0); 3482 BasicBlock *QFB = QBI->getSuccessor(1); 3483 BasicBlock *PostBB = QFB->getSingleSuccessor(); 3484 3485 // Make sure we have a good guess for PostBB. If QTB's only successor is 3486 // QFB, then QFB is a better PostBB. 3487 if (QTB->getSingleSuccessor() == QFB) 3488 PostBB = QFB; 3489 3490 // If we couldn't find a good PostBB, stop. 3491 if (!PostBB) 3492 return false; 3493 3494 bool InvertPCond = false, InvertQCond = false; 3495 // Canonicalize fallthroughs to the true branches. 3496 if (PFB == QBI->getParent()) { 3497 std::swap(PFB, PTB); 3498 InvertPCond = true; 3499 } 3500 if (QFB == PostBB) { 3501 std::swap(QFB, QTB); 3502 InvertQCond = true; 3503 } 3504 3505 // From this point on we can assume PTB or QTB may be fallthroughs but PFB 3506 // and QFB may not. Model fallthroughs as a nullptr block. 3507 if (PTB == QBI->getParent()) 3508 PTB = nullptr; 3509 if (QTB == PostBB) 3510 QTB = nullptr; 3511 3512 // Legality bailouts. We must have at least the non-fallthrough blocks and 3513 // the post-dominating block, and the non-fallthroughs must only have one 3514 // predecessor. 3515 auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) { 3516 return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S; 3517 }; 3518 if (!HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) || 3519 !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB)) 3520 return false; 3521 if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) || 3522 (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB))) 3523 return false; 3524 if (!QBI->getParent()->hasNUses(2)) 3525 return false; 3526 3527 // OK, this is a sequence of two diamonds or triangles. 3528 // Check if there are stores in PTB or PFB that are repeated in QTB or QFB. 3529 SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses; 3530 for (auto *BB : {PTB, PFB}) { 3531 if (!BB) 3532 continue; 3533 for (auto &I : *BB) 3534 if (StoreInst *SI = dyn_cast<StoreInst>(&I)) 3535 PStoreAddresses.insert(SI->getPointerOperand()); 3536 } 3537 for (auto *BB : {QTB, QFB}) { 3538 if (!BB) 3539 continue; 3540 for (auto &I : *BB) 3541 if (StoreInst *SI = dyn_cast<StoreInst>(&I)) 3542 QStoreAddresses.insert(SI->getPointerOperand()); 3543 } 3544 3545 set_intersect(PStoreAddresses, QStoreAddresses); 3546 // set_intersect mutates PStoreAddresses in place. Rename it here to make it 3547 // clear what it contains. 3548 auto &CommonAddresses = PStoreAddresses; 3549 3550 bool Changed = false; 3551 for (auto *Address : CommonAddresses) 3552 Changed |= 3553 mergeConditionalStoreToAddress(PTB, PFB, QTB, QFB, PostBB, Address, 3554 InvertPCond, InvertQCond, DTU, DL, TTI); 3555 return Changed; 3556 } 3557 3558 /// If the previous block ended with a widenable branch, determine if reusing 3559 /// the target block is profitable and legal. This will have the effect of 3560 /// "widening" PBI, but doesn't require us to reason about hosting safety. 3561 static bool tryWidenCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI, 3562 DomTreeUpdater *DTU) { 3563 // TODO: This can be generalized in two important ways: 3564 // 1) We can allow phi nodes in IfFalseBB and simply reuse all the input 3565 // values from the PBI edge. 3566 // 2) We can sink side effecting instructions into BI's fallthrough 3567 // successor provided they doesn't contribute to computation of 3568 // BI's condition. 3569 Value *CondWB, *WC; 3570 BasicBlock *IfTrueBB, *IfFalseBB; 3571 if (!parseWidenableBranch(PBI, CondWB, WC, IfTrueBB, IfFalseBB) || 3572 IfTrueBB != BI->getParent() || !BI->getParent()->getSinglePredecessor()) 3573 return false; 3574 if (!IfFalseBB->phis().empty()) 3575 return false; // TODO 3576 // Use lambda to lazily compute expensive condition after cheap ones. 3577 auto NoSideEffects = [](BasicBlock &BB) { 3578 return !llvm::any_of(BB, [](const Instruction &I) { 3579 return I.mayWriteToMemory() || I.mayHaveSideEffects(); 3580 }); 3581 }; 3582 if (BI->getSuccessor(1) != IfFalseBB && // no inf looping 3583 BI->getSuccessor(1)->getTerminatingDeoptimizeCall() && // profitability 3584 NoSideEffects(*BI->getParent())) { 3585 auto *OldSuccessor = BI->getSuccessor(1); 3586 OldSuccessor->removePredecessor(BI->getParent()); 3587 BI->setSuccessor(1, IfFalseBB); 3588 if (DTU) 3589 DTU->applyUpdates( 3590 {{DominatorTree::Insert, BI->getParent(), IfFalseBB}, 3591 {DominatorTree::Delete, BI->getParent(), OldSuccessor}}); 3592 return true; 3593 } 3594 if (BI->getSuccessor(0) != IfFalseBB && // no inf looping 3595 BI->getSuccessor(0)->getTerminatingDeoptimizeCall() && // profitability 3596 NoSideEffects(*BI->getParent())) { 3597 auto *OldSuccessor = BI->getSuccessor(0); 3598 OldSuccessor->removePredecessor(BI->getParent()); 3599 BI->setSuccessor(0, IfFalseBB); 3600 if (DTU) 3601 DTU->applyUpdates( 3602 {{DominatorTree::Insert, BI->getParent(), IfFalseBB}, 3603 {DominatorTree::Delete, BI->getParent(), OldSuccessor}}); 3604 return true; 3605 } 3606 return false; 3607 } 3608 3609 /// If we have a conditional branch as a predecessor of another block, 3610 /// this function tries to simplify it. We know 3611 /// that PBI and BI are both conditional branches, and BI is in one of the 3612 /// successor blocks of PBI - PBI branches to BI. 3613 static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI, 3614 DomTreeUpdater *DTU, 3615 const DataLayout &DL, 3616 const TargetTransformInfo &TTI) { 3617 assert(PBI->isConditional() && BI->isConditional()); 3618 BasicBlock *BB = BI->getParent(); 3619 3620 // If this block ends with a branch instruction, and if there is a 3621 // predecessor that ends on a branch of the same condition, make 3622 // this conditional branch redundant. 3623 if (PBI->getCondition() == BI->getCondition() && 3624 PBI->getSuccessor(0) != PBI->getSuccessor(1)) { 3625 // Okay, the outcome of this conditional branch is statically 3626 // knowable. If this block had a single pred, handle specially. 3627 if (BB->getSinglePredecessor()) { 3628 // Turn this into a branch on constant. 3629 bool CondIsTrue = PBI->getSuccessor(0) == BB; 3630 BI->setCondition( 3631 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue)); 3632 return true; // Nuke the branch on constant. 3633 } 3634 3635 // Otherwise, if there are multiple predecessors, insert a PHI that merges 3636 // in the constant and simplify the block result. Subsequent passes of 3637 // simplifycfg will thread the block. 3638 if (BlockIsSimpleEnoughToThreadThrough(BB)) { 3639 pred_iterator PB = pred_begin(BB), PE = pred_end(BB); 3640 PHINode *NewPN = PHINode::Create( 3641 Type::getInt1Ty(BB->getContext()), std::distance(PB, PE), 3642 BI->getCondition()->getName() + ".pr", &BB->front()); 3643 // Okay, we're going to insert the PHI node. Since PBI is not the only 3644 // predecessor, compute the PHI'd conditional value for all of the preds. 3645 // Any predecessor where the condition is not computable we keep symbolic. 3646 for (pred_iterator PI = PB; PI != PE; ++PI) { 3647 BasicBlock *P = *PI; 3648 if ((PBI = dyn_cast<BranchInst>(P->getTerminator())) && PBI != BI && 3649 PBI->isConditional() && PBI->getCondition() == BI->getCondition() && 3650 PBI->getSuccessor(0) != PBI->getSuccessor(1)) { 3651 bool CondIsTrue = PBI->getSuccessor(0) == BB; 3652 NewPN->addIncoming( 3653 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue), 3654 P); 3655 } else { 3656 NewPN->addIncoming(BI->getCondition(), P); 3657 } 3658 } 3659 3660 BI->setCondition(NewPN); 3661 return true; 3662 } 3663 } 3664 3665 // If the previous block ended with a widenable branch, determine if reusing 3666 // the target block is profitable and legal. This will have the effect of 3667 // "widening" PBI, but doesn't require us to reason about hosting safety. 3668 if (tryWidenCondBranchToCondBranch(PBI, BI, DTU)) 3669 return true; 3670 3671 if (auto *CE = dyn_cast<ConstantExpr>(BI->getCondition())) 3672 if (CE->canTrap()) 3673 return false; 3674 3675 // If both branches are conditional and both contain stores to the same 3676 // address, remove the stores from the conditionals and create a conditional 3677 // merged store at the end. 3678 if (MergeCondStores && mergeConditionalStores(PBI, BI, DTU, DL, TTI)) 3679 return true; 3680 3681 // If this is a conditional branch in an empty block, and if any 3682 // predecessors are a conditional branch to one of our destinations, 3683 // fold the conditions into logical ops and one cond br. 3684 3685 // Ignore dbg intrinsics. 3686 if (&*BB->instructionsWithoutDebug().begin() != BI) 3687 return false; 3688 3689 int PBIOp, BIOp; 3690 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) { 3691 PBIOp = 0; 3692 BIOp = 0; 3693 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) { 3694 PBIOp = 0; 3695 BIOp = 1; 3696 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) { 3697 PBIOp = 1; 3698 BIOp = 0; 3699 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) { 3700 PBIOp = 1; 3701 BIOp = 1; 3702 } else { 3703 return false; 3704 } 3705 3706 // Check to make sure that the other destination of this branch 3707 // isn't BB itself. If so, this is an infinite loop that will 3708 // keep getting unwound. 3709 if (PBI->getSuccessor(PBIOp) == BB) 3710 return false; 3711 3712 // Do not perform this transformation if it would require 3713 // insertion of a large number of select instructions. For targets 3714 // without predication/cmovs, this is a big pessimization. 3715 3716 // Also do not perform this transformation if any phi node in the common 3717 // destination block can trap when reached by BB or PBB (PR17073). In that 3718 // case, it would be unsafe to hoist the operation into a select instruction. 3719 3720 BasicBlock *CommonDest = PBI->getSuccessor(PBIOp); 3721 BasicBlock *RemovedDest = PBI->getSuccessor(PBIOp ^ 1); 3722 unsigned NumPhis = 0; 3723 for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II); 3724 ++II, ++NumPhis) { 3725 if (NumPhis > 2) // Disable this xform. 3726 return false; 3727 3728 PHINode *PN = cast<PHINode>(II); 3729 Value *BIV = PN->getIncomingValueForBlock(BB); 3730 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(BIV)) 3731 if (CE->canTrap()) 3732 return false; 3733 3734 unsigned PBBIdx = PN->getBasicBlockIndex(PBI->getParent()); 3735 Value *PBIV = PN->getIncomingValue(PBBIdx); 3736 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(PBIV)) 3737 if (CE->canTrap()) 3738 return false; 3739 } 3740 3741 // Finally, if everything is ok, fold the branches to logical ops. 3742 BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1); 3743 3744 LLVM_DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent() 3745 << "AND: " << *BI->getParent()); 3746 3747 SmallVector<DominatorTree::UpdateType, 5> Updates; 3748 3749 // If OtherDest *is* BB, then BB is a basic block with a single conditional 3750 // branch in it, where one edge (OtherDest) goes back to itself but the other 3751 // exits. We don't *know* that the program avoids the infinite loop 3752 // (even though that seems likely). If we do this xform naively, we'll end up 3753 // recursively unpeeling the loop. Since we know that (after the xform is 3754 // done) that the block *is* infinite if reached, we just make it an obviously 3755 // infinite loop with no cond branch. 3756 if (OtherDest == BB) { 3757 // Insert it at the end of the function, because it's either code, 3758 // or it won't matter if it's hot. :) 3759 BasicBlock *InfLoopBlock = 3760 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent()); 3761 BranchInst::Create(InfLoopBlock, InfLoopBlock); 3762 if (DTU) 3763 Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock}); 3764 OtherDest = InfLoopBlock; 3765 } 3766 3767 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent()); 3768 3769 // BI may have other predecessors. Because of this, we leave 3770 // it alone, but modify PBI. 3771 3772 // Make sure we get to CommonDest on True&True directions. 3773 Value *PBICond = PBI->getCondition(); 3774 IRBuilder<NoFolder> Builder(PBI); 3775 if (PBIOp) 3776 PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not"); 3777 3778 Value *BICond = BI->getCondition(); 3779 if (BIOp) 3780 BICond = Builder.CreateNot(BICond, BICond->getName() + ".not"); 3781 3782 // Merge the conditions. 3783 Value *Cond = 3784 createLogicalOp(Builder, Instruction::Or, PBICond, BICond, "brmerge"); 3785 3786 // Modify PBI to branch on the new condition to the new dests. 3787 PBI->setCondition(Cond); 3788 PBI->setSuccessor(0, CommonDest); 3789 PBI->setSuccessor(1, OtherDest); 3790 3791 if (DTU) { 3792 Updates.push_back({DominatorTree::Insert, PBI->getParent(), OtherDest}); 3793 Updates.push_back({DominatorTree::Delete, PBI->getParent(), RemovedDest}); 3794 3795 DTU->applyUpdates(Updates); 3796 } 3797 3798 // Update branch weight for PBI. 3799 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight; 3800 uint64_t PredCommon, PredOther, SuccCommon, SuccOther; 3801 bool HasWeights = 3802 extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight, 3803 SuccTrueWeight, SuccFalseWeight); 3804 if (HasWeights) { 3805 PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight; 3806 PredOther = PBIOp ? PredTrueWeight : PredFalseWeight; 3807 SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight; 3808 SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight; 3809 // The weight to CommonDest should be PredCommon * SuccTotal + 3810 // PredOther * SuccCommon. 3811 // The weight to OtherDest should be PredOther * SuccOther. 3812 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) + 3813 PredOther * SuccCommon, 3814 PredOther * SuccOther}; 3815 // Halve the weights if any of them cannot fit in an uint32_t 3816 FitWeights(NewWeights); 3817 3818 setBranchWeights(PBI, NewWeights[0], NewWeights[1]); 3819 } 3820 3821 // OtherDest may have phi nodes. If so, add an entry from PBI's 3822 // block that are identical to the entries for BI's block. 3823 AddPredecessorToBlock(OtherDest, PBI->getParent(), BB); 3824 3825 // We know that the CommonDest already had an edge from PBI to 3826 // it. If it has PHIs though, the PHIs may have different 3827 // entries for BB and PBI's BB. If so, insert a select to make 3828 // them agree. 3829 for (PHINode &PN : CommonDest->phis()) { 3830 Value *BIV = PN.getIncomingValueForBlock(BB); 3831 unsigned PBBIdx = PN.getBasicBlockIndex(PBI->getParent()); 3832 Value *PBIV = PN.getIncomingValue(PBBIdx); 3833 if (BIV != PBIV) { 3834 // Insert a select in PBI to pick the right value. 3835 SelectInst *NV = cast<SelectInst>( 3836 Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux")); 3837 PN.setIncomingValue(PBBIdx, NV); 3838 // Although the select has the same condition as PBI, the original branch 3839 // weights for PBI do not apply to the new select because the select's 3840 // 'logical' edges are incoming edges of the phi that is eliminated, not 3841 // the outgoing edges of PBI. 3842 if (HasWeights) { 3843 uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight; 3844 uint64_t PredOther = PBIOp ? PredTrueWeight : PredFalseWeight; 3845 uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight; 3846 uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight; 3847 // The weight to PredCommonDest should be PredCommon * SuccTotal. 3848 // The weight to PredOtherDest should be PredOther * SuccCommon. 3849 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther), 3850 PredOther * SuccCommon}; 3851 3852 FitWeights(NewWeights); 3853 3854 setBranchWeights(NV, NewWeights[0], NewWeights[1]); 3855 } 3856 } 3857 } 3858 3859 LLVM_DEBUG(dbgs() << "INTO: " << *PBI->getParent()); 3860 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent()); 3861 3862 // This basic block is probably dead. We know it has at least 3863 // one fewer predecessor. 3864 return true; 3865 } 3866 3867 // Simplifies a terminator by replacing it with a branch to TrueBB if Cond is 3868 // true or to FalseBB if Cond is false. 3869 // Takes care of updating the successors and removing the old terminator. 3870 // Also makes sure not to introduce new successors by assuming that edges to 3871 // non-successor TrueBBs and FalseBBs aren't reachable. 3872 bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm, 3873 Value *Cond, BasicBlock *TrueBB, 3874 BasicBlock *FalseBB, 3875 uint32_t TrueWeight, 3876 uint32_t FalseWeight) { 3877 auto *BB = OldTerm->getParent(); 3878 // Remove any superfluous successor edges from the CFG. 3879 // First, figure out which successors to preserve. 3880 // If TrueBB and FalseBB are equal, only try to preserve one copy of that 3881 // successor. 3882 BasicBlock *KeepEdge1 = TrueBB; 3883 BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr; 3884 3885 SmallPtrSet<BasicBlock *, 2> RemovedSuccessors; 3886 3887 // Then remove the rest. 3888 for (BasicBlock *Succ : successors(OldTerm)) { 3889 // Make sure only to keep exactly one copy of each edge. 3890 if (Succ == KeepEdge1) 3891 KeepEdge1 = nullptr; 3892 else if (Succ == KeepEdge2) 3893 KeepEdge2 = nullptr; 3894 else { 3895 Succ->removePredecessor(BB, 3896 /*KeepOneInputPHIs=*/true); 3897 3898 if (Succ != TrueBB && Succ != FalseBB) 3899 RemovedSuccessors.insert(Succ); 3900 } 3901 } 3902 3903 IRBuilder<> Builder(OldTerm); 3904 Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc()); 3905 3906 // Insert an appropriate new terminator. 3907 if (!KeepEdge1 && !KeepEdge2) { 3908 if (TrueBB == FalseBB) { 3909 // We were only looking for one successor, and it was present. 3910 // Create an unconditional branch to it. 3911 Builder.CreateBr(TrueBB); 3912 } else { 3913 // We found both of the successors we were looking for. 3914 // Create a conditional branch sharing the condition of the select. 3915 BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB); 3916 if (TrueWeight != FalseWeight) 3917 setBranchWeights(NewBI, TrueWeight, FalseWeight); 3918 } 3919 } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) { 3920 // Neither of the selected blocks were successors, so this 3921 // terminator must be unreachable. 3922 new UnreachableInst(OldTerm->getContext(), OldTerm); 3923 } else { 3924 // One of the selected values was a successor, but the other wasn't. 3925 // Insert an unconditional branch to the one that was found; 3926 // the edge to the one that wasn't must be unreachable. 3927 if (!KeepEdge1) { 3928 // Only TrueBB was found. 3929 Builder.CreateBr(TrueBB); 3930 } else { 3931 // Only FalseBB was found. 3932 Builder.CreateBr(FalseBB); 3933 } 3934 } 3935 3936 EraseTerminatorAndDCECond(OldTerm); 3937 3938 if (DTU) { 3939 SmallVector<DominatorTree::UpdateType, 2> Updates; 3940 Updates.reserve(RemovedSuccessors.size()); 3941 for (auto *RemovedSuccessor : RemovedSuccessors) 3942 Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor}); 3943 DTU->applyUpdates(Updates); 3944 } 3945 3946 return true; 3947 } 3948 3949 // Replaces 3950 // (switch (select cond, X, Y)) on constant X, Y 3951 // with a branch - conditional if X and Y lead to distinct BBs, 3952 // unconditional otherwise. 3953 bool SimplifyCFGOpt::SimplifySwitchOnSelect(SwitchInst *SI, 3954 SelectInst *Select) { 3955 // Check for constant integer values in the select. 3956 ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue()); 3957 ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue()); 3958 if (!TrueVal || !FalseVal) 3959 return false; 3960 3961 // Find the relevant condition and destinations. 3962 Value *Condition = Select->getCondition(); 3963 BasicBlock *TrueBB = SI->findCaseValue(TrueVal)->getCaseSuccessor(); 3964 BasicBlock *FalseBB = SI->findCaseValue(FalseVal)->getCaseSuccessor(); 3965 3966 // Get weight for TrueBB and FalseBB. 3967 uint32_t TrueWeight = 0, FalseWeight = 0; 3968 SmallVector<uint64_t, 8> Weights; 3969 bool HasWeights = HasBranchWeights(SI); 3970 if (HasWeights) { 3971 GetBranchWeights(SI, Weights); 3972 if (Weights.size() == 1 + SI->getNumCases()) { 3973 TrueWeight = 3974 (uint32_t)Weights[SI->findCaseValue(TrueVal)->getSuccessorIndex()]; 3975 FalseWeight = 3976 (uint32_t)Weights[SI->findCaseValue(FalseVal)->getSuccessorIndex()]; 3977 } 3978 } 3979 3980 // Perform the actual simplification. 3981 return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight, 3982 FalseWeight); 3983 } 3984 3985 // Replaces 3986 // (indirectbr (select cond, blockaddress(@fn, BlockA), 3987 // blockaddress(@fn, BlockB))) 3988 // with 3989 // (br cond, BlockA, BlockB). 3990 bool SimplifyCFGOpt::SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, 3991 SelectInst *SI) { 3992 // Check that both operands of the select are block addresses. 3993 BlockAddress *TBA = dyn_cast<BlockAddress>(SI->getTrueValue()); 3994 BlockAddress *FBA = dyn_cast<BlockAddress>(SI->getFalseValue()); 3995 if (!TBA || !FBA) 3996 return false; 3997 3998 // Extract the actual blocks. 3999 BasicBlock *TrueBB = TBA->getBasicBlock(); 4000 BasicBlock *FalseBB = FBA->getBasicBlock(); 4001 4002 // Perform the actual simplification. 4003 return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB, 0, 4004 0); 4005 } 4006 4007 /// This is called when we find an icmp instruction 4008 /// (a seteq/setne with a constant) as the only instruction in a 4009 /// block that ends with an uncond branch. We are looking for a very specific 4010 /// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified. In 4011 /// this case, we merge the first two "or's of icmp" into a switch, but then the 4012 /// default value goes to an uncond block with a seteq in it, we get something 4013 /// like: 4014 /// 4015 /// switch i8 %A, label %DEFAULT [ i8 1, label %end i8 2, label %end ] 4016 /// DEFAULT: 4017 /// %tmp = icmp eq i8 %A, 92 4018 /// br label %end 4019 /// end: 4020 /// ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ] 4021 /// 4022 /// We prefer to split the edge to 'end' so that there is a true/false entry to 4023 /// the PHI, merging the third icmp into the switch. 4024 bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt( 4025 ICmpInst *ICI, IRBuilder<> &Builder) { 4026 BasicBlock *BB = ICI->getParent(); 4027 4028 // If the block has any PHIs in it or the icmp has multiple uses, it is too 4029 // complex. 4030 if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse()) 4031 return false; 4032 4033 Value *V = ICI->getOperand(0); 4034 ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1)); 4035 4036 // The pattern we're looking for is where our only predecessor is a switch on 4037 // 'V' and this block is the default case for the switch. In this case we can 4038 // fold the compared value into the switch to simplify things. 4039 BasicBlock *Pred = BB->getSinglePredecessor(); 4040 if (!Pred || !isa<SwitchInst>(Pred->getTerminator())) 4041 return false; 4042 4043 SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator()); 4044 if (SI->getCondition() != V) 4045 return false; 4046 4047 // If BB is reachable on a non-default case, then we simply know the value of 4048 // V in this block. Substitute it and constant fold the icmp instruction 4049 // away. 4050 if (SI->getDefaultDest() != BB) { 4051 ConstantInt *VVal = SI->findCaseDest(BB); 4052 assert(VVal && "Should have a unique destination value"); 4053 ICI->setOperand(0, VVal); 4054 4055 if (Value *V = SimplifyInstruction(ICI, {DL, ICI})) { 4056 ICI->replaceAllUsesWith(V); 4057 ICI->eraseFromParent(); 4058 } 4059 // BB is now empty, so it is likely to simplify away. 4060 return requestResimplify(); 4061 } 4062 4063 // Ok, the block is reachable from the default dest. If the constant we're 4064 // comparing exists in one of the other edges, then we can constant fold ICI 4065 // and zap it. 4066 if (SI->findCaseValue(Cst) != SI->case_default()) { 4067 Value *V; 4068 if (ICI->getPredicate() == ICmpInst::ICMP_EQ) 4069 V = ConstantInt::getFalse(BB->getContext()); 4070 else 4071 V = ConstantInt::getTrue(BB->getContext()); 4072 4073 ICI->replaceAllUsesWith(V); 4074 ICI->eraseFromParent(); 4075 // BB is now empty, so it is likely to simplify away. 4076 return requestResimplify(); 4077 } 4078 4079 // The use of the icmp has to be in the 'end' block, by the only PHI node in 4080 // the block. 4081 BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0); 4082 PHINode *PHIUse = dyn_cast<PHINode>(ICI->user_back()); 4083 if (PHIUse == nullptr || PHIUse != &SuccBlock->front() || 4084 isa<PHINode>(++BasicBlock::iterator(PHIUse))) 4085 return false; 4086 4087 // If the icmp is a SETEQ, then the default dest gets false, the new edge gets 4088 // true in the PHI. 4089 Constant *DefaultCst = ConstantInt::getTrue(BB->getContext()); 4090 Constant *NewCst = ConstantInt::getFalse(BB->getContext()); 4091 4092 if (ICI->getPredicate() == ICmpInst::ICMP_EQ) 4093 std::swap(DefaultCst, NewCst); 4094 4095 // Replace ICI (which is used by the PHI for the default value) with true or 4096 // false depending on if it is EQ or NE. 4097 ICI->replaceAllUsesWith(DefaultCst); 4098 ICI->eraseFromParent(); 4099 4100 SmallVector<DominatorTree::UpdateType, 2> Updates; 4101 4102 // Okay, the switch goes to this block on a default value. Add an edge from 4103 // the switch to the merge point on the compared value. 4104 BasicBlock *NewBB = 4105 BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB); 4106 { 4107 SwitchInstProfUpdateWrapper SIW(*SI); 4108 auto W0 = SIW.getSuccessorWeight(0); 4109 SwitchInstProfUpdateWrapper::CaseWeightOpt NewW; 4110 if (W0) { 4111 NewW = ((uint64_t(*W0) + 1) >> 1); 4112 SIW.setSuccessorWeight(0, *NewW); 4113 } 4114 SIW.addCase(Cst, NewBB, NewW); 4115 if (DTU) 4116 Updates.push_back({DominatorTree::Insert, Pred, NewBB}); 4117 } 4118 4119 // NewBB branches to the phi block, add the uncond branch and the phi entry. 4120 Builder.SetInsertPoint(NewBB); 4121 Builder.SetCurrentDebugLocation(SI->getDebugLoc()); 4122 Builder.CreateBr(SuccBlock); 4123 PHIUse->addIncoming(NewCst, NewBB); 4124 if (DTU) { 4125 Updates.push_back({DominatorTree::Insert, NewBB, SuccBlock}); 4126 DTU->applyUpdates(Updates); 4127 } 4128 return true; 4129 } 4130 4131 /// The specified branch is a conditional branch. 4132 /// Check to see if it is branching on an or/and chain of icmp instructions, and 4133 /// fold it into a switch instruction if so. 4134 bool SimplifyCFGOpt::SimplifyBranchOnICmpChain(BranchInst *BI, 4135 IRBuilder<> &Builder, 4136 const DataLayout &DL) { 4137 Instruction *Cond = dyn_cast<Instruction>(BI->getCondition()); 4138 if (!Cond) 4139 return false; 4140 4141 // Change br (X == 0 | X == 1), T, F into a switch instruction. 4142 // If this is a bunch of seteq's or'd together, or if it's a bunch of 4143 // 'setne's and'ed together, collect them. 4144 4145 // Try to gather values from a chain of and/or to be turned into a switch 4146 ConstantComparesGatherer ConstantCompare(Cond, DL); 4147 // Unpack the result 4148 SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals; 4149 Value *CompVal = ConstantCompare.CompValue; 4150 unsigned UsedICmps = ConstantCompare.UsedICmps; 4151 Value *ExtraCase = ConstantCompare.Extra; 4152 4153 // If we didn't have a multiply compared value, fail. 4154 if (!CompVal) 4155 return false; 4156 4157 // Avoid turning single icmps into a switch. 4158 if (UsedICmps <= 1) 4159 return false; 4160 4161 bool TrueWhenEqual = match(Cond, m_LogicalOr(m_Value(), m_Value())); 4162 4163 // There might be duplicate constants in the list, which the switch 4164 // instruction can't handle, remove them now. 4165 array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate); 4166 Values.erase(std::unique(Values.begin(), Values.end()), Values.end()); 4167 4168 // If Extra was used, we require at least two switch values to do the 4169 // transformation. A switch with one value is just a conditional branch. 4170 if (ExtraCase && Values.size() < 2) 4171 return false; 4172 4173 // TODO: Preserve branch weight metadata, similarly to how 4174 // FoldValueComparisonIntoPredecessors preserves it. 4175 4176 // Figure out which block is which destination. 4177 BasicBlock *DefaultBB = BI->getSuccessor(1); 4178 BasicBlock *EdgeBB = BI->getSuccessor(0); 4179 if (!TrueWhenEqual) 4180 std::swap(DefaultBB, EdgeBB); 4181 4182 BasicBlock *BB = BI->getParent(); 4183 4184 LLVM_DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size() 4185 << " cases into SWITCH. BB is:\n" 4186 << *BB); 4187 4188 SmallVector<DominatorTree::UpdateType, 2> Updates; 4189 4190 // If there are any extra values that couldn't be folded into the switch 4191 // then we evaluate them with an explicit branch first. Split the block 4192 // right before the condbr to handle it. 4193 if (ExtraCase) { 4194 BasicBlock *NewBB = SplitBlock(BB, BI, DTU, /*LI=*/nullptr, 4195 /*MSSAU=*/nullptr, "switch.early.test"); 4196 4197 // Remove the uncond branch added to the old block. 4198 Instruction *OldTI = BB->getTerminator(); 4199 Builder.SetInsertPoint(OldTI); 4200 4201 // There can be an unintended UB if extra values are Poison. Before the 4202 // transformation, extra values may not be evaluated according to the 4203 // condition, and it will not raise UB. But after transformation, we are 4204 // evaluating extra values before checking the condition, and it will raise 4205 // UB. It can be solved by adding freeze instruction to extra values. 4206 AssumptionCache *AC = Options.AC; 4207 4208 if (!isGuaranteedNotToBeUndefOrPoison(ExtraCase, AC, BI, nullptr)) 4209 ExtraCase = Builder.CreateFreeze(ExtraCase); 4210 4211 if (TrueWhenEqual) 4212 Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB); 4213 else 4214 Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB); 4215 4216 OldTI->eraseFromParent(); 4217 4218 if (DTU) 4219 Updates.push_back({DominatorTree::Insert, BB, EdgeBB}); 4220 4221 // If there are PHI nodes in EdgeBB, then we need to add a new entry to them 4222 // for the edge we just added. 4223 AddPredecessorToBlock(EdgeBB, BB, NewBB); 4224 4225 LLVM_DEBUG(dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCase 4226 << "\nEXTRABB = " << *BB); 4227 BB = NewBB; 4228 } 4229 4230 Builder.SetInsertPoint(BI); 4231 // Convert pointer to int before we switch. 4232 if (CompVal->getType()->isPointerTy()) { 4233 CompVal = Builder.CreatePtrToInt( 4234 CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr"); 4235 } 4236 4237 // Create the new switch instruction now. 4238 SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size()); 4239 4240 // Add all of the 'cases' to the switch instruction. 4241 for (unsigned i = 0, e = Values.size(); i != e; ++i) 4242 New->addCase(Values[i], EdgeBB); 4243 4244 // We added edges from PI to the EdgeBB. As such, if there were any 4245 // PHI nodes in EdgeBB, they need entries to be added corresponding to 4246 // the number of edges added. 4247 for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) { 4248 PHINode *PN = cast<PHINode>(BBI); 4249 Value *InVal = PN->getIncomingValueForBlock(BB); 4250 for (unsigned i = 0, e = Values.size() - 1; i != e; ++i) 4251 PN->addIncoming(InVal, BB); 4252 } 4253 4254 // Erase the old branch instruction. 4255 EraseTerminatorAndDCECond(BI); 4256 if (DTU) 4257 DTU->applyUpdates(Updates); 4258 4259 LLVM_DEBUG(dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n'); 4260 return true; 4261 } 4262 4263 bool SimplifyCFGOpt::simplifyResume(ResumeInst *RI, IRBuilder<> &Builder) { 4264 if (isa<PHINode>(RI->getValue())) 4265 return simplifyCommonResume(RI); 4266 else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) && 4267 RI->getValue() == RI->getParent()->getFirstNonPHI()) 4268 // The resume must unwind the exception that caused control to branch here. 4269 return simplifySingleResume(RI); 4270 4271 return false; 4272 } 4273 4274 // Check if cleanup block is empty 4275 static bool isCleanupBlockEmpty(iterator_range<BasicBlock::iterator> R) { 4276 for (Instruction &I : R) { 4277 auto *II = dyn_cast<IntrinsicInst>(&I); 4278 if (!II) 4279 return false; 4280 4281 Intrinsic::ID IntrinsicID = II->getIntrinsicID(); 4282 switch (IntrinsicID) { 4283 case Intrinsic::dbg_declare: 4284 case Intrinsic::dbg_value: 4285 case Intrinsic::dbg_label: 4286 case Intrinsic::lifetime_end: 4287 break; 4288 default: 4289 return false; 4290 } 4291 } 4292 return true; 4293 } 4294 4295 // Simplify resume that is shared by several landing pads (phi of landing pad). 4296 bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) { 4297 BasicBlock *BB = RI->getParent(); 4298 4299 // Check that there are no other instructions except for debug and lifetime 4300 // intrinsics between the phi's and resume instruction. 4301 if (!isCleanupBlockEmpty( 4302 make_range(RI->getParent()->getFirstNonPHI(), BB->getTerminator()))) 4303 return false; 4304 4305 SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks; 4306 auto *PhiLPInst = cast<PHINode>(RI->getValue()); 4307 4308 // Check incoming blocks to see if any of them are trivial. 4309 for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End; 4310 Idx++) { 4311 auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx); 4312 auto *IncomingValue = PhiLPInst->getIncomingValue(Idx); 4313 4314 // If the block has other successors, we can not delete it because 4315 // it has other dependents. 4316 if (IncomingBB->getUniqueSuccessor() != BB) 4317 continue; 4318 4319 auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI()); 4320 // Not the landing pad that caused the control to branch here. 4321 if (IncomingValue != LandingPad) 4322 continue; 4323 4324 if (isCleanupBlockEmpty( 4325 make_range(LandingPad->getNextNode(), IncomingBB->getTerminator()))) 4326 TrivialUnwindBlocks.insert(IncomingBB); 4327 } 4328 4329 // If no trivial unwind blocks, don't do any simplifications. 4330 if (TrivialUnwindBlocks.empty()) 4331 return false; 4332 4333 // Turn all invokes that unwind here into calls. 4334 for (auto *TrivialBB : TrivialUnwindBlocks) { 4335 // Blocks that will be simplified should be removed from the phi node. 4336 // Note there could be multiple edges to the resume block, and we need 4337 // to remove them all. 4338 while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1) 4339 BB->removePredecessor(TrivialBB, true); 4340 4341 for (BasicBlock *Pred : 4342 llvm::make_early_inc_range(predecessors(TrivialBB))) { 4343 removeUnwindEdge(Pred, DTU); 4344 ++NumInvokes; 4345 } 4346 4347 // In each SimplifyCFG run, only the current processed block can be erased. 4348 // Otherwise, it will break the iteration of SimplifyCFG pass. So instead 4349 // of erasing TrivialBB, we only remove the branch to the common resume 4350 // block so that we can later erase the resume block since it has no 4351 // predecessors. 4352 TrivialBB->getTerminator()->eraseFromParent(); 4353 new UnreachableInst(RI->getContext(), TrivialBB); 4354 if (DTU) 4355 DTU->applyUpdates({{DominatorTree::Delete, TrivialBB, BB}}); 4356 } 4357 4358 // Delete the resume block if all its predecessors have been removed. 4359 if (pred_empty(BB)) 4360 DeleteDeadBlock(BB, DTU); 4361 4362 return !TrivialUnwindBlocks.empty(); 4363 } 4364 4365 // Simplify resume that is only used by a single (non-phi) landing pad. 4366 bool SimplifyCFGOpt::simplifySingleResume(ResumeInst *RI) { 4367 BasicBlock *BB = RI->getParent(); 4368 auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHI()); 4369 assert(RI->getValue() == LPInst && 4370 "Resume must unwind the exception that caused control to here"); 4371 4372 // Check that there are no other instructions except for debug intrinsics. 4373 if (!isCleanupBlockEmpty( 4374 make_range<Instruction *>(LPInst->getNextNode(), RI))) 4375 return false; 4376 4377 // Turn all invokes that unwind here into calls and delete the basic block. 4378 for (BasicBlock *Pred : llvm::make_early_inc_range(predecessors(BB))) { 4379 removeUnwindEdge(Pred, DTU); 4380 ++NumInvokes; 4381 } 4382 4383 // The landingpad is now unreachable. Zap it. 4384 DeleteDeadBlock(BB, DTU); 4385 return true; 4386 } 4387 4388 static bool removeEmptyCleanup(CleanupReturnInst *RI, DomTreeUpdater *DTU) { 4389 // If this is a trivial cleanup pad that executes no instructions, it can be 4390 // eliminated. If the cleanup pad continues to the caller, any predecessor 4391 // that is an EH pad will be updated to continue to the caller and any 4392 // predecessor that terminates with an invoke instruction will have its invoke 4393 // instruction converted to a call instruction. If the cleanup pad being 4394 // simplified does not continue to the caller, each predecessor will be 4395 // updated to continue to the unwind destination of the cleanup pad being 4396 // simplified. 4397 BasicBlock *BB = RI->getParent(); 4398 CleanupPadInst *CPInst = RI->getCleanupPad(); 4399 if (CPInst->getParent() != BB) 4400 // This isn't an empty cleanup. 4401 return false; 4402 4403 // We cannot kill the pad if it has multiple uses. This typically arises 4404 // from unreachable basic blocks. 4405 if (!CPInst->hasOneUse()) 4406 return false; 4407 4408 // Check that there are no other instructions except for benign intrinsics. 4409 if (!isCleanupBlockEmpty( 4410 make_range<Instruction *>(CPInst->getNextNode(), RI))) 4411 return false; 4412 4413 // If the cleanup return we are simplifying unwinds to the caller, this will 4414 // set UnwindDest to nullptr. 4415 BasicBlock *UnwindDest = RI->getUnwindDest(); 4416 Instruction *DestEHPad = UnwindDest ? UnwindDest->getFirstNonPHI() : nullptr; 4417 4418 // We're about to remove BB from the control flow. Before we do, sink any 4419 // PHINodes into the unwind destination. Doing this before changing the 4420 // control flow avoids some potentially slow checks, since we can currently 4421 // be certain that UnwindDest and BB have no common predecessors (since they 4422 // are both EH pads). 4423 if (UnwindDest) { 4424 // First, go through the PHI nodes in UnwindDest and update any nodes that 4425 // reference the block we are removing 4426 for (PHINode &DestPN : UnwindDest->phis()) { 4427 int Idx = DestPN.getBasicBlockIndex(BB); 4428 // Since BB unwinds to UnwindDest, it has to be in the PHI node. 4429 assert(Idx != -1); 4430 // This PHI node has an incoming value that corresponds to a control 4431 // path through the cleanup pad we are removing. If the incoming 4432 // value is in the cleanup pad, it must be a PHINode (because we 4433 // verified above that the block is otherwise empty). Otherwise, the 4434 // value is either a constant or a value that dominates the cleanup 4435 // pad being removed. 4436 // 4437 // Because BB and UnwindDest are both EH pads, all of their 4438 // predecessors must unwind to these blocks, and since no instruction 4439 // can have multiple unwind destinations, there will be no overlap in 4440 // incoming blocks between SrcPN and DestPN. 4441 Value *SrcVal = DestPN.getIncomingValue(Idx); 4442 PHINode *SrcPN = dyn_cast<PHINode>(SrcVal); 4443 4444 bool NeedPHITranslation = SrcPN && SrcPN->getParent() == BB; 4445 for (auto *Pred : predecessors(BB)) { 4446 Value *Incoming = 4447 NeedPHITranslation ? SrcPN->getIncomingValueForBlock(Pred) : SrcVal; 4448 DestPN.addIncoming(Incoming, Pred); 4449 } 4450 } 4451 4452 // Sink any remaining PHI nodes directly into UnwindDest. 4453 Instruction *InsertPt = DestEHPad; 4454 for (PHINode &PN : make_early_inc_range(BB->phis())) { 4455 if (PN.use_empty() || !PN.isUsedOutsideOfBlock(BB)) 4456 // If the PHI node has no uses or all of its uses are in this basic 4457 // block (meaning they are debug or lifetime intrinsics), just leave 4458 // it. It will be erased when we erase BB below. 4459 continue; 4460 4461 // Otherwise, sink this PHI node into UnwindDest. 4462 // Any predecessors to UnwindDest which are not already represented 4463 // must be back edges which inherit the value from the path through 4464 // BB. In this case, the PHI value must reference itself. 4465 for (auto *pred : predecessors(UnwindDest)) 4466 if (pred != BB) 4467 PN.addIncoming(&PN, pred); 4468 PN.moveBefore(InsertPt); 4469 // Also, add a dummy incoming value for the original BB itself, 4470 // so that the PHI is well-formed until we drop said predecessor. 4471 PN.addIncoming(UndefValue::get(PN.getType()), BB); 4472 } 4473 } 4474 4475 std::vector<DominatorTree::UpdateType> Updates; 4476 4477 // We use make_early_inc_range here because we will remove all predecessors. 4478 for (BasicBlock *PredBB : llvm::make_early_inc_range(predecessors(BB))) { 4479 if (UnwindDest == nullptr) { 4480 if (DTU) { 4481 DTU->applyUpdates(Updates); 4482 Updates.clear(); 4483 } 4484 removeUnwindEdge(PredBB, DTU); 4485 ++NumInvokes; 4486 } else { 4487 BB->removePredecessor(PredBB); 4488 Instruction *TI = PredBB->getTerminator(); 4489 TI->replaceUsesOfWith(BB, UnwindDest); 4490 if (DTU) { 4491 Updates.push_back({DominatorTree::Insert, PredBB, UnwindDest}); 4492 Updates.push_back({DominatorTree::Delete, PredBB, BB}); 4493 } 4494 } 4495 } 4496 4497 if (DTU) 4498 DTU->applyUpdates(Updates); 4499 4500 DeleteDeadBlock(BB, DTU); 4501 4502 return true; 4503 } 4504 4505 // Try to merge two cleanuppads together. 4506 static bool mergeCleanupPad(CleanupReturnInst *RI) { 4507 // Skip any cleanuprets which unwind to caller, there is nothing to merge 4508 // with. 4509 BasicBlock *UnwindDest = RI->getUnwindDest(); 4510 if (!UnwindDest) 4511 return false; 4512 4513 // This cleanupret isn't the only predecessor of this cleanuppad, it wouldn't 4514 // be safe to merge without code duplication. 4515 if (UnwindDest->getSinglePredecessor() != RI->getParent()) 4516 return false; 4517 4518 // Verify that our cleanuppad's unwind destination is another cleanuppad. 4519 auto *SuccessorCleanupPad = dyn_cast<CleanupPadInst>(&UnwindDest->front()); 4520 if (!SuccessorCleanupPad) 4521 return false; 4522 4523 CleanupPadInst *PredecessorCleanupPad = RI->getCleanupPad(); 4524 // Replace any uses of the successor cleanupad with the predecessor pad 4525 // The only cleanuppad uses should be this cleanupret, it's cleanupret and 4526 // funclet bundle operands. 4527 SuccessorCleanupPad->replaceAllUsesWith(PredecessorCleanupPad); 4528 // Remove the old cleanuppad. 4529 SuccessorCleanupPad->eraseFromParent(); 4530 // Now, we simply replace the cleanupret with a branch to the unwind 4531 // destination. 4532 BranchInst::Create(UnwindDest, RI->getParent()); 4533 RI->eraseFromParent(); 4534 4535 return true; 4536 } 4537 4538 bool SimplifyCFGOpt::simplifyCleanupReturn(CleanupReturnInst *RI) { 4539 // It is possible to transiantly have an undef cleanuppad operand because we 4540 // have deleted some, but not all, dead blocks. 4541 // Eventually, this block will be deleted. 4542 if (isa<UndefValue>(RI->getOperand(0))) 4543 return false; 4544 4545 if (mergeCleanupPad(RI)) 4546 return true; 4547 4548 if (removeEmptyCleanup(RI, DTU)) 4549 return true; 4550 4551 return false; 4552 } 4553 4554 // WARNING: keep in sync with InstCombinerImpl::visitUnreachableInst()! 4555 bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) { 4556 BasicBlock *BB = UI->getParent(); 4557 4558 bool Changed = false; 4559 4560 // If there are any instructions immediately before the unreachable that can 4561 // be removed, do so. 4562 while (UI->getIterator() != BB->begin()) { 4563 BasicBlock::iterator BBI = UI->getIterator(); 4564 --BBI; 4565 4566 if (!isGuaranteedToTransferExecutionToSuccessor(&*BBI)) 4567 break; // Can not drop any more instructions. We're done here. 4568 // Otherwise, this instruction can be freely erased, 4569 // even if it is not side-effect free. 4570 4571 // Note that deleting EH's here is in fact okay, although it involves a bit 4572 // of subtle reasoning. If this inst is an EH, all the predecessors of this 4573 // block will be the unwind edges of Invoke/CatchSwitch/CleanupReturn, 4574 // and we can therefore guarantee this block will be erased. 4575 4576 // Delete this instruction (any uses are guaranteed to be dead) 4577 BBI->replaceAllUsesWith(PoisonValue::get(BBI->getType())); 4578 BBI->eraseFromParent(); 4579 Changed = true; 4580 } 4581 4582 // If the unreachable instruction is the first in the block, take a gander 4583 // at all of the predecessors of this instruction, and simplify them. 4584 if (&BB->front() != UI) 4585 return Changed; 4586 4587 std::vector<DominatorTree::UpdateType> Updates; 4588 4589 SmallSetVector<BasicBlock *, 8> Preds(pred_begin(BB), pred_end(BB)); 4590 for (unsigned i = 0, e = Preds.size(); i != e; ++i) { 4591 auto *Predecessor = Preds[i]; 4592 Instruction *TI = Predecessor->getTerminator(); 4593 IRBuilder<> Builder(TI); 4594 if (auto *BI = dyn_cast<BranchInst>(TI)) { 4595 // We could either have a proper unconditional branch, 4596 // or a degenerate conditional branch with matching destinations. 4597 if (all_of(BI->successors(), 4598 [BB](auto *Successor) { return Successor == BB; })) { 4599 new UnreachableInst(TI->getContext(), TI); 4600 TI->eraseFromParent(); 4601 Changed = true; 4602 } else { 4603 assert(BI->isConditional() && "Can't get here with an uncond branch."); 4604 Value* Cond = BI->getCondition(); 4605 assert(BI->getSuccessor(0) != BI->getSuccessor(1) && 4606 "The destinations are guaranteed to be different here."); 4607 if (BI->getSuccessor(0) == BB) { 4608 Builder.CreateAssumption(Builder.CreateNot(Cond)); 4609 Builder.CreateBr(BI->getSuccessor(1)); 4610 } else { 4611 assert(BI->getSuccessor(1) == BB && "Incorrect CFG"); 4612 Builder.CreateAssumption(Cond); 4613 Builder.CreateBr(BI->getSuccessor(0)); 4614 } 4615 EraseTerminatorAndDCECond(BI); 4616 Changed = true; 4617 } 4618 if (DTU) 4619 Updates.push_back({DominatorTree::Delete, Predecessor, BB}); 4620 } else if (auto *SI = dyn_cast<SwitchInst>(TI)) { 4621 SwitchInstProfUpdateWrapper SU(*SI); 4622 for (auto i = SU->case_begin(), e = SU->case_end(); i != e;) { 4623 if (i->getCaseSuccessor() != BB) { 4624 ++i; 4625 continue; 4626 } 4627 BB->removePredecessor(SU->getParent()); 4628 i = SU.removeCase(i); 4629 e = SU->case_end(); 4630 Changed = true; 4631 } 4632 // Note that the default destination can't be removed! 4633 if (DTU && SI->getDefaultDest() != BB) 4634 Updates.push_back({DominatorTree::Delete, Predecessor, BB}); 4635 } else if (auto *II = dyn_cast<InvokeInst>(TI)) { 4636 if (II->getUnwindDest() == BB) { 4637 if (DTU) { 4638 DTU->applyUpdates(Updates); 4639 Updates.clear(); 4640 } 4641 removeUnwindEdge(TI->getParent(), DTU); 4642 Changed = true; 4643 } 4644 } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) { 4645 if (CSI->getUnwindDest() == BB) { 4646 if (DTU) { 4647 DTU->applyUpdates(Updates); 4648 Updates.clear(); 4649 } 4650 removeUnwindEdge(TI->getParent(), DTU); 4651 Changed = true; 4652 continue; 4653 } 4654 4655 for (CatchSwitchInst::handler_iterator I = CSI->handler_begin(), 4656 E = CSI->handler_end(); 4657 I != E; ++I) { 4658 if (*I == BB) { 4659 CSI->removeHandler(I); 4660 --I; 4661 --E; 4662 Changed = true; 4663 } 4664 } 4665 if (DTU) 4666 Updates.push_back({DominatorTree::Delete, Predecessor, BB}); 4667 if (CSI->getNumHandlers() == 0) { 4668 if (CSI->hasUnwindDest()) { 4669 // Redirect all predecessors of the block containing CatchSwitchInst 4670 // to instead branch to the CatchSwitchInst's unwind destination. 4671 if (DTU) { 4672 for (auto *PredecessorOfPredecessor : predecessors(Predecessor)) { 4673 Updates.push_back({DominatorTree::Insert, 4674 PredecessorOfPredecessor, 4675 CSI->getUnwindDest()}); 4676 Updates.push_back({DominatorTree::Delete, 4677 PredecessorOfPredecessor, Predecessor}); 4678 } 4679 } 4680 Predecessor->replaceAllUsesWith(CSI->getUnwindDest()); 4681 } else { 4682 // Rewrite all preds to unwind to caller (or from invoke to call). 4683 if (DTU) { 4684 DTU->applyUpdates(Updates); 4685 Updates.clear(); 4686 } 4687 SmallVector<BasicBlock *, 8> EHPreds(predecessors(Predecessor)); 4688 for (BasicBlock *EHPred : EHPreds) 4689 removeUnwindEdge(EHPred, DTU); 4690 } 4691 // The catchswitch is no longer reachable. 4692 new UnreachableInst(CSI->getContext(), CSI); 4693 CSI->eraseFromParent(); 4694 Changed = true; 4695 } 4696 } else if (auto *CRI = dyn_cast<CleanupReturnInst>(TI)) { 4697 (void)CRI; 4698 assert(CRI->hasUnwindDest() && CRI->getUnwindDest() == BB && 4699 "Expected to always have an unwind to BB."); 4700 if (DTU) 4701 Updates.push_back({DominatorTree::Delete, Predecessor, BB}); 4702 new UnreachableInst(TI->getContext(), TI); 4703 TI->eraseFromParent(); 4704 Changed = true; 4705 } 4706 } 4707 4708 if (DTU) 4709 DTU->applyUpdates(Updates); 4710 4711 // If this block is now dead, remove it. 4712 if (pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) { 4713 DeleteDeadBlock(BB, DTU); 4714 return true; 4715 } 4716 4717 return Changed; 4718 } 4719 4720 static bool CasesAreContiguous(SmallVectorImpl<ConstantInt *> &Cases) { 4721 assert(Cases.size() >= 1); 4722 4723 array_pod_sort(Cases.begin(), Cases.end(), ConstantIntSortPredicate); 4724 for (size_t I = 1, E = Cases.size(); I != E; ++I) { 4725 if (Cases[I - 1]->getValue() != Cases[I]->getValue() + 1) 4726 return false; 4727 } 4728 return true; 4729 } 4730 4731 static void createUnreachableSwitchDefault(SwitchInst *Switch, 4732 DomTreeUpdater *DTU) { 4733 LLVM_DEBUG(dbgs() << "SimplifyCFG: switch default is dead.\n"); 4734 auto *BB = Switch->getParent(); 4735 BasicBlock *NewDefaultBlock = SplitBlockPredecessors( 4736 Switch->getDefaultDest(), Switch->getParent(), "", DTU); 4737 auto *OrigDefaultBlock = Switch->getDefaultDest(); 4738 Switch->setDefaultDest(&*NewDefaultBlock); 4739 if (DTU) 4740 DTU->applyUpdates({{DominatorTree::Insert, BB, &*NewDefaultBlock}, 4741 {DominatorTree::Delete, BB, OrigDefaultBlock}}); 4742 SplitBlock(&*NewDefaultBlock, &NewDefaultBlock->front(), DTU); 4743 SmallVector<DominatorTree::UpdateType, 2> Updates; 4744 if (DTU) 4745 for (auto *Successor : successors(NewDefaultBlock)) 4746 Updates.push_back({DominatorTree::Delete, NewDefaultBlock, Successor}); 4747 auto *NewTerminator = NewDefaultBlock->getTerminator(); 4748 new UnreachableInst(Switch->getContext(), NewTerminator); 4749 EraseTerminatorAndDCECond(NewTerminator); 4750 if (DTU) 4751 DTU->applyUpdates(Updates); 4752 } 4753 4754 /// Turn a switch with two reachable destinations into an integer range 4755 /// comparison and branch. 4756 bool SimplifyCFGOpt::TurnSwitchRangeIntoICmp(SwitchInst *SI, 4757 IRBuilder<> &Builder) { 4758 assert(SI->getNumCases() > 1 && "Degenerate switch?"); 4759 4760 bool HasDefault = 4761 !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg()); 4762 4763 auto *BB = SI->getParent(); 4764 4765 // Partition the cases into two sets with different destinations. 4766 BasicBlock *DestA = HasDefault ? SI->getDefaultDest() : nullptr; 4767 BasicBlock *DestB = nullptr; 4768 SmallVector<ConstantInt *, 16> CasesA; 4769 SmallVector<ConstantInt *, 16> CasesB; 4770 4771 for (auto Case : SI->cases()) { 4772 BasicBlock *Dest = Case.getCaseSuccessor(); 4773 if (!DestA) 4774 DestA = Dest; 4775 if (Dest == DestA) { 4776 CasesA.push_back(Case.getCaseValue()); 4777 continue; 4778 } 4779 if (!DestB) 4780 DestB = Dest; 4781 if (Dest == DestB) { 4782 CasesB.push_back(Case.getCaseValue()); 4783 continue; 4784 } 4785 return false; // More than two destinations. 4786 } 4787 4788 assert(DestA && DestB && 4789 "Single-destination switch should have been folded."); 4790 assert(DestA != DestB); 4791 assert(DestB != SI->getDefaultDest()); 4792 assert(!CasesB.empty() && "There must be non-default cases."); 4793 assert(!CasesA.empty() || HasDefault); 4794 4795 // Figure out if one of the sets of cases form a contiguous range. 4796 SmallVectorImpl<ConstantInt *> *ContiguousCases = nullptr; 4797 BasicBlock *ContiguousDest = nullptr; 4798 BasicBlock *OtherDest = nullptr; 4799 if (!CasesA.empty() && CasesAreContiguous(CasesA)) { 4800 ContiguousCases = &CasesA; 4801 ContiguousDest = DestA; 4802 OtherDest = DestB; 4803 } else if (CasesAreContiguous(CasesB)) { 4804 ContiguousCases = &CasesB; 4805 ContiguousDest = DestB; 4806 OtherDest = DestA; 4807 } else 4808 return false; 4809 4810 // Start building the compare and branch. 4811 4812 Constant *Offset = ConstantExpr::getNeg(ContiguousCases->back()); 4813 Constant *NumCases = 4814 ConstantInt::get(Offset->getType(), ContiguousCases->size()); 4815 4816 Value *Sub = SI->getCondition(); 4817 if (!Offset->isNullValue()) 4818 Sub = Builder.CreateAdd(Sub, Offset, Sub->getName() + ".off"); 4819 4820 Value *Cmp; 4821 // If NumCases overflowed, then all possible values jump to the successor. 4822 if (NumCases->isNullValue() && !ContiguousCases->empty()) 4823 Cmp = ConstantInt::getTrue(SI->getContext()); 4824 else 4825 Cmp = Builder.CreateICmpULT(Sub, NumCases, "switch"); 4826 BranchInst *NewBI = Builder.CreateCondBr(Cmp, ContiguousDest, OtherDest); 4827 4828 // Update weight for the newly-created conditional branch. 4829 if (HasBranchWeights(SI)) { 4830 SmallVector<uint64_t, 8> Weights; 4831 GetBranchWeights(SI, Weights); 4832 if (Weights.size() == 1 + SI->getNumCases()) { 4833 uint64_t TrueWeight = 0; 4834 uint64_t FalseWeight = 0; 4835 for (size_t I = 0, E = Weights.size(); I != E; ++I) { 4836 if (SI->getSuccessor(I) == ContiguousDest) 4837 TrueWeight += Weights[I]; 4838 else 4839 FalseWeight += Weights[I]; 4840 } 4841 while (TrueWeight > UINT32_MAX || FalseWeight > UINT32_MAX) { 4842 TrueWeight /= 2; 4843 FalseWeight /= 2; 4844 } 4845 setBranchWeights(NewBI, TrueWeight, FalseWeight); 4846 } 4847 } 4848 4849 // Prune obsolete incoming values off the successors' PHI nodes. 4850 for (auto BBI = ContiguousDest->begin(); isa<PHINode>(BBI); ++BBI) { 4851 unsigned PreviousEdges = ContiguousCases->size(); 4852 if (ContiguousDest == SI->getDefaultDest()) 4853 ++PreviousEdges; 4854 for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I) 4855 cast<PHINode>(BBI)->removeIncomingValue(SI->getParent()); 4856 } 4857 for (auto BBI = OtherDest->begin(); isa<PHINode>(BBI); ++BBI) { 4858 unsigned PreviousEdges = SI->getNumCases() - ContiguousCases->size(); 4859 if (OtherDest == SI->getDefaultDest()) 4860 ++PreviousEdges; 4861 for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I) 4862 cast<PHINode>(BBI)->removeIncomingValue(SI->getParent()); 4863 } 4864 4865 // Clean up the default block - it may have phis or other instructions before 4866 // the unreachable terminator. 4867 if (!HasDefault) 4868 createUnreachableSwitchDefault(SI, DTU); 4869 4870 auto *UnreachableDefault = SI->getDefaultDest(); 4871 4872 // Drop the switch. 4873 SI->eraseFromParent(); 4874 4875 if (!HasDefault && DTU) 4876 DTU->applyUpdates({{DominatorTree::Delete, BB, UnreachableDefault}}); 4877 4878 return true; 4879 } 4880 4881 /// Compute masked bits for the condition of a switch 4882 /// and use it to remove dead cases. 4883 static bool eliminateDeadSwitchCases(SwitchInst *SI, DomTreeUpdater *DTU, 4884 AssumptionCache *AC, 4885 const DataLayout &DL) { 4886 Value *Cond = SI->getCondition(); 4887 unsigned Bits = Cond->getType()->getIntegerBitWidth(); 4888 KnownBits Known = computeKnownBits(Cond, DL, 0, AC, SI); 4889 4890 // We can also eliminate cases by determining that their values are outside of 4891 // the limited range of the condition based on how many significant (non-sign) 4892 // bits are in the condition value. 4893 unsigned ExtraSignBits = ComputeNumSignBits(Cond, DL, 0, AC, SI) - 1; 4894 unsigned MaxSignificantBitsInCond = Bits - ExtraSignBits; 4895 4896 // Gather dead cases. 4897 SmallVector<ConstantInt *, 8> DeadCases; 4898 SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases; 4899 for (auto &Case : SI->cases()) { 4900 auto *Successor = Case.getCaseSuccessor(); 4901 if (DTU) 4902 ++NumPerSuccessorCases[Successor]; 4903 const APInt &CaseVal = Case.getCaseValue()->getValue(); 4904 if (Known.Zero.intersects(CaseVal) || !Known.One.isSubsetOf(CaseVal) || 4905 (CaseVal.getMinSignedBits() > MaxSignificantBitsInCond)) { 4906 DeadCases.push_back(Case.getCaseValue()); 4907 if (DTU) 4908 --NumPerSuccessorCases[Successor]; 4909 LLVM_DEBUG(dbgs() << "SimplifyCFG: switch case " << CaseVal 4910 << " is dead.\n"); 4911 } 4912 } 4913 4914 // If we can prove that the cases must cover all possible values, the 4915 // default destination becomes dead and we can remove it. If we know some 4916 // of the bits in the value, we can use that to more precisely compute the 4917 // number of possible unique case values. 4918 bool HasDefault = 4919 !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg()); 4920 const unsigned NumUnknownBits = 4921 Bits - (Known.Zero | Known.One).countPopulation(); 4922 assert(NumUnknownBits <= Bits); 4923 if (HasDefault && DeadCases.empty() && 4924 NumUnknownBits < 64 /* avoid overflow */ && 4925 SI->getNumCases() == (1ULL << NumUnknownBits)) { 4926 createUnreachableSwitchDefault(SI, DTU); 4927 return true; 4928 } 4929 4930 if (DeadCases.empty()) 4931 return false; 4932 4933 SwitchInstProfUpdateWrapper SIW(*SI); 4934 for (ConstantInt *DeadCase : DeadCases) { 4935 SwitchInst::CaseIt CaseI = SI->findCaseValue(DeadCase); 4936 assert(CaseI != SI->case_default() && 4937 "Case was not found. Probably mistake in DeadCases forming."); 4938 // Prune unused values from PHI nodes. 4939 CaseI->getCaseSuccessor()->removePredecessor(SI->getParent()); 4940 SIW.removeCase(CaseI); 4941 } 4942 4943 if (DTU) { 4944 std::vector<DominatorTree::UpdateType> Updates; 4945 for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases) 4946 if (I.second == 0) 4947 Updates.push_back({DominatorTree::Delete, SI->getParent(), I.first}); 4948 DTU->applyUpdates(Updates); 4949 } 4950 4951 return true; 4952 } 4953 4954 /// If BB would be eligible for simplification by 4955 /// TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated 4956 /// by an unconditional branch), look at the phi node for BB in the successor 4957 /// block and see if the incoming value is equal to CaseValue. If so, return 4958 /// the phi node, and set PhiIndex to BB's index in the phi node. 4959 static PHINode *FindPHIForConditionForwarding(ConstantInt *CaseValue, 4960 BasicBlock *BB, int *PhiIndex) { 4961 if (BB->getFirstNonPHIOrDbg() != BB->getTerminator()) 4962 return nullptr; // BB must be empty to be a candidate for simplification. 4963 if (!BB->getSinglePredecessor()) 4964 return nullptr; // BB must be dominated by the switch. 4965 4966 BranchInst *Branch = dyn_cast<BranchInst>(BB->getTerminator()); 4967 if (!Branch || !Branch->isUnconditional()) 4968 return nullptr; // Terminator must be unconditional branch. 4969 4970 BasicBlock *Succ = Branch->getSuccessor(0); 4971 4972 for (PHINode &PHI : Succ->phis()) { 4973 int Idx = PHI.getBasicBlockIndex(BB); 4974 assert(Idx >= 0 && "PHI has no entry for predecessor?"); 4975 4976 Value *InValue = PHI.getIncomingValue(Idx); 4977 if (InValue != CaseValue) 4978 continue; 4979 4980 *PhiIndex = Idx; 4981 return &PHI; 4982 } 4983 4984 return nullptr; 4985 } 4986 4987 /// Try to forward the condition of a switch instruction to a phi node 4988 /// dominated by the switch, if that would mean that some of the destination 4989 /// blocks of the switch can be folded away. Return true if a change is made. 4990 static bool ForwardSwitchConditionToPHI(SwitchInst *SI) { 4991 using ForwardingNodesMap = DenseMap<PHINode *, SmallVector<int, 4>>; 4992 4993 ForwardingNodesMap ForwardingNodes; 4994 BasicBlock *SwitchBlock = SI->getParent(); 4995 bool Changed = false; 4996 for (auto &Case : SI->cases()) { 4997 ConstantInt *CaseValue = Case.getCaseValue(); 4998 BasicBlock *CaseDest = Case.getCaseSuccessor(); 4999 5000 // Replace phi operands in successor blocks that are using the constant case 5001 // value rather than the switch condition variable: 5002 // switchbb: 5003 // switch i32 %x, label %default [ 5004 // i32 17, label %succ 5005 // ... 5006 // succ: 5007 // %r = phi i32 ... [ 17, %switchbb ] ... 5008 // --> 5009 // %r = phi i32 ... [ %x, %switchbb ] ... 5010 5011 for (PHINode &Phi : CaseDest->phis()) { 5012 // This only works if there is exactly 1 incoming edge from the switch to 5013 // a phi. If there is >1, that means multiple cases of the switch map to 1 5014 // value in the phi, and that phi value is not the switch condition. Thus, 5015 // this transform would not make sense (the phi would be invalid because 5016 // a phi can't have different incoming values from the same block). 5017 int SwitchBBIdx = Phi.getBasicBlockIndex(SwitchBlock); 5018 if (Phi.getIncomingValue(SwitchBBIdx) == CaseValue && 5019 count(Phi.blocks(), SwitchBlock) == 1) { 5020 Phi.setIncomingValue(SwitchBBIdx, SI->getCondition()); 5021 Changed = true; 5022 } 5023 } 5024 5025 // Collect phi nodes that are indirectly using this switch's case constants. 5026 int PhiIdx; 5027 if (auto *Phi = FindPHIForConditionForwarding(CaseValue, CaseDest, &PhiIdx)) 5028 ForwardingNodes[Phi].push_back(PhiIdx); 5029 } 5030 5031 for (auto &ForwardingNode : ForwardingNodes) { 5032 PHINode *Phi = ForwardingNode.first; 5033 SmallVectorImpl<int> &Indexes = ForwardingNode.second; 5034 if (Indexes.size() < 2) 5035 continue; 5036 5037 for (int Index : Indexes) 5038 Phi->setIncomingValue(Index, SI->getCondition()); 5039 Changed = true; 5040 } 5041 5042 return Changed; 5043 } 5044 5045 /// Return true if the backend will be able to handle 5046 /// initializing an array of constants like C. 5047 static bool ValidLookupTableConstant(Constant *C, const TargetTransformInfo &TTI) { 5048 if (C->isThreadDependent()) 5049 return false; 5050 if (C->isDLLImportDependent()) 5051 return false; 5052 5053 if (!isa<ConstantFP>(C) && !isa<ConstantInt>(C) && 5054 !isa<ConstantPointerNull>(C) && !isa<GlobalValue>(C) && 5055 !isa<UndefValue>(C) && !isa<ConstantExpr>(C)) 5056 return false; 5057 5058 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { 5059 if (!CE->isGEPWithNoNotionalOverIndexing()) 5060 return false; 5061 if (!ValidLookupTableConstant(CE->getOperand(0), TTI)) 5062 return false; 5063 } 5064 5065 if (!TTI.shouldBuildLookupTablesForConstant(C)) 5066 return false; 5067 5068 return true; 5069 } 5070 5071 /// If V is a Constant, return it. Otherwise, try to look up 5072 /// its constant value in ConstantPool, returning 0 if it's not there. 5073 static Constant * 5074 LookupConstant(Value *V, 5075 const SmallDenseMap<Value *, Constant *> &ConstantPool) { 5076 if (Constant *C = dyn_cast<Constant>(V)) 5077 return C; 5078 return ConstantPool.lookup(V); 5079 } 5080 5081 /// Try to fold instruction I into a constant. This works for 5082 /// simple instructions such as binary operations where both operands are 5083 /// constant or can be replaced by constants from the ConstantPool. Returns the 5084 /// resulting constant on success, 0 otherwise. 5085 static Constant * 5086 ConstantFold(Instruction *I, const DataLayout &DL, 5087 const SmallDenseMap<Value *, Constant *> &ConstantPool) { 5088 if (SelectInst *Select = dyn_cast<SelectInst>(I)) { 5089 Constant *A = LookupConstant(Select->getCondition(), ConstantPool); 5090 if (!A) 5091 return nullptr; 5092 if (A->isAllOnesValue()) 5093 return LookupConstant(Select->getTrueValue(), ConstantPool); 5094 if (A->isNullValue()) 5095 return LookupConstant(Select->getFalseValue(), ConstantPool); 5096 return nullptr; 5097 } 5098 5099 SmallVector<Constant *, 4> COps; 5100 for (unsigned N = 0, E = I->getNumOperands(); N != E; ++N) { 5101 if (Constant *A = LookupConstant(I->getOperand(N), ConstantPool)) 5102 COps.push_back(A); 5103 else 5104 return nullptr; 5105 } 5106 5107 if (CmpInst *Cmp = dyn_cast<CmpInst>(I)) { 5108 return ConstantFoldCompareInstOperands(Cmp->getPredicate(), COps[0], 5109 COps[1], DL); 5110 } 5111 5112 return ConstantFoldInstOperands(I, COps, DL); 5113 } 5114 5115 /// Try to determine the resulting constant values in phi nodes 5116 /// at the common destination basic block, *CommonDest, for one of the case 5117 /// destionations CaseDest corresponding to value CaseVal (0 for the default 5118 /// case), of a switch instruction SI. 5119 static bool 5120 GetCaseResults(SwitchInst *SI, ConstantInt *CaseVal, BasicBlock *CaseDest, 5121 BasicBlock **CommonDest, 5122 SmallVectorImpl<std::pair<PHINode *, Constant *>> &Res, 5123 const DataLayout &DL, const TargetTransformInfo &TTI) { 5124 // The block from which we enter the common destination. 5125 BasicBlock *Pred = SI->getParent(); 5126 5127 // If CaseDest is empty except for some side-effect free instructions through 5128 // which we can constant-propagate the CaseVal, continue to its successor. 5129 SmallDenseMap<Value *, Constant *> ConstantPool; 5130 ConstantPool.insert(std::make_pair(SI->getCondition(), CaseVal)); 5131 for (Instruction &I :CaseDest->instructionsWithoutDebug()) { 5132 if (I.isTerminator()) { 5133 // If the terminator is a simple branch, continue to the next block. 5134 if (I.getNumSuccessors() != 1 || I.isExceptionalTerminator()) 5135 return false; 5136 Pred = CaseDest; 5137 CaseDest = I.getSuccessor(0); 5138 } else if (Constant *C = ConstantFold(&I, DL, ConstantPool)) { 5139 // Instruction is side-effect free and constant. 5140 5141 // If the instruction has uses outside this block or a phi node slot for 5142 // the block, it is not safe to bypass the instruction since it would then 5143 // no longer dominate all its uses. 5144 for (auto &Use : I.uses()) { 5145 User *User = Use.getUser(); 5146 if (Instruction *I = dyn_cast<Instruction>(User)) 5147 if (I->getParent() == CaseDest) 5148 continue; 5149 if (PHINode *Phi = dyn_cast<PHINode>(User)) 5150 if (Phi->getIncomingBlock(Use) == CaseDest) 5151 continue; 5152 return false; 5153 } 5154 5155 ConstantPool.insert(std::make_pair(&I, C)); 5156 } else { 5157 break; 5158 } 5159 } 5160 5161 // If we did not have a CommonDest before, use the current one. 5162 if (!*CommonDest) 5163 *CommonDest = CaseDest; 5164 // If the destination isn't the common one, abort. 5165 if (CaseDest != *CommonDest) 5166 return false; 5167 5168 // Get the values for this case from phi nodes in the destination block. 5169 for (PHINode &PHI : (*CommonDest)->phis()) { 5170 int Idx = PHI.getBasicBlockIndex(Pred); 5171 if (Idx == -1) 5172 continue; 5173 5174 Constant *ConstVal = 5175 LookupConstant(PHI.getIncomingValue(Idx), ConstantPool); 5176 if (!ConstVal) 5177 return false; 5178 5179 // Be conservative about which kinds of constants we support. 5180 if (!ValidLookupTableConstant(ConstVal, TTI)) 5181 return false; 5182 5183 Res.push_back(std::make_pair(&PHI, ConstVal)); 5184 } 5185 5186 return Res.size() > 0; 5187 } 5188 5189 // Helper function used to add CaseVal to the list of cases that generate 5190 // Result. Returns the updated number of cases that generate this result. 5191 static uintptr_t MapCaseToResult(ConstantInt *CaseVal, 5192 SwitchCaseResultVectorTy &UniqueResults, 5193 Constant *Result) { 5194 for (auto &I : UniqueResults) { 5195 if (I.first == Result) { 5196 I.second.push_back(CaseVal); 5197 return I.second.size(); 5198 } 5199 } 5200 UniqueResults.push_back( 5201 std::make_pair(Result, SmallVector<ConstantInt *, 4>(1, CaseVal))); 5202 return 1; 5203 } 5204 5205 // Helper function that initializes a map containing 5206 // results for the PHI node of the common destination block for a switch 5207 // instruction. Returns false if multiple PHI nodes have been found or if 5208 // there is not a common destination block for the switch. 5209 static bool 5210 InitializeUniqueCases(SwitchInst *SI, PHINode *&PHI, BasicBlock *&CommonDest, 5211 SwitchCaseResultVectorTy &UniqueResults, 5212 Constant *&DefaultResult, const DataLayout &DL, 5213 const TargetTransformInfo &TTI, 5214 uintptr_t MaxUniqueResults, uintptr_t MaxCasesPerResult) { 5215 for (auto &I : SI->cases()) { 5216 ConstantInt *CaseVal = I.getCaseValue(); 5217 5218 // Resulting value at phi nodes for this case value. 5219 SwitchCaseResultsTy Results; 5220 if (!GetCaseResults(SI, CaseVal, I.getCaseSuccessor(), &CommonDest, Results, 5221 DL, TTI)) 5222 return false; 5223 5224 // Only one value per case is permitted. 5225 if (Results.size() > 1) 5226 return false; 5227 5228 // Add the case->result mapping to UniqueResults. 5229 const uintptr_t NumCasesForResult = 5230 MapCaseToResult(CaseVal, UniqueResults, Results.begin()->second); 5231 5232 // Early out if there are too many cases for this result. 5233 if (NumCasesForResult > MaxCasesPerResult) 5234 return false; 5235 5236 // Early out if there are too many unique results. 5237 if (UniqueResults.size() > MaxUniqueResults) 5238 return false; 5239 5240 // Check the PHI consistency. 5241 if (!PHI) 5242 PHI = Results[0].first; 5243 else if (PHI != Results[0].first) 5244 return false; 5245 } 5246 // Find the default result value. 5247 SmallVector<std::pair<PHINode *, Constant *>, 1> DefaultResults; 5248 BasicBlock *DefaultDest = SI->getDefaultDest(); 5249 GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, DefaultResults, 5250 DL, TTI); 5251 // If the default value is not found abort unless the default destination 5252 // is unreachable. 5253 DefaultResult = 5254 DefaultResults.size() == 1 ? DefaultResults.begin()->second : nullptr; 5255 if ((!DefaultResult && 5256 !isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg()))) 5257 return false; 5258 5259 return true; 5260 } 5261 5262 // Helper function that checks if it is possible to transform a switch with only 5263 // two cases (or two cases + default) that produces a result into a select. 5264 // Example: 5265 // switch (a) { 5266 // case 10: %0 = icmp eq i32 %a, 10 5267 // return 10; %1 = select i1 %0, i32 10, i32 4 5268 // case 20: ----> %2 = icmp eq i32 %a, 20 5269 // return 2; %3 = select i1 %2, i32 2, i32 %1 5270 // default: 5271 // return 4; 5272 // } 5273 static Value *ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy &ResultVector, 5274 Constant *DefaultResult, Value *Condition, 5275 IRBuilder<> &Builder) { 5276 // If we are selecting between only two cases transform into a simple 5277 // select or a two-way select if default is possible. 5278 if (ResultVector.size() == 2 && ResultVector[0].second.size() == 1 && 5279 ResultVector[1].second.size() == 1) { 5280 ConstantInt *const FirstCase = ResultVector[0].second[0]; 5281 ConstantInt *const SecondCase = ResultVector[1].second[0]; 5282 5283 bool DefaultCanTrigger = DefaultResult; 5284 Value *SelectValue = ResultVector[1].first; 5285 if (DefaultCanTrigger) { 5286 Value *const ValueCompare = 5287 Builder.CreateICmpEQ(Condition, SecondCase, "switch.selectcmp"); 5288 SelectValue = Builder.CreateSelect(ValueCompare, ResultVector[1].first, 5289 DefaultResult, "switch.select"); 5290 } 5291 Value *const ValueCompare = 5292 Builder.CreateICmpEQ(Condition, FirstCase, "switch.selectcmp"); 5293 return Builder.CreateSelect(ValueCompare, ResultVector[0].first, 5294 SelectValue, "switch.select"); 5295 } 5296 5297 // Handle the degenerate case where two cases have the same value. 5298 if (ResultVector.size() == 1 && ResultVector[0].second.size() == 2 && 5299 DefaultResult) { 5300 Value *Cmp1 = Builder.CreateICmpEQ( 5301 Condition, ResultVector[0].second[0], "switch.selectcmp.case1"); 5302 Value *Cmp2 = Builder.CreateICmpEQ( 5303 Condition, ResultVector[0].second[1], "switch.selectcmp.case2"); 5304 Value *Cmp = Builder.CreateOr(Cmp1, Cmp2, "switch.selectcmp"); 5305 return Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult); 5306 } 5307 5308 return nullptr; 5309 } 5310 5311 // Helper function to cleanup a switch instruction that has been converted into 5312 // a select, fixing up PHI nodes and basic blocks. 5313 static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI, 5314 Value *SelectValue, 5315 IRBuilder<> &Builder, 5316 DomTreeUpdater *DTU) { 5317 std::vector<DominatorTree::UpdateType> Updates; 5318 5319 BasicBlock *SelectBB = SI->getParent(); 5320 BasicBlock *DestBB = PHI->getParent(); 5321 5322 if (DTU && !is_contained(predecessors(DestBB), SelectBB)) 5323 Updates.push_back({DominatorTree::Insert, SelectBB, DestBB}); 5324 Builder.CreateBr(DestBB); 5325 5326 // Remove the switch. 5327 5328 while (PHI->getBasicBlockIndex(SelectBB) >= 0) 5329 PHI->removeIncomingValue(SelectBB); 5330 PHI->addIncoming(SelectValue, SelectBB); 5331 5332 SmallPtrSet<BasicBlock *, 4> RemovedSuccessors; 5333 for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) { 5334 BasicBlock *Succ = SI->getSuccessor(i); 5335 5336 if (Succ == DestBB) 5337 continue; 5338 Succ->removePredecessor(SelectBB); 5339 if (DTU && RemovedSuccessors.insert(Succ).second) 5340 Updates.push_back({DominatorTree::Delete, SelectBB, Succ}); 5341 } 5342 SI->eraseFromParent(); 5343 if (DTU) 5344 DTU->applyUpdates(Updates); 5345 } 5346 5347 /// If the switch is only used to initialize one or more 5348 /// phi nodes in a common successor block with only two different 5349 /// constant values, replace the switch with select. 5350 static bool switchToSelect(SwitchInst *SI, IRBuilder<> &Builder, 5351 DomTreeUpdater *DTU, const DataLayout &DL, 5352 const TargetTransformInfo &TTI) { 5353 Value *const Cond = SI->getCondition(); 5354 PHINode *PHI = nullptr; 5355 BasicBlock *CommonDest = nullptr; 5356 Constant *DefaultResult; 5357 SwitchCaseResultVectorTy UniqueResults; 5358 // Collect all the cases that will deliver the same value from the switch. 5359 if (!InitializeUniqueCases(SI, PHI, CommonDest, UniqueResults, DefaultResult, 5360 DL, TTI, /*MaxUniqueResults*/2, 5361 /*MaxCasesPerResult*/2)) 5362 return false; 5363 assert(PHI != nullptr && "PHI for value select not found"); 5364 5365 Builder.SetInsertPoint(SI); 5366 Value *SelectValue = 5367 ConvertTwoCaseSwitch(UniqueResults, DefaultResult, Cond, Builder); 5368 if (SelectValue) { 5369 RemoveSwitchAfterSelectConversion(SI, PHI, SelectValue, Builder, DTU); 5370 return true; 5371 } 5372 // The switch couldn't be converted into a select. 5373 return false; 5374 } 5375 5376 namespace { 5377 5378 /// This class represents a lookup table that can be used to replace a switch. 5379 class SwitchLookupTable { 5380 public: 5381 /// Create a lookup table to use as a switch replacement with the contents 5382 /// of Values, using DefaultValue to fill any holes in the table. 5383 SwitchLookupTable( 5384 Module &M, uint64_t TableSize, ConstantInt *Offset, 5385 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values, 5386 Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName); 5387 5388 /// Build instructions with Builder to retrieve the value at 5389 /// the position given by Index in the lookup table. 5390 Value *BuildLookup(Value *Index, IRBuilder<> &Builder); 5391 5392 /// Return true if a table with TableSize elements of 5393 /// type ElementType would fit in a target-legal register. 5394 static bool WouldFitInRegister(const DataLayout &DL, uint64_t TableSize, 5395 Type *ElementType); 5396 5397 private: 5398 // Depending on the contents of the table, it can be represented in 5399 // different ways. 5400 enum { 5401 // For tables where each element contains the same value, we just have to 5402 // store that single value and return it for each lookup. 5403 SingleValueKind, 5404 5405 // For tables where there is a linear relationship between table index 5406 // and values. We calculate the result with a simple multiplication 5407 // and addition instead of a table lookup. 5408 LinearMapKind, 5409 5410 // For small tables with integer elements, we can pack them into a bitmap 5411 // that fits into a target-legal register. Values are retrieved by 5412 // shift and mask operations. 5413 BitMapKind, 5414 5415 // The table is stored as an array of values. Values are retrieved by load 5416 // instructions from the table. 5417 ArrayKind 5418 } Kind; 5419 5420 // For SingleValueKind, this is the single value. 5421 Constant *SingleValue = nullptr; 5422 5423 // For BitMapKind, this is the bitmap. 5424 ConstantInt *BitMap = nullptr; 5425 IntegerType *BitMapElementTy = nullptr; 5426 5427 // For LinearMapKind, these are the constants used to derive the value. 5428 ConstantInt *LinearOffset = nullptr; 5429 ConstantInt *LinearMultiplier = nullptr; 5430 5431 // For ArrayKind, this is the array. 5432 GlobalVariable *Array = nullptr; 5433 }; 5434 5435 } // end anonymous namespace 5436 5437 SwitchLookupTable::SwitchLookupTable( 5438 Module &M, uint64_t TableSize, ConstantInt *Offset, 5439 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values, 5440 Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName) { 5441 assert(Values.size() && "Can't build lookup table without values!"); 5442 assert(TableSize >= Values.size() && "Can't fit values in table!"); 5443 5444 // If all values in the table are equal, this is that value. 5445 SingleValue = Values.begin()->second; 5446 5447 Type *ValueType = Values.begin()->second->getType(); 5448 5449 // Build up the table contents. 5450 SmallVector<Constant *, 64> TableContents(TableSize); 5451 for (size_t I = 0, E = Values.size(); I != E; ++I) { 5452 ConstantInt *CaseVal = Values[I].first; 5453 Constant *CaseRes = Values[I].second; 5454 assert(CaseRes->getType() == ValueType); 5455 5456 uint64_t Idx = (CaseVal->getValue() - Offset->getValue()).getLimitedValue(); 5457 TableContents[Idx] = CaseRes; 5458 5459 if (CaseRes != SingleValue) 5460 SingleValue = nullptr; 5461 } 5462 5463 // Fill in any holes in the table with the default result. 5464 if (Values.size() < TableSize) { 5465 assert(DefaultValue && 5466 "Need a default value to fill the lookup table holes."); 5467 assert(DefaultValue->getType() == ValueType); 5468 for (uint64_t I = 0; I < TableSize; ++I) { 5469 if (!TableContents[I]) 5470 TableContents[I] = DefaultValue; 5471 } 5472 5473 if (DefaultValue != SingleValue) 5474 SingleValue = nullptr; 5475 } 5476 5477 // If each element in the table contains the same value, we only need to store 5478 // that single value. 5479 if (SingleValue) { 5480 Kind = SingleValueKind; 5481 return; 5482 } 5483 5484 // Check if we can derive the value with a linear transformation from the 5485 // table index. 5486 if (isa<IntegerType>(ValueType)) { 5487 bool LinearMappingPossible = true; 5488 APInt PrevVal; 5489 APInt DistToPrev; 5490 assert(TableSize >= 2 && "Should be a SingleValue table."); 5491 // Check if there is the same distance between two consecutive values. 5492 for (uint64_t I = 0; I < TableSize; ++I) { 5493 ConstantInt *ConstVal = dyn_cast<ConstantInt>(TableContents[I]); 5494 if (!ConstVal) { 5495 // This is an undef. We could deal with it, but undefs in lookup tables 5496 // are very seldom. It's probably not worth the additional complexity. 5497 LinearMappingPossible = false; 5498 break; 5499 } 5500 const APInt &Val = ConstVal->getValue(); 5501 if (I != 0) { 5502 APInt Dist = Val - PrevVal; 5503 if (I == 1) { 5504 DistToPrev = Dist; 5505 } else if (Dist != DistToPrev) { 5506 LinearMappingPossible = false; 5507 break; 5508 } 5509 } 5510 PrevVal = Val; 5511 } 5512 if (LinearMappingPossible) { 5513 LinearOffset = cast<ConstantInt>(TableContents[0]); 5514 LinearMultiplier = ConstantInt::get(M.getContext(), DistToPrev); 5515 Kind = LinearMapKind; 5516 ++NumLinearMaps; 5517 return; 5518 } 5519 } 5520 5521 // If the type is integer and the table fits in a register, build a bitmap. 5522 if (WouldFitInRegister(DL, TableSize, ValueType)) { 5523 IntegerType *IT = cast<IntegerType>(ValueType); 5524 APInt TableInt(TableSize * IT->getBitWidth(), 0); 5525 for (uint64_t I = TableSize; I > 0; --I) { 5526 TableInt <<= IT->getBitWidth(); 5527 // Insert values into the bitmap. Undef values are set to zero. 5528 if (!isa<UndefValue>(TableContents[I - 1])) { 5529 ConstantInt *Val = cast<ConstantInt>(TableContents[I - 1]); 5530 TableInt |= Val->getValue().zext(TableInt.getBitWidth()); 5531 } 5532 } 5533 BitMap = ConstantInt::get(M.getContext(), TableInt); 5534 BitMapElementTy = IT; 5535 Kind = BitMapKind; 5536 ++NumBitMaps; 5537 return; 5538 } 5539 5540 // Store the table in an array. 5541 ArrayType *ArrayTy = ArrayType::get(ValueType, TableSize); 5542 Constant *Initializer = ConstantArray::get(ArrayTy, TableContents); 5543 5544 Array = new GlobalVariable(M, ArrayTy, /*isConstant=*/true, 5545 GlobalVariable::PrivateLinkage, Initializer, 5546 "switch.table." + FuncName); 5547 Array->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); 5548 // Set the alignment to that of an array items. We will be only loading one 5549 // value out of it. 5550 Array->setAlignment(Align(DL.getPrefTypeAlignment(ValueType))); 5551 Kind = ArrayKind; 5552 } 5553 5554 Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) { 5555 switch (Kind) { 5556 case SingleValueKind: 5557 return SingleValue; 5558 case LinearMapKind: { 5559 // Derive the result value from the input value. 5560 Value *Result = Builder.CreateIntCast(Index, LinearMultiplier->getType(), 5561 false, "switch.idx.cast"); 5562 if (!LinearMultiplier->isOne()) 5563 Result = Builder.CreateMul(Result, LinearMultiplier, "switch.idx.mult"); 5564 if (!LinearOffset->isZero()) 5565 Result = Builder.CreateAdd(Result, LinearOffset, "switch.offset"); 5566 return Result; 5567 } 5568 case BitMapKind: { 5569 // Type of the bitmap (e.g. i59). 5570 IntegerType *MapTy = BitMap->getType(); 5571 5572 // Cast Index to the same type as the bitmap. 5573 // Note: The Index is <= the number of elements in the table, so 5574 // truncating it to the width of the bitmask is safe. 5575 Value *ShiftAmt = Builder.CreateZExtOrTrunc(Index, MapTy, "switch.cast"); 5576 5577 // Multiply the shift amount by the element width. 5578 ShiftAmt = Builder.CreateMul( 5579 ShiftAmt, ConstantInt::get(MapTy, BitMapElementTy->getBitWidth()), 5580 "switch.shiftamt"); 5581 5582 // Shift down. 5583 Value *DownShifted = 5584 Builder.CreateLShr(BitMap, ShiftAmt, "switch.downshift"); 5585 // Mask off. 5586 return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked"); 5587 } 5588 case ArrayKind: { 5589 // Make sure the table index will not overflow when treated as signed. 5590 IntegerType *IT = cast<IntegerType>(Index->getType()); 5591 uint64_t TableSize = 5592 Array->getInitializer()->getType()->getArrayNumElements(); 5593 if (TableSize > (1ULL << (IT->getBitWidth() - 1))) 5594 Index = Builder.CreateZExt( 5595 Index, IntegerType::get(IT->getContext(), IT->getBitWidth() + 1), 5596 "switch.tableidx.zext"); 5597 5598 Value *GEPIndices[] = {Builder.getInt32(0), Index}; 5599 Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array, 5600 GEPIndices, "switch.gep"); 5601 return Builder.CreateLoad( 5602 cast<ArrayType>(Array->getValueType())->getElementType(), GEP, 5603 "switch.load"); 5604 } 5605 } 5606 llvm_unreachable("Unknown lookup table kind!"); 5607 } 5608 5609 bool SwitchLookupTable::WouldFitInRegister(const DataLayout &DL, 5610 uint64_t TableSize, 5611 Type *ElementType) { 5612 auto *IT = dyn_cast<IntegerType>(ElementType); 5613 if (!IT) 5614 return false; 5615 // FIXME: If the type is wider than it needs to be, e.g. i8 but all values 5616 // are <= 15, we could try to narrow the type. 5617 5618 // Avoid overflow, fitsInLegalInteger uses unsigned int for the width. 5619 if (TableSize >= UINT_MAX / IT->getBitWidth()) 5620 return false; 5621 return DL.fitsInLegalInteger(TableSize * IT->getBitWidth()); 5622 } 5623 5624 /// Determine whether a lookup table should be built for this switch, based on 5625 /// the number of cases, size of the table, and the types of the results. 5626 static bool 5627 ShouldBuildLookupTable(SwitchInst *SI, uint64_t TableSize, 5628 const TargetTransformInfo &TTI, const DataLayout &DL, 5629 const SmallDenseMap<PHINode *, Type *> &ResultTypes) { 5630 if (SI->getNumCases() > TableSize || TableSize >= UINT64_MAX / 10) 5631 return false; // TableSize overflowed, or mul below might overflow. 5632 5633 bool AllTablesFitInRegister = true; 5634 bool HasIllegalType = false; 5635 for (const auto &I : ResultTypes) { 5636 Type *Ty = I.second; 5637 5638 // Saturate this flag to true. 5639 HasIllegalType = HasIllegalType || !TTI.isTypeLegal(Ty); 5640 5641 // Saturate this flag to false. 5642 AllTablesFitInRegister = 5643 AllTablesFitInRegister && 5644 SwitchLookupTable::WouldFitInRegister(DL, TableSize, Ty); 5645 5646 // If both flags saturate, we're done. NOTE: This *only* works with 5647 // saturating flags, and all flags have to saturate first due to the 5648 // non-deterministic behavior of iterating over a dense map. 5649 if (HasIllegalType && !AllTablesFitInRegister) 5650 break; 5651 } 5652 5653 // If each table would fit in a register, we should build it anyway. 5654 if (AllTablesFitInRegister) 5655 return true; 5656 5657 // Don't build a table that doesn't fit in-register if it has illegal types. 5658 if (HasIllegalType) 5659 return false; 5660 5661 // The table density should be at least 40%. This is the same criterion as for 5662 // jump tables, see SelectionDAGBuilder::handleJTSwitchCase. 5663 // FIXME: Find the best cut-off. 5664 return SI->getNumCases() * 10 >= TableSize * 4; 5665 } 5666 5667 /// Try to reuse the switch table index compare. Following pattern: 5668 /// \code 5669 /// if (idx < tablesize) 5670 /// r = table[idx]; // table does not contain default_value 5671 /// else 5672 /// r = default_value; 5673 /// if (r != default_value) 5674 /// ... 5675 /// \endcode 5676 /// Is optimized to: 5677 /// \code 5678 /// cond = idx < tablesize; 5679 /// if (cond) 5680 /// r = table[idx]; 5681 /// else 5682 /// r = default_value; 5683 /// if (cond) 5684 /// ... 5685 /// \endcode 5686 /// Jump threading will then eliminate the second if(cond). 5687 static void reuseTableCompare( 5688 User *PhiUser, BasicBlock *PhiBlock, BranchInst *RangeCheckBranch, 5689 Constant *DefaultValue, 5690 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values) { 5691 ICmpInst *CmpInst = dyn_cast<ICmpInst>(PhiUser); 5692 if (!CmpInst) 5693 return; 5694 5695 // We require that the compare is in the same block as the phi so that jump 5696 // threading can do its work afterwards. 5697 if (CmpInst->getParent() != PhiBlock) 5698 return; 5699 5700 Constant *CmpOp1 = dyn_cast<Constant>(CmpInst->getOperand(1)); 5701 if (!CmpOp1) 5702 return; 5703 5704 Value *RangeCmp = RangeCheckBranch->getCondition(); 5705 Constant *TrueConst = ConstantInt::getTrue(RangeCmp->getType()); 5706 Constant *FalseConst = ConstantInt::getFalse(RangeCmp->getType()); 5707 5708 // Check if the compare with the default value is constant true or false. 5709 Constant *DefaultConst = ConstantExpr::getICmp(CmpInst->getPredicate(), 5710 DefaultValue, CmpOp1, true); 5711 if (DefaultConst != TrueConst && DefaultConst != FalseConst) 5712 return; 5713 5714 // Check if the compare with the case values is distinct from the default 5715 // compare result. 5716 for (auto ValuePair : Values) { 5717 Constant *CaseConst = ConstantExpr::getICmp(CmpInst->getPredicate(), 5718 ValuePair.second, CmpOp1, true); 5719 if (!CaseConst || CaseConst == DefaultConst || isa<UndefValue>(CaseConst)) 5720 return; 5721 assert((CaseConst == TrueConst || CaseConst == FalseConst) && 5722 "Expect true or false as compare result."); 5723 } 5724 5725 // Check if the branch instruction dominates the phi node. It's a simple 5726 // dominance check, but sufficient for our needs. 5727 // Although this check is invariant in the calling loops, it's better to do it 5728 // at this late stage. Practically we do it at most once for a switch. 5729 BasicBlock *BranchBlock = RangeCheckBranch->getParent(); 5730 for (BasicBlock *Pred : predecessors(PhiBlock)) { 5731 if (Pred != BranchBlock && Pred->getUniquePredecessor() != BranchBlock) 5732 return; 5733 } 5734 5735 if (DefaultConst == FalseConst) { 5736 // The compare yields the same result. We can replace it. 5737 CmpInst->replaceAllUsesWith(RangeCmp); 5738 ++NumTableCmpReuses; 5739 } else { 5740 // The compare yields the same result, just inverted. We can replace it. 5741 Value *InvertedTableCmp = BinaryOperator::CreateXor( 5742 RangeCmp, ConstantInt::get(RangeCmp->getType(), 1), "inverted.cmp", 5743 RangeCheckBranch); 5744 CmpInst->replaceAllUsesWith(InvertedTableCmp); 5745 ++NumTableCmpReuses; 5746 } 5747 } 5748 5749 /// If the switch is only used to initialize one or more phi nodes in a common 5750 /// successor block with different constant values, replace the switch with 5751 /// lookup tables. 5752 static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder, 5753 DomTreeUpdater *DTU, const DataLayout &DL, 5754 const TargetTransformInfo &TTI) { 5755 assert(SI->getNumCases() > 1 && "Degenerate switch?"); 5756 5757 BasicBlock *BB = SI->getParent(); 5758 Function *Fn = BB->getParent(); 5759 // Only build lookup table when we have a target that supports it or the 5760 // attribute is not set. 5761 if (!TTI.shouldBuildLookupTables() || 5762 (Fn->getFnAttribute("no-jump-tables").getValueAsBool())) 5763 return false; 5764 5765 // FIXME: If the switch is too sparse for a lookup table, perhaps we could 5766 // split off a dense part and build a lookup table for that. 5767 5768 // FIXME: This creates arrays of GEPs to constant strings, which means each 5769 // GEP needs a runtime relocation in PIC code. We should just build one big 5770 // string and lookup indices into that. 5771 5772 // Ignore switches with less than three cases. Lookup tables will not make 5773 // them faster, so we don't analyze them. 5774 if (SI->getNumCases() < 3) 5775 return false; 5776 5777 // Figure out the corresponding result for each case value and phi node in the 5778 // common destination, as well as the min and max case values. 5779 assert(!SI->cases().empty()); 5780 SwitchInst::CaseIt CI = SI->case_begin(); 5781 ConstantInt *MinCaseVal = CI->getCaseValue(); 5782 ConstantInt *MaxCaseVal = CI->getCaseValue(); 5783 5784 BasicBlock *CommonDest = nullptr; 5785 5786 using ResultListTy = SmallVector<std::pair<ConstantInt *, Constant *>, 4>; 5787 SmallDenseMap<PHINode *, ResultListTy> ResultLists; 5788 5789 SmallDenseMap<PHINode *, Constant *> DefaultResults; 5790 SmallDenseMap<PHINode *, Type *> ResultTypes; 5791 SmallVector<PHINode *, 4> PHIs; 5792 5793 for (SwitchInst::CaseIt E = SI->case_end(); CI != E; ++CI) { 5794 ConstantInt *CaseVal = CI->getCaseValue(); 5795 if (CaseVal->getValue().slt(MinCaseVal->getValue())) 5796 MinCaseVal = CaseVal; 5797 if (CaseVal->getValue().sgt(MaxCaseVal->getValue())) 5798 MaxCaseVal = CaseVal; 5799 5800 // Resulting value at phi nodes for this case value. 5801 using ResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>; 5802 ResultsTy Results; 5803 if (!GetCaseResults(SI, CaseVal, CI->getCaseSuccessor(), &CommonDest, 5804 Results, DL, TTI)) 5805 return false; 5806 5807 // Append the result from this case to the list for each phi. 5808 for (const auto &I : Results) { 5809 PHINode *PHI = I.first; 5810 Constant *Value = I.second; 5811 if (!ResultLists.count(PHI)) 5812 PHIs.push_back(PHI); 5813 ResultLists[PHI].push_back(std::make_pair(CaseVal, Value)); 5814 } 5815 } 5816 5817 // Keep track of the result types. 5818 for (PHINode *PHI : PHIs) { 5819 ResultTypes[PHI] = ResultLists[PHI][0].second->getType(); 5820 } 5821 5822 uint64_t NumResults = ResultLists[PHIs[0]].size(); 5823 APInt RangeSpread = MaxCaseVal->getValue() - MinCaseVal->getValue(); 5824 uint64_t TableSize = RangeSpread.getLimitedValue() + 1; 5825 bool TableHasHoles = (NumResults < TableSize); 5826 5827 // If the table has holes, we need a constant result for the default case 5828 // or a bitmask that fits in a register. 5829 SmallVector<std::pair<PHINode *, Constant *>, 4> DefaultResultsList; 5830 bool HasDefaultResults = 5831 GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, 5832 DefaultResultsList, DL, TTI); 5833 5834 bool NeedMask = (TableHasHoles && !HasDefaultResults); 5835 if (NeedMask) { 5836 // As an extra penalty for the validity test we require more cases. 5837 if (SI->getNumCases() < 4) // FIXME: Find best threshold value (benchmark). 5838 return false; 5839 if (!DL.fitsInLegalInteger(TableSize)) 5840 return false; 5841 } 5842 5843 for (const auto &I : DefaultResultsList) { 5844 PHINode *PHI = I.first; 5845 Constant *Result = I.second; 5846 DefaultResults[PHI] = Result; 5847 } 5848 5849 if (!ShouldBuildLookupTable(SI, TableSize, TTI, DL, ResultTypes)) 5850 return false; 5851 5852 std::vector<DominatorTree::UpdateType> Updates; 5853 5854 // Create the BB that does the lookups. 5855 Module &Mod = *CommonDest->getParent()->getParent(); 5856 BasicBlock *LookupBB = BasicBlock::Create( 5857 Mod.getContext(), "switch.lookup", CommonDest->getParent(), CommonDest); 5858 5859 // Compute the table index value. 5860 Builder.SetInsertPoint(SI); 5861 Value *TableIndex; 5862 if (MinCaseVal->isNullValue()) 5863 TableIndex = SI->getCondition(); 5864 else 5865 TableIndex = Builder.CreateSub(SI->getCondition(), MinCaseVal, 5866 "switch.tableidx"); 5867 5868 // Compute the maximum table size representable by the integer type we are 5869 // switching upon. 5870 unsigned CaseSize = MinCaseVal->getType()->getPrimitiveSizeInBits(); 5871 uint64_t MaxTableSize = CaseSize > 63 ? UINT64_MAX : 1ULL << CaseSize; 5872 assert(MaxTableSize >= TableSize && 5873 "It is impossible for a switch to have more entries than the max " 5874 "representable value of its input integer type's size."); 5875 5876 // If the default destination is unreachable, or if the lookup table covers 5877 // all values of the conditional variable, branch directly to the lookup table 5878 // BB. Otherwise, check that the condition is within the case range. 5879 const bool DefaultIsReachable = 5880 !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg()); 5881 const bool GeneratingCoveredLookupTable = (MaxTableSize == TableSize); 5882 BranchInst *RangeCheckBranch = nullptr; 5883 5884 if (!DefaultIsReachable || GeneratingCoveredLookupTable) { 5885 Builder.CreateBr(LookupBB); 5886 if (DTU) 5887 Updates.push_back({DominatorTree::Insert, BB, LookupBB}); 5888 // Note: We call removeProdecessor later since we need to be able to get the 5889 // PHI value for the default case in case we're using a bit mask. 5890 } else { 5891 Value *Cmp = Builder.CreateICmpULT( 5892 TableIndex, ConstantInt::get(MinCaseVal->getType(), TableSize)); 5893 RangeCheckBranch = 5894 Builder.CreateCondBr(Cmp, LookupBB, SI->getDefaultDest()); 5895 if (DTU) 5896 Updates.push_back({DominatorTree::Insert, BB, LookupBB}); 5897 } 5898 5899 // Populate the BB that does the lookups. 5900 Builder.SetInsertPoint(LookupBB); 5901 5902 if (NeedMask) { 5903 // Before doing the lookup, we do the hole check. The LookupBB is therefore 5904 // re-purposed to do the hole check, and we create a new LookupBB. 5905 BasicBlock *MaskBB = LookupBB; 5906 MaskBB->setName("switch.hole_check"); 5907 LookupBB = BasicBlock::Create(Mod.getContext(), "switch.lookup", 5908 CommonDest->getParent(), CommonDest); 5909 5910 // Make the mask's bitwidth at least 8-bit and a power-of-2 to avoid 5911 // unnecessary illegal types. 5912 uint64_t TableSizePowOf2 = NextPowerOf2(std::max(7ULL, TableSize - 1ULL)); 5913 APInt MaskInt(TableSizePowOf2, 0); 5914 APInt One(TableSizePowOf2, 1); 5915 // Build bitmask; fill in a 1 bit for every case. 5916 const ResultListTy &ResultList = ResultLists[PHIs[0]]; 5917 for (size_t I = 0, E = ResultList.size(); I != E; ++I) { 5918 uint64_t Idx = (ResultList[I].first->getValue() - MinCaseVal->getValue()) 5919 .getLimitedValue(); 5920 MaskInt |= One << Idx; 5921 } 5922 ConstantInt *TableMask = ConstantInt::get(Mod.getContext(), MaskInt); 5923 5924 // Get the TableIndex'th bit of the bitmask. 5925 // If this bit is 0 (meaning hole) jump to the default destination, 5926 // else continue with table lookup. 5927 IntegerType *MapTy = TableMask->getType(); 5928 Value *MaskIndex = 5929 Builder.CreateZExtOrTrunc(TableIndex, MapTy, "switch.maskindex"); 5930 Value *Shifted = Builder.CreateLShr(TableMask, MaskIndex, "switch.shifted"); 5931 Value *LoBit = Builder.CreateTrunc( 5932 Shifted, Type::getInt1Ty(Mod.getContext()), "switch.lobit"); 5933 Builder.CreateCondBr(LoBit, LookupBB, SI->getDefaultDest()); 5934 if (DTU) { 5935 Updates.push_back({DominatorTree::Insert, MaskBB, LookupBB}); 5936 Updates.push_back({DominatorTree::Insert, MaskBB, SI->getDefaultDest()}); 5937 } 5938 Builder.SetInsertPoint(LookupBB); 5939 AddPredecessorToBlock(SI->getDefaultDest(), MaskBB, BB); 5940 } 5941 5942 if (!DefaultIsReachable || GeneratingCoveredLookupTable) { 5943 // We cached PHINodes in PHIs. To avoid accessing deleted PHINodes later, 5944 // do not delete PHINodes here. 5945 SI->getDefaultDest()->removePredecessor(BB, 5946 /*KeepOneInputPHIs=*/true); 5947 if (DTU) 5948 Updates.push_back({DominatorTree::Delete, BB, SI->getDefaultDest()}); 5949 } 5950 5951 for (PHINode *PHI : PHIs) { 5952 const ResultListTy &ResultList = ResultLists[PHI]; 5953 5954 // If using a bitmask, use any value to fill the lookup table holes. 5955 Constant *DV = NeedMask ? ResultLists[PHI][0].second : DefaultResults[PHI]; 5956 StringRef FuncName = Fn->getName(); 5957 SwitchLookupTable Table(Mod, TableSize, MinCaseVal, ResultList, DV, DL, 5958 FuncName); 5959 5960 Value *Result = Table.BuildLookup(TableIndex, Builder); 5961 5962 // Do a small peephole optimization: re-use the switch table compare if 5963 // possible. 5964 if (!TableHasHoles && HasDefaultResults && RangeCheckBranch) { 5965 BasicBlock *PhiBlock = PHI->getParent(); 5966 // Search for compare instructions which use the phi. 5967 for (auto *User : PHI->users()) { 5968 reuseTableCompare(User, PhiBlock, RangeCheckBranch, DV, ResultList); 5969 } 5970 } 5971 5972 PHI->addIncoming(Result, LookupBB); 5973 } 5974 5975 Builder.CreateBr(CommonDest); 5976 if (DTU) 5977 Updates.push_back({DominatorTree::Insert, LookupBB, CommonDest}); 5978 5979 // Remove the switch. 5980 SmallPtrSet<BasicBlock *, 8> RemovedSuccessors; 5981 for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) { 5982 BasicBlock *Succ = SI->getSuccessor(i); 5983 5984 if (Succ == SI->getDefaultDest()) 5985 continue; 5986 Succ->removePredecessor(BB); 5987 RemovedSuccessors.insert(Succ); 5988 } 5989 SI->eraseFromParent(); 5990 5991 if (DTU) { 5992 for (BasicBlock *RemovedSuccessor : RemovedSuccessors) 5993 Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor}); 5994 DTU->applyUpdates(Updates); 5995 } 5996 5997 ++NumLookupTables; 5998 if (NeedMask) 5999 ++NumLookupTablesHoles; 6000 return true; 6001 } 6002 6003 static bool isSwitchDense(ArrayRef<int64_t> Values) { 6004 // See also SelectionDAGBuilder::isDense(), which this function was based on. 6005 uint64_t Diff = (uint64_t)Values.back() - (uint64_t)Values.front(); 6006 uint64_t Range = Diff + 1; 6007 uint64_t NumCases = Values.size(); 6008 // 40% is the default density for building a jump table in optsize/minsize mode. 6009 uint64_t MinDensity = 40; 6010 6011 return NumCases * 100 >= Range * MinDensity; 6012 } 6013 6014 /// Try to transform a switch that has "holes" in it to a contiguous sequence 6015 /// of cases. 6016 /// 6017 /// A switch such as: switch(i) {case 5: case 9: case 13: case 17:} can be 6018 /// range-reduced to: switch ((i-5) / 4) {case 0: case 1: case 2: case 3:}. 6019 /// 6020 /// This converts a sparse switch into a dense switch which allows better 6021 /// lowering and could also allow transforming into a lookup table. 6022 static bool ReduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder, 6023 const DataLayout &DL, 6024 const TargetTransformInfo &TTI) { 6025 auto *CondTy = cast<IntegerType>(SI->getCondition()->getType()); 6026 if (CondTy->getIntegerBitWidth() > 64 || 6027 !DL.fitsInLegalInteger(CondTy->getIntegerBitWidth())) 6028 return false; 6029 // Only bother with this optimization if there are more than 3 switch cases; 6030 // SDAG will only bother creating jump tables for 4 or more cases. 6031 if (SI->getNumCases() < 4) 6032 return false; 6033 6034 // This transform is agnostic to the signedness of the input or case values. We 6035 // can treat the case values as signed or unsigned. We can optimize more common 6036 // cases such as a sequence crossing zero {-4,0,4,8} if we interpret case values 6037 // as signed. 6038 SmallVector<int64_t,4> Values; 6039 for (auto &C : SI->cases()) 6040 Values.push_back(C.getCaseValue()->getValue().getSExtValue()); 6041 llvm::sort(Values); 6042 6043 // If the switch is already dense, there's nothing useful to do here. 6044 if (isSwitchDense(Values)) 6045 return false; 6046 6047 // First, transform the values such that they start at zero and ascend. 6048 int64_t Base = Values[0]; 6049 for (auto &V : Values) 6050 V -= (uint64_t)(Base); 6051 6052 // Now we have signed numbers that have been shifted so that, given enough 6053 // precision, there are no negative values. Since the rest of the transform 6054 // is bitwise only, we switch now to an unsigned representation. 6055 6056 // This transform can be done speculatively because it is so cheap - it 6057 // results in a single rotate operation being inserted. 6058 // FIXME: It's possible that optimizing a switch on powers of two might also 6059 // be beneficial - flag values are often powers of two and we could use a CLZ 6060 // as the key function. 6061 6062 // countTrailingZeros(0) returns 64. As Values is guaranteed to have more than 6063 // one element and LLVM disallows duplicate cases, Shift is guaranteed to be 6064 // less than 64. 6065 unsigned Shift = 64; 6066 for (auto &V : Values) 6067 Shift = std::min(Shift, countTrailingZeros((uint64_t)V)); 6068 assert(Shift < 64); 6069 if (Shift > 0) 6070 for (auto &V : Values) 6071 V = (int64_t)((uint64_t)V >> Shift); 6072 6073 if (!isSwitchDense(Values)) 6074 // Transform didn't create a dense switch. 6075 return false; 6076 6077 // The obvious transform is to shift the switch condition right and emit a 6078 // check that the condition actually cleanly divided by GCD, i.e. 6079 // C & (1 << Shift - 1) == 0 6080 // inserting a new CFG edge to handle the case where it didn't divide cleanly. 6081 // 6082 // A cheaper way of doing this is a simple ROTR(C, Shift). This performs the 6083 // shift and puts the shifted-off bits in the uppermost bits. If any of these 6084 // are nonzero then the switch condition will be very large and will hit the 6085 // default case. 6086 6087 auto *Ty = cast<IntegerType>(SI->getCondition()->getType()); 6088 Builder.SetInsertPoint(SI); 6089 auto *ShiftC = ConstantInt::get(Ty, Shift); 6090 auto *Sub = Builder.CreateSub(SI->getCondition(), ConstantInt::get(Ty, Base)); 6091 auto *LShr = Builder.CreateLShr(Sub, ShiftC); 6092 auto *Shl = Builder.CreateShl(Sub, Ty->getBitWidth() - Shift); 6093 auto *Rot = Builder.CreateOr(LShr, Shl); 6094 SI->replaceUsesOfWith(SI->getCondition(), Rot); 6095 6096 for (auto Case : SI->cases()) { 6097 auto *Orig = Case.getCaseValue(); 6098 auto Sub = Orig->getValue() - APInt(Ty->getBitWidth(), Base); 6099 Case.setValue( 6100 cast<ConstantInt>(ConstantInt::get(Ty, Sub.lshr(ShiftC->getValue())))); 6101 } 6102 return true; 6103 } 6104 6105 bool SimplifyCFGOpt::simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) { 6106 BasicBlock *BB = SI->getParent(); 6107 6108 if (isValueEqualityComparison(SI)) { 6109 // If we only have one predecessor, and if it is a branch on this value, 6110 // see if that predecessor totally determines the outcome of this switch. 6111 if (BasicBlock *OnlyPred = BB->getSinglePredecessor()) 6112 if (SimplifyEqualityComparisonWithOnlyPredecessor(SI, OnlyPred, Builder)) 6113 return requestResimplify(); 6114 6115 Value *Cond = SI->getCondition(); 6116 if (SelectInst *Select = dyn_cast<SelectInst>(Cond)) 6117 if (SimplifySwitchOnSelect(SI, Select)) 6118 return requestResimplify(); 6119 6120 // If the block only contains the switch, see if we can fold the block 6121 // away into any preds. 6122 if (SI == &*BB->instructionsWithoutDebug().begin()) 6123 if (FoldValueComparisonIntoPredecessors(SI, Builder)) 6124 return requestResimplify(); 6125 } 6126 6127 // Try to transform the switch into an icmp and a branch. 6128 if (TurnSwitchRangeIntoICmp(SI, Builder)) 6129 return requestResimplify(); 6130 6131 // Remove unreachable cases. 6132 if (eliminateDeadSwitchCases(SI, DTU, Options.AC, DL)) 6133 return requestResimplify(); 6134 6135 if (switchToSelect(SI, Builder, DTU, DL, TTI)) 6136 return requestResimplify(); 6137 6138 if (Options.ForwardSwitchCondToPhi && ForwardSwitchConditionToPHI(SI)) 6139 return requestResimplify(); 6140 6141 // The conversion from switch to lookup tables results in difficult-to-analyze 6142 // code and makes pruning branches much harder. This is a problem if the 6143 // switch expression itself can still be restricted as a result of inlining or 6144 // CVP. Therefore, only apply this transformation during late stages of the 6145 // optimisation pipeline. 6146 if (Options.ConvertSwitchToLookupTable && 6147 SwitchToLookupTable(SI, Builder, DTU, DL, TTI)) 6148 return requestResimplify(); 6149 6150 if (ReduceSwitchRange(SI, Builder, DL, TTI)) 6151 return requestResimplify(); 6152 6153 return false; 6154 } 6155 6156 bool SimplifyCFGOpt::simplifyIndirectBr(IndirectBrInst *IBI) { 6157 BasicBlock *BB = IBI->getParent(); 6158 bool Changed = false; 6159 6160 // Eliminate redundant destinations. 6161 SmallPtrSet<Value *, 8> Succs; 6162 SmallPtrSet<BasicBlock *, 8> RemovedSuccs; 6163 for (unsigned i = 0, e = IBI->getNumDestinations(); i != e; ++i) { 6164 BasicBlock *Dest = IBI->getDestination(i); 6165 if (!Dest->hasAddressTaken() || !Succs.insert(Dest).second) { 6166 if (!Dest->hasAddressTaken()) 6167 RemovedSuccs.insert(Dest); 6168 Dest->removePredecessor(BB); 6169 IBI->removeDestination(i); 6170 --i; 6171 --e; 6172 Changed = true; 6173 } 6174 } 6175 6176 if (DTU) { 6177 std::vector<DominatorTree::UpdateType> Updates; 6178 Updates.reserve(RemovedSuccs.size()); 6179 for (auto *RemovedSucc : RemovedSuccs) 6180 Updates.push_back({DominatorTree::Delete, BB, RemovedSucc}); 6181 DTU->applyUpdates(Updates); 6182 } 6183 6184 if (IBI->getNumDestinations() == 0) { 6185 // If the indirectbr has no successors, change it to unreachable. 6186 new UnreachableInst(IBI->getContext(), IBI); 6187 EraseTerminatorAndDCECond(IBI); 6188 return true; 6189 } 6190 6191 if (IBI->getNumDestinations() == 1) { 6192 // If the indirectbr has one successor, change it to a direct branch. 6193 BranchInst::Create(IBI->getDestination(0), IBI); 6194 EraseTerminatorAndDCECond(IBI); 6195 return true; 6196 } 6197 6198 if (SelectInst *SI = dyn_cast<SelectInst>(IBI->getAddress())) { 6199 if (SimplifyIndirectBrOnSelect(IBI, SI)) 6200 return requestResimplify(); 6201 } 6202 return Changed; 6203 } 6204 6205 /// Given an block with only a single landing pad and a unconditional branch 6206 /// try to find another basic block which this one can be merged with. This 6207 /// handles cases where we have multiple invokes with unique landing pads, but 6208 /// a shared handler. 6209 /// 6210 /// We specifically choose to not worry about merging non-empty blocks 6211 /// here. That is a PRE/scheduling problem and is best solved elsewhere. In 6212 /// practice, the optimizer produces empty landing pad blocks quite frequently 6213 /// when dealing with exception dense code. (see: instcombine, gvn, if-else 6214 /// sinking in this file) 6215 /// 6216 /// This is primarily a code size optimization. We need to avoid performing 6217 /// any transform which might inhibit optimization (such as our ability to 6218 /// specialize a particular handler via tail commoning). We do this by not 6219 /// merging any blocks which require us to introduce a phi. Since the same 6220 /// values are flowing through both blocks, we don't lose any ability to 6221 /// specialize. If anything, we make such specialization more likely. 6222 /// 6223 /// TODO - This transformation could remove entries from a phi in the target 6224 /// block when the inputs in the phi are the same for the two blocks being 6225 /// merged. In some cases, this could result in removal of the PHI entirely. 6226 static bool TryToMergeLandingPad(LandingPadInst *LPad, BranchInst *BI, 6227 BasicBlock *BB, DomTreeUpdater *DTU) { 6228 auto Succ = BB->getUniqueSuccessor(); 6229 assert(Succ); 6230 // If there's a phi in the successor block, we'd likely have to introduce 6231 // a phi into the merged landing pad block. 6232 if (isa<PHINode>(*Succ->begin())) 6233 return false; 6234 6235 for (BasicBlock *OtherPred : predecessors(Succ)) { 6236 if (BB == OtherPred) 6237 continue; 6238 BasicBlock::iterator I = OtherPred->begin(); 6239 LandingPadInst *LPad2 = dyn_cast<LandingPadInst>(I); 6240 if (!LPad2 || !LPad2->isIdenticalTo(LPad)) 6241 continue; 6242 for (++I; isa<DbgInfoIntrinsic>(I); ++I) 6243 ; 6244 BranchInst *BI2 = dyn_cast<BranchInst>(I); 6245 if (!BI2 || !BI2->isIdenticalTo(BI)) 6246 continue; 6247 6248 std::vector<DominatorTree::UpdateType> Updates; 6249 6250 // We've found an identical block. Update our predecessors to take that 6251 // path instead and make ourselves dead. 6252 SmallPtrSet<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB)); 6253 for (BasicBlock *Pred : Preds) { 6254 InvokeInst *II = cast<InvokeInst>(Pred->getTerminator()); 6255 assert(II->getNormalDest() != BB && II->getUnwindDest() == BB && 6256 "unexpected successor"); 6257 II->setUnwindDest(OtherPred); 6258 if (DTU) { 6259 Updates.push_back({DominatorTree::Insert, Pred, OtherPred}); 6260 Updates.push_back({DominatorTree::Delete, Pred, BB}); 6261 } 6262 } 6263 6264 // The debug info in OtherPred doesn't cover the merged control flow that 6265 // used to go through BB. We need to delete it or update it. 6266 for (auto I = OtherPred->begin(), E = OtherPred->end(); I != E;) { 6267 Instruction &Inst = *I; 6268 I++; 6269 if (isa<DbgInfoIntrinsic>(Inst)) 6270 Inst.eraseFromParent(); 6271 } 6272 6273 SmallPtrSet<BasicBlock *, 16> Succs(succ_begin(BB), succ_end(BB)); 6274 for (BasicBlock *Succ : Succs) { 6275 Succ->removePredecessor(BB); 6276 if (DTU) 6277 Updates.push_back({DominatorTree::Delete, BB, Succ}); 6278 } 6279 6280 IRBuilder<> Builder(BI); 6281 Builder.CreateUnreachable(); 6282 BI->eraseFromParent(); 6283 if (DTU) 6284 DTU->applyUpdates(Updates); 6285 return true; 6286 } 6287 return false; 6288 } 6289 6290 bool SimplifyCFGOpt::simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder) { 6291 return Branch->isUnconditional() ? simplifyUncondBranch(Branch, Builder) 6292 : simplifyCondBranch(Branch, Builder); 6293 } 6294 6295 bool SimplifyCFGOpt::simplifyUncondBranch(BranchInst *BI, 6296 IRBuilder<> &Builder) { 6297 BasicBlock *BB = BI->getParent(); 6298 BasicBlock *Succ = BI->getSuccessor(0); 6299 6300 // If the Terminator is the only non-phi instruction, simplify the block. 6301 // If LoopHeader is provided, check if the block or its successor is a loop 6302 // header. (This is for early invocations before loop simplify and 6303 // vectorization to keep canonical loop forms for nested loops. These blocks 6304 // can be eliminated when the pass is invoked later in the back-end.) 6305 // Note that if BB has only one predecessor then we do not introduce new 6306 // backedge, so we can eliminate BB. 6307 bool NeedCanonicalLoop = 6308 Options.NeedCanonicalLoop && 6309 (!LoopHeaders.empty() && BB->hasNPredecessorsOrMore(2) && 6310 (is_contained(LoopHeaders, BB) || is_contained(LoopHeaders, Succ))); 6311 BasicBlock::iterator I = BB->getFirstNonPHIOrDbg(true)->getIterator(); 6312 if (I->isTerminator() && BB != &BB->getParent()->getEntryBlock() && 6313 !NeedCanonicalLoop && TryToSimplifyUncondBranchFromEmptyBlock(BB, DTU)) 6314 return true; 6315 6316 // If the only instruction in the block is a seteq/setne comparison against a 6317 // constant, try to simplify the block. 6318 if (ICmpInst *ICI = dyn_cast<ICmpInst>(I)) 6319 if (ICI->isEquality() && isa<ConstantInt>(ICI->getOperand(1))) { 6320 for (++I; isa<DbgInfoIntrinsic>(I); ++I) 6321 ; 6322 if (I->isTerminator() && 6323 tryToSimplifyUncondBranchWithICmpInIt(ICI, Builder)) 6324 return true; 6325 } 6326 6327 // See if we can merge an empty landing pad block with another which is 6328 // equivalent. 6329 if (LandingPadInst *LPad = dyn_cast<LandingPadInst>(I)) { 6330 for (++I; isa<DbgInfoIntrinsic>(I); ++I) 6331 ; 6332 if (I->isTerminator() && TryToMergeLandingPad(LPad, BI, BB, DTU)) 6333 return true; 6334 } 6335 6336 // If this basic block is ONLY a compare and a branch, and if a predecessor 6337 // branches to us and our successor, fold the comparison into the 6338 // predecessor and use logical operations to update the incoming value 6339 // for PHI nodes in common successor. 6340 if (FoldBranchToCommonDest(BI, DTU, /*MSSAU=*/nullptr, &TTI, 6341 Options.BonusInstThreshold)) 6342 return requestResimplify(); 6343 return false; 6344 } 6345 6346 static BasicBlock *allPredecessorsComeFromSameSource(BasicBlock *BB) { 6347 BasicBlock *PredPred = nullptr; 6348 for (auto *P : predecessors(BB)) { 6349 BasicBlock *PPred = P->getSinglePredecessor(); 6350 if (!PPred || (PredPred && PredPred != PPred)) 6351 return nullptr; 6352 PredPred = PPred; 6353 } 6354 return PredPred; 6355 } 6356 6357 bool SimplifyCFGOpt::simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) { 6358 BasicBlock *BB = BI->getParent(); 6359 if (!Options.SimplifyCondBranch) 6360 return false; 6361 6362 // Conditional branch 6363 if (isValueEqualityComparison(BI)) { 6364 // If we only have one predecessor, and if it is a branch on this value, 6365 // see if that predecessor totally determines the outcome of this 6366 // switch. 6367 if (BasicBlock *OnlyPred = BB->getSinglePredecessor()) 6368 if (SimplifyEqualityComparisonWithOnlyPredecessor(BI, OnlyPred, Builder)) 6369 return requestResimplify(); 6370 6371 // This block must be empty, except for the setcond inst, if it exists. 6372 // Ignore dbg and pseudo intrinsics. 6373 auto I = BB->instructionsWithoutDebug(true).begin(); 6374 if (&*I == BI) { 6375 if (FoldValueComparisonIntoPredecessors(BI, Builder)) 6376 return requestResimplify(); 6377 } else if (&*I == cast<Instruction>(BI->getCondition())) { 6378 ++I; 6379 if (&*I == BI && FoldValueComparisonIntoPredecessors(BI, Builder)) 6380 return requestResimplify(); 6381 } 6382 } 6383 6384 // Try to turn "br (X == 0 | X == 1), T, F" into a switch instruction. 6385 if (SimplifyBranchOnICmpChain(BI, Builder, DL)) 6386 return true; 6387 6388 // If this basic block has dominating predecessor blocks and the dominating 6389 // blocks' conditions imply BI's condition, we know the direction of BI. 6390 Optional<bool> Imp = isImpliedByDomCondition(BI->getCondition(), BI, DL); 6391 if (Imp) { 6392 // Turn this into a branch on constant. 6393 auto *OldCond = BI->getCondition(); 6394 ConstantInt *TorF = *Imp ? ConstantInt::getTrue(BB->getContext()) 6395 : ConstantInt::getFalse(BB->getContext()); 6396 BI->setCondition(TorF); 6397 RecursivelyDeleteTriviallyDeadInstructions(OldCond); 6398 return requestResimplify(); 6399 } 6400 6401 // If this basic block is ONLY a compare and a branch, and if a predecessor 6402 // branches to us and one of our successors, fold the comparison into the 6403 // predecessor and use logical operations to pick the right destination. 6404 if (FoldBranchToCommonDest(BI, DTU, /*MSSAU=*/nullptr, &TTI, 6405 Options.BonusInstThreshold)) 6406 return requestResimplify(); 6407 6408 // We have a conditional branch to two blocks that are only reachable 6409 // from BI. We know that the condbr dominates the two blocks, so see if 6410 // there is any identical code in the "then" and "else" blocks. If so, we 6411 // can hoist it up to the branching block. 6412 if (BI->getSuccessor(0)->getSinglePredecessor()) { 6413 if (BI->getSuccessor(1)->getSinglePredecessor()) { 6414 if (HoistCommon && 6415 HoistThenElseCodeToIf(BI, TTI, !Options.HoistCommonInsts)) 6416 return requestResimplify(); 6417 } else { 6418 // If Successor #1 has multiple preds, we may be able to conditionally 6419 // execute Successor #0 if it branches to Successor #1. 6420 Instruction *Succ0TI = BI->getSuccessor(0)->getTerminator(); 6421 if (Succ0TI->getNumSuccessors() == 1 && 6422 Succ0TI->getSuccessor(0) == BI->getSuccessor(1)) 6423 if (SpeculativelyExecuteBB(BI, BI->getSuccessor(0), TTI)) 6424 return requestResimplify(); 6425 } 6426 } else if (BI->getSuccessor(1)->getSinglePredecessor()) { 6427 // If Successor #0 has multiple preds, we may be able to conditionally 6428 // execute Successor #1 if it branches to Successor #0. 6429 Instruction *Succ1TI = BI->getSuccessor(1)->getTerminator(); 6430 if (Succ1TI->getNumSuccessors() == 1 && 6431 Succ1TI->getSuccessor(0) == BI->getSuccessor(0)) 6432 if (SpeculativelyExecuteBB(BI, BI->getSuccessor(1), TTI)) 6433 return requestResimplify(); 6434 } 6435 6436 // If this is a branch on a phi node in the current block, thread control 6437 // through this block if any PHI node entries are constants. 6438 if (PHINode *PN = dyn_cast<PHINode>(BI->getCondition())) 6439 if (PN->getParent() == BI->getParent()) 6440 if (FoldCondBranchOnPHI(BI, DTU, DL, Options.AC)) 6441 return requestResimplify(); 6442 6443 // Scan predecessor blocks for conditional branches. 6444 for (BasicBlock *Pred : predecessors(BB)) 6445 if (BranchInst *PBI = dyn_cast<BranchInst>(Pred->getTerminator())) 6446 if (PBI != BI && PBI->isConditional()) 6447 if (SimplifyCondBranchToCondBranch(PBI, BI, DTU, DL, TTI)) 6448 return requestResimplify(); 6449 6450 // Look for diamond patterns. 6451 if (MergeCondStores) 6452 if (BasicBlock *PrevBB = allPredecessorsComeFromSameSource(BB)) 6453 if (BranchInst *PBI = dyn_cast<BranchInst>(PrevBB->getTerminator())) 6454 if (PBI != BI && PBI->isConditional()) 6455 if (mergeConditionalStores(PBI, BI, DTU, DL, TTI)) 6456 return requestResimplify(); 6457 6458 return false; 6459 } 6460 6461 /// Check if passing a value to an instruction will cause undefined behavior. 6462 static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified) { 6463 Constant *C = dyn_cast<Constant>(V); 6464 if (!C) 6465 return false; 6466 6467 if (I->use_empty()) 6468 return false; 6469 6470 if (C->isNullValue() || isa<UndefValue>(C)) { 6471 // Only look at the first use, avoid hurting compile time with long uselists 6472 User *Use = *I->user_begin(); 6473 6474 // Now make sure that there are no instructions in between that can alter 6475 // control flow (eg. calls) 6476 for (BasicBlock::iterator 6477 i = ++BasicBlock::iterator(I), 6478 UI = BasicBlock::iterator(dyn_cast<Instruction>(Use)); 6479 i != UI; ++i) { 6480 if (i == I->getParent()->end()) 6481 return false; 6482 if (!isGuaranteedToTransferExecutionToSuccessor(&*i)) 6483 return false; 6484 } 6485 6486 // Look through GEPs. A load from a GEP derived from NULL is still undefined 6487 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Use)) 6488 if (GEP->getPointerOperand() == I) { 6489 if (!GEP->isInBounds() || !GEP->hasAllZeroIndices()) 6490 PtrValueMayBeModified = true; 6491 return passingValueIsAlwaysUndefined(V, GEP, PtrValueMayBeModified); 6492 } 6493 6494 // Look through bitcasts. 6495 if (BitCastInst *BC = dyn_cast<BitCastInst>(Use)) 6496 return passingValueIsAlwaysUndefined(V, BC, PtrValueMayBeModified); 6497 6498 // Load from null is undefined. 6499 if (LoadInst *LI = dyn_cast<LoadInst>(Use)) 6500 if (!LI->isVolatile()) 6501 return !NullPointerIsDefined(LI->getFunction(), 6502 LI->getPointerAddressSpace()); 6503 6504 // Store to null is undefined. 6505 if (StoreInst *SI = dyn_cast<StoreInst>(Use)) 6506 if (!SI->isVolatile()) 6507 return (!NullPointerIsDefined(SI->getFunction(), 6508 SI->getPointerAddressSpace())) && 6509 SI->getPointerOperand() == I; 6510 6511 if (auto *CB = dyn_cast<CallBase>(Use)) { 6512 if (C->isNullValue() && NullPointerIsDefined(CB->getFunction())) 6513 return false; 6514 // A call to null is undefined. 6515 if (CB->getCalledOperand() == I) 6516 return true; 6517 6518 if (C->isNullValue()) { 6519 for (const llvm::Use &Arg : CB->args()) 6520 if (Arg == I) { 6521 unsigned ArgIdx = CB->getArgOperandNo(&Arg); 6522 if (CB->isPassingUndefUB(ArgIdx) && 6523 CB->paramHasAttr(ArgIdx, Attribute::NonNull)) { 6524 // Passing null to a nonnnull+noundef argument is undefined. 6525 return !PtrValueMayBeModified; 6526 } 6527 } 6528 } else if (isa<UndefValue>(C)) { 6529 // Passing undef to a noundef argument is undefined. 6530 for (const llvm::Use &Arg : CB->args()) 6531 if (Arg == I) { 6532 unsigned ArgIdx = CB->getArgOperandNo(&Arg); 6533 if (CB->isPassingUndefUB(ArgIdx)) { 6534 // Passing undef to a noundef argument is undefined. 6535 return true; 6536 } 6537 } 6538 } 6539 } 6540 } 6541 return false; 6542 } 6543 6544 /// If BB has an incoming value that will always trigger undefined behavior 6545 /// (eg. null pointer dereference), remove the branch leading here. 6546 static bool removeUndefIntroducingPredecessor(BasicBlock *BB, 6547 DomTreeUpdater *DTU) { 6548 for (PHINode &PHI : BB->phis()) 6549 for (unsigned i = 0, e = PHI.getNumIncomingValues(); i != e; ++i) 6550 if (passingValueIsAlwaysUndefined(PHI.getIncomingValue(i), &PHI)) { 6551 BasicBlock *Predecessor = PHI.getIncomingBlock(i); 6552 Instruction *T = Predecessor->getTerminator(); 6553 IRBuilder<> Builder(T); 6554 if (BranchInst *BI = dyn_cast<BranchInst>(T)) { 6555 BB->removePredecessor(Predecessor); 6556 // Turn uncoditional branches into unreachables and remove the dead 6557 // destination from conditional branches. 6558 if (BI->isUnconditional()) 6559 Builder.CreateUnreachable(); 6560 else 6561 Builder.CreateBr(BI->getSuccessor(0) == BB ? BI->getSuccessor(1) 6562 : BI->getSuccessor(0)); 6563 BI->eraseFromParent(); 6564 if (DTU) 6565 DTU->applyUpdates({{DominatorTree::Delete, Predecessor, BB}}); 6566 return true; 6567 } 6568 // TODO: SwitchInst. 6569 } 6570 6571 return false; 6572 } 6573 6574 bool SimplifyCFGOpt::simplifyOnceImpl(BasicBlock *BB) { 6575 bool Changed = false; 6576 6577 assert(BB && BB->getParent() && "Block not embedded in function!"); 6578 assert(BB->getTerminator() && "Degenerate basic block encountered!"); 6579 6580 // Remove basic blocks that have no predecessors (except the entry block)... 6581 // or that just have themself as a predecessor. These are unreachable. 6582 if ((pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) || 6583 BB->getSinglePredecessor() == BB) { 6584 LLVM_DEBUG(dbgs() << "Removing BB: \n" << *BB); 6585 DeleteDeadBlock(BB, DTU); 6586 return true; 6587 } 6588 6589 // Check to see if we can constant propagate this terminator instruction 6590 // away... 6591 Changed |= ConstantFoldTerminator(BB, /*DeleteDeadConditions=*/true, 6592 /*TLI=*/nullptr, DTU); 6593 6594 // Check for and eliminate duplicate PHI nodes in this block. 6595 Changed |= EliminateDuplicatePHINodes(BB); 6596 6597 // Check for and remove branches that will always cause undefined behavior. 6598 Changed |= removeUndefIntroducingPredecessor(BB, DTU); 6599 6600 // Merge basic blocks into their predecessor if there is only one distinct 6601 // pred, and if there is only one distinct successor of the predecessor, and 6602 // if there are no PHI nodes. 6603 if (MergeBlockIntoPredecessor(BB, DTU)) 6604 return true; 6605 6606 if (SinkCommon && Options.SinkCommonInsts) 6607 if (SinkCommonCodeFromPredecessors(BB, DTU)) { 6608 // SinkCommonCodeFromPredecessors() does not automatically CSE PHI's, 6609 // so we may now how duplicate PHI's. 6610 // Let's rerun EliminateDuplicatePHINodes() first, 6611 // before FoldTwoEntryPHINode() potentially converts them into select's, 6612 // after which we'd need a whole EarlyCSE pass run to cleanup them. 6613 return true; 6614 } 6615 6616 IRBuilder<> Builder(BB); 6617 6618 if (Options.FoldTwoEntryPHINode) { 6619 // If there is a trivial two-entry PHI node in this basic block, and we can 6620 // eliminate it, do so now. 6621 if (auto *PN = dyn_cast<PHINode>(BB->begin())) 6622 if (PN->getNumIncomingValues() == 2) 6623 Changed |= FoldTwoEntryPHINode(PN, TTI, DTU, DL); 6624 } 6625 6626 Instruction *Terminator = BB->getTerminator(); 6627 Builder.SetInsertPoint(Terminator); 6628 switch (Terminator->getOpcode()) { 6629 case Instruction::Br: 6630 Changed |= simplifyBranch(cast<BranchInst>(Terminator), Builder); 6631 break; 6632 case Instruction::Resume: 6633 Changed |= simplifyResume(cast<ResumeInst>(Terminator), Builder); 6634 break; 6635 case Instruction::CleanupRet: 6636 Changed |= simplifyCleanupReturn(cast<CleanupReturnInst>(Terminator)); 6637 break; 6638 case Instruction::Switch: 6639 Changed |= simplifySwitch(cast<SwitchInst>(Terminator), Builder); 6640 break; 6641 case Instruction::Unreachable: 6642 Changed |= simplifyUnreachable(cast<UnreachableInst>(Terminator)); 6643 break; 6644 case Instruction::IndirectBr: 6645 Changed |= simplifyIndirectBr(cast<IndirectBrInst>(Terminator)); 6646 break; 6647 } 6648 6649 return Changed; 6650 } 6651 6652 bool SimplifyCFGOpt::simplifyOnce(BasicBlock *BB) { 6653 bool Changed = simplifyOnceImpl(BB); 6654 6655 return Changed; 6656 } 6657 6658 bool SimplifyCFGOpt::run(BasicBlock *BB) { 6659 bool Changed = false; 6660 6661 // Repeated simplify BB as long as resimplification is requested. 6662 do { 6663 Resimplify = false; 6664 6665 // Perform one round of simplifcation. Resimplify flag will be set if 6666 // another iteration is requested. 6667 Changed |= simplifyOnce(BB); 6668 } while (Resimplify); 6669 6670 return Changed; 6671 } 6672 6673 bool llvm::simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI, 6674 DomTreeUpdater *DTU, const SimplifyCFGOptions &Options, 6675 ArrayRef<WeakVH> LoopHeaders) { 6676 return SimplifyCFGOpt(TTI, DTU, BB->getModule()->getDataLayout(), LoopHeaders, 6677 Options) 6678 .run(BB); 6679 } 6680