1 //===- InstCombineCompares.cpp --------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the visitICmp and visitFCmp functions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "InstCombineInternal.h" 14 #include "llvm/ADT/APSInt.h" 15 #include "llvm/ADT/SetVector.h" 16 #include "llvm/ADT/Statistic.h" 17 #include "llvm/Analysis/CmpInstAnalysis.h" 18 #include "llvm/Analysis/ConstantFolding.h" 19 #include "llvm/Analysis/InstructionSimplify.h" 20 #include "llvm/IR/ConstantRange.h" 21 #include "llvm/IR/DataLayout.h" 22 #include "llvm/IR/GetElementPtrTypeIterator.h" 23 #include "llvm/IR/IntrinsicInst.h" 24 #include "llvm/IR/PatternMatch.h" 25 #include "llvm/Support/KnownBits.h" 26 #include "llvm/Transforms/InstCombine/InstCombiner.h" 27 28 using namespace llvm; 29 using namespace PatternMatch; 30 31 #define DEBUG_TYPE "instcombine" 32 33 // How many times is a select replaced by one of its operands? 34 STATISTIC(NumSel, "Number of select opts"); 35 36 37 /// Compute Result = In1+In2, returning true if the result overflowed for this 38 /// type. 39 static bool addWithOverflow(APInt &Result, const APInt &In1, 40 const APInt &In2, bool IsSigned = false) { 41 bool Overflow; 42 if (IsSigned) 43 Result = In1.sadd_ov(In2, Overflow); 44 else 45 Result = In1.uadd_ov(In2, Overflow); 46 47 return Overflow; 48 } 49 50 /// Compute Result = In1-In2, returning true if the result overflowed for this 51 /// type. 52 static bool subWithOverflow(APInt &Result, const APInt &In1, 53 const APInt &In2, bool IsSigned = false) { 54 bool Overflow; 55 if (IsSigned) 56 Result = In1.ssub_ov(In2, Overflow); 57 else 58 Result = In1.usub_ov(In2, Overflow); 59 60 return Overflow; 61 } 62 63 /// Given an icmp instruction, return true if any use of this comparison is a 64 /// branch on sign bit comparison. 65 static bool hasBranchUse(ICmpInst &I) { 66 for (auto *U : I.users()) 67 if (isa<BranchInst>(U)) 68 return true; 69 return false; 70 } 71 72 /// Returns true if the exploded icmp can be expressed as a signed comparison 73 /// to zero and updates the predicate accordingly. 74 /// The signedness of the comparison is preserved. 75 /// TODO: Refactor with decomposeBitTestICmp()? 76 static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) { 77 if (!ICmpInst::isSigned(Pred)) 78 return false; 79 80 if (C.isZero()) 81 return ICmpInst::isRelational(Pred); 82 83 if (C.isOne()) { 84 if (Pred == ICmpInst::ICMP_SLT) { 85 Pred = ICmpInst::ICMP_SLE; 86 return true; 87 } 88 } else if (C.isAllOnes()) { 89 if (Pred == ICmpInst::ICMP_SGT) { 90 Pred = ICmpInst::ICMP_SGE; 91 return true; 92 } 93 } 94 95 return false; 96 } 97 98 /// This is called when we see this pattern: 99 /// cmp pred (load (gep GV, ...)), cmpcst 100 /// where GV is a global variable with a constant initializer. Try to simplify 101 /// this into some simple computation that does not need the load. For example 102 /// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3". 103 /// 104 /// If AndCst is non-null, then the loaded value is masked with that constant 105 /// before doing the comparison. This handles cases like "A[i]&4 == 0". 106 Instruction *InstCombinerImpl::foldCmpLoadFromIndexedGlobal( 107 LoadInst *LI, GetElementPtrInst *GEP, GlobalVariable *GV, CmpInst &ICI, 108 ConstantInt *AndCst) { 109 if (LI->isVolatile() || LI->getType() != GEP->getResultElementType() || 110 GV->getValueType() != GEP->getSourceElementType() || 111 !GV->isConstant() || !GV->hasDefinitiveInitializer()) 112 return nullptr; 113 114 Constant *Init = GV->getInitializer(); 115 if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init)) 116 return nullptr; 117 118 uint64_t ArrayElementCount = Init->getType()->getArrayNumElements(); 119 // Don't blow up on huge arrays. 120 if (ArrayElementCount > MaxArraySizeForCombine) 121 return nullptr; 122 123 // There are many forms of this optimization we can handle, for now, just do 124 // the simple index into a single-dimensional array. 125 // 126 // Require: GEP GV, 0, i {{, constant indices}} 127 if (GEP->getNumOperands() < 3 || 128 !isa<ConstantInt>(GEP->getOperand(1)) || 129 !cast<ConstantInt>(GEP->getOperand(1))->isZero() || 130 isa<Constant>(GEP->getOperand(2))) 131 return nullptr; 132 133 // Check that indices after the variable are constants and in-range for the 134 // type they index. Collect the indices. This is typically for arrays of 135 // structs. 136 SmallVector<unsigned, 4> LaterIndices; 137 138 Type *EltTy = Init->getType()->getArrayElementType(); 139 for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) { 140 ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i)); 141 if (!Idx) return nullptr; // Variable index. 142 143 uint64_t IdxVal = Idx->getZExtValue(); 144 if ((unsigned)IdxVal != IdxVal) return nullptr; // Too large array index. 145 146 if (StructType *STy = dyn_cast<StructType>(EltTy)) 147 EltTy = STy->getElementType(IdxVal); 148 else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) { 149 if (IdxVal >= ATy->getNumElements()) return nullptr; 150 EltTy = ATy->getElementType(); 151 } else { 152 return nullptr; // Unknown type. 153 } 154 155 LaterIndices.push_back(IdxVal); 156 } 157 158 enum { Overdefined = -3, Undefined = -2 }; 159 160 // Variables for our state machines. 161 162 // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form 163 // "i == 47 | i == 87", where 47 is the first index the condition is true for, 164 // and 87 is the second (and last) index. FirstTrueElement is -2 when 165 // undefined, otherwise set to the first true element. SecondTrueElement is 166 // -2 when undefined, -3 when overdefined and >= 0 when that index is true. 167 int FirstTrueElement = Undefined, SecondTrueElement = Undefined; 168 169 // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the 170 // form "i != 47 & i != 87". Same state transitions as for true elements. 171 int FirstFalseElement = Undefined, SecondFalseElement = Undefined; 172 173 /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these 174 /// define a state machine that triggers for ranges of values that the index 175 /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'. 176 /// This is -2 when undefined, -3 when overdefined, and otherwise the last 177 /// index in the range (inclusive). We use -2 for undefined here because we 178 /// use relative comparisons and don't want 0-1 to match -1. 179 int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined; 180 181 // MagicBitvector - This is a magic bitvector where we set a bit if the 182 // comparison is true for element 'i'. If there are 64 elements or less in 183 // the array, this will fully represent all the comparison results. 184 uint64_t MagicBitvector = 0; 185 186 // Scan the array and see if one of our patterns matches. 187 Constant *CompareRHS = cast<Constant>(ICI.getOperand(1)); 188 for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) { 189 Constant *Elt = Init->getAggregateElement(i); 190 if (!Elt) return nullptr; 191 192 // If this is indexing an array of structures, get the structure element. 193 if (!LaterIndices.empty()) { 194 Elt = ConstantFoldExtractValueInstruction(Elt, LaterIndices); 195 if (!Elt) 196 return nullptr; 197 } 198 199 // If the element is masked, handle it. 200 if (AndCst) Elt = ConstantExpr::getAnd(Elt, AndCst); 201 202 // Find out if the comparison would be true or false for the i'th element. 203 Constant *C = ConstantFoldCompareInstOperands(ICI.getPredicate(), Elt, 204 CompareRHS, DL, &TLI); 205 // If the result is undef for this element, ignore it. 206 if (isa<UndefValue>(C)) { 207 // Extend range state machines to cover this element in case there is an 208 // undef in the middle of the range. 209 if (TrueRangeEnd == (int)i-1) 210 TrueRangeEnd = i; 211 if (FalseRangeEnd == (int)i-1) 212 FalseRangeEnd = i; 213 continue; 214 } 215 216 // If we can't compute the result for any of the elements, we have to give 217 // up evaluating the entire conditional. 218 if (!isa<ConstantInt>(C)) return nullptr; 219 220 // Otherwise, we know if the comparison is true or false for this element, 221 // update our state machines. 222 bool IsTrueForElt = !cast<ConstantInt>(C)->isZero(); 223 224 // State machine for single/double/range index comparison. 225 if (IsTrueForElt) { 226 // Update the TrueElement state machine. 227 if (FirstTrueElement == Undefined) 228 FirstTrueElement = TrueRangeEnd = i; // First true element. 229 else { 230 // Update double-compare state machine. 231 if (SecondTrueElement == Undefined) 232 SecondTrueElement = i; 233 else 234 SecondTrueElement = Overdefined; 235 236 // Update range state machine. 237 if (TrueRangeEnd == (int)i-1) 238 TrueRangeEnd = i; 239 else 240 TrueRangeEnd = Overdefined; 241 } 242 } else { 243 // Update the FalseElement state machine. 244 if (FirstFalseElement == Undefined) 245 FirstFalseElement = FalseRangeEnd = i; // First false element. 246 else { 247 // Update double-compare state machine. 248 if (SecondFalseElement == Undefined) 249 SecondFalseElement = i; 250 else 251 SecondFalseElement = Overdefined; 252 253 // Update range state machine. 254 if (FalseRangeEnd == (int)i-1) 255 FalseRangeEnd = i; 256 else 257 FalseRangeEnd = Overdefined; 258 } 259 } 260 261 // If this element is in range, update our magic bitvector. 262 if (i < 64 && IsTrueForElt) 263 MagicBitvector |= 1ULL << i; 264 265 // If all of our states become overdefined, bail out early. Since the 266 // predicate is expensive, only check it every 8 elements. This is only 267 // really useful for really huge arrays. 268 if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined && 269 SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined && 270 FalseRangeEnd == Overdefined) 271 return nullptr; 272 } 273 274 // Now that we've scanned the entire array, emit our new comparison(s). We 275 // order the state machines in complexity of the generated code. 276 Value *Idx = GEP->getOperand(2); 277 278 // If the index is larger than the pointer size of the target, truncate the 279 // index down like the GEP would do implicitly. We don't have to do this for 280 // an inbounds GEP because the index can't be out of range. 281 if (!GEP->isInBounds()) { 282 Type *IntPtrTy = DL.getIntPtrType(GEP->getType()); 283 unsigned PtrSize = IntPtrTy->getIntegerBitWidth(); 284 if (Idx->getType()->getPrimitiveSizeInBits().getFixedSize() > PtrSize) 285 Idx = Builder.CreateTrunc(Idx, IntPtrTy); 286 } 287 288 // If inbounds keyword is not present, Idx * ElementSize can overflow. 289 // Let's assume that ElementSize is 2 and the wanted value is at offset 0. 290 // Then, there are two possible values for Idx to match offset 0: 291 // 0x00..00, 0x80..00. 292 // Emitting 'icmp eq Idx, 0' isn't correct in this case because the 293 // comparison is false if Idx was 0x80..00. 294 // We need to erase the highest countTrailingZeros(ElementSize) bits of Idx. 295 unsigned ElementSize = 296 DL.getTypeAllocSize(Init->getType()->getArrayElementType()); 297 auto MaskIdx = [&](Value* Idx){ 298 if (!GEP->isInBounds() && countTrailingZeros(ElementSize) != 0) { 299 Value *Mask = ConstantInt::get(Idx->getType(), -1); 300 Mask = Builder.CreateLShr(Mask, countTrailingZeros(ElementSize)); 301 Idx = Builder.CreateAnd(Idx, Mask); 302 } 303 return Idx; 304 }; 305 306 // If the comparison is only true for one or two elements, emit direct 307 // comparisons. 308 if (SecondTrueElement != Overdefined) { 309 Idx = MaskIdx(Idx); 310 // None true -> false. 311 if (FirstTrueElement == Undefined) 312 return replaceInstUsesWith(ICI, Builder.getFalse()); 313 314 Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement); 315 316 // True for one element -> 'i == 47'. 317 if (SecondTrueElement == Undefined) 318 return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx); 319 320 // True for two elements -> 'i == 47 | i == 72'. 321 Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx); 322 Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement); 323 Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx); 324 return BinaryOperator::CreateOr(C1, C2); 325 } 326 327 // If the comparison is only false for one or two elements, emit direct 328 // comparisons. 329 if (SecondFalseElement != Overdefined) { 330 Idx = MaskIdx(Idx); 331 // None false -> true. 332 if (FirstFalseElement == Undefined) 333 return replaceInstUsesWith(ICI, Builder.getTrue()); 334 335 Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement); 336 337 // False for one element -> 'i != 47'. 338 if (SecondFalseElement == Undefined) 339 return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx); 340 341 // False for two elements -> 'i != 47 & i != 72'. 342 Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx); 343 Value *SecondFalseIdx = ConstantInt::get(Idx->getType(),SecondFalseElement); 344 Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx); 345 return BinaryOperator::CreateAnd(C1, C2); 346 } 347 348 // If the comparison can be replaced with a range comparison for the elements 349 // where it is true, emit the range check. 350 if (TrueRangeEnd != Overdefined) { 351 assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare"); 352 Idx = MaskIdx(Idx); 353 354 // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1). 355 if (FirstTrueElement) { 356 Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement); 357 Idx = Builder.CreateAdd(Idx, Offs); 358 } 359 360 Value *End = ConstantInt::get(Idx->getType(), 361 TrueRangeEnd-FirstTrueElement+1); 362 return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End); 363 } 364 365 // False range check. 366 if (FalseRangeEnd != Overdefined) { 367 assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare"); 368 Idx = MaskIdx(Idx); 369 // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse). 370 if (FirstFalseElement) { 371 Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement); 372 Idx = Builder.CreateAdd(Idx, Offs); 373 } 374 375 Value *End = ConstantInt::get(Idx->getType(), 376 FalseRangeEnd-FirstFalseElement); 377 return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End); 378 } 379 380 // If a magic bitvector captures the entire comparison state 381 // of this load, replace it with computation that does: 382 // ((magic_cst >> i) & 1) != 0 383 { 384 Type *Ty = nullptr; 385 386 // Look for an appropriate type: 387 // - The type of Idx if the magic fits 388 // - The smallest fitting legal type 389 if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth()) 390 Ty = Idx->getType(); 391 else 392 Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount); 393 394 if (Ty) { 395 Idx = MaskIdx(Idx); 396 Value *V = Builder.CreateIntCast(Idx, Ty, false); 397 V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V); 398 V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V); 399 return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0)); 400 } 401 } 402 403 return nullptr; 404 } 405 406 /// Return a value that can be used to compare the *offset* implied by a GEP to 407 /// zero. For example, if we have &A[i], we want to return 'i' for 408 /// "icmp ne i, 0". Note that, in general, indices can be complex, and scales 409 /// are involved. The above expression would also be legal to codegen as 410 /// "icmp ne (i*4), 0" (assuming A is a pointer to i32). 411 /// This latter form is less amenable to optimization though, and we are allowed 412 /// to generate the first by knowing that pointer arithmetic doesn't overflow. 413 /// 414 /// If we can't emit an optimized form for this expression, this returns null. 415 /// 416 static Value *evaluateGEPOffsetExpression(User *GEP, InstCombinerImpl &IC, 417 const DataLayout &DL) { 418 gep_type_iterator GTI = gep_type_begin(GEP); 419 420 // Check to see if this gep only has a single variable index. If so, and if 421 // any constant indices are a multiple of its scale, then we can compute this 422 // in terms of the scale of the variable index. For example, if the GEP 423 // implies an offset of "12 + i*4", then we can codegen this as "3 + i", 424 // because the expression will cross zero at the same point. 425 unsigned i, e = GEP->getNumOperands(); 426 int64_t Offset = 0; 427 for (i = 1; i != e; ++i, ++GTI) { 428 if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { 429 // Compute the aggregate offset of constant indices. 430 if (CI->isZero()) continue; 431 432 // Handle a struct index, which adds its field offset to the pointer. 433 if (StructType *STy = GTI.getStructTypeOrNull()) { 434 Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); 435 } else { 436 uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); 437 Offset += Size*CI->getSExtValue(); 438 } 439 } else { 440 // Found our variable index. 441 break; 442 } 443 } 444 445 // If there are no variable indices, we must have a constant offset, just 446 // evaluate it the general way. 447 if (i == e) return nullptr; 448 449 Value *VariableIdx = GEP->getOperand(i); 450 // Determine the scale factor of the variable element. For example, this is 451 // 4 if the variable index is into an array of i32. 452 uint64_t VariableScale = DL.getTypeAllocSize(GTI.getIndexedType()); 453 454 // Verify that there are no other variable indices. If so, emit the hard way. 455 for (++i, ++GTI; i != e; ++i, ++GTI) { 456 ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i)); 457 if (!CI) return nullptr; 458 459 // Compute the aggregate offset of constant indices. 460 if (CI->isZero()) continue; 461 462 // Handle a struct index, which adds its field offset to the pointer. 463 if (StructType *STy = GTI.getStructTypeOrNull()) { 464 Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); 465 } else { 466 uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); 467 Offset += Size*CI->getSExtValue(); 468 } 469 } 470 471 // Okay, we know we have a single variable index, which must be a 472 // pointer/array/vector index. If there is no offset, life is simple, return 473 // the index. 474 Type *IntPtrTy = DL.getIntPtrType(GEP->getOperand(0)->getType()); 475 unsigned IntPtrWidth = IntPtrTy->getIntegerBitWidth(); 476 if (Offset == 0) { 477 // Cast to intptrty in case a truncation occurs. If an extension is needed, 478 // we don't need to bother extending: the extension won't affect where the 479 // computation crosses zero. 480 if (VariableIdx->getType()->getPrimitiveSizeInBits().getFixedSize() > 481 IntPtrWidth) { 482 VariableIdx = IC.Builder.CreateTrunc(VariableIdx, IntPtrTy); 483 } 484 return VariableIdx; 485 } 486 487 // Otherwise, there is an index. The computation we will do will be modulo 488 // the pointer size. 489 Offset = SignExtend64(Offset, IntPtrWidth); 490 VariableScale = SignExtend64(VariableScale, IntPtrWidth); 491 492 // To do this transformation, any constant index must be a multiple of the 493 // variable scale factor. For example, we can evaluate "12 + 4*i" as "3 + i", 494 // but we can't evaluate "10 + 3*i" in terms of i. Check that the offset is a 495 // multiple of the variable scale. 496 int64_t NewOffs = Offset / (int64_t)VariableScale; 497 if (Offset != NewOffs*(int64_t)VariableScale) 498 return nullptr; 499 500 // Okay, we can do this evaluation. Start by converting the index to intptr. 501 if (VariableIdx->getType() != IntPtrTy) 502 VariableIdx = IC.Builder.CreateIntCast(VariableIdx, IntPtrTy, 503 true /*Signed*/); 504 Constant *OffsetVal = ConstantInt::get(IntPtrTy, NewOffs); 505 return IC.Builder.CreateAdd(VariableIdx, OffsetVal, "offset"); 506 } 507 508 /// Returns true if we can rewrite Start as a GEP with pointer Base 509 /// and some integer offset. The nodes that need to be re-written 510 /// for this transformation will be added to Explored. 511 static bool canRewriteGEPAsOffset(Type *ElemTy, Value *Start, Value *Base, 512 const DataLayout &DL, 513 SetVector<Value *> &Explored) { 514 SmallVector<Value *, 16> WorkList(1, Start); 515 Explored.insert(Base); 516 517 // The following traversal gives us an order which can be used 518 // when doing the final transformation. Since in the final 519 // transformation we create the PHI replacement instructions first, 520 // we don't have to get them in any particular order. 521 // 522 // However, for other instructions we will have to traverse the 523 // operands of an instruction first, which means that we have to 524 // do a post-order traversal. 525 while (!WorkList.empty()) { 526 SetVector<PHINode *> PHIs; 527 528 while (!WorkList.empty()) { 529 if (Explored.size() >= 100) 530 return false; 531 532 Value *V = WorkList.back(); 533 534 if (Explored.contains(V)) { 535 WorkList.pop_back(); 536 continue; 537 } 538 539 if (!isa<IntToPtrInst>(V) && !isa<PtrToIntInst>(V) && 540 !isa<GetElementPtrInst>(V) && !isa<PHINode>(V)) 541 // We've found some value that we can't explore which is different from 542 // the base. Therefore we can't do this transformation. 543 return false; 544 545 if (isa<IntToPtrInst>(V) || isa<PtrToIntInst>(V)) { 546 auto *CI = cast<CastInst>(V); 547 if (!CI->isNoopCast(DL)) 548 return false; 549 550 if (!Explored.contains(CI->getOperand(0))) 551 WorkList.push_back(CI->getOperand(0)); 552 } 553 554 if (auto *GEP = dyn_cast<GEPOperator>(V)) { 555 // We're limiting the GEP to having one index. This will preserve 556 // the original pointer type. We could handle more cases in the 557 // future. 558 if (GEP->getNumIndices() != 1 || !GEP->isInBounds() || 559 GEP->getSourceElementType() != ElemTy) 560 return false; 561 562 if (!Explored.contains(GEP->getOperand(0))) 563 WorkList.push_back(GEP->getOperand(0)); 564 } 565 566 if (WorkList.back() == V) { 567 WorkList.pop_back(); 568 // We've finished visiting this node, mark it as such. 569 Explored.insert(V); 570 } 571 572 if (auto *PN = dyn_cast<PHINode>(V)) { 573 // We cannot transform PHIs on unsplittable basic blocks. 574 if (isa<CatchSwitchInst>(PN->getParent()->getTerminator())) 575 return false; 576 Explored.insert(PN); 577 PHIs.insert(PN); 578 } 579 } 580 581 // Explore the PHI nodes further. 582 for (auto *PN : PHIs) 583 for (Value *Op : PN->incoming_values()) 584 if (!Explored.contains(Op)) 585 WorkList.push_back(Op); 586 } 587 588 // Make sure that we can do this. Since we can't insert GEPs in a basic 589 // block before a PHI node, we can't easily do this transformation if 590 // we have PHI node users of transformed instructions. 591 for (Value *Val : Explored) { 592 for (Value *Use : Val->uses()) { 593 594 auto *PHI = dyn_cast<PHINode>(Use); 595 auto *Inst = dyn_cast<Instruction>(Val); 596 597 if (Inst == Base || Inst == PHI || !Inst || !PHI || 598 !Explored.contains(PHI)) 599 continue; 600 601 if (PHI->getParent() == Inst->getParent()) 602 return false; 603 } 604 } 605 return true; 606 } 607 608 // Sets the appropriate insert point on Builder where we can add 609 // a replacement Instruction for V (if that is possible). 610 static void setInsertionPoint(IRBuilder<> &Builder, Value *V, 611 bool Before = true) { 612 if (auto *PHI = dyn_cast<PHINode>(V)) { 613 Builder.SetInsertPoint(&*PHI->getParent()->getFirstInsertionPt()); 614 return; 615 } 616 if (auto *I = dyn_cast<Instruction>(V)) { 617 if (!Before) 618 I = &*std::next(I->getIterator()); 619 Builder.SetInsertPoint(I); 620 return; 621 } 622 if (auto *A = dyn_cast<Argument>(V)) { 623 // Set the insertion point in the entry block. 624 BasicBlock &Entry = A->getParent()->getEntryBlock(); 625 Builder.SetInsertPoint(&*Entry.getFirstInsertionPt()); 626 return; 627 } 628 // Otherwise, this is a constant and we don't need to set a new 629 // insertion point. 630 assert(isa<Constant>(V) && "Setting insertion point for unknown value!"); 631 } 632 633 /// Returns a re-written value of Start as an indexed GEP using Base as a 634 /// pointer. 635 static Value *rewriteGEPAsOffset(Type *ElemTy, Value *Start, Value *Base, 636 const DataLayout &DL, 637 SetVector<Value *> &Explored) { 638 // Perform all the substitutions. This is a bit tricky because we can 639 // have cycles in our use-def chains. 640 // 1. Create the PHI nodes without any incoming values. 641 // 2. Create all the other values. 642 // 3. Add the edges for the PHI nodes. 643 // 4. Emit GEPs to get the original pointers. 644 // 5. Remove the original instructions. 645 Type *IndexType = IntegerType::get( 646 Base->getContext(), DL.getIndexTypeSizeInBits(Start->getType())); 647 648 DenseMap<Value *, Value *> NewInsts; 649 NewInsts[Base] = ConstantInt::getNullValue(IndexType); 650 651 // Create the new PHI nodes, without adding any incoming values. 652 for (Value *Val : Explored) { 653 if (Val == Base) 654 continue; 655 // Create empty phi nodes. This avoids cyclic dependencies when creating 656 // the remaining instructions. 657 if (auto *PHI = dyn_cast<PHINode>(Val)) 658 NewInsts[PHI] = PHINode::Create(IndexType, PHI->getNumIncomingValues(), 659 PHI->getName() + ".idx", PHI); 660 } 661 IRBuilder<> Builder(Base->getContext()); 662 663 // Create all the other instructions. 664 for (Value *Val : Explored) { 665 666 if (NewInsts.find(Val) != NewInsts.end()) 667 continue; 668 669 if (auto *CI = dyn_cast<CastInst>(Val)) { 670 // Don't get rid of the intermediate variable here; the store can grow 671 // the map which will invalidate the reference to the input value. 672 Value *V = NewInsts[CI->getOperand(0)]; 673 NewInsts[CI] = V; 674 continue; 675 } 676 if (auto *GEP = dyn_cast<GEPOperator>(Val)) { 677 Value *Index = NewInsts[GEP->getOperand(1)] ? NewInsts[GEP->getOperand(1)] 678 : GEP->getOperand(1); 679 setInsertionPoint(Builder, GEP); 680 // Indices might need to be sign extended. GEPs will magically do 681 // this, but we need to do it ourselves here. 682 if (Index->getType()->getScalarSizeInBits() != 683 NewInsts[GEP->getOperand(0)]->getType()->getScalarSizeInBits()) { 684 Index = Builder.CreateSExtOrTrunc( 685 Index, NewInsts[GEP->getOperand(0)]->getType(), 686 GEP->getOperand(0)->getName() + ".sext"); 687 } 688 689 auto *Op = NewInsts[GEP->getOperand(0)]; 690 if (isa<ConstantInt>(Op) && cast<ConstantInt>(Op)->isZero()) 691 NewInsts[GEP] = Index; 692 else 693 NewInsts[GEP] = Builder.CreateNSWAdd( 694 Op, Index, GEP->getOperand(0)->getName() + ".add"); 695 continue; 696 } 697 if (isa<PHINode>(Val)) 698 continue; 699 700 llvm_unreachable("Unexpected instruction type"); 701 } 702 703 // Add the incoming values to the PHI nodes. 704 for (Value *Val : Explored) { 705 if (Val == Base) 706 continue; 707 // All the instructions have been created, we can now add edges to the 708 // phi nodes. 709 if (auto *PHI = dyn_cast<PHINode>(Val)) { 710 PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]); 711 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) { 712 Value *NewIncoming = PHI->getIncomingValue(I); 713 714 if (NewInsts.find(NewIncoming) != NewInsts.end()) 715 NewIncoming = NewInsts[NewIncoming]; 716 717 NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I)); 718 } 719 } 720 } 721 722 PointerType *PtrTy = 723 ElemTy->getPointerTo(Start->getType()->getPointerAddressSpace()); 724 for (Value *Val : Explored) { 725 if (Val == Base) 726 continue; 727 728 // Depending on the type, for external users we have to emit 729 // a GEP or a GEP + ptrtoint. 730 setInsertionPoint(Builder, Val, false); 731 732 // Cast base to the expected type. 733 Value *NewVal = Builder.CreateBitOrPointerCast( 734 Base, PtrTy, Start->getName() + "to.ptr"); 735 NewVal = Builder.CreateInBoundsGEP( 736 ElemTy, NewVal, makeArrayRef(NewInsts[Val]), Val->getName() + ".ptr"); 737 NewVal = Builder.CreateBitOrPointerCast( 738 NewVal, Val->getType(), Val->getName() + ".conv"); 739 Val->replaceAllUsesWith(NewVal); 740 } 741 742 return NewInsts[Start]; 743 } 744 745 /// Looks through GEPs, IntToPtrInsts and PtrToIntInsts in order to express 746 /// the input Value as a constant indexed GEP. Returns a pair containing 747 /// the GEPs Pointer and Index. 748 static std::pair<Value *, Value *> 749 getAsConstantIndexedAddress(Type *ElemTy, Value *V, const DataLayout &DL) { 750 Type *IndexType = IntegerType::get(V->getContext(), 751 DL.getIndexTypeSizeInBits(V->getType())); 752 753 Constant *Index = ConstantInt::getNullValue(IndexType); 754 while (true) { 755 if (GEPOperator *GEP = dyn_cast<GEPOperator>(V)) { 756 // We accept only inbouds GEPs here to exclude the possibility of 757 // overflow. 758 if (!GEP->isInBounds()) 759 break; 760 if (GEP->hasAllConstantIndices() && GEP->getNumIndices() == 1 && 761 GEP->getSourceElementType() == ElemTy) { 762 V = GEP->getOperand(0); 763 Constant *GEPIndex = static_cast<Constant *>(GEP->getOperand(1)); 764 Index = ConstantExpr::getAdd( 765 Index, ConstantExpr::getSExtOrTrunc(GEPIndex, IndexType)); 766 continue; 767 } 768 break; 769 } 770 if (auto *CI = dyn_cast<IntToPtrInst>(V)) { 771 if (!CI->isNoopCast(DL)) 772 break; 773 V = CI->getOperand(0); 774 continue; 775 } 776 if (auto *CI = dyn_cast<PtrToIntInst>(V)) { 777 if (!CI->isNoopCast(DL)) 778 break; 779 V = CI->getOperand(0); 780 continue; 781 } 782 break; 783 } 784 return {V, Index}; 785 } 786 787 /// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant. 788 /// We can look through PHIs, GEPs and casts in order to determine a common base 789 /// between GEPLHS and RHS. 790 static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, 791 ICmpInst::Predicate Cond, 792 const DataLayout &DL) { 793 // FIXME: Support vector of pointers. 794 if (GEPLHS->getType()->isVectorTy()) 795 return nullptr; 796 797 if (!GEPLHS->hasAllConstantIndices()) 798 return nullptr; 799 800 Type *ElemTy = GEPLHS->getSourceElementType(); 801 Value *PtrBase, *Index; 802 std::tie(PtrBase, Index) = getAsConstantIndexedAddress(ElemTy, GEPLHS, DL); 803 804 // The set of nodes that will take part in this transformation. 805 SetVector<Value *> Nodes; 806 807 if (!canRewriteGEPAsOffset(ElemTy, RHS, PtrBase, DL, Nodes)) 808 return nullptr; 809 810 // We know we can re-write this as 811 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) 812 // Since we've only looked through inbouds GEPs we know that we 813 // can't have overflow on either side. We can therefore re-write 814 // this as: 815 // OFFSET1 cmp OFFSET2 816 Value *NewRHS = rewriteGEPAsOffset(ElemTy, RHS, PtrBase, DL, Nodes); 817 818 // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written 819 // GEP having PtrBase as the pointer base, and has returned in NewRHS the 820 // offset. Since Index is the offset of LHS to the base pointer, we will now 821 // compare the offsets instead of comparing the pointers. 822 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Index, NewRHS); 823 } 824 825 /// Fold comparisons between a GEP instruction and something else. At this point 826 /// we know that the GEP is on the LHS of the comparison. 827 Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, 828 ICmpInst::Predicate Cond, 829 Instruction &I) { 830 // Don't transform signed compares of GEPs into index compares. Even if the 831 // GEP is inbounds, the final add of the base pointer can have signed overflow 832 // and would change the result of the icmp. 833 // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be 834 // the maximum signed value for the pointer type. 835 if (ICmpInst::isSigned(Cond)) 836 return nullptr; 837 838 // Look through bitcasts and addrspacecasts. We do not however want to remove 839 // 0 GEPs. 840 if (!isa<GetElementPtrInst>(RHS)) 841 RHS = RHS->stripPointerCasts(); 842 843 Value *PtrBase = GEPLHS->getOperand(0); 844 // FIXME: Support vector pointer GEPs. 845 if (PtrBase == RHS && GEPLHS->isInBounds() && 846 !GEPLHS->getType()->isVectorTy()) { 847 // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0). 848 // This transformation (ignoring the base and scales) is valid because we 849 // know pointers can't overflow since the gep is inbounds. See if we can 850 // output an optimized form. 851 Value *Offset = evaluateGEPOffsetExpression(GEPLHS, *this, DL); 852 853 // If not, synthesize the offset the hard way. 854 if (!Offset) 855 Offset = EmitGEPOffset(GEPLHS); 856 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Offset, 857 Constant::getNullValue(Offset->getType())); 858 } 859 860 if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) && 861 isa<Constant>(RHS) && cast<Constant>(RHS)->isNullValue() && 862 !NullPointerIsDefined(I.getFunction(), 863 RHS->getType()->getPointerAddressSpace())) { 864 // For most address spaces, an allocation can't be placed at null, but null 865 // itself is treated as a 0 size allocation in the in bounds rules. Thus, 866 // the only valid inbounds address derived from null, is null itself. 867 // Thus, we have four cases to consider: 868 // 1) Base == nullptr, Offset == 0 -> inbounds, null 869 // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds 870 // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations) 871 // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison) 872 // 873 // (Note if we're indexing a type of size 0, that simply collapses into one 874 // of the buckets above.) 875 // 876 // In general, we're allowed to make values less poison (i.e. remove 877 // sources of full UB), so in this case, we just select between the two 878 // non-poison cases (1 and 4 above). 879 // 880 // For vectors, we apply the same reasoning on a per-lane basis. 881 auto *Base = GEPLHS->getPointerOperand(); 882 if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) { 883 auto EC = cast<VectorType>(GEPLHS->getType())->getElementCount(); 884 Base = Builder.CreateVectorSplat(EC, Base); 885 } 886 return new ICmpInst(Cond, Base, 887 ConstantExpr::getPointerBitCastOrAddrSpaceCast( 888 cast<Constant>(RHS), Base->getType())); 889 } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(RHS)) { 890 // If the base pointers are different, but the indices are the same, just 891 // compare the base pointer. 892 if (PtrBase != GEPRHS->getOperand(0)) { 893 bool IndicesTheSame = 894 GEPLHS->getNumOperands() == GEPRHS->getNumOperands() && 895 GEPLHS->getPointerOperand()->getType() == 896 GEPRHS->getPointerOperand()->getType() && 897 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType(); 898 if (IndicesTheSame) 899 for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i) 900 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { 901 IndicesTheSame = false; 902 break; 903 } 904 905 // If all indices are the same, just compare the base pointers. 906 Type *BaseType = GEPLHS->getOperand(0)->getType(); 907 if (IndicesTheSame && CmpInst::makeCmpResultType(BaseType) == I.getType()) 908 return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0)); 909 910 // If we're comparing GEPs with two base pointers that only differ in type 911 // and both GEPs have only constant indices or just one use, then fold 912 // the compare with the adjusted indices. 913 // FIXME: Support vector of pointers. 914 if (GEPLHS->isInBounds() && GEPRHS->isInBounds() && 915 (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) && 916 (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) && 917 PtrBase->stripPointerCasts() == 918 GEPRHS->getOperand(0)->stripPointerCasts() && 919 !GEPLHS->getType()->isVectorTy()) { 920 Value *LOffset = EmitGEPOffset(GEPLHS); 921 Value *ROffset = EmitGEPOffset(GEPRHS); 922 923 // If we looked through an addrspacecast between different sized address 924 // spaces, the LHS and RHS pointers are different sized 925 // integers. Truncate to the smaller one. 926 Type *LHSIndexTy = LOffset->getType(); 927 Type *RHSIndexTy = ROffset->getType(); 928 if (LHSIndexTy != RHSIndexTy) { 929 if (LHSIndexTy->getPrimitiveSizeInBits().getFixedSize() < 930 RHSIndexTy->getPrimitiveSizeInBits().getFixedSize()) { 931 ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy); 932 } else 933 LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy); 934 } 935 936 Value *Cmp = Builder.CreateICmp(ICmpInst::getSignedPredicate(Cond), 937 LOffset, ROffset); 938 return replaceInstUsesWith(I, Cmp); 939 } 940 941 // Otherwise, the base pointers are different and the indices are 942 // different. Try convert this to an indexed compare by looking through 943 // PHIs/casts. 944 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); 945 } 946 947 // If one of the GEPs has all zero indices, recurse. 948 // FIXME: Handle vector of pointers. 949 if (!GEPLHS->getType()->isVectorTy() && GEPLHS->hasAllZeroIndices()) 950 return foldGEPICmp(GEPRHS, GEPLHS->getOperand(0), 951 ICmpInst::getSwappedPredicate(Cond), I); 952 953 // If the other GEP has all zero indices, recurse. 954 // FIXME: Handle vector of pointers. 955 if (!GEPRHS->getType()->isVectorTy() && GEPRHS->hasAllZeroIndices()) 956 return foldGEPICmp(GEPLHS, GEPRHS->getOperand(0), Cond, I); 957 958 bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds(); 959 if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands() && 960 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType()) { 961 // If the GEPs only differ by one index, compare it. 962 unsigned NumDifferences = 0; // Keep track of # differences. 963 unsigned DiffOperand = 0; // The operand that differs. 964 for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i) 965 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { 966 Type *LHSType = GEPLHS->getOperand(i)->getType(); 967 Type *RHSType = GEPRHS->getOperand(i)->getType(); 968 // FIXME: Better support for vector of pointers. 969 if (LHSType->getPrimitiveSizeInBits() != 970 RHSType->getPrimitiveSizeInBits() || 971 (GEPLHS->getType()->isVectorTy() && 972 (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) { 973 // Irreconcilable differences. 974 NumDifferences = 2; 975 break; 976 } 977 978 if (NumDifferences++) break; 979 DiffOperand = i; 980 } 981 982 if (NumDifferences == 0) // SAME GEP? 983 return replaceInstUsesWith(I, // No comparison is needed here. 984 ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond))); 985 986 else if (NumDifferences == 1 && GEPsInBounds) { 987 Value *LHSV = GEPLHS->getOperand(DiffOperand); 988 Value *RHSV = GEPRHS->getOperand(DiffOperand); 989 // Make sure we do a signed comparison here. 990 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), LHSV, RHSV); 991 } 992 } 993 994 // Only lower this if the icmp is the only user of the GEP or if we expect 995 // the result to fold to a constant! 996 if (GEPsInBounds && (isa<ConstantExpr>(GEPLHS) || GEPLHS->hasOneUse()) && 997 (isa<ConstantExpr>(GEPRHS) || GEPRHS->hasOneUse())) { 998 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2) 999 Value *L = EmitGEPOffset(GEPLHS); 1000 Value *R = EmitGEPOffset(GEPRHS); 1001 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), L, R); 1002 } 1003 } 1004 1005 // Try convert this to an indexed compare by looking through PHIs/casts as a 1006 // last resort. 1007 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); 1008 } 1009 1010 Instruction *InstCombinerImpl::foldAllocaCmp(ICmpInst &ICI, 1011 const AllocaInst *Alloca) { 1012 assert(ICI.isEquality() && "Cannot fold non-equality comparison."); 1013 1014 // It would be tempting to fold away comparisons between allocas and any 1015 // pointer not based on that alloca (e.g. an argument). However, even 1016 // though such pointers cannot alias, they can still compare equal. 1017 // 1018 // But LLVM doesn't specify where allocas get their memory, so if the alloca 1019 // doesn't escape we can argue that it's impossible to guess its value, and we 1020 // can therefore act as if any such guesses are wrong. 1021 // 1022 // The code below checks that the alloca doesn't escape, and that it's only 1023 // used in a comparison once (the current instruction). The 1024 // single-comparison-use condition ensures that we're trivially folding all 1025 // comparisons against the alloca consistently, and avoids the risk of 1026 // erroneously folding a comparison of the pointer with itself. 1027 1028 unsigned MaxIter = 32; // Break cycles and bound to constant-time. 1029 1030 SmallVector<const Use *, 32> Worklist; 1031 for (const Use &U : Alloca->uses()) { 1032 if (Worklist.size() >= MaxIter) 1033 return nullptr; 1034 Worklist.push_back(&U); 1035 } 1036 1037 unsigned NumCmps = 0; 1038 while (!Worklist.empty()) { 1039 assert(Worklist.size() <= MaxIter); 1040 const Use *U = Worklist.pop_back_val(); 1041 const Value *V = U->getUser(); 1042 --MaxIter; 1043 1044 if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V) || isa<PHINode>(V) || 1045 isa<SelectInst>(V)) { 1046 // Track the uses. 1047 } else if (isa<LoadInst>(V)) { 1048 // Loading from the pointer doesn't escape it. 1049 continue; 1050 } else if (const auto *SI = dyn_cast<StoreInst>(V)) { 1051 // Storing *to* the pointer is fine, but storing the pointer escapes it. 1052 if (SI->getValueOperand() == U->get()) 1053 return nullptr; 1054 continue; 1055 } else if (isa<ICmpInst>(V)) { 1056 if (NumCmps++) 1057 return nullptr; // Found more than one cmp. 1058 continue; 1059 } else if (const auto *Intrin = dyn_cast<IntrinsicInst>(V)) { 1060 switch (Intrin->getIntrinsicID()) { 1061 // These intrinsics don't escape or compare the pointer. Memset is safe 1062 // because we don't allow ptrtoint. Memcpy and memmove are safe because 1063 // we don't allow stores, so src cannot point to V. 1064 case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: 1065 case Intrinsic::memcpy: case Intrinsic::memmove: case Intrinsic::memset: 1066 continue; 1067 default: 1068 return nullptr; 1069 } 1070 } else { 1071 return nullptr; 1072 } 1073 for (const Use &U : V->uses()) { 1074 if (Worklist.size() >= MaxIter) 1075 return nullptr; 1076 Worklist.push_back(&U); 1077 } 1078 } 1079 1080 auto *Res = ConstantInt::get(ICI.getType(), 1081 !CmpInst::isTrueWhenEqual(ICI.getPredicate())); 1082 return replaceInstUsesWith(ICI, Res); 1083 } 1084 1085 /// Fold "icmp pred (X+C), X". 1086 Instruction *InstCombinerImpl::foldICmpAddOpConst(Value *X, const APInt &C, 1087 ICmpInst::Predicate Pred) { 1088 // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0, 1089 // so the values can never be equal. Similarly for all other "or equals" 1090 // operators. 1091 assert(!!C && "C should not be zero!"); 1092 1093 // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255 1094 // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253 1095 // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0 1096 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) { 1097 Constant *R = ConstantInt::get(X->getType(), 1098 APInt::getMaxValue(C.getBitWidth()) - C); 1099 return new ICmpInst(ICmpInst::ICMP_UGT, X, R); 1100 } 1101 1102 // (X+1) >u X --> X <u (0-1) --> X != 255 1103 // (X+2) >u X --> X <u (0-2) --> X <u 254 1104 // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0 1105 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) 1106 return new ICmpInst(ICmpInst::ICMP_ULT, X, 1107 ConstantInt::get(X->getType(), -C)); 1108 1109 APInt SMax = APInt::getSignedMaxValue(C.getBitWidth()); 1110 1111 // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127 1112 // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125 1113 // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0 1114 // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1 1115 // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126 1116 // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127 1117 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) 1118 return new ICmpInst(ICmpInst::ICMP_SGT, X, 1119 ConstantInt::get(X->getType(), SMax - C)); 1120 1121 // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127 1122 // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126 1123 // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1 1124 // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2 1125 // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126 1126 // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128 1127 1128 assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE); 1129 return new ICmpInst(ICmpInst::ICMP_SLT, X, 1130 ConstantInt::get(X->getType(), SMax - (C - 1))); 1131 } 1132 1133 /// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> 1134 /// (icmp eq/ne A, Log2(AP2/AP1)) -> 1135 /// (icmp eq/ne A, Log2(AP2) - Log2(AP1)). 1136 Instruction *InstCombinerImpl::foldICmpShrConstConst(ICmpInst &I, Value *A, 1137 const APInt &AP1, 1138 const APInt &AP2) { 1139 assert(I.isEquality() && "Cannot fold icmp gt/lt"); 1140 1141 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { 1142 if (I.getPredicate() == I.ICMP_NE) 1143 Pred = CmpInst::getInversePredicate(Pred); 1144 return new ICmpInst(Pred, LHS, RHS); 1145 }; 1146 1147 // Don't bother doing any work for cases which InstSimplify handles. 1148 if (AP2.isZero()) 1149 return nullptr; 1150 1151 bool IsAShr = isa<AShrOperator>(I.getOperand(0)); 1152 if (IsAShr) { 1153 if (AP2.isAllOnes()) 1154 return nullptr; 1155 if (AP2.isNegative() != AP1.isNegative()) 1156 return nullptr; 1157 if (AP2.sgt(AP1)) 1158 return nullptr; 1159 } 1160 1161 if (!AP1) 1162 // 'A' must be large enough to shift out the highest set bit. 1163 return getICmp(I.ICMP_UGT, A, 1164 ConstantInt::get(A->getType(), AP2.logBase2())); 1165 1166 if (AP1 == AP2) 1167 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); 1168 1169 int Shift; 1170 if (IsAShr && AP1.isNegative()) 1171 Shift = AP1.countLeadingOnes() - AP2.countLeadingOnes(); 1172 else 1173 Shift = AP1.countLeadingZeros() - AP2.countLeadingZeros(); 1174 1175 if (Shift > 0) { 1176 if (IsAShr && AP1 == AP2.ashr(Shift)) { 1177 // There are multiple solutions if we are comparing against -1 and the LHS 1178 // of the ashr is not a power of two. 1179 if (AP1.isAllOnes() && !AP2.isPowerOf2()) 1180 return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift)); 1181 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); 1182 } else if (AP1 == AP2.lshr(Shift)) { 1183 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); 1184 } 1185 } 1186 1187 // Shifting const2 will never be equal to const1. 1188 // FIXME: This should always be handled by InstSimplify? 1189 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); 1190 return replaceInstUsesWith(I, TorF); 1191 } 1192 1193 /// Handle "(icmp eq/ne (shl AP2, A), AP1)" -> 1194 /// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)). 1195 Instruction *InstCombinerImpl::foldICmpShlConstConst(ICmpInst &I, Value *A, 1196 const APInt &AP1, 1197 const APInt &AP2) { 1198 assert(I.isEquality() && "Cannot fold icmp gt/lt"); 1199 1200 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { 1201 if (I.getPredicate() == I.ICMP_NE) 1202 Pred = CmpInst::getInversePredicate(Pred); 1203 return new ICmpInst(Pred, LHS, RHS); 1204 }; 1205 1206 // Don't bother doing any work for cases which InstSimplify handles. 1207 if (AP2.isZero()) 1208 return nullptr; 1209 1210 unsigned AP2TrailingZeros = AP2.countTrailingZeros(); 1211 1212 if (!AP1 && AP2TrailingZeros != 0) 1213 return getICmp( 1214 I.ICMP_UGE, A, 1215 ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros)); 1216 1217 if (AP1 == AP2) 1218 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); 1219 1220 // Get the distance between the lowest bits that are set. 1221 int Shift = AP1.countTrailingZeros() - AP2TrailingZeros; 1222 1223 if (Shift > 0 && AP2.shl(Shift) == AP1) 1224 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); 1225 1226 // Shifting const2 will never be equal to const1. 1227 // FIXME: This should always be handled by InstSimplify? 1228 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); 1229 return replaceInstUsesWith(I, TorF); 1230 } 1231 1232 /// The caller has matched a pattern of the form: 1233 /// I = icmp ugt (add (add A, B), CI2), CI1 1234 /// If this is of the form: 1235 /// sum = a + b 1236 /// if (sum+128 >u 255) 1237 /// Then replace it with llvm.sadd.with.overflow.i8. 1238 /// 1239 static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, 1240 ConstantInt *CI2, ConstantInt *CI1, 1241 InstCombinerImpl &IC) { 1242 // The transformation we're trying to do here is to transform this into an 1243 // llvm.sadd.with.overflow. To do this, we have to replace the original add 1244 // with a narrower add, and discard the add-with-constant that is part of the 1245 // range check (if we can't eliminate it, this isn't profitable). 1246 1247 // In order to eliminate the add-with-constant, the compare can be its only 1248 // use. 1249 Instruction *AddWithCst = cast<Instruction>(I.getOperand(0)); 1250 if (!AddWithCst->hasOneUse()) 1251 return nullptr; 1252 1253 // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow. 1254 if (!CI2->getValue().isPowerOf2()) 1255 return nullptr; 1256 unsigned NewWidth = CI2->getValue().countTrailingZeros(); 1257 if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31) 1258 return nullptr; 1259 1260 // The width of the new add formed is 1 more than the bias. 1261 ++NewWidth; 1262 1263 // Check to see that CI1 is an all-ones value with NewWidth bits. 1264 if (CI1->getBitWidth() == NewWidth || 1265 CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth)) 1266 return nullptr; 1267 1268 // This is only really a signed overflow check if the inputs have been 1269 // sign-extended; check for that condition. For example, if CI2 is 2^31 and 1270 // the operands of the add are 64 bits wide, we need at least 33 sign bits. 1271 if (IC.ComputeMaxSignificantBits(A, 0, &I) > NewWidth || 1272 IC.ComputeMaxSignificantBits(B, 0, &I) > NewWidth) 1273 return nullptr; 1274 1275 // In order to replace the original add with a narrower 1276 // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant 1277 // and truncates that discard the high bits of the add. Verify that this is 1278 // the case. 1279 Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0)); 1280 for (User *U : OrigAdd->users()) { 1281 if (U == AddWithCst) 1282 continue; 1283 1284 // Only accept truncates for now. We would really like a nice recursive 1285 // predicate like SimplifyDemandedBits, but which goes downwards the use-def 1286 // chain to see which bits of a value are actually demanded. If the 1287 // original add had another add which was then immediately truncated, we 1288 // could still do the transformation. 1289 TruncInst *TI = dyn_cast<TruncInst>(U); 1290 if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth) 1291 return nullptr; 1292 } 1293 1294 // If the pattern matches, truncate the inputs to the narrower type and 1295 // use the sadd_with_overflow intrinsic to efficiently compute both the 1296 // result and the overflow bit. 1297 Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth); 1298 Function *F = Intrinsic::getDeclaration( 1299 I.getModule(), Intrinsic::sadd_with_overflow, NewType); 1300 1301 InstCombiner::BuilderTy &Builder = IC.Builder; 1302 1303 // Put the new code above the original add, in case there are any uses of the 1304 // add between the add and the compare. 1305 Builder.SetInsertPoint(OrigAdd); 1306 1307 Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc"); 1308 Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc"); 1309 CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd"); 1310 Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result"); 1311 Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType()); 1312 1313 // The inner add was the result of the narrow add, zero extended to the 1314 // wider type. Replace it with the result computed by the intrinsic. 1315 IC.replaceInstUsesWith(*OrigAdd, ZExt); 1316 IC.eraseInstFromFunction(*OrigAdd); 1317 1318 // The original icmp gets replaced with the overflow value. 1319 return ExtractValueInst::Create(Call, 1, "sadd.overflow"); 1320 } 1321 1322 /// If we have: 1323 /// icmp eq/ne (urem/srem %x, %y), 0 1324 /// iff %y is a power-of-two, we can replace this with a bit test: 1325 /// icmp eq/ne (and %x, (add %y, -1)), 0 1326 Instruction *InstCombinerImpl::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) { 1327 // This fold is only valid for equality predicates. 1328 if (!I.isEquality()) 1329 return nullptr; 1330 ICmpInst::Predicate Pred; 1331 Value *X, *Y, *Zero; 1332 if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))), 1333 m_CombineAnd(m_Zero(), m_Value(Zero))))) 1334 return nullptr; 1335 if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, 0, &I)) 1336 return nullptr; 1337 // This may increase instruction count, we don't enforce that Y is a constant. 1338 Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType())); 1339 Value *Masked = Builder.CreateAnd(X, Mask); 1340 return ICmpInst::Create(Instruction::ICmp, Pred, Masked, Zero); 1341 } 1342 1343 /// Fold equality-comparison between zero and any (maybe truncated) right-shift 1344 /// by one-less-than-bitwidth into a sign test on the original value. 1345 Instruction *InstCombinerImpl::foldSignBitTest(ICmpInst &I) { 1346 Instruction *Val; 1347 ICmpInst::Predicate Pred; 1348 if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero()))) 1349 return nullptr; 1350 1351 Value *X; 1352 Type *XTy; 1353 1354 Constant *C; 1355 if (match(Val, m_TruncOrSelf(m_Shr(m_Value(X), m_Constant(C))))) { 1356 XTy = X->getType(); 1357 unsigned XBitWidth = XTy->getScalarSizeInBits(); 1358 if (!match(C, m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_EQ, 1359 APInt(XBitWidth, XBitWidth - 1)))) 1360 return nullptr; 1361 } else if (isa<BinaryOperator>(Val) && 1362 (X = reassociateShiftAmtsOfTwoSameDirectionShifts( 1363 cast<BinaryOperator>(Val), SQ.getWithInstruction(Val), 1364 /*AnalyzeForSignBitExtraction=*/true))) { 1365 XTy = X->getType(); 1366 } else 1367 return nullptr; 1368 1369 return ICmpInst::Create(Instruction::ICmp, 1370 Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SGE 1371 : ICmpInst::ICMP_SLT, 1372 X, ConstantInt::getNullValue(XTy)); 1373 } 1374 1375 // Handle icmp pred X, 0 1376 Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) { 1377 CmpInst::Predicate Pred = Cmp.getPredicate(); 1378 if (!match(Cmp.getOperand(1), m_Zero())) 1379 return nullptr; 1380 1381 // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0) 1382 if (Pred == ICmpInst::ICMP_SGT) { 1383 Value *A, *B; 1384 if (match(Cmp.getOperand(0), m_SMin(m_Value(A), m_Value(B)))) { 1385 if (isKnownPositive(A, DL, 0, &AC, &Cmp, &DT)) 1386 return new ICmpInst(Pred, B, Cmp.getOperand(1)); 1387 if (isKnownPositive(B, DL, 0, &AC, &Cmp, &DT)) 1388 return new ICmpInst(Pred, A, Cmp.getOperand(1)); 1389 } 1390 } 1391 1392 if (Instruction *New = foldIRemByPowerOfTwoToBitTest(Cmp)) 1393 return New; 1394 1395 // Given: 1396 // icmp eq/ne (urem %x, %y), 0 1397 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem': 1398 // icmp eq/ne %x, 0 1399 Value *X, *Y; 1400 if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) && 1401 ICmpInst::isEquality(Pred)) { 1402 KnownBits XKnown = computeKnownBits(X, 0, &Cmp); 1403 KnownBits YKnown = computeKnownBits(Y, 0, &Cmp); 1404 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2) 1405 return new ICmpInst(Pred, X, Cmp.getOperand(1)); 1406 } 1407 1408 return nullptr; 1409 } 1410 1411 /// Fold icmp Pred X, C. 1412 /// TODO: This code structure does not make sense. The saturating add fold 1413 /// should be moved to some other helper and extended as noted below (it is also 1414 /// possible that code has been made unnecessary - do we canonicalize IR to 1415 /// overflow/saturating intrinsics or not?). 1416 Instruction *InstCombinerImpl::foldICmpWithConstant(ICmpInst &Cmp) { 1417 // Match the following pattern, which is a common idiom when writing 1418 // overflow-safe integer arithmetic functions. The source performs an addition 1419 // in wider type and explicitly checks for overflow using comparisons against 1420 // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic. 1421 // 1422 // TODO: This could probably be generalized to handle other overflow-safe 1423 // operations if we worked out the formulas to compute the appropriate magic 1424 // constants. 1425 // 1426 // sum = a + b 1427 // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8 1428 CmpInst::Predicate Pred = Cmp.getPredicate(); 1429 Value *Op0 = Cmp.getOperand(0), *Op1 = Cmp.getOperand(1); 1430 Value *A, *B; 1431 ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI 1432 if (Pred == ICmpInst::ICMP_UGT && match(Op1, m_ConstantInt(CI)) && 1433 match(Op0, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2)))) 1434 if (Instruction *Res = processUGT_ADDCST_ADD(Cmp, A, B, CI2, CI, *this)) 1435 return Res; 1436 1437 // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...). 1438 Constant *C = dyn_cast<Constant>(Op1); 1439 if (!C || C->canTrap()) 1440 return nullptr; 1441 1442 if (auto *Phi = dyn_cast<PHINode>(Op0)) 1443 if (all_of(Phi->operands(), [](Value *V) { return isa<Constant>(V); })) { 1444 Type *Ty = Cmp.getType(); 1445 Builder.SetInsertPoint(Phi); 1446 PHINode *NewPhi = 1447 Builder.CreatePHI(Ty, Phi->getNumOperands()); 1448 for (BasicBlock *Predecessor : predecessors(Phi->getParent())) { 1449 auto *Input = 1450 cast<Constant>(Phi->getIncomingValueForBlock(Predecessor)); 1451 auto *BoolInput = ConstantExpr::getCompare(Pred, Input, C); 1452 NewPhi->addIncoming(BoolInput, Predecessor); 1453 } 1454 NewPhi->takeName(&Cmp); 1455 return replaceInstUsesWith(Cmp, NewPhi); 1456 } 1457 1458 return nullptr; 1459 } 1460 1461 /// Canonicalize icmp instructions based on dominating conditions. 1462 Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) { 1463 // This is a cheap/incomplete check for dominance - just match a single 1464 // predecessor with a conditional branch. 1465 BasicBlock *CmpBB = Cmp.getParent(); 1466 BasicBlock *DomBB = CmpBB->getSinglePredecessor(); 1467 if (!DomBB) 1468 return nullptr; 1469 1470 Value *DomCond; 1471 BasicBlock *TrueBB, *FalseBB; 1472 if (!match(DomBB->getTerminator(), m_Br(m_Value(DomCond), TrueBB, FalseBB))) 1473 return nullptr; 1474 1475 assert((TrueBB == CmpBB || FalseBB == CmpBB) && 1476 "Predecessor block does not point to successor?"); 1477 1478 // The branch should get simplified. Don't bother simplifying this condition. 1479 if (TrueBB == FalseBB) 1480 return nullptr; 1481 1482 // Try to simplify this compare to T/F based on the dominating condition. 1483 Optional<bool> Imp = isImpliedCondition(DomCond, &Cmp, DL, TrueBB == CmpBB); 1484 if (Imp) 1485 return replaceInstUsesWith(Cmp, ConstantInt::get(Cmp.getType(), *Imp)); 1486 1487 CmpInst::Predicate Pred = Cmp.getPredicate(); 1488 Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1); 1489 ICmpInst::Predicate DomPred; 1490 const APInt *C, *DomC; 1491 if (match(DomCond, m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))) && 1492 match(Y, m_APInt(C))) { 1493 // We have 2 compares of a variable with constants. Calculate the constant 1494 // ranges of those compares to see if we can transform the 2nd compare: 1495 // DomBB: 1496 // DomCond = icmp DomPred X, DomC 1497 // br DomCond, CmpBB, FalseBB 1498 // CmpBB: 1499 // Cmp = icmp Pred X, C 1500 ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, *C); 1501 ConstantRange DominatingCR = 1502 (CmpBB == TrueBB) ? ConstantRange::makeExactICmpRegion(DomPred, *DomC) 1503 : ConstantRange::makeExactICmpRegion( 1504 CmpInst::getInversePredicate(DomPred), *DomC); 1505 ConstantRange Intersection = DominatingCR.intersectWith(CR); 1506 ConstantRange Difference = DominatingCR.difference(CR); 1507 if (Intersection.isEmptySet()) 1508 return replaceInstUsesWith(Cmp, Builder.getFalse()); 1509 if (Difference.isEmptySet()) 1510 return replaceInstUsesWith(Cmp, Builder.getTrue()); 1511 1512 // Canonicalizing a sign bit comparison that gets used in a branch, 1513 // pessimizes codegen by generating branch on zero instruction instead 1514 // of a test and branch. So we avoid canonicalizing in such situations 1515 // because test and branch instruction has better branch displacement 1516 // than compare and branch instruction. 1517 bool UnusedBit; 1518 bool IsSignBit = isSignBitCheck(Pred, *C, UnusedBit); 1519 if (Cmp.isEquality() || (IsSignBit && hasBranchUse(Cmp))) 1520 return nullptr; 1521 1522 // Avoid an infinite loop with min/max canonicalization. 1523 // TODO: This will be unnecessary if we canonicalize to min/max intrinsics. 1524 if (Cmp.hasOneUse() && 1525 match(Cmp.user_back(), m_MaxOrMin(m_Value(), m_Value()))) 1526 return nullptr; 1527 1528 if (const APInt *EqC = Intersection.getSingleElement()) 1529 return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC)); 1530 if (const APInt *NeC = Difference.getSingleElement()) 1531 return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC)); 1532 } 1533 1534 return nullptr; 1535 } 1536 1537 /// Fold icmp (trunc X), C. 1538 Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp, 1539 TruncInst *Trunc, 1540 const APInt &C) { 1541 ICmpInst::Predicate Pred = Cmp.getPredicate(); 1542 Value *X = Trunc->getOperand(0); 1543 if (C.isOne() && C.getBitWidth() > 1) { 1544 // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1 1545 Value *V = nullptr; 1546 if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V)))) 1547 return new ICmpInst(ICmpInst::ICMP_SLT, V, 1548 ConstantInt::get(V->getType(), 1)); 1549 } 1550 1551 unsigned DstBits = Trunc->getType()->getScalarSizeInBits(), 1552 SrcBits = X->getType()->getScalarSizeInBits(); 1553 if (Cmp.isEquality() && Trunc->hasOneUse()) { 1554 // Canonicalize to a mask and wider compare if the wide type is suitable: 1555 // (trunc X to i8) == C --> (X & 0xff) == (zext C) 1556 if (!X->getType()->isVectorTy() && shouldChangeType(DstBits, SrcBits)) { 1557 Constant *Mask = ConstantInt::get(X->getType(), 1558 APInt::getLowBitsSet(SrcBits, DstBits)); 1559 Value *And = Builder.CreateAnd(X, Mask); 1560 Constant *WideC = ConstantInt::get(X->getType(), C.zext(SrcBits)); 1561 return new ICmpInst(Pred, And, WideC); 1562 } 1563 1564 // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all 1565 // of the high bits truncated out of x are known. 1566 KnownBits Known = computeKnownBits(X, 0, &Cmp); 1567 1568 // If all the high bits are known, we can do this xform. 1569 if ((Known.Zero | Known.One).countLeadingOnes() >= SrcBits - DstBits) { 1570 // Pull in the high bits from known-ones set. 1571 APInt NewRHS = C.zext(SrcBits); 1572 NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits); 1573 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), NewRHS)); 1574 } 1575 } 1576 1577 // Look through truncated right-shift of the sign-bit for a sign-bit check: 1578 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0 --> ShOp < 0 1579 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1 1580 Value *ShOp; 1581 const APInt *ShAmtC; 1582 bool TrueIfSigned; 1583 if (isSignBitCheck(Pred, C, TrueIfSigned) && 1584 match(X, m_Shr(m_Value(ShOp), m_APInt(ShAmtC))) && 1585 DstBits == SrcBits - ShAmtC->getZExtValue()) { 1586 return TrueIfSigned 1587 ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp, 1588 ConstantInt::getNullValue(X->getType())) 1589 : new ICmpInst(ICmpInst::ICMP_SGT, ShOp, 1590 ConstantInt::getAllOnesValue(X->getType())); 1591 } 1592 1593 return nullptr; 1594 } 1595 1596 /// Fold icmp (xor X, Y), C. 1597 Instruction *InstCombinerImpl::foldICmpXorConstant(ICmpInst &Cmp, 1598 BinaryOperator *Xor, 1599 const APInt &C) { 1600 Value *X = Xor->getOperand(0); 1601 Value *Y = Xor->getOperand(1); 1602 const APInt *XorC; 1603 if (!match(Y, m_APInt(XorC))) 1604 return nullptr; 1605 1606 // If this is a comparison that tests the signbit (X < 0) or (x > -1), 1607 // fold the xor. 1608 ICmpInst::Predicate Pred = Cmp.getPredicate(); 1609 bool TrueIfSigned = false; 1610 if (isSignBitCheck(Cmp.getPredicate(), C, TrueIfSigned)) { 1611 1612 // If the sign bit of the XorCst is not set, there is no change to 1613 // the operation, just stop using the Xor. 1614 if (!XorC->isNegative()) 1615 return replaceOperand(Cmp, 0, X); 1616 1617 // Emit the opposite comparison. 1618 if (TrueIfSigned) 1619 return new ICmpInst(ICmpInst::ICMP_SGT, X, 1620 ConstantInt::getAllOnesValue(X->getType())); 1621 else 1622 return new ICmpInst(ICmpInst::ICMP_SLT, X, 1623 ConstantInt::getNullValue(X->getType())); 1624 } 1625 1626 if (Xor->hasOneUse()) { 1627 // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask)) 1628 if (!Cmp.isEquality() && XorC->isSignMask()) { 1629 Pred = Cmp.getFlippedSignednessPredicate(); 1630 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC)); 1631 } 1632 1633 // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask)) 1634 if (!Cmp.isEquality() && XorC->isMaxSignedValue()) { 1635 Pred = Cmp.getFlippedSignednessPredicate(); 1636 Pred = Cmp.getSwappedPredicate(Pred); 1637 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC)); 1638 } 1639 } 1640 1641 // Mask constant magic can eliminate an 'xor' with unsigned compares. 1642 if (Pred == ICmpInst::ICMP_UGT) { 1643 // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2) 1644 if (*XorC == ~C && (C + 1).isPowerOf2()) 1645 return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); 1646 // (xor X, C) >u C --> X >u C (when C+1 is a power of 2) 1647 if (*XorC == C && (C + 1).isPowerOf2()) 1648 return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); 1649 } 1650 if (Pred == ICmpInst::ICMP_ULT) { 1651 // (xor X, -C) <u C --> X >u ~C (when C is a power of 2) 1652 if (*XorC == -C && C.isPowerOf2()) 1653 return new ICmpInst(ICmpInst::ICMP_UGT, X, 1654 ConstantInt::get(X->getType(), ~C)); 1655 // (xor X, C) <u C --> X >u ~C (when -C is a power of 2) 1656 if (*XorC == C && (-C).isPowerOf2()) 1657 return new ICmpInst(ICmpInst::ICMP_UGT, X, 1658 ConstantInt::get(X->getType(), ~C)); 1659 } 1660 return nullptr; 1661 } 1662 1663 /// Fold icmp (and (sh X, Y), C2), C1. 1664 Instruction *InstCombinerImpl::foldICmpAndShift(ICmpInst &Cmp, 1665 BinaryOperator *And, 1666 const APInt &C1, 1667 const APInt &C2) { 1668 BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0)); 1669 if (!Shift || !Shift->isShift()) 1670 return nullptr; 1671 1672 // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could 1673 // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in 1674 // code produced by the clang front-end, for bitfield access. 1675 // This seemingly simple opportunity to fold away a shift turns out to be 1676 // rather complicated. See PR17827 for details. 1677 unsigned ShiftOpcode = Shift->getOpcode(); 1678 bool IsShl = ShiftOpcode == Instruction::Shl; 1679 const APInt *C3; 1680 if (match(Shift->getOperand(1), m_APInt(C3))) { 1681 APInt NewAndCst, NewCmpCst; 1682 bool AnyCmpCstBitsShiftedOut; 1683 if (ShiftOpcode == Instruction::Shl) { 1684 // For a left shift, we can fold if the comparison is not signed. We can 1685 // also fold a signed comparison if the mask value and comparison value 1686 // are not negative. These constraints may not be obvious, but we can 1687 // prove that they are correct using an SMT solver. 1688 if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative())) 1689 return nullptr; 1690 1691 NewCmpCst = C1.lshr(*C3); 1692 NewAndCst = C2.lshr(*C3); 1693 AnyCmpCstBitsShiftedOut = NewCmpCst.shl(*C3) != C1; 1694 } else if (ShiftOpcode == Instruction::LShr) { 1695 // For a logical right shift, we can fold if the comparison is not signed. 1696 // We can also fold a signed comparison if the shifted mask value and the 1697 // shifted comparison value are not negative. These constraints may not be 1698 // obvious, but we can prove that they are correct using an SMT solver. 1699 NewCmpCst = C1.shl(*C3); 1700 NewAndCst = C2.shl(*C3); 1701 AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(*C3) != C1; 1702 if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative())) 1703 return nullptr; 1704 } else { 1705 // For an arithmetic shift, check that both constants don't use (in a 1706 // signed sense) the top bits being shifted out. 1707 assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode"); 1708 NewCmpCst = C1.shl(*C3); 1709 NewAndCst = C2.shl(*C3); 1710 AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(*C3) != C1; 1711 if (NewAndCst.ashr(*C3) != C2) 1712 return nullptr; 1713 } 1714 1715 if (AnyCmpCstBitsShiftedOut) { 1716 // If we shifted bits out, the fold is not going to work out. As a 1717 // special case, check to see if this means that the result is always 1718 // true or false now. 1719 if (Cmp.getPredicate() == ICmpInst::ICMP_EQ) 1720 return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType())); 1721 if (Cmp.getPredicate() == ICmpInst::ICMP_NE) 1722 return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType())); 1723 } else { 1724 Value *NewAnd = Builder.CreateAnd( 1725 Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst)); 1726 return new ICmpInst(Cmp.getPredicate(), 1727 NewAnd, ConstantInt::get(And->getType(), NewCmpCst)); 1728 } 1729 } 1730 1731 // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is 1732 // preferable because it allows the C2 << Y expression to be hoisted out of a 1733 // loop if Y is invariant and X is not. 1734 if (Shift->hasOneUse() && C1.isZero() && Cmp.isEquality() && 1735 !Shift->isArithmeticShift() && !isa<Constant>(Shift->getOperand(0))) { 1736 // Compute C2 << Y. 1737 Value *NewShift = 1738 IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1)) 1739 : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1)); 1740 1741 // Compute X & (C2 << Y). 1742 Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift); 1743 return replaceOperand(Cmp, 0, NewAnd); 1744 } 1745 1746 return nullptr; 1747 } 1748 1749 /// Fold icmp (and X, C2), C1. 1750 Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp, 1751 BinaryOperator *And, 1752 const APInt &C1) { 1753 bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE; 1754 1755 // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1 1756 // TODO: We canonicalize to the longer form for scalars because we have 1757 // better analysis/folds for icmp, and codegen may be better with icmp. 1758 if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isZero() && 1759 match(And->getOperand(1), m_One())) 1760 return new TruncInst(And->getOperand(0), Cmp.getType()); 1761 1762 const APInt *C2; 1763 Value *X; 1764 if (!match(And, m_And(m_Value(X), m_APInt(C2)))) 1765 return nullptr; 1766 1767 // Don't perform the following transforms if the AND has multiple uses 1768 if (!And->hasOneUse()) 1769 return nullptr; 1770 1771 if (Cmp.isEquality() && C1.isZero()) { 1772 // Restrict this fold to single-use 'and' (PR10267). 1773 // Replace (and X, (1 << size(X)-1) != 0) with X s< 0 1774 if (C2->isSignMask()) { 1775 Constant *Zero = Constant::getNullValue(X->getType()); 1776 auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE; 1777 return new ICmpInst(NewPred, X, Zero); 1778 } 1779 1780 // Restrict this fold only for single-use 'and' (PR10267). 1781 // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two. 1782 if ((~(*C2) + 1).isPowerOf2()) { 1783 Constant *NegBOC = 1784 ConstantExpr::getNeg(cast<Constant>(And->getOperand(1))); 1785 auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; 1786 return new ICmpInst(NewPred, X, NegBOC); 1787 } 1788 } 1789 1790 // If the LHS is an 'and' of a truncate and we can widen the and/compare to 1791 // the input width without changing the value produced, eliminate the cast: 1792 // 1793 // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1' 1794 // 1795 // We can do this transformation if the constants do not have their sign bits 1796 // set or if it is an equality comparison. Extending a relational comparison 1797 // when we're checking the sign bit would not work. 1798 Value *W; 1799 if (match(And->getOperand(0), m_OneUse(m_Trunc(m_Value(W)))) && 1800 (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) { 1801 // TODO: Is this a good transform for vectors? Wider types may reduce 1802 // throughput. Should this transform be limited (even for scalars) by using 1803 // shouldChangeType()? 1804 if (!Cmp.getType()->isVectorTy()) { 1805 Type *WideType = W->getType(); 1806 unsigned WideScalarBits = WideType->getScalarSizeInBits(); 1807 Constant *ZextC1 = ConstantInt::get(WideType, C1.zext(WideScalarBits)); 1808 Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits)); 1809 Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName()); 1810 return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1); 1811 } 1812 } 1813 1814 if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2)) 1815 return I; 1816 1817 // (icmp pred (and (or (lshr A, B), A), 1), 0) --> 1818 // (icmp pred (and A, (or (shl 1, B), 1), 0)) 1819 // 1820 // iff pred isn't signed 1821 if (!Cmp.isSigned() && C1.isZero() && And->getOperand(0)->hasOneUse() && 1822 match(And->getOperand(1), m_One())) { 1823 Constant *One = cast<Constant>(And->getOperand(1)); 1824 Value *Or = And->getOperand(0); 1825 Value *A, *B, *LShr; 1826 if (match(Or, m_Or(m_Value(LShr), m_Value(A))) && 1827 match(LShr, m_LShr(m_Specific(A), m_Value(B)))) { 1828 unsigned UsesRemoved = 0; 1829 if (And->hasOneUse()) 1830 ++UsesRemoved; 1831 if (Or->hasOneUse()) 1832 ++UsesRemoved; 1833 if (LShr->hasOneUse()) 1834 ++UsesRemoved; 1835 1836 // Compute A & ((1 << B) | 1) 1837 Value *NewOr = nullptr; 1838 if (auto *C = dyn_cast<Constant>(B)) { 1839 if (UsesRemoved >= 1) 1840 NewOr = ConstantExpr::getOr(ConstantExpr::getNUWShl(One, C), One); 1841 } else { 1842 if (UsesRemoved >= 3) 1843 NewOr = Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(), 1844 /*HasNUW=*/true), 1845 One, Or->getName()); 1846 } 1847 if (NewOr) { 1848 Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName()); 1849 return replaceOperand(Cmp, 0, NewAnd); 1850 } 1851 } 1852 } 1853 1854 return nullptr; 1855 } 1856 1857 /// Fold icmp (and X, Y), C. 1858 Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp, 1859 BinaryOperator *And, 1860 const APInt &C) { 1861 if (Instruction *I = foldICmpAndConstConst(Cmp, And, C)) 1862 return I; 1863 1864 const ICmpInst::Predicate Pred = Cmp.getPredicate(); 1865 bool TrueIfNeg; 1866 if (isSignBitCheck(Pred, C, TrueIfNeg)) { 1867 // ((X - 1) & ~X) < 0 --> X == 0 1868 // ((X - 1) & ~X) >= 0 --> X != 0 1869 Value *X; 1870 if (match(And->getOperand(0), m_Add(m_Value(X), m_AllOnes())) && 1871 match(And->getOperand(1), m_Not(m_Specific(X)))) { 1872 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE; 1873 return new ICmpInst(NewPred, X, ConstantInt::getNullValue(X->getType())); 1874 } 1875 } 1876 1877 // TODO: These all require that Y is constant too, so refactor with the above. 1878 1879 // Try to optimize things like "A[i] & 42 == 0" to index computations. 1880 Value *X = And->getOperand(0); 1881 Value *Y = And->getOperand(1); 1882 if (auto *C2 = dyn_cast<ConstantInt>(Y)) 1883 if (auto *LI = dyn_cast<LoadInst>(X)) 1884 if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0))) 1885 if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) 1886 if (Instruction *Res = 1887 foldCmpLoadFromIndexedGlobal(LI, GEP, GV, Cmp, C2)) 1888 return Res; 1889 1890 if (!Cmp.isEquality()) 1891 return nullptr; 1892 1893 // X & -C == -C -> X > u ~C 1894 // X & -C != -C -> X <= u ~C 1895 // iff C is a power of 2 1896 if (Cmp.getOperand(1) == Y && C.isNegatedPowerOf2()) { 1897 auto NewPred = 1898 Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT : CmpInst::ICMP_ULE; 1899 return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1)))); 1900 } 1901 1902 return nullptr; 1903 } 1904 1905 /// Fold icmp (or X, Y), C. 1906 Instruction *InstCombinerImpl::foldICmpOrConstant(ICmpInst &Cmp, 1907 BinaryOperator *Or, 1908 const APInt &C) { 1909 ICmpInst::Predicate Pred = Cmp.getPredicate(); 1910 if (C.isOne()) { 1911 // icmp slt signum(V) 1 --> icmp slt V, 1 1912 Value *V = nullptr; 1913 if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V)))) 1914 return new ICmpInst(ICmpInst::ICMP_SLT, V, 1915 ConstantInt::get(V->getType(), 1)); 1916 } 1917 1918 Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1); 1919 const APInt *MaskC; 1920 if (match(OrOp1, m_APInt(MaskC)) && Cmp.isEquality()) { 1921 if (*MaskC == C && (C + 1).isPowerOf2()) { 1922 // X | C == C --> X <=u C 1923 // X | C != C --> X >u C 1924 // iff C+1 is a power of 2 (C is a bitmask of the low bits) 1925 Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT; 1926 return new ICmpInst(Pred, OrOp0, OrOp1); 1927 } 1928 1929 // More general: canonicalize 'equality with set bits mask' to 1930 // 'equality with clear bits mask'. 1931 // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC 1932 // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC 1933 if (Or->hasOneUse()) { 1934 Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC)); 1935 Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC)); 1936 return new ICmpInst(Pred, And, NewC); 1937 } 1938 } 1939 1940 // (X | (X-1)) s< 0 --> X s< 1 1941 // (X | (X-1)) s> -1 --> X s> 0 1942 Value *X; 1943 bool TrueIfSigned; 1944 if (isSignBitCheck(Pred, C, TrueIfSigned) && 1945 match(Or, m_c_Or(m_Add(m_Value(X), m_AllOnes()), m_Deferred(X)))) { 1946 auto NewPred = TrueIfSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGT; 1947 Constant *NewC = ConstantInt::get(X->getType(), TrueIfSigned ? 1 : 0); 1948 return new ICmpInst(NewPred, X, NewC); 1949 } 1950 1951 if (!Cmp.isEquality() || !C.isZero() || !Or->hasOneUse()) 1952 return nullptr; 1953 1954 Value *P, *Q; 1955 if (match(Or, m_Or(m_PtrToInt(m_Value(P)), m_PtrToInt(m_Value(Q))))) { 1956 // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0 1957 // -> and (icmp eq P, null), (icmp eq Q, null). 1958 Value *CmpP = 1959 Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType())); 1960 Value *CmpQ = 1961 Builder.CreateICmp(Pred, Q, ConstantInt::getNullValue(Q->getType())); 1962 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; 1963 return BinaryOperator::Create(BOpc, CmpP, CmpQ); 1964 } 1965 1966 // Are we using xors to bitwise check for a pair of (in)equalities? Convert to 1967 // a shorter form that has more potential to be folded even further. 1968 Value *X1, *X2, *X3, *X4; 1969 if (match(OrOp0, m_OneUse(m_Xor(m_Value(X1), m_Value(X2)))) && 1970 match(OrOp1, m_OneUse(m_Xor(m_Value(X3), m_Value(X4))))) { 1971 // ((X1 ^ X2) || (X3 ^ X4)) == 0 --> (X1 == X2) && (X3 == X4) 1972 // ((X1 ^ X2) || (X3 ^ X4)) != 0 --> (X1 != X2) || (X3 != X4) 1973 Value *Cmp12 = Builder.CreateICmp(Pred, X1, X2); 1974 Value *Cmp34 = Builder.CreateICmp(Pred, X3, X4); 1975 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; 1976 return BinaryOperator::Create(BOpc, Cmp12, Cmp34); 1977 } 1978 1979 return nullptr; 1980 } 1981 1982 /// Fold icmp (mul X, Y), C. 1983 Instruction *InstCombinerImpl::foldICmpMulConstant(ICmpInst &Cmp, 1984 BinaryOperator *Mul, 1985 const APInt &C) { 1986 const APInt *MulC; 1987 if (!match(Mul->getOperand(1), m_APInt(MulC))) 1988 return nullptr; 1989 1990 // If this is a test of the sign bit and the multiply is sign-preserving with 1991 // a constant operand, use the multiply LHS operand instead. 1992 ICmpInst::Predicate Pred = Cmp.getPredicate(); 1993 if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) { 1994 if (MulC->isNegative()) 1995 Pred = ICmpInst::getSwappedPredicate(Pred); 1996 return new ICmpInst(Pred, Mul->getOperand(0), 1997 Constant::getNullValue(Mul->getType())); 1998 } 1999 2000 // If the multiply does not wrap, try to divide the compare constant by the 2001 // multiplication factor. 2002 if (Cmp.isEquality() && !MulC->isZero()) { 2003 // (mul nsw X, MulC) == C --> X == C /s MulC 2004 if (Mul->hasNoSignedWrap() && C.srem(*MulC).isZero()) { 2005 Constant *NewC = ConstantInt::get(Mul->getType(), C.sdiv(*MulC)); 2006 return new ICmpInst(Pred, Mul->getOperand(0), NewC); 2007 } 2008 // (mul nuw X, MulC) == C --> X == C /u MulC 2009 if (Mul->hasNoUnsignedWrap() && C.urem(*MulC).isZero()) { 2010 Constant *NewC = ConstantInt::get(Mul->getType(), C.udiv(*MulC)); 2011 return new ICmpInst(Pred, Mul->getOperand(0), NewC); 2012 } 2013 } 2014 2015 return nullptr; 2016 } 2017 2018 /// Fold icmp (shl 1, Y), C. 2019 static Instruction *foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl, 2020 const APInt &C) { 2021 Value *Y; 2022 if (!match(Shl, m_Shl(m_One(), m_Value(Y)))) 2023 return nullptr; 2024 2025 Type *ShiftType = Shl->getType(); 2026 unsigned TypeBits = C.getBitWidth(); 2027 bool CIsPowerOf2 = C.isPowerOf2(); 2028 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2029 if (Cmp.isUnsigned()) { 2030 // (1 << Y) pred C -> Y pred Log2(C) 2031 if (!CIsPowerOf2) { 2032 // (1 << Y) < 30 -> Y <= 4 2033 // (1 << Y) <= 30 -> Y <= 4 2034 // (1 << Y) >= 30 -> Y > 4 2035 // (1 << Y) > 30 -> Y > 4 2036 if (Pred == ICmpInst::ICMP_ULT) 2037 Pred = ICmpInst::ICMP_ULE; 2038 else if (Pred == ICmpInst::ICMP_UGE) 2039 Pred = ICmpInst::ICMP_UGT; 2040 } 2041 2042 // (1 << Y) >= 2147483648 -> Y >= 31 -> Y == 31 2043 // (1 << Y) < 2147483648 -> Y < 31 -> Y != 31 2044 unsigned CLog2 = C.logBase2(); 2045 if (CLog2 == TypeBits - 1) { 2046 if (Pred == ICmpInst::ICMP_UGE) 2047 Pred = ICmpInst::ICMP_EQ; 2048 else if (Pred == ICmpInst::ICMP_ULT) 2049 Pred = ICmpInst::ICMP_NE; 2050 } 2051 return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2)); 2052 } else if (Cmp.isSigned()) { 2053 Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1); 2054 if (C.isAllOnes()) { 2055 // (1 << Y) <= -1 -> Y == 31 2056 if (Pred == ICmpInst::ICMP_SLE) 2057 return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); 2058 2059 // (1 << Y) > -1 -> Y != 31 2060 if (Pred == ICmpInst::ICMP_SGT) 2061 return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); 2062 } else if (!C) { 2063 // (1 << Y) < 0 -> Y == 31 2064 // (1 << Y) <= 0 -> Y == 31 2065 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) 2066 return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); 2067 2068 // (1 << Y) >= 0 -> Y != 31 2069 // (1 << Y) > 0 -> Y != 31 2070 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) 2071 return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); 2072 } 2073 } else if (Cmp.isEquality() && CIsPowerOf2) { 2074 return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, C.logBase2())); 2075 } 2076 2077 return nullptr; 2078 } 2079 2080 /// Fold icmp (shl X, Y), C. 2081 Instruction *InstCombinerImpl::foldICmpShlConstant(ICmpInst &Cmp, 2082 BinaryOperator *Shl, 2083 const APInt &C) { 2084 const APInt *ShiftVal; 2085 if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal))) 2086 return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal); 2087 2088 const APInt *ShiftAmt; 2089 if (!match(Shl->getOperand(1), m_APInt(ShiftAmt))) 2090 return foldICmpShlOne(Cmp, Shl, C); 2091 2092 // Check that the shift amount is in range. If not, don't perform undefined 2093 // shifts. When the shift is visited, it will be simplified. 2094 unsigned TypeBits = C.getBitWidth(); 2095 if (ShiftAmt->uge(TypeBits)) 2096 return nullptr; 2097 2098 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2099 Value *X = Shl->getOperand(0); 2100 Type *ShType = Shl->getType(); 2101 2102 // NSW guarantees that we are only shifting out sign bits from the high bits, 2103 // so we can ASHR the compare constant without needing a mask and eliminate 2104 // the shift. 2105 if (Shl->hasNoSignedWrap()) { 2106 if (Pred == ICmpInst::ICMP_SGT) { 2107 // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt) 2108 APInt ShiftedC = C.ashr(*ShiftAmt); 2109 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2110 } 2111 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && 2112 C.ashr(*ShiftAmt).shl(*ShiftAmt) == C) { 2113 APInt ShiftedC = C.ashr(*ShiftAmt); 2114 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2115 } 2116 if (Pred == ICmpInst::ICMP_SLT) { 2117 // SLE is the same as above, but SLE is canonicalized to SLT, so convert: 2118 // (X << S) <=s C is equiv to X <=s (C >> S) for all C 2119 // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX 2120 // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN 2121 assert(!C.isMinSignedValue() && "Unexpected icmp slt"); 2122 APInt ShiftedC = (C - 1).ashr(*ShiftAmt) + 1; 2123 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2124 } 2125 // If this is a signed comparison to 0 and the shift is sign preserving, 2126 // use the shift LHS operand instead; isSignTest may change 'Pred', so only 2127 // do that if we're sure to not continue on in this function. 2128 if (isSignTest(Pred, C)) 2129 return new ICmpInst(Pred, X, Constant::getNullValue(ShType)); 2130 } 2131 2132 // NUW guarantees that we are only shifting out zero bits from the high bits, 2133 // so we can LSHR the compare constant without needing a mask and eliminate 2134 // the shift. 2135 if (Shl->hasNoUnsignedWrap()) { 2136 if (Pred == ICmpInst::ICMP_UGT) { 2137 // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt) 2138 APInt ShiftedC = C.lshr(*ShiftAmt); 2139 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2140 } 2141 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && 2142 C.lshr(*ShiftAmt).shl(*ShiftAmt) == C) { 2143 APInt ShiftedC = C.lshr(*ShiftAmt); 2144 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2145 } 2146 if (Pred == ICmpInst::ICMP_ULT) { 2147 // ULE is the same as above, but ULE is canonicalized to ULT, so convert: 2148 // (X << S) <=u C is equiv to X <=u (C >> S) for all C 2149 // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u 2150 // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0 2151 assert(C.ugt(0) && "ult 0 should have been eliminated"); 2152 APInt ShiftedC = (C - 1).lshr(*ShiftAmt) + 1; 2153 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2154 } 2155 } 2156 2157 if (Cmp.isEquality() && Shl->hasOneUse()) { 2158 // Strength-reduce the shift into an 'and'. 2159 Constant *Mask = ConstantInt::get( 2160 ShType, 2161 APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue())); 2162 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); 2163 Constant *LShrC = ConstantInt::get(ShType, C.lshr(*ShiftAmt)); 2164 return new ICmpInst(Pred, And, LShrC); 2165 } 2166 2167 // Otherwise, if this is a comparison of the sign bit, simplify to and/test. 2168 bool TrueIfSigned = false; 2169 if (Shl->hasOneUse() && isSignBitCheck(Pred, C, TrueIfSigned)) { 2170 // (X << 31) <s 0 --> (X & 1) != 0 2171 Constant *Mask = ConstantInt::get( 2172 ShType, 2173 APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1)); 2174 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); 2175 return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ, 2176 And, Constant::getNullValue(ShType)); 2177 } 2178 2179 // Simplify 'shl' inequality test into 'and' equality test. 2180 if (Cmp.isUnsigned() && Shl->hasOneUse()) { 2181 // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0 2182 if ((C + 1).isPowerOf2() && 2183 (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) { 2184 Value *And = Builder.CreateAnd(X, (~C).lshr(ShiftAmt->getZExtValue())); 2185 return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ 2186 : ICmpInst::ICMP_NE, 2187 And, Constant::getNullValue(ShType)); 2188 } 2189 // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0 2190 if (C.isPowerOf2() && 2191 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) { 2192 Value *And = 2193 Builder.CreateAnd(X, (~(C - 1)).lshr(ShiftAmt->getZExtValue())); 2194 return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ 2195 : ICmpInst::ICMP_NE, 2196 And, Constant::getNullValue(ShType)); 2197 } 2198 } 2199 2200 // Transform (icmp pred iM (shl iM %v, N), C) 2201 // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N)) 2202 // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N. 2203 // This enables us to get rid of the shift in favor of a trunc that may be 2204 // free on the target. It has the additional benefit of comparing to a 2205 // smaller constant that may be more target-friendly. 2206 unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1); 2207 if (Shl->hasOneUse() && Amt != 0 && C.countTrailingZeros() >= Amt && 2208 DL.isLegalInteger(TypeBits - Amt)) { 2209 Type *TruncTy = IntegerType::get(Cmp.getContext(), TypeBits - Amt); 2210 if (auto *ShVTy = dyn_cast<VectorType>(ShType)) 2211 TruncTy = VectorType::get(TruncTy, ShVTy->getElementCount()); 2212 Constant *NewC = 2213 ConstantInt::get(TruncTy, C.ashr(*ShiftAmt).trunc(TypeBits - Amt)); 2214 return new ICmpInst(Pred, Builder.CreateTrunc(X, TruncTy), NewC); 2215 } 2216 2217 return nullptr; 2218 } 2219 2220 /// Fold icmp ({al}shr X, Y), C. 2221 Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp, 2222 BinaryOperator *Shr, 2223 const APInt &C) { 2224 // An exact shr only shifts out zero bits, so: 2225 // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0 2226 Value *X = Shr->getOperand(0); 2227 CmpInst::Predicate Pred = Cmp.getPredicate(); 2228 if (Cmp.isEquality() && Shr->isExact() && C.isZero()) 2229 return new ICmpInst(Pred, X, Cmp.getOperand(1)); 2230 2231 bool IsAShr = Shr->getOpcode() == Instruction::AShr; 2232 const APInt *ShiftValC; 2233 if (match(Shr->getOperand(0), m_APInt(ShiftValC))) { 2234 if (Cmp.isEquality()) 2235 return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftValC); 2236 2237 // If the shifted constant is a power-of-2, test the shift amount directly: 2238 // (ShiftValC >> X) >u C --> X <u (LZ(C) - LZ(ShiftValC)) 2239 // (ShiftValC >> X) <u C --> X >=u (LZ(C-1) - LZ(ShiftValC)) 2240 if (!IsAShr && ShiftValC->isPowerOf2() && 2241 (Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_ULT)) { 2242 bool IsUGT = Pred == CmpInst::ICMP_UGT; 2243 assert(ShiftValC->uge(C) && "Expected simplify of compare"); 2244 assert((IsUGT || !C.isZero()) && "Expected X u< 0 to simplify"); 2245 2246 unsigned CmpLZ = 2247 IsUGT ? C.countLeadingZeros() : (C - 1).countLeadingZeros(); 2248 unsigned ShiftLZ = ShiftValC->countLeadingZeros(); 2249 Constant *NewC = ConstantInt::get(Shr->getType(), CmpLZ - ShiftLZ); 2250 auto NewPred = IsUGT ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE; 2251 return new ICmpInst(NewPred, Shr->getOperand(1), NewC); 2252 } 2253 } 2254 2255 const APInt *ShiftAmtC; 2256 if (!match(Shr->getOperand(1), m_APInt(ShiftAmtC))) 2257 return nullptr; 2258 2259 // Check that the shift amount is in range. If not, don't perform undefined 2260 // shifts. When the shift is visited it will be simplified. 2261 unsigned TypeBits = C.getBitWidth(); 2262 unsigned ShAmtVal = ShiftAmtC->getLimitedValue(TypeBits); 2263 if (ShAmtVal >= TypeBits || ShAmtVal == 0) 2264 return nullptr; 2265 2266 bool IsExact = Shr->isExact(); 2267 Type *ShrTy = Shr->getType(); 2268 // TODO: If we could guarantee that InstSimplify would handle all of the 2269 // constant-value-based preconditions in the folds below, then we could assert 2270 // those conditions rather than checking them. This is difficult because of 2271 // undef/poison (PR34838). 2272 if (IsAShr) { 2273 if (IsExact || Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) { 2274 // When ShAmtC can be shifted losslessly: 2275 // icmp PRED (ashr exact X, ShAmtC), C --> icmp PRED X, (C << ShAmtC) 2276 // icmp slt/ult (ashr X, ShAmtC), C --> icmp slt/ult X, (C << ShAmtC) 2277 APInt ShiftedC = C.shl(ShAmtVal); 2278 if (ShiftedC.ashr(ShAmtVal) == C) 2279 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); 2280 } 2281 if (Pred == CmpInst::ICMP_SGT) { 2282 // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1 2283 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; 2284 if (!C.isMaxSignedValue() && !(C + 1).shl(ShAmtVal).isMinSignedValue() && 2285 (ShiftedC + 1).ashr(ShAmtVal) == (C + 1)) 2286 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); 2287 } 2288 if (Pred == CmpInst::ICMP_UGT) { 2289 // icmp ugt (ashr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1 2290 // 'C + 1 << ShAmtC' can overflow as a signed number, so the 2nd 2291 // clause accounts for that pattern. 2292 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; 2293 if ((ShiftedC + 1).ashr(ShAmtVal) == (C + 1) || 2294 (C + 1).shl(ShAmtVal).isMinSignedValue()) 2295 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); 2296 } 2297 2298 // If the compare constant has significant bits above the lowest sign-bit, 2299 // then convert an unsigned cmp to a test of the sign-bit: 2300 // (ashr X, ShiftC) u> C --> X s< 0 2301 // (ashr X, ShiftC) u< C --> X s> -1 2302 if (C.getBitWidth() > 2 && C.getNumSignBits() <= ShAmtVal) { 2303 if (Pred == CmpInst::ICMP_UGT) { 2304 return new ICmpInst(CmpInst::ICMP_SLT, X, 2305 ConstantInt::getNullValue(ShrTy)); 2306 } 2307 if (Pred == CmpInst::ICMP_ULT) { 2308 return new ICmpInst(CmpInst::ICMP_SGT, X, 2309 ConstantInt::getAllOnesValue(ShrTy)); 2310 } 2311 } 2312 } else { 2313 if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) { 2314 // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC) 2315 // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC) 2316 APInt ShiftedC = C.shl(ShAmtVal); 2317 if (ShiftedC.lshr(ShAmtVal) == C) 2318 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); 2319 } 2320 if (Pred == CmpInst::ICMP_UGT) { 2321 // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1 2322 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; 2323 if ((ShiftedC + 1).lshr(ShAmtVal) == (C + 1)) 2324 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); 2325 } 2326 } 2327 2328 if (!Cmp.isEquality()) 2329 return nullptr; 2330 2331 // Handle equality comparisons of shift-by-constant. 2332 2333 // If the comparison constant changes with the shift, the comparison cannot 2334 // succeed (bits of the comparison constant cannot match the shifted value). 2335 // This should be known by InstSimplify and already be folded to true/false. 2336 assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || 2337 (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && 2338 "Expected icmp+shr simplify did not occur."); 2339 2340 // If the bits shifted out are known zero, compare the unshifted value: 2341 // (X & 4) >> 1 == 2 --> (X & 4) == 4. 2342 if (Shr->isExact()) 2343 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal)); 2344 2345 if (C.isZero()) { 2346 // == 0 is u< 1. 2347 if (Pred == CmpInst::ICMP_EQ) 2348 return new ICmpInst(CmpInst::ICMP_ULT, X, 2349 ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal))); 2350 else 2351 return new ICmpInst(CmpInst::ICMP_UGT, X, 2352 ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal) - 1)); 2353 } 2354 2355 if (Shr->hasOneUse()) { 2356 // Canonicalize the shift into an 'and': 2357 // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt) 2358 APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal)); 2359 Constant *Mask = ConstantInt::get(ShrTy, Val); 2360 Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask"); 2361 return new ICmpInst(Pred, And, ConstantInt::get(ShrTy, C << ShAmtVal)); 2362 } 2363 2364 return nullptr; 2365 } 2366 2367 Instruction *InstCombinerImpl::foldICmpSRemConstant(ICmpInst &Cmp, 2368 BinaryOperator *SRem, 2369 const APInt &C) { 2370 // Match an 'is positive' or 'is negative' comparison of remainder by a 2371 // constant power-of-2 value: 2372 // (X % pow2C) sgt/slt 0 2373 const ICmpInst::Predicate Pred = Cmp.getPredicate(); 2374 if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT && 2375 Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE) 2376 return nullptr; 2377 2378 // TODO: The one-use check is standard because we do not typically want to 2379 // create longer instruction sequences, but this might be a special-case 2380 // because srem is not good for analysis or codegen. 2381 if (!SRem->hasOneUse()) 2382 return nullptr; 2383 2384 const APInt *DivisorC; 2385 if (!match(SRem->getOperand(1), m_Power2(DivisorC))) 2386 return nullptr; 2387 2388 // For cmp_sgt/cmp_slt only zero valued C is handled. 2389 // For cmp_eq/cmp_ne only positive valued C is handled. 2390 if (((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT) && 2391 !C.isZero()) || 2392 ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && 2393 !C.isStrictlyPositive())) 2394 return nullptr; 2395 2396 // Mask off the sign bit and the modulo bits (low-bits). 2397 Type *Ty = SRem->getType(); 2398 APInt SignMask = APInt::getSignMask(Ty->getScalarSizeInBits()); 2399 Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1)); 2400 Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC); 2401 2402 if (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) 2403 return new ICmpInst(Pred, And, ConstantInt::get(Ty, C)); 2404 2405 // For 'is positive?' check that the sign-bit is clear and at least 1 masked 2406 // bit is set. Example: 2407 // (i8 X % 32) s> 0 --> (X & 159) s> 0 2408 if (Pred == ICmpInst::ICMP_SGT) 2409 return new ICmpInst(ICmpInst::ICMP_SGT, And, ConstantInt::getNullValue(Ty)); 2410 2411 // For 'is negative?' check that the sign-bit is set and at least 1 masked 2412 // bit is set. Example: 2413 // (i16 X % 4) s< 0 --> (X & 32771) u> 32768 2414 return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, SignMask)); 2415 } 2416 2417 /// Fold icmp (udiv X, Y), C. 2418 Instruction *InstCombinerImpl::foldICmpUDivConstant(ICmpInst &Cmp, 2419 BinaryOperator *UDiv, 2420 const APInt &C) { 2421 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2422 Value *X = UDiv->getOperand(0); 2423 Value *Y = UDiv->getOperand(1); 2424 Type *Ty = UDiv->getType(); 2425 2426 const APInt *C2; 2427 if (!match(X, m_APInt(C2))) 2428 return nullptr; 2429 2430 assert(*C2 != 0 && "udiv 0, X should have been simplified already."); 2431 2432 // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1)) 2433 if (Pred == ICmpInst::ICMP_UGT) { 2434 assert(!C.isMaxValue() && 2435 "icmp ugt X, UINT_MAX should have been simplified already."); 2436 return new ICmpInst(ICmpInst::ICMP_ULE, Y, 2437 ConstantInt::get(Ty, C2->udiv(C + 1))); 2438 } 2439 2440 // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C) 2441 if (Pred == ICmpInst::ICMP_ULT) { 2442 assert(C != 0 && "icmp ult X, 0 should have been simplified already."); 2443 return new ICmpInst(ICmpInst::ICMP_UGT, Y, 2444 ConstantInt::get(Ty, C2->udiv(C))); 2445 } 2446 2447 return nullptr; 2448 } 2449 2450 /// Fold icmp ({su}div X, Y), C. 2451 Instruction *InstCombinerImpl::foldICmpDivConstant(ICmpInst &Cmp, 2452 BinaryOperator *Div, 2453 const APInt &C) { 2454 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2455 Value *X = Div->getOperand(0); 2456 Value *Y = Div->getOperand(1); 2457 Type *Ty = Div->getType(); 2458 bool DivIsSigned = Div->getOpcode() == Instruction::SDiv; 2459 2460 // If unsigned division and the compare constant is bigger than 2461 // UMAX/2 (negative), there's only one pair of values that satisfies an 2462 // equality check, so eliminate the division: 2463 // (X u/ Y) == C --> (X == C) && (Y == 1) 2464 // (X u/ Y) != C --> (X != C) || (Y != 1) 2465 // Similarly, if signed division and the compare constant is exactly SMIN: 2466 // (X s/ Y) == SMIN --> (X == SMIN) && (Y == 1) 2467 // (X s/ Y) != SMIN --> (X != SMIN) || (Y != 1) 2468 if (Cmp.isEquality() && Div->hasOneUse() && C.isSignBitSet() && 2469 (!DivIsSigned || C.isMinSignedValue())) { 2470 Value *XBig = Builder.CreateICmp(Pred, X, ConstantInt::get(Ty, C)); 2471 Value *YOne = Builder.CreateICmp(Pred, Y, ConstantInt::get(Ty, 1)); 2472 auto Logic = Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; 2473 return BinaryOperator::Create(Logic, XBig, YOne); 2474 } 2475 2476 // Fold: icmp pred ([us]div X, C2), C -> range test 2477 // Fold this div into the comparison, producing a range check. 2478 // Determine, based on the divide type, what the range is being 2479 // checked. If there is an overflow on the low or high side, remember 2480 // it, otherwise compute the range [low, hi) bounding the new value. 2481 // See: InsertRangeTest above for the kinds of replacements possible. 2482 const APInt *C2; 2483 if (!match(Y, m_APInt(C2))) 2484 return nullptr; 2485 2486 // FIXME: If the operand types don't match the type of the divide 2487 // then don't attempt this transform. The code below doesn't have the 2488 // logic to deal with a signed divide and an unsigned compare (and 2489 // vice versa). This is because (x /s C2) <s C produces different 2490 // results than (x /s C2) <u C or (x /u C2) <s C or even 2491 // (x /u C2) <u C. Simply casting the operands and result won't 2492 // work. :( The if statement below tests that condition and bails 2493 // if it finds it. 2494 if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned()) 2495 return nullptr; 2496 2497 // The ProdOV computation fails on divide by 0 and divide by -1. Cases with 2498 // INT_MIN will also fail if the divisor is 1. Although folds of all these 2499 // division-by-constant cases should be present, we can not assert that they 2500 // have happened before we reach this icmp instruction. 2501 if (C2->isZero() || C2->isOne() || (DivIsSigned && C2->isAllOnes())) 2502 return nullptr; 2503 2504 // Compute Prod = C * C2. We are essentially solving an equation of 2505 // form X / C2 = C. We solve for X by multiplying C2 and C. 2506 // By solving for X, we can turn this into a range check instead of computing 2507 // a divide. 2508 APInt Prod = C * *C2; 2509 2510 // Determine if the product overflows by seeing if the product is not equal to 2511 // the divide. Make sure we do the same kind of divide as in the LHS 2512 // instruction that we're folding. 2513 bool ProdOV = (DivIsSigned ? Prod.sdiv(*C2) : Prod.udiv(*C2)) != C; 2514 2515 // If the division is known to be exact, then there is no remainder from the 2516 // divide, so the covered range size is unit, otherwise it is the divisor. 2517 APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2; 2518 2519 // Figure out the interval that is being checked. For example, a comparison 2520 // like "X /u 5 == 0" is really checking that X is in the interval [0, 5). 2521 // Compute this interval based on the constants involved and the signedness of 2522 // the compare/divide. This computes a half-open interval, keeping track of 2523 // whether either value in the interval overflows. After analysis each 2524 // overflow variable is set to 0 if it's corresponding bound variable is valid 2525 // -1 if overflowed off the bottom end, or +1 if overflowed off the top end. 2526 int LoOverflow = 0, HiOverflow = 0; 2527 APInt LoBound, HiBound; 2528 2529 if (!DivIsSigned) { // udiv 2530 // e.g. X/5 op 3 --> [15, 20) 2531 LoBound = Prod; 2532 HiOverflow = LoOverflow = ProdOV; 2533 if (!HiOverflow) { 2534 // If this is not an exact divide, then many values in the range collapse 2535 // to the same result value. 2536 HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false); 2537 } 2538 } else if (C2->isStrictlyPositive()) { // Divisor is > 0. 2539 if (C.isZero()) { // (X / pos) op 0 2540 // Can't overflow. e.g. X/2 op 0 --> [-1, 2) 2541 LoBound = -(RangeSize - 1); 2542 HiBound = RangeSize; 2543 } else if (C.isStrictlyPositive()) { // (X / pos) op pos 2544 LoBound = Prod; // e.g. X/5 op 3 --> [15, 20) 2545 HiOverflow = LoOverflow = ProdOV; 2546 if (!HiOverflow) 2547 HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true); 2548 } else { // (X / pos) op neg 2549 // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14) 2550 HiBound = Prod + 1; 2551 LoOverflow = HiOverflow = ProdOV ? -1 : 0; 2552 if (!LoOverflow) { 2553 APInt DivNeg = -RangeSize; 2554 LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0; 2555 } 2556 } 2557 } else if (C2->isNegative()) { // Divisor is < 0. 2558 if (Div->isExact()) 2559 RangeSize.negate(); 2560 if (C.isZero()) { // (X / neg) op 0 2561 // e.g. X/-5 op 0 --> [-4, 5) 2562 LoBound = RangeSize + 1; 2563 HiBound = -RangeSize; 2564 if (HiBound == *C2) { // -INTMIN = INTMIN 2565 HiOverflow = 1; // [INTMIN+1, overflow) 2566 HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN 2567 } 2568 } else if (C.isStrictlyPositive()) { // (X / neg) op pos 2569 // e.g. X/-5 op 3 --> [-19, -14) 2570 HiBound = Prod + 1; 2571 HiOverflow = LoOverflow = ProdOV ? -1 : 0; 2572 if (!LoOverflow) 2573 LoOverflow = 2574 addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1 : 0; 2575 } else { // (X / neg) op neg 2576 LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20) 2577 LoOverflow = HiOverflow = ProdOV; 2578 if (!HiOverflow) 2579 HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true); 2580 } 2581 2582 // Dividing by a negative swaps the condition. LT <-> GT 2583 Pred = ICmpInst::getSwappedPredicate(Pred); 2584 } 2585 2586 switch (Pred) { 2587 default: 2588 llvm_unreachable("Unhandled icmp predicate!"); 2589 case ICmpInst::ICMP_EQ: 2590 if (LoOverflow && HiOverflow) 2591 return replaceInstUsesWith(Cmp, Builder.getFalse()); 2592 if (HiOverflow) 2593 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, 2594 X, ConstantInt::get(Ty, LoBound)); 2595 if (LoOverflow) 2596 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT, 2597 X, ConstantInt::get(Ty, HiBound)); 2598 return replaceInstUsesWith( 2599 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, true)); 2600 case ICmpInst::ICMP_NE: 2601 if (LoOverflow && HiOverflow) 2602 return replaceInstUsesWith(Cmp, Builder.getTrue()); 2603 if (HiOverflow) 2604 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT, 2605 X, ConstantInt::get(Ty, LoBound)); 2606 if (LoOverflow) 2607 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, 2608 X, ConstantInt::get(Ty, HiBound)); 2609 return replaceInstUsesWith( 2610 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, false)); 2611 case ICmpInst::ICMP_ULT: 2612 case ICmpInst::ICMP_SLT: 2613 if (LoOverflow == +1) // Low bound is greater than input range. 2614 return replaceInstUsesWith(Cmp, Builder.getTrue()); 2615 if (LoOverflow == -1) // Low bound is less than input range. 2616 return replaceInstUsesWith(Cmp, Builder.getFalse()); 2617 return new ICmpInst(Pred, X, ConstantInt::get(Ty, LoBound)); 2618 case ICmpInst::ICMP_UGT: 2619 case ICmpInst::ICMP_SGT: 2620 if (HiOverflow == +1) // High bound greater than input range. 2621 return replaceInstUsesWith(Cmp, Builder.getFalse()); 2622 if (HiOverflow == -1) // High bound less than input range. 2623 return replaceInstUsesWith(Cmp, Builder.getTrue()); 2624 if (Pred == ICmpInst::ICMP_UGT) 2625 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, HiBound)); 2626 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, HiBound)); 2627 } 2628 2629 return nullptr; 2630 } 2631 2632 /// Fold icmp (sub X, Y), C. 2633 Instruction *InstCombinerImpl::foldICmpSubConstant(ICmpInst &Cmp, 2634 BinaryOperator *Sub, 2635 const APInt &C) { 2636 Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1); 2637 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2638 Type *Ty = Sub->getType(); 2639 2640 // (SubC - Y) == C) --> Y == (SubC - C) 2641 // (SubC - Y) != C) --> Y != (SubC - C) 2642 Constant *SubC; 2643 if (Cmp.isEquality() && match(X, m_ImmConstant(SubC))) { 2644 return new ICmpInst(Pred, Y, 2645 ConstantExpr::getSub(SubC, ConstantInt::get(Ty, C))); 2646 } 2647 2648 // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C) 2649 const APInt *C2; 2650 APInt SubResult; 2651 ICmpInst::Predicate SwappedPred = Cmp.getSwappedPredicate(); 2652 bool HasNSW = Sub->hasNoSignedWrap(); 2653 bool HasNUW = Sub->hasNoUnsignedWrap(); 2654 if (match(X, m_APInt(C2)) && 2655 ((Cmp.isUnsigned() && HasNUW) || (Cmp.isSigned() && HasNSW)) && 2656 !subWithOverflow(SubResult, *C2, C, Cmp.isSigned())) 2657 return new ICmpInst(SwappedPred, Y, ConstantInt::get(Ty, SubResult)); 2658 2659 // X - Y == 0 --> X == Y. 2660 // X - Y != 0 --> X != Y. 2661 // TODO: We allow this with multiple uses as long as the other uses are not 2662 // in phis. The phi use check is guarding against a codegen regression 2663 // for a loop test. If the backend could undo this (and possibly 2664 // subsequent transforms), we would not need this hack. 2665 if (Cmp.isEquality() && C.isZero() && 2666 none_of((Sub->users()), [](const User *U) { return isa<PHINode>(U); })) 2667 return new ICmpInst(Pred, X, Y); 2668 2669 // The following transforms are only worth it if the only user of the subtract 2670 // is the icmp. 2671 // TODO: This is an artificial restriction for all of the transforms below 2672 // that only need a single replacement icmp. Can these use the phi test 2673 // like the transform above here? 2674 if (!Sub->hasOneUse()) 2675 return nullptr; 2676 2677 if (Sub->hasNoSignedWrap()) { 2678 // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y) 2679 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes()) 2680 return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); 2681 2682 // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y) 2683 if (Pred == ICmpInst::ICMP_SGT && C.isZero()) 2684 return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); 2685 2686 // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y) 2687 if (Pred == ICmpInst::ICMP_SLT && C.isZero()) 2688 return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); 2689 2690 // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y) 2691 if (Pred == ICmpInst::ICMP_SLT && C.isOne()) 2692 return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); 2693 } 2694 2695 if (!match(X, m_APInt(C2))) 2696 return nullptr; 2697 2698 // C2 - Y <u C -> (Y | (C - 1)) == C2 2699 // iff (C2 & (C - 1)) == C - 1 and C is a power of 2 2700 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && 2701 (*C2 & (C - 1)) == (C - 1)) 2702 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, C - 1), X); 2703 2704 // C2 - Y >u C -> (Y | C) != C2 2705 // iff C2 & C == C and C + 1 is a power of 2 2706 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C) 2707 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, C), X); 2708 2709 // We have handled special cases that reduce. 2710 // Canonicalize any remaining sub to add as: 2711 // (C2 - Y) > C --> (Y + ~C2) < ~C 2712 Value *Add = Builder.CreateAdd(Y, ConstantInt::get(Ty, ~(*C2)), "notsub", 2713 HasNUW, HasNSW); 2714 return new ICmpInst(SwappedPred, Add, ConstantInt::get(Ty, ~C)); 2715 } 2716 2717 /// Fold icmp (add X, Y), C. 2718 Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp, 2719 BinaryOperator *Add, 2720 const APInt &C) { 2721 Value *Y = Add->getOperand(1); 2722 const APInt *C2; 2723 if (Cmp.isEquality() || !match(Y, m_APInt(C2))) 2724 return nullptr; 2725 2726 // Fold icmp pred (add X, C2), C. 2727 Value *X = Add->getOperand(0); 2728 Type *Ty = Add->getType(); 2729 const CmpInst::Predicate Pred = Cmp.getPredicate(); 2730 2731 // If the add does not wrap, we can always adjust the compare by subtracting 2732 // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE 2733 // are canonicalized to SGT/SLT/UGT/ULT. 2734 if ((Add->hasNoSignedWrap() && 2735 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) || 2736 (Add->hasNoUnsignedWrap() && 2737 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) { 2738 bool Overflow; 2739 APInt NewC = 2740 Cmp.isSigned() ? C.ssub_ov(*C2, Overflow) : C.usub_ov(*C2, Overflow); 2741 // If there is overflow, the result must be true or false. 2742 // TODO: Can we assert there is no overflow because InstSimplify always 2743 // handles those cases? 2744 if (!Overflow) 2745 // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2) 2746 return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC)); 2747 } 2748 2749 auto CR = ConstantRange::makeExactICmpRegion(Pred, C).subtract(*C2); 2750 const APInt &Upper = CR.getUpper(); 2751 const APInt &Lower = CR.getLower(); 2752 if (Cmp.isSigned()) { 2753 if (Lower.isSignMask()) 2754 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper)); 2755 if (Upper.isSignMask()) 2756 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower)); 2757 } else { 2758 if (Lower.isMinValue()) 2759 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper)); 2760 if (Upper.isMinValue()) 2761 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower)); 2762 } 2763 2764 // This set of folds is intentionally placed after folds that use no-wrapping 2765 // flags because those folds are likely better for later analysis/codegen. 2766 const APInt SMax = APInt::getSignedMaxValue(Ty->getScalarSizeInBits()); 2767 const APInt SMin = APInt::getSignedMinValue(Ty->getScalarSizeInBits()); 2768 2769 // Fold compare with offset to opposite sign compare if it eliminates offset: 2770 // (X + C2) >u C --> X <s -C2 (if C == C2 + SMAX) 2771 if (Pred == CmpInst::ICMP_UGT && C == *C2 + SMax) 2772 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, -(*C2))); 2773 2774 // (X + C2) <u C --> X >s ~C2 (if C == C2 + SMIN) 2775 if (Pred == CmpInst::ICMP_ULT && C == *C2 + SMin) 2776 return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantInt::get(Ty, ~(*C2))); 2777 2778 // (X + C2) >s C --> X <u (SMAX - C) (if C == C2 - 1) 2779 if (Pred == CmpInst::ICMP_SGT && C == *C2 - 1) 2780 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, SMax - C)); 2781 2782 // (X + C2) <s C --> X >u (C ^ SMAX) (if C == C2) 2783 if (Pred == CmpInst::ICMP_SLT && C == *C2) 2784 return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, C ^ SMax)); 2785 2786 if (!Add->hasOneUse()) 2787 return nullptr; 2788 2789 // X+C <u C2 -> (X & -C2) == C 2790 // iff C & (C2-1) == 0 2791 // C2 is a power of 2 2792 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0) 2793 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateAnd(X, -C), 2794 ConstantExpr::getNeg(cast<Constant>(Y))); 2795 2796 // X+C >u C2 -> (X & ~C2) != C 2797 // iff C & C2 == 0 2798 // C2+1 is a power of 2 2799 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0) 2800 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(X, ~C), 2801 ConstantExpr::getNeg(cast<Constant>(Y))); 2802 2803 // The range test idiom can use either ult or ugt. Arbitrarily canonicalize 2804 // to the ult form. 2805 // X+C2 >u C -> X+(C2-C-1) <u ~C 2806 if (Pred == ICmpInst::ICMP_UGT) 2807 return new ICmpInst(ICmpInst::ICMP_ULT, 2808 Builder.CreateAdd(X, ConstantInt::get(Ty, *C2 - C - 1)), 2809 ConstantInt::get(Ty, ~C)); 2810 2811 return nullptr; 2812 } 2813 2814 bool InstCombinerImpl::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, 2815 Value *&RHS, ConstantInt *&Less, 2816 ConstantInt *&Equal, 2817 ConstantInt *&Greater) { 2818 // TODO: Generalize this to work with other comparison idioms or ensure 2819 // they get canonicalized into this form. 2820 2821 // select i1 (a == b), 2822 // i32 Equal, 2823 // i32 (select i1 (a < b), i32 Less, i32 Greater) 2824 // where Equal, Less and Greater are placeholders for any three constants. 2825 ICmpInst::Predicate PredA; 2826 if (!match(SI->getCondition(), m_ICmp(PredA, m_Value(LHS), m_Value(RHS))) || 2827 !ICmpInst::isEquality(PredA)) 2828 return false; 2829 Value *EqualVal = SI->getTrueValue(); 2830 Value *UnequalVal = SI->getFalseValue(); 2831 // We still can get non-canonical predicate here, so canonicalize. 2832 if (PredA == ICmpInst::ICMP_NE) 2833 std::swap(EqualVal, UnequalVal); 2834 if (!match(EqualVal, m_ConstantInt(Equal))) 2835 return false; 2836 ICmpInst::Predicate PredB; 2837 Value *LHS2, *RHS2; 2838 if (!match(UnequalVal, m_Select(m_ICmp(PredB, m_Value(LHS2), m_Value(RHS2)), 2839 m_ConstantInt(Less), m_ConstantInt(Greater)))) 2840 return false; 2841 // We can get predicate mismatch here, so canonicalize if possible: 2842 // First, ensure that 'LHS' match. 2843 if (LHS2 != LHS) { 2844 // x sgt y <--> y slt x 2845 std::swap(LHS2, RHS2); 2846 PredB = ICmpInst::getSwappedPredicate(PredB); 2847 } 2848 if (LHS2 != LHS) 2849 return false; 2850 // We also need to canonicalize 'RHS'. 2851 if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) { 2852 // x sgt C-1 <--> x sge C <--> not(x slt C) 2853 auto FlippedStrictness = 2854 InstCombiner::getFlippedStrictnessPredicateAndConstant( 2855 PredB, cast<Constant>(RHS2)); 2856 if (!FlippedStrictness) 2857 return false; 2858 assert(FlippedStrictness->first == ICmpInst::ICMP_SGE && 2859 "basic correctness failure"); 2860 RHS2 = FlippedStrictness->second; 2861 // And kind-of perform the result swap. 2862 std::swap(Less, Greater); 2863 PredB = ICmpInst::ICMP_SLT; 2864 } 2865 return PredB == ICmpInst::ICMP_SLT && RHS == RHS2; 2866 } 2867 2868 Instruction *InstCombinerImpl::foldICmpSelectConstant(ICmpInst &Cmp, 2869 SelectInst *Select, 2870 ConstantInt *C) { 2871 2872 assert(C && "Cmp RHS should be a constant int!"); 2873 // If we're testing a constant value against the result of a three way 2874 // comparison, the result can be expressed directly in terms of the 2875 // original values being compared. Note: We could possibly be more 2876 // aggressive here and remove the hasOneUse test. The original select is 2877 // really likely to simplify or sink when we remove a test of the result. 2878 Value *OrigLHS, *OrigRHS; 2879 ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan; 2880 if (Cmp.hasOneUse() && 2881 matchThreeWayIntCompare(Select, OrigLHS, OrigRHS, C1LessThan, C2Equal, 2882 C3GreaterThan)) { 2883 assert(C1LessThan && C2Equal && C3GreaterThan); 2884 2885 bool TrueWhenLessThan = 2886 ConstantExpr::getCompare(Cmp.getPredicate(), C1LessThan, C) 2887 ->isAllOnesValue(); 2888 bool TrueWhenEqual = 2889 ConstantExpr::getCompare(Cmp.getPredicate(), C2Equal, C) 2890 ->isAllOnesValue(); 2891 bool TrueWhenGreaterThan = 2892 ConstantExpr::getCompare(Cmp.getPredicate(), C3GreaterThan, C) 2893 ->isAllOnesValue(); 2894 2895 // This generates the new instruction that will replace the original Cmp 2896 // Instruction. Instead of enumerating the various combinations when 2897 // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus 2898 // false, we rely on chaining of ORs and future passes of InstCombine to 2899 // simplify the OR further (i.e. a s< b || a == b becomes a s<= b). 2900 2901 // When none of the three constants satisfy the predicate for the RHS (C), 2902 // the entire original Cmp can be simplified to a false. 2903 Value *Cond = Builder.getFalse(); 2904 if (TrueWhenLessThan) 2905 Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT, 2906 OrigLHS, OrigRHS)); 2907 if (TrueWhenEqual) 2908 Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ, 2909 OrigLHS, OrigRHS)); 2910 if (TrueWhenGreaterThan) 2911 Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT, 2912 OrigLHS, OrigRHS)); 2913 2914 return replaceInstUsesWith(Cmp, Cond); 2915 } 2916 return nullptr; 2917 } 2918 2919 Instruction *InstCombinerImpl::foldICmpBitCast(ICmpInst &Cmp) { 2920 auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0)); 2921 if (!Bitcast) 2922 return nullptr; 2923 2924 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2925 Value *Op1 = Cmp.getOperand(1); 2926 Value *BCSrcOp = Bitcast->getOperand(0); 2927 Type *SrcType = Bitcast->getSrcTy(); 2928 Type *DstType = Bitcast->getType(); 2929 2930 // Make sure the bitcast doesn't change between scalar and vector and 2931 // doesn't change the number of vector elements. 2932 if (SrcType->isVectorTy() == DstType->isVectorTy() && 2933 SrcType->getScalarSizeInBits() == DstType->getScalarSizeInBits()) { 2934 // Zero-equality and sign-bit checks are preserved through sitofp + bitcast. 2935 Value *X; 2936 if (match(BCSrcOp, m_SIToFP(m_Value(X)))) { 2937 // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0 2938 // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0 2939 // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0 2940 // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0 2941 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT || 2942 Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) && 2943 match(Op1, m_Zero())) 2944 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType())); 2945 2946 // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1 2947 if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One())) 2948 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1)); 2949 2950 // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1 2951 if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes())) 2952 return new ICmpInst(Pred, X, 2953 ConstantInt::getAllOnesValue(X->getType())); 2954 } 2955 2956 // Zero-equality checks are preserved through unsigned floating-point casts: 2957 // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0 2958 // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0 2959 if (match(BCSrcOp, m_UIToFP(m_Value(X)))) 2960 if (Cmp.isEquality() && match(Op1, m_Zero())) 2961 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType())); 2962 2963 // If this is a sign-bit test of a bitcast of a casted FP value, eliminate 2964 // the FP extend/truncate because that cast does not change the sign-bit. 2965 // This is true for all standard IEEE-754 types and the X86 80-bit type. 2966 // The sign-bit is always the most significant bit in those types. 2967 const APInt *C; 2968 bool TrueIfSigned; 2969 if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse() && 2970 InstCombiner::isSignBitCheck(Pred, *C, TrueIfSigned)) { 2971 if (match(BCSrcOp, m_FPExt(m_Value(X))) || 2972 match(BCSrcOp, m_FPTrunc(m_Value(X)))) { 2973 // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0 2974 // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1 2975 Type *XType = X->getType(); 2976 2977 // We can't currently handle Power style floating point operations here. 2978 if (!(XType->isPPC_FP128Ty() || SrcType->isPPC_FP128Ty())) { 2979 Type *NewType = Builder.getIntNTy(XType->getScalarSizeInBits()); 2980 if (auto *XVTy = dyn_cast<VectorType>(XType)) 2981 NewType = VectorType::get(NewType, XVTy->getElementCount()); 2982 Value *NewBitcast = Builder.CreateBitCast(X, NewType); 2983 if (TrueIfSigned) 2984 return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast, 2985 ConstantInt::getNullValue(NewType)); 2986 else 2987 return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast, 2988 ConstantInt::getAllOnesValue(NewType)); 2989 } 2990 } 2991 } 2992 } 2993 2994 // Test to see if the operands of the icmp are casted versions of other 2995 // values. If the ptr->ptr cast can be stripped off both arguments, do so. 2996 if (DstType->isPointerTy() && (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) { 2997 // If operand #1 is a bitcast instruction, it must also be a ptr->ptr cast 2998 // so eliminate it as well. 2999 if (auto *BC2 = dyn_cast<BitCastInst>(Op1)) 3000 Op1 = BC2->getOperand(0); 3001 3002 Op1 = Builder.CreateBitCast(Op1, SrcType); 3003 return new ICmpInst(Pred, BCSrcOp, Op1); 3004 } 3005 3006 const APInt *C; 3007 if (!match(Cmp.getOperand(1), m_APInt(C)) || !DstType->isIntegerTy() || 3008 !SrcType->isIntOrIntVectorTy()) 3009 return nullptr; 3010 3011 // If this is checking if all elements of a vector compare are set or not, 3012 // invert the casted vector equality compare and test if all compare 3013 // elements are clear or not. Compare against zero is generally easier for 3014 // analysis and codegen. 3015 // icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0 3016 // Example: are all elements equal? --> are zero elements not equal? 3017 // TODO: Try harder to reduce compare of 2 freely invertible operands? 3018 if (Cmp.isEquality() && C->isAllOnes() && Bitcast->hasOneUse() && 3019 isFreeToInvert(BCSrcOp, BCSrcOp->hasOneUse())) { 3020 Value *Cast = Builder.CreateBitCast(Builder.CreateNot(BCSrcOp), DstType); 3021 return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(DstType)); 3022 } 3023 3024 // If this is checking if all elements of an extended vector are clear or not, 3025 // compare in a narrow type to eliminate the extend: 3026 // icmp eq/ne (bitcast (ext X) to iN), 0 --> icmp eq/ne (bitcast X to iM), 0 3027 Value *X; 3028 if (Cmp.isEquality() && C->isZero() && Bitcast->hasOneUse() && 3029 match(BCSrcOp, m_ZExtOrSExt(m_Value(X)))) { 3030 if (auto *VecTy = dyn_cast<FixedVectorType>(X->getType())) { 3031 Type *NewType = Builder.getIntNTy(VecTy->getPrimitiveSizeInBits()); 3032 Value *NewCast = Builder.CreateBitCast(X, NewType); 3033 return new ICmpInst(Pred, NewCast, ConstantInt::getNullValue(NewType)); 3034 } 3035 } 3036 3037 // Folding: icmp <pred> iN X, C 3038 // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN 3039 // and C is a splat of a K-bit pattern 3040 // and SC is a constant vector = <C', C', C', ..., C'> 3041 // Into: 3042 // %E = extractelement <M x iK> %vec, i32 C' 3043 // icmp <pred> iK %E, trunc(C) 3044 Value *Vec; 3045 ArrayRef<int> Mask; 3046 if (match(BCSrcOp, m_Shuffle(m_Value(Vec), m_Undef(), m_Mask(Mask)))) { 3047 // Check whether every element of Mask is the same constant 3048 if (is_splat(Mask)) { 3049 auto *VecTy = cast<VectorType>(SrcType); 3050 auto *EltTy = cast<IntegerType>(VecTy->getElementType()); 3051 if (C->isSplat(EltTy->getBitWidth())) { 3052 // Fold the icmp based on the value of C 3053 // If C is M copies of an iK sized bit pattern, 3054 // then: 3055 // => %E = extractelement <N x iK> %vec, i32 Elem 3056 // icmp <pred> iK %SplatVal, <pattern> 3057 Value *Elem = Builder.getInt32(Mask[0]); 3058 Value *Extract = Builder.CreateExtractElement(Vec, Elem); 3059 Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth())); 3060 return new ICmpInst(Pred, Extract, NewC); 3061 } 3062 } 3063 } 3064 return nullptr; 3065 } 3066 3067 /// Try to fold integer comparisons with a constant operand: icmp Pred X, C 3068 /// where X is some kind of instruction. 3069 Instruction *InstCombinerImpl::foldICmpInstWithConstant(ICmpInst &Cmp) { 3070 const APInt *C; 3071 3072 if (match(Cmp.getOperand(1), m_APInt(C))) { 3073 if (auto *BO = dyn_cast<BinaryOperator>(Cmp.getOperand(0))) 3074 if (Instruction *I = foldICmpBinOpWithConstant(Cmp, BO, *C)) 3075 return I; 3076 3077 if (auto *SI = dyn_cast<SelectInst>(Cmp.getOperand(0))) 3078 // For now, we only support constant integers while folding the 3079 // ICMP(SELECT)) pattern. We can extend this to support vector of integers 3080 // similar to the cases handled by binary ops above. 3081 if (auto *ConstRHS = dyn_cast<ConstantInt>(Cmp.getOperand(1))) 3082 if (Instruction *I = foldICmpSelectConstant(Cmp, SI, ConstRHS)) 3083 return I; 3084 3085 if (auto *TI = dyn_cast<TruncInst>(Cmp.getOperand(0))) 3086 if (Instruction *I = foldICmpTruncConstant(Cmp, TI, *C)) 3087 return I; 3088 3089 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) 3090 if (Instruction *I = foldICmpIntrinsicWithConstant(Cmp, II, *C)) 3091 return I; 3092 } 3093 3094 if (match(Cmp.getOperand(1), m_APIntAllowUndef(C))) 3095 return foldICmpInstWithConstantAllowUndef(Cmp, *C); 3096 3097 return nullptr; 3098 } 3099 3100 /// Fold an icmp equality instruction with binary operator LHS and constant RHS: 3101 /// icmp eq/ne BO, C. 3102 Instruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant( 3103 ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) { 3104 // TODO: Some of these folds could work with arbitrary constants, but this 3105 // function is limited to scalar and vector splat constants. 3106 if (!Cmp.isEquality()) 3107 return nullptr; 3108 3109 ICmpInst::Predicate Pred = Cmp.getPredicate(); 3110 bool isICMP_NE = Pred == ICmpInst::ICMP_NE; 3111 Constant *RHS = cast<Constant>(Cmp.getOperand(1)); 3112 Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1); 3113 3114 switch (BO->getOpcode()) { 3115 case Instruction::SRem: 3116 // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one. 3117 if (C.isZero() && BO->hasOneUse()) { 3118 const APInt *BOC; 3119 if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) { 3120 Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName()); 3121 return new ICmpInst(Pred, NewRem, 3122 Constant::getNullValue(BO->getType())); 3123 } 3124 } 3125 break; 3126 case Instruction::Add: { 3127 // Replace ((add A, B) != C) with (A != C-B) if B & C are constants. 3128 if (Constant *BOC = dyn_cast<Constant>(BOp1)) { 3129 if (BO->hasOneUse()) 3130 return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, BOC)); 3131 } else if (C.isZero()) { 3132 // Replace ((add A, B) != 0) with (A != -B) if A or B is 3133 // efficiently invertible, or if the add has just this one use. 3134 if (Value *NegVal = dyn_castNegVal(BOp1)) 3135 return new ICmpInst(Pred, BOp0, NegVal); 3136 if (Value *NegVal = dyn_castNegVal(BOp0)) 3137 return new ICmpInst(Pred, NegVal, BOp1); 3138 if (BO->hasOneUse()) { 3139 Value *Neg = Builder.CreateNeg(BOp1); 3140 Neg->takeName(BO); 3141 return new ICmpInst(Pred, BOp0, Neg); 3142 } 3143 } 3144 break; 3145 } 3146 case Instruction::Xor: 3147 if (BO->hasOneUse()) { 3148 if (Constant *BOC = dyn_cast<Constant>(BOp1)) { 3149 // For the xor case, we can xor two constants together, eliminating 3150 // the explicit xor. 3151 return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC)); 3152 } else if (C.isZero()) { 3153 // Replace ((xor A, B) != 0) with (A != B) 3154 return new ICmpInst(Pred, BOp0, BOp1); 3155 } 3156 } 3157 break; 3158 case Instruction::Or: { 3159 const APInt *BOC; 3160 if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) { 3161 // Comparing if all bits outside of a constant mask are set? 3162 // Replace (X | C) == -1 with (X & ~C) == ~C. 3163 // This removes the -1 constant. 3164 Constant *NotBOC = ConstantExpr::getNot(cast<Constant>(BOp1)); 3165 Value *And = Builder.CreateAnd(BOp0, NotBOC); 3166 return new ICmpInst(Pred, And, NotBOC); 3167 } 3168 break; 3169 } 3170 case Instruction::And: { 3171 const APInt *BOC; 3172 if (match(BOp1, m_APInt(BOC))) { 3173 // If we have ((X & C) == C), turn it into ((X & C) != 0). 3174 if (C == *BOC && C.isPowerOf2()) 3175 return new ICmpInst(isICMP_NE ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE, 3176 BO, Constant::getNullValue(RHS->getType())); 3177 } 3178 break; 3179 } 3180 case Instruction::UDiv: 3181 if (C.isZero()) { 3182 // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A) 3183 auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; 3184 return new ICmpInst(NewPred, BOp1, BOp0); 3185 } 3186 break; 3187 default: 3188 break; 3189 } 3190 return nullptr; 3191 } 3192 3193 /// Fold an equality icmp with LLVM intrinsic and constant operand. 3194 Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant( 3195 ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) { 3196 Type *Ty = II->getType(); 3197 unsigned BitWidth = C.getBitWidth(); 3198 const ICmpInst::Predicate Pred = Cmp.getPredicate(); 3199 3200 switch (II->getIntrinsicID()) { 3201 case Intrinsic::abs: 3202 // abs(A) == 0 -> A == 0 3203 // abs(A) == INT_MIN -> A == INT_MIN 3204 if (C.isZero() || C.isMinSignedValue()) 3205 return new ICmpInst(Pred, II->getArgOperand(0), ConstantInt::get(Ty, C)); 3206 break; 3207 3208 case Intrinsic::bswap: 3209 // bswap(A) == C -> A == bswap(C) 3210 return new ICmpInst(Pred, II->getArgOperand(0), 3211 ConstantInt::get(Ty, C.byteSwap())); 3212 3213 case Intrinsic::ctlz: 3214 case Intrinsic::cttz: { 3215 // ctz(A) == bitwidth(A) -> A == 0 and likewise for != 3216 if (C == BitWidth) 3217 return new ICmpInst(Pred, II->getArgOperand(0), 3218 ConstantInt::getNullValue(Ty)); 3219 3220 // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set 3221 // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits. 3222 // Limit to one use to ensure we don't increase instruction count. 3223 unsigned Num = C.getLimitedValue(BitWidth); 3224 if (Num != BitWidth && II->hasOneUse()) { 3225 bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz; 3226 APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1) 3227 : APInt::getHighBitsSet(BitWidth, Num + 1); 3228 APInt Mask2 = IsTrailing 3229 ? APInt::getOneBitSet(BitWidth, Num) 3230 : APInt::getOneBitSet(BitWidth, BitWidth - Num - 1); 3231 return new ICmpInst(Pred, Builder.CreateAnd(II->getArgOperand(0), Mask1), 3232 ConstantInt::get(Ty, Mask2)); 3233 } 3234 break; 3235 } 3236 3237 case Intrinsic::ctpop: { 3238 // popcount(A) == 0 -> A == 0 and likewise for != 3239 // popcount(A) == bitwidth(A) -> A == -1 and likewise for != 3240 bool IsZero = C.isZero(); 3241 if (IsZero || C == BitWidth) 3242 return new ICmpInst(Pred, II->getArgOperand(0), 3243 IsZero ? Constant::getNullValue(Ty) 3244 : Constant::getAllOnesValue(Ty)); 3245 3246 break; 3247 } 3248 3249 case Intrinsic::fshl: 3250 case Intrinsic::fshr: 3251 if (II->getArgOperand(0) == II->getArgOperand(1)) { 3252 const APInt *RotAmtC; 3253 // ror(X, RotAmtC) == C --> X == rol(C, RotAmtC) 3254 // rol(X, RotAmtC) == C --> X == ror(C, RotAmtC) 3255 if (match(II->getArgOperand(2), m_APInt(RotAmtC))) 3256 return new ICmpInst(Pred, II->getArgOperand(0), 3257 II->getIntrinsicID() == Intrinsic::fshl 3258 ? ConstantInt::get(Ty, C.rotr(*RotAmtC)) 3259 : ConstantInt::get(Ty, C.rotl(*RotAmtC))); 3260 } 3261 break; 3262 3263 case Intrinsic::uadd_sat: { 3264 // uadd.sat(a, b) == 0 -> (a | b) == 0 3265 if (C.isZero()) { 3266 Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1)); 3267 return new ICmpInst(Pred, Or, Constant::getNullValue(Ty)); 3268 } 3269 break; 3270 } 3271 3272 case Intrinsic::usub_sat: { 3273 // usub.sat(a, b) == 0 -> a <= b 3274 if (C.isZero()) { 3275 ICmpInst::Predicate NewPred = 3276 Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; 3277 return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1)); 3278 } 3279 break; 3280 } 3281 default: 3282 break; 3283 } 3284 3285 return nullptr; 3286 } 3287 3288 /// Fold an icmp with LLVM intrinsics 3289 static Instruction *foldICmpIntrinsicWithIntrinsic(ICmpInst &Cmp) { 3290 assert(Cmp.isEquality()); 3291 3292 ICmpInst::Predicate Pred = Cmp.getPredicate(); 3293 Value *Op0 = Cmp.getOperand(0); 3294 Value *Op1 = Cmp.getOperand(1); 3295 const auto *IIOp0 = dyn_cast<IntrinsicInst>(Op0); 3296 const auto *IIOp1 = dyn_cast<IntrinsicInst>(Op1); 3297 if (!IIOp0 || !IIOp1 || IIOp0->getIntrinsicID() != IIOp1->getIntrinsicID()) 3298 return nullptr; 3299 3300 switch (IIOp0->getIntrinsicID()) { 3301 case Intrinsic::bswap: 3302 case Intrinsic::bitreverse: 3303 // If both operands are byte-swapped or bit-reversed, just compare the 3304 // original values. 3305 return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0)); 3306 case Intrinsic::fshl: 3307 case Intrinsic::fshr: 3308 // If both operands are rotated by same amount, just compare the 3309 // original values. 3310 if (IIOp0->getOperand(0) != IIOp0->getOperand(1)) 3311 break; 3312 if (IIOp1->getOperand(0) != IIOp1->getOperand(1)) 3313 break; 3314 if (IIOp0->getOperand(2) != IIOp1->getOperand(2)) 3315 break; 3316 return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0)); 3317 default: 3318 break; 3319 } 3320 3321 return nullptr; 3322 } 3323 3324 /// Try to fold integer comparisons with a constant operand: icmp Pred X, C 3325 /// where X is some kind of instruction and C is AllowUndef. 3326 /// TODO: Move more folds which allow undef to this function. 3327 Instruction * 3328 InstCombinerImpl::foldICmpInstWithConstantAllowUndef(ICmpInst &Cmp, 3329 const APInt &C) { 3330 const ICmpInst::Predicate Pred = Cmp.getPredicate(); 3331 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) { 3332 switch (II->getIntrinsicID()) { 3333 default: 3334 break; 3335 case Intrinsic::fshl: 3336 case Intrinsic::fshr: 3337 if (Cmp.isEquality() && II->getArgOperand(0) == II->getArgOperand(1)) { 3338 // (rot X, ?) == 0/-1 --> X == 0/-1 3339 if (C.isZero() || C.isAllOnes()) 3340 return new ICmpInst(Pred, II->getArgOperand(0), Cmp.getOperand(1)); 3341 } 3342 break; 3343 } 3344 } 3345 3346 return nullptr; 3347 } 3348 3349 /// Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C. 3350 Instruction *InstCombinerImpl::foldICmpBinOpWithConstant(ICmpInst &Cmp, 3351 BinaryOperator *BO, 3352 const APInt &C) { 3353 switch (BO->getOpcode()) { 3354 case Instruction::Xor: 3355 if (Instruction *I = foldICmpXorConstant(Cmp, BO, C)) 3356 return I; 3357 break; 3358 case Instruction::And: 3359 if (Instruction *I = foldICmpAndConstant(Cmp, BO, C)) 3360 return I; 3361 break; 3362 case Instruction::Or: 3363 if (Instruction *I = foldICmpOrConstant(Cmp, BO, C)) 3364 return I; 3365 break; 3366 case Instruction::Mul: 3367 if (Instruction *I = foldICmpMulConstant(Cmp, BO, C)) 3368 return I; 3369 break; 3370 case Instruction::Shl: 3371 if (Instruction *I = foldICmpShlConstant(Cmp, BO, C)) 3372 return I; 3373 break; 3374 case Instruction::LShr: 3375 case Instruction::AShr: 3376 if (Instruction *I = foldICmpShrConstant(Cmp, BO, C)) 3377 return I; 3378 break; 3379 case Instruction::SRem: 3380 if (Instruction *I = foldICmpSRemConstant(Cmp, BO, C)) 3381 return I; 3382 break; 3383 case Instruction::UDiv: 3384 if (Instruction *I = foldICmpUDivConstant(Cmp, BO, C)) 3385 return I; 3386 LLVM_FALLTHROUGH; 3387 case Instruction::SDiv: 3388 if (Instruction *I = foldICmpDivConstant(Cmp, BO, C)) 3389 return I; 3390 break; 3391 case Instruction::Sub: 3392 if (Instruction *I = foldICmpSubConstant(Cmp, BO, C)) 3393 return I; 3394 break; 3395 case Instruction::Add: 3396 if (Instruction *I = foldICmpAddConstant(Cmp, BO, C)) 3397 return I; 3398 break; 3399 default: 3400 break; 3401 } 3402 3403 // TODO: These folds could be refactored to be part of the above calls. 3404 return foldICmpBinOpEqualityWithConstant(Cmp, BO, C); 3405 } 3406 3407 /// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C. 3408 Instruction *InstCombinerImpl::foldICmpIntrinsicWithConstant(ICmpInst &Cmp, 3409 IntrinsicInst *II, 3410 const APInt &C) { 3411 if (Cmp.isEquality()) 3412 return foldICmpEqIntrinsicWithConstant(Cmp, II, C); 3413 3414 Type *Ty = II->getType(); 3415 unsigned BitWidth = C.getBitWidth(); 3416 ICmpInst::Predicate Pred = Cmp.getPredicate(); 3417 switch (II->getIntrinsicID()) { 3418 case Intrinsic::ctpop: { 3419 // (ctpop X > BitWidth - 1) --> X == -1 3420 Value *X = II->getArgOperand(0); 3421 if (C == BitWidth - 1 && Pred == ICmpInst::ICMP_UGT) 3422 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, X, 3423 ConstantInt::getAllOnesValue(Ty)); 3424 // (ctpop X < BitWidth) --> X != -1 3425 if (C == BitWidth && Pred == ICmpInst::ICMP_ULT) 3426 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, X, 3427 ConstantInt::getAllOnesValue(Ty)); 3428 break; 3429 } 3430 case Intrinsic::ctlz: { 3431 // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000 3432 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) { 3433 unsigned Num = C.getLimitedValue(); 3434 APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1); 3435 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, 3436 II->getArgOperand(0), ConstantInt::get(Ty, Limit)); 3437 } 3438 3439 // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111 3440 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) { 3441 unsigned Num = C.getLimitedValue(); 3442 APInt Limit = APInt::getLowBitsSet(BitWidth, BitWidth - Num); 3443 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, 3444 II->getArgOperand(0), ConstantInt::get(Ty, Limit)); 3445 } 3446 break; 3447 } 3448 case Intrinsic::cttz: { 3449 // Limit to one use to ensure we don't increase instruction count. 3450 if (!II->hasOneUse()) 3451 return nullptr; 3452 3453 // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0 3454 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) { 3455 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1); 3456 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, 3457 Builder.CreateAnd(II->getArgOperand(0), Mask), 3458 ConstantInt::getNullValue(Ty)); 3459 } 3460 3461 // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0 3462 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) { 3463 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue()); 3464 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, 3465 Builder.CreateAnd(II->getArgOperand(0), Mask), 3466 ConstantInt::getNullValue(Ty)); 3467 } 3468 break; 3469 } 3470 default: 3471 break; 3472 } 3473 3474 return nullptr; 3475 } 3476 3477 /// Handle icmp with constant (but not simple integer constant) RHS. 3478 Instruction *InstCombinerImpl::foldICmpInstWithConstantNotInt(ICmpInst &I) { 3479 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 3480 Constant *RHSC = dyn_cast<Constant>(Op1); 3481 Instruction *LHSI = dyn_cast<Instruction>(Op0); 3482 if (!RHSC || !LHSI) 3483 return nullptr; 3484 3485 switch (LHSI->getOpcode()) { 3486 case Instruction::GetElementPtr: 3487 // icmp pred GEP (P, int 0, int 0, int 0), null -> icmp pred P, null 3488 if (RHSC->isNullValue() && 3489 cast<GetElementPtrInst>(LHSI)->hasAllZeroIndices()) 3490 return new ICmpInst( 3491 I.getPredicate(), LHSI->getOperand(0), 3492 Constant::getNullValue(LHSI->getOperand(0)->getType())); 3493 break; 3494 case Instruction::PHI: 3495 // Only fold icmp into the PHI if the phi and icmp are in the same 3496 // block. If in the same block, we're encouraging jump threading. If 3497 // not, we are just pessimizing the code by making an i1 phi. 3498 if (LHSI->getParent() == I.getParent()) 3499 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) 3500 return NV; 3501 break; 3502 case Instruction::IntToPtr: 3503 // icmp pred inttoptr(X), null -> icmp pred X, 0 3504 if (RHSC->isNullValue() && 3505 DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(0)->getType()) 3506 return new ICmpInst( 3507 I.getPredicate(), LHSI->getOperand(0), 3508 Constant::getNullValue(LHSI->getOperand(0)->getType())); 3509 break; 3510 3511 case Instruction::Load: 3512 // Try to optimize things like "A[i] > 4" to index computations. 3513 if (GetElementPtrInst *GEP = 3514 dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) 3515 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) 3516 if (Instruction *Res = 3517 foldCmpLoadFromIndexedGlobal(cast<LoadInst>(LHSI), GEP, GV, I)) 3518 return Res; 3519 break; 3520 } 3521 3522 return nullptr; 3523 } 3524 3525 Instruction *InstCombinerImpl::foldSelectICmp(ICmpInst::Predicate Pred, 3526 SelectInst *SI, Value *RHS, 3527 const ICmpInst &I) { 3528 // Try to fold the comparison into the select arms, which will cause the 3529 // select to be converted into a logical and/or. 3530 auto SimplifyOp = [&](Value *Op, bool SelectCondIsTrue) -> Value * { 3531 if (Value *Res = simplifyICmpInst(Pred, Op, RHS, SQ)) 3532 return Res; 3533 if (Optional<bool> Impl = isImpliedCondition(SI->getCondition(), Pred, Op, 3534 RHS, DL, SelectCondIsTrue)) 3535 return ConstantInt::get(I.getType(), *Impl); 3536 return nullptr; 3537 }; 3538 3539 ConstantInt *CI = nullptr; 3540 Value *Op1 = SimplifyOp(SI->getOperand(1), true); 3541 if (Op1) 3542 CI = dyn_cast<ConstantInt>(Op1); 3543 3544 Value *Op2 = SimplifyOp(SI->getOperand(2), false); 3545 if (Op2) 3546 CI = dyn_cast<ConstantInt>(Op2); 3547 3548 // We only want to perform this transformation if it will not lead to 3549 // additional code. This is true if either both sides of the select 3550 // fold to a constant (in which case the icmp is replaced with a select 3551 // which will usually simplify) or this is the only user of the 3552 // select (in which case we are trading a select+icmp for a simpler 3553 // select+icmp) or all uses of the select can be replaced based on 3554 // dominance information ("Global cases"). 3555 bool Transform = false; 3556 if (Op1 && Op2) 3557 Transform = true; 3558 else if (Op1 || Op2) { 3559 // Local case 3560 if (SI->hasOneUse()) 3561 Transform = true; 3562 // Global cases 3563 else if (CI && !CI->isZero()) 3564 // When Op1 is constant try replacing select with second operand. 3565 // Otherwise Op2 is constant and try replacing select with first 3566 // operand. 3567 Transform = replacedSelectWithOperand(SI, &I, Op1 ? 2 : 1); 3568 } 3569 if (Transform) { 3570 if (!Op1) 3571 Op1 = Builder.CreateICmp(Pred, SI->getOperand(1), RHS, I.getName()); 3572 if (!Op2) 3573 Op2 = Builder.CreateICmp(Pred, SI->getOperand(2), RHS, I.getName()); 3574 return SelectInst::Create(SI->getOperand(0), Op1, Op2); 3575 } 3576 3577 return nullptr; 3578 } 3579 3580 /// Some comparisons can be simplified. 3581 /// In this case, we are looking for comparisons that look like 3582 /// a check for a lossy truncation. 3583 /// Folds: 3584 /// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask 3585 /// Where Mask is some pattern that produces all-ones in low bits: 3586 /// (-1 >> y) 3587 /// ((-1 << y) >> y) <- non-canonical, has extra uses 3588 /// ~(-1 << y) 3589 /// ((1 << y) + (-1)) <- non-canonical, has extra uses 3590 /// The Mask can be a constant, too. 3591 /// For some predicates, the operands are commutative. 3592 /// For others, x can only be on a specific side. 3593 static Value *foldICmpWithLowBitMaskedVal(ICmpInst &I, 3594 InstCombiner::BuilderTy &Builder) { 3595 ICmpInst::Predicate SrcPred; 3596 Value *X, *M, *Y; 3597 auto m_VariableMask = m_CombineOr( 3598 m_CombineOr(m_Not(m_Shl(m_AllOnes(), m_Value())), 3599 m_Add(m_Shl(m_One(), m_Value()), m_AllOnes())), 3600 m_CombineOr(m_LShr(m_AllOnes(), m_Value()), 3601 m_LShr(m_Shl(m_AllOnes(), m_Value(Y)), m_Deferred(Y)))); 3602 auto m_Mask = m_CombineOr(m_VariableMask, m_LowBitMask()); 3603 if (!match(&I, m_c_ICmp(SrcPred, 3604 m_c_And(m_CombineAnd(m_Mask, m_Value(M)), m_Value(X)), 3605 m_Deferred(X)))) 3606 return nullptr; 3607 3608 ICmpInst::Predicate DstPred; 3609 switch (SrcPred) { 3610 case ICmpInst::Predicate::ICMP_EQ: 3611 // x & (-1 >> y) == x -> x u<= (-1 >> y) 3612 DstPred = ICmpInst::Predicate::ICMP_ULE; 3613 break; 3614 case ICmpInst::Predicate::ICMP_NE: 3615 // x & (-1 >> y) != x -> x u> (-1 >> y) 3616 DstPred = ICmpInst::Predicate::ICMP_UGT; 3617 break; 3618 case ICmpInst::Predicate::ICMP_ULT: 3619 // x & (-1 >> y) u< x -> x u> (-1 >> y) 3620 // x u> x & (-1 >> y) -> x u> (-1 >> y) 3621 DstPred = ICmpInst::Predicate::ICMP_UGT; 3622 break; 3623 case ICmpInst::Predicate::ICMP_UGE: 3624 // x & (-1 >> y) u>= x -> x u<= (-1 >> y) 3625 // x u<= x & (-1 >> y) -> x u<= (-1 >> y) 3626 DstPred = ICmpInst::Predicate::ICMP_ULE; 3627 break; 3628 case ICmpInst::Predicate::ICMP_SLT: 3629 // x & (-1 >> y) s< x -> x s> (-1 >> y) 3630 // x s> x & (-1 >> y) -> x s> (-1 >> y) 3631 if (!match(M, m_Constant())) // Can not do this fold with non-constant. 3632 return nullptr; 3633 if (!match(M, m_NonNegative())) // Must not have any -1 vector elements. 3634 return nullptr; 3635 DstPred = ICmpInst::Predicate::ICMP_SGT; 3636 break; 3637 case ICmpInst::Predicate::ICMP_SGE: 3638 // x & (-1 >> y) s>= x -> x s<= (-1 >> y) 3639 // x s<= x & (-1 >> y) -> x s<= (-1 >> y) 3640 if (!match(M, m_Constant())) // Can not do this fold with non-constant. 3641 return nullptr; 3642 if (!match(M, m_NonNegative())) // Must not have any -1 vector elements. 3643 return nullptr; 3644 DstPred = ICmpInst::Predicate::ICMP_SLE; 3645 break; 3646 case ICmpInst::Predicate::ICMP_SGT: 3647 case ICmpInst::Predicate::ICMP_SLE: 3648 return nullptr; 3649 case ICmpInst::Predicate::ICMP_UGT: 3650 case ICmpInst::Predicate::ICMP_ULE: 3651 llvm_unreachable("Instsimplify took care of commut. variant"); 3652 break; 3653 default: 3654 llvm_unreachable("All possible folds are handled."); 3655 } 3656 3657 // The mask value may be a vector constant that has undefined elements. But it 3658 // may not be safe to propagate those undefs into the new compare, so replace 3659 // those elements by copying an existing, defined, and safe scalar constant. 3660 Type *OpTy = M->getType(); 3661 auto *VecC = dyn_cast<Constant>(M); 3662 auto *OpVTy = dyn_cast<FixedVectorType>(OpTy); 3663 if (OpVTy && VecC && VecC->containsUndefOrPoisonElement()) { 3664 Constant *SafeReplacementConstant = nullptr; 3665 for (unsigned i = 0, e = OpVTy->getNumElements(); i != e; ++i) { 3666 if (!isa<UndefValue>(VecC->getAggregateElement(i))) { 3667 SafeReplacementConstant = VecC->getAggregateElement(i); 3668 break; 3669 } 3670 } 3671 assert(SafeReplacementConstant && "Failed to find undef replacement"); 3672 M = Constant::replaceUndefsWith(VecC, SafeReplacementConstant); 3673 } 3674 3675 return Builder.CreateICmp(DstPred, X, M); 3676 } 3677 3678 /// Some comparisons can be simplified. 3679 /// In this case, we are looking for comparisons that look like 3680 /// a check for a lossy signed truncation. 3681 /// Folds: (MaskedBits is a constant.) 3682 /// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x 3683 /// Into: 3684 /// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits) 3685 /// Where KeptBits = bitwidth(%x) - MaskedBits 3686 static Value * 3687 foldICmpWithTruncSignExtendedVal(ICmpInst &I, 3688 InstCombiner::BuilderTy &Builder) { 3689 ICmpInst::Predicate SrcPred; 3690 Value *X; 3691 const APInt *C0, *C1; // FIXME: non-splats, potentially with undef. 3692 // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use. 3693 if (!match(&I, m_c_ICmp(SrcPred, 3694 m_OneUse(m_AShr(m_Shl(m_Value(X), m_APInt(C0)), 3695 m_APInt(C1))), 3696 m_Deferred(X)))) 3697 return nullptr; 3698 3699 // Potential handling of non-splats: for each element: 3700 // * if both are undef, replace with constant 0. 3701 // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0. 3702 // * if both are not undef, and are different, bailout. 3703 // * else, only one is undef, then pick the non-undef one. 3704 3705 // The shift amount must be equal. 3706 if (*C0 != *C1) 3707 return nullptr; 3708 const APInt &MaskedBits = *C0; 3709 assert(MaskedBits != 0 && "shift by zero should be folded away already."); 3710 3711 ICmpInst::Predicate DstPred; 3712 switch (SrcPred) { 3713 case ICmpInst::Predicate::ICMP_EQ: 3714 // ((%x << MaskedBits) a>> MaskedBits) == %x 3715 // => 3716 // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits) 3717 DstPred = ICmpInst::Predicate::ICMP_ULT; 3718 break; 3719 case ICmpInst::Predicate::ICMP_NE: 3720 // ((%x << MaskedBits) a>> MaskedBits) != %x 3721 // => 3722 // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits) 3723 DstPred = ICmpInst::Predicate::ICMP_UGE; 3724 break; 3725 // FIXME: are more folds possible? 3726 default: 3727 return nullptr; 3728 } 3729 3730 auto *XType = X->getType(); 3731 const unsigned XBitWidth = XType->getScalarSizeInBits(); 3732 const APInt BitWidth = APInt(XBitWidth, XBitWidth); 3733 assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched"); 3734 3735 // KeptBits = bitwidth(%x) - MaskedBits 3736 const APInt KeptBits = BitWidth - MaskedBits; 3737 assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable"); 3738 // ICmpCst = (1 << KeptBits) 3739 const APInt ICmpCst = APInt(XBitWidth, 1).shl(KeptBits); 3740 assert(ICmpCst.isPowerOf2()); 3741 // AddCst = (1 << (KeptBits-1)) 3742 const APInt AddCst = ICmpCst.lshr(1); 3743 assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2()); 3744 3745 // T0 = add %x, AddCst 3746 Value *T0 = Builder.CreateAdd(X, ConstantInt::get(XType, AddCst)); 3747 // T1 = T0 DstPred ICmpCst 3748 Value *T1 = Builder.CreateICmp(DstPred, T0, ConstantInt::get(XType, ICmpCst)); 3749 3750 return T1; 3751 } 3752 3753 // Given pattern: 3754 // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 3755 // we should move shifts to the same hand of 'and', i.e. rewrite as 3756 // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) 3757 // We are only interested in opposite logical shifts here. 3758 // One of the shifts can be truncated. 3759 // If we can, we want to end up creating 'lshr' shift. 3760 static Value * 3761 foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ, 3762 InstCombiner::BuilderTy &Builder) { 3763 if (!I.isEquality() || !match(I.getOperand(1), m_Zero()) || 3764 !I.getOperand(0)->hasOneUse()) 3765 return nullptr; 3766 3767 auto m_AnyLogicalShift = m_LogicalShift(m_Value(), m_Value()); 3768 3769 // Look for an 'and' of two logical shifts, one of which may be truncated. 3770 // We use m_TruncOrSelf() on the RHS to correctly handle commutative case. 3771 Instruction *XShift, *MaybeTruncation, *YShift; 3772 if (!match( 3773 I.getOperand(0), 3774 m_c_And(m_CombineAnd(m_AnyLogicalShift, m_Instruction(XShift)), 3775 m_CombineAnd(m_TruncOrSelf(m_CombineAnd( 3776 m_AnyLogicalShift, m_Instruction(YShift))), 3777 m_Instruction(MaybeTruncation))))) 3778 return nullptr; 3779 3780 // We potentially looked past 'trunc', but only when matching YShift, 3781 // therefore YShift must have the widest type. 3782 Instruction *WidestShift = YShift; 3783 // Therefore XShift must have the shallowest type. 3784 // Or they both have identical types if there was no truncation. 3785 Instruction *NarrowestShift = XShift; 3786 3787 Type *WidestTy = WidestShift->getType(); 3788 Type *NarrowestTy = NarrowestShift->getType(); 3789 assert(NarrowestTy == I.getOperand(0)->getType() && 3790 "We did not look past any shifts while matching XShift though."); 3791 bool HadTrunc = WidestTy != I.getOperand(0)->getType(); 3792 3793 // If YShift is a 'lshr', swap the shifts around. 3794 if (match(YShift, m_LShr(m_Value(), m_Value()))) 3795 std::swap(XShift, YShift); 3796 3797 // The shifts must be in opposite directions. 3798 auto XShiftOpcode = XShift->getOpcode(); 3799 if (XShiftOpcode == YShift->getOpcode()) 3800 return nullptr; // Do not care about same-direction shifts here. 3801 3802 Value *X, *XShAmt, *Y, *YShAmt; 3803 match(XShift, m_BinOp(m_Value(X), m_ZExtOrSelf(m_Value(XShAmt)))); 3804 match(YShift, m_BinOp(m_Value(Y), m_ZExtOrSelf(m_Value(YShAmt)))); 3805 3806 // If one of the values being shifted is a constant, then we will end with 3807 // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not, 3808 // however, we will need to ensure that we won't increase instruction count. 3809 if (!isa<Constant>(X) && !isa<Constant>(Y)) { 3810 // At least one of the hands of the 'and' should be one-use shift. 3811 if (!match(I.getOperand(0), 3812 m_c_And(m_OneUse(m_AnyLogicalShift), m_Value()))) 3813 return nullptr; 3814 if (HadTrunc) { 3815 // Due to the 'trunc', we will need to widen X. For that either the old 3816 // 'trunc' or the shift amt in the non-truncated shift should be one-use. 3817 if (!MaybeTruncation->hasOneUse() && 3818 !NarrowestShift->getOperand(1)->hasOneUse()) 3819 return nullptr; 3820 } 3821 } 3822 3823 // We have two shift amounts from two different shifts. The types of those 3824 // shift amounts may not match. If that's the case let's bailout now. 3825 if (XShAmt->getType() != YShAmt->getType()) 3826 return nullptr; 3827 3828 // As input, we have the following pattern: 3829 // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 3830 // We want to rewrite that as: 3831 // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) 3832 // While we know that originally (Q+K) would not overflow 3833 // (because 2 * (N-1) u<= iN -1), we have looked past extensions of 3834 // shift amounts. so it may now overflow in smaller bitwidth. 3835 // To ensure that does not happen, we need to ensure that the total maximal 3836 // shift amount is still representable in that smaller bit width. 3837 unsigned MaximalPossibleTotalShiftAmount = 3838 (WidestTy->getScalarSizeInBits() - 1) + 3839 (NarrowestTy->getScalarSizeInBits() - 1); 3840 APInt MaximalRepresentableShiftAmount = 3841 APInt::getAllOnes(XShAmt->getType()->getScalarSizeInBits()); 3842 if (MaximalRepresentableShiftAmount.ult(MaximalPossibleTotalShiftAmount)) 3843 return nullptr; 3844 3845 // Can we fold (XShAmt+YShAmt) ? 3846 auto *NewShAmt = dyn_cast_or_null<Constant>( 3847 simplifyAddInst(XShAmt, YShAmt, /*isNSW=*/false, 3848 /*isNUW=*/false, SQ.getWithInstruction(&I))); 3849 if (!NewShAmt) 3850 return nullptr; 3851 NewShAmt = ConstantExpr::getZExtOrBitCast(NewShAmt, WidestTy); 3852 unsigned WidestBitWidth = WidestTy->getScalarSizeInBits(); 3853 3854 // Is the new shift amount smaller than the bit width? 3855 // FIXME: could also rely on ConstantRange. 3856 if (!match(NewShAmt, 3857 m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_ULT, 3858 APInt(WidestBitWidth, WidestBitWidth)))) 3859 return nullptr; 3860 3861 // An extra legality check is needed if we had trunc-of-lshr. 3862 if (HadTrunc && match(WidestShift, m_LShr(m_Value(), m_Value()))) { 3863 auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ, 3864 WidestShift]() { 3865 // It isn't obvious whether it's worth it to analyze non-constants here. 3866 // Also, let's basically give up on non-splat cases, pessimizing vectors. 3867 // If *any* of these preconditions matches we can perform the fold. 3868 Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy() 3869 ? NewShAmt->getSplatValue() 3870 : NewShAmt; 3871 // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold. 3872 if (NewShAmtSplat && 3873 (NewShAmtSplat->isNullValue() || 3874 NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1)) 3875 return true; 3876 // We consider *min* leading zeros so a single outlier 3877 // blocks the transform as opposed to allowing it. 3878 if (auto *C = dyn_cast<Constant>(NarrowestShift->getOperand(0))) { 3879 KnownBits Known = computeKnownBits(C, SQ.DL); 3880 unsigned MinLeadZero = Known.countMinLeadingZeros(); 3881 // If the value being shifted has at most lowest bit set we can fold. 3882 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; 3883 if (MaxActiveBits <= 1) 3884 return true; 3885 // Precondition: NewShAmt u<= countLeadingZeros(C) 3886 if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(MinLeadZero)) 3887 return true; 3888 } 3889 if (auto *C = dyn_cast<Constant>(WidestShift->getOperand(0))) { 3890 KnownBits Known = computeKnownBits(C, SQ.DL); 3891 unsigned MinLeadZero = Known.countMinLeadingZeros(); 3892 // If the value being shifted has at most lowest bit set we can fold. 3893 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; 3894 if (MaxActiveBits <= 1) 3895 return true; 3896 // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C) 3897 if (NewShAmtSplat) { 3898 APInt AdjNewShAmt = 3899 (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger(); 3900 if (AdjNewShAmt.ule(MinLeadZero)) 3901 return true; 3902 } 3903 } 3904 return false; // Can't tell if it's ok. 3905 }; 3906 if (!CanFold()) 3907 return nullptr; 3908 } 3909 3910 // All good, we can do this fold. 3911 X = Builder.CreateZExt(X, WidestTy); 3912 Y = Builder.CreateZExt(Y, WidestTy); 3913 // The shift is the same that was for X. 3914 Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr 3915 ? Builder.CreateLShr(X, NewShAmt) 3916 : Builder.CreateShl(X, NewShAmt); 3917 Value *T1 = Builder.CreateAnd(T0, Y); 3918 return Builder.CreateICmp(I.getPredicate(), T1, 3919 Constant::getNullValue(WidestTy)); 3920 } 3921 3922 /// Fold 3923 /// (-1 u/ x) u< y 3924 /// ((x * y) ?/ x) != y 3925 /// to 3926 /// @llvm.?mul.with.overflow(x, y) plus extraction of overflow bit 3927 /// Note that the comparison is commutative, while inverted (u>=, ==) predicate 3928 /// will mean that we are looking for the opposite answer. 3929 Value *InstCombinerImpl::foldMultiplicationOverflowCheck(ICmpInst &I) { 3930 ICmpInst::Predicate Pred; 3931 Value *X, *Y; 3932 Instruction *Mul; 3933 Instruction *Div; 3934 bool NeedNegation; 3935 // Look for: (-1 u/ x) u</u>= y 3936 if (!I.isEquality() && 3937 match(&I, m_c_ICmp(Pred, 3938 m_CombineAnd(m_OneUse(m_UDiv(m_AllOnes(), m_Value(X))), 3939 m_Instruction(Div)), 3940 m_Value(Y)))) { 3941 Mul = nullptr; 3942 3943 // Are we checking that overflow does not happen, or does happen? 3944 switch (Pred) { 3945 case ICmpInst::Predicate::ICMP_ULT: 3946 NeedNegation = false; 3947 break; // OK 3948 case ICmpInst::Predicate::ICMP_UGE: 3949 NeedNegation = true; 3950 break; // OK 3951 default: 3952 return nullptr; // Wrong predicate. 3953 } 3954 } else // Look for: ((x * y) / x) !=/== y 3955 if (I.isEquality() && 3956 match(&I, 3957 m_c_ICmp(Pred, m_Value(Y), 3958 m_CombineAnd( 3959 m_OneUse(m_IDiv(m_CombineAnd(m_c_Mul(m_Deferred(Y), 3960 m_Value(X)), 3961 m_Instruction(Mul)), 3962 m_Deferred(X))), 3963 m_Instruction(Div))))) { 3964 NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ; 3965 } else 3966 return nullptr; 3967 3968 BuilderTy::InsertPointGuard Guard(Builder); 3969 // If the pattern included (x * y), we'll want to insert new instructions 3970 // right before that original multiplication so that we can replace it. 3971 bool MulHadOtherUses = Mul && !Mul->hasOneUse(); 3972 if (MulHadOtherUses) 3973 Builder.SetInsertPoint(Mul); 3974 3975 Function *F = Intrinsic::getDeclaration(I.getModule(), 3976 Div->getOpcode() == Instruction::UDiv 3977 ? Intrinsic::umul_with_overflow 3978 : Intrinsic::smul_with_overflow, 3979 X->getType()); 3980 CallInst *Call = Builder.CreateCall(F, {X, Y}, "mul"); 3981 3982 // If the multiplication was used elsewhere, to ensure that we don't leave 3983 // "duplicate" instructions, replace uses of that original multiplication 3984 // with the multiplication result from the with.overflow intrinsic. 3985 if (MulHadOtherUses) 3986 replaceInstUsesWith(*Mul, Builder.CreateExtractValue(Call, 0, "mul.val")); 3987 3988 Value *Res = Builder.CreateExtractValue(Call, 1, "mul.ov"); 3989 if (NeedNegation) // This technically increases instruction count. 3990 Res = Builder.CreateNot(Res, "mul.not.ov"); 3991 3992 // If we replaced the mul, erase it. Do this after all uses of Builder, 3993 // as the mul is used as insertion point. 3994 if (MulHadOtherUses) 3995 eraseInstFromFunction(*Mul); 3996 3997 return Res; 3998 } 3999 4000 static Instruction *foldICmpXNegX(ICmpInst &I) { 4001 CmpInst::Predicate Pred; 4002 Value *X; 4003 if (!match(&I, m_c_ICmp(Pred, m_NSWNeg(m_Value(X)), m_Deferred(X)))) 4004 return nullptr; 4005 4006 if (ICmpInst::isSigned(Pred)) 4007 Pred = ICmpInst::getSwappedPredicate(Pred); 4008 else if (ICmpInst::isUnsigned(Pred)) 4009 Pred = ICmpInst::getSignedPredicate(Pred); 4010 // else for equality-comparisons just keep the predicate. 4011 4012 return ICmpInst::Create(Instruction::ICmp, Pred, X, 4013 Constant::getNullValue(X->getType()), I.getName()); 4014 } 4015 4016 /// Try to fold icmp (binop), X or icmp X, (binop). 4017 /// TODO: A large part of this logic is duplicated in InstSimplify's 4018 /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code 4019 /// duplication. 4020 Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I, 4021 const SimplifyQuery &SQ) { 4022 const SimplifyQuery Q = SQ.getWithInstruction(&I); 4023 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 4024 4025 // Special logic for binary operators. 4026 BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Op0); 4027 BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Op1); 4028 if (!BO0 && !BO1) 4029 return nullptr; 4030 4031 if (Instruction *NewICmp = foldICmpXNegX(I)) 4032 return NewICmp; 4033 4034 const CmpInst::Predicate Pred = I.getPredicate(); 4035 Value *X; 4036 4037 // Convert add-with-unsigned-overflow comparisons into a 'not' with compare. 4038 // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X 4039 if (match(Op0, m_OneUse(m_c_Add(m_Specific(Op1), m_Value(X)))) && 4040 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) 4041 return new ICmpInst(Pred, Builder.CreateNot(Op1), X); 4042 // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0 4043 if (match(Op1, m_OneUse(m_c_Add(m_Specific(Op0), m_Value(X)))) && 4044 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) 4045 return new ICmpInst(Pred, X, Builder.CreateNot(Op0)); 4046 4047 { 4048 // (Op1 + X) + C u</u>= Op1 --> ~C - X u</u>= Op1 4049 Constant *C; 4050 if (match(Op0, m_OneUse(m_Add(m_c_Add(m_Specific(Op1), m_Value(X)), 4051 m_ImmConstant(C)))) && 4052 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) { 4053 Constant *C2 = ConstantExpr::getNot(C); 4054 return new ICmpInst(Pred, Builder.CreateSub(C2, X), Op1); 4055 } 4056 // Op0 u>/u<= (Op0 + X) + C --> Op0 u>/u<= ~C - X 4057 if (match(Op1, m_OneUse(m_Add(m_c_Add(m_Specific(Op0), m_Value(X)), 4058 m_ImmConstant(C)))) && 4059 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) { 4060 Constant *C2 = ConstantExpr::getNot(C); 4061 return new ICmpInst(Pred, Op0, Builder.CreateSub(C2, X)); 4062 } 4063 } 4064 4065 { 4066 // Similar to above: an unsigned overflow comparison may use offset + mask: 4067 // ((Op1 + C) & C) u< Op1 --> Op1 != 0 4068 // ((Op1 + C) & C) u>= Op1 --> Op1 == 0 4069 // Op0 u> ((Op0 + C) & C) --> Op0 != 0 4070 // Op0 u<= ((Op0 + C) & C) --> Op0 == 0 4071 BinaryOperator *BO; 4072 const APInt *C; 4073 if ((Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) && 4074 match(Op0, m_And(m_BinOp(BO), m_LowBitMask(C))) && 4075 match(BO, m_Add(m_Specific(Op1), m_SpecificIntAllowUndef(*C)))) { 4076 CmpInst::Predicate NewPred = 4077 Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ; 4078 Constant *Zero = ConstantInt::getNullValue(Op1->getType()); 4079 return new ICmpInst(NewPred, Op1, Zero); 4080 } 4081 4082 if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) && 4083 match(Op1, m_And(m_BinOp(BO), m_LowBitMask(C))) && 4084 match(BO, m_Add(m_Specific(Op0), m_SpecificIntAllowUndef(*C)))) { 4085 CmpInst::Predicate NewPred = 4086 Pred == ICmpInst::ICMP_UGT ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ; 4087 Constant *Zero = ConstantInt::getNullValue(Op1->getType()); 4088 return new ICmpInst(NewPred, Op0, Zero); 4089 } 4090 } 4091 4092 bool NoOp0WrapProblem = false, NoOp1WrapProblem = false; 4093 if (BO0 && isa<OverflowingBinaryOperator>(BO0)) 4094 NoOp0WrapProblem = 4095 ICmpInst::isEquality(Pred) || 4096 (CmpInst::isUnsigned(Pred) && BO0->hasNoUnsignedWrap()) || 4097 (CmpInst::isSigned(Pred) && BO0->hasNoSignedWrap()); 4098 if (BO1 && isa<OverflowingBinaryOperator>(BO1)) 4099 NoOp1WrapProblem = 4100 ICmpInst::isEquality(Pred) || 4101 (CmpInst::isUnsigned(Pred) && BO1->hasNoUnsignedWrap()) || 4102 (CmpInst::isSigned(Pred) && BO1->hasNoSignedWrap()); 4103 4104 // Analyze the case when either Op0 or Op1 is an add instruction. 4105 // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null). 4106 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr; 4107 if (BO0 && BO0->getOpcode() == Instruction::Add) { 4108 A = BO0->getOperand(0); 4109 B = BO0->getOperand(1); 4110 } 4111 if (BO1 && BO1->getOpcode() == Instruction::Add) { 4112 C = BO1->getOperand(0); 4113 D = BO1->getOperand(1); 4114 } 4115 4116 // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow. 4117 // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow. 4118 if ((A == Op1 || B == Op1) && NoOp0WrapProblem) 4119 return new ICmpInst(Pred, A == Op1 ? B : A, 4120 Constant::getNullValue(Op1->getType())); 4121 4122 // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow. 4123 // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow. 4124 if ((C == Op0 || D == Op0) && NoOp1WrapProblem) 4125 return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()), 4126 C == Op0 ? D : C); 4127 4128 // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow. 4129 if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem && 4130 NoOp1WrapProblem) { 4131 // Determine Y and Z in the form icmp (X+Y), (X+Z). 4132 Value *Y, *Z; 4133 if (A == C) { 4134 // C + B == C + D -> B == D 4135 Y = B; 4136 Z = D; 4137 } else if (A == D) { 4138 // D + B == C + D -> B == C 4139 Y = B; 4140 Z = C; 4141 } else if (B == C) { 4142 // A + C == C + D -> A == D 4143 Y = A; 4144 Z = D; 4145 } else { 4146 assert(B == D); 4147 // A + D == C + D -> A == C 4148 Y = A; 4149 Z = C; 4150 } 4151 return new ICmpInst(Pred, Y, Z); 4152 } 4153 4154 // icmp slt (A + -1), Op1 -> icmp sle A, Op1 4155 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT && 4156 match(B, m_AllOnes())) 4157 return new ICmpInst(CmpInst::ICMP_SLE, A, Op1); 4158 4159 // icmp sge (A + -1), Op1 -> icmp sgt A, Op1 4160 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE && 4161 match(B, m_AllOnes())) 4162 return new ICmpInst(CmpInst::ICMP_SGT, A, Op1); 4163 4164 // icmp sle (A + 1), Op1 -> icmp slt A, Op1 4165 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One())) 4166 return new ICmpInst(CmpInst::ICMP_SLT, A, Op1); 4167 4168 // icmp sgt (A + 1), Op1 -> icmp sge A, Op1 4169 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One())) 4170 return new ICmpInst(CmpInst::ICMP_SGE, A, Op1); 4171 4172 // icmp sgt Op0, (C + -1) -> icmp sge Op0, C 4173 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT && 4174 match(D, m_AllOnes())) 4175 return new ICmpInst(CmpInst::ICMP_SGE, Op0, C); 4176 4177 // icmp sle Op0, (C + -1) -> icmp slt Op0, C 4178 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE && 4179 match(D, m_AllOnes())) 4180 return new ICmpInst(CmpInst::ICMP_SLT, Op0, C); 4181 4182 // icmp sge Op0, (C + 1) -> icmp sgt Op0, C 4183 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One())) 4184 return new ICmpInst(CmpInst::ICMP_SGT, Op0, C); 4185 4186 // icmp slt Op0, (C + 1) -> icmp sle Op0, C 4187 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One())) 4188 return new ICmpInst(CmpInst::ICMP_SLE, Op0, C); 4189 4190 // TODO: The subtraction-related identities shown below also hold, but 4191 // canonicalization from (X -nuw 1) to (X + -1) means that the combinations 4192 // wouldn't happen even if they were implemented. 4193 // 4194 // icmp ult (A - 1), Op1 -> icmp ule A, Op1 4195 // icmp uge (A - 1), Op1 -> icmp ugt A, Op1 4196 // icmp ugt Op0, (C - 1) -> icmp uge Op0, C 4197 // icmp ule Op0, (C - 1) -> icmp ult Op0, C 4198 4199 // icmp ule (A + 1), Op0 -> icmp ult A, Op1 4200 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One())) 4201 return new ICmpInst(CmpInst::ICMP_ULT, A, Op1); 4202 4203 // icmp ugt (A + 1), Op0 -> icmp uge A, Op1 4204 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One())) 4205 return new ICmpInst(CmpInst::ICMP_UGE, A, Op1); 4206 4207 // icmp uge Op0, (C + 1) -> icmp ugt Op0, C 4208 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One())) 4209 return new ICmpInst(CmpInst::ICMP_UGT, Op0, C); 4210 4211 // icmp ult Op0, (C + 1) -> icmp ule Op0, C 4212 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One())) 4213 return new ICmpInst(CmpInst::ICMP_ULE, Op0, C); 4214 4215 // if C1 has greater magnitude than C2: 4216 // icmp (A + C1), (C + C2) -> icmp (A + C3), C 4217 // s.t. C3 = C1 - C2 4218 // 4219 // if C2 has greater magnitude than C1: 4220 // icmp (A + C1), (C + C2) -> icmp A, (C + C3) 4221 // s.t. C3 = C2 - C1 4222 if (A && C && NoOp0WrapProblem && NoOp1WrapProblem && 4223 (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) { 4224 const APInt *AP1, *AP2; 4225 // TODO: Support non-uniform vectors. 4226 // TODO: Allow undef passthrough if B AND D's element is undef. 4227 if (match(B, m_APIntAllowUndef(AP1)) && match(D, m_APIntAllowUndef(AP2)) && 4228 AP1->isNegative() == AP2->isNegative()) { 4229 APInt AP1Abs = AP1->abs(); 4230 APInt AP2Abs = AP2->abs(); 4231 if (AP1Abs.uge(AP2Abs)) { 4232 APInt Diff = *AP1 - *AP2; 4233 bool HasNUW = BO0->hasNoUnsignedWrap() && Diff.ule(*AP1); 4234 bool HasNSW = BO0->hasNoSignedWrap(); 4235 Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff); 4236 Value *NewAdd = Builder.CreateAdd(A, C3, "", HasNUW, HasNSW); 4237 return new ICmpInst(Pred, NewAdd, C); 4238 } else { 4239 APInt Diff = *AP2 - *AP1; 4240 bool HasNUW = BO1->hasNoUnsignedWrap() && Diff.ule(*AP2); 4241 bool HasNSW = BO1->hasNoSignedWrap(); 4242 Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff); 4243 Value *NewAdd = Builder.CreateAdd(C, C3, "", HasNUW, HasNSW); 4244 return new ICmpInst(Pred, A, NewAdd); 4245 } 4246 } 4247 Constant *Cst1, *Cst2; 4248 if (match(B, m_ImmConstant(Cst1)) && match(D, m_ImmConstant(Cst2)) && 4249 ICmpInst::isEquality(Pred)) { 4250 Constant *Diff = ConstantExpr::getSub(Cst2, Cst1); 4251 Value *NewAdd = Builder.CreateAdd(C, Diff); 4252 return new ICmpInst(Pred, A, NewAdd); 4253 } 4254 } 4255 4256 // Analyze the case when either Op0 or Op1 is a sub instruction. 4257 // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null). 4258 A = nullptr; 4259 B = nullptr; 4260 C = nullptr; 4261 D = nullptr; 4262 if (BO0 && BO0->getOpcode() == Instruction::Sub) { 4263 A = BO0->getOperand(0); 4264 B = BO0->getOperand(1); 4265 } 4266 if (BO1 && BO1->getOpcode() == Instruction::Sub) { 4267 C = BO1->getOperand(0); 4268 D = BO1->getOperand(1); 4269 } 4270 4271 // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow. 4272 if (A == Op1 && NoOp0WrapProblem) 4273 return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B); 4274 // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow. 4275 if (C == Op0 && NoOp1WrapProblem) 4276 return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType())); 4277 4278 // Convert sub-with-unsigned-overflow comparisons into a comparison of args. 4279 // (A - B) u>/u<= A --> B u>/u<= A 4280 if (A == Op1 && (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) 4281 return new ICmpInst(Pred, B, A); 4282 // C u</u>= (C - D) --> C u</u>= D 4283 if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) 4284 return new ICmpInst(Pred, C, D); 4285 // (A - B) u>=/u< A --> B u>/u<= A iff B != 0 4286 if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) && 4287 isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) 4288 return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), B, A); 4289 // C u<=/u> (C - D) --> C u</u>= D iff B != 0 4290 if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) && 4291 isKnownNonZero(D, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) 4292 return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), C, D); 4293 4294 // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow. 4295 if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem) 4296 return new ICmpInst(Pred, A, C); 4297 4298 // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow. 4299 if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem) 4300 return new ICmpInst(Pred, D, B); 4301 4302 // icmp (0-X) < cst --> x > -cst 4303 if (NoOp0WrapProblem && ICmpInst::isSigned(Pred)) { 4304 Value *X; 4305 if (match(BO0, m_Neg(m_Value(X)))) 4306 if (Constant *RHSC = dyn_cast<Constant>(Op1)) 4307 if (RHSC->isNotMinSignedValue()) 4308 return new ICmpInst(I.getSwappedPredicate(), X, 4309 ConstantExpr::getNeg(RHSC)); 4310 } 4311 4312 { 4313 // Try to remove shared constant multiplier from equality comparison: 4314 // X * C == Y * C (with no overflowing/aliasing) --> X == Y 4315 Value *X, *Y; 4316 const APInt *C; 4317 if (match(Op0, m_Mul(m_Value(X), m_APInt(C))) && *C != 0 && 4318 match(Op1, m_Mul(m_Value(Y), m_SpecificInt(*C))) && I.isEquality()) 4319 if (!C->countTrailingZeros() || 4320 (BO0 && BO1 && BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap()) || 4321 (BO0 && BO1 && BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap())) 4322 return new ICmpInst(Pred, X, Y); 4323 } 4324 4325 BinaryOperator *SRem = nullptr; 4326 // icmp (srem X, Y), Y 4327 if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(1)) 4328 SRem = BO0; 4329 // icmp Y, (srem X, Y) 4330 else if (BO1 && BO1->getOpcode() == Instruction::SRem && 4331 Op0 == BO1->getOperand(1)) 4332 SRem = BO1; 4333 if (SRem) { 4334 // We don't check hasOneUse to avoid increasing register pressure because 4335 // the value we use is the same value this instruction was already using. 4336 switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) { 4337 default: 4338 break; 4339 case ICmpInst::ICMP_EQ: 4340 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 4341 case ICmpInst::ICMP_NE: 4342 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 4343 case ICmpInst::ICMP_SGT: 4344 case ICmpInst::ICMP_SGE: 4345 return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1), 4346 Constant::getAllOnesValue(SRem->getType())); 4347 case ICmpInst::ICMP_SLT: 4348 case ICmpInst::ICMP_SLE: 4349 return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1), 4350 Constant::getNullValue(SRem->getType())); 4351 } 4352 } 4353 4354 if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() && BO0->hasOneUse() && 4355 BO1->hasOneUse() && BO0->getOperand(1) == BO1->getOperand(1)) { 4356 switch (BO0->getOpcode()) { 4357 default: 4358 break; 4359 case Instruction::Add: 4360 case Instruction::Sub: 4361 case Instruction::Xor: { 4362 if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b 4363 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); 4364 4365 const APInt *C; 4366 if (match(BO0->getOperand(1), m_APInt(C))) { 4367 // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b 4368 if (C->isSignMask()) { 4369 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate(); 4370 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); 4371 } 4372 4373 // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b 4374 if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) { 4375 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate(); 4376 NewPred = I.getSwappedPredicate(NewPred); 4377 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); 4378 } 4379 } 4380 break; 4381 } 4382 case Instruction::Mul: { 4383 if (!I.isEquality()) 4384 break; 4385 4386 const APInt *C; 4387 if (match(BO0->getOperand(1), m_APInt(C)) && !C->isZero() && 4388 !C->isOne()) { 4389 // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask) 4390 // Mask = -1 >> count-trailing-zeros(C). 4391 if (unsigned TZs = C->countTrailingZeros()) { 4392 Constant *Mask = ConstantInt::get( 4393 BO0->getType(), 4394 APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs)); 4395 Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask); 4396 Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask); 4397 return new ICmpInst(Pred, And1, And2); 4398 } 4399 } 4400 break; 4401 } 4402 case Instruction::UDiv: 4403 case Instruction::LShr: 4404 if (I.isSigned() || !BO0->isExact() || !BO1->isExact()) 4405 break; 4406 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); 4407 4408 case Instruction::SDiv: 4409 if (!I.isEquality() || !BO0->isExact() || !BO1->isExact()) 4410 break; 4411 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); 4412 4413 case Instruction::AShr: 4414 if (!BO0->isExact() || !BO1->isExact()) 4415 break; 4416 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); 4417 4418 case Instruction::Shl: { 4419 bool NUW = BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap(); 4420 bool NSW = BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap(); 4421 if (!NUW && !NSW) 4422 break; 4423 if (!NSW && I.isSigned()) 4424 break; 4425 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); 4426 } 4427 } 4428 } 4429 4430 if (BO0) { 4431 // Transform A & (L - 1) `ult` L --> L != 0 4432 auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes()); 4433 auto BitwiseAnd = m_c_And(m_Value(), LSubOne); 4434 4435 if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) { 4436 auto *Zero = Constant::getNullValue(BO0->getType()); 4437 return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero); 4438 } 4439 } 4440 4441 if (Value *V = foldMultiplicationOverflowCheck(I)) 4442 return replaceInstUsesWith(I, V); 4443 4444 if (Value *V = foldICmpWithLowBitMaskedVal(I, Builder)) 4445 return replaceInstUsesWith(I, V); 4446 4447 if (Value *V = foldICmpWithTruncSignExtendedVal(I, Builder)) 4448 return replaceInstUsesWith(I, V); 4449 4450 if (Value *V = foldShiftIntoShiftInAnotherHandOfAndInICmp(I, SQ, Builder)) 4451 return replaceInstUsesWith(I, V); 4452 4453 return nullptr; 4454 } 4455 4456 /// Fold icmp Pred min|max(X, Y), X. 4457 static Instruction *foldICmpWithMinMax(ICmpInst &Cmp) { 4458 ICmpInst::Predicate Pred = Cmp.getPredicate(); 4459 Value *Op0 = Cmp.getOperand(0); 4460 Value *X = Cmp.getOperand(1); 4461 4462 // Canonicalize minimum or maximum operand to LHS of the icmp. 4463 if (match(X, m_c_SMin(m_Specific(Op0), m_Value())) || 4464 match(X, m_c_SMax(m_Specific(Op0), m_Value())) || 4465 match(X, m_c_UMin(m_Specific(Op0), m_Value())) || 4466 match(X, m_c_UMax(m_Specific(Op0), m_Value()))) { 4467 std::swap(Op0, X); 4468 Pred = Cmp.getSwappedPredicate(); 4469 } 4470 4471 Value *Y; 4472 if (match(Op0, m_c_SMin(m_Specific(X), m_Value(Y)))) { 4473 // smin(X, Y) == X --> X s<= Y 4474 // smin(X, Y) s>= X --> X s<= Y 4475 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SGE) 4476 return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); 4477 4478 // smin(X, Y) != X --> X s> Y 4479 // smin(X, Y) s< X --> X s> Y 4480 if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SLT) 4481 return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); 4482 4483 // These cases should be handled in InstSimplify: 4484 // smin(X, Y) s<= X --> true 4485 // smin(X, Y) s> X --> false 4486 return nullptr; 4487 } 4488 4489 if (match(Op0, m_c_SMax(m_Specific(X), m_Value(Y)))) { 4490 // smax(X, Y) == X --> X s>= Y 4491 // smax(X, Y) s<= X --> X s>= Y 4492 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SLE) 4493 return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); 4494 4495 // smax(X, Y) != X --> X s< Y 4496 // smax(X, Y) s> X --> X s< Y 4497 if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SGT) 4498 return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); 4499 4500 // These cases should be handled in InstSimplify: 4501 // smax(X, Y) s>= X --> true 4502 // smax(X, Y) s< X --> false 4503 return nullptr; 4504 } 4505 4506 if (match(Op0, m_c_UMin(m_Specific(X), m_Value(Y)))) { 4507 // umin(X, Y) == X --> X u<= Y 4508 // umin(X, Y) u>= X --> X u<= Y 4509 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_UGE) 4510 return new ICmpInst(ICmpInst::ICMP_ULE, X, Y); 4511 4512 // umin(X, Y) != X --> X u> Y 4513 // umin(X, Y) u< X --> X u> Y 4514 if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_ULT) 4515 return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); 4516 4517 // These cases should be handled in InstSimplify: 4518 // umin(X, Y) u<= X --> true 4519 // umin(X, Y) u> X --> false 4520 return nullptr; 4521 } 4522 4523 if (match(Op0, m_c_UMax(m_Specific(X), m_Value(Y)))) { 4524 // umax(X, Y) == X --> X u>= Y 4525 // umax(X, Y) u<= X --> X u>= Y 4526 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_ULE) 4527 return new ICmpInst(ICmpInst::ICMP_UGE, X, Y); 4528 4529 // umax(X, Y) != X --> X u< Y 4530 // umax(X, Y) u> X --> X u< Y 4531 if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_UGT) 4532 return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); 4533 4534 // These cases should be handled in InstSimplify: 4535 // umax(X, Y) u>= X --> true 4536 // umax(X, Y) u< X --> false 4537 return nullptr; 4538 } 4539 4540 return nullptr; 4541 } 4542 4543 Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) { 4544 if (!I.isEquality()) 4545 return nullptr; 4546 4547 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 4548 const CmpInst::Predicate Pred = I.getPredicate(); 4549 Value *A, *B, *C, *D; 4550 if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) { 4551 if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0 4552 Value *OtherVal = A == Op1 ? B : A; 4553 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType())); 4554 } 4555 4556 if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) { 4557 // A^c1 == C^c2 --> A == C^(c1^c2) 4558 ConstantInt *C1, *C2; 4559 if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) && 4560 Op1->hasOneUse()) { 4561 Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue()); 4562 Value *Xor = Builder.CreateXor(C, NC); 4563 return new ICmpInst(Pred, A, Xor); 4564 } 4565 4566 // A^B == A^D -> B == D 4567 if (A == C) 4568 return new ICmpInst(Pred, B, D); 4569 if (A == D) 4570 return new ICmpInst(Pred, B, C); 4571 if (B == C) 4572 return new ICmpInst(Pred, A, D); 4573 if (B == D) 4574 return new ICmpInst(Pred, A, C); 4575 } 4576 } 4577 4578 if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) { 4579 // A == (A^B) -> B == 0 4580 Value *OtherVal = A == Op0 ? B : A; 4581 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType())); 4582 } 4583 4584 // (X&Z) == (Y&Z) -> (X^Y) & Z == 0 4585 if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B)))) && 4586 match(Op1, m_OneUse(m_And(m_Value(C), m_Value(D))))) { 4587 Value *X = nullptr, *Y = nullptr, *Z = nullptr; 4588 4589 if (A == C) { 4590 X = B; 4591 Y = D; 4592 Z = A; 4593 } else if (A == D) { 4594 X = B; 4595 Y = C; 4596 Z = A; 4597 } else if (B == C) { 4598 X = A; 4599 Y = D; 4600 Z = B; 4601 } else if (B == D) { 4602 X = A; 4603 Y = C; 4604 Z = B; 4605 } 4606 4607 if (X) { // Build (X^Y) & Z 4608 Op1 = Builder.CreateXor(X, Y); 4609 Op1 = Builder.CreateAnd(Op1, Z); 4610 return new ICmpInst(Pred, Op1, Constant::getNullValue(Op1->getType())); 4611 } 4612 } 4613 4614 { 4615 // Similar to above, but specialized for constant because invert is needed: 4616 // (X | C) == (Y | C) --> (X ^ Y) & ~C == 0 4617 Value *X, *Y; 4618 Constant *C; 4619 if (match(Op0, m_OneUse(m_Or(m_Value(X), m_Constant(C)))) && 4620 match(Op1, m_OneUse(m_Or(m_Value(Y), m_Specific(C))))) { 4621 Value *Xor = Builder.CreateXor(X, Y); 4622 Value *And = Builder.CreateAnd(Xor, ConstantExpr::getNot(C)); 4623 return new ICmpInst(Pred, And, Constant::getNullValue(And->getType())); 4624 } 4625 } 4626 4627 // Transform (zext A) == (B & (1<<X)-1) --> A == (trunc B) 4628 // and (B & (1<<X)-1) == (zext A) --> A == (trunc B) 4629 ConstantInt *Cst1; 4630 if ((Op0->hasOneUse() && match(Op0, m_ZExt(m_Value(A))) && 4631 match(Op1, m_And(m_Value(B), m_ConstantInt(Cst1)))) || 4632 (Op1->hasOneUse() && match(Op0, m_And(m_Value(B), m_ConstantInt(Cst1))) && 4633 match(Op1, m_ZExt(m_Value(A))))) { 4634 APInt Pow2 = Cst1->getValue() + 1; 4635 if (Pow2.isPowerOf2() && isa<IntegerType>(A->getType()) && 4636 Pow2.logBase2() == cast<IntegerType>(A->getType())->getBitWidth()) 4637 return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType())); 4638 } 4639 4640 // (A >> C) == (B >> C) --> (A^B) u< (1 << C) 4641 // For lshr and ashr pairs. 4642 const APInt *AP1, *AP2; 4643 if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_APIntAllowUndef(AP1)))) && 4644 match(Op1, m_OneUse(m_LShr(m_Value(B), m_APIntAllowUndef(AP2))))) || 4645 (match(Op0, m_OneUse(m_AShr(m_Value(A), m_APIntAllowUndef(AP1)))) && 4646 match(Op1, m_OneUse(m_AShr(m_Value(B), m_APIntAllowUndef(AP2)))))) { 4647 if (AP1 != AP2) 4648 return nullptr; 4649 unsigned TypeBits = AP1->getBitWidth(); 4650 unsigned ShAmt = AP1->getLimitedValue(TypeBits); 4651 if (ShAmt < TypeBits && ShAmt != 0) { 4652 ICmpInst::Predicate NewPred = 4653 Pred == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; 4654 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); 4655 APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt); 4656 return new ICmpInst(NewPred, Xor, ConstantInt::get(A->getType(), CmpVal)); 4657 } 4658 } 4659 4660 // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0 4661 if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) && 4662 match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) { 4663 unsigned TypeBits = Cst1->getBitWidth(); 4664 unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); 4665 if (ShAmt < TypeBits && ShAmt != 0) { 4666 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); 4667 APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt); 4668 Value *And = Builder.CreateAnd(Xor, Builder.getInt(AndVal), 4669 I.getName() + ".mask"); 4670 return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType())); 4671 } 4672 } 4673 4674 // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to 4675 // "icmp (and X, mask), cst" 4676 uint64_t ShAmt = 0; 4677 if (Op0->hasOneUse() && 4678 match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) && 4679 match(Op1, m_ConstantInt(Cst1)) && 4680 // Only do this when A has multiple uses. This is most important to do 4681 // when it exposes other optimizations. 4682 !A->hasOneUse()) { 4683 unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits(); 4684 4685 if (ShAmt < ASize) { 4686 APInt MaskV = 4687 APInt::getLowBitsSet(ASize, Op0->getType()->getPrimitiveSizeInBits()); 4688 MaskV <<= ShAmt; 4689 4690 APInt CmpV = Cst1->getValue().zext(ASize); 4691 CmpV <<= ShAmt; 4692 4693 Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV)); 4694 return new ICmpInst(Pred, Mask, Builder.getInt(CmpV)); 4695 } 4696 } 4697 4698 if (Instruction *ICmp = foldICmpIntrinsicWithIntrinsic(I)) 4699 return ICmp; 4700 4701 // Canonicalize checking for a power-of-2-or-zero value: 4702 // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants) 4703 // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants) 4704 if (!match(Op0, m_OneUse(m_c_And(m_Add(m_Value(A), m_AllOnes()), 4705 m_Deferred(A)))) || 4706 !match(Op1, m_ZeroInt())) 4707 A = nullptr; 4708 4709 // (A & -A) == A --> ctpop(A) < 2 (four commuted variants) 4710 // (-A & A) != A --> ctpop(A) > 1 (four commuted variants) 4711 if (match(Op0, m_OneUse(m_c_And(m_Neg(m_Specific(Op1)), m_Specific(Op1))))) 4712 A = Op1; 4713 else if (match(Op1, 4714 m_OneUse(m_c_And(m_Neg(m_Specific(Op0)), m_Specific(Op0))))) 4715 A = Op0; 4716 4717 if (A) { 4718 Type *Ty = A->getType(); 4719 CallInst *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, A); 4720 return Pred == ICmpInst::ICMP_EQ 4721 ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop, ConstantInt::get(Ty, 2)) 4722 : new ICmpInst(ICmpInst::ICMP_UGT, CtPop, ConstantInt::get(Ty, 1)); 4723 } 4724 4725 // Match icmp eq (trunc (lshr A, BW), (ashr (trunc A), BW-1)), which checks the 4726 // top BW/2 + 1 bits are all the same. Create "A >=s INT_MIN && A <=s INT_MAX", 4727 // which we generate as "icmp ult (add A, 2^(BW-1)), 2^BW" to skip a few steps 4728 // of instcombine. 4729 unsigned BitWidth = Op0->getType()->getScalarSizeInBits(); 4730 if (match(Op0, m_AShr(m_Trunc(m_Value(A)), m_SpecificInt(BitWidth - 1))) && 4731 match(Op1, m_Trunc(m_LShr(m_Specific(A), m_SpecificInt(BitWidth)))) && 4732 A->getType()->getScalarSizeInBits() == BitWidth * 2 && 4733 (I.getOperand(0)->hasOneUse() || I.getOperand(1)->hasOneUse())) { 4734 APInt C = APInt::getOneBitSet(BitWidth * 2, BitWidth - 1); 4735 Value *Add = Builder.CreateAdd(A, ConstantInt::get(A->getType(), C)); 4736 return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT 4737 : ICmpInst::ICMP_UGE, 4738 Add, ConstantInt::get(A->getType(), C.shl(1))); 4739 } 4740 4741 return nullptr; 4742 } 4743 4744 static Instruction *foldICmpWithTrunc(ICmpInst &ICmp, 4745 InstCombiner::BuilderTy &Builder) { 4746 ICmpInst::Predicate Pred = ICmp.getPredicate(); 4747 Value *Op0 = ICmp.getOperand(0), *Op1 = ICmp.getOperand(1); 4748 4749 // Try to canonicalize trunc + compare-to-constant into a mask + cmp. 4750 // The trunc masks high bits while the compare may effectively mask low bits. 4751 Value *X; 4752 const APInt *C; 4753 if (!match(Op0, m_OneUse(m_Trunc(m_Value(X)))) || !match(Op1, m_APInt(C))) 4754 return nullptr; 4755 4756 // This matches patterns corresponding to tests of the signbit as well as: 4757 // (trunc X) u< C --> (X & -C) == 0 (are all masked-high-bits clear?) 4758 // (trunc X) u> C --> (X & ~C) != 0 (are any masked-high-bits set?) 4759 APInt Mask; 4760 if (decomposeBitTestICmp(Op0, Op1, Pred, X, Mask, true /* WithTrunc */)) { 4761 Value *And = Builder.CreateAnd(X, Mask); 4762 Constant *Zero = ConstantInt::getNullValue(X->getType()); 4763 return new ICmpInst(Pred, And, Zero); 4764 } 4765 4766 unsigned SrcBits = X->getType()->getScalarSizeInBits(); 4767 if (Pred == ICmpInst::ICMP_ULT && C->isNegatedPowerOf2()) { 4768 // If C is a negative power-of-2 (high-bit mask): 4769 // (trunc X) u< C --> (X & C) != C (are any masked-high-bits clear?) 4770 Constant *MaskC = ConstantInt::get(X->getType(), C->zext(SrcBits)); 4771 Value *And = Builder.CreateAnd(X, MaskC); 4772 return new ICmpInst(ICmpInst::ICMP_NE, And, MaskC); 4773 } 4774 4775 if (Pred == ICmpInst::ICMP_UGT && (~*C).isPowerOf2()) { 4776 // If C is not-of-power-of-2 (one clear bit): 4777 // (trunc X) u> C --> (X & (C+1)) == C+1 (are all masked-high-bits set?) 4778 Constant *MaskC = ConstantInt::get(X->getType(), (*C + 1).zext(SrcBits)); 4779 Value *And = Builder.CreateAnd(X, MaskC); 4780 return new ICmpInst(ICmpInst::ICMP_EQ, And, MaskC); 4781 } 4782 4783 return nullptr; 4784 } 4785 4786 Instruction *InstCombinerImpl::foldICmpWithZextOrSext(ICmpInst &ICmp) { 4787 assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0"); 4788 auto *CastOp0 = cast<CastInst>(ICmp.getOperand(0)); 4789 Value *X; 4790 if (!match(CastOp0, m_ZExtOrSExt(m_Value(X)))) 4791 return nullptr; 4792 4793 bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt; 4794 bool IsSignedCmp = ICmp.isSigned(); 4795 4796 // icmp Pred (ext X), (ext Y) 4797 Value *Y; 4798 if (match(ICmp.getOperand(1), m_ZExtOrSExt(m_Value(Y)))) { 4799 bool IsZext0 = isa<ZExtOperator>(ICmp.getOperand(0)); 4800 bool IsZext1 = isa<ZExtOperator>(ICmp.getOperand(1)); 4801 4802 // If we have mismatched casts, treat the zext of a non-negative source as 4803 // a sext to simulate matching casts. Otherwise, we are done. 4804 // TODO: Can we handle some predicates (equality) without non-negative? 4805 if (IsZext0 != IsZext1) { 4806 if ((IsZext0 && isKnownNonNegative(X, DL, 0, &AC, &ICmp, &DT)) || 4807 (IsZext1 && isKnownNonNegative(Y, DL, 0, &AC, &ICmp, &DT))) 4808 IsSignedExt = true; 4809 else 4810 return nullptr; 4811 } 4812 4813 // Not an extension from the same type? 4814 Type *XTy = X->getType(), *YTy = Y->getType(); 4815 if (XTy != YTy) { 4816 // One of the casts must have one use because we are creating a new cast. 4817 if (!ICmp.getOperand(0)->hasOneUse() && !ICmp.getOperand(1)->hasOneUse()) 4818 return nullptr; 4819 // Extend the narrower operand to the type of the wider operand. 4820 CastInst::CastOps CastOpcode = 4821 IsSignedExt ? Instruction::SExt : Instruction::ZExt; 4822 if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits()) 4823 X = Builder.CreateCast(CastOpcode, X, YTy); 4824 else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits()) 4825 Y = Builder.CreateCast(CastOpcode, Y, XTy); 4826 else 4827 return nullptr; 4828 } 4829 4830 // (zext X) == (zext Y) --> X == Y 4831 // (sext X) == (sext Y) --> X == Y 4832 if (ICmp.isEquality()) 4833 return new ICmpInst(ICmp.getPredicate(), X, Y); 4834 4835 // A signed comparison of sign extended values simplifies into a 4836 // signed comparison. 4837 if (IsSignedCmp && IsSignedExt) 4838 return new ICmpInst(ICmp.getPredicate(), X, Y); 4839 4840 // The other three cases all fold into an unsigned comparison. 4841 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y); 4842 } 4843 4844 // Below here, we are only folding a compare with constant. 4845 auto *C = dyn_cast<Constant>(ICmp.getOperand(1)); 4846 if (!C) 4847 return nullptr; 4848 4849 // Compute the constant that would happen if we truncated to SrcTy then 4850 // re-extended to DestTy. 4851 Type *SrcTy = CastOp0->getSrcTy(); 4852 Type *DestTy = CastOp0->getDestTy(); 4853 Constant *Res1 = ConstantExpr::getTrunc(C, SrcTy); 4854 Constant *Res2 = ConstantExpr::getCast(CastOp0->getOpcode(), Res1, DestTy); 4855 4856 // If the re-extended constant didn't change... 4857 if (Res2 == C) { 4858 if (ICmp.isEquality()) 4859 return new ICmpInst(ICmp.getPredicate(), X, Res1); 4860 4861 // A signed comparison of sign extended values simplifies into a 4862 // signed comparison. 4863 if (IsSignedExt && IsSignedCmp) 4864 return new ICmpInst(ICmp.getPredicate(), X, Res1); 4865 4866 // The other three cases all fold into an unsigned comparison. 4867 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res1); 4868 } 4869 4870 // The re-extended constant changed, partly changed (in the case of a vector), 4871 // or could not be determined to be equal (in the case of a constant 4872 // expression), so the constant cannot be represented in the shorter type. 4873 // All the cases that fold to true or false will have already been handled 4874 // by simplifyICmpInst, so only deal with the tricky case. 4875 if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(C)) 4876 return nullptr; 4877 4878 // Is source op positive? 4879 // icmp ult (sext X), C --> icmp sgt X, -1 4880 if (ICmp.getPredicate() == ICmpInst::ICMP_ULT) 4881 return new ICmpInst(CmpInst::ICMP_SGT, X, Constant::getAllOnesValue(SrcTy)); 4882 4883 // Is source op negative? 4884 // icmp ugt (sext X), C --> icmp slt X, 0 4885 assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!"); 4886 return new ICmpInst(CmpInst::ICMP_SLT, X, Constant::getNullValue(SrcTy)); 4887 } 4888 4889 /// Handle icmp (cast x), (cast or constant). 4890 Instruction *InstCombinerImpl::foldICmpWithCastOp(ICmpInst &ICmp) { 4891 // If any operand of ICmp is a inttoptr roundtrip cast then remove it as 4892 // icmp compares only pointer's value. 4893 // icmp (inttoptr (ptrtoint p1)), p2 --> icmp p1, p2. 4894 Value *SimplifiedOp0 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(0)); 4895 Value *SimplifiedOp1 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(1)); 4896 if (SimplifiedOp0 || SimplifiedOp1) 4897 return new ICmpInst(ICmp.getPredicate(), 4898 SimplifiedOp0 ? SimplifiedOp0 : ICmp.getOperand(0), 4899 SimplifiedOp1 ? SimplifiedOp1 : ICmp.getOperand(1)); 4900 4901 auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0)); 4902 if (!CastOp0) 4903 return nullptr; 4904 if (!isa<Constant>(ICmp.getOperand(1)) && !isa<CastInst>(ICmp.getOperand(1))) 4905 return nullptr; 4906 4907 Value *Op0Src = CastOp0->getOperand(0); 4908 Type *SrcTy = CastOp0->getSrcTy(); 4909 Type *DestTy = CastOp0->getDestTy(); 4910 4911 // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the 4912 // integer type is the same size as the pointer type. 4913 auto CompatibleSizes = [&](Type *SrcTy, Type *DestTy) { 4914 if (isa<VectorType>(SrcTy)) { 4915 SrcTy = cast<VectorType>(SrcTy)->getElementType(); 4916 DestTy = cast<VectorType>(DestTy)->getElementType(); 4917 } 4918 return DL.getPointerTypeSizeInBits(SrcTy) == DestTy->getIntegerBitWidth(); 4919 }; 4920 if (CastOp0->getOpcode() == Instruction::PtrToInt && 4921 CompatibleSizes(SrcTy, DestTy)) { 4922 Value *NewOp1 = nullptr; 4923 if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(ICmp.getOperand(1))) { 4924 Value *PtrSrc = PtrToIntOp1->getOperand(0); 4925 if (PtrSrc->getType()->getPointerAddressSpace() == 4926 Op0Src->getType()->getPointerAddressSpace()) { 4927 NewOp1 = PtrToIntOp1->getOperand(0); 4928 // If the pointer types don't match, insert a bitcast. 4929 if (Op0Src->getType() != NewOp1->getType()) 4930 NewOp1 = Builder.CreateBitCast(NewOp1, Op0Src->getType()); 4931 } 4932 } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) { 4933 NewOp1 = ConstantExpr::getIntToPtr(RHSC, SrcTy); 4934 } 4935 4936 if (NewOp1) 4937 return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1); 4938 } 4939 4940 if (Instruction *R = foldICmpWithTrunc(ICmp, Builder)) 4941 return R; 4942 4943 return foldICmpWithZextOrSext(ICmp); 4944 } 4945 4946 static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS) { 4947 switch (BinaryOp) { 4948 default: 4949 llvm_unreachable("Unsupported binary op"); 4950 case Instruction::Add: 4951 case Instruction::Sub: 4952 return match(RHS, m_Zero()); 4953 case Instruction::Mul: 4954 return match(RHS, m_One()); 4955 } 4956 } 4957 4958 OverflowResult 4959 InstCombinerImpl::computeOverflow(Instruction::BinaryOps BinaryOp, 4960 bool IsSigned, Value *LHS, Value *RHS, 4961 Instruction *CxtI) const { 4962 switch (BinaryOp) { 4963 default: 4964 llvm_unreachable("Unsupported binary op"); 4965 case Instruction::Add: 4966 if (IsSigned) 4967 return computeOverflowForSignedAdd(LHS, RHS, CxtI); 4968 else 4969 return computeOverflowForUnsignedAdd(LHS, RHS, CxtI); 4970 case Instruction::Sub: 4971 if (IsSigned) 4972 return computeOverflowForSignedSub(LHS, RHS, CxtI); 4973 else 4974 return computeOverflowForUnsignedSub(LHS, RHS, CxtI); 4975 case Instruction::Mul: 4976 if (IsSigned) 4977 return computeOverflowForSignedMul(LHS, RHS, CxtI); 4978 else 4979 return computeOverflowForUnsignedMul(LHS, RHS, CxtI); 4980 } 4981 } 4982 4983 bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp, 4984 bool IsSigned, Value *LHS, 4985 Value *RHS, Instruction &OrigI, 4986 Value *&Result, 4987 Constant *&Overflow) { 4988 if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS)) 4989 std::swap(LHS, RHS); 4990 4991 // If the overflow check was an add followed by a compare, the insertion point 4992 // may be pointing to the compare. We want to insert the new instructions 4993 // before the add in case there are uses of the add between the add and the 4994 // compare. 4995 Builder.SetInsertPoint(&OrigI); 4996 4997 Type *OverflowTy = Type::getInt1Ty(LHS->getContext()); 4998 if (auto *LHSTy = dyn_cast<VectorType>(LHS->getType())) 4999 OverflowTy = VectorType::get(OverflowTy, LHSTy->getElementCount()); 5000 5001 if (isNeutralValue(BinaryOp, RHS)) { 5002 Result = LHS; 5003 Overflow = ConstantInt::getFalse(OverflowTy); 5004 return true; 5005 } 5006 5007 switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) { 5008 case OverflowResult::MayOverflow: 5009 return false; 5010 case OverflowResult::AlwaysOverflowsLow: 5011 case OverflowResult::AlwaysOverflowsHigh: 5012 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS); 5013 Result->takeName(&OrigI); 5014 Overflow = ConstantInt::getTrue(OverflowTy); 5015 return true; 5016 case OverflowResult::NeverOverflows: 5017 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS); 5018 Result->takeName(&OrigI); 5019 Overflow = ConstantInt::getFalse(OverflowTy); 5020 if (auto *Inst = dyn_cast<Instruction>(Result)) { 5021 if (IsSigned) 5022 Inst->setHasNoSignedWrap(); 5023 else 5024 Inst->setHasNoUnsignedWrap(); 5025 } 5026 return true; 5027 } 5028 5029 llvm_unreachable("Unexpected overflow result"); 5030 } 5031 5032 /// Recognize and process idiom involving test for multiplication 5033 /// overflow. 5034 /// 5035 /// The caller has matched a pattern of the form: 5036 /// I = cmp u (mul(zext A, zext B), V 5037 /// The function checks if this is a test for overflow and if so replaces 5038 /// multiplication with call to 'mul.with.overflow' intrinsic. 5039 /// 5040 /// \param I Compare instruction. 5041 /// \param MulVal Result of 'mult' instruction. It is one of the arguments of 5042 /// the compare instruction. Must be of integer type. 5043 /// \param OtherVal The other argument of compare instruction. 5044 /// \returns Instruction which must replace the compare instruction, NULL if no 5045 /// replacement required. 5046 static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal, 5047 Value *OtherVal, 5048 InstCombinerImpl &IC) { 5049 // Don't bother doing this transformation for pointers, don't do it for 5050 // vectors. 5051 if (!isa<IntegerType>(MulVal->getType())) 5052 return nullptr; 5053 5054 assert(I.getOperand(0) == MulVal || I.getOperand(1) == MulVal); 5055 assert(I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal); 5056 auto *MulInstr = dyn_cast<Instruction>(MulVal); 5057 if (!MulInstr) 5058 return nullptr; 5059 assert(MulInstr->getOpcode() == Instruction::Mul); 5060 5061 auto *LHS = cast<ZExtOperator>(MulInstr->getOperand(0)), 5062 *RHS = cast<ZExtOperator>(MulInstr->getOperand(1)); 5063 assert(LHS->getOpcode() == Instruction::ZExt); 5064 assert(RHS->getOpcode() == Instruction::ZExt); 5065 Value *A = LHS->getOperand(0), *B = RHS->getOperand(0); 5066 5067 // Calculate type and width of the result produced by mul.with.overflow. 5068 Type *TyA = A->getType(), *TyB = B->getType(); 5069 unsigned WidthA = TyA->getPrimitiveSizeInBits(), 5070 WidthB = TyB->getPrimitiveSizeInBits(); 5071 unsigned MulWidth; 5072 Type *MulType; 5073 if (WidthB > WidthA) { 5074 MulWidth = WidthB; 5075 MulType = TyB; 5076 } else { 5077 MulWidth = WidthA; 5078 MulType = TyA; 5079 } 5080 5081 // In order to replace the original mul with a narrower mul.with.overflow, 5082 // all uses must ignore upper bits of the product. The number of used low 5083 // bits must be not greater than the width of mul.with.overflow. 5084 if (MulVal->hasNUsesOrMore(2)) 5085 for (User *U : MulVal->users()) { 5086 if (U == &I) 5087 continue; 5088 if (TruncInst *TI = dyn_cast<TruncInst>(U)) { 5089 // Check if truncation ignores bits above MulWidth. 5090 unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits(); 5091 if (TruncWidth > MulWidth) 5092 return nullptr; 5093 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { 5094 // Check if AND ignores bits above MulWidth. 5095 if (BO->getOpcode() != Instruction::And) 5096 return nullptr; 5097 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) { 5098 const APInt &CVal = CI->getValue(); 5099 if (CVal.getBitWidth() - CVal.countLeadingZeros() > MulWidth) 5100 return nullptr; 5101 } else { 5102 // In this case we could have the operand of the binary operation 5103 // being defined in another block, and performing the replacement 5104 // could break the dominance relation. 5105 return nullptr; 5106 } 5107 } else { 5108 // Other uses prohibit this transformation. 5109 return nullptr; 5110 } 5111 } 5112 5113 // Recognize patterns 5114 switch (I.getPredicate()) { 5115 case ICmpInst::ICMP_EQ: 5116 case ICmpInst::ICMP_NE: 5117 // Recognize pattern: 5118 // mulval = mul(zext A, zext B) 5119 // cmp eq/neq mulval, and(mulval, mask), mask selects low MulWidth bits. 5120 ConstantInt *CI; 5121 Value *ValToMask; 5122 if (match(OtherVal, m_And(m_Value(ValToMask), m_ConstantInt(CI)))) { 5123 if (ValToMask != MulVal) 5124 return nullptr; 5125 const APInt &CVal = CI->getValue() + 1; 5126 if (CVal.isPowerOf2()) { 5127 unsigned MaskWidth = CVal.logBase2(); 5128 if (MaskWidth == MulWidth) 5129 break; // Recognized 5130 } 5131 } 5132 return nullptr; 5133 5134 case ICmpInst::ICMP_UGT: 5135 // Recognize pattern: 5136 // mulval = mul(zext A, zext B) 5137 // cmp ugt mulval, max 5138 if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { 5139 APInt MaxVal = APInt::getMaxValue(MulWidth); 5140 MaxVal = MaxVal.zext(CI->getBitWidth()); 5141 if (MaxVal.eq(CI->getValue())) 5142 break; // Recognized 5143 } 5144 return nullptr; 5145 5146 case ICmpInst::ICMP_UGE: 5147 // Recognize pattern: 5148 // mulval = mul(zext A, zext B) 5149 // cmp uge mulval, max+1 5150 if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { 5151 APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); 5152 if (MaxVal.eq(CI->getValue())) 5153 break; // Recognized 5154 } 5155 return nullptr; 5156 5157 case ICmpInst::ICMP_ULE: 5158 // Recognize pattern: 5159 // mulval = mul(zext A, zext B) 5160 // cmp ule mulval, max 5161 if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { 5162 APInt MaxVal = APInt::getMaxValue(MulWidth); 5163 MaxVal = MaxVal.zext(CI->getBitWidth()); 5164 if (MaxVal.eq(CI->getValue())) 5165 break; // Recognized 5166 } 5167 return nullptr; 5168 5169 case ICmpInst::ICMP_ULT: 5170 // Recognize pattern: 5171 // mulval = mul(zext A, zext B) 5172 // cmp ule mulval, max + 1 5173 if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { 5174 APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); 5175 if (MaxVal.eq(CI->getValue())) 5176 break; // Recognized 5177 } 5178 return nullptr; 5179 5180 default: 5181 return nullptr; 5182 } 5183 5184 InstCombiner::BuilderTy &Builder = IC.Builder; 5185 Builder.SetInsertPoint(MulInstr); 5186 5187 // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B) 5188 Value *MulA = A, *MulB = B; 5189 if (WidthA < MulWidth) 5190 MulA = Builder.CreateZExt(A, MulType); 5191 if (WidthB < MulWidth) 5192 MulB = Builder.CreateZExt(B, MulType); 5193 Function *F = Intrinsic::getDeclaration( 5194 I.getModule(), Intrinsic::umul_with_overflow, MulType); 5195 CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul"); 5196 IC.addToWorklist(MulInstr); 5197 5198 // If there are uses of mul result other than the comparison, we know that 5199 // they are truncation or binary AND. Change them to use result of 5200 // mul.with.overflow and adjust properly mask/size. 5201 if (MulVal->hasNUsesOrMore(2)) { 5202 Value *Mul = Builder.CreateExtractValue(Call, 0, "umul.value"); 5203 for (User *U : make_early_inc_range(MulVal->users())) { 5204 if (U == &I || U == OtherVal) 5205 continue; 5206 if (TruncInst *TI = dyn_cast<TruncInst>(U)) { 5207 if (TI->getType()->getPrimitiveSizeInBits() == MulWidth) 5208 IC.replaceInstUsesWith(*TI, Mul); 5209 else 5210 TI->setOperand(0, Mul); 5211 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { 5212 assert(BO->getOpcode() == Instruction::And); 5213 // Replace (mul & mask) --> zext (mul.with.overflow & short_mask) 5214 ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1)); 5215 APInt ShortMask = CI->getValue().trunc(MulWidth); 5216 Value *ShortAnd = Builder.CreateAnd(Mul, ShortMask); 5217 Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType()); 5218 IC.replaceInstUsesWith(*BO, Zext); 5219 } else { 5220 llvm_unreachable("Unexpected Binary operation"); 5221 } 5222 IC.addToWorklist(cast<Instruction>(U)); 5223 } 5224 } 5225 if (isa<Instruction>(OtherVal)) 5226 IC.addToWorklist(cast<Instruction>(OtherVal)); 5227 5228 // The original icmp gets replaced with the overflow value, maybe inverted 5229 // depending on predicate. 5230 bool Inverse = false; 5231 switch (I.getPredicate()) { 5232 case ICmpInst::ICMP_NE: 5233 break; 5234 case ICmpInst::ICMP_EQ: 5235 Inverse = true; 5236 break; 5237 case ICmpInst::ICMP_UGT: 5238 case ICmpInst::ICMP_UGE: 5239 if (I.getOperand(0) == MulVal) 5240 break; 5241 Inverse = true; 5242 break; 5243 case ICmpInst::ICMP_ULT: 5244 case ICmpInst::ICMP_ULE: 5245 if (I.getOperand(1) == MulVal) 5246 break; 5247 Inverse = true; 5248 break; 5249 default: 5250 llvm_unreachable("Unexpected predicate"); 5251 } 5252 if (Inverse) { 5253 Value *Res = Builder.CreateExtractValue(Call, 1); 5254 return BinaryOperator::CreateNot(Res); 5255 } 5256 5257 return ExtractValueInst::Create(Call, 1); 5258 } 5259 5260 /// When performing a comparison against a constant, it is possible that not all 5261 /// the bits in the LHS are demanded. This helper method computes the mask that 5262 /// IS demanded. 5263 static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth) { 5264 const APInt *RHS; 5265 if (!match(I.getOperand(1), m_APInt(RHS))) 5266 return APInt::getAllOnes(BitWidth); 5267 5268 // If this is a normal comparison, it demands all bits. If it is a sign bit 5269 // comparison, it only demands the sign bit. 5270 bool UnusedBit; 5271 if (InstCombiner::isSignBitCheck(I.getPredicate(), *RHS, UnusedBit)) 5272 return APInt::getSignMask(BitWidth); 5273 5274 switch (I.getPredicate()) { 5275 // For a UGT comparison, we don't care about any bits that 5276 // correspond to the trailing ones of the comparand. The value of these 5277 // bits doesn't impact the outcome of the comparison, because any value 5278 // greater than the RHS must differ in a bit higher than these due to carry. 5279 case ICmpInst::ICMP_UGT: 5280 return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingOnes()); 5281 5282 // Similarly, for a ULT comparison, we don't care about the trailing zeros. 5283 // Any value less than the RHS must differ in a higher bit because of carries. 5284 case ICmpInst::ICMP_ULT: 5285 return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingZeros()); 5286 5287 default: 5288 return APInt::getAllOnes(BitWidth); 5289 } 5290 } 5291 5292 /// Check if the order of \p Op0 and \p Op1 as operands in an ICmpInst 5293 /// should be swapped. 5294 /// The decision is based on how many times these two operands are reused 5295 /// as subtract operands and their positions in those instructions. 5296 /// The rationale is that several architectures use the same instruction for 5297 /// both subtract and cmp. Thus, it is better if the order of those operands 5298 /// match. 5299 /// \return true if Op0 and Op1 should be swapped. 5300 static bool swapMayExposeCSEOpportunities(const Value *Op0, const Value *Op1) { 5301 // Filter out pointer values as those cannot appear directly in subtract. 5302 // FIXME: we may want to go through inttoptrs or bitcasts. 5303 if (Op0->getType()->isPointerTy()) 5304 return false; 5305 // If a subtract already has the same operands as a compare, swapping would be 5306 // bad. If a subtract has the same operands as a compare but in reverse order, 5307 // then swapping is good. 5308 int GoodToSwap = 0; 5309 for (const User *U : Op0->users()) { 5310 if (match(U, m_Sub(m_Specific(Op1), m_Specific(Op0)))) 5311 GoodToSwap++; 5312 else if (match(U, m_Sub(m_Specific(Op0), m_Specific(Op1)))) 5313 GoodToSwap--; 5314 } 5315 return GoodToSwap > 0; 5316 } 5317 5318 /// Check that one use is in the same block as the definition and all 5319 /// other uses are in blocks dominated by a given block. 5320 /// 5321 /// \param DI Definition 5322 /// \param UI Use 5323 /// \param DB Block that must dominate all uses of \p DI outside 5324 /// the parent block 5325 /// \return true when \p UI is the only use of \p DI in the parent block 5326 /// and all other uses of \p DI are in blocks dominated by \p DB. 5327 /// 5328 bool InstCombinerImpl::dominatesAllUses(const Instruction *DI, 5329 const Instruction *UI, 5330 const BasicBlock *DB) const { 5331 assert(DI && UI && "Instruction not defined\n"); 5332 // Ignore incomplete definitions. 5333 if (!DI->getParent()) 5334 return false; 5335 // DI and UI must be in the same block. 5336 if (DI->getParent() != UI->getParent()) 5337 return false; 5338 // Protect from self-referencing blocks. 5339 if (DI->getParent() == DB) 5340 return false; 5341 for (const User *U : DI->users()) { 5342 auto *Usr = cast<Instruction>(U); 5343 if (Usr != UI && !DT.dominates(DB, Usr->getParent())) 5344 return false; 5345 } 5346 return true; 5347 } 5348 5349 /// Return true when the instruction sequence within a block is select-cmp-br. 5350 static bool isChainSelectCmpBranch(const SelectInst *SI) { 5351 const BasicBlock *BB = SI->getParent(); 5352 if (!BB) 5353 return false; 5354 auto *BI = dyn_cast_or_null<BranchInst>(BB->getTerminator()); 5355 if (!BI || BI->getNumSuccessors() != 2) 5356 return false; 5357 auto *IC = dyn_cast<ICmpInst>(BI->getCondition()); 5358 if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI)) 5359 return false; 5360 return true; 5361 } 5362 5363 /// True when a select result is replaced by one of its operands 5364 /// in select-icmp sequence. This will eventually result in the elimination 5365 /// of the select. 5366 /// 5367 /// \param SI Select instruction 5368 /// \param Icmp Compare instruction 5369 /// \param SIOpd Operand that replaces the select 5370 /// 5371 /// Notes: 5372 /// - The replacement is global and requires dominator information 5373 /// - The caller is responsible for the actual replacement 5374 /// 5375 /// Example: 5376 /// 5377 /// entry: 5378 /// %4 = select i1 %3, %C* %0, %C* null 5379 /// %5 = icmp eq %C* %4, null 5380 /// br i1 %5, label %9, label %7 5381 /// ... 5382 /// ; <label>:7 ; preds = %entry 5383 /// %8 = getelementptr inbounds %C* %4, i64 0, i32 0 5384 /// ... 5385 /// 5386 /// can be transformed to 5387 /// 5388 /// %5 = icmp eq %C* %0, null 5389 /// %6 = select i1 %3, i1 %5, i1 true 5390 /// br i1 %6, label %9, label %7 5391 /// ... 5392 /// ; <label>:7 ; preds = %entry 5393 /// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0! 5394 /// 5395 /// Similar when the first operand of the select is a constant or/and 5396 /// the compare is for not equal rather than equal. 5397 /// 5398 /// NOTE: The function is only called when the select and compare constants 5399 /// are equal, the optimization can work only for EQ predicates. This is not a 5400 /// major restriction since a NE compare should be 'normalized' to an equal 5401 /// compare, which usually happens in the combiner and test case 5402 /// select-cmp-br.ll checks for it. 5403 bool InstCombinerImpl::replacedSelectWithOperand(SelectInst *SI, 5404 const ICmpInst *Icmp, 5405 const unsigned SIOpd) { 5406 assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!"); 5407 if (isChainSelectCmpBranch(SI) && Icmp->getPredicate() == ICmpInst::ICMP_EQ) { 5408 BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1); 5409 // The check for the single predecessor is not the best that can be 5410 // done. But it protects efficiently against cases like when SI's 5411 // home block has two successors, Succ and Succ1, and Succ1 predecessor 5412 // of Succ. Then SI can't be replaced by SIOpd because the use that gets 5413 // replaced can be reached on either path. So the uniqueness check 5414 // guarantees that the path all uses of SI (outside SI's parent) are on 5415 // is disjoint from all other paths out of SI. But that information 5416 // is more expensive to compute, and the trade-off here is in favor 5417 // of compile-time. It should also be noticed that we check for a single 5418 // predecessor and not only uniqueness. This to handle the situation when 5419 // Succ and Succ1 points to the same basic block. 5420 if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) { 5421 NumSel++; 5422 SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent()); 5423 return true; 5424 } 5425 } 5426 return false; 5427 } 5428 5429 /// Try to fold the comparison based on range information we can get by checking 5430 /// whether bits are known to be zero or one in the inputs. 5431 Instruction *InstCombinerImpl::foldICmpUsingKnownBits(ICmpInst &I) { 5432 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 5433 Type *Ty = Op0->getType(); 5434 ICmpInst::Predicate Pred = I.getPredicate(); 5435 5436 // Get scalar or pointer size. 5437 unsigned BitWidth = Ty->isIntOrIntVectorTy() 5438 ? Ty->getScalarSizeInBits() 5439 : DL.getPointerTypeSizeInBits(Ty->getScalarType()); 5440 5441 if (!BitWidth) 5442 return nullptr; 5443 5444 KnownBits Op0Known(BitWidth); 5445 KnownBits Op1Known(BitWidth); 5446 5447 if (SimplifyDemandedBits(&I, 0, 5448 getDemandedBitsLHSMask(I, BitWidth), 5449 Op0Known, 0)) 5450 return &I; 5451 5452 if (SimplifyDemandedBits(&I, 1, APInt::getAllOnes(BitWidth), Op1Known, 0)) 5453 return &I; 5454 5455 // Given the known and unknown bits, compute a range that the LHS could be 5456 // in. Compute the Min, Max and RHS values based on the known bits. For the 5457 // EQ and NE we use unsigned values. 5458 APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0); 5459 APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0); 5460 if (I.isSigned()) { 5461 Op0Min = Op0Known.getSignedMinValue(); 5462 Op0Max = Op0Known.getSignedMaxValue(); 5463 Op1Min = Op1Known.getSignedMinValue(); 5464 Op1Max = Op1Known.getSignedMaxValue(); 5465 } else { 5466 Op0Min = Op0Known.getMinValue(); 5467 Op0Max = Op0Known.getMaxValue(); 5468 Op1Min = Op1Known.getMinValue(); 5469 Op1Max = Op1Known.getMaxValue(); 5470 } 5471 5472 // If Min and Max are known to be the same, then SimplifyDemandedBits figured 5473 // out that the LHS or RHS is a constant. Constant fold this now, so that 5474 // code below can assume that Min != Max. 5475 if (!isa<Constant>(Op0) && Op0Min == Op0Max) 5476 return new ICmpInst(Pred, ConstantExpr::getIntegerValue(Ty, Op0Min), Op1); 5477 if (!isa<Constant>(Op1) && Op1Min == Op1Max) 5478 return new ICmpInst(Pred, Op0, ConstantExpr::getIntegerValue(Ty, Op1Min)); 5479 5480 // Don't break up a clamp pattern -- (min(max X, Y), Z) -- by replacing a 5481 // min/max canonical compare with some other compare. That could lead to 5482 // conflict with select canonicalization and infinite looping. 5483 // FIXME: This constraint may go away if min/max intrinsics are canonical. 5484 auto isMinMaxCmp = [&](Instruction &Cmp) { 5485 if (!Cmp.hasOneUse()) 5486 return false; 5487 Value *A, *B; 5488 SelectPatternFlavor SPF = matchSelectPattern(Cmp.user_back(), A, B).Flavor; 5489 if (!SelectPatternResult::isMinOrMax(SPF)) 5490 return false; 5491 return match(Op0, m_MaxOrMin(m_Value(), m_Value())) || 5492 match(Op1, m_MaxOrMin(m_Value(), m_Value())); 5493 }; 5494 if (!isMinMaxCmp(I)) { 5495 switch (Pred) { 5496 default: 5497 break; 5498 case ICmpInst::ICMP_ULT: { 5499 if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B) 5500 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); 5501 const APInt *CmpC; 5502 if (match(Op1, m_APInt(CmpC))) { 5503 // A <u C -> A == C-1 if min(A)+1 == C 5504 if (*CmpC == Op0Min + 1) 5505 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, 5506 ConstantInt::get(Op1->getType(), *CmpC - 1)); 5507 // X <u C --> X == 0, if the number of zero bits in the bottom of X 5508 // exceeds the log2 of C. 5509 if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2()) 5510 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, 5511 Constant::getNullValue(Op1->getType())); 5512 } 5513 break; 5514 } 5515 case ICmpInst::ICMP_UGT: { 5516 if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B) 5517 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); 5518 const APInt *CmpC; 5519 if (match(Op1, m_APInt(CmpC))) { 5520 // A >u C -> A == C+1 if max(a)-1 == C 5521 if (*CmpC == Op0Max - 1) 5522 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, 5523 ConstantInt::get(Op1->getType(), *CmpC + 1)); 5524 // X >u C --> X != 0, if the number of zero bits in the bottom of X 5525 // exceeds the log2 of C. 5526 if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits()) 5527 return new ICmpInst(ICmpInst::ICMP_NE, Op0, 5528 Constant::getNullValue(Op1->getType())); 5529 } 5530 break; 5531 } 5532 case ICmpInst::ICMP_SLT: { 5533 if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B) 5534 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); 5535 const APInt *CmpC; 5536 if (match(Op1, m_APInt(CmpC))) { 5537 if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C 5538 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, 5539 ConstantInt::get(Op1->getType(), *CmpC - 1)); 5540 } 5541 break; 5542 } 5543 case ICmpInst::ICMP_SGT: { 5544 if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B) 5545 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); 5546 const APInt *CmpC; 5547 if (match(Op1, m_APInt(CmpC))) { 5548 if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C 5549 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, 5550 ConstantInt::get(Op1->getType(), *CmpC + 1)); 5551 } 5552 break; 5553 } 5554 } 5555 } 5556 5557 // Based on the range information we know about the LHS, see if we can 5558 // simplify this comparison. For example, (x&4) < 8 is always true. 5559 switch (Pred) { 5560 default: 5561 llvm_unreachable("Unknown icmp opcode!"); 5562 case ICmpInst::ICMP_EQ: 5563 case ICmpInst::ICMP_NE: { 5564 if (Op0Max.ult(Op1Min) || Op0Min.ugt(Op1Max)) 5565 return replaceInstUsesWith( 5566 I, ConstantInt::getBool(I.getType(), Pred == CmpInst::ICMP_NE)); 5567 5568 // If all bits are known zero except for one, then we know at most one bit 5569 // is set. If the comparison is against zero, then this is a check to see if 5570 // *that* bit is set. 5571 APInt Op0KnownZeroInverted = ~Op0Known.Zero; 5572 if (Op1Known.isZero()) { 5573 // If the LHS is an AND with the same constant, look through it. 5574 Value *LHS = nullptr; 5575 const APInt *LHSC; 5576 if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) || 5577 *LHSC != Op0KnownZeroInverted) 5578 LHS = Op0; 5579 5580 Value *X; 5581 const APInt *C1; 5582 if (match(LHS, m_Shl(m_Power2(C1), m_Value(X)))) { 5583 Type *XTy = X->getType(); 5584 unsigned Log2C1 = C1->countTrailingZeros(); 5585 APInt C2 = Op0KnownZeroInverted; 5586 APInt C2Pow2 = (C2 & ~(*C1 - 1)) + *C1; 5587 if (C2Pow2.isPowerOf2()) { 5588 // iff (C1 is pow2) & ((C2 & ~(C1-1)) + C1) is pow2): 5589 // ((C1 << X) & C2) == 0 -> X >= (Log2(C2+C1) - Log2(C1)) 5590 // ((C1 << X) & C2) != 0 -> X < (Log2(C2+C1) - Log2(C1)) 5591 unsigned Log2C2 = C2Pow2.countTrailingZeros(); 5592 auto *CmpC = ConstantInt::get(XTy, Log2C2 - Log2C1); 5593 auto NewPred = 5594 Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT; 5595 return new ICmpInst(NewPred, X, CmpC); 5596 } 5597 } 5598 } 5599 break; 5600 } 5601 case ICmpInst::ICMP_ULT: { 5602 if (Op0Max.ult(Op1Min)) // A <u B -> true if max(A) < min(B) 5603 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5604 if (Op0Min.uge(Op1Max)) // A <u B -> false if min(A) >= max(B) 5605 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5606 break; 5607 } 5608 case ICmpInst::ICMP_UGT: { 5609 if (Op0Min.ugt(Op1Max)) // A >u B -> true if min(A) > max(B) 5610 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5611 if (Op0Max.ule(Op1Min)) // A >u B -> false if max(A) <= max(B) 5612 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5613 break; 5614 } 5615 case ICmpInst::ICMP_SLT: { 5616 if (Op0Max.slt(Op1Min)) // A <s B -> true if max(A) < min(C) 5617 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5618 if (Op0Min.sge(Op1Max)) // A <s B -> false if min(A) >= max(C) 5619 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5620 break; 5621 } 5622 case ICmpInst::ICMP_SGT: { 5623 if (Op0Min.sgt(Op1Max)) // A >s B -> true if min(A) > max(B) 5624 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5625 if (Op0Max.sle(Op1Min)) // A >s B -> false if max(A) <= min(B) 5626 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5627 break; 5628 } 5629 case ICmpInst::ICMP_SGE: 5630 assert(!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!"); 5631 if (Op0Min.sge(Op1Max)) // A >=s B -> true if min(A) >= max(B) 5632 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5633 if (Op0Max.slt(Op1Min)) // A >=s B -> false if max(A) < min(B) 5634 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5635 if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B) 5636 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); 5637 break; 5638 case ICmpInst::ICMP_SLE: 5639 assert(!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!"); 5640 if (Op0Max.sle(Op1Min)) // A <=s B -> true if max(A) <= min(B) 5641 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5642 if (Op0Min.sgt(Op1Max)) // A <=s B -> false if min(A) > max(B) 5643 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5644 if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B) 5645 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); 5646 break; 5647 case ICmpInst::ICMP_UGE: 5648 assert(!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!"); 5649 if (Op0Min.uge(Op1Max)) // A >=u B -> true if min(A) >= max(B) 5650 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5651 if (Op0Max.ult(Op1Min)) // A >=u B -> false if max(A) < min(B) 5652 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5653 if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B) 5654 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); 5655 break; 5656 case ICmpInst::ICMP_ULE: 5657 assert(!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!"); 5658 if (Op0Max.ule(Op1Min)) // A <=u B -> true if max(A) <= min(B) 5659 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5660 if (Op0Min.ugt(Op1Max)) // A <=u B -> false if min(A) > max(B) 5661 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5662 if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B) 5663 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); 5664 break; 5665 } 5666 5667 // Turn a signed comparison into an unsigned one if both operands are known to 5668 // have the same sign. 5669 if (I.isSigned() && 5670 ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) || 5671 (Op0Known.One.isNegative() && Op1Known.One.isNegative()))) 5672 return new ICmpInst(I.getUnsignedPredicate(), Op0, Op1); 5673 5674 return nullptr; 5675 } 5676 5677 /// If one operand of an icmp is effectively a bool (value range of {0,1}), 5678 /// then try to reduce patterns based on that limit. 5679 static Instruction *foldICmpUsingBoolRange(ICmpInst &I, 5680 InstCombiner::BuilderTy &Builder) { 5681 Value *X, *Y; 5682 ICmpInst::Predicate Pred; 5683 5684 // X must be 0 and bool must be true for "ULT": 5685 // X <u (zext i1 Y) --> (X == 0) & Y 5686 if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_ZExt(m_Value(Y))))) && 5687 Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULT) 5688 return BinaryOperator::CreateAnd(Builder.CreateIsNull(X), Y); 5689 5690 // X must be 0 or bool must be true for "ULE": 5691 // X <=u (sext i1 Y) --> (X == 0) | Y 5692 if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_SExt(m_Value(Y))))) && 5693 Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULE) 5694 return BinaryOperator::CreateOr(Builder.CreateIsNull(X), Y); 5695 5696 return nullptr; 5697 } 5698 5699 llvm::Optional<std::pair<CmpInst::Predicate, Constant *>> 5700 InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred, 5701 Constant *C) { 5702 assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && 5703 "Only for relational integer predicates."); 5704 5705 Type *Type = C->getType(); 5706 bool IsSigned = ICmpInst::isSigned(Pred); 5707 5708 CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred); 5709 bool WillIncrement = 5710 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT; 5711 5712 // Check if the constant operand can be safely incremented/decremented 5713 // without overflowing/underflowing. 5714 auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) { 5715 return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned); 5716 }; 5717 5718 Constant *SafeReplacementConstant = nullptr; 5719 if (auto *CI = dyn_cast<ConstantInt>(C)) { 5720 // Bail out if the constant can't be safely incremented/decremented. 5721 if (!ConstantIsOk(CI)) 5722 return llvm::None; 5723 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Type)) { 5724 unsigned NumElts = FVTy->getNumElements(); 5725 for (unsigned i = 0; i != NumElts; ++i) { 5726 Constant *Elt = C->getAggregateElement(i); 5727 if (!Elt) 5728 return llvm::None; 5729 5730 if (isa<UndefValue>(Elt)) 5731 continue; 5732 5733 // Bail out if we can't determine if this constant is min/max or if we 5734 // know that this constant is min/max. 5735 auto *CI = dyn_cast<ConstantInt>(Elt); 5736 if (!CI || !ConstantIsOk(CI)) 5737 return llvm::None; 5738 5739 if (!SafeReplacementConstant) 5740 SafeReplacementConstant = CI; 5741 } 5742 } else { 5743 // ConstantExpr? 5744 return llvm::None; 5745 } 5746 5747 // It may not be safe to change a compare predicate in the presence of 5748 // undefined elements, so replace those elements with the first safe constant 5749 // that we found. 5750 // TODO: in case of poison, it is safe; let's replace undefs only. 5751 if (C->containsUndefOrPoisonElement()) { 5752 assert(SafeReplacementConstant && "Replacement constant not set"); 5753 C = Constant::replaceUndefsWith(C, SafeReplacementConstant); 5754 } 5755 5756 CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(Pred); 5757 5758 // Increment or decrement the constant. 5759 Constant *OneOrNegOne = ConstantInt::get(Type, WillIncrement ? 1 : -1, true); 5760 Constant *NewC = ConstantExpr::getAdd(C, OneOrNegOne); 5761 5762 return std::make_pair(NewPred, NewC); 5763 } 5764 5765 /// If we have an icmp le or icmp ge instruction with a constant operand, turn 5766 /// it into the appropriate icmp lt or icmp gt instruction. This transform 5767 /// allows them to be folded in visitICmpInst. 5768 static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) { 5769 ICmpInst::Predicate Pred = I.getPredicate(); 5770 if (ICmpInst::isEquality(Pred) || !ICmpInst::isIntPredicate(Pred) || 5771 InstCombiner::isCanonicalPredicate(Pred)) 5772 return nullptr; 5773 5774 Value *Op0 = I.getOperand(0); 5775 Value *Op1 = I.getOperand(1); 5776 auto *Op1C = dyn_cast<Constant>(Op1); 5777 if (!Op1C) 5778 return nullptr; 5779 5780 auto FlippedStrictness = 5781 InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, Op1C); 5782 if (!FlippedStrictness) 5783 return nullptr; 5784 5785 return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second); 5786 } 5787 5788 /// If we have a comparison with a non-canonical predicate, if we can update 5789 /// all the users, invert the predicate and adjust all the users. 5790 CmpInst *InstCombinerImpl::canonicalizeICmpPredicate(CmpInst &I) { 5791 // Is the predicate already canonical? 5792 CmpInst::Predicate Pred = I.getPredicate(); 5793 if (InstCombiner::isCanonicalPredicate(Pred)) 5794 return nullptr; 5795 5796 // Can all users be adjusted to predicate inversion? 5797 if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr)) 5798 return nullptr; 5799 5800 // Ok, we can canonicalize comparison! 5801 // Let's first invert the comparison's predicate. 5802 I.setPredicate(CmpInst::getInversePredicate(Pred)); 5803 I.setName(I.getName() + ".not"); 5804 5805 // And, adapt users. 5806 freelyInvertAllUsersOf(&I); 5807 5808 return &I; 5809 } 5810 5811 /// Integer compare with boolean values can always be turned into bitwise ops. 5812 static Instruction *canonicalizeICmpBool(ICmpInst &I, 5813 InstCombiner::BuilderTy &Builder) { 5814 Value *A = I.getOperand(0), *B = I.getOperand(1); 5815 assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only"); 5816 5817 // A boolean compared to true/false can be simplified to Op0/true/false in 5818 // 14 out of the 20 (10 predicates * 2 constants) possible combinations. 5819 // Cases not handled by InstSimplify are always 'not' of Op0. 5820 if (match(B, m_Zero())) { 5821 switch (I.getPredicate()) { 5822 case CmpInst::ICMP_EQ: // A == 0 -> !A 5823 case CmpInst::ICMP_ULE: // A <=u 0 -> !A 5824 case CmpInst::ICMP_SGE: // A >=s 0 -> !A 5825 return BinaryOperator::CreateNot(A); 5826 default: 5827 llvm_unreachable("ICmp i1 X, C not simplified as expected."); 5828 } 5829 } else if (match(B, m_One())) { 5830 switch (I.getPredicate()) { 5831 case CmpInst::ICMP_NE: // A != 1 -> !A 5832 case CmpInst::ICMP_ULT: // A <u 1 -> !A 5833 case CmpInst::ICMP_SGT: // A >s -1 -> !A 5834 return BinaryOperator::CreateNot(A); 5835 default: 5836 llvm_unreachable("ICmp i1 X, C not simplified as expected."); 5837 } 5838 } 5839 5840 switch (I.getPredicate()) { 5841 default: 5842 llvm_unreachable("Invalid icmp instruction!"); 5843 case ICmpInst::ICMP_EQ: 5844 // icmp eq i1 A, B -> ~(A ^ B) 5845 return BinaryOperator::CreateNot(Builder.CreateXor(A, B)); 5846 5847 case ICmpInst::ICMP_NE: 5848 // icmp ne i1 A, B -> A ^ B 5849 return BinaryOperator::CreateXor(A, B); 5850 5851 case ICmpInst::ICMP_UGT: 5852 // icmp ugt -> icmp ult 5853 std::swap(A, B); 5854 LLVM_FALLTHROUGH; 5855 case ICmpInst::ICMP_ULT: 5856 // icmp ult i1 A, B -> ~A & B 5857 return BinaryOperator::CreateAnd(Builder.CreateNot(A), B); 5858 5859 case ICmpInst::ICMP_SGT: 5860 // icmp sgt -> icmp slt 5861 std::swap(A, B); 5862 LLVM_FALLTHROUGH; 5863 case ICmpInst::ICMP_SLT: 5864 // icmp slt i1 A, B -> A & ~B 5865 return BinaryOperator::CreateAnd(Builder.CreateNot(B), A); 5866 5867 case ICmpInst::ICMP_UGE: 5868 // icmp uge -> icmp ule 5869 std::swap(A, B); 5870 LLVM_FALLTHROUGH; 5871 case ICmpInst::ICMP_ULE: 5872 // icmp ule i1 A, B -> ~A | B 5873 return BinaryOperator::CreateOr(Builder.CreateNot(A), B); 5874 5875 case ICmpInst::ICMP_SGE: 5876 // icmp sge -> icmp sle 5877 std::swap(A, B); 5878 LLVM_FALLTHROUGH; 5879 case ICmpInst::ICMP_SLE: 5880 // icmp sle i1 A, B -> A | ~B 5881 return BinaryOperator::CreateOr(Builder.CreateNot(B), A); 5882 } 5883 } 5884 5885 // Transform pattern like: 5886 // (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X 5887 // (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X 5888 // Into: 5889 // (X l>> Y) != 0 5890 // (X l>> Y) == 0 5891 static Instruction *foldICmpWithHighBitMask(ICmpInst &Cmp, 5892 InstCombiner::BuilderTy &Builder) { 5893 ICmpInst::Predicate Pred, NewPred; 5894 Value *X, *Y; 5895 if (match(&Cmp, 5896 m_c_ICmp(Pred, m_OneUse(m_Shl(m_One(), m_Value(Y))), m_Value(X)))) { 5897 switch (Pred) { 5898 case ICmpInst::ICMP_ULE: 5899 NewPred = ICmpInst::ICMP_NE; 5900 break; 5901 case ICmpInst::ICMP_UGT: 5902 NewPred = ICmpInst::ICMP_EQ; 5903 break; 5904 default: 5905 return nullptr; 5906 } 5907 } else if (match(&Cmp, m_c_ICmp(Pred, 5908 m_OneUse(m_CombineOr( 5909 m_Not(m_Shl(m_AllOnes(), m_Value(Y))), 5910 m_Add(m_Shl(m_One(), m_Value(Y)), 5911 m_AllOnes()))), 5912 m_Value(X)))) { 5913 // The variant with 'add' is not canonical, (the variant with 'not' is) 5914 // we only get it because it has extra uses, and can't be canonicalized, 5915 5916 switch (Pred) { 5917 case ICmpInst::ICMP_ULT: 5918 NewPred = ICmpInst::ICMP_NE; 5919 break; 5920 case ICmpInst::ICMP_UGE: 5921 NewPred = ICmpInst::ICMP_EQ; 5922 break; 5923 default: 5924 return nullptr; 5925 } 5926 } else 5927 return nullptr; 5928 5929 Value *NewX = Builder.CreateLShr(X, Y, X->getName() + ".highbits"); 5930 Constant *Zero = Constant::getNullValue(NewX->getType()); 5931 return CmpInst::Create(Instruction::ICmp, NewPred, NewX, Zero); 5932 } 5933 5934 static Instruction *foldVectorCmp(CmpInst &Cmp, 5935 InstCombiner::BuilderTy &Builder) { 5936 const CmpInst::Predicate Pred = Cmp.getPredicate(); 5937 Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1); 5938 Value *V1, *V2; 5939 ArrayRef<int> M; 5940 if (!match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(M)))) 5941 return nullptr; 5942 5943 // If both arguments of the cmp are shuffles that use the same mask and 5944 // shuffle within a single vector, move the shuffle after the cmp: 5945 // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M 5946 Type *V1Ty = V1->getType(); 5947 if (match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(M))) && 5948 V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) { 5949 Value *NewCmp = Builder.CreateCmp(Pred, V1, V2); 5950 return new ShuffleVectorInst(NewCmp, M); 5951 } 5952 5953 // Try to canonicalize compare with splatted operand and splat constant. 5954 // TODO: We could generalize this for more than splats. See/use the code in 5955 // InstCombiner::foldVectorBinop(). 5956 Constant *C; 5957 if (!LHS->hasOneUse() || !match(RHS, m_Constant(C))) 5958 return nullptr; 5959 5960 // Length-changing splats are ok, so adjust the constants as needed: 5961 // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M 5962 Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true); 5963 int MaskSplatIndex; 5964 if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) { 5965 // We allow undefs in matching, but this transform removes those for safety. 5966 // Demanded elements analysis should be able to recover some/all of that. 5967 C = ConstantVector::getSplat(cast<VectorType>(V1Ty)->getElementCount(), 5968 ScalarC); 5969 SmallVector<int, 8> NewM(M.size(), MaskSplatIndex); 5970 Value *NewCmp = Builder.CreateCmp(Pred, V1, C); 5971 return new ShuffleVectorInst(NewCmp, NewM); 5972 } 5973 5974 return nullptr; 5975 } 5976 5977 // extract(uadd.with.overflow(A, B), 0) ult A 5978 // -> extract(uadd.with.overflow(A, B), 1) 5979 static Instruction *foldICmpOfUAddOv(ICmpInst &I) { 5980 CmpInst::Predicate Pred = I.getPredicate(); 5981 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 5982 5983 Value *UAddOv; 5984 Value *A, *B; 5985 auto UAddOvResultPat = m_ExtractValue<0>( 5986 m_Intrinsic<Intrinsic::uadd_with_overflow>(m_Value(A), m_Value(B))); 5987 if (match(Op0, UAddOvResultPat) && 5988 ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) || 5989 (Pred == ICmpInst::ICMP_EQ && match(Op1, m_ZeroInt()) && 5990 (match(A, m_One()) || match(B, m_One()))) || 5991 (Pred == ICmpInst::ICMP_NE && match(Op1, m_AllOnes()) && 5992 (match(A, m_AllOnes()) || match(B, m_AllOnes()))))) 5993 // extract(uadd.with.overflow(A, B), 0) < A 5994 // extract(uadd.with.overflow(A, 1), 0) == 0 5995 // extract(uadd.with.overflow(A, -1), 0) != -1 5996 UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand(); 5997 else if (match(Op1, UAddOvResultPat) && 5998 Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B)) 5999 // A > extract(uadd.with.overflow(A, B), 0) 6000 UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand(); 6001 else 6002 return nullptr; 6003 6004 return ExtractValueInst::Create(UAddOv, 1); 6005 } 6006 6007 static Instruction *foldICmpInvariantGroup(ICmpInst &I) { 6008 if (!I.getOperand(0)->getType()->isPointerTy() || 6009 NullPointerIsDefined( 6010 I.getParent()->getParent(), 6011 I.getOperand(0)->getType()->getPointerAddressSpace())) { 6012 return nullptr; 6013 } 6014 Instruction *Op; 6015 if (match(I.getOperand(0), m_Instruction(Op)) && 6016 match(I.getOperand(1), m_Zero()) && 6017 Op->isLaunderOrStripInvariantGroup()) { 6018 return ICmpInst::Create(Instruction::ICmp, I.getPredicate(), 6019 Op->getOperand(0), I.getOperand(1)); 6020 } 6021 return nullptr; 6022 } 6023 6024 /// This function folds patterns produced by lowering of reduce idioms, such as 6025 /// llvm.vector.reduce.and which are lowered into instruction chains. This code 6026 /// attempts to generate fewer number of scalar comparisons instead of vector 6027 /// comparisons when possible. 6028 static Instruction *foldReductionIdiom(ICmpInst &I, 6029 InstCombiner::BuilderTy &Builder, 6030 const DataLayout &DL) { 6031 if (I.getType()->isVectorTy()) 6032 return nullptr; 6033 ICmpInst::Predicate OuterPred, InnerPred; 6034 Value *LHS, *RHS; 6035 6036 // Match lowering of @llvm.vector.reduce.and. Turn 6037 /// %vec_ne = icmp ne <8 x i8> %lhs, %rhs 6038 /// %scalar_ne = bitcast <8 x i1> %vec_ne to i8 6039 /// %res = icmp <pred> i8 %scalar_ne, 0 6040 /// 6041 /// into 6042 /// 6043 /// %lhs.scalar = bitcast <8 x i8> %lhs to i64 6044 /// %rhs.scalar = bitcast <8 x i8> %rhs to i64 6045 /// %res = icmp <pred> i64 %lhs.scalar, %rhs.scalar 6046 /// 6047 /// for <pred> in {ne, eq}. 6048 if (!match(&I, m_ICmp(OuterPred, 6049 m_OneUse(m_BitCast(m_OneUse( 6050 m_ICmp(InnerPred, m_Value(LHS), m_Value(RHS))))), 6051 m_Zero()))) 6052 return nullptr; 6053 auto *LHSTy = dyn_cast<FixedVectorType>(LHS->getType()); 6054 if (!LHSTy || !LHSTy->getElementType()->isIntegerTy()) 6055 return nullptr; 6056 unsigned NumBits = 6057 LHSTy->getNumElements() * LHSTy->getElementType()->getIntegerBitWidth(); 6058 // TODO: Relax this to "not wider than max legal integer type"? 6059 if (!DL.isLegalInteger(NumBits)) 6060 return nullptr; 6061 6062 if (ICmpInst::isEquality(OuterPred) && InnerPred == ICmpInst::ICMP_NE) { 6063 auto *ScalarTy = Builder.getIntNTy(NumBits); 6064 LHS = Builder.CreateBitCast(LHS, ScalarTy, LHS->getName() + ".scalar"); 6065 RHS = Builder.CreateBitCast(RHS, ScalarTy, RHS->getName() + ".scalar"); 6066 return ICmpInst::Create(Instruction::ICmp, OuterPred, LHS, RHS, 6067 I.getName()); 6068 } 6069 6070 return nullptr; 6071 } 6072 6073 Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) { 6074 bool Changed = false; 6075 const SimplifyQuery Q = SQ.getWithInstruction(&I); 6076 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 6077 unsigned Op0Cplxity = getComplexity(Op0); 6078 unsigned Op1Cplxity = getComplexity(Op1); 6079 6080 /// Orders the operands of the compare so that they are listed from most 6081 /// complex to least complex. This puts constants before unary operators, 6082 /// before binary operators. 6083 if (Op0Cplxity < Op1Cplxity || 6084 (Op0Cplxity == Op1Cplxity && swapMayExposeCSEOpportunities(Op0, Op1))) { 6085 I.swapOperands(); 6086 std::swap(Op0, Op1); 6087 Changed = true; 6088 } 6089 6090 if (Value *V = simplifyICmpInst(I.getPredicate(), Op0, Op1, Q)) 6091 return replaceInstUsesWith(I, V); 6092 6093 // Comparing -val or val with non-zero is the same as just comparing val 6094 // ie, abs(val) != 0 -> val != 0 6095 if (I.getPredicate() == ICmpInst::ICMP_NE && match(Op1, m_Zero())) { 6096 Value *Cond, *SelectTrue, *SelectFalse; 6097 if (match(Op0, m_Select(m_Value(Cond), m_Value(SelectTrue), 6098 m_Value(SelectFalse)))) { 6099 if (Value *V = dyn_castNegVal(SelectTrue)) { 6100 if (V == SelectFalse) 6101 return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); 6102 } 6103 else if (Value *V = dyn_castNegVal(SelectFalse)) { 6104 if (V == SelectTrue) 6105 return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); 6106 } 6107 } 6108 } 6109 6110 if (Op0->getType()->isIntOrIntVectorTy(1)) 6111 if (Instruction *Res = canonicalizeICmpBool(I, Builder)) 6112 return Res; 6113 6114 if (Instruction *Res = canonicalizeCmpWithConstant(I)) 6115 return Res; 6116 6117 if (Instruction *Res = canonicalizeICmpPredicate(I)) 6118 return Res; 6119 6120 if (Instruction *Res = foldICmpWithConstant(I)) 6121 return Res; 6122 6123 if (Instruction *Res = foldICmpWithDominatingICmp(I)) 6124 return Res; 6125 6126 if (Instruction *Res = foldICmpUsingBoolRange(I, Builder)) 6127 return Res; 6128 6129 if (Instruction *Res = foldICmpUsingKnownBits(I)) 6130 return Res; 6131 6132 // Test if the ICmpInst instruction is used exclusively by a select as 6133 // part of a minimum or maximum operation. If so, refrain from doing 6134 // any other folding. This helps out other analyses which understand 6135 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution 6136 // and CodeGen. And in this case, at least one of the comparison 6137 // operands has at least one user besides the compare (the select), 6138 // which would often largely negate the benefit of folding anyway. 6139 // 6140 // Do the same for the other patterns recognized by matchSelectPattern. 6141 if (I.hasOneUse()) 6142 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) { 6143 Value *A, *B; 6144 SelectPatternResult SPR = matchSelectPattern(SI, A, B); 6145 if (SPR.Flavor != SPF_UNKNOWN) 6146 return nullptr; 6147 } 6148 6149 // Do this after checking for min/max to prevent infinite looping. 6150 if (Instruction *Res = foldICmpWithZero(I)) 6151 return Res; 6152 6153 // FIXME: We only do this after checking for min/max to prevent infinite 6154 // looping caused by a reverse canonicalization of these patterns for min/max. 6155 // FIXME: The organization of folds is a mess. These would naturally go into 6156 // canonicalizeCmpWithConstant(), but we can't move all of the above folds 6157 // down here after the min/max restriction. 6158 ICmpInst::Predicate Pred = I.getPredicate(); 6159 const APInt *C; 6160 if (match(Op1, m_APInt(C))) { 6161 // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set 6162 if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) { 6163 Constant *Zero = Constant::getNullValue(Op0->getType()); 6164 return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero); 6165 } 6166 6167 // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear 6168 if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) { 6169 Constant *AllOnes = Constant::getAllOnesValue(Op0->getType()); 6170 return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes); 6171 } 6172 } 6173 6174 // The folds in here may rely on wrapping flags and special constants, so 6175 // they can break up min/max idioms in some cases but not seemingly similar 6176 // patterns. 6177 // FIXME: It may be possible to enhance select folding to make this 6178 // unnecessary. It may also be moot if we canonicalize to min/max 6179 // intrinsics. 6180 if (Instruction *Res = foldICmpBinOp(I, Q)) 6181 return Res; 6182 6183 if (Instruction *Res = foldICmpInstWithConstant(I)) 6184 return Res; 6185 6186 // Try to match comparison as a sign bit test. Intentionally do this after 6187 // foldICmpInstWithConstant() to potentially let other folds to happen first. 6188 if (Instruction *New = foldSignBitTest(I)) 6189 return New; 6190 6191 if (Instruction *Res = foldICmpInstWithConstantNotInt(I)) 6192 return Res; 6193 6194 // Try to optimize 'icmp GEP, P' or 'icmp P, GEP'. 6195 if (auto *GEP = dyn_cast<GEPOperator>(Op0)) 6196 if (Instruction *NI = foldGEPICmp(GEP, Op1, I.getPredicate(), I)) 6197 return NI; 6198 if (auto *GEP = dyn_cast<GEPOperator>(Op1)) 6199 if (Instruction *NI = foldGEPICmp(GEP, Op0, I.getSwappedPredicate(), I)) 6200 return NI; 6201 6202 if (auto *SI = dyn_cast<SelectInst>(Op0)) 6203 if (Instruction *NI = foldSelectICmp(I.getPredicate(), SI, Op1, I)) 6204 return NI; 6205 if (auto *SI = dyn_cast<SelectInst>(Op1)) 6206 if (Instruction *NI = foldSelectICmp(I.getSwappedPredicate(), SI, Op0, I)) 6207 return NI; 6208 6209 // Try to optimize equality comparisons against alloca-based pointers. 6210 if (Op0->getType()->isPointerTy() && I.isEquality()) { 6211 assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?"); 6212 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0))) 6213 if (Instruction *New = foldAllocaCmp(I, Alloca)) 6214 return New; 6215 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1))) 6216 if (Instruction *New = foldAllocaCmp(I, Alloca)) 6217 return New; 6218 } 6219 6220 if (Instruction *Res = foldICmpBitCast(I)) 6221 return Res; 6222 6223 // TODO: Hoist this above the min/max bailout. 6224 if (Instruction *R = foldICmpWithCastOp(I)) 6225 return R; 6226 6227 if (Instruction *Res = foldICmpWithMinMax(I)) 6228 return Res; 6229 6230 { 6231 Value *A, *B; 6232 // Transform (A & ~B) == 0 --> (A & B) != 0 6233 // and (A & ~B) != 0 --> (A & B) == 0 6234 // if A is a power of 2. 6235 if (match(Op0, m_And(m_Value(A), m_Not(m_Value(B)))) && 6236 match(Op1, m_Zero()) && 6237 isKnownToBeAPowerOfTwo(A, false, 0, &I) && I.isEquality()) 6238 return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(A, B), 6239 Op1); 6240 6241 // ~X < ~Y --> Y < X 6242 // ~X < C --> X > ~C 6243 if (match(Op0, m_Not(m_Value(A)))) { 6244 if (match(Op1, m_Not(m_Value(B)))) 6245 return new ICmpInst(I.getPredicate(), B, A); 6246 6247 const APInt *C; 6248 if (match(Op1, m_APInt(C))) 6249 return new ICmpInst(I.getSwappedPredicate(), A, 6250 ConstantInt::get(Op1->getType(), ~(*C))); 6251 } 6252 6253 Instruction *AddI = nullptr; 6254 if (match(&I, m_UAddWithOverflow(m_Value(A), m_Value(B), 6255 m_Instruction(AddI))) && 6256 isa<IntegerType>(A->getType())) { 6257 Value *Result; 6258 Constant *Overflow; 6259 // m_UAddWithOverflow can match patterns that do not include an explicit 6260 // "add" instruction, so check the opcode of the matched op. 6261 if (AddI->getOpcode() == Instruction::Add && 6262 OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, A, B, *AddI, 6263 Result, Overflow)) { 6264 replaceInstUsesWith(*AddI, Result); 6265 eraseInstFromFunction(*AddI); 6266 return replaceInstUsesWith(I, Overflow); 6267 } 6268 } 6269 6270 // (zext a) * (zext b) --> llvm.umul.with.overflow. 6271 if (match(Op0, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { 6272 if (Instruction *R = processUMulZExtIdiom(I, Op0, Op1, *this)) 6273 return R; 6274 } 6275 if (match(Op1, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { 6276 if (Instruction *R = processUMulZExtIdiom(I, Op1, Op0, *this)) 6277 return R; 6278 } 6279 } 6280 6281 if (Instruction *Res = foldICmpEquality(I)) 6282 return Res; 6283 6284 if (Instruction *Res = foldICmpOfUAddOv(I)) 6285 return Res; 6286 6287 // The 'cmpxchg' instruction returns an aggregate containing the old value and 6288 // an i1 which indicates whether or not we successfully did the swap. 6289 // 6290 // Replace comparisons between the old value and the expected value with the 6291 // indicator that 'cmpxchg' returns. 6292 // 6293 // N.B. This transform is only valid when the 'cmpxchg' is not permitted to 6294 // spuriously fail. In those cases, the old value may equal the expected 6295 // value but it is possible for the swap to not occur. 6296 if (I.getPredicate() == ICmpInst::ICMP_EQ) 6297 if (auto *EVI = dyn_cast<ExtractValueInst>(Op0)) 6298 if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand())) 6299 if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 && 6300 !ACXI->isWeak()) 6301 return ExtractValueInst::Create(ACXI, 1); 6302 6303 { 6304 Value *X; 6305 const APInt *C; 6306 // icmp X+Cst, X 6307 if (match(Op0, m_Add(m_Value(X), m_APInt(C))) && Op1 == X) 6308 return foldICmpAddOpConst(X, *C, I.getPredicate()); 6309 6310 // icmp X, X+Cst 6311 if (match(Op1, m_Add(m_Value(X), m_APInt(C))) && Op0 == X) 6312 return foldICmpAddOpConst(X, *C, I.getSwappedPredicate()); 6313 } 6314 6315 if (Instruction *Res = foldICmpWithHighBitMask(I, Builder)) 6316 return Res; 6317 6318 if (I.getType()->isVectorTy()) 6319 if (Instruction *Res = foldVectorCmp(I, Builder)) 6320 return Res; 6321 6322 if (Instruction *Res = foldICmpInvariantGroup(I)) 6323 return Res; 6324 6325 if (Instruction *Res = foldReductionIdiom(I, Builder, DL)) 6326 return Res; 6327 6328 return Changed ? &I : nullptr; 6329 } 6330 6331 /// Fold fcmp ([us]itofp x, cst) if possible. 6332 Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, 6333 Instruction *LHSI, 6334 Constant *RHSC) { 6335 if (!isa<ConstantFP>(RHSC)) return nullptr; 6336 const APFloat &RHS = cast<ConstantFP>(RHSC)->getValueAPF(); 6337 6338 // Get the width of the mantissa. We don't want to hack on conversions that 6339 // might lose information from the integer, e.g. "i64 -> float" 6340 int MantissaWidth = LHSI->getType()->getFPMantissaWidth(); 6341 if (MantissaWidth == -1) return nullptr; // Unknown. 6342 6343 IntegerType *IntTy = cast<IntegerType>(LHSI->getOperand(0)->getType()); 6344 6345 bool LHSUnsigned = isa<UIToFPInst>(LHSI); 6346 6347 if (I.isEquality()) { 6348 FCmpInst::Predicate P = I.getPredicate(); 6349 bool IsExact = false; 6350 APSInt RHSCvt(IntTy->getBitWidth(), LHSUnsigned); 6351 RHS.convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact); 6352 6353 // If the floating point constant isn't an integer value, we know if we will 6354 // ever compare equal / not equal to it. 6355 if (!IsExact) { 6356 // TODO: Can never be -0.0 and other non-representable values 6357 APFloat RHSRoundInt(RHS); 6358 RHSRoundInt.roundToIntegral(APFloat::rmNearestTiesToEven); 6359 if (RHS != RHSRoundInt) { 6360 if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ) 6361 return replaceInstUsesWith(I, Builder.getFalse()); 6362 6363 assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE); 6364 return replaceInstUsesWith(I, Builder.getTrue()); 6365 } 6366 } 6367 6368 // TODO: If the constant is exactly representable, is it always OK to do 6369 // equality compares as integer? 6370 } 6371 6372 // Check to see that the input is converted from an integer type that is small 6373 // enough that preserves all bits. TODO: check here for "known" sign bits. 6374 // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e. 6375 unsigned InputSize = IntTy->getScalarSizeInBits(); 6376 6377 // Following test does NOT adjust InputSize downwards for signed inputs, 6378 // because the most negative value still requires all the mantissa bits 6379 // to distinguish it from one less than that value. 6380 if ((int)InputSize > MantissaWidth) { 6381 // Conversion would lose accuracy. Check if loss can impact comparison. 6382 int Exp = ilogb(RHS); 6383 if (Exp == APFloat::IEK_Inf) { 6384 int MaxExponent = ilogb(APFloat::getLargest(RHS.getSemantics())); 6385 if (MaxExponent < (int)InputSize - !LHSUnsigned) 6386 // Conversion could create infinity. 6387 return nullptr; 6388 } else { 6389 // Note that if RHS is zero or NaN, then Exp is negative 6390 // and first condition is trivially false. 6391 if (MantissaWidth <= Exp && Exp <= (int)InputSize - !LHSUnsigned) 6392 // Conversion could affect comparison. 6393 return nullptr; 6394 } 6395 } 6396 6397 // Otherwise, we can potentially simplify the comparison. We know that it 6398 // will always come through as an integer value and we know the constant is 6399 // not a NAN (it would have been previously simplified). 6400 assert(!RHS.isNaN() && "NaN comparison not already folded!"); 6401 6402 ICmpInst::Predicate Pred; 6403 switch (I.getPredicate()) { 6404 default: llvm_unreachable("Unexpected predicate!"); 6405 case FCmpInst::FCMP_UEQ: 6406 case FCmpInst::FCMP_OEQ: 6407 Pred = ICmpInst::ICMP_EQ; 6408 break; 6409 case FCmpInst::FCMP_UGT: 6410 case FCmpInst::FCMP_OGT: 6411 Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT; 6412 break; 6413 case FCmpInst::FCMP_UGE: 6414 case FCmpInst::FCMP_OGE: 6415 Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE; 6416 break; 6417 case FCmpInst::FCMP_ULT: 6418 case FCmpInst::FCMP_OLT: 6419 Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT; 6420 break; 6421 case FCmpInst::FCMP_ULE: 6422 case FCmpInst::FCMP_OLE: 6423 Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE; 6424 break; 6425 case FCmpInst::FCMP_UNE: 6426 case FCmpInst::FCMP_ONE: 6427 Pred = ICmpInst::ICMP_NE; 6428 break; 6429 case FCmpInst::FCMP_ORD: 6430 return replaceInstUsesWith(I, Builder.getTrue()); 6431 case FCmpInst::FCMP_UNO: 6432 return replaceInstUsesWith(I, Builder.getFalse()); 6433 } 6434 6435 // Now we know that the APFloat is a normal number, zero or inf. 6436 6437 // See if the FP constant is too large for the integer. For example, 6438 // comparing an i8 to 300.0. 6439 unsigned IntWidth = IntTy->getScalarSizeInBits(); 6440 6441 if (!LHSUnsigned) { 6442 // If the RHS value is > SignedMax, fold the comparison. This handles +INF 6443 // and large values. 6444 APFloat SMax(RHS.getSemantics()); 6445 SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true, 6446 APFloat::rmNearestTiesToEven); 6447 if (SMax < RHS) { // smax < 13123.0 6448 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT || 6449 Pred == ICmpInst::ICMP_SLE) 6450 return replaceInstUsesWith(I, Builder.getTrue()); 6451 return replaceInstUsesWith(I, Builder.getFalse()); 6452 } 6453 } else { 6454 // If the RHS value is > UnsignedMax, fold the comparison. This handles 6455 // +INF and large values. 6456 APFloat UMax(RHS.getSemantics()); 6457 UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false, 6458 APFloat::rmNearestTiesToEven); 6459 if (UMax < RHS) { // umax < 13123.0 6460 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT || 6461 Pred == ICmpInst::ICMP_ULE) 6462 return replaceInstUsesWith(I, Builder.getTrue()); 6463 return replaceInstUsesWith(I, Builder.getFalse()); 6464 } 6465 } 6466 6467 if (!LHSUnsigned) { 6468 // See if the RHS value is < SignedMin. 6469 APFloat SMin(RHS.getSemantics()); 6470 SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true, 6471 APFloat::rmNearestTiesToEven); 6472 if (SMin > RHS) { // smin > 12312.0 6473 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT || 6474 Pred == ICmpInst::ICMP_SGE) 6475 return replaceInstUsesWith(I, Builder.getTrue()); 6476 return replaceInstUsesWith(I, Builder.getFalse()); 6477 } 6478 } else { 6479 // See if the RHS value is < UnsignedMin. 6480 APFloat UMin(RHS.getSemantics()); 6481 UMin.convertFromAPInt(APInt::getMinValue(IntWidth), false, 6482 APFloat::rmNearestTiesToEven); 6483 if (UMin > RHS) { // umin > 12312.0 6484 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT || 6485 Pred == ICmpInst::ICMP_UGE) 6486 return replaceInstUsesWith(I, Builder.getTrue()); 6487 return replaceInstUsesWith(I, Builder.getFalse()); 6488 } 6489 } 6490 6491 // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or 6492 // [0, UMAX], but it may still be fractional. See if it is fractional by 6493 // casting the FP value to the integer value and back, checking for equality. 6494 // Don't do this for zero, because -0.0 is not fractional. 6495 Constant *RHSInt = LHSUnsigned 6496 ? ConstantExpr::getFPToUI(RHSC, IntTy) 6497 : ConstantExpr::getFPToSI(RHSC, IntTy); 6498 if (!RHS.isZero()) { 6499 bool Equal = LHSUnsigned 6500 ? ConstantExpr::getUIToFP(RHSInt, RHSC->getType()) == RHSC 6501 : ConstantExpr::getSIToFP(RHSInt, RHSC->getType()) == RHSC; 6502 if (!Equal) { 6503 // If we had a comparison against a fractional value, we have to adjust 6504 // the compare predicate and sometimes the value. RHSC is rounded towards 6505 // zero at this point. 6506 switch (Pred) { 6507 default: llvm_unreachable("Unexpected integer comparison!"); 6508 case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true 6509 return replaceInstUsesWith(I, Builder.getTrue()); 6510 case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false 6511 return replaceInstUsesWith(I, Builder.getFalse()); 6512 case ICmpInst::ICMP_ULE: 6513 // (float)int <= 4.4 --> int <= 4 6514 // (float)int <= -4.4 --> false 6515 if (RHS.isNegative()) 6516 return replaceInstUsesWith(I, Builder.getFalse()); 6517 break; 6518 case ICmpInst::ICMP_SLE: 6519 // (float)int <= 4.4 --> int <= 4 6520 // (float)int <= -4.4 --> int < -4 6521 if (RHS.isNegative()) 6522 Pred = ICmpInst::ICMP_SLT; 6523 break; 6524 case ICmpInst::ICMP_ULT: 6525 // (float)int < -4.4 --> false 6526 // (float)int < 4.4 --> int <= 4 6527 if (RHS.isNegative()) 6528 return replaceInstUsesWith(I, Builder.getFalse()); 6529 Pred = ICmpInst::ICMP_ULE; 6530 break; 6531 case ICmpInst::ICMP_SLT: 6532 // (float)int < -4.4 --> int < -4 6533 // (float)int < 4.4 --> int <= 4 6534 if (!RHS.isNegative()) 6535 Pred = ICmpInst::ICMP_SLE; 6536 break; 6537 case ICmpInst::ICMP_UGT: 6538 // (float)int > 4.4 --> int > 4 6539 // (float)int > -4.4 --> true 6540 if (RHS.isNegative()) 6541 return replaceInstUsesWith(I, Builder.getTrue()); 6542 break; 6543 case ICmpInst::ICMP_SGT: 6544 // (float)int > 4.4 --> int > 4 6545 // (float)int > -4.4 --> int >= -4 6546 if (RHS.isNegative()) 6547 Pred = ICmpInst::ICMP_SGE; 6548 break; 6549 case ICmpInst::ICMP_UGE: 6550 // (float)int >= -4.4 --> true 6551 // (float)int >= 4.4 --> int > 4 6552 if (RHS.isNegative()) 6553 return replaceInstUsesWith(I, Builder.getTrue()); 6554 Pred = ICmpInst::ICMP_UGT; 6555 break; 6556 case ICmpInst::ICMP_SGE: 6557 // (float)int >= -4.4 --> int >= -4 6558 // (float)int >= 4.4 --> int > 4 6559 if (!RHS.isNegative()) 6560 Pred = ICmpInst::ICMP_SGT; 6561 break; 6562 } 6563 } 6564 } 6565 6566 // Lower this FP comparison into an appropriate integer version of the 6567 // comparison. 6568 return new ICmpInst(Pred, LHSI->getOperand(0), RHSInt); 6569 } 6570 6571 /// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary. 6572 static Instruction *foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI, 6573 Constant *RHSC) { 6574 // When C is not 0.0 and infinities are not allowed: 6575 // (C / X) < 0.0 is a sign-bit test of X 6576 // (C / X) < 0.0 --> X < 0.0 (if C is positive) 6577 // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate) 6578 // 6579 // Proof: 6580 // Multiply (C / X) < 0.0 by X * X / C. 6581 // - X is non zero, if it is the flag 'ninf' is violated. 6582 // - C defines the sign of X * X * C. Thus it also defines whether to swap 6583 // the predicate. C is also non zero by definition. 6584 // 6585 // Thus X * X / C is non zero and the transformation is valid. [qed] 6586 6587 FCmpInst::Predicate Pred = I.getPredicate(); 6588 6589 // Check that predicates are valid. 6590 if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) && 6591 (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE)) 6592 return nullptr; 6593 6594 // Check that RHS operand is zero. 6595 if (!match(RHSC, m_AnyZeroFP())) 6596 return nullptr; 6597 6598 // Check fastmath flags ('ninf'). 6599 if (!LHSI->hasNoInfs() || !I.hasNoInfs()) 6600 return nullptr; 6601 6602 // Check the properties of the dividend. It must not be zero to avoid a 6603 // division by zero (see Proof). 6604 const APFloat *C; 6605 if (!match(LHSI->getOperand(0), m_APFloat(C))) 6606 return nullptr; 6607 6608 if (C->isZero()) 6609 return nullptr; 6610 6611 // Get swapped predicate if necessary. 6612 if (C->isNegative()) 6613 Pred = I.getSwappedPredicate(); 6614 6615 return new FCmpInst(Pred, LHSI->getOperand(1), RHSC, "", &I); 6616 } 6617 6618 /// Optimize fabs(X) compared with zero. 6619 static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC) { 6620 Value *X; 6621 if (!match(I.getOperand(0), m_FAbs(m_Value(X))) || 6622 !match(I.getOperand(1), m_PosZeroFP())) 6623 return nullptr; 6624 6625 auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) { 6626 I->setPredicate(P); 6627 return IC.replaceOperand(*I, 0, X); 6628 }; 6629 6630 switch (I.getPredicate()) { 6631 case FCmpInst::FCMP_UGE: 6632 case FCmpInst::FCMP_OLT: 6633 // fabs(X) >= 0.0 --> true 6634 // fabs(X) < 0.0 --> false 6635 llvm_unreachable("fcmp should have simplified"); 6636 6637 case FCmpInst::FCMP_OGT: 6638 // fabs(X) > 0.0 --> X != 0.0 6639 return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X); 6640 6641 case FCmpInst::FCMP_UGT: 6642 // fabs(X) u> 0.0 --> X u!= 0.0 6643 return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X); 6644 6645 case FCmpInst::FCMP_OLE: 6646 // fabs(X) <= 0.0 --> X == 0.0 6647 return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X); 6648 6649 case FCmpInst::FCMP_ULE: 6650 // fabs(X) u<= 0.0 --> X u== 0.0 6651 return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X); 6652 6653 case FCmpInst::FCMP_OGE: 6654 // fabs(X) >= 0.0 --> !isnan(X) 6655 assert(!I.hasNoNaNs() && "fcmp should have simplified"); 6656 return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X); 6657 6658 case FCmpInst::FCMP_ULT: 6659 // fabs(X) u< 0.0 --> isnan(X) 6660 assert(!I.hasNoNaNs() && "fcmp should have simplified"); 6661 return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X); 6662 6663 case FCmpInst::FCMP_OEQ: 6664 case FCmpInst::FCMP_UEQ: 6665 case FCmpInst::FCMP_ONE: 6666 case FCmpInst::FCMP_UNE: 6667 case FCmpInst::FCMP_ORD: 6668 case FCmpInst::FCMP_UNO: 6669 // Look through the fabs() because it doesn't change anything but the sign. 6670 // fabs(X) == 0.0 --> X == 0.0, 6671 // fabs(X) != 0.0 --> X != 0.0 6672 // isnan(fabs(X)) --> isnan(X) 6673 // !isnan(fabs(X) --> !isnan(X) 6674 return replacePredAndOp0(&I, I.getPredicate(), X); 6675 6676 default: 6677 return nullptr; 6678 } 6679 } 6680 6681 static Instruction *foldFCmpFNegCommonOp(FCmpInst &I) { 6682 CmpInst::Predicate Pred = I.getPredicate(); 6683 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 6684 6685 // Canonicalize fneg as Op1. 6686 if (match(Op0, m_FNeg(m_Value())) && !match(Op1, m_FNeg(m_Value()))) { 6687 std::swap(Op0, Op1); 6688 Pred = I.getSwappedPredicate(); 6689 } 6690 6691 if (!match(Op1, m_FNeg(m_Specific(Op0)))) 6692 return nullptr; 6693 6694 // Replace the negated operand with 0.0: 6695 // fcmp Pred Op0, -Op0 --> fcmp Pred Op0, 0.0 6696 Constant *Zero = ConstantFP::getNullValue(Op0->getType()); 6697 return new FCmpInst(Pred, Op0, Zero, "", &I); 6698 } 6699 6700 Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) { 6701 bool Changed = false; 6702 6703 /// Orders the operands of the compare so that they are listed from most 6704 /// complex to least complex. This puts constants before unary operators, 6705 /// before binary operators. 6706 if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) { 6707 I.swapOperands(); 6708 Changed = true; 6709 } 6710 6711 const CmpInst::Predicate Pred = I.getPredicate(); 6712 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 6713 if (Value *V = simplifyFCmpInst(Pred, Op0, Op1, I.getFastMathFlags(), 6714 SQ.getWithInstruction(&I))) 6715 return replaceInstUsesWith(I, V); 6716 6717 // Simplify 'fcmp pred X, X' 6718 Type *OpType = Op0->getType(); 6719 assert(OpType == Op1->getType() && "fcmp with different-typed operands?"); 6720 if (Op0 == Op1) { 6721 switch (Pred) { 6722 default: break; 6723 case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y) 6724 case FCmpInst::FCMP_ULT: // True if unordered or less than 6725 case FCmpInst::FCMP_UGT: // True if unordered or greater than 6726 case FCmpInst::FCMP_UNE: // True if unordered or not equal 6727 // Canonicalize these to be 'fcmp uno %X, 0.0'. 6728 I.setPredicate(FCmpInst::FCMP_UNO); 6729 I.setOperand(1, Constant::getNullValue(OpType)); 6730 return &I; 6731 6732 case FCmpInst::FCMP_ORD: // True if ordered (no nans) 6733 case FCmpInst::FCMP_OEQ: // True if ordered and equal 6734 case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal 6735 case FCmpInst::FCMP_OLE: // True if ordered and less than or equal 6736 // Canonicalize these to be 'fcmp ord %X, 0.0'. 6737 I.setPredicate(FCmpInst::FCMP_ORD); 6738 I.setOperand(1, Constant::getNullValue(OpType)); 6739 return &I; 6740 } 6741 } 6742 6743 // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand, 6744 // then canonicalize the operand to 0.0. 6745 if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) { 6746 if (!match(Op0, m_PosZeroFP()) && isKnownNeverNaN(Op0, &TLI)) 6747 return replaceOperand(I, 0, ConstantFP::getNullValue(OpType)); 6748 6749 if (!match(Op1, m_PosZeroFP()) && isKnownNeverNaN(Op1, &TLI)) 6750 return replaceOperand(I, 1, ConstantFP::getNullValue(OpType)); 6751 } 6752 6753 // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y 6754 Value *X, *Y; 6755 if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y)))) 6756 return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I); 6757 6758 if (Instruction *R = foldFCmpFNegCommonOp(I)) 6759 return R; 6760 6761 // Test if the FCmpInst instruction is used exclusively by a select as 6762 // part of a minimum or maximum operation. If so, refrain from doing 6763 // any other folding. This helps out other analyses which understand 6764 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution 6765 // and CodeGen. And in this case, at least one of the comparison 6766 // operands has at least one user besides the compare (the select), 6767 // which would often largely negate the benefit of folding anyway. 6768 if (I.hasOneUse()) 6769 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) { 6770 Value *A, *B; 6771 SelectPatternResult SPR = matchSelectPattern(SI, A, B); 6772 if (SPR.Flavor != SPF_UNKNOWN) 6773 return nullptr; 6774 } 6775 6776 // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0: 6777 // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0 6778 if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP())) 6779 return replaceOperand(I, 1, ConstantFP::getNullValue(OpType)); 6780 6781 // Handle fcmp with instruction LHS and constant RHS. 6782 Instruction *LHSI; 6783 Constant *RHSC; 6784 if (match(Op0, m_Instruction(LHSI)) && match(Op1, m_Constant(RHSC))) { 6785 switch (LHSI->getOpcode()) { 6786 case Instruction::PHI: 6787 // Only fold fcmp into the PHI if the phi and fcmp are in the same 6788 // block. If in the same block, we're encouraging jump threading. If 6789 // not, we are just pessimizing the code by making an i1 phi. 6790 if (LHSI->getParent() == I.getParent()) 6791 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) 6792 return NV; 6793 break; 6794 case Instruction::SIToFP: 6795 case Instruction::UIToFP: 6796 if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC)) 6797 return NV; 6798 break; 6799 case Instruction::FDiv: 6800 if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC)) 6801 return NV; 6802 break; 6803 case Instruction::Load: 6804 if (auto *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) 6805 if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) 6806 if (Instruction *Res = foldCmpLoadFromIndexedGlobal( 6807 cast<LoadInst>(LHSI), GEP, GV, I)) 6808 return Res; 6809 break; 6810 } 6811 } 6812 6813 if (Instruction *R = foldFabsWithFcmpZero(I, *this)) 6814 return R; 6815 6816 if (match(Op0, m_FNeg(m_Value(X)))) { 6817 // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C 6818 Constant *C; 6819 if (match(Op1, m_Constant(C))) { 6820 Constant *NegC = ConstantExpr::getFNeg(C); 6821 return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I); 6822 } 6823 } 6824 6825 if (match(Op0, m_FPExt(m_Value(X)))) { 6826 // fcmp (fpext X), (fpext Y) -> fcmp X, Y 6827 if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType()) 6828 return new FCmpInst(Pred, X, Y, "", &I); 6829 6830 const APFloat *C; 6831 if (match(Op1, m_APFloat(C))) { 6832 const fltSemantics &FPSem = 6833 X->getType()->getScalarType()->getFltSemantics(); 6834 bool Lossy; 6835 APFloat TruncC = *C; 6836 TruncC.convert(FPSem, APFloat::rmNearestTiesToEven, &Lossy); 6837 6838 if (Lossy) { 6839 // X can't possibly equal the higher-precision constant, so reduce any 6840 // equality comparison. 6841 // TODO: Other predicates can be handled via getFCmpCode(). 6842 switch (Pred) { 6843 case FCmpInst::FCMP_OEQ: 6844 // X is ordered and equal to an impossible constant --> false 6845 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 6846 case FCmpInst::FCMP_ONE: 6847 // X is ordered and not equal to an impossible constant --> ordered 6848 return new FCmpInst(FCmpInst::FCMP_ORD, X, 6849 ConstantFP::getNullValue(X->getType())); 6850 case FCmpInst::FCMP_UEQ: 6851 // X is unordered or equal to an impossible constant --> unordered 6852 return new FCmpInst(FCmpInst::FCMP_UNO, X, 6853 ConstantFP::getNullValue(X->getType())); 6854 case FCmpInst::FCMP_UNE: 6855 // X is unordered or not equal to an impossible constant --> true 6856 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 6857 default: 6858 break; 6859 } 6860 } 6861 6862 // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless 6863 // Avoid lossy conversions and denormals. 6864 // Zero is a special case that's OK to convert. 6865 APFloat Fabs = TruncC; 6866 Fabs.clearSign(); 6867 if (!Lossy && 6868 (!(Fabs < APFloat::getSmallestNormalized(FPSem)) || Fabs.isZero())) { 6869 Constant *NewC = ConstantFP::get(X->getType(), TruncC); 6870 return new FCmpInst(Pred, X, NewC, "", &I); 6871 } 6872 } 6873 } 6874 6875 // Convert a sign-bit test of an FP value into a cast and integer compare. 6876 // TODO: Simplify if the copysign constant is 0.0 or NaN. 6877 // TODO: Handle non-zero compare constants. 6878 // TODO: Handle other predicates. 6879 const APFloat *C; 6880 if (match(Op0, m_OneUse(m_Intrinsic<Intrinsic::copysign>(m_APFloat(C), 6881 m_Value(X)))) && 6882 match(Op1, m_AnyZeroFP()) && !C->isZero() && !C->isNaN()) { 6883 Type *IntType = Builder.getIntNTy(X->getType()->getScalarSizeInBits()); 6884 if (auto *VecTy = dyn_cast<VectorType>(OpType)) 6885 IntType = VectorType::get(IntType, VecTy->getElementCount()); 6886 6887 // copysign(non-zero constant, X) < 0.0 --> (bitcast X) < 0 6888 if (Pred == FCmpInst::FCMP_OLT) { 6889 Value *IntX = Builder.CreateBitCast(X, IntType); 6890 return new ICmpInst(ICmpInst::ICMP_SLT, IntX, 6891 ConstantInt::getNullValue(IntType)); 6892 } 6893 } 6894 6895 if (I.getType()->isVectorTy()) 6896 if (Instruction *Res = foldVectorCmp(I, Builder)) 6897 return Res; 6898 6899 return Changed ? &I : nullptr; 6900 } 6901