1 //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This implements the SelectionDAG class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/CodeGen/SelectionDAG.h" 14 #include "SDNodeDbgValue.h" 15 #include "llvm/ADT/APFloat.h" 16 #include "llvm/ADT/APInt.h" 17 #include "llvm/ADT/APSInt.h" 18 #include "llvm/ADT/ArrayRef.h" 19 #include "llvm/ADT/BitVector.h" 20 #include "llvm/ADT/FoldingSet.h" 21 #include "llvm/ADT/None.h" 22 #include "llvm/ADT/STLExtras.h" 23 #include "llvm/ADT/SmallPtrSet.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/Triple.h" 26 #include "llvm/ADT/Twine.h" 27 #include "llvm/Analysis/BlockFrequencyInfo.h" 28 #include "llvm/Analysis/MemoryLocation.h" 29 #include "llvm/Analysis/ProfileSummaryInfo.h" 30 #include "llvm/Analysis/ValueTracking.h" 31 #include "llvm/CodeGen/FunctionLoweringInfo.h" 32 #include "llvm/CodeGen/ISDOpcodes.h" 33 #include "llvm/CodeGen/MachineBasicBlock.h" 34 #include "llvm/CodeGen/MachineConstantPool.h" 35 #include "llvm/CodeGen/MachineFrameInfo.h" 36 #include "llvm/CodeGen/MachineFunction.h" 37 #include "llvm/CodeGen/MachineMemOperand.h" 38 #include "llvm/CodeGen/RuntimeLibcalls.h" 39 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h" 40 #include "llvm/CodeGen/SelectionDAGNodes.h" 41 #include "llvm/CodeGen/SelectionDAGTargetInfo.h" 42 #include "llvm/CodeGen/TargetFrameLowering.h" 43 #include "llvm/CodeGen/TargetLowering.h" 44 #include "llvm/CodeGen/TargetRegisterInfo.h" 45 #include "llvm/CodeGen/TargetSubtargetInfo.h" 46 #include "llvm/CodeGen/ValueTypes.h" 47 #include "llvm/IR/Constant.h" 48 #include "llvm/IR/Constants.h" 49 #include "llvm/IR/DataLayout.h" 50 #include "llvm/IR/DebugInfoMetadata.h" 51 #include "llvm/IR/DebugLoc.h" 52 #include "llvm/IR/DerivedTypes.h" 53 #include "llvm/IR/Function.h" 54 #include "llvm/IR/GlobalValue.h" 55 #include "llvm/IR/Metadata.h" 56 #include "llvm/IR/Type.h" 57 #include "llvm/IR/Value.h" 58 #include "llvm/Support/Casting.h" 59 #include "llvm/Support/CodeGen.h" 60 #include "llvm/Support/Compiler.h" 61 #include "llvm/Support/Debug.h" 62 #include "llvm/Support/ErrorHandling.h" 63 #include "llvm/Support/KnownBits.h" 64 #include "llvm/Support/MachineValueType.h" 65 #include "llvm/Support/ManagedStatic.h" 66 #include "llvm/Support/MathExtras.h" 67 #include "llvm/Support/Mutex.h" 68 #include "llvm/Support/raw_ostream.h" 69 #include "llvm/Target/TargetMachine.h" 70 #include "llvm/Target/TargetOptions.h" 71 #include "llvm/Transforms/Utils/SizeOpts.h" 72 #include <algorithm> 73 #include <cassert> 74 #include <cstdint> 75 #include <cstdlib> 76 #include <limits> 77 #include <set> 78 #include <string> 79 #include <utility> 80 #include <vector> 81 82 using namespace llvm; 83 84 /// makeVTList - Return an instance of the SDVTList struct initialized with the 85 /// specified members. 86 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 87 SDVTList Res = {VTs, NumVTs}; 88 return Res; 89 } 90 91 // Default null implementations of the callbacks. 92 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} 93 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} 94 void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {} 95 96 void SelectionDAG::DAGNodeDeletedListener::anchor() {} 97 98 #define DEBUG_TYPE "selectiondag" 99 100 static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt", 101 cl::Hidden, cl::init(true), 102 cl::desc("Gang up loads and stores generated by inlining of memcpy")); 103 104 static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max", 105 cl::desc("Number limit for gluing ld/st of memcpy."), 106 cl::Hidden, cl::init(0)); 107 108 static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) { 109 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G);); 110 } 111 112 //===----------------------------------------------------------------------===// 113 // ConstantFPSDNode Class 114 //===----------------------------------------------------------------------===// 115 116 /// isExactlyValue - We don't rely on operator== working on double values, as 117 /// it returns true for things that are clearly not equal, like -0.0 and 0.0. 118 /// As such, this method can be used to do an exact bit-for-bit comparison of 119 /// two floating point values. 120 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 121 return getValueAPF().bitwiseIsEqual(V); 122 } 123 124 bool ConstantFPSDNode::isValueValidForType(EVT VT, 125 const APFloat& Val) { 126 assert(VT.isFloatingPoint() && "Can only convert between FP types"); 127 128 // convert modifies in place, so make a copy. 129 APFloat Val2 = APFloat(Val); 130 bool losesInfo; 131 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), 132 APFloat::rmNearestTiesToEven, 133 &losesInfo); 134 return !losesInfo; 135 } 136 137 //===----------------------------------------------------------------------===// 138 // ISD Namespace 139 //===----------------------------------------------------------------------===// 140 141 bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { 142 if (N->getOpcode() == ISD::SPLAT_VECTOR) { 143 unsigned EltSize = 144 N->getValueType(0).getVectorElementType().getSizeInBits(); 145 if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 146 SplatVal = Op0->getAPIntValue().truncOrSelf(EltSize); 147 return true; 148 } 149 if (auto *Op0 = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) { 150 SplatVal = Op0->getValueAPF().bitcastToAPInt().truncOrSelf(EltSize); 151 return true; 152 } 153 } 154 155 auto *BV = dyn_cast<BuildVectorSDNode>(N); 156 if (!BV) 157 return false; 158 159 APInt SplatUndef; 160 unsigned SplatBitSize; 161 bool HasUndefs; 162 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); 163 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs, 164 EltSize) && 165 EltSize == SplatBitSize; 166 } 167 168 // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be 169 // specializations of the more general isConstantSplatVector()? 170 171 bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { 172 // Look through a bit convert. 173 while (N->getOpcode() == ISD::BITCAST) 174 N = N->getOperand(0).getNode(); 175 176 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { 177 APInt SplatVal; 178 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnesValue(); 179 } 180 181 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 182 183 unsigned i = 0, e = N->getNumOperands(); 184 185 // Skip over all of the undef values. 186 while (i != e && N->getOperand(i).isUndef()) 187 ++i; 188 189 // Do not accept an all-undef vector. 190 if (i == e) return false; 191 192 // Do not accept build_vectors that aren't all constants or which have non-~0 193 // elements. We have to be a bit careful here, as the type of the constant 194 // may not be the same as the type of the vector elements due to type 195 // legalization (the elements are promoted to a legal type for the target and 196 // a vector of a type may be legal when the base element type is not). 197 // We only want to check enough bits to cover the vector elements, because 198 // we care if the resultant vector is all ones, not whether the individual 199 // constants are. 200 SDValue NotZero = N->getOperand(i); 201 unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); 202 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) { 203 if (CN->getAPIntValue().countTrailingOnes() < EltSize) 204 return false; 205 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) { 206 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) 207 return false; 208 } else 209 return false; 210 211 // Okay, we have at least one ~0 value, check to see if the rest match or are 212 // undefs. Even with the above element type twiddling, this should be OK, as 213 // the same type legalization should have applied to all the elements. 214 for (++i; i != e; ++i) 215 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef()) 216 return false; 217 return true; 218 } 219 220 bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { 221 // Look through a bit convert. 222 while (N->getOpcode() == ISD::BITCAST) 223 N = N->getOperand(0).getNode(); 224 225 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { 226 APInt SplatVal; 227 return isConstantSplatVector(N, SplatVal) && SplatVal.isNullValue(); 228 } 229 230 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 231 232 bool IsAllUndef = true; 233 for (const SDValue &Op : N->op_values()) { 234 if (Op.isUndef()) 235 continue; 236 IsAllUndef = false; 237 // Do not accept build_vectors that aren't all constants or which have non-0 238 // elements. We have to be a bit careful here, as the type of the constant 239 // may not be the same as the type of the vector elements due to type 240 // legalization (the elements are promoted to a legal type for the target 241 // and a vector of a type may be legal when the base element type is not). 242 // We only want to check enough bits to cover the vector elements, because 243 // we care if the resultant vector is all zeros, not whether the individual 244 // constants are. 245 unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); 246 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) { 247 if (CN->getAPIntValue().countTrailingZeros() < EltSize) 248 return false; 249 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) { 250 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize) 251 return false; 252 } else 253 return false; 254 } 255 256 // Do not accept an all-undef vector. 257 if (IsAllUndef) 258 return false; 259 return true; 260 } 261 262 bool ISD::isBuildVectorAllOnes(const SDNode *N) { 263 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true); 264 } 265 266 bool ISD::isBuildVectorAllZeros(const SDNode *N) { 267 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true); 268 } 269 270 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { 271 if (N->getOpcode() != ISD::BUILD_VECTOR) 272 return false; 273 274 for (const SDValue &Op : N->op_values()) { 275 if (Op.isUndef()) 276 continue; 277 if (!isa<ConstantSDNode>(Op)) 278 return false; 279 } 280 return true; 281 } 282 283 bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { 284 if (N->getOpcode() != ISD::BUILD_VECTOR) 285 return false; 286 287 for (const SDValue &Op : N->op_values()) { 288 if (Op.isUndef()) 289 continue; 290 if (!isa<ConstantFPSDNode>(Op)) 291 return false; 292 } 293 return true; 294 } 295 296 bool ISD::allOperandsUndef(const SDNode *N) { 297 // Return false if the node has no operands. 298 // This is "logically inconsistent" with the definition of "all" but 299 // is probably the desired behavior. 300 if (N->getNumOperands() == 0) 301 return false; 302 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); }); 303 } 304 305 bool ISD::matchUnaryPredicate(SDValue Op, 306 std::function<bool(ConstantSDNode *)> Match, 307 bool AllowUndefs) { 308 // FIXME: Add support for scalar UNDEF cases? 309 if (auto *Cst = dyn_cast<ConstantSDNode>(Op)) 310 return Match(Cst); 311 312 // FIXME: Add support for vector UNDEF cases? 313 if (ISD::BUILD_VECTOR != Op.getOpcode() && 314 ISD::SPLAT_VECTOR != Op.getOpcode()) 315 return false; 316 317 EVT SVT = Op.getValueType().getScalarType(); 318 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { 319 if (AllowUndefs && Op.getOperand(i).isUndef()) { 320 if (!Match(nullptr)) 321 return false; 322 continue; 323 } 324 325 auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i)); 326 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst)) 327 return false; 328 } 329 return true; 330 } 331 332 bool ISD::matchBinaryPredicate( 333 SDValue LHS, SDValue RHS, 334 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match, 335 bool AllowUndefs, bool AllowTypeMismatch) { 336 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType()) 337 return false; 338 339 // TODO: Add support for scalar UNDEF cases? 340 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS)) 341 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS)) 342 return Match(LHSCst, RHSCst); 343 344 // TODO: Add support for vector UNDEF cases? 345 if (LHS.getOpcode() != RHS.getOpcode() || 346 (LHS.getOpcode() != ISD::BUILD_VECTOR && 347 LHS.getOpcode() != ISD::SPLAT_VECTOR)) 348 return false; 349 350 EVT SVT = LHS.getValueType().getScalarType(); 351 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) { 352 SDValue LHSOp = LHS.getOperand(i); 353 SDValue RHSOp = RHS.getOperand(i); 354 bool LHSUndef = AllowUndefs && LHSOp.isUndef(); 355 bool RHSUndef = AllowUndefs && RHSOp.isUndef(); 356 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp); 357 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp); 358 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef)) 359 return false; 360 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT || 361 LHSOp.getValueType() != RHSOp.getValueType())) 362 return false; 363 if (!Match(LHSCst, RHSCst)) 364 return false; 365 } 366 return true; 367 } 368 369 ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { 370 switch (VecReduceOpcode) { 371 default: 372 llvm_unreachable("Expected VECREDUCE opcode"); 373 case ISD::VECREDUCE_FADD: 374 case ISD::VECREDUCE_SEQ_FADD: 375 return ISD::FADD; 376 case ISD::VECREDUCE_FMUL: 377 case ISD::VECREDUCE_SEQ_FMUL: 378 return ISD::FMUL; 379 case ISD::VECREDUCE_ADD: 380 return ISD::ADD; 381 case ISD::VECREDUCE_MUL: 382 return ISD::MUL; 383 case ISD::VECREDUCE_AND: 384 return ISD::AND; 385 case ISD::VECREDUCE_OR: 386 return ISD::OR; 387 case ISD::VECREDUCE_XOR: 388 return ISD::XOR; 389 case ISD::VECREDUCE_SMAX: 390 return ISD::SMAX; 391 case ISD::VECREDUCE_SMIN: 392 return ISD::SMIN; 393 case ISD::VECREDUCE_UMAX: 394 return ISD::UMAX; 395 case ISD::VECREDUCE_UMIN: 396 return ISD::UMIN; 397 case ISD::VECREDUCE_FMAX: 398 return ISD::FMAXNUM; 399 case ISD::VECREDUCE_FMIN: 400 return ISD::FMINNUM; 401 } 402 } 403 404 bool ISD::isVPOpcode(unsigned Opcode) { 405 switch (Opcode) { 406 default: 407 return false; 408 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \ 409 case ISD::SDOPC: \ 410 return true; 411 #include "llvm/IR/VPIntrinsics.def" 412 } 413 } 414 415 /// The operand position of the vector mask. 416 Optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) { 417 switch (Opcode) { 418 default: 419 return None; 420 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, ...) \ 421 case ISD::SDOPC: \ 422 return MASKPOS; 423 #include "llvm/IR/VPIntrinsics.def" 424 } 425 } 426 427 /// The operand position of the explicit vector length parameter. 428 Optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) { 429 switch (Opcode) { 430 default: 431 return None; 432 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \ 433 case ISD::SDOPC: \ 434 return EVLPOS; 435 #include "llvm/IR/VPIntrinsics.def" 436 } 437 } 438 439 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { 440 switch (ExtType) { 441 case ISD::EXTLOAD: 442 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND; 443 case ISD::SEXTLOAD: 444 return ISD::SIGN_EXTEND; 445 case ISD::ZEXTLOAD: 446 return ISD::ZERO_EXTEND; 447 default: 448 break; 449 } 450 451 llvm_unreachable("Invalid LoadExtType"); 452 } 453 454 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 455 // To perform this operation, we just need to swap the L and G bits of the 456 // operation. 457 unsigned OldL = (Operation >> 2) & 1; 458 unsigned OldG = (Operation >> 1) & 1; 459 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 460 (OldL << 1) | // New G bit 461 (OldG << 2)); // New L bit. 462 } 463 464 static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) { 465 unsigned Operation = Op; 466 if (isIntegerLike) 467 Operation ^= 7; // Flip L, G, E bits, but not U. 468 else 469 Operation ^= 15; // Flip all of the condition bits. 470 471 if (Operation > ISD::SETTRUE2) 472 Operation &= ~8; // Don't let N and U bits get set. 473 474 return ISD::CondCode(Operation); 475 } 476 477 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) { 478 return getSetCCInverseImpl(Op, Type.isInteger()); 479 } 480 481 ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op, 482 bool isIntegerLike) { 483 return getSetCCInverseImpl(Op, isIntegerLike); 484 } 485 486 /// For an integer comparison, return 1 if the comparison is a signed operation 487 /// and 2 if the result is an unsigned comparison. Return zero if the operation 488 /// does not depend on the sign of the input (setne and seteq). 489 static int isSignedOp(ISD::CondCode Opcode) { 490 switch (Opcode) { 491 default: llvm_unreachable("Illegal integer setcc operation!"); 492 case ISD::SETEQ: 493 case ISD::SETNE: return 0; 494 case ISD::SETLT: 495 case ISD::SETLE: 496 case ISD::SETGT: 497 case ISD::SETGE: return 1; 498 case ISD::SETULT: 499 case ISD::SETULE: 500 case ISD::SETUGT: 501 case ISD::SETUGE: return 2; 502 } 503 } 504 505 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 506 EVT Type) { 507 bool IsInteger = Type.isInteger(); 508 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 509 // Cannot fold a signed integer setcc with an unsigned integer setcc. 510 return ISD::SETCC_INVALID; 511 512 unsigned Op = Op1 | Op2; // Combine all of the condition bits. 513 514 // If the N and U bits get set, then the resultant comparison DOES suddenly 515 // care about orderedness, and it is true when ordered. 516 if (Op > ISD::SETTRUE2) 517 Op &= ~16; // Clear the U bit if the N bit is set. 518 519 // Canonicalize illegal integer setcc's. 520 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 521 Op = ISD::SETNE; 522 523 return ISD::CondCode(Op); 524 } 525 526 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 527 EVT Type) { 528 bool IsInteger = Type.isInteger(); 529 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 530 // Cannot fold a signed setcc with an unsigned setcc. 531 return ISD::SETCC_INVALID; 532 533 // Combine all of the condition bits. 534 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 535 536 // Canonicalize illegal integer setcc's. 537 if (IsInteger) { 538 switch (Result) { 539 default: break; 540 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 541 case ISD::SETOEQ: // SETEQ & SETU[LG]E 542 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 543 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 544 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 545 } 546 } 547 548 return Result; 549 } 550 551 //===----------------------------------------------------------------------===// 552 // SDNode Profile Support 553 //===----------------------------------------------------------------------===// 554 555 /// AddNodeIDOpcode - Add the node opcode to the NodeID data. 556 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 557 ID.AddInteger(OpC); 558 } 559 560 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 561 /// solely with their pointer. 562 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 563 ID.AddPointer(VTList.VTs); 564 } 565 566 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 567 static void AddNodeIDOperands(FoldingSetNodeID &ID, 568 ArrayRef<SDValue> Ops) { 569 for (auto& Op : Ops) { 570 ID.AddPointer(Op.getNode()); 571 ID.AddInteger(Op.getResNo()); 572 } 573 } 574 575 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 576 static void AddNodeIDOperands(FoldingSetNodeID &ID, 577 ArrayRef<SDUse> Ops) { 578 for (auto& Op : Ops) { 579 ID.AddPointer(Op.getNode()); 580 ID.AddInteger(Op.getResNo()); 581 } 582 } 583 584 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC, 585 SDVTList VTList, ArrayRef<SDValue> OpList) { 586 AddNodeIDOpcode(ID, OpC); 587 AddNodeIDValueTypes(ID, VTList); 588 AddNodeIDOperands(ID, OpList); 589 } 590 591 /// If this is an SDNode with special info, add this info to the NodeID data. 592 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 593 switch (N->getOpcode()) { 594 case ISD::TargetExternalSymbol: 595 case ISD::ExternalSymbol: 596 case ISD::MCSymbol: 597 llvm_unreachable("Should only be used on nodes with operands"); 598 default: break; // Normal nodes don't need extra info. 599 case ISD::TargetConstant: 600 case ISD::Constant: { 601 const ConstantSDNode *C = cast<ConstantSDNode>(N); 602 ID.AddPointer(C->getConstantIntValue()); 603 ID.AddBoolean(C->isOpaque()); 604 break; 605 } 606 case ISD::TargetConstantFP: 607 case ISD::ConstantFP: 608 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 609 break; 610 case ISD::TargetGlobalAddress: 611 case ISD::GlobalAddress: 612 case ISD::TargetGlobalTLSAddress: 613 case ISD::GlobalTLSAddress: { 614 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 615 ID.AddPointer(GA->getGlobal()); 616 ID.AddInteger(GA->getOffset()); 617 ID.AddInteger(GA->getTargetFlags()); 618 break; 619 } 620 case ISD::BasicBlock: 621 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 622 break; 623 case ISD::Register: 624 ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 625 break; 626 case ISD::RegisterMask: 627 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 628 break; 629 case ISD::SRCVALUE: 630 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 631 break; 632 case ISD::FrameIndex: 633 case ISD::TargetFrameIndex: 634 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 635 break; 636 case ISD::LIFETIME_START: 637 case ISD::LIFETIME_END: 638 if (cast<LifetimeSDNode>(N)->hasOffset()) { 639 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize()); 640 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset()); 641 } 642 break; 643 case ISD::PSEUDO_PROBE: 644 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid()); 645 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex()); 646 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes()); 647 break; 648 case ISD::JumpTable: 649 case ISD::TargetJumpTable: 650 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 651 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 652 break; 653 case ISD::ConstantPool: 654 case ISD::TargetConstantPool: { 655 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 656 ID.AddInteger(CP->getAlign().value()); 657 ID.AddInteger(CP->getOffset()); 658 if (CP->isMachineConstantPoolEntry()) 659 CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 660 else 661 ID.AddPointer(CP->getConstVal()); 662 ID.AddInteger(CP->getTargetFlags()); 663 break; 664 } 665 case ISD::TargetIndex: { 666 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); 667 ID.AddInteger(TI->getIndex()); 668 ID.AddInteger(TI->getOffset()); 669 ID.AddInteger(TI->getTargetFlags()); 670 break; 671 } 672 case ISD::LOAD: { 673 const LoadSDNode *LD = cast<LoadSDNode>(N); 674 ID.AddInteger(LD->getMemoryVT().getRawBits()); 675 ID.AddInteger(LD->getRawSubclassData()); 676 ID.AddInteger(LD->getPointerInfo().getAddrSpace()); 677 break; 678 } 679 case ISD::STORE: { 680 const StoreSDNode *ST = cast<StoreSDNode>(N); 681 ID.AddInteger(ST->getMemoryVT().getRawBits()); 682 ID.AddInteger(ST->getRawSubclassData()); 683 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 684 break; 685 } 686 case ISD::MLOAD: { 687 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N); 688 ID.AddInteger(MLD->getMemoryVT().getRawBits()); 689 ID.AddInteger(MLD->getRawSubclassData()); 690 ID.AddInteger(MLD->getPointerInfo().getAddrSpace()); 691 break; 692 } 693 case ISD::MSTORE: { 694 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); 695 ID.AddInteger(MST->getMemoryVT().getRawBits()); 696 ID.AddInteger(MST->getRawSubclassData()); 697 ID.AddInteger(MST->getPointerInfo().getAddrSpace()); 698 break; 699 } 700 case ISD::MGATHER: { 701 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N); 702 ID.AddInteger(MG->getMemoryVT().getRawBits()); 703 ID.AddInteger(MG->getRawSubclassData()); 704 ID.AddInteger(MG->getPointerInfo().getAddrSpace()); 705 break; 706 } 707 case ISD::MSCATTER: { 708 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N); 709 ID.AddInteger(MS->getMemoryVT().getRawBits()); 710 ID.AddInteger(MS->getRawSubclassData()); 711 ID.AddInteger(MS->getPointerInfo().getAddrSpace()); 712 break; 713 } 714 case ISD::ATOMIC_CMP_SWAP: 715 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 716 case ISD::ATOMIC_SWAP: 717 case ISD::ATOMIC_LOAD_ADD: 718 case ISD::ATOMIC_LOAD_SUB: 719 case ISD::ATOMIC_LOAD_AND: 720 case ISD::ATOMIC_LOAD_CLR: 721 case ISD::ATOMIC_LOAD_OR: 722 case ISD::ATOMIC_LOAD_XOR: 723 case ISD::ATOMIC_LOAD_NAND: 724 case ISD::ATOMIC_LOAD_MIN: 725 case ISD::ATOMIC_LOAD_MAX: 726 case ISD::ATOMIC_LOAD_UMIN: 727 case ISD::ATOMIC_LOAD_UMAX: 728 case ISD::ATOMIC_LOAD: 729 case ISD::ATOMIC_STORE: { 730 const AtomicSDNode *AT = cast<AtomicSDNode>(N); 731 ID.AddInteger(AT->getMemoryVT().getRawBits()); 732 ID.AddInteger(AT->getRawSubclassData()); 733 ID.AddInteger(AT->getPointerInfo().getAddrSpace()); 734 break; 735 } 736 case ISD::PREFETCH: { 737 const MemSDNode *PF = cast<MemSDNode>(N); 738 ID.AddInteger(PF->getPointerInfo().getAddrSpace()); 739 break; 740 } 741 case ISD::VECTOR_SHUFFLE: { 742 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 743 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 744 i != e; ++i) 745 ID.AddInteger(SVN->getMaskElt(i)); 746 break; 747 } 748 case ISD::TargetBlockAddress: 749 case ISD::BlockAddress: { 750 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); 751 ID.AddPointer(BA->getBlockAddress()); 752 ID.AddInteger(BA->getOffset()); 753 ID.AddInteger(BA->getTargetFlags()); 754 break; 755 } 756 } // end switch (N->getOpcode()) 757 758 // Target specific memory nodes could also have address spaces to check. 759 if (N->isTargetMemoryOpcode()) 760 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); 761 } 762 763 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 764 /// data. 765 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 766 AddNodeIDOpcode(ID, N->getOpcode()); 767 // Add the return value info. 768 AddNodeIDValueTypes(ID, N->getVTList()); 769 // Add the operand info. 770 AddNodeIDOperands(ID, N->ops()); 771 772 // Handle SDNode leafs with special info. 773 AddNodeIDCustom(ID, N); 774 } 775 776 //===----------------------------------------------------------------------===// 777 // SelectionDAG Class 778 //===----------------------------------------------------------------------===// 779 780 /// doNotCSE - Return true if CSE should not be performed for this node. 781 static bool doNotCSE(SDNode *N) { 782 if (N->getValueType(0) == MVT::Glue) 783 return true; // Never CSE anything that produces a flag. 784 785 switch (N->getOpcode()) { 786 default: break; 787 case ISD::HANDLENODE: 788 case ISD::EH_LABEL: 789 return true; // Never CSE these nodes. 790 } 791 792 // Check that remaining values produced are not flags. 793 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 794 if (N->getValueType(i) == MVT::Glue) 795 return true; // Never CSE anything that produces a flag. 796 797 return false; 798 } 799 800 /// RemoveDeadNodes - This method deletes all unreachable nodes in the 801 /// SelectionDAG. 802 void SelectionDAG::RemoveDeadNodes() { 803 // Create a dummy node (which is not added to allnodes), that adds a reference 804 // to the root node, preventing it from being deleted. 805 HandleSDNode Dummy(getRoot()); 806 807 SmallVector<SDNode*, 128> DeadNodes; 808 809 // Add all obviously-dead nodes to the DeadNodes worklist. 810 for (SDNode &Node : allnodes()) 811 if (Node.use_empty()) 812 DeadNodes.push_back(&Node); 813 814 RemoveDeadNodes(DeadNodes); 815 816 // If the root changed (e.g. it was a dead load, update the root). 817 setRoot(Dummy.getValue()); 818 } 819 820 /// RemoveDeadNodes - This method deletes the unreachable nodes in the 821 /// given list, and any nodes that become unreachable as a result. 822 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { 823 824 // Process the worklist, deleting the nodes and adding their uses to the 825 // worklist. 826 while (!DeadNodes.empty()) { 827 SDNode *N = DeadNodes.pop_back_val(); 828 // Skip to next node if we've already managed to delete the node. This could 829 // happen if replacing a node causes a node previously added to the node to 830 // be deleted. 831 if (N->getOpcode() == ISD::DELETED_NODE) 832 continue; 833 834 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 835 DUL->NodeDeleted(N, nullptr); 836 837 // Take the node out of the appropriate CSE map. 838 RemoveNodeFromCSEMaps(N); 839 840 // Next, brutally remove the operand list. This is safe to do, as there are 841 // no cycles in the graph. 842 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 843 SDUse &Use = *I++; 844 SDNode *Operand = Use.getNode(); 845 Use.set(SDValue()); 846 847 // Now that we removed this operand, see if there are no uses of it left. 848 if (Operand->use_empty()) 849 DeadNodes.push_back(Operand); 850 } 851 852 DeallocateNode(N); 853 } 854 } 855 856 void SelectionDAG::RemoveDeadNode(SDNode *N){ 857 SmallVector<SDNode*, 16> DeadNodes(1, N); 858 859 // Create a dummy node that adds a reference to the root node, preventing 860 // it from being deleted. (This matters if the root is an operand of the 861 // dead node.) 862 HandleSDNode Dummy(getRoot()); 863 864 RemoveDeadNodes(DeadNodes); 865 } 866 867 void SelectionDAG::DeleteNode(SDNode *N) { 868 // First take this out of the appropriate CSE map. 869 RemoveNodeFromCSEMaps(N); 870 871 // Finally, remove uses due to operands of this node, remove from the 872 // AllNodes list, and delete the node. 873 DeleteNodeNotInCSEMaps(N); 874 } 875 876 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 877 assert(N->getIterator() != AllNodes.begin() && 878 "Cannot delete the entry node!"); 879 assert(N->use_empty() && "Cannot delete a node that is not dead!"); 880 881 // Drop all of the operands and decrement used node's use counts. 882 N->DropOperands(); 883 884 DeallocateNode(N); 885 } 886 887 void SDDbgInfo::add(SDDbgValue *V, bool isParameter) { 888 assert(!(V->isVariadic() && isParameter)); 889 if (isParameter) 890 ByvalParmDbgValues.push_back(V); 891 else 892 DbgValues.push_back(V); 893 for (const SDNode *Node : V->getSDNodes()) 894 if (Node) 895 DbgValMap[Node].push_back(V); 896 } 897 898 void SDDbgInfo::erase(const SDNode *Node) { 899 DbgValMapType::iterator I = DbgValMap.find(Node); 900 if (I == DbgValMap.end()) 901 return; 902 for (auto &Val: I->second) 903 Val->setIsInvalidated(); 904 DbgValMap.erase(I); 905 } 906 907 void SelectionDAG::DeallocateNode(SDNode *N) { 908 // If we have operands, deallocate them. 909 removeOperands(N); 910 911 NodeAllocator.Deallocate(AllNodes.remove(N)); 912 913 // Set the opcode to DELETED_NODE to help catch bugs when node 914 // memory is reallocated. 915 // FIXME: There are places in SDag that have grown a dependency on the opcode 916 // value in the released node. 917 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType)); 918 N->NodeType = ISD::DELETED_NODE; 919 920 // If any of the SDDbgValue nodes refer to this SDNode, invalidate 921 // them and forget about that node. 922 DbgInfo->erase(N); 923 } 924 925 #ifndef NDEBUG 926 /// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 927 static void VerifySDNode(SDNode *N) { 928 switch (N->getOpcode()) { 929 default: 930 break; 931 case ISD::BUILD_PAIR: { 932 EVT VT = N->getValueType(0); 933 assert(N->getNumValues() == 1 && "Too many results!"); 934 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 935 "Wrong return type!"); 936 assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 937 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 938 "Mismatched operand types!"); 939 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 940 "Wrong operand type!"); 941 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 942 "Wrong return type size"); 943 break; 944 } 945 case ISD::BUILD_VECTOR: { 946 assert(N->getNumValues() == 1 && "Too many results!"); 947 assert(N->getValueType(0).isVector() && "Wrong return type!"); 948 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 949 "Wrong number of operands!"); 950 EVT EltVT = N->getValueType(0).getVectorElementType(); 951 for (const SDUse &Op : N->ops()) { 952 assert((Op.getValueType() == EltVT || 953 (EltVT.isInteger() && Op.getValueType().isInteger() && 954 EltVT.bitsLE(Op.getValueType()))) && 955 "Wrong operand type!"); 956 assert(Op.getValueType() == N->getOperand(0).getValueType() && 957 "Operands must all have the same type"); 958 } 959 break; 960 } 961 } 962 } 963 #endif // NDEBUG 964 965 /// Insert a newly allocated node into the DAG. 966 /// 967 /// Handles insertion into the all nodes list and CSE map, as well as 968 /// verification and other common operations when a new node is allocated. 969 void SelectionDAG::InsertNode(SDNode *N) { 970 AllNodes.push_back(N); 971 #ifndef NDEBUG 972 N->PersistentId = NextPersistentId++; 973 VerifySDNode(N); 974 #endif 975 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 976 DUL->NodeInserted(N); 977 } 978 979 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 980 /// correspond to it. This is useful when we're about to delete or repurpose 981 /// the node. We don't want future request for structurally identical nodes 982 /// to return N anymore. 983 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 984 bool Erased = false; 985 switch (N->getOpcode()) { 986 case ISD::HANDLENODE: return false; // noop. 987 case ISD::CONDCODE: 988 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 989 "Cond code doesn't exist!"); 990 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr; 991 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr; 992 break; 993 case ISD::ExternalSymbol: 994 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 995 break; 996 case ISD::TargetExternalSymbol: { 997 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 998 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>( 999 ESN->getSymbol(), ESN->getTargetFlags())); 1000 break; 1001 } 1002 case ISD::MCSymbol: { 1003 auto *MCSN = cast<MCSymbolSDNode>(N); 1004 Erased = MCSymbols.erase(MCSN->getMCSymbol()); 1005 break; 1006 } 1007 case ISD::VALUETYPE: { 1008 EVT VT = cast<VTSDNode>(N)->getVT(); 1009 if (VT.isExtended()) { 1010 Erased = ExtendedValueTypeNodes.erase(VT); 1011 } else { 1012 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr; 1013 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr; 1014 } 1015 break; 1016 } 1017 default: 1018 // Remove it from the CSE Map. 1019 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 1020 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 1021 Erased = CSEMap.RemoveNode(N); 1022 break; 1023 } 1024 #ifndef NDEBUG 1025 // Verify that the node was actually in one of the CSE maps, unless it has a 1026 // flag result (which cannot be CSE'd) or is one of the special cases that are 1027 // not subject to CSE. 1028 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 1029 !N->isMachineOpcode() && !doNotCSE(N)) { 1030 N->dump(this); 1031 dbgs() << "\n"; 1032 llvm_unreachable("Node is not in map!"); 1033 } 1034 #endif 1035 return Erased; 1036 } 1037 1038 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 1039 /// maps and modified in place. Add it back to the CSE maps, unless an identical 1040 /// node already exists, in which case transfer all its users to the existing 1041 /// node. This transfer can potentially trigger recursive merging. 1042 void 1043 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { 1044 // For node types that aren't CSE'd, just act as if no identical node 1045 // already exists. 1046 if (!doNotCSE(N)) { 1047 SDNode *Existing = CSEMap.GetOrInsertNode(N); 1048 if (Existing != N) { 1049 // If there was already an existing matching node, use ReplaceAllUsesWith 1050 // to replace the dead one with the existing one. This can cause 1051 // recursive merging of other unrelated nodes down the line. 1052 ReplaceAllUsesWith(N, Existing); 1053 1054 // N is now dead. Inform the listeners and delete it. 1055 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 1056 DUL->NodeDeleted(N, Existing); 1057 DeleteNodeNotInCSEMaps(N); 1058 return; 1059 } 1060 } 1061 1062 // If the node doesn't already exist, we updated it. Inform listeners. 1063 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 1064 DUL->NodeUpdated(N); 1065 } 1066 1067 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 1068 /// were replaced with those specified. If this node is never memoized, 1069 /// return null, otherwise return a pointer to the slot it would take. If a 1070 /// node already exists with these operands, the slot will be non-null. 1071 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 1072 void *&InsertPos) { 1073 if (doNotCSE(N)) 1074 return nullptr; 1075 1076 SDValue Ops[] = { Op }; 1077 FoldingSetNodeID ID; 1078 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); 1079 AddNodeIDCustom(ID, N); 1080 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); 1081 if (Node) 1082 Node->intersectFlagsWith(N->getFlags()); 1083 return Node; 1084 } 1085 1086 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 1087 /// were replaced with those specified. If this node is never memoized, 1088 /// return null, otherwise return a pointer to the slot it would take. If a 1089 /// node already exists with these operands, the slot will be non-null. 1090 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 1091 SDValue Op1, SDValue Op2, 1092 void *&InsertPos) { 1093 if (doNotCSE(N)) 1094 return nullptr; 1095 1096 SDValue Ops[] = { Op1, Op2 }; 1097 FoldingSetNodeID ID; 1098 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); 1099 AddNodeIDCustom(ID, N); 1100 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); 1101 if (Node) 1102 Node->intersectFlagsWith(N->getFlags()); 1103 return Node; 1104 } 1105 1106 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 1107 /// were replaced with those specified. If this node is never memoized, 1108 /// return null, otherwise return a pointer to the slot it would take. If a 1109 /// node already exists with these operands, the slot will be non-null. 1110 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops, 1111 void *&InsertPos) { 1112 if (doNotCSE(N)) 1113 return nullptr; 1114 1115 FoldingSetNodeID ID; 1116 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); 1117 AddNodeIDCustom(ID, N); 1118 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); 1119 if (Node) 1120 Node->intersectFlagsWith(N->getFlags()); 1121 return Node; 1122 } 1123 1124 Align SelectionDAG::getEVTAlign(EVT VT) const { 1125 Type *Ty = VT == MVT::iPTR ? 1126 PointerType::get(Type::getInt8Ty(*getContext()), 0) : 1127 VT.getTypeForEVT(*getContext()); 1128 1129 return getDataLayout().getABITypeAlign(Ty); 1130 } 1131 1132 // EntryNode could meaningfully have debug info if we can find it... 1133 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 1134 : TM(tm), OptLevel(OL), 1135 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), 1136 Root(getEntryNode()) { 1137 InsertNode(&EntryNode); 1138 DbgInfo = new SDDbgInfo(); 1139 } 1140 1141 void SelectionDAG::init(MachineFunction &NewMF, 1142 OptimizationRemarkEmitter &NewORE, 1143 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, 1144 LegacyDivergenceAnalysis * Divergence, 1145 ProfileSummaryInfo *PSIin, 1146 BlockFrequencyInfo *BFIin) { 1147 MF = &NewMF; 1148 SDAGISelPass = PassPtr; 1149 ORE = &NewORE; 1150 TLI = getSubtarget().getTargetLowering(); 1151 TSI = getSubtarget().getSelectionDAGInfo(); 1152 LibInfo = LibraryInfo; 1153 Context = &MF->getFunction().getContext(); 1154 DA = Divergence; 1155 PSI = PSIin; 1156 BFI = BFIin; 1157 } 1158 1159 SelectionDAG::~SelectionDAG() { 1160 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); 1161 allnodes_clear(); 1162 OperandRecycler.clear(OperandAllocator); 1163 delete DbgInfo; 1164 } 1165 1166 bool SelectionDAG::shouldOptForSize() const { 1167 return MF->getFunction().hasOptSize() || 1168 llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI); 1169 } 1170 1171 void SelectionDAG::allnodes_clear() { 1172 assert(&*AllNodes.begin() == &EntryNode); 1173 AllNodes.remove(AllNodes.begin()); 1174 while (!AllNodes.empty()) 1175 DeallocateNode(&AllNodes.front()); 1176 #ifndef NDEBUG 1177 NextPersistentId = 0; 1178 #endif 1179 } 1180 1181 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, 1182 void *&InsertPos) { 1183 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 1184 if (N) { 1185 switch (N->getOpcode()) { 1186 default: break; 1187 case ISD::Constant: 1188 case ISD::ConstantFP: 1189 llvm_unreachable("Querying for Constant and ConstantFP nodes requires " 1190 "debug location. Use another overload."); 1191 } 1192 } 1193 return N; 1194 } 1195 1196 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, 1197 const SDLoc &DL, void *&InsertPos) { 1198 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 1199 if (N) { 1200 switch (N->getOpcode()) { 1201 case ISD::Constant: 1202 case ISD::ConstantFP: 1203 // Erase debug location from the node if the node is used at several 1204 // different places. Do not propagate one location to all uses as it 1205 // will cause a worse single stepping debugging experience. 1206 if (N->getDebugLoc() != DL.getDebugLoc()) 1207 N->setDebugLoc(DebugLoc()); 1208 break; 1209 default: 1210 // When the node's point of use is located earlier in the instruction 1211 // sequence than its prior point of use, update its debug info to the 1212 // earlier location. 1213 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder()) 1214 N->setDebugLoc(DL.getDebugLoc()); 1215 break; 1216 } 1217 } 1218 return N; 1219 } 1220 1221 void SelectionDAG::clear() { 1222 allnodes_clear(); 1223 OperandRecycler.clear(OperandAllocator); 1224 OperandAllocator.Reset(); 1225 CSEMap.clear(); 1226 1227 ExtendedValueTypeNodes.clear(); 1228 ExternalSymbols.clear(); 1229 TargetExternalSymbols.clear(); 1230 MCSymbols.clear(); 1231 SDCallSiteDbgInfo.clear(); 1232 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 1233 static_cast<CondCodeSDNode*>(nullptr)); 1234 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 1235 static_cast<SDNode*>(nullptr)); 1236 1237 EntryNode.UseList = nullptr; 1238 InsertNode(&EntryNode); 1239 Root = getEntryNode(); 1240 DbgInfo->clear(); 1241 } 1242 1243 SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) { 1244 return VT.bitsGT(Op.getValueType()) 1245 ? getNode(ISD::FP_EXTEND, DL, VT, Op) 1246 : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL)); 1247 } 1248 1249 std::pair<SDValue, SDValue> 1250 SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain, 1251 const SDLoc &DL, EVT VT) { 1252 assert(!VT.bitsEq(Op.getValueType()) && 1253 "Strict no-op FP extend/round not allowed."); 1254 SDValue Res = 1255 VT.bitsGT(Op.getValueType()) 1256 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}) 1257 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other}, 1258 {Chain, Op, getIntPtrConstant(0, DL)}); 1259 1260 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1)); 1261 } 1262 1263 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1264 return VT.bitsGT(Op.getValueType()) ? 1265 getNode(ISD::ANY_EXTEND, DL, VT, Op) : 1266 getNode(ISD::TRUNCATE, DL, VT, Op); 1267 } 1268 1269 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1270 return VT.bitsGT(Op.getValueType()) ? 1271 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 1272 getNode(ISD::TRUNCATE, DL, VT, Op); 1273 } 1274 1275 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1276 return VT.bitsGT(Op.getValueType()) ? 1277 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 1278 getNode(ISD::TRUNCATE, DL, VT, Op); 1279 } 1280 1281 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, 1282 EVT OpVT) { 1283 if (VT.bitsLE(Op.getValueType())) 1284 return getNode(ISD::TRUNCATE, SL, VT, Op); 1285 1286 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT); 1287 return getNode(TLI->getExtendForContent(BType), SL, VT, Op); 1288 } 1289 1290 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { 1291 EVT OpVT = Op.getValueType(); 1292 assert(VT.isInteger() && OpVT.isInteger() && 1293 "Cannot getZeroExtendInReg FP types"); 1294 assert(VT.isVector() == OpVT.isVector() && 1295 "getZeroExtendInReg type should be vector iff the operand " 1296 "type is vector!"); 1297 assert((!VT.isVector() || 1298 VT.getVectorElementCount() == OpVT.getVectorElementCount()) && 1299 "Vector element counts must match in getZeroExtendInReg"); 1300 assert(VT.bitsLE(OpVT) && "Not extending!"); 1301 if (OpVT == VT) 1302 return Op; 1303 APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(), 1304 VT.getScalarSizeInBits()); 1305 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT)); 1306 } 1307 1308 SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1309 // Only unsigned pointer semantics are supported right now. In the future this 1310 // might delegate to TLI to check pointer signedness. 1311 return getZExtOrTrunc(Op, DL, VT); 1312 } 1313 1314 SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { 1315 // Only unsigned pointer semantics are supported right now. In the future this 1316 // might delegate to TLI to check pointer signedness. 1317 return getZeroExtendInReg(Op, DL, VT); 1318 } 1319 1320 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 1321 SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) { 1322 EVT EltVT = VT.getScalarType(); 1323 SDValue NegOne = 1324 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT); 1325 return getNode(ISD::XOR, DL, VT, Val, NegOne); 1326 } 1327 1328 SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) { 1329 SDValue TrueValue = getBoolConstant(true, DL, VT, VT); 1330 return getNode(ISD::XOR, DL, VT, Val, TrueValue); 1331 } 1332 1333 SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT, 1334 EVT OpVT) { 1335 if (!V) 1336 return getConstant(0, DL, VT); 1337 1338 switch (TLI->getBooleanContents(OpVT)) { 1339 case TargetLowering::ZeroOrOneBooleanContent: 1340 case TargetLowering::UndefinedBooleanContent: 1341 return getConstant(1, DL, VT); 1342 case TargetLowering::ZeroOrNegativeOneBooleanContent: 1343 return getAllOnesConstant(DL, VT); 1344 } 1345 llvm_unreachable("Unexpected boolean content enum!"); 1346 } 1347 1348 SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT, 1349 bool isT, bool isO) { 1350 EVT EltVT = VT.getScalarType(); 1351 assert((EltVT.getSizeInBits() >= 64 || 1352 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 1353 "getConstant with a uint64_t value that doesn't fit in the type!"); 1354 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO); 1355 } 1356 1357 SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT, 1358 bool isT, bool isO) { 1359 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO); 1360 } 1361 1362 SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, 1363 EVT VT, bool isT, bool isO) { 1364 assert(VT.isInteger() && "Cannot create FP integer constant!"); 1365 1366 EVT EltVT = VT.getScalarType(); 1367 const ConstantInt *Elt = &Val; 1368 1369 // In some cases the vector type is legal but the element type is illegal and 1370 // needs to be promoted, for example v8i8 on ARM. In this case, promote the 1371 // inserted value (the type does not need to match the vector element type). 1372 // Any extra bits introduced will be truncated away. 1373 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == 1374 TargetLowering::TypePromoteInteger) { 1375 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 1376 APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits()); 1377 Elt = ConstantInt::get(*getContext(), NewVal); 1378 } 1379 // In other cases the element type is illegal and needs to be expanded, for 1380 // example v2i64 on MIPS32. In this case, find the nearest legal type, split 1381 // the value into n parts and use a vector type with n-times the elements. 1382 // Then bitcast to the type requested. 1383 // Legalizing constants too early makes the DAGCombiner's job harder so we 1384 // only legalize if the DAG tells us we must produce legal types. 1385 else if (NewNodesMustHaveLegalTypes && VT.isVector() && 1386 TLI->getTypeAction(*getContext(), EltVT) == 1387 TargetLowering::TypeExpandInteger) { 1388 const APInt &NewVal = Elt->getValue(); 1389 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 1390 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); 1391 1392 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node. 1393 if (VT.isScalableVector()) { 1394 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 && 1395 "Can only handle an even split!"); 1396 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits; 1397 1398 SmallVector<SDValue, 2> ScalarParts; 1399 for (unsigned i = 0; i != Parts; ++i) 1400 ScalarParts.push_back(getConstant( 1401 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL, 1402 ViaEltVT, isT, isO)); 1403 1404 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts); 1405 } 1406 1407 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; 1408 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts); 1409 1410 // Check the temporary vector is the correct size. If this fails then 1411 // getTypeToTransformTo() probably returned a type whose size (in bits) 1412 // isn't a power-of-2 factor of the requested type size. 1413 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); 1414 1415 SmallVector<SDValue, 2> EltParts; 1416 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) 1417 EltParts.push_back(getConstant( 1418 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL, 1419 ViaEltVT, isT, isO)); 1420 1421 // EltParts is currently in little endian order. If we actually want 1422 // big-endian order then reverse it now. 1423 if (getDataLayout().isBigEndian()) 1424 std::reverse(EltParts.begin(), EltParts.end()); 1425 1426 // The elements must be reversed when the element order is different 1427 // to the endianness of the elements (because the BITCAST is itself a 1428 // vector shuffle in this situation). However, we do not need any code to 1429 // perform this reversal because getConstant() is producing a vector 1430 // splat. 1431 // This situation occurs in MIPS MSA. 1432 1433 SmallVector<SDValue, 8> Ops; 1434 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) 1435 llvm::append_range(Ops, EltParts); 1436 1437 SDValue V = 1438 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); 1439 return V; 1440 } 1441 1442 assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 1443 "APInt size does not match type size!"); 1444 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 1445 FoldingSetNodeID ID; 1446 AddNodeIDNode(ID, Opc, getVTList(EltVT), None); 1447 ID.AddPointer(Elt); 1448 ID.AddBoolean(isO); 1449 void *IP = nullptr; 1450 SDNode *N = nullptr; 1451 if ((N = FindNodeOrInsertPos(ID, DL, IP))) 1452 if (!VT.isVector()) 1453 return SDValue(N, 0); 1454 1455 if (!N) { 1456 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT); 1457 CSEMap.InsertNode(N, IP); 1458 InsertNode(N); 1459 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this); 1460 } 1461 1462 SDValue Result(N, 0); 1463 if (VT.isScalableVector()) 1464 Result = getSplatVector(VT, DL, Result); 1465 else if (VT.isVector()) 1466 Result = getSplatBuildVector(VT, DL, Result); 1467 1468 return Result; 1469 } 1470 1471 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL, 1472 bool isTarget) { 1473 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget); 1474 } 1475 1476 SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT, 1477 const SDLoc &DL, bool LegalTypes) { 1478 assert(VT.isInteger() && "Shift amount is not an integer type!"); 1479 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes); 1480 return getConstant(Val, DL, ShiftVT); 1481 } 1482 1483 SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL, 1484 bool isTarget) { 1485 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget); 1486 } 1487 1488 SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT, 1489 bool isTarget) { 1490 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget); 1491 } 1492 1493 SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL, 1494 EVT VT, bool isTarget) { 1495 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 1496 1497 EVT EltVT = VT.getScalarType(); 1498 1499 // Do the map lookup using the actual bit pattern for the floating point 1500 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1501 // we don't have issues with SNANs. 1502 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1503 FoldingSetNodeID ID; 1504 AddNodeIDNode(ID, Opc, getVTList(EltVT), None); 1505 ID.AddPointer(&V); 1506 void *IP = nullptr; 1507 SDNode *N = nullptr; 1508 if ((N = FindNodeOrInsertPos(ID, DL, IP))) 1509 if (!VT.isVector()) 1510 return SDValue(N, 0); 1511 1512 if (!N) { 1513 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT); 1514 CSEMap.InsertNode(N, IP); 1515 InsertNode(N); 1516 } 1517 1518 SDValue Result(N, 0); 1519 if (VT.isScalableVector()) 1520 Result = getSplatVector(VT, DL, Result); 1521 else if (VT.isVector()) 1522 Result = getSplatBuildVector(VT, DL, Result); 1523 NewSDValueDbgMsg(Result, "Creating fp constant: ", this); 1524 return Result; 1525 } 1526 1527 SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT, 1528 bool isTarget) { 1529 EVT EltVT = VT.getScalarType(); 1530 if (EltVT == MVT::f32) 1531 return getConstantFP(APFloat((float)Val), DL, VT, isTarget); 1532 if (EltVT == MVT::f64) 1533 return getConstantFP(APFloat(Val), DL, VT, isTarget); 1534 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 || 1535 EltVT == MVT::f16 || EltVT == MVT::bf16) { 1536 bool Ignored; 1537 APFloat APF = APFloat(Val); 1538 APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1539 &Ignored); 1540 return getConstantFP(APF, DL, VT, isTarget); 1541 } 1542 llvm_unreachable("Unsupported type in getConstantFP"); 1543 } 1544 1545 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, 1546 EVT VT, int64_t Offset, bool isTargetGA, 1547 unsigned TargetFlags) { 1548 assert((TargetFlags == 0 || isTargetGA) && 1549 "Cannot set target flags on target-independent globals"); 1550 1551 // Truncate (with sign-extension) the offset value to the pointer size. 1552 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); 1553 if (BitWidth < 64) 1554 Offset = SignExtend64(Offset, BitWidth); 1555 1556 unsigned Opc; 1557 if (GV->isThreadLocal()) 1558 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1559 else 1560 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1561 1562 FoldingSetNodeID ID; 1563 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1564 ID.AddPointer(GV); 1565 ID.AddInteger(Offset); 1566 ID.AddInteger(TargetFlags); 1567 void *IP = nullptr; 1568 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 1569 return SDValue(E, 0); 1570 1571 auto *N = newSDNode<GlobalAddressSDNode>( 1572 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags); 1573 CSEMap.InsertNode(N, IP); 1574 InsertNode(N); 1575 return SDValue(N, 0); 1576 } 1577 1578 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1579 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1580 FoldingSetNodeID ID; 1581 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1582 ID.AddInteger(FI); 1583 void *IP = nullptr; 1584 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1585 return SDValue(E, 0); 1586 1587 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget); 1588 CSEMap.InsertNode(N, IP); 1589 InsertNode(N); 1590 return SDValue(N, 0); 1591 } 1592 1593 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1594 unsigned TargetFlags) { 1595 assert((TargetFlags == 0 || isTarget) && 1596 "Cannot set target flags on target-independent jump tables"); 1597 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1598 FoldingSetNodeID ID; 1599 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1600 ID.AddInteger(JTI); 1601 ID.AddInteger(TargetFlags); 1602 void *IP = nullptr; 1603 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1604 return SDValue(E, 0); 1605 1606 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags); 1607 CSEMap.InsertNode(N, IP); 1608 InsertNode(N); 1609 return SDValue(N, 0); 1610 } 1611 1612 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1613 MaybeAlign Alignment, int Offset, 1614 bool isTarget, unsigned TargetFlags) { 1615 assert((TargetFlags == 0 || isTarget) && 1616 "Cannot set target flags on target-independent globals"); 1617 if (!Alignment) 1618 Alignment = shouldOptForSize() 1619 ? getDataLayout().getABITypeAlign(C->getType()) 1620 : getDataLayout().getPrefTypeAlign(C->getType()); 1621 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1622 FoldingSetNodeID ID; 1623 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1624 ID.AddInteger(Alignment->value()); 1625 ID.AddInteger(Offset); 1626 ID.AddPointer(C); 1627 ID.AddInteger(TargetFlags); 1628 void *IP = nullptr; 1629 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1630 return SDValue(E, 0); 1631 1632 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment, 1633 TargetFlags); 1634 CSEMap.InsertNode(N, IP); 1635 InsertNode(N); 1636 SDValue V = SDValue(N, 0); 1637 NewSDValueDbgMsg(V, "Creating new constant pool: ", this); 1638 return V; 1639 } 1640 1641 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1642 MaybeAlign Alignment, int Offset, 1643 bool isTarget, unsigned TargetFlags) { 1644 assert((TargetFlags == 0 || isTarget) && 1645 "Cannot set target flags on target-independent globals"); 1646 if (!Alignment) 1647 Alignment = getDataLayout().getPrefTypeAlign(C->getType()); 1648 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1649 FoldingSetNodeID ID; 1650 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1651 ID.AddInteger(Alignment->value()); 1652 ID.AddInteger(Offset); 1653 C->addSelectionDAGCSEId(ID); 1654 ID.AddInteger(TargetFlags); 1655 void *IP = nullptr; 1656 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1657 return SDValue(E, 0); 1658 1659 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment, 1660 TargetFlags); 1661 CSEMap.InsertNode(N, IP); 1662 InsertNode(N); 1663 return SDValue(N, 0); 1664 } 1665 1666 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, 1667 unsigned TargetFlags) { 1668 FoldingSetNodeID ID; 1669 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None); 1670 ID.AddInteger(Index); 1671 ID.AddInteger(Offset); 1672 ID.AddInteger(TargetFlags); 1673 void *IP = nullptr; 1674 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1675 return SDValue(E, 0); 1676 1677 auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags); 1678 CSEMap.InsertNode(N, IP); 1679 InsertNode(N); 1680 return SDValue(N, 0); 1681 } 1682 1683 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1684 FoldingSetNodeID ID; 1685 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None); 1686 ID.AddPointer(MBB); 1687 void *IP = nullptr; 1688 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1689 return SDValue(E, 0); 1690 1691 auto *N = newSDNode<BasicBlockSDNode>(MBB); 1692 CSEMap.InsertNode(N, IP); 1693 InsertNode(N); 1694 return SDValue(N, 0); 1695 } 1696 1697 SDValue SelectionDAG::getValueType(EVT VT) { 1698 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1699 ValueTypeNodes.size()) 1700 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1701 1702 SDNode *&N = VT.isExtended() ? 1703 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1704 1705 if (N) return SDValue(N, 0); 1706 N = newSDNode<VTSDNode>(VT); 1707 InsertNode(N); 1708 return SDValue(N, 0); 1709 } 1710 1711 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1712 SDNode *&N = ExternalSymbols[Sym]; 1713 if (N) return SDValue(N, 0); 1714 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT); 1715 InsertNode(N); 1716 return SDValue(N, 0); 1717 } 1718 1719 SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) { 1720 SDNode *&N = MCSymbols[Sym]; 1721 if (N) 1722 return SDValue(N, 0); 1723 N = newSDNode<MCSymbolSDNode>(Sym, VT); 1724 InsertNode(N); 1725 return SDValue(N, 0); 1726 } 1727 1728 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1729 unsigned TargetFlags) { 1730 SDNode *&N = 1731 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)]; 1732 if (N) return SDValue(N, 0); 1733 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT); 1734 InsertNode(N); 1735 return SDValue(N, 0); 1736 } 1737 1738 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1739 if ((unsigned)Cond >= CondCodeNodes.size()) 1740 CondCodeNodes.resize(Cond+1); 1741 1742 if (!CondCodeNodes[Cond]) { 1743 auto *N = newSDNode<CondCodeSDNode>(Cond); 1744 CondCodeNodes[Cond] = N; 1745 InsertNode(N); 1746 } 1747 1748 return SDValue(CondCodeNodes[Cond], 0); 1749 } 1750 1751 SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) { 1752 APInt One(ResVT.getScalarSizeInBits(), 1); 1753 return getStepVector(DL, ResVT, One); 1754 } 1755 1756 SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) { 1757 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth()); 1758 if (ResVT.isScalableVector()) 1759 return getNode( 1760 ISD::STEP_VECTOR, DL, ResVT, 1761 getTargetConstant(StepVal, DL, ResVT.getVectorElementType())); 1762 1763 SmallVector<SDValue, 16> OpsStepConstants; 1764 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++) 1765 OpsStepConstants.push_back( 1766 getConstant(StepVal * i, DL, ResVT.getVectorElementType())); 1767 return getBuildVector(ResVT, DL, OpsStepConstants); 1768 } 1769 1770 /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that 1771 /// point at N1 to point at N2 and indices that point at N2 to point at N1. 1772 static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) { 1773 std::swap(N1, N2); 1774 ShuffleVectorSDNode::commuteMask(M); 1775 } 1776 1777 SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, 1778 SDValue N2, ArrayRef<int> Mask) { 1779 assert(VT.getVectorNumElements() == Mask.size() && 1780 "Must have the same number of vector elements as mask elements!"); 1781 assert(VT == N1.getValueType() && VT == N2.getValueType() && 1782 "Invalid VECTOR_SHUFFLE"); 1783 1784 // Canonicalize shuffle undef, undef -> undef 1785 if (N1.isUndef() && N2.isUndef()) 1786 return getUNDEF(VT); 1787 1788 // Validate that all indices in Mask are within the range of the elements 1789 // input to the shuffle. 1790 int NElts = Mask.size(); 1791 assert(llvm::all_of(Mask, 1792 [&](int M) { return M < (NElts * 2) && M >= -1; }) && 1793 "Index out of range"); 1794 1795 // Copy the mask so we can do any needed cleanup. 1796 SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end()); 1797 1798 // Canonicalize shuffle v, v -> v, undef 1799 if (N1 == N2) { 1800 N2 = getUNDEF(VT); 1801 for (int i = 0; i != NElts; ++i) 1802 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts; 1803 } 1804 1805 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1806 if (N1.isUndef()) 1807 commuteShuffle(N1, N2, MaskVec); 1808 1809 if (TLI->hasVectorBlend()) { 1810 // If shuffling a splat, try to blend the splat instead. We do this here so 1811 // that even when this arises during lowering we don't have to re-handle it. 1812 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) { 1813 BitVector UndefElements; 1814 SDValue Splat = BV->getSplatValue(&UndefElements); 1815 if (!Splat) 1816 return; 1817 1818 for (int i = 0; i < NElts; ++i) { 1819 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts)) 1820 continue; 1821 1822 // If this input comes from undef, mark it as such. 1823 if (UndefElements[MaskVec[i] - Offset]) { 1824 MaskVec[i] = -1; 1825 continue; 1826 } 1827 1828 // If we can blend a non-undef lane, use that instead. 1829 if (!UndefElements[i]) 1830 MaskVec[i] = i + Offset; 1831 } 1832 }; 1833 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1)) 1834 BlendSplat(N1BV, 0); 1835 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2)) 1836 BlendSplat(N2BV, NElts); 1837 } 1838 1839 // Canonicalize all index into lhs, -> shuffle lhs, undef 1840 // Canonicalize all index into rhs, -> shuffle rhs, undef 1841 bool AllLHS = true, AllRHS = true; 1842 bool N2Undef = N2.isUndef(); 1843 for (int i = 0; i != NElts; ++i) { 1844 if (MaskVec[i] >= NElts) { 1845 if (N2Undef) 1846 MaskVec[i] = -1; 1847 else 1848 AllLHS = false; 1849 } else if (MaskVec[i] >= 0) { 1850 AllRHS = false; 1851 } 1852 } 1853 if (AllLHS && AllRHS) 1854 return getUNDEF(VT); 1855 if (AllLHS && !N2Undef) 1856 N2 = getUNDEF(VT); 1857 if (AllRHS) { 1858 N1 = getUNDEF(VT); 1859 commuteShuffle(N1, N2, MaskVec); 1860 } 1861 // Reset our undef status after accounting for the mask. 1862 N2Undef = N2.isUndef(); 1863 // Re-check whether both sides ended up undef. 1864 if (N1.isUndef() && N2Undef) 1865 return getUNDEF(VT); 1866 1867 // If Identity shuffle return that node. 1868 bool Identity = true, AllSame = true; 1869 for (int i = 0; i != NElts; ++i) { 1870 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false; 1871 if (MaskVec[i] != MaskVec[0]) AllSame = false; 1872 } 1873 if (Identity && NElts) 1874 return N1; 1875 1876 // Shuffling a constant splat doesn't change the result. 1877 if (N2Undef) { 1878 SDValue V = N1; 1879 1880 // Look through any bitcasts. We check that these don't change the number 1881 // (and size) of elements and just changes their types. 1882 while (V.getOpcode() == ISD::BITCAST) 1883 V = V->getOperand(0); 1884 1885 // A splat should always show up as a build vector node. 1886 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) { 1887 BitVector UndefElements; 1888 SDValue Splat = BV->getSplatValue(&UndefElements); 1889 // If this is a splat of an undef, shuffling it is also undef. 1890 if (Splat && Splat.isUndef()) 1891 return getUNDEF(VT); 1892 1893 bool SameNumElts = 1894 V.getValueType().getVectorNumElements() == VT.getVectorNumElements(); 1895 1896 // We only have a splat which can skip shuffles if there is a splatted 1897 // value and no undef lanes rearranged by the shuffle. 1898 if (Splat && UndefElements.none()) { 1899 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the 1900 // number of elements match or the value splatted is a zero constant. 1901 if (SameNumElts) 1902 return N1; 1903 if (auto *C = dyn_cast<ConstantSDNode>(Splat)) 1904 if (C->isNullValue()) 1905 return N1; 1906 } 1907 1908 // If the shuffle itself creates a splat, build the vector directly. 1909 if (AllSame && SameNumElts) { 1910 EVT BuildVT = BV->getValueType(0); 1911 const SDValue &Splatted = BV->getOperand(MaskVec[0]); 1912 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted); 1913 1914 // We may have jumped through bitcasts, so the type of the 1915 // BUILD_VECTOR may not match the type of the shuffle. 1916 if (BuildVT != VT) 1917 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV); 1918 return NewBV; 1919 } 1920 } 1921 } 1922 1923 FoldingSetNodeID ID; 1924 SDValue Ops[2] = { N1, N2 }; 1925 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops); 1926 for (int i = 0; i != NElts; ++i) 1927 ID.AddInteger(MaskVec[i]); 1928 1929 void* IP = nullptr; 1930 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 1931 return SDValue(E, 0); 1932 1933 // Allocate the mask array for the node out of the BumpPtrAllocator, since 1934 // SDNode doesn't have access to it. This memory will be "leaked" when 1935 // the node is deallocated, but recovered when the NodeAllocator is released. 1936 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1937 llvm::copy(MaskVec, MaskAlloc); 1938 1939 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(), 1940 dl.getDebugLoc(), MaskAlloc); 1941 createOperands(N, Ops); 1942 1943 CSEMap.InsertNode(N, IP); 1944 InsertNode(N); 1945 SDValue V = SDValue(N, 0); 1946 NewSDValueDbgMsg(V, "Creating new node: ", this); 1947 return V; 1948 } 1949 1950 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) { 1951 EVT VT = SV.getValueType(0); 1952 SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end()); 1953 ShuffleVectorSDNode::commuteMask(MaskVec); 1954 1955 SDValue Op0 = SV.getOperand(0); 1956 SDValue Op1 = SV.getOperand(1); 1957 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec); 1958 } 1959 1960 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1961 FoldingSetNodeID ID; 1962 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None); 1963 ID.AddInteger(RegNo); 1964 void *IP = nullptr; 1965 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1966 return SDValue(E, 0); 1967 1968 auto *N = newSDNode<RegisterSDNode>(RegNo, VT); 1969 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA); 1970 CSEMap.InsertNode(N, IP); 1971 InsertNode(N); 1972 return SDValue(N, 0); 1973 } 1974 1975 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 1976 FoldingSetNodeID ID; 1977 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None); 1978 ID.AddPointer(RegMask); 1979 void *IP = nullptr; 1980 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1981 return SDValue(E, 0); 1982 1983 auto *N = newSDNode<RegisterMaskSDNode>(RegMask); 1984 CSEMap.InsertNode(N, IP); 1985 InsertNode(N); 1986 return SDValue(N, 0); 1987 } 1988 1989 SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root, 1990 MCSymbol *Label) { 1991 return getLabelNode(ISD::EH_LABEL, dl, Root, Label); 1992 } 1993 1994 SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl, 1995 SDValue Root, MCSymbol *Label) { 1996 FoldingSetNodeID ID; 1997 SDValue Ops[] = { Root }; 1998 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops); 1999 ID.AddPointer(Label); 2000 void *IP = nullptr; 2001 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 2002 return SDValue(E, 0); 2003 2004 auto *N = 2005 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label); 2006 createOperands(N, Ops); 2007 2008 CSEMap.InsertNode(N, IP); 2009 InsertNode(N); 2010 return SDValue(N, 0); 2011 } 2012 2013 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 2014 int64_t Offset, bool isTarget, 2015 unsigned TargetFlags) { 2016 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 2017 2018 FoldingSetNodeID ID; 2019 AddNodeIDNode(ID, Opc, getVTList(VT), None); 2020 ID.AddPointer(BA); 2021 ID.AddInteger(Offset); 2022 ID.AddInteger(TargetFlags); 2023 void *IP = nullptr; 2024 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 2025 return SDValue(E, 0); 2026 2027 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags); 2028 CSEMap.InsertNode(N, IP); 2029 InsertNode(N); 2030 return SDValue(N, 0); 2031 } 2032 2033 SDValue SelectionDAG::getSrcValue(const Value *V) { 2034 FoldingSetNodeID ID; 2035 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None); 2036 ID.AddPointer(V); 2037 2038 void *IP = nullptr; 2039 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 2040 return SDValue(E, 0); 2041 2042 auto *N = newSDNode<SrcValueSDNode>(V); 2043 CSEMap.InsertNode(N, IP); 2044 InsertNode(N); 2045 return SDValue(N, 0); 2046 } 2047 2048 SDValue SelectionDAG::getMDNode(const MDNode *MD) { 2049 FoldingSetNodeID ID; 2050 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None); 2051 ID.AddPointer(MD); 2052 2053 void *IP = nullptr; 2054 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 2055 return SDValue(E, 0); 2056 2057 auto *N = newSDNode<MDNodeSDNode>(MD); 2058 CSEMap.InsertNode(N, IP); 2059 InsertNode(N); 2060 return SDValue(N, 0); 2061 } 2062 2063 SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) { 2064 if (VT == V.getValueType()) 2065 return V; 2066 2067 return getNode(ISD::BITCAST, SDLoc(V), VT, V); 2068 } 2069 2070 SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, 2071 unsigned SrcAS, unsigned DestAS) { 2072 SDValue Ops[] = {Ptr}; 2073 FoldingSetNodeID ID; 2074 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops); 2075 ID.AddInteger(SrcAS); 2076 ID.AddInteger(DestAS); 2077 2078 void *IP = nullptr; 2079 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 2080 return SDValue(E, 0); 2081 2082 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(), 2083 VT, SrcAS, DestAS); 2084 createOperands(N, Ops); 2085 2086 CSEMap.InsertNode(N, IP); 2087 InsertNode(N); 2088 return SDValue(N, 0); 2089 } 2090 2091 SDValue SelectionDAG::getFreeze(SDValue V) { 2092 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V); 2093 } 2094 2095 /// getShiftAmountOperand - Return the specified value casted to 2096 /// the target's desired shift amount type. 2097 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 2098 EVT OpTy = Op.getValueType(); 2099 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout()); 2100 if (OpTy == ShTy || OpTy.isVector()) return Op; 2101 2102 return getZExtOrTrunc(Op, SDLoc(Op), ShTy); 2103 } 2104 2105 SDValue SelectionDAG::expandVAArg(SDNode *Node) { 2106 SDLoc dl(Node); 2107 const TargetLowering &TLI = getTargetLoweringInfo(); 2108 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 2109 EVT VT = Node->getValueType(0); 2110 SDValue Tmp1 = Node->getOperand(0); 2111 SDValue Tmp2 = Node->getOperand(1); 2112 const MaybeAlign MA(Node->getConstantOperandVal(3)); 2113 2114 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1, 2115 Tmp2, MachinePointerInfo(V)); 2116 SDValue VAList = VAListLoad; 2117 2118 if (MA && *MA > TLI.getMinStackArgumentAlignment()) { 2119 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, 2120 getConstant(MA->value() - 1, dl, VAList.getValueType())); 2121 2122 VAList = 2123 getNode(ISD::AND, dl, VAList.getValueType(), VAList, 2124 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType())); 2125 } 2126 2127 // Increment the pointer, VAList, to the next vaarg 2128 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, 2129 getConstant(getDataLayout().getTypeAllocSize( 2130 VT.getTypeForEVT(*getContext())), 2131 dl, VAList.getValueType())); 2132 // Store the incremented VAList to the legalized pointer 2133 Tmp1 = 2134 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V)); 2135 // Load the actual argument out of the pointer VAList 2136 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo()); 2137 } 2138 2139 SDValue SelectionDAG::expandVACopy(SDNode *Node) { 2140 SDLoc dl(Node); 2141 const TargetLowering &TLI = getTargetLoweringInfo(); 2142 // This defaults to loading a pointer from the input and storing it to the 2143 // output, returning the chain. 2144 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 2145 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 2146 SDValue Tmp1 = 2147 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0), 2148 Node->getOperand(2), MachinePointerInfo(VS)); 2149 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), 2150 MachinePointerInfo(VD)); 2151 } 2152 2153 Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) { 2154 const DataLayout &DL = getDataLayout(); 2155 Type *Ty = VT.getTypeForEVT(*getContext()); 2156 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty); 2157 2158 if (TLI->isTypeLegal(VT) || !VT.isVector()) 2159 return RedAlign; 2160 2161 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); 2162 const Align StackAlign = TFI->getStackAlign(); 2163 2164 // See if we can choose a smaller ABI alignment in cases where it's an 2165 // illegal vector type that will get broken down. 2166 if (RedAlign > StackAlign) { 2167 EVT IntermediateVT; 2168 MVT RegisterVT; 2169 unsigned NumIntermediates; 2170 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT, 2171 NumIntermediates, RegisterVT); 2172 Ty = IntermediateVT.getTypeForEVT(*getContext()); 2173 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty); 2174 if (RedAlign2 < RedAlign) 2175 RedAlign = RedAlign2; 2176 } 2177 2178 return RedAlign; 2179 } 2180 2181 SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) { 2182 MachineFrameInfo &MFI = MF->getFrameInfo(); 2183 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); 2184 int StackID = 0; 2185 if (Bytes.isScalable()) 2186 StackID = TFI->getStackIDForScalableVectors(); 2187 // The stack id gives an indication of whether the object is scalable or 2188 // not, so it's safe to pass in the minimum size here. 2189 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinSize(), Alignment, 2190 false, nullptr, StackID); 2191 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout())); 2192 } 2193 2194 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 2195 Type *Ty = VT.getTypeForEVT(*getContext()); 2196 Align StackAlign = 2197 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign)); 2198 return CreateStackTemporary(VT.getStoreSize(), StackAlign); 2199 } 2200 2201 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 2202 TypeSize VT1Size = VT1.getStoreSize(); 2203 TypeSize VT2Size = VT2.getStoreSize(); 2204 assert(VT1Size.isScalable() == VT2Size.isScalable() && 2205 "Don't know how to choose the maximum size when creating a stack " 2206 "temporary"); 2207 TypeSize Bytes = 2208 VT1Size.getKnownMinSize() > VT2Size.getKnownMinSize() ? VT1Size : VT2Size; 2209 2210 Type *Ty1 = VT1.getTypeForEVT(*getContext()); 2211 Type *Ty2 = VT2.getTypeForEVT(*getContext()); 2212 const DataLayout &DL = getDataLayout(); 2213 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2)); 2214 return CreateStackTemporary(Bytes, Align); 2215 } 2216 2217 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, 2218 ISD::CondCode Cond, const SDLoc &dl) { 2219 EVT OpVT = N1.getValueType(); 2220 2221 // These setcc operations always fold. 2222 switch (Cond) { 2223 default: break; 2224 case ISD::SETFALSE: 2225 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT); 2226 case ISD::SETTRUE: 2227 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT); 2228 2229 case ISD::SETOEQ: 2230 case ISD::SETOGT: 2231 case ISD::SETOGE: 2232 case ISD::SETOLT: 2233 case ISD::SETOLE: 2234 case ISD::SETONE: 2235 case ISD::SETO: 2236 case ISD::SETUO: 2237 case ISD::SETUEQ: 2238 case ISD::SETUNE: 2239 assert(!OpVT.isInteger() && "Illegal setcc for integer!"); 2240 break; 2241 } 2242 2243 if (OpVT.isInteger()) { 2244 // For EQ and NE, we can always pick a value for the undef to make the 2245 // predicate pass or fail, so we can return undef. 2246 // Matches behavior in llvm::ConstantFoldCompareInstruction. 2247 // icmp eq/ne X, undef -> undef. 2248 if ((N1.isUndef() || N2.isUndef()) && 2249 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) 2250 return getUNDEF(VT); 2251 2252 // If both operands are undef, we can return undef for int comparison. 2253 // icmp undef, undef -> undef. 2254 if (N1.isUndef() && N2.isUndef()) 2255 return getUNDEF(VT); 2256 2257 // icmp X, X -> true/false 2258 // icmp X, undef -> true/false because undef could be X. 2259 if (N1 == N2) 2260 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT); 2261 } 2262 2263 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) { 2264 const APInt &C2 = N2C->getAPIntValue(); 2265 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) { 2266 const APInt &C1 = N1C->getAPIntValue(); 2267 2268 switch (Cond) { 2269 default: llvm_unreachable("Unknown integer setcc!"); 2270 case ISD::SETEQ: return getBoolConstant(C1 == C2, dl, VT, OpVT); 2271 case ISD::SETNE: return getBoolConstant(C1 != C2, dl, VT, OpVT); 2272 case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT); 2273 case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT); 2274 case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT); 2275 case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT); 2276 case ISD::SETLT: return getBoolConstant(C1.slt(C2), dl, VT, OpVT); 2277 case ISD::SETGT: return getBoolConstant(C1.sgt(C2), dl, VT, OpVT); 2278 case ISD::SETLE: return getBoolConstant(C1.sle(C2), dl, VT, OpVT); 2279 case ISD::SETGE: return getBoolConstant(C1.sge(C2), dl, VT, OpVT); 2280 } 2281 } 2282 } 2283 2284 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 2285 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 2286 2287 if (N1CFP && N2CFP) { 2288 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF()); 2289 switch (Cond) { 2290 default: break; 2291 case ISD::SETEQ: if (R==APFloat::cmpUnordered) 2292 return getUNDEF(VT); 2293 LLVM_FALLTHROUGH; 2294 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT, 2295 OpVT); 2296 case ISD::SETNE: if (R==APFloat::cmpUnordered) 2297 return getUNDEF(VT); 2298 LLVM_FALLTHROUGH; 2299 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan || 2300 R==APFloat::cmpLessThan, dl, VT, 2301 OpVT); 2302 case ISD::SETLT: if (R==APFloat::cmpUnordered) 2303 return getUNDEF(VT); 2304 LLVM_FALLTHROUGH; 2305 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT, 2306 OpVT); 2307 case ISD::SETGT: if (R==APFloat::cmpUnordered) 2308 return getUNDEF(VT); 2309 LLVM_FALLTHROUGH; 2310 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl, 2311 VT, OpVT); 2312 case ISD::SETLE: if (R==APFloat::cmpUnordered) 2313 return getUNDEF(VT); 2314 LLVM_FALLTHROUGH; 2315 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan || 2316 R==APFloat::cmpEqual, dl, VT, 2317 OpVT); 2318 case ISD::SETGE: if (R==APFloat::cmpUnordered) 2319 return getUNDEF(VT); 2320 LLVM_FALLTHROUGH; 2321 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan || 2322 R==APFloat::cmpEqual, dl, VT, OpVT); 2323 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT, 2324 OpVT); 2325 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT, 2326 OpVT); 2327 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered || 2328 R==APFloat::cmpEqual, dl, VT, 2329 OpVT); 2330 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT, 2331 OpVT); 2332 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered || 2333 R==APFloat::cmpLessThan, dl, VT, 2334 OpVT); 2335 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan || 2336 R==APFloat::cmpUnordered, dl, VT, 2337 OpVT); 2338 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl, 2339 VT, OpVT); 2340 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT, 2341 OpVT); 2342 } 2343 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) { 2344 // Ensure that the constant occurs on the RHS. 2345 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); 2346 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT())) 2347 return SDValue(); 2348 return getSetCC(dl, VT, N2, N1, SwappedCond); 2349 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) || 2350 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) { 2351 // If an operand is known to be a nan (or undef that could be a nan), we can 2352 // fold it. 2353 // Choosing NaN for the undef will always make unordered comparison succeed 2354 // and ordered comparison fails. 2355 // Matches behavior in llvm::ConstantFoldCompareInstruction. 2356 switch (ISD::getUnorderedFlavor(Cond)) { 2357 default: 2358 llvm_unreachable("Unknown flavor!"); 2359 case 0: // Known false. 2360 return getBoolConstant(false, dl, VT, OpVT); 2361 case 1: // Known true. 2362 return getBoolConstant(true, dl, VT, OpVT); 2363 case 2: // Undefined. 2364 return getUNDEF(VT); 2365 } 2366 } 2367 2368 // Could not fold it. 2369 return SDValue(); 2370 } 2371 2372 /// See if the specified operand can be simplified with the knowledge that only 2373 /// the bits specified by DemandedBits are used. 2374 /// TODO: really we should be making this into the DAG equivalent of 2375 /// SimplifyMultipleUseDemandedBits and not generate any new nodes. 2376 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) { 2377 EVT VT = V.getValueType(); 2378 2379 if (VT.isScalableVector()) 2380 return SDValue(); 2381 2382 APInt DemandedElts = VT.isVector() 2383 ? APInt::getAllOnesValue(VT.getVectorNumElements()) 2384 : APInt(1, 1); 2385 return GetDemandedBits(V, DemandedBits, DemandedElts); 2386 } 2387 2388 /// See if the specified operand can be simplified with the knowledge that only 2389 /// the bits specified by DemandedBits are used in the elements specified by 2390 /// DemandedElts. 2391 /// TODO: really we should be making this into the DAG equivalent of 2392 /// SimplifyMultipleUseDemandedBits and not generate any new nodes. 2393 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits, 2394 const APInt &DemandedElts) { 2395 switch (V.getOpcode()) { 2396 default: 2397 return TLI->SimplifyMultipleUseDemandedBits(V, DemandedBits, DemandedElts, 2398 *this, 0); 2399 case ISD::Constant: { 2400 const APInt &CVal = cast<ConstantSDNode>(V)->getAPIntValue(); 2401 APInt NewVal = CVal & DemandedBits; 2402 if (NewVal != CVal) 2403 return getConstant(NewVal, SDLoc(V), V.getValueType()); 2404 break; 2405 } 2406 case ISD::SRL: 2407 // Only look at single-use SRLs. 2408 if (!V.getNode()->hasOneUse()) 2409 break; 2410 if (auto *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) { 2411 // See if we can recursively simplify the LHS. 2412 unsigned Amt = RHSC->getZExtValue(); 2413 2414 // Watch out for shift count overflow though. 2415 if (Amt >= DemandedBits.getBitWidth()) 2416 break; 2417 APInt SrcDemandedBits = DemandedBits << Amt; 2418 if (SDValue SimplifyLHS = 2419 GetDemandedBits(V.getOperand(0), SrcDemandedBits)) 2420 return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS, 2421 V.getOperand(1)); 2422 } 2423 break; 2424 } 2425 return SDValue(); 2426 } 2427 2428 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 2429 /// use this predicate to simplify operations downstream. 2430 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 2431 unsigned BitWidth = Op.getScalarValueSizeInBits(); 2432 return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth); 2433 } 2434 2435 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 2436 /// this predicate to simplify operations downstream. Mask is known to be zero 2437 /// for bits that V cannot have. 2438 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, 2439 unsigned Depth) const { 2440 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero); 2441 } 2442 2443 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in 2444 /// DemandedElts. We use this predicate to simplify operations downstream. 2445 /// Mask is known to be zero for bits that V cannot have. 2446 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, 2447 const APInt &DemandedElts, 2448 unsigned Depth) const { 2449 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero); 2450 } 2451 2452 /// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'. 2453 bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask, 2454 unsigned Depth) const { 2455 return Mask.isSubsetOf(computeKnownBits(V, Depth).One); 2456 } 2457 2458 /// isSplatValue - Return true if the vector V has the same value 2459 /// across all DemandedElts. For scalable vectors it does not make 2460 /// sense to specify which elements are demanded or undefined, therefore 2461 /// they are simply ignored. 2462 bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, 2463 APInt &UndefElts, unsigned Depth) { 2464 EVT VT = V.getValueType(); 2465 assert(VT.isVector() && "Vector type expected"); 2466 2467 if (!VT.isScalableVector() && !DemandedElts) 2468 return false; // No demanded elts, better to assume we don't know anything. 2469 2470 if (Depth >= MaxRecursionDepth) 2471 return false; // Limit search depth. 2472 2473 // Deal with some common cases here that work for both fixed and scalable 2474 // vector types. 2475 switch (V.getOpcode()) { 2476 case ISD::SPLAT_VECTOR: 2477 UndefElts = V.getOperand(0).isUndef() 2478 ? APInt::getAllOnesValue(DemandedElts.getBitWidth()) 2479 : APInt(DemandedElts.getBitWidth(), 0); 2480 return true; 2481 case ISD::ADD: 2482 case ISD::SUB: 2483 case ISD::AND: 2484 case ISD::XOR: 2485 case ISD::OR: { 2486 APInt UndefLHS, UndefRHS; 2487 SDValue LHS = V.getOperand(0); 2488 SDValue RHS = V.getOperand(1); 2489 if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) && 2490 isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) { 2491 UndefElts = UndefLHS | UndefRHS; 2492 return true; 2493 } 2494 return false; 2495 } 2496 case ISD::ABS: 2497 case ISD::TRUNCATE: 2498 case ISD::SIGN_EXTEND: 2499 case ISD::ZERO_EXTEND: 2500 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1); 2501 } 2502 2503 // We don't support other cases than those above for scalable vectors at 2504 // the moment. 2505 if (VT.isScalableVector()) 2506 return false; 2507 2508 unsigned NumElts = VT.getVectorNumElements(); 2509 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch"); 2510 UndefElts = APInt::getNullValue(NumElts); 2511 2512 switch (V.getOpcode()) { 2513 case ISD::BUILD_VECTOR: { 2514 SDValue Scl; 2515 for (unsigned i = 0; i != NumElts; ++i) { 2516 SDValue Op = V.getOperand(i); 2517 if (Op.isUndef()) { 2518 UndefElts.setBit(i); 2519 continue; 2520 } 2521 if (!DemandedElts[i]) 2522 continue; 2523 if (Scl && Scl != Op) 2524 return false; 2525 Scl = Op; 2526 } 2527 return true; 2528 } 2529 case ISD::VECTOR_SHUFFLE: { 2530 // Check if this is a shuffle node doing a splat. 2531 // TODO: Do we need to handle shuffle(splat, undef, mask)? 2532 int SplatIndex = -1; 2533 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask(); 2534 for (int i = 0; i != (int)NumElts; ++i) { 2535 int M = Mask[i]; 2536 if (M < 0) { 2537 UndefElts.setBit(i); 2538 continue; 2539 } 2540 if (!DemandedElts[i]) 2541 continue; 2542 if (0 <= SplatIndex && SplatIndex != M) 2543 return false; 2544 SplatIndex = M; 2545 } 2546 return true; 2547 } 2548 case ISD::EXTRACT_SUBVECTOR: { 2549 // Offset the demanded elts by the subvector index. 2550 SDValue Src = V.getOperand(0); 2551 // We don't support scalable vectors at the moment. 2552 if (Src.getValueType().isScalableVector()) 2553 return false; 2554 uint64_t Idx = V.getConstantOperandVal(1); 2555 unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); 2556 APInt UndefSrcElts; 2557 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx); 2558 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) { 2559 UndefElts = UndefSrcElts.extractBits(NumElts, Idx); 2560 return true; 2561 } 2562 break; 2563 } 2564 } 2565 2566 return false; 2567 } 2568 2569 /// Helper wrapper to main isSplatValue function. 2570 bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) { 2571 EVT VT = V.getValueType(); 2572 assert(VT.isVector() && "Vector type expected"); 2573 2574 APInt UndefElts; 2575 APInt DemandedElts; 2576 2577 // For now we don't support this with scalable vectors. 2578 if (!VT.isScalableVector()) 2579 DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements()); 2580 return isSplatValue(V, DemandedElts, UndefElts) && 2581 (AllowUndefs || !UndefElts); 2582 } 2583 2584 SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) { 2585 V = peekThroughExtractSubvectors(V); 2586 2587 EVT VT = V.getValueType(); 2588 unsigned Opcode = V.getOpcode(); 2589 switch (Opcode) { 2590 default: { 2591 APInt UndefElts; 2592 APInt DemandedElts; 2593 2594 if (!VT.isScalableVector()) 2595 DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements()); 2596 2597 if (isSplatValue(V, DemandedElts, UndefElts)) { 2598 if (VT.isScalableVector()) { 2599 // DemandedElts and UndefElts are ignored for scalable vectors, since 2600 // the only supported cases are SPLAT_VECTOR nodes. 2601 SplatIdx = 0; 2602 } else { 2603 // Handle case where all demanded elements are UNDEF. 2604 if (DemandedElts.isSubsetOf(UndefElts)) { 2605 SplatIdx = 0; 2606 return getUNDEF(VT); 2607 } 2608 SplatIdx = (UndefElts & DemandedElts).countTrailingOnes(); 2609 } 2610 return V; 2611 } 2612 break; 2613 } 2614 case ISD::SPLAT_VECTOR: 2615 SplatIdx = 0; 2616 return V; 2617 case ISD::VECTOR_SHUFFLE: { 2618 if (VT.isScalableVector()) 2619 return SDValue(); 2620 2621 // Check if this is a shuffle node doing a splat. 2622 // TODO - remove this and rely purely on SelectionDAG::isSplatValue, 2623 // getTargetVShiftNode currently struggles without the splat source. 2624 auto *SVN = cast<ShuffleVectorSDNode>(V); 2625 if (!SVN->isSplat()) 2626 break; 2627 int Idx = SVN->getSplatIndex(); 2628 int NumElts = V.getValueType().getVectorNumElements(); 2629 SplatIdx = Idx % NumElts; 2630 return V.getOperand(Idx / NumElts); 2631 } 2632 } 2633 2634 return SDValue(); 2635 } 2636 2637 SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) { 2638 int SplatIdx; 2639 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) { 2640 EVT SVT = SrcVector.getValueType().getScalarType(); 2641 EVT LegalSVT = SVT; 2642 if (LegalTypes && !TLI->isTypeLegal(SVT)) { 2643 if (!SVT.isInteger()) 2644 return SDValue(); 2645 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); 2646 if (LegalSVT.bitsLT(SVT)) 2647 return SDValue(); 2648 } 2649 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector, 2650 getVectorIdxConstant(SplatIdx, SDLoc(V))); 2651 } 2652 return SDValue(); 2653 } 2654 2655 const APInt * 2656 SelectionDAG::getValidShiftAmountConstant(SDValue V, 2657 const APInt &DemandedElts) const { 2658 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || 2659 V.getOpcode() == ISD::SRA) && 2660 "Unknown shift node"); 2661 unsigned BitWidth = V.getScalarValueSizeInBits(); 2662 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) { 2663 // Shifting more than the bitwidth is not valid. 2664 const APInt &ShAmt = SA->getAPIntValue(); 2665 if (ShAmt.ult(BitWidth)) 2666 return &ShAmt; 2667 } 2668 return nullptr; 2669 } 2670 2671 const APInt *SelectionDAG::getValidMinimumShiftAmountConstant( 2672 SDValue V, const APInt &DemandedElts) const { 2673 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || 2674 V.getOpcode() == ISD::SRA) && 2675 "Unknown shift node"); 2676 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts)) 2677 return ValidAmt; 2678 unsigned BitWidth = V.getScalarValueSizeInBits(); 2679 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1)); 2680 if (!BV) 2681 return nullptr; 2682 const APInt *MinShAmt = nullptr; 2683 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { 2684 if (!DemandedElts[i]) 2685 continue; 2686 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i)); 2687 if (!SA) 2688 return nullptr; 2689 // Shifting more than the bitwidth is not valid. 2690 const APInt &ShAmt = SA->getAPIntValue(); 2691 if (ShAmt.uge(BitWidth)) 2692 return nullptr; 2693 if (MinShAmt && MinShAmt->ule(ShAmt)) 2694 continue; 2695 MinShAmt = &ShAmt; 2696 } 2697 return MinShAmt; 2698 } 2699 2700 const APInt *SelectionDAG::getValidMaximumShiftAmountConstant( 2701 SDValue V, const APInt &DemandedElts) const { 2702 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || 2703 V.getOpcode() == ISD::SRA) && 2704 "Unknown shift node"); 2705 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts)) 2706 return ValidAmt; 2707 unsigned BitWidth = V.getScalarValueSizeInBits(); 2708 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1)); 2709 if (!BV) 2710 return nullptr; 2711 const APInt *MaxShAmt = nullptr; 2712 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { 2713 if (!DemandedElts[i]) 2714 continue; 2715 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i)); 2716 if (!SA) 2717 return nullptr; 2718 // Shifting more than the bitwidth is not valid. 2719 const APInt &ShAmt = SA->getAPIntValue(); 2720 if (ShAmt.uge(BitWidth)) 2721 return nullptr; 2722 if (MaxShAmt && MaxShAmt->uge(ShAmt)) 2723 continue; 2724 MaxShAmt = &ShAmt; 2725 } 2726 return MaxShAmt; 2727 } 2728 2729 /// Determine which bits of Op are known to be either zero or one and return 2730 /// them in Known. For vectors, the known bits are those that are shared by 2731 /// every vector element. 2732 KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const { 2733 EVT VT = Op.getValueType(); 2734 2735 // TOOD: Until we have a plan for how to represent demanded elements for 2736 // scalable vectors, we can just bail out for now. 2737 if (Op.getValueType().isScalableVector()) { 2738 unsigned BitWidth = Op.getScalarValueSizeInBits(); 2739 return KnownBits(BitWidth); 2740 } 2741 2742 APInt DemandedElts = VT.isVector() 2743 ? APInt::getAllOnesValue(VT.getVectorNumElements()) 2744 : APInt(1, 1); 2745 return computeKnownBits(Op, DemandedElts, Depth); 2746 } 2747 2748 /// Determine which bits of Op are known to be either zero or one and return 2749 /// them in Known. The DemandedElts argument allows us to only collect the known 2750 /// bits that are shared by the requested vector elements. 2751 KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, 2752 unsigned Depth) const { 2753 unsigned BitWidth = Op.getScalarValueSizeInBits(); 2754 2755 KnownBits Known(BitWidth); // Don't know anything. 2756 2757 // TOOD: Until we have a plan for how to represent demanded elements for 2758 // scalable vectors, we can just bail out for now. 2759 if (Op.getValueType().isScalableVector()) 2760 return Known; 2761 2762 if (auto *C = dyn_cast<ConstantSDNode>(Op)) { 2763 // We know all of the bits for a constant! 2764 return KnownBits::makeConstant(C->getAPIntValue()); 2765 } 2766 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) { 2767 // We know all of the bits for a constant fp! 2768 return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt()); 2769 } 2770 2771 if (Depth >= MaxRecursionDepth) 2772 return Known; // Limit search depth. 2773 2774 KnownBits Known2; 2775 unsigned NumElts = DemandedElts.getBitWidth(); 2776 assert((!Op.getValueType().isVector() || 2777 NumElts == Op.getValueType().getVectorNumElements()) && 2778 "Unexpected vector size"); 2779 2780 if (!DemandedElts) 2781 return Known; // No demanded elts, better to assume we don't know anything. 2782 2783 unsigned Opcode = Op.getOpcode(); 2784 switch (Opcode) { 2785 case ISD::BUILD_VECTOR: 2786 // Collect the known bits that are shared by every demanded vector element. 2787 Known.Zero.setAllBits(); Known.One.setAllBits(); 2788 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { 2789 if (!DemandedElts[i]) 2790 continue; 2791 2792 SDValue SrcOp = Op.getOperand(i); 2793 Known2 = computeKnownBits(SrcOp, Depth + 1); 2794 2795 // BUILD_VECTOR can implicitly truncate sources, we must handle this. 2796 if (SrcOp.getValueSizeInBits() != BitWidth) { 2797 assert(SrcOp.getValueSizeInBits() > BitWidth && 2798 "Expected BUILD_VECTOR implicit truncation"); 2799 Known2 = Known2.trunc(BitWidth); 2800 } 2801 2802 // Known bits are the values that are shared by every demanded element. 2803 Known = KnownBits::commonBits(Known, Known2); 2804 2805 // If we don't know any bits, early out. 2806 if (Known.isUnknown()) 2807 break; 2808 } 2809 break; 2810 case ISD::VECTOR_SHUFFLE: { 2811 // Collect the known bits that are shared by every vector element referenced 2812 // by the shuffle. 2813 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0); 2814 Known.Zero.setAllBits(); Known.One.setAllBits(); 2815 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op); 2816 assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); 2817 for (unsigned i = 0; i != NumElts; ++i) { 2818 if (!DemandedElts[i]) 2819 continue; 2820 2821 int M = SVN->getMaskElt(i); 2822 if (M < 0) { 2823 // For UNDEF elements, we don't know anything about the common state of 2824 // the shuffle result. 2825 Known.resetAll(); 2826 DemandedLHS.clearAllBits(); 2827 DemandedRHS.clearAllBits(); 2828 break; 2829 } 2830 2831 if ((unsigned)M < NumElts) 2832 DemandedLHS.setBit((unsigned)M % NumElts); 2833 else 2834 DemandedRHS.setBit((unsigned)M % NumElts); 2835 } 2836 // Known bits are the values that are shared by every demanded element. 2837 if (!!DemandedLHS) { 2838 SDValue LHS = Op.getOperand(0); 2839 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1); 2840 Known = KnownBits::commonBits(Known, Known2); 2841 } 2842 // If we don't know any bits, early out. 2843 if (Known.isUnknown()) 2844 break; 2845 if (!!DemandedRHS) { 2846 SDValue RHS = Op.getOperand(1); 2847 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1); 2848 Known = KnownBits::commonBits(Known, Known2); 2849 } 2850 break; 2851 } 2852 case ISD::CONCAT_VECTORS: { 2853 // Split DemandedElts and test each of the demanded subvectors. 2854 Known.Zero.setAllBits(); Known.One.setAllBits(); 2855 EVT SubVectorVT = Op.getOperand(0).getValueType(); 2856 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); 2857 unsigned NumSubVectors = Op.getNumOperands(); 2858 for (unsigned i = 0; i != NumSubVectors; ++i) { 2859 APInt DemandedSub = 2860 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts); 2861 if (!!DemandedSub) { 2862 SDValue Sub = Op.getOperand(i); 2863 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1); 2864 Known = KnownBits::commonBits(Known, Known2); 2865 } 2866 // If we don't know any bits, early out. 2867 if (Known.isUnknown()) 2868 break; 2869 } 2870 break; 2871 } 2872 case ISD::INSERT_SUBVECTOR: { 2873 // Demand any elements from the subvector and the remainder from the src its 2874 // inserted into. 2875 SDValue Src = Op.getOperand(0); 2876 SDValue Sub = Op.getOperand(1); 2877 uint64_t Idx = Op.getConstantOperandVal(2); 2878 unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); 2879 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); 2880 APInt DemandedSrcElts = DemandedElts; 2881 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx); 2882 2883 Known.One.setAllBits(); 2884 Known.Zero.setAllBits(); 2885 if (!!DemandedSubElts) { 2886 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1); 2887 if (Known.isUnknown()) 2888 break; // early-out. 2889 } 2890 if (!!DemandedSrcElts) { 2891 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1); 2892 Known = KnownBits::commonBits(Known, Known2); 2893 } 2894 break; 2895 } 2896 case ISD::EXTRACT_SUBVECTOR: { 2897 // Offset the demanded elts by the subvector index. 2898 SDValue Src = Op.getOperand(0); 2899 // Bail until we can represent demanded elements for scalable vectors. 2900 if (Src.getValueType().isScalableVector()) 2901 break; 2902 uint64_t Idx = Op.getConstantOperandVal(1); 2903 unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); 2904 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx); 2905 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1); 2906 break; 2907 } 2908 case ISD::SCALAR_TO_VECTOR: { 2909 // We know about scalar_to_vector as much as we know about it source, 2910 // which becomes the first element of otherwise unknown vector. 2911 if (DemandedElts != 1) 2912 break; 2913 2914 SDValue N0 = Op.getOperand(0); 2915 Known = computeKnownBits(N0, Depth + 1); 2916 if (N0.getValueSizeInBits() != BitWidth) 2917 Known = Known.trunc(BitWidth); 2918 2919 break; 2920 } 2921 case ISD::BITCAST: { 2922 SDValue N0 = Op.getOperand(0); 2923 EVT SubVT = N0.getValueType(); 2924 unsigned SubBitWidth = SubVT.getScalarSizeInBits(); 2925 2926 // Ignore bitcasts from unsupported types. 2927 if (!(SubVT.isInteger() || SubVT.isFloatingPoint())) 2928 break; 2929 2930 // Fast handling of 'identity' bitcasts. 2931 if (BitWidth == SubBitWidth) { 2932 Known = computeKnownBits(N0, DemandedElts, Depth + 1); 2933 break; 2934 } 2935 2936 bool IsLE = getDataLayout().isLittleEndian(); 2937 2938 // Bitcast 'small element' vector to 'large element' scalar/vector. 2939 if ((BitWidth % SubBitWidth) == 0) { 2940 assert(N0.getValueType().isVector() && "Expected bitcast from vector"); 2941 2942 // Collect known bits for the (larger) output by collecting the known 2943 // bits from each set of sub elements and shift these into place. 2944 // We need to separately call computeKnownBits for each set of 2945 // sub elements as the knownbits for each is likely to be different. 2946 unsigned SubScale = BitWidth / SubBitWidth; 2947 APInt SubDemandedElts(NumElts * SubScale, 0); 2948 for (unsigned i = 0; i != NumElts; ++i) 2949 if (DemandedElts[i]) 2950 SubDemandedElts.setBit(i * SubScale); 2951 2952 for (unsigned i = 0; i != SubScale; ++i) { 2953 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i), 2954 Depth + 1); 2955 unsigned Shifts = IsLE ? i : SubScale - 1 - i; 2956 Known.insertBits(Known2, SubBitWidth * Shifts); 2957 } 2958 } 2959 2960 // Bitcast 'large element' scalar/vector to 'small element' vector. 2961 if ((SubBitWidth % BitWidth) == 0) { 2962 assert(Op.getValueType().isVector() && "Expected bitcast to vector"); 2963 2964 // Collect known bits for the (smaller) output by collecting the known 2965 // bits from the overlapping larger input elements and extracting the 2966 // sub sections we actually care about. 2967 unsigned SubScale = SubBitWidth / BitWidth; 2968 APInt SubDemandedElts(NumElts / SubScale, 0); 2969 for (unsigned i = 0; i != NumElts; ++i) 2970 if (DemandedElts[i]) 2971 SubDemandedElts.setBit(i / SubScale); 2972 2973 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1); 2974 2975 Known.Zero.setAllBits(); Known.One.setAllBits(); 2976 for (unsigned i = 0; i != NumElts; ++i) 2977 if (DemandedElts[i]) { 2978 unsigned Shifts = IsLE ? i : NumElts - 1 - i; 2979 unsigned Offset = (Shifts % SubScale) * BitWidth; 2980 Known = KnownBits::commonBits(Known, 2981 Known2.extractBits(BitWidth, Offset)); 2982 // If we don't know any bits, early out. 2983 if (Known.isUnknown()) 2984 break; 2985 } 2986 } 2987 break; 2988 } 2989 case ISD::AND: 2990 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2991 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2992 2993 Known &= Known2; 2994 break; 2995 case ISD::OR: 2996 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 2997 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 2998 2999 Known |= Known2; 3000 break; 3001 case ISD::XOR: 3002 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3003 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3004 3005 Known ^= Known2; 3006 break; 3007 case ISD::MUL: { 3008 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3009 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3010 Known = KnownBits::mul(Known, Known2); 3011 break; 3012 } 3013 case ISD::MULHU: { 3014 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3015 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3016 Known = KnownBits::mulhu(Known, Known2); 3017 break; 3018 } 3019 case ISD::MULHS: { 3020 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3021 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3022 Known = KnownBits::mulhs(Known, Known2); 3023 break; 3024 } 3025 case ISD::UMUL_LOHI: { 3026 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result"); 3027 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3028 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3029 if (Op.getResNo() == 0) 3030 Known = KnownBits::mul(Known, Known2); 3031 else 3032 Known = KnownBits::mulhu(Known, Known2); 3033 break; 3034 } 3035 case ISD::SMUL_LOHI: { 3036 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result"); 3037 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3038 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3039 if (Op.getResNo() == 0) 3040 Known = KnownBits::mul(Known, Known2); 3041 else 3042 Known = KnownBits::mulhs(Known, Known2); 3043 break; 3044 } 3045 case ISD::UDIV: { 3046 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3047 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3048 Known = KnownBits::udiv(Known, Known2); 3049 break; 3050 } 3051 case ISD::SELECT: 3052 case ISD::VSELECT: 3053 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1); 3054 // If we don't know any bits, early out. 3055 if (Known.isUnknown()) 3056 break; 3057 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1); 3058 3059 // Only known if known in both the LHS and RHS. 3060 Known = KnownBits::commonBits(Known, Known2); 3061 break; 3062 case ISD::SELECT_CC: 3063 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1); 3064 // If we don't know any bits, early out. 3065 if (Known.isUnknown()) 3066 break; 3067 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1); 3068 3069 // Only known if known in both the LHS and RHS. 3070 Known = KnownBits::commonBits(Known, Known2); 3071 break; 3072 case ISD::SMULO: 3073 case ISD::UMULO: 3074 if (Op.getResNo() != 1) 3075 break; 3076 // The boolean result conforms to getBooleanContents. 3077 // If we know the result of a setcc has the top bits zero, use this info. 3078 // We know that we have an integer-based boolean since these operations 3079 // are only available for integer. 3080 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) == 3081 TargetLowering::ZeroOrOneBooleanContent && 3082 BitWidth > 1) 3083 Known.Zero.setBitsFrom(1); 3084 break; 3085 case ISD::SETCC: 3086 case ISD::STRICT_FSETCC: 3087 case ISD::STRICT_FSETCCS: { 3088 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; 3089 // If we know the result of a setcc has the top bits zero, use this info. 3090 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) == 3091 TargetLowering::ZeroOrOneBooleanContent && 3092 BitWidth > 1) 3093 Known.Zero.setBitsFrom(1); 3094 break; 3095 } 3096 case ISD::SHL: 3097 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3098 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3099 Known = KnownBits::shl(Known, Known2); 3100 3101 // Minimum shift low bits are known zero. 3102 if (const APInt *ShMinAmt = 3103 getValidMinimumShiftAmountConstant(Op, DemandedElts)) 3104 Known.Zero.setLowBits(ShMinAmt->getZExtValue()); 3105 break; 3106 case ISD::SRL: 3107 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3108 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3109 Known = KnownBits::lshr(Known, Known2); 3110 3111 // Minimum shift high bits are known zero. 3112 if (const APInt *ShMinAmt = 3113 getValidMinimumShiftAmountConstant(Op, DemandedElts)) 3114 Known.Zero.setHighBits(ShMinAmt->getZExtValue()); 3115 break; 3116 case ISD::SRA: 3117 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3118 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3119 Known = KnownBits::ashr(Known, Known2); 3120 // TODO: Add minimum shift high known sign bits. 3121 break; 3122 case ISD::FSHL: 3123 case ISD::FSHR: 3124 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) { 3125 unsigned Amt = C->getAPIntValue().urem(BitWidth); 3126 3127 // For fshl, 0-shift returns the 1st arg. 3128 // For fshr, 0-shift returns the 2nd arg. 3129 if (Amt == 0) { 3130 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1), 3131 DemandedElts, Depth + 1); 3132 break; 3133 } 3134 3135 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) 3136 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) 3137 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3138 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3139 if (Opcode == ISD::FSHL) { 3140 Known.One <<= Amt; 3141 Known.Zero <<= Amt; 3142 Known2.One.lshrInPlace(BitWidth - Amt); 3143 Known2.Zero.lshrInPlace(BitWidth - Amt); 3144 } else { 3145 Known.One <<= BitWidth - Amt; 3146 Known.Zero <<= BitWidth - Amt; 3147 Known2.One.lshrInPlace(Amt); 3148 Known2.Zero.lshrInPlace(Amt); 3149 } 3150 Known.One |= Known2.One; 3151 Known.Zero |= Known2.Zero; 3152 } 3153 break; 3154 case ISD::SIGN_EXTEND_INREG: { 3155 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3156 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 3157 Known = Known.sextInReg(EVT.getScalarSizeInBits()); 3158 break; 3159 } 3160 case ISD::CTTZ: 3161 case ISD::CTTZ_ZERO_UNDEF: { 3162 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3163 // If we have a known 1, its position is our upper bound. 3164 unsigned PossibleTZ = Known2.countMaxTrailingZeros(); 3165 unsigned LowBits = Log2_32(PossibleTZ) + 1; 3166 Known.Zero.setBitsFrom(LowBits); 3167 break; 3168 } 3169 case ISD::CTLZ: 3170 case ISD::CTLZ_ZERO_UNDEF: { 3171 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3172 // If we have a known 1, its position is our upper bound. 3173 unsigned PossibleLZ = Known2.countMaxLeadingZeros(); 3174 unsigned LowBits = Log2_32(PossibleLZ) + 1; 3175 Known.Zero.setBitsFrom(LowBits); 3176 break; 3177 } 3178 case ISD::CTPOP: { 3179 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3180 // If we know some of the bits are zero, they can't be one. 3181 unsigned PossibleOnes = Known2.countMaxPopulation(); 3182 Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1); 3183 break; 3184 } 3185 case ISD::PARITY: { 3186 // Parity returns 0 everywhere but the LSB. 3187 Known.Zero.setBitsFrom(1); 3188 break; 3189 } 3190 case ISD::LOAD: { 3191 LoadSDNode *LD = cast<LoadSDNode>(Op); 3192 const Constant *Cst = TLI->getTargetConstantFromLoad(LD); 3193 if (ISD::isNON_EXTLoad(LD) && Cst) { 3194 // Determine any common known bits from the loaded constant pool value. 3195 Type *CstTy = Cst->getType(); 3196 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits()) { 3197 // If its a vector splat, then we can (quickly) reuse the scalar path. 3198 // NOTE: We assume all elements match and none are UNDEF. 3199 if (CstTy->isVectorTy()) { 3200 if (const Constant *Splat = Cst->getSplatValue()) { 3201 Cst = Splat; 3202 CstTy = Cst->getType(); 3203 } 3204 } 3205 // TODO - do we need to handle different bitwidths? 3206 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) { 3207 // Iterate across all vector elements finding common known bits. 3208 Known.One.setAllBits(); 3209 Known.Zero.setAllBits(); 3210 for (unsigned i = 0; i != NumElts; ++i) { 3211 if (!DemandedElts[i]) 3212 continue; 3213 if (Constant *Elt = Cst->getAggregateElement(i)) { 3214 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) { 3215 const APInt &Value = CInt->getValue(); 3216 Known.One &= Value; 3217 Known.Zero &= ~Value; 3218 continue; 3219 } 3220 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) { 3221 APInt Value = CFP->getValueAPF().bitcastToAPInt(); 3222 Known.One &= Value; 3223 Known.Zero &= ~Value; 3224 continue; 3225 } 3226 } 3227 Known.One.clearAllBits(); 3228 Known.Zero.clearAllBits(); 3229 break; 3230 } 3231 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) { 3232 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) { 3233 Known = KnownBits::makeConstant(CInt->getValue()); 3234 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) { 3235 Known = 3236 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt()); 3237 } 3238 } 3239 } 3240 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { 3241 // If this is a ZEXTLoad and we are looking at the loaded value. 3242 EVT VT = LD->getMemoryVT(); 3243 unsigned MemBits = VT.getScalarSizeInBits(); 3244 Known.Zero.setBitsFrom(MemBits); 3245 } else if (const MDNode *Ranges = LD->getRanges()) { 3246 if (LD->getExtensionType() == ISD::NON_EXTLOAD) 3247 computeKnownBitsFromRangeMetadata(*Ranges, Known); 3248 } 3249 break; 3250 } 3251 case ISD::ZERO_EXTEND_VECTOR_INREG: { 3252 EVT InVT = Op.getOperand(0).getValueType(); 3253 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements()); 3254 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); 3255 Known = Known.zext(BitWidth); 3256 break; 3257 } 3258 case ISD::ZERO_EXTEND: { 3259 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3260 Known = Known.zext(BitWidth); 3261 break; 3262 } 3263 case ISD::SIGN_EXTEND_VECTOR_INREG: { 3264 EVT InVT = Op.getOperand(0).getValueType(); 3265 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements()); 3266 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); 3267 // If the sign bit is known to be zero or one, then sext will extend 3268 // it to the top bits, else it will just zext. 3269 Known = Known.sext(BitWidth); 3270 break; 3271 } 3272 case ISD::SIGN_EXTEND: { 3273 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3274 // If the sign bit is known to be zero or one, then sext will extend 3275 // it to the top bits, else it will just zext. 3276 Known = Known.sext(BitWidth); 3277 break; 3278 } 3279 case ISD::ANY_EXTEND_VECTOR_INREG: { 3280 EVT InVT = Op.getOperand(0).getValueType(); 3281 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements()); 3282 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); 3283 Known = Known.anyext(BitWidth); 3284 break; 3285 } 3286 case ISD::ANY_EXTEND: { 3287 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3288 Known = Known.anyext(BitWidth); 3289 break; 3290 } 3291 case ISD::TRUNCATE: { 3292 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3293 Known = Known.trunc(BitWidth); 3294 break; 3295 } 3296 case ISD::AssertZext: { 3297 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 3298 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 3299 Known = computeKnownBits(Op.getOperand(0), Depth+1); 3300 Known.Zero |= (~InMask); 3301 Known.One &= (~Known.Zero); 3302 break; 3303 } 3304 case ISD::AssertAlign: { 3305 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign()); 3306 assert(LogOfAlign != 0); 3307 // If a node is guaranteed to be aligned, set low zero bits accordingly as 3308 // well as clearing one bits. 3309 Known.Zero.setLowBits(LogOfAlign); 3310 Known.One.clearLowBits(LogOfAlign); 3311 break; 3312 } 3313 case ISD::FGETSIGN: 3314 // All bits are zero except the low bit. 3315 Known.Zero.setBitsFrom(1); 3316 break; 3317 case ISD::USUBO: 3318 case ISD::SSUBO: 3319 if (Op.getResNo() == 1) { 3320 // If we know the result of a setcc has the top bits zero, use this info. 3321 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == 3322 TargetLowering::ZeroOrOneBooleanContent && 3323 BitWidth > 1) 3324 Known.Zero.setBitsFrom(1); 3325 break; 3326 } 3327 LLVM_FALLTHROUGH; 3328 case ISD::SUB: 3329 case ISD::SUBC: { 3330 assert(Op.getResNo() == 0 && 3331 "We only compute knownbits for the difference here."); 3332 3333 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3334 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3335 Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false, 3336 Known, Known2); 3337 break; 3338 } 3339 case ISD::UADDO: 3340 case ISD::SADDO: 3341 case ISD::ADDCARRY: 3342 if (Op.getResNo() == 1) { 3343 // If we know the result of a setcc has the top bits zero, use this info. 3344 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == 3345 TargetLowering::ZeroOrOneBooleanContent && 3346 BitWidth > 1) 3347 Known.Zero.setBitsFrom(1); 3348 break; 3349 } 3350 LLVM_FALLTHROUGH; 3351 case ISD::ADD: 3352 case ISD::ADDC: 3353 case ISD::ADDE: { 3354 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here."); 3355 3356 // With ADDE and ADDCARRY, a carry bit may be added in. 3357 KnownBits Carry(1); 3358 if (Opcode == ISD::ADDE) 3359 // Can't track carry from glue, set carry to unknown. 3360 Carry.resetAll(); 3361 else if (Opcode == ISD::ADDCARRY) 3362 // TODO: Compute known bits for the carry operand. Not sure if it is worth 3363 // the trouble (how often will we find a known carry bit). And I haven't 3364 // tested this very much yet, but something like this might work: 3365 // Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1); 3366 // Carry = Carry.zextOrTrunc(1, false); 3367 Carry.resetAll(); 3368 else 3369 Carry.setAllZero(); 3370 3371 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3372 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3373 Known = KnownBits::computeForAddCarry(Known, Known2, Carry); 3374 break; 3375 } 3376 case ISD::SREM: { 3377 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3378 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3379 Known = KnownBits::srem(Known, Known2); 3380 break; 3381 } 3382 case ISD::UREM: { 3383 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3384 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3385 Known = KnownBits::urem(Known, Known2); 3386 break; 3387 } 3388 case ISD::EXTRACT_ELEMENT: { 3389 Known = computeKnownBits(Op.getOperand(0), Depth+1); 3390 const unsigned Index = Op.getConstantOperandVal(1); 3391 const unsigned EltBitWidth = Op.getValueSizeInBits(); 3392 3393 // Remove low part of known bits mask 3394 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth); 3395 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth); 3396 3397 // Remove high part of known bit mask 3398 Known = Known.trunc(EltBitWidth); 3399 break; 3400 } 3401 case ISD::EXTRACT_VECTOR_ELT: { 3402 SDValue InVec = Op.getOperand(0); 3403 SDValue EltNo = Op.getOperand(1); 3404 EVT VecVT = InVec.getValueType(); 3405 // computeKnownBits not yet implemented for scalable vectors. 3406 if (VecVT.isScalableVector()) 3407 break; 3408 const unsigned EltBitWidth = VecVT.getScalarSizeInBits(); 3409 const unsigned NumSrcElts = VecVT.getVectorNumElements(); 3410 3411 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know 3412 // anything about the extended bits. 3413 if (BitWidth > EltBitWidth) 3414 Known = Known.trunc(EltBitWidth); 3415 3416 // If we know the element index, just demand that vector element, else for 3417 // an unknown element index, ignore DemandedElts and demand them all. 3418 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts); 3419 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo); 3420 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) 3421 DemandedSrcElts = 3422 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue()); 3423 3424 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1); 3425 if (BitWidth > EltBitWidth) 3426 Known = Known.anyext(BitWidth); 3427 break; 3428 } 3429 case ISD::INSERT_VECTOR_ELT: { 3430 // If we know the element index, split the demand between the 3431 // source vector and the inserted element, otherwise assume we need 3432 // the original demanded vector elements and the value. 3433 SDValue InVec = Op.getOperand(0); 3434 SDValue InVal = Op.getOperand(1); 3435 SDValue EltNo = Op.getOperand(2); 3436 bool DemandedVal = true; 3437 APInt DemandedVecElts = DemandedElts; 3438 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo); 3439 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { 3440 unsigned EltIdx = CEltNo->getZExtValue(); 3441 DemandedVal = !!DemandedElts[EltIdx]; 3442 DemandedVecElts.clearBit(EltIdx); 3443 } 3444 Known.One.setAllBits(); 3445 Known.Zero.setAllBits(); 3446 if (DemandedVal) { 3447 Known2 = computeKnownBits(InVal, Depth + 1); 3448 Known = KnownBits::commonBits(Known, Known2.zextOrTrunc(BitWidth)); 3449 } 3450 if (!!DemandedVecElts) { 3451 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1); 3452 Known = KnownBits::commonBits(Known, Known2); 3453 } 3454 break; 3455 } 3456 case ISD::BITREVERSE: { 3457 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3458 Known = Known2.reverseBits(); 3459 break; 3460 } 3461 case ISD::BSWAP: { 3462 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3463 Known = Known2.byteSwap(); 3464 break; 3465 } 3466 case ISD::ABS: { 3467 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3468 Known = Known2.abs(); 3469 break; 3470 } 3471 case ISD::USUBSAT: { 3472 // The result of usubsat will never be larger than the LHS. 3473 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3474 Known.Zero.setHighBits(Known2.countMinLeadingZeros()); 3475 break; 3476 } 3477 case ISD::UMIN: { 3478 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3479 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3480 Known = KnownBits::umin(Known, Known2); 3481 break; 3482 } 3483 case ISD::UMAX: { 3484 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3485 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3486 Known = KnownBits::umax(Known, Known2); 3487 break; 3488 } 3489 case ISD::SMIN: 3490 case ISD::SMAX: { 3491 // If we have a clamp pattern, we know that the number of sign bits will be 3492 // the minimum of the clamp min/max range. 3493 bool IsMax = (Opcode == ISD::SMAX); 3494 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; 3495 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts))) 3496 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) 3497 CstHigh = 3498 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts); 3499 if (CstLow && CstHigh) { 3500 if (!IsMax) 3501 std::swap(CstLow, CstHigh); 3502 3503 const APInt &ValueLow = CstLow->getAPIntValue(); 3504 const APInt &ValueHigh = CstHigh->getAPIntValue(); 3505 if (ValueLow.sle(ValueHigh)) { 3506 unsigned LowSignBits = ValueLow.getNumSignBits(); 3507 unsigned HighSignBits = ValueHigh.getNumSignBits(); 3508 unsigned MinSignBits = std::min(LowSignBits, HighSignBits); 3509 if (ValueLow.isNegative() && ValueHigh.isNegative()) { 3510 Known.One.setHighBits(MinSignBits); 3511 break; 3512 } 3513 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) { 3514 Known.Zero.setHighBits(MinSignBits); 3515 break; 3516 } 3517 } 3518 } 3519 3520 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3521 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3522 if (IsMax) 3523 Known = KnownBits::smax(Known, Known2); 3524 else 3525 Known = KnownBits::smin(Known, Known2); 3526 break; 3527 } 3528 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 3529 if (Op.getResNo() == 1) { 3530 // The boolean result conforms to getBooleanContents. 3531 // If we know the result of a setcc has the top bits zero, use this info. 3532 // We know that we have an integer-based boolean since these operations 3533 // are only available for integer. 3534 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) == 3535 TargetLowering::ZeroOrOneBooleanContent && 3536 BitWidth > 1) 3537 Known.Zero.setBitsFrom(1); 3538 break; 3539 } 3540 LLVM_FALLTHROUGH; 3541 case ISD::ATOMIC_CMP_SWAP: 3542 case ISD::ATOMIC_SWAP: 3543 case ISD::ATOMIC_LOAD_ADD: 3544 case ISD::ATOMIC_LOAD_SUB: 3545 case ISD::ATOMIC_LOAD_AND: 3546 case ISD::ATOMIC_LOAD_CLR: 3547 case ISD::ATOMIC_LOAD_OR: 3548 case ISD::ATOMIC_LOAD_XOR: 3549 case ISD::ATOMIC_LOAD_NAND: 3550 case ISD::ATOMIC_LOAD_MIN: 3551 case ISD::ATOMIC_LOAD_MAX: 3552 case ISD::ATOMIC_LOAD_UMIN: 3553 case ISD::ATOMIC_LOAD_UMAX: 3554 case ISD::ATOMIC_LOAD: { 3555 unsigned MemBits = 3556 cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits(); 3557 // If we are looking at the loaded value. 3558 if (Op.getResNo() == 0) { 3559 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) 3560 Known.Zero.setBitsFrom(MemBits); 3561 } 3562 break; 3563 } 3564 case ISD::FrameIndex: 3565 case ISD::TargetFrameIndex: 3566 TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(), 3567 Known, getMachineFunction()); 3568 break; 3569 3570 default: 3571 if (Opcode < ISD::BUILTIN_OP_END) 3572 break; 3573 LLVM_FALLTHROUGH; 3574 case ISD::INTRINSIC_WO_CHAIN: 3575 case ISD::INTRINSIC_W_CHAIN: 3576 case ISD::INTRINSIC_VOID: 3577 // Allow the target to implement this method for its nodes. 3578 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth); 3579 break; 3580 } 3581 3582 assert(!Known.hasConflict() && "Bits known to be one AND zero?"); 3583 return Known; 3584 } 3585 3586 SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0, 3587 SDValue N1) const { 3588 // X + 0 never overflow 3589 if (isNullConstant(N1)) 3590 return OFK_Never; 3591 3592 KnownBits N1Known = computeKnownBits(N1); 3593 if (N1Known.Zero.getBoolValue()) { 3594 KnownBits N0Known = computeKnownBits(N0); 3595 3596 bool overflow; 3597 (void)N0Known.getMaxValue().uadd_ov(N1Known.getMaxValue(), overflow); 3598 if (!overflow) 3599 return OFK_Never; 3600 } 3601 3602 // mulhi + 1 never overflow 3603 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 && 3604 (N1Known.getMaxValue() & 0x01) == N1Known.getMaxValue()) 3605 return OFK_Never; 3606 3607 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) { 3608 KnownBits N0Known = computeKnownBits(N0); 3609 3610 if ((N0Known.getMaxValue() & 0x01) == N0Known.getMaxValue()) 3611 return OFK_Never; 3612 } 3613 3614 return OFK_Sometime; 3615 } 3616 3617 bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const { 3618 EVT OpVT = Val.getValueType(); 3619 unsigned BitWidth = OpVT.getScalarSizeInBits(); 3620 3621 // Is the constant a known power of 2? 3622 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val)) 3623 return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); 3624 3625 // A left-shift of a constant one will have exactly one bit set because 3626 // shifting the bit off the end is undefined. 3627 if (Val.getOpcode() == ISD::SHL) { 3628 auto *C = isConstOrConstSplat(Val.getOperand(0)); 3629 if (C && C->getAPIntValue() == 1) 3630 return true; 3631 } 3632 3633 // Similarly, a logical right-shift of a constant sign-bit will have exactly 3634 // one bit set. 3635 if (Val.getOpcode() == ISD::SRL) { 3636 auto *C = isConstOrConstSplat(Val.getOperand(0)); 3637 if (C && C->getAPIntValue().isSignMask()) 3638 return true; 3639 } 3640 3641 // Are all operands of a build vector constant powers of two? 3642 if (Val.getOpcode() == ISD::BUILD_VECTOR) 3643 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) { 3644 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E)) 3645 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); 3646 return false; 3647 })) 3648 return true; 3649 3650 // More could be done here, though the above checks are enough 3651 // to handle some common cases. 3652 3653 // Fall back to computeKnownBits to catch other known cases. 3654 KnownBits Known = computeKnownBits(Val); 3655 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1); 3656 } 3657 3658 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const { 3659 EVT VT = Op.getValueType(); 3660 3661 // TODO: Assume we don't know anything for now. 3662 if (VT.isScalableVector()) 3663 return 1; 3664 3665 APInt DemandedElts = VT.isVector() 3666 ? APInt::getAllOnesValue(VT.getVectorNumElements()) 3667 : APInt(1, 1); 3668 return ComputeNumSignBits(Op, DemandedElts, Depth); 3669 } 3670 3671 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, 3672 unsigned Depth) const { 3673 EVT VT = Op.getValueType(); 3674 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!"); 3675 unsigned VTBits = VT.getScalarSizeInBits(); 3676 unsigned NumElts = DemandedElts.getBitWidth(); 3677 unsigned Tmp, Tmp2; 3678 unsigned FirstAnswer = 1; 3679 3680 if (auto *C = dyn_cast<ConstantSDNode>(Op)) { 3681 const APInt &Val = C->getAPIntValue(); 3682 return Val.getNumSignBits(); 3683 } 3684 3685 if (Depth >= MaxRecursionDepth) 3686 return 1; // Limit search depth. 3687 3688 if (!DemandedElts || VT.isScalableVector()) 3689 return 1; // No demanded elts, better to assume we don't know anything. 3690 3691 unsigned Opcode = Op.getOpcode(); 3692 switch (Opcode) { 3693 default: break; 3694 case ISD::AssertSext: 3695 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 3696 return VTBits-Tmp+1; 3697 case ISD::AssertZext: 3698 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 3699 return VTBits-Tmp; 3700 3701 case ISD::BUILD_VECTOR: 3702 Tmp = VTBits; 3703 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) { 3704 if (!DemandedElts[i]) 3705 continue; 3706 3707 SDValue SrcOp = Op.getOperand(i); 3708 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1); 3709 3710 // BUILD_VECTOR can implicitly truncate sources, we must handle this. 3711 if (SrcOp.getValueSizeInBits() != VTBits) { 3712 assert(SrcOp.getValueSizeInBits() > VTBits && 3713 "Expected BUILD_VECTOR implicit truncation"); 3714 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits; 3715 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1); 3716 } 3717 Tmp = std::min(Tmp, Tmp2); 3718 } 3719 return Tmp; 3720 3721 case ISD::VECTOR_SHUFFLE: { 3722 // Collect the minimum number of sign bits that are shared by every vector 3723 // element referenced by the shuffle. 3724 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0); 3725 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op); 3726 assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); 3727 for (unsigned i = 0; i != NumElts; ++i) { 3728 int M = SVN->getMaskElt(i); 3729 if (!DemandedElts[i]) 3730 continue; 3731 // For UNDEF elements, we don't know anything about the common state of 3732 // the shuffle result. 3733 if (M < 0) 3734 return 1; 3735 if ((unsigned)M < NumElts) 3736 DemandedLHS.setBit((unsigned)M % NumElts); 3737 else 3738 DemandedRHS.setBit((unsigned)M % NumElts); 3739 } 3740 Tmp = std::numeric_limits<unsigned>::max(); 3741 if (!!DemandedLHS) 3742 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1); 3743 if (!!DemandedRHS) { 3744 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1); 3745 Tmp = std::min(Tmp, Tmp2); 3746 } 3747 // If we don't know anything, early out and try computeKnownBits fall-back. 3748 if (Tmp == 1) 3749 break; 3750 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 3751 return Tmp; 3752 } 3753 3754 case ISD::BITCAST: { 3755 SDValue N0 = Op.getOperand(0); 3756 EVT SrcVT = N0.getValueType(); 3757 unsigned SrcBits = SrcVT.getScalarSizeInBits(); 3758 3759 // Ignore bitcasts from unsupported types.. 3760 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint())) 3761 break; 3762 3763 // Fast handling of 'identity' bitcasts. 3764 if (VTBits == SrcBits) 3765 return ComputeNumSignBits(N0, DemandedElts, Depth + 1); 3766 3767 bool IsLE = getDataLayout().isLittleEndian(); 3768 3769 // Bitcast 'large element' scalar/vector to 'small element' vector. 3770 if ((SrcBits % VTBits) == 0) { 3771 assert(VT.isVector() && "Expected bitcast to vector"); 3772 3773 unsigned Scale = SrcBits / VTBits; 3774 APInt SrcDemandedElts(NumElts / Scale, 0); 3775 for (unsigned i = 0; i != NumElts; ++i) 3776 if (DemandedElts[i]) 3777 SrcDemandedElts.setBit(i / Scale); 3778 3779 // Fast case - sign splat can be simply split across the small elements. 3780 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1); 3781 if (Tmp == SrcBits) 3782 return VTBits; 3783 3784 // Slow case - determine how far the sign extends into each sub-element. 3785 Tmp2 = VTBits; 3786 for (unsigned i = 0; i != NumElts; ++i) 3787 if (DemandedElts[i]) { 3788 unsigned SubOffset = i % Scale; 3789 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset); 3790 SubOffset = SubOffset * VTBits; 3791 if (Tmp <= SubOffset) 3792 return 1; 3793 Tmp2 = std::min(Tmp2, Tmp - SubOffset); 3794 } 3795 return Tmp2; 3796 } 3797 break; 3798 } 3799 3800 case ISD::SIGN_EXTEND: 3801 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits(); 3802 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp; 3803 case ISD::SIGN_EXTEND_INREG: 3804 // Max of the input and what this extends. 3805 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits(); 3806 Tmp = VTBits-Tmp+1; 3807 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); 3808 return std::max(Tmp, Tmp2); 3809 case ISD::SIGN_EXTEND_VECTOR_INREG: { 3810 SDValue Src = Op.getOperand(0); 3811 EVT SrcVT = Src.getValueType(); 3812 APInt DemandedSrcElts = DemandedElts.zextOrSelf(SrcVT.getVectorNumElements()); 3813 Tmp = VTBits - SrcVT.getScalarSizeInBits(); 3814 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp; 3815 } 3816 case ISD::SRA: 3817 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3818 // SRA X, C -> adds C sign bits. 3819 if (const APInt *ShAmt = 3820 getValidMinimumShiftAmountConstant(Op, DemandedElts)) 3821 Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits); 3822 return Tmp; 3823 case ISD::SHL: 3824 if (const APInt *ShAmt = 3825 getValidMaximumShiftAmountConstant(Op, DemandedElts)) { 3826 // shl destroys sign bits, ensure it doesn't shift out all sign bits. 3827 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3828 if (ShAmt->ult(Tmp)) 3829 return Tmp - ShAmt->getZExtValue(); 3830 } 3831 break; 3832 case ISD::AND: 3833 case ISD::OR: 3834 case ISD::XOR: // NOT is handled here. 3835 // Logical binary ops preserve the number of sign bits at the worst. 3836 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); 3837 if (Tmp != 1) { 3838 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1); 3839 FirstAnswer = std::min(Tmp, Tmp2); 3840 // We computed what we know about the sign bits as our first 3841 // answer. Now proceed to the generic code that uses 3842 // computeKnownBits, and pick whichever answer is better. 3843 } 3844 break; 3845 3846 case ISD::SELECT: 3847 case ISD::VSELECT: 3848 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1); 3849 if (Tmp == 1) return 1; // Early out. 3850 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1); 3851 return std::min(Tmp, Tmp2); 3852 case ISD::SELECT_CC: 3853 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1); 3854 if (Tmp == 1) return 1; // Early out. 3855 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1); 3856 return std::min(Tmp, Tmp2); 3857 3858 case ISD::SMIN: 3859 case ISD::SMAX: { 3860 // If we have a clamp pattern, we know that the number of sign bits will be 3861 // the minimum of the clamp min/max range. 3862 bool IsMax = (Opcode == ISD::SMAX); 3863 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; 3864 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts))) 3865 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) 3866 CstHigh = 3867 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts); 3868 if (CstLow && CstHigh) { 3869 if (!IsMax) 3870 std::swap(CstLow, CstHigh); 3871 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) { 3872 Tmp = CstLow->getAPIntValue().getNumSignBits(); 3873 Tmp2 = CstHigh->getAPIntValue().getNumSignBits(); 3874 return std::min(Tmp, Tmp2); 3875 } 3876 } 3877 3878 // Fallback - just get the minimum number of sign bits of the operands. 3879 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3880 if (Tmp == 1) 3881 return 1; // Early out. 3882 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); 3883 return std::min(Tmp, Tmp2); 3884 } 3885 case ISD::UMIN: 3886 case ISD::UMAX: 3887 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3888 if (Tmp == 1) 3889 return 1; // Early out. 3890 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); 3891 return std::min(Tmp, Tmp2); 3892 case ISD::SADDO: 3893 case ISD::UADDO: 3894 case ISD::SSUBO: 3895 case ISD::USUBO: 3896 case ISD::SMULO: 3897 case ISD::UMULO: 3898 if (Op.getResNo() != 1) 3899 break; 3900 // The boolean result conforms to getBooleanContents. Fall through. 3901 // If setcc returns 0/-1, all bits are sign bits. 3902 // We know that we have an integer-based boolean since these operations 3903 // are only available for integer. 3904 if (TLI->getBooleanContents(VT.isVector(), false) == 3905 TargetLowering::ZeroOrNegativeOneBooleanContent) 3906 return VTBits; 3907 break; 3908 case ISD::SETCC: 3909 case ISD::STRICT_FSETCC: 3910 case ISD::STRICT_FSETCCS: { 3911 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; 3912 // If setcc returns 0/-1, all bits are sign bits. 3913 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) == 3914 TargetLowering::ZeroOrNegativeOneBooleanContent) 3915 return VTBits; 3916 break; 3917 } 3918 case ISD::ROTL: 3919 case ISD::ROTR: 3920 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3921 3922 // If we're rotating an 0/-1 value, then it stays an 0/-1 value. 3923 if (Tmp == VTBits) 3924 return VTBits; 3925 3926 if (ConstantSDNode *C = 3927 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) { 3928 unsigned RotAmt = C->getAPIntValue().urem(VTBits); 3929 3930 // Handle rotate right by N like a rotate left by 32-N. 3931 if (Opcode == ISD::ROTR) 3932 RotAmt = (VTBits - RotAmt) % VTBits; 3933 3934 // If we aren't rotating out all of the known-in sign bits, return the 3935 // number that are left. This handles rotl(sext(x), 1) for example. 3936 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt); 3937 } 3938 break; 3939 case ISD::ADD: 3940 case ISD::ADDC: 3941 // Add can have at most one carry bit. Thus we know that the output 3942 // is, at worst, one more bit than the inputs. 3943 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3944 if (Tmp == 1) return 1; // Early out. 3945 3946 // Special case decrementing a value (ADD X, -1): 3947 if (ConstantSDNode *CRHS = 3948 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) 3949 if (CRHS->isAllOnesValue()) { 3950 KnownBits Known = 3951 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); 3952 3953 // If the input is known to be 0 or 1, the output is 0/-1, which is all 3954 // sign bits set. 3955 if ((Known.Zero | 1).isAllOnesValue()) 3956 return VTBits; 3957 3958 // If we are subtracting one from a positive number, there is no carry 3959 // out of the result. 3960 if (Known.isNonNegative()) 3961 return Tmp; 3962 } 3963 3964 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); 3965 if (Tmp2 == 1) return 1; // Early out. 3966 return std::min(Tmp, Tmp2) - 1; 3967 case ISD::SUB: 3968 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); 3969 if (Tmp2 == 1) return 1; // Early out. 3970 3971 // Handle NEG. 3972 if (ConstantSDNode *CLHS = 3973 isConstOrConstSplat(Op.getOperand(0), DemandedElts)) 3974 if (CLHS->isNullValue()) { 3975 KnownBits Known = 3976 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); 3977 // If the input is known to be 0 or 1, the output is 0/-1, which is all 3978 // sign bits set. 3979 if ((Known.Zero | 1).isAllOnesValue()) 3980 return VTBits; 3981 3982 // If the input is known to be positive (the sign bit is known clear), 3983 // the output of the NEG has the same number of sign bits as the input. 3984 if (Known.isNonNegative()) 3985 return Tmp2; 3986 3987 // Otherwise, we treat this like a SUB. 3988 } 3989 3990 // Sub can have at most one carry bit. Thus we know that the output 3991 // is, at worst, one more bit than the inputs. 3992 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 3993 if (Tmp == 1) return 1; // Early out. 3994 return std::min(Tmp, Tmp2) - 1; 3995 case ISD::MUL: { 3996 // The output of the Mul can be at most twice the valid bits in the inputs. 3997 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1); 3998 if (SignBitsOp0 == 1) 3999 break; 4000 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1); 4001 if (SignBitsOp1 == 1) 4002 break; 4003 unsigned OutValidBits = 4004 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1); 4005 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1; 4006 } 4007 case ISD::SREM: 4008 // The sign bit is the LHS's sign bit, except when the result of the 4009 // remainder is zero. The magnitude of the result should be less than or 4010 // equal to the magnitude of the LHS. Therefore, the result should have 4011 // at least as many sign bits as the left hand side. 4012 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); 4013 case ISD::TRUNCATE: { 4014 // Check if the sign bits of source go down as far as the truncated value. 4015 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits(); 4016 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1); 4017 if (NumSrcSignBits > (NumSrcBits - VTBits)) 4018 return NumSrcSignBits - (NumSrcBits - VTBits); 4019 break; 4020 } 4021 case ISD::EXTRACT_ELEMENT: { 4022 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1); 4023 const int BitWidth = Op.getValueSizeInBits(); 4024 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth; 4025 4026 // Get reverse index (starting from 1), Op1 value indexes elements from 4027 // little end. Sign starts at big end. 4028 const int rIndex = Items - 1 - Op.getConstantOperandVal(1); 4029 4030 // If the sign portion ends in our element the subtraction gives correct 4031 // result. Otherwise it gives either negative or > bitwidth result 4032 return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0); 4033 } 4034 case ISD::INSERT_VECTOR_ELT: { 4035 // If we know the element index, split the demand between the 4036 // source vector and the inserted element, otherwise assume we need 4037 // the original demanded vector elements and the value. 4038 SDValue InVec = Op.getOperand(0); 4039 SDValue InVal = Op.getOperand(1); 4040 SDValue EltNo = Op.getOperand(2); 4041 bool DemandedVal = true; 4042 APInt DemandedVecElts = DemandedElts; 4043 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo); 4044 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { 4045 unsigned EltIdx = CEltNo->getZExtValue(); 4046 DemandedVal = !!DemandedElts[EltIdx]; 4047 DemandedVecElts.clearBit(EltIdx); 4048 } 4049 Tmp = std::numeric_limits<unsigned>::max(); 4050 if (DemandedVal) { 4051 // TODO - handle implicit truncation of inserted elements. 4052 if (InVal.getScalarValueSizeInBits() != VTBits) 4053 break; 4054 Tmp2 = ComputeNumSignBits(InVal, Depth + 1); 4055 Tmp = std::min(Tmp, Tmp2); 4056 } 4057 if (!!DemandedVecElts) { 4058 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1); 4059 Tmp = std::min(Tmp, Tmp2); 4060 } 4061 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 4062 return Tmp; 4063 } 4064 case ISD::EXTRACT_VECTOR_ELT: { 4065 SDValue InVec = Op.getOperand(0); 4066 SDValue EltNo = Op.getOperand(1); 4067 EVT VecVT = InVec.getValueType(); 4068 // ComputeNumSignBits not yet implemented for scalable vectors. 4069 if (VecVT.isScalableVector()) 4070 break; 4071 const unsigned BitWidth = Op.getValueSizeInBits(); 4072 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits(); 4073 const unsigned NumSrcElts = VecVT.getVectorNumElements(); 4074 4075 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know 4076 // anything about sign bits. But if the sizes match we can derive knowledge 4077 // about sign bits from the vector operand. 4078 if (BitWidth != EltBitWidth) 4079 break; 4080 4081 // If we know the element index, just demand that vector element, else for 4082 // an unknown element index, ignore DemandedElts and demand them all. 4083 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts); 4084 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo); 4085 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) 4086 DemandedSrcElts = 4087 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue()); 4088 4089 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1); 4090 } 4091 case ISD::EXTRACT_SUBVECTOR: { 4092 // Offset the demanded elts by the subvector index. 4093 SDValue Src = Op.getOperand(0); 4094 // Bail until we can represent demanded elements for scalable vectors. 4095 if (Src.getValueType().isScalableVector()) 4096 break; 4097 uint64_t Idx = Op.getConstantOperandVal(1); 4098 unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); 4099 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx); 4100 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1); 4101 } 4102 case ISD::CONCAT_VECTORS: { 4103 // Determine the minimum number of sign bits across all demanded 4104 // elts of the input vectors. Early out if the result is already 1. 4105 Tmp = std::numeric_limits<unsigned>::max(); 4106 EVT SubVectorVT = Op.getOperand(0).getValueType(); 4107 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); 4108 unsigned NumSubVectors = Op.getNumOperands(); 4109 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) { 4110 APInt DemandedSub = 4111 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts); 4112 if (!DemandedSub) 4113 continue; 4114 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1); 4115 Tmp = std::min(Tmp, Tmp2); 4116 } 4117 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 4118 return Tmp; 4119 } 4120 case ISD::INSERT_SUBVECTOR: { 4121 // Demand any elements from the subvector and the remainder from the src its 4122 // inserted into. 4123 SDValue Src = Op.getOperand(0); 4124 SDValue Sub = Op.getOperand(1); 4125 uint64_t Idx = Op.getConstantOperandVal(2); 4126 unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); 4127 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); 4128 APInt DemandedSrcElts = DemandedElts; 4129 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx); 4130 4131 Tmp = std::numeric_limits<unsigned>::max(); 4132 if (!!DemandedSubElts) { 4133 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1); 4134 if (Tmp == 1) 4135 return 1; // early-out 4136 } 4137 if (!!DemandedSrcElts) { 4138 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1); 4139 Tmp = std::min(Tmp, Tmp2); 4140 } 4141 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 4142 return Tmp; 4143 } 4144 case ISD::ATOMIC_CMP_SWAP: 4145 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 4146 case ISD::ATOMIC_SWAP: 4147 case ISD::ATOMIC_LOAD_ADD: 4148 case ISD::ATOMIC_LOAD_SUB: 4149 case ISD::ATOMIC_LOAD_AND: 4150 case ISD::ATOMIC_LOAD_CLR: 4151 case ISD::ATOMIC_LOAD_OR: 4152 case ISD::ATOMIC_LOAD_XOR: 4153 case ISD::ATOMIC_LOAD_NAND: 4154 case ISD::ATOMIC_LOAD_MIN: 4155 case ISD::ATOMIC_LOAD_MAX: 4156 case ISD::ATOMIC_LOAD_UMIN: 4157 case ISD::ATOMIC_LOAD_UMAX: 4158 case ISD::ATOMIC_LOAD: { 4159 Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits(); 4160 // If we are looking at the loaded value. 4161 if (Op.getResNo() == 0) { 4162 if (Tmp == VTBits) 4163 return 1; // early-out 4164 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND) 4165 return VTBits - Tmp + 1; 4166 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) 4167 return VTBits - Tmp; 4168 } 4169 break; 4170 } 4171 } 4172 4173 // If we are looking at the loaded value of the SDNode. 4174 if (Op.getResNo() == 0) { 4175 // Handle LOADX separately here. EXTLOAD case will fallthrough. 4176 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 4177 unsigned ExtType = LD->getExtensionType(); 4178 switch (ExtType) { 4179 default: break; 4180 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known. 4181 Tmp = LD->getMemoryVT().getScalarSizeInBits(); 4182 return VTBits - Tmp + 1; 4183 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known. 4184 Tmp = LD->getMemoryVT().getScalarSizeInBits(); 4185 return VTBits - Tmp; 4186 case ISD::NON_EXTLOAD: 4187 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) { 4188 // We only need to handle vectors - computeKnownBits should handle 4189 // scalar cases. 4190 Type *CstTy = Cst->getType(); 4191 if (CstTy->isVectorTy() && 4192 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits()) { 4193 Tmp = VTBits; 4194 for (unsigned i = 0; i != NumElts; ++i) { 4195 if (!DemandedElts[i]) 4196 continue; 4197 if (Constant *Elt = Cst->getAggregateElement(i)) { 4198 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) { 4199 const APInt &Value = CInt->getValue(); 4200 Tmp = std::min(Tmp, Value.getNumSignBits()); 4201 continue; 4202 } 4203 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) { 4204 APInt Value = CFP->getValueAPF().bitcastToAPInt(); 4205 Tmp = std::min(Tmp, Value.getNumSignBits()); 4206 continue; 4207 } 4208 } 4209 // Unknown type. Conservatively assume no bits match sign bit. 4210 return 1; 4211 } 4212 return Tmp; 4213 } 4214 } 4215 break; 4216 } 4217 } 4218 } 4219 4220 // Allow the target to implement this method for its nodes. 4221 if (Opcode >= ISD::BUILTIN_OP_END || 4222 Opcode == ISD::INTRINSIC_WO_CHAIN || 4223 Opcode == ISD::INTRINSIC_W_CHAIN || 4224 Opcode == ISD::INTRINSIC_VOID) { 4225 unsigned NumBits = 4226 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth); 4227 if (NumBits > 1) 4228 FirstAnswer = std::max(FirstAnswer, NumBits); 4229 } 4230 4231 // Finally, if we can prove that the top bits of the result are 0's or 1's, 4232 // use this information. 4233 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth); 4234 4235 APInt Mask; 4236 if (Known.isNonNegative()) { // sign bit is 0 4237 Mask = Known.Zero; 4238 } else if (Known.isNegative()) { // sign bit is 1; 4239 Mask = Known.One; 4240 } else { 4241 // Nothing known. 4242 return FirstAnswer; 4243 } 4244 4245 // Okay, we know that the sign bit in Mask is set. Use CLO to determine 4246 // the number of identical bits in the top of the input value. 4247 Mask <<= Mask.getBitWidth()-VTBits; 4248 return std::max(FirstAnswer, Mask.countLeadingOnes()); 4249 } 4250 4251 bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly, 4252 unsigned Depth) const { 4253 // Early out for FREEZE. 4254 if (Op.getOpcode() == ISD::FREEZE) 4255 return true; 4256 4257 // TODO: Assume we don't know anything for now. 4258 EVT VT = Op.getValueType(); 4259 if (VT.isScalableVector()) 4260 return false; 4261 4262 APInt DemandedElts = VT.isVector() 4263 ? APInt::getAllOnesValue(VT.getVectorNumElements()) 4264 : APInt(1, 1); 4265 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth); 4266 } 4267 4268 bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, 4269 const APInt &DemandedElts, 4270 bool PoisonOnly, 4271 unsigned Depth) const { 4272 unsigned Opcode = Op.getOpcode(); 4273 4274 // Early out for FREEZE. 4275 if (Opcode == ISD::FREEZE) 4276 return true; 4277 4278 if (Depth >= MaxRecursionDepth) 4279 return false; // Limit search depth. 4280 4281 if (isIntOrFPConstant(Op)) 4282 return true; 4283 4284 switch (Opcode) { 4285 case ISD::UNDEF: 4286 return PoisonOnly; 4287 4288 // TODO: ISD::BUILD_VECTOR handling 4289 4290 // TODO: Search for noundef attributes from library functions. 4291 4292 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef. 4293 4294 default: 4295 // Allow the target to implement this method for its nodes. 4296 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || 4297 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) 4298 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode( 4299 Op, DemandedElts, *this, PoisonOnly, Depth); 4300 break; 4301 } 4302 4303 return false; 4304 } 4305 4306 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 4307 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 4308 !isa<ConstantSDNode>(Op.getOperand(1))) 4309 return false; 4310 4311 if (Op.getOpcode() == ISD::OR && 4312 !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1))) 4313 return false; 4314 4315 return true; 4316 } 4317 4318 bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const { 4319 // If we're told that NaNs won't happen, assume they won't. 4320 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs()) 4321 return true; 4322 4323 if (Depth >= MaxRecursionDepth) 4324 return false; // Limit search depth. 4325 4326 // TODO: Handle vectors. 4327 // If the value is a constant, we can obviously see if it is a NaN or not. 4328 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) { 4329 return !C->getValueAPF().isNaN() || 4330 (SNaN && !C->getValueAPF().isSignaling()); 4331 } 4332 4333 unsigned Opcode = Op.getOpcode(); 4334 switch (Opcode) { 4335 case ISD::FADD: 4336 case ISD::FSUB: 4337 case ISD::FMUL: 4338 case ISD::FDIV: 4339 case ISD::FREM: 4340 case ISD::FSIN: 4341 case ISD::FCOS: { 4342 if (SNaN) 4343 return true; 4344 // TODO: Need isKnownNeverInfinity 4345 return false; 4346 } 4347 case ISD::FCANONICALIZE: 4348 case ISD::FEXP: 4349 case ISD::FEXP2: 4350 case ISD::FTRUNC: 4351 case ISD::FFLOOR: 4352 case ISD::FCEIL: 4353 case ISD::FROUND: 4354 case ISD::FROUNDEVEN: 4355 case ISD::FRINT: 4356 case ISD::FNEARBYINT: { 4357 if (SNaN) 4358 return true; 4359 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); 4360 } 4361 case ISD::FABS: 4362 case ISD::FNEG: 4363 case ISD::FCOPYSIGN: { 4364 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); 4365 } 4366 case ISD::SELECT: 4367 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && 4368 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1); 4369 case ISD::FP_EXTEND: 4370 case ISD::FP_ROUND: { 4371 if (SNaN) 4372 return true; 4373 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); 4374 } 4375 case ISD::SINT_TO_FP: 4376 case ISD::UINT_TO_FP: 4377 return true; 4378 case ISD::FMA: 4379 case ISD::FMAD: { 4380 if (SNaN) 4381 return true; 4382 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) && 4383 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && 4384 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1); 4385 } 4386 case ISD::FSQRT: // Need is known positive 4387 case ISD::FLOG: 4388 case ISD::FLOG2: 4389 case ISD::FLOG10: 4390 case ISD::FPOWI: 4391 case ISD::FPOW: { 4392 if (SNaN) 4393 return true; 4394 // TODO: Refine on operand 4395 return false; 4396 } 4397 case ISD::FMINNUM: 4398 case ISD::FMAXNUM: { 4399 // Only one needs to be known not-nan, since it will be returned if the 4400 // other ends up being one. 4401 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) || 4402 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); 4403 } 4404 case ISD::FMINNUM_IEEE: 4405 case ISD::FMAXNUM_IEEE: { 4406 if (SNaN) 4407 return true; 4408 // This can return a NaN if either operand is an sNaN, or if both operands 4409 // are NaN. 4410 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) && 4411 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) || 4412 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) && 4413 isKnownNeverSNaN(Op.getOperand(0), Depth + 1)); 4414 } 4415 case ISD::FMINIMUM: 4416 case ISD::FMAXIMUM: { 4417 // TODO: Does this quiet or return the origina NaN as-is? 4418 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) && 4419 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); 4420 } 4421 case ISD::EXTRACT_VECTOR_ELT: { 4422 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); 4423 } 4424 default: 4425 if (Opcode >= ISD::BUILTIN_OP_END || 4426 Opcode == ISD::INTRINSIC_WO_CHAIN || 4427 Opcode == ISD::INTRINSIC_W_CHAIN || 4428 Opcode == ISD::INTRINSIC_VOID) { 4429 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth); 4430 } 4431 4432 return false; 4433 } 4434 } 4435 4436 bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const { 4437 assert(Op.getValueType().isFloatingPoint() && 4438 "Floating point type expected"); 4439 4440 // If the value is a constant, we can obviously see if it is a zero or not. 4441 // TODO: Add BuildVector support. 4442 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 4443 return !C->isZero(); 4444 return false; 4445 } 4446 4447 bool SelectionDAG::isKnownNeverZero(SDValue Op) const { 4448 assert(!Op.getValueType().isFloatingPoint() && 4449 "Floating point types unsupported - use isKnownNeverZeroFloat"); 4450 4451 // If the value is a constant, we can obviously see if it is a zero or not. 4452 if (ISD::matchUnaryPredicate( 4453 Op, [](ConstantSDNode *C) { return !C->isNullValue(); })) 4454 return true; 4455 4456 // TODO: Recognize more cases here. 4457 switch (Op.getOpcode()) { 4458 default: break; 4459 case ISD::OR: 4460 if (isKnownNeverZero(Op.getOperand(1)) || 4461 isKnownNeverZero(Op.getOperand(0))) 4462 return true; 4463 break; 4464 } 4465 4466 return false; 4467 } 4468 4469 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 4470 // Check the obvious case. 4471 if (A == B) return true; 4472 4473 // For for negative and positive zero. 4474 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 4475 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 4476 if (CA->isZero() && CB->isZero()) return true; 4477 4478 // Otherwise they may not be equal. 4479 return false; 4480 } 4481 4482 // FIXME: unify with llvm::haveNoCommonBitsSet. 4483 // FIXME: could also handle masked merge pattern (X & ~M) op (Y & M) 4484 bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const { 4485 assert(A.getValueType() == B.getValueType() && 4486 "Values must have the same type"); 4487 return KnownBits::haveNoCommonBitsSet(computeKnownBits(A), 4488 computeKnownBits(B)); 4489 } 4490 4491 static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, 4492 SelectionDAG &DAG) { 4493 if (cast<ConstantSDNode>(Step)->isNullValue()) 4494 return DAG.getConstant(0, DL, VT); 4495 4496 return SDValue(); 4497 } 4498 4499 static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, 4500 ArrayRef<SDValue> Ops, 4501 SelectionDAG &DAG) { 4502 int NumOps = Ops.size(); 4503 assert(NumOps != 0 && "Can't build an empty vector!"); 4504 assert(!VT.isScalableVector() && 4505 "BUILD_VECTOR cannot be used with scalable types"); 4506 assert(VT.getVectorNumElements() == (unsigned)NumOps && 4507 "Incorrect element count in BUILD_VECTOR!"); 4508 4509 // BUILD_VECTOR of UNDEFs is UNDEF. 4510 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); })) 4511 return DAG.getUNDEF(VT); 4512 4513 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity. 4514 SDValue IdentitySrc; 4515 bool IsIdentity = true; 4516 for (int i = 0; i != NumOps; ++i) { 4517 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT || 4518 Ops[i].getOperand(0).getValueType() != VT || 4519 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) || 4520 !isa<ConstantSDNode>(Ops[i].getOperand(1)) || 4521 cast<ConstantSDNode>(Ops[i].getOperand(1))->getAPIntValue() != i) { 4522 IsIdentity = false; 4523 break; 4524 } 4525 IdentitySrc = Ops[i].getOperand(0); 4526 } 4527 if (IsIdentity) 4528 return IdentitySrc; 4529 4530 return SDValue(); 4531 } 4532 4533 /// Try to simplify vector concatenation to an input value, undef, or build 4534 /// vector. 4535 static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, 4536 ArrayRef<SDValue> Ops, 4537 SelectionDAG &DAG) { 4538 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!"); 4539 assert(llvm::all_of(Ops, 4540 [Ops](SDValue Op) { 4541 return Ops[0].getValueType() == Op.getValueType(); 4542 }) && 4543 "Concatenation of vectors with inconsistent value types!"); 4544 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) == 4545 VT.getVectorElementCount() && 4546 "Incorrect element count in vector concatenation!"); 4547 4548 if (Ops.size() == 1) 4549 return Ops[0]; 4550 4551 // Concat of UNDEFs is UNDEF. 4552 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); })) 4553 return DAG.getUNDEF(VT); 4554 4555 // Scan the operands and look for extract operations from a single source 4556 // that correspond to insertion at the same location via this concatenation: 4557 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ... 4558 SDValue IdentitySrc; 4559 bool IsIdentity = true; 4560 for (unsigned i = 0, e = Ops.size(); i != e; ++i) { 4561 SDValue Op = Ops[i]; 4562 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements(); 4563 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR || 4564 Op.getOperand(0).getValueType() != VT || 4565 (IdentitySrc && Op.getOperand(0) != IdentitySrc) || 4566 Op.getConstantOperandVal(1) != IdentityIndex) { 4567 IsIdentity = false; 4568 break; 4569 } 4570 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) && 4571 "Unexpected identity source vector for concat of extracts"); 4572 IdentitySrc = Op.getOperand(0); 4573 } 4574 if (IsIdentity) { 4575 assert(IdentitySrc && "Failed to set source vector of extracts"); 4576 return IdentitySrc; 4577 } 4578 4579 // The code below this point is only designed to work for fixed width 4580 // vectors, so we bail out for now. 4581 if (VT.isScalableVector()) 4582 return SDValue(); 4583 4584 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be 4585 // simplified to one big BUILD_VECTOR. 4586 // FIXME: Add support for SCALAR_TO_VECTOR as well. 4587 EVT SVT = VT.getScalarType(); 4588 SmallVector<SDValue, 16> Elts; 4589 for (SDValue Op : Ops) { 4590 EVT OpVT = Op.getValueType(); 4591 if (Op.isUndef()) 4592 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT)); 4593 else if (Op.getOpcode() == ISD::BUILD_VECTOR) 4594 Elts.append(Op->op_begin(), Op->op_end()); 4595 else 4596 return SDValue(); 4597 } 4598 4599 // BUILD_VECTOR requires all inputs to be of the same type, find the 4600 // maximum type and extend them all. 4601 for (SDValue Op : Elts) 4602 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT); 4603 4604 if (SVT.bitsGT(VT.getScalarType())) { 4605 for (SDValue &Op : Elts) { 4606 if (Op.isUndef()) 4607 Op = DAG.getUNDEF(SVT); 4608 else 4609 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT) 4610 ? DAG.getZExtOrTrunc(Op, DL, SVT) 4611 : DAG.getSExtOrTrunc(Op, DL, SVT); 4612 } 4613 } 4614 4615 SDValue V = DAG.getBuildVector(VT, DL, Elts); 4616 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG); 4617 return V; 4618 } 4619 4620 /// Gets or creates the specified node. 4621 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) { 4622 FoldingSetNodeID ID; 4623 AddNodeIDNode(ID, Opcode, getVTList(VT), None); 4624 void *IP = nullptr; 4625 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 4626 return SDValue(E, 0); 4627 4628 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4629 getVTList(VT)); 4630 CSEMap.InsertNode(N, IP); 4631 4632 InsertNode(N); 4633 SDValue V = SDValue(N, 0); 4634 NewSDValueDbgMsg(V, "Creating new node: ", this); 4635 return V; 4636 } 4637 4638 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 4639 SDValue Operand) { 4640 SDNodeFlags Flags; 4641 if (Inserter) 4642 Flags = Inserter->getFlags(); 4643 return getNode(Opcode, DL, VT, Operand, Flags); 4644 } 4645 4646 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 4647 SDValue Operand, const SDNodeFlags Flags) { 4648 assert(Operand.getOpcode() != ISD::DELETED_NODE && 4649 "Operand is DELETED_NODE!"); 4650 // Constant fold unary operations with an integer constant operand. Even 4651 // opaque constant will be folded, because the folding of unary operations 4652 // doesn't create new constants with different values. Nevertheless, the 4653 // opaque flag is preserved during folding to prevent future folding with 4654 // other constants. 4655 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) { 4656 const APInt &Val = C->getAPIntValue(); 4657 switch (Opcode) { 4658 default: break; 4659 case ISD::SIGN_EXTEND: 4660 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT, 4661 C->isTargetOpcode(), C->isOpaque()); 4662 case ISD::TRUNCATE: 4663 if (C->isOpaque()) 4664 break; 4665 LLVM_FALLTHROUGH; 4666 case ISD::ZERO_EXTEND: 4667 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT, 4668 C->isTargetOpcode(), C->isOpaque()); 4669 case ISD::ANY_EXTEND: 4670 // Some targets like RISCV prefer to sign extend some types. 4671 if (TLI->isSExtCheaperThanZExt(Operand.getValueType(), VT)) 4672 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT, 4673 C->isTargetOpcode(), C->isOpaque()); 4674 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT, 4675 C->isTargetOpcode(), C->isOpaque()); 4676 case ISD::UINT_TO_FP: 4677 case ISD::SINT_TO_FP: { 4678 APFloat apf(EVTToAPFloatSemantics(VT), 4679 APInt::getNullValue(VT.getSizeInBits())); 4680 (void)apf.convertFromAPInt(Val, 4681 Opcode==ISD::SINT_TO_FP, 4682 APFloat::rmNearestTiesToEven); 4683 return getConstantFP(apf, DL, VT); 4684 } 4685 case ISD::BITCAST: 4686 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16) 4687 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT); 4688 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 4689 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT); 4690 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 4691 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT); 4692 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128) 4693 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT); 4694 break; 4695 case ISD::ABS: 4696 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(), 4697 C->isOpaque()); 4698 case ISD::BITREVERSE: 4699 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(), 4700 C->isOpaque()); 4701 case ISD::BSWAP: 4702 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(), 4703 C->isOpaque()); 4704 case ISD::CTPOP: 4705 return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(), 4706 C->isOpaque()); 4707 case ISD::CTLZ: 4708 case ISD::CTLZ_ZERO_UNDEF: 4709 return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(), 4710 C->isOpaque()); 4711 case ISD::CTTZ: 4712 case ISD::CTTZ_ZERO_UNDEF: 4713 return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(), 4714 C->isOpaque()); 4715 case ISD::FP16_TO_FP: { 4716 bool Ignored; 4717 APFloat FPV(APFloat::IEEEhalf(), 4718 (Val.getBitWidth() == 16) ? Val : Val.trunc(16)); 4719 4720 // This can return overflow, underflow, or inexact; we don't care. 4721 // FIXME need to be more flexible about rounding mode. 4722 (void)FPV.convert(EVTToAPFloatSemantics(VT), 4723 APFloat::rmNearestTiesToEven, &Ignored); 4724 return getConstantFP(FPV, DL, VT); 4725 } 4726 case ISD::STEP_VECTOR: { 4727 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Operand, *this)) 4728 return V; 4729 break; 4730 } 4731 } 4732 } 4733 4734 // Constant fold unary operations with a floating point constant operand. 4735 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) { 4736 APFloat V = C->getValueAPF(); // make copy 4737 switch (Opcode) { 4738 case ISD::FNEG: 4739 V.changeSign(); 4740 return getConstantFP(V, DL, VT); 4741 case ISD::FABS: 4742 V.clearSign(); 4743 return getConstantFP(V, DL, VT); 4744 case ISD::FCEIL: { 4745 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); 4746 if (fs == APFloat::opOK || fs == APFloat::opInexact) 4747 return getConstantFP(V, DL, VT); 4748 break; 4749 } 4750 case ISD::FTRUNC: { 4751 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); 4752 if (fs == APFloat::opOK || fs == APFloat::opInexact) 4753 return getConstantFP(V, DL, VT); 4754 break; 4755 } 4756 case ISD::FFLOOR: { 4757 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); 4758 if (fs == APFloat::opOK || fs == APFloat::opInexact) 4759 return getConstantFP(V, DL, VT); 4760 break; 4761 } 4762 case ISD::FP_EXTEND: { 4763 bool ignored; 4764 // This can return overflow, underflow, or inexact; we don't care. 4765 // FIXME need to be more flexible about rounding mode. 4766 (void)V.convert(EVTToAPFloatSemantics(VT), 4767 APFloat::rmNearestTiesToEven, &ignored); 4768 return getConstantFP(V, DL, VT); 4769 } 4770 case ISD::FP_TO_SINT: 4771 case ISD::FP_TO_UINT: { 4772 bool ignored; 4773 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT); 4774 // FIXME need to be more flexible about rounding mode. 4775 APFloat::opStatus s = 4776 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored); 4777 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual 4778 break; 4779 return getConstant(IntVal, DL, VT); 4780 } 4781 case ISD::BITCAST: 4782 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16) 4783 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT); 4784 if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16) 4785 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT); 4786 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 4787 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT); 4788 if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 4789 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT); 4790 break; 4791 case ISD::FP_TO_FP16: { 4792 bool Ignored; 4793 // This can return overflow, underflow, or inexact; we don't care. 4794 // FIXME need to be more flexible about rounding mode. 4795 (void)V.convert(APFloat::IEEEhalf(), 4796 APFloat::rmNearestTiesToEven, &Ignored); 4797 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT); 4798 } 4799 } 4800 } 4801 4802 // Constant fold unary operations with a vector integer or float operand. 4803 switch (Opcode) { 4804 default: 4805 // FIXME: Entirely reasonable to perform folding of other unary 4806 // operations here as the need arises. 4807 break; 4808 case ISD::FNEG: 4809 case ISD::FABS: 4810 case ISD::FCEIL: 4811 case ISD::FTRUNC: 4812 case ISD::FFLOOR: 4813 case ISD::FP_EXTEND: 4814 case ISD::FP_TO_SINT: 4815 case ISD::FP_TO_UINT: 4816 case ISD::TRUNCATE: 4817 case ISD::ANY_EXTEND: 4818 case ISD::ZERO_EXTEND: 4819 case ISD::SIGN_EXTEND: 4820 case ISD::UINT_TO_FP: 4821 case ISD::SINT_TO_FP: 4822 case ISD::ABS: 4823 case ISD::BITREVERSE: 4824 case ISD::BSWAP: 4825 case ISD::CTLZ: 4826 case ISD::CTLZ_ZERO_UNDEF: 4827 case ISD::CTTZ: 4828 case ISD::CTTZ_ZERO_UNDEF: 4829 case ISD::CTPOP: { 4830 SDValue Ops = {Operand}; 4831 if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) 4832 return Fold; 4833 } 4834 } 4835 4836 unsigned OpOpcode = Operand.getNode()->getOpcode(); 4837 switch (Opcode) { 4838 case ISD::STEP_VECTOR: 4839 assert(VT.isScalableVector() && 4840 "STEP_VECTOR can only be used with scalable types"); 4841 assert(OpOpcode == ISD::TargetConstant && 4842 VT.getVectorElementType() == Operand.getValueType() && 4843 "Unexpected step operand"); 4844 break; 4845 case ISD::FREEZE: 4846 assert(VT == Operand.getValueType() && "Unexpected VT!"); 4847 break; 4848 case ISD::TokenFactor: 4849 case ISD::MERGE_VALUES: 4850 case ISD::CONCAT_VECTORS: 4851 return Operand; // Factor, merge or concat of one node? No need. 4852 case ISD::BUILD_VECTOR: { 4853 // Attempt to simplify BUILD_VECTOR. 4854 SDValue Ops[] = {Operand}; 4855 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) 4856 return V; 4857 break; 4858 } 4859 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 4860 case ISD::FP_EXTEND: 4861 assert(VT.isFloatingPoint() && 4862 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 4863 if (Operand.getValueType() == VT) return Operand; // noop conversion. 4864 assert((!VT.isVector() || 4865 VT.getVectorElementCount() == 4866 Operand.getValueType().getVectorElementCount()) && 4867 "Vector element count mismatch!"); 4868 assert(Operand.getValueType().bitsLT(VT) && 4869 "Invalid fpext node, dst < src!"); 4870 if (Operand.isUndef()) 4871 return getUNDEF(VT); 4872 break; 4873 case ISD::FP_TO_SINT: 4874 case ISD::FP_TO_UINT: 4875 if (Operand.isUndef()) 4876 return getUNDEF(VT); 4877 break; 4878 case ISD::SINT_TO_FP: 4879 case ISD::UINT_TO_FP: 4880 // [us]itofp(undef) = 0, because the result value is bounded. 4881 if (Operand.isUndef()) 4882 return getConstantFP(0.0, DL, VT); 4883 break; 4884 case ISD::SIGN_EXTEND: 4885 assert(VT.isInteger() && Operand.getValueType().isInteger() && 4886 "Invalid SIGN_EXTEND!"); 4887 assert(VT.isVector() == Operand.getValueType().isVector() && 4888 "SIGN_EXTEND result type type should be vector iff the operand " 4889 "type is vector!"); 4890 if (Operand.getValueType() == VT) return Operand; // noop extension 4891 assert((!VT.isVector() || 4892 VT.getVectorElementCount() == 4893 Operand.getValueType().getVectorElementCount()) && 4894 "Vector element count mismatch!"); 4895 assert(Operand.getValueType().bitsLT(VT) && 4896 "Invalid sext node, dst < src!"); 4897 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 4898 return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); 4899 if (OpOpcode == ISD::UNDEF) 4900 // sext(undef) = 0, because the top bits will all be the same. 4901 return getConstant(0, DL, VT); 4902 break; 4903 case ISD::ZERO_EXTEND: 4904 assert(VT.isInteger() && Operand.getValueType().isInteger() && 4905 "Invalid ZERO_EXTEND!"); 4906 assert(VT.isVector() == Operand.getValueType().isVector() && 4907 "ZERO_EXTEND result type type should be vector iff the operand " 4908 "type is vector!"); 4909 if (Operand.getValueType() == VT) return Operand; // noop extension 4910 assert((!VT.isVector() || 4911 VT.getVectorElementCount() == 4912 Operand.getValueType().getVectorElementCount()) && 4913 "Vector element count mismatch!"); 4914 assert(Operand.getValueType().bitsLT(VT) && 4915 "Invalid zext node, dst < src!"); 4916 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 4917 return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0)); 4918 if (OpOpcode == ISD::UNDEF) 4919 // zext(undef) = 0, because the top bits will be zero. 4920 return getConstant(0, DL, VT); 4921 break; 4922 case ISD::ANY_EXTEND: 4923 assert(VT.isInteger() && Operand.getValueType().isInteger() && 4924 "Invalid ANY_EXTEND!"); 4925 assert(VT.isVector() == Operand.getValueType().isVector() && 4926 "ANY_EXTEND result type type should be vector iff the operand " 4927 "type is vector!"); 4928 if (Operand.getValueType() == VT) return Operand; // noop extension 4929 assert((!VT.isVector() || 4930 VT.getVectorElementCount() == 4931 Operand.getValueType().getVectorElementCount()) && 4932 "Vector element count mismatch!"); 4933 assert(Operand.getValueType().bitsLT(VT) && 4934 "Invalid anyext node, dst < src!"); 4935 4936 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 4937 OpOpcode == ISD::ANY_EXTEND) 4938 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 4939 return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); 4940 if (OpOpcode == ISD::UNDEF) 4941 return getUNDEF(VT); 4942 4943 // (ext (trunc x)) -> x 4944 if (OpOpcode == ISD::TRUNCATE) { 4945 SDValue OpOp = Operand.getOperand(0); 4946 if (OpOp.getValueType() == VT) { 4947 transferDbgValues(Operand, OpOp); 4948 return OpOp; 4949 } 4950 } 4951 break; 4952 case ISD::TRUNCATE: 4953 assert(VT.isInteger() && Operand.getValueType().isInteger() && 4954 "Invalid TRUNCATE!"); 4955 assert(VT.isVector() == Operand.getValueType().isVector() && 4956 "TRUNCATE result type type should be vector iff the operand " 4957 "type is vector!"); 4958 if (Operand.getValueType() == VT) return Operand; // noop truncate 4959 assert((!VT.isVector() || 4960 VT.getVectorElementCount() == 4961 Operand.getValueType().getVectorElementCount()) && 4962 "Vector element count mismatch!"); 4963 assert(Operand.getValueType().bitsGT(VT) && 4964 "Invalid truncate node, src < dst!"); 4965 if (OpOpcode == ISD::TRUNCATE) 4966 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0)); 4967 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 4968 OpOpcode == ISD::ANY_EXTEND) { 4969 // If the source is smaller than the dest, we still need an extend. 4970 if (Operand.getOperand(0).getValueType().getScalarType() 4971 .bitsLT(VT.getScalarType())) 4972 return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); 4973 if (Operand.getOperand(0).getValueType().bitsGT(VT)) 4974 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0)); 4975 return Operand.getOperand(0); 4976 } 4977 if (OpOpcode == ISD::UNDEF) 4978 return getUNDEF(VT); 4979 break; 4980 case ISD::ANY_EXTEND_VECTOR_INREG: 4981 case ISD::ZERO_EXTEND_VECTOR_INREG: 4982 case ISD::SIGN_EXTEND_VECTOR_INREG: 4983 assert(VT.isVector() && "This DAG node is restricted to vector types."); 4984 assert(Operand.getValueType().bitsLE(VT) && 4985 "The input must be the same size or smaller than the result."); 4986 assert(VT.getVectorMinNumElements() < 4987 Operand.getValueType().getVectorMinNumElements() && 4988 "The destination vector type must have fewer lanes than the input."); 4989 break; 4990 case ISD::ABS: 4991 assert(VT.isInteger() && VT == Operand.getValueType() && 4992 "Invalid ABS!"); 4993 if (OpOpcode == ISD::UNDEF) 4994 return getUNDEF(VT); 4995 break; 4996 case ISD::BSWAP: 4997 assert(VT.isInteger() && VT == Operand.getValueType() && 4998 "Invalid BSWAP!"); 4999 assert((VT.getScalarSizeInBits() % 16 == 0) && 5000 "BSWAP types must be a multiple of 16 bits!"); 5001 if (OpOpcode == ISD::UNDEF) 5002 return getUNDEF(VT); 5003 break; 5004 case ISD::BITREVERSE: 5005 assert(VT.isInteger() && VT == Operand.getValueType() && 5006 "Invalid BITREVERSE!"); 5007 if (OpOpcode == ISD::UNDEF) 5008 return getUNDEF(VT); 5009 break; 5010 case ISD::BITCAST: 5011 // Basic sanity checking. 5012 assert(VT.getSizeInBits() == Operand.getValueSizeInBits() && 5013 "Cannot BITCAST between types of different sizes!"); 5014 if (VT == Operand.getValueType()) return Operand; // noop conversion. 5015 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 5016 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 5017 if (OpOpcode == ISD::UNDEF) 5018 return getUNDEF(VT); 5019 break; 5020 case ISD::SCALAR_TO_VECTOR: 5021 assert(VT.isVector() && !Operand.getValueType().isVector() && 5022 (VT.getVectorElementType() == Operand.getValueType() || 5023 (VT.getVectorElementType().isInteger() && 5024 Operand.getValueType().isInteger() && 5025 VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 5026 "Illegal SCALAR_TO_VECTOR node!"); 5027 if (OpOpcode == ISD::UNDEF) 5028 return getUNDEF(VT); 5029 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 5030 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 5031 isa<ConstantSDNode>(Operand.getOperand(1)) && 5032 Operand.getConstantOperandVal(1) == 0 && 5033 Operand.getOperand(0).getValueType() == VT) 5034 return Operand.getOperand(0); 5035 break; 5036 case ISD::FNEG: 5037 // Negation of an unknown bag of bits is still completely undefined. 5038 if (OpOpcode == ISD::UNDEF) 5039 return getUNDEF(VT); 5040 5041 if (OpOpcode == ISD::FNEG) // --X -> X 5042 return Operand.getOperand(0); 5043 break; 5044 case ISD::FABS: 5045 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 5046 return getNode(ISD::FABS, DL, VT, Operand.getOperand(0)); 5047 break; 5048 case ISD::VSCALE: 5049 assert(VT == Operand.getValueType() && "Unexpected VT!"); 5050 break; 5051 case ISD::CTPOP: 5052 if (Operand.getValueType().getScalarType() == MVT::i1) 5053 return Operand; 5054 break; 5055 case ISD::CTLZ: 5056 case ISD::CTTZ: 5057 if (Operand.getValueType().getScalarType() == MVT::i1) 5058 return getNOT(DL, Operand, Operand.getValueType()); 5059 break; 5060 case ISD::VECREDUCE_SMIN: 5061 case ISD::VECREDUCE_UMAX: 5062 if (Operand.getValueType().getScalarType() == MVT::i1) 5063 return getNode(ISD::VECREDUCE_OR, DL, VT, Operand); 5064 break; 5065 case ISD::VECREDUCE_SMAX: 5066 case ISD::VECREDUCE_UMIN: 5067 if (Operand.getValueType().getScalarType() == MVT::i1) 5068 return getNode(ISD::VECREDUCE_AND, DL, VT, Operand); 5069 break; 5070 } 5071 5072 SDNode *N; 5073 SDVTList VTs = getVTList(VT); 5074 SDValue Ops[] = {Operand}; 5075 if (VT != MVT::Glue) { // Don't CSE flag producing nodes 5076 FoldingSetNodeID ID; 5077 AddNodeIDNode(ID, Opcode, VTs, Ops); 5078 void *IP = nullptr; 5079 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 5080 E->intersectFlagsWith(Flags); 5081 return SDValue(E, 0); 5082 } 5083 5084 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 5085 N->setFlags(Flags); 5086 createOperands(N, Ops); 5087 CSEMap.InsertNode(N, IP); 5088 } else { 5089 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 5090 createOperands(N, Ops); 5091 } 5092 5093 InsertNode(N); 5094 SDValue V = SDValue(N, 0); 5095 NewSDValueDbgMsg(V, "Creating new node: ", this); 5096 return V; 5097 } 5098 5099 static llvm::Optional<APInt> FoldValue(unsigned Opcode, const APInt &C1, 5100 const APInt &C2) { 5101 switch (Opcode) { 5102 case ISD::ADD: return C1 + C2; 5103 case ISD::SUB: return C1 - C2; 5104 case ISD::MUL: return C1 * C2; 5105 case ISD::AND: return C1 & C2; 5106 case ISD::OR: return C1 | C2; 5107 case ISD::XOR: return C1 ^ C2; 5108 case ISD::SHL: return C1 << C2; 5109 case ISD::SRL: return C1.lshr(C2); 5110 case ISD::SRA: return C1.ashr(C2); 5111 case ISD::ROTL: return C1.rotl(C2); 5112 case ISD::ROTR: return C1.rotr(C2); 5113 case ISD::SMIN: return C1.sle(C2) ? C1 : C2; 5114 case ISD::SMAX: return C1.sge(C2) ? C1 : C2; 5115 case ISD::UMIN: return C1.ule(C2) ? C1 : C2; 5116 case ISD::UMAX: return C1.uge(C2) ? C1 : C2; 5117 case ISD::SADDSAT: return C1.sadd_sat(C2); 5118 case ISD::UADDSAT: return C1.uadd_sat(C2); 5119 case ISD::SSUBSAT: return C1.ssub_sat(C2); 5120 case ISD::USUBSAT: return C1.usub_sat(C2); 5121 case ISD::UDIV: 5122 if (!C2.getBoolValue()) 5123 break; 5124 return C1.udiv(C2); 5125 case ISD::UREM: 5126 if (!C2.getBoolValue()) 5127 break; 5128 return C1.urem(C2); 5129 case ISD::SDIV: 5130 if (!C2.getBoolValue()) 5131 break; 5132 return C1.sdiv(C2); 5133 case ISD::SREM: 5134 if (!C2.getBoolValue()) 5135 break; 5136 return C1.srem(C2); 5137 case ISD::MULHS: { 5138 unsigned FullWidth = C1.getBitWidth() * 2; 5139 APInt C1Ext = C1.sext(FullWidth); 5140 APInt C2Ext = C2.sext(FullWidth); 5141 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth()); 5142 } 5143 case ISD::MULHU: { 5144 unsigned FullWidth = C1.getBitWidth() * 2; 5145 APInt C1Ext = C1.zext(FullWidth); 5146 APInt C2Ext = C2.zext(FullWidth); 5147 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth()); 5148 } 5149 } 5150 return llvm::None; 5151 } 5152 5153 SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT, 5154 const GlobalAddressSDNode *GA, 5155 const SDNode *N2) { 5156 if (GA->getOpcode() != ISD::GlobalAddress) 5157 return SDValue(); 5158 if (!TLI->isOffsetFoldingLegal(GA)) 5159 return SDValue(); 5160 auto *C2 = dyn_cast<ConstantSDNode>(N2); 5161 if (!C2) 5162 return SDValue(); 5163 int64_t Offset = C2->getSExtValue(); 5164 switch (Opcode) { 5165 case ISD::ADD: break; 5166 case ISD::SUB: Offset = -uint64_t(Offset); break; 5167 default: return SDValue(); 5168 } 5169 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT, 5170 GA->getOffset() + uint64_t(Offset)); 5171 } 5172 5173 bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) { 5174 switch (Opcode) { 5175 case ISD::SDIV: 5176 case ISD::UDIV: 5177 case ISD::SREM: 5178 case ISD::UREM: { 5179 // If a divisor is zero/undef or any element of a divisor vector is 5180 // zero/undef, the whole op is undef. 5181 assert(Ops.size() == 2 && "Div/rem should have 2 operands"); 5182 SDValue Divisor = Ops[1]; 5183 if (Divisor.isUndef() || isNullConstant(Divisor)) 5184 return true; 5185 5186 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) && 5187 llvm::any_of(Divisor->op_values(), 5188 [](SDValue V) { return V.isUndef() || 5189 isNullConstant(V); }); 5190 // TODO: Handle signed overflow. 5191 } 5192 // TODO: Handle oversized shifts. 5193 default: 5194 return false; 5195 } 5196 } 5197 5198 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, 5199 EVT VT, ArrayRef<SDValue> Ops) { 5200 // If the opcode is a target-specific ISD node, there's nothing we can 5201 // do here and the operand rules may not line up with the below, so 5202 // bail early. 5203 // We can't create a scalar CONCAT_VECTORS so skip it. It will break 5204 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by 5205 // foldCONCAT_VECTORS in getNode before this is called. 5206 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS) 5207 return SDValue(); 5208 5209 // For now, the array Ops should only contain two values. 5210 // This enforcement will be removed once this function is merged with 5211 // FoldConstantVectorArithmetic 5212 if (Ops.size() != 2) 5213 return SDValue(); 5214 5215 if (isUndef(Opcode, Ops)) 5216 return getUNDEF(VT); 5217 5218 SDNode *N1 = Ops[0].getNode(); 5219 SDNode *N2 = Ops[1].getNode(); 5220 5221 // Handle the case of two scalars. 5222 if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) { 5223 if (auto *C2 = dyn_cast<ConstantSDNode>(N2)) { 5224 if (C1->isOpaque() || C2->isOpaque()) 5225 return SDValue(); 5226 5227 Optional<APInt> FoldAttempt = 5228 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue()); 5229 if (!FoldAttempt) 5230 return SDValue(); 5231 5232 SDValue Folded = getConstant(FoldAttempt.getValue(), DL, VT); 5233 assert((!Folded || !VT.isVector()) && 5234 "Can't fold vectors ops with scalar operands"); 5235 return Folded; 5236 } 5237 } 5238 5239 // fold (add Sym, c) -> Sym+c 5240 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N1)) 5241 return FoldSymbolOffset(Opcode, VT, GA, N2); 5242 if (TLI->isCommutativeBinOp(Opcode)) 5243 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2)) 5244 return FoldSymbolOffset(Opcode, VT, GA, N1); 5245 5246 // For fixed width vectors, extract each constant element and fold them 5247 // individually. Either input may be an undef value. 5248 bool IsBVOrSV1 = N1->getOpcode() == ISD::BUILD_VECTOR || 5249 N1->getOpcode() == ISD::SPLAT_VECTOR; 5250 if (!IsBVOrSV1 && !N1->isUndef()) 5251 return SDValue(); 5252 bool IsBVOrSV2 = N2->getOpcode() == ISD::BUILD_VECTOR || 5253 N2->getOpcode() == ISD::SPLAT_VECTOR; 5254 if (!IsBVOrSV2 && !N2->isUndef()) 5255 return SDValue(); 5256 // If both operands are undef, that's handled the same way as scalars. 5257 if (!IsBVOrSV1 && !IsBVOrSV2) 5258 return SDValue(); 5259 5260 EVT SVT = VT.getScalarType(); 5261 EVT LegalSVT = SVT; 5262 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) { 5263 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); 5264 if (LegalSVT.bitsLT(SVT)) 5265 return SDValue(); 5266 } 5267 5268 SmallVector<SDValue, 4> Outputs; 5269 unsigned NumOps = 0; 5270 if (IsBVOrSV1) 5271 NumOps = std::max(NumOps, N1->getNumOperands()); 5272 if (IsBVOrSV2) 5273 NumOps = std::max(NumOps, N2->getNumOperands()); 5274 assert(NumOps != 0 && "Expected non-zero operands"); 5275 // Scalable vectors should only be SPLAT_VECTOR or UNDEF here. We only need 5276 // one iteration for that. 5277 assert((!VT.isScalableVector() || NumOps == 1) && 5278 "Scalable vector should only have one scalar"); 5279 5280 for (unsigned I = 0; I != NumOps; ++I) { 5281 // We can have a fixed length SPLAT_VECTOR and a BUILD_VECTOR so we need 5282 // to use operand 0 of the SPLAT_VECTOR for each fixed element. 5283 SDValue V1; 5284 if (N1->getOpcode() == ISD::BUILD_VECTOR) 5285 V1 = N1->getOperand(I); 5286 else if (N1->getOpcode() == ISD::SPLAT_VECTOR) 5287 V1 = N1->getOperand(0); 5288 else 5289 V1 = getUNDEF(SVT); 5290 5291 SDValue V2; 5292 if (N2->getOpcode() == ISD::BUILD_VECTOR) 5293 V2 = N2->getOperand(I); 5294 else if (N2->getOpcode() == ISD::SPLAT_VECTOR) 5295 V2 = N2->getOperand(0); 5296 else 5297 V2 = getUNDEF(SVT); 5298 5299 if (SVT.isInteger()) { 5300 if (V1.getValueType().bitsGT(SVT)) 5301 V1 = getNode(ISD::TRUNCATE, DL, SVT, V1); 5302 if (V2.getValueType().bitsGT(SVT)) 5303 V2 = getNode(ISD::TRUNCATE, DL, SVT, V2); 5304 } 5305 5306 if (V1.getValueType() != SVT || V2.getValueType() != SVT) 5307 return SDValue(); 5308 5309 // Fold one vector element. 5310 SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2); 5311 if (LegalSVT != SVT) 5312 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult); 5313 5314 // Scalar folding only succeeded if the result is a constant or UNDEF. 5315 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && 5316 ScalarResult.getOpcode() != ISD::ConstantFP) 5317 return SDValue(); 5318 Outputs.push_back(ScalarResult); 5319 } 5320 5321 if (N1->getOpcode() == ISD::BUILD_VECTOR || 5322 N2->getOpcode() == ISD::BUILD_VECTOR) { 5323 assert(VT.getVectorNumElements() == Outputs.size() && 5324 "Vector size mismatch!"); 5325 5326 // Build a big vector out of the scalar elements we generated. 5327 return getBuildVector(VT, SDLoc(), Outputs); 5328 } 5329 5330 assert((N1->getOpcode() == ISD::SPLAT_VECTOR || 5331 N2->getOpcode() == ISD::SPLAT_VECTOR) && 5332 "One operand should be a splat vector"); 5333 5334 assert(Outputs.size() == 1 && "Vector size mismatch!"); 5335 return getSplatVector(VT, SDLoc(), Outputs[0]); 5336 } 5337 5338 // TODO: Merge with FoldConstantArithmetic 5339 SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode, 5340 const SDLoc &DL, EVT VT, 5341 ArrayRef<SDValue> Ops, 5342 const SDNodeFlags Flags) { 5343 // If the opcode is a target-specific ISD node, there's nothing we can 5344 // do here and the operand rules may not line up with the below, so 5345 // bail early. 5346 if (Opcode >= ISD::BUILTIN_OP_END) 5347 return SDValue(); 5348 5349 if (isUndef(Opcode, Ops)) 5350 return getUNDEF(VT); 5351 5352 // We can only fold vectors - maybe merge with FoldConstantArithmetic someday? 5353 if (!VT.isVector()) 5354 return SDValue(); 5355 5356 ElementCount NumElts = VT.getVectorElementCount(); 5357 5358 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) { 5359 return !Op.getValueType().isVector() || 5360 Op.getValueType().getVectorElementCount() == NumElts; 5361 }; 5362 5363 auto IsConstantBuildVectorSplatVectorOrUndef = [](const SDValue &Op) { 5364 APInt SplatVal; 5365 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op); 5366 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE || 5367 (BV && BV->isConstant()) || 5368 (Op.getOpcode() == ISD::SPLAT_VECTOR && 5369 ISD::isConstantSplatVector(Op.getNode(), SplatVal)); 5370 }; 5371 5372 // All operands must be vector types with the same number of elements as 5373 // the result type and must be either UNDEF or a build vector of constant 5374 // or UNDEF scalars. 5375 if (!llvm::all_of(Ops, IsConstantBuildVectorSplatVectorOrUndef) || 5376 !llvm::all_of(Ops, IsScalarOrSameVectorSize)) 5377 return SDValue(); 5378 5379 // If we are comparing vectors, then the result needs to be a i1 boolean 5380 // that is then sign-extended back to the legal result type. 5381 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType()); 5382 5383 // Find legal integer scalar type for constant promotion and 5384 // ensure that its scalar size is at least as large as source. 5385 EVT LegalSVT = VT.getScalarType(); 5386 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) { 5387 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); 5388 if (LegalSVT.bitsLT(VT.getScalarType())) 5389 return SDValue(); 5390 } 5391 5392 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We 5393 // only have one operand to check. For fixed-length vector types we may have 5394 // a combination of BUILD_VECTOR and SPLAT_VECTOR. 5395 unsigned NumOperands = NumElts.isScalable() ? 1 : NumElts.getFixedValue(); 5396 5397 // Constant fold each scalar lane separately. 5398 SmallVector<SDValue, 4> ScalarResults; 5399 for (unsigned I = 0; I != NumOperands; I++) { 5400 SmallVector<SDValue, 4> ScalarOps; 5401 for (SDValue Op : Ops) { 5402 EVT InSVT = Op.getValueType().getScalarType(); 5403 if (Op.getOpcode() != ISD::BUILD_VECTOR && 5404 Op.getOpcode() != ISD::SPLAT_VECTOR) { 5405 // We've checked that this is UNDEF or a constant of some kind. 5406 if (Op.isUndef()) 5407 ScalarOps.push_back(getUNDEF(InSVT)); 5408 else 5409 ScalarOps.push_back(Op); 5410 continue; 5411 } 5412 5413 SDValue ScalarOp = 5414 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I); 5415 EVT ScalarVT = ScalarOp.getValueType(); 5416 5417 // Build vector (integer) scalar operands may need implicit 5418 // truncation - do this before constant folding. 5419 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) 5420 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp); 5421 5422 ScalarOps.push_back(ScalarOp); 5423 } 5424 5425 // Constant fold the scalar operands. 5426 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags); 5427 5428 // Legalize the (integer) scalar constant if necessary. 5429 if (LegalSVT != SVT) 5430 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult); 5431 5432 // Scalar folding only succeeded if the result is a constant or UNDEF. 5433 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && 5434 ScalarResult.getOpcode() != ISD::ConstantFP) 5435 return SDValue(); 5436 ScalarResults.push_back(ScalarResult); 5437 } 5438 5439 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0]) 5440 : getBuildVector(VT, DL, ScalarResults); 5441 NewSDValueDbgMsg(V, "New node fold constant vector: ", this); 5442 return V; 5443 } 5444 5445 SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL, 5446 EVT VT, SDValue N1, SDValue N2) { 5447 // TODO: We don't do any constant folding for strict FP opcodes here, but we 5448 // should. That will require dealing with a potentially non-default 5449 // rounding mode, checking the "opStatus" return value from the APFloat 5450 // math calculations, and possibly other variations. 5451 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 5452 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 5453 if (N1CFP && N2CFP) { 5454 APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF(); 5455 switch (Opcode) { 5456 case ISD::FADD: 5457 C1.add(C2, APFloat::rmNearestTiesToEven); 5458 return getConstantFP(C1, DL, VT); 5459 case ISD::FSUB: 5460 C1.subtract(C2, APFloat::rmNearestTiesToEven); 5461 return getConstantFP(C1, DL, VT); 5462 case ISD::FMUL: 5463 C1.multiply(C2, APFloat::rmNearestTiesToEven); 5464 return getConstantFP(C1, DL, VT); 5465 case ISD::FDIV: 5466 C1.divide(C2, APFloat::rmNearestTiesToEven); 5467 return getConstantFP(C1, DL, VT); 5468 case ISD::FREM: 5469 C1.mod(C2); 5470 return getConstantFP(C1, DL, VT); 5471 case ISD::FCOPYSIGN: 5472 C1.copySign(C2); 5473 return getConstantFP(C1, DL, VT); 5474 default: break; 5475 } 5476 } 5477 if (N1CFP && Opcode == ISD::FP_ROUND) { 5478 APFloat C1 = N1CFP->getValueAPF(); // make copy 5479 bool Unused; 5480 // This can return overflow, underflow, or inexact; we don't care. 5481 // FIXME need to be more flexible about rounding mode. 5482 (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, 5483 &Unused); 5484 return getConstantFP(C1, DL, VT); 5485 } 5486 5487 switch (Opcode) { 5488 case ISD::FSUB: 5489 // -0.0 - undef --> undef (consistent with "fneg undef") 5490 if (N1CFP && N1CFP->getValueAPF().isNegZero() && N2.isUndef()) 5491 return getUNDEF(VT); 5492 LLVM_FALLTHROUGH; 5493 5494 case ISD::FADD: 5495 case ISD::FMUL: 5496 case ISD::FDIV: 5497 case ISD::FREM: 5498 // If both operands are undef, the result is undef. If 1 operand is undef, 5499 // the result is NaN. This should match the behavior of the IR optimizer. 5500 if (N1.isUndef() && N2.isUndef()) 5501 return getUNDEF(VT); 5502 if (N1.isUndef() || N2.isUndef()) 5503 return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT); 5504 } 5505 return SDValue(); 5506 } 5507 5508 SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) { 5509 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!"); 5510 5511 // There's no need to assert on a byte-aligned pointer. All pointers are at 5512 // least byte aligned. 5513 if (A == Align(1)) 5514 return Val; 5515 5516 FoldingSetNodeID ID; 5517 AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val}); 5518 ID.AddInteger(A.value()); 5519 5520 void *IP = nullptr; 5521 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 5522 return SDValue(E, 0); 5523 5524 auto *N = newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(), 5525 Val.getValueType(), A); 5526 createOperands(N, {Val}); 5527 5528 CSEMap.InsertNode(N, IP); 5529 InsertNode(N); 5530 5531 SDValue V(N, 0); 5532 NewSDValueDbgMsg(V, "Creating new node: ", this); 5533 return V; 5534 } 5535 5536 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 5537 SDValue N1, SDValue N2) { 5538 SDNodeFlags Flags; 5539 if (Inserter) 5540 Flags = Inserter->getFlags(); 5541 return getNode(Opcode, DL, VT, N1, N2, Flags); 5542 } 5543 5544 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 5545 SDValue N1, SDValue N2, const SDNodeFlags Flags) { 5546 assert(N1.getOpcode() != ISD::DELETED_NODE && 5547 N2.getOpcode() != ISD::DELETED_NODE && 5548 "Operand is DELETED_NODE!"); 5549 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 5550 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); 5551 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 5552 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 5553 5554 // Canonicalize constant to RHS if commutative. 5555 if (TLI->isCommutativeBinOp(Opcode)) { 5556 if (N1C && !N2C) { 5557 std::swap(N1C, N2C); 5558 std::swap(N1, N2); 5559 } else if (N1CFP && !N2CFP) { 5560 std::swap(N1CFP, N2CFP); 5561 std::swap(N1, N2); 5562 } 5563 } 5564 5565 switch (Opcode) { 5566 default: break; 5567 case ISD::TokenFactor: 5568 assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 5569 N2.getValueType() == MVT::Other && "Invalid token factor!"); 5570 // Fold trivial token factors. 5571 if (N1.getOpcode() == ISD::EntryToken) return N2; 5572 if (N2.getOpcode() == ISD::EntryToken) return N1; 5573 if (N1 == N2) return N1; 5574 break; 5575 case ISD::BUILD_VECTOR: { 5576 // Attempt to simplify BUILD_VECTOR. 5577 SDValue Ops[] = {N1, N2}; 5578 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) 5579 return V; 5580 break; 5581 } 5582 case ISD::CONCAT_VECTORS: { 5583 SDValue Ops[] = {N1, N2}; 5584 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) 5585 return V; 5586 break; 5587 } 5588 case ISD::AND: 5589 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5590 assert(N1.getValueType() == N2.getValueType() && 5591 N1.getValueType() == VT && "Binary operator types must match!"); 5592 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 5593 // worth handling here. 5594 if (N2C && N2C->isNullValue()) 5595 return N2; 5596 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 5597 return N1; 5598 break; 5599 case ISD::OR: 5600 case ISD::XOR: 5601 case ISD::ADD: 5602 case ISD::SUB: 5603 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5604 assert(N1.getValueType() == N2.getValueType() && 5605 N1.getValueType() == VT && "Binary operator types must match!"); 5606 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 5607 // it's worth handling here. 5608 if (N2C && N2C->isNullValue()) 5609 return N1; 5610 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() && 5611 VT.getVectorElementType() == MVT::i1) 5612 return getNode(ISD::XOR, DL, VT, N1, N2); 5613 break; 5614 case ISD::MUL: 5615 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5616 assert(N1.getValueType() == N2.getValueType() && 5617 N1.getValueType() == VT && "Binary operator types must match!"); 5618 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) 5619 return getNode(ISD::AND, DL, VT, N1, N2); 5620 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { 5621 const APInt &MulImm = N1->getConstantOperandAPInt(0); 5622 const APInt &N2CImm = N2C->getAPIntValue(); 5623 return getVScale(DL, VT, MulImm * N2CImm); 5624 } 5625 break; 5626 case ISD::UDIV: 5627 case ISD::UREM: 5628 case ISD::MULHU: 5629 case ISD::MULHS: 5630 case ISD::SDIV: 5631 case ISD::SREM: 5632 case ISD::SADDSAT: 5633 case ISD::SSUBSAT: 5634 case ISD::UADDSAT: 5635 case ISD::USUBSAT: 5636 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5637 assert(N1.getValueType() == N2.getValueType() && 5638 N1.getValueType() == VT && "Binary operator types must match!"); 5639 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) { 5640 // fold (add_sat x, y) -> (or x, y) for bool types. 5641 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT) 5642 return getNode(ISD::OR, DL, VT, N1, N2); 5643 // fold (sub_sat x, y) -> (and x, ~y) for bool types. 5644 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT) 5645 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT)); 5646 } 5647 break; 5648 case ISD::SMIN: 5649 case ISD::UMAX: 5650 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5651 assert(N1.getValueType() == N2.getValueType() && 5652 N1.getValueType() == VT && "Binary operator types must match!"); 5653 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) 5654 return getNode(ISD::OR, DL, VT, N1, N2); 5655 break; 5656 case ISD::SMAX: 5657 case ISD::UMIN: 5658 assert(VT.isInteger() && "This operator does not apply to FP types!"); 5659 assert(N1.getValueType() == N2.getValueType() && 5660 N1.getValueType() == VT && "Binary operator types must match!"); 5661 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) 5662 return getNode(ISD::AND, DL, VT, N1, N2); 5663 break; 5664 case ISD::FADD: 5665 case ISD::FSUB: 5666 case ISD::FMUL: 5667 case ISD::FDIV: 5668 case ISD::FREM: 5669 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 5670 assert(N1.getValueType() == N2.getValueType() && 5671 N1.getValueType() == VT && "Binary operator types must match!"); 5672 if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags)) 5673 return V; 5674 break; 5675 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 5676 assert(N1.getValueType() == VT && 5677 N1.getValueType().isFloatingPoint() && 5678 N2.getValueType().isFloatingPoint() && 5679 "Invalid FCOPYSIGN!"); 5680 break; 5681 case ISD::SHL: 5682 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { 5683 const APInt &MulImm = N1->getConstantOperandAPInt(0); 5684 const APInt &ShiftImm = N2C->getAPIntValue(); 5685 return getVScale(DL, VT, MulImm << ShiftImm); 5686 } 5687 LLVM_FALLTHROUGH; 5688 case ISD::SRA: 5689 case ISD::SRL: 5690 if (SDValue V = simplifyShift(N1, N2)) 5691 return V; 5692 LLVM_FALLTHROUGH; 5693 case ISD::ROTL: 5694 case ISD::ROTR: 5695 assert(VT == N1.getValueType() && 5696 "Shift operators return type must be the same as their first arg"); 5697 assert(VT.isInteger() && N2.getValueType().isInteger() && 5698 "Shifts only work on integers"); 5699 assert((!VT.isVector() || VT == N2.getValueType()) && 5700 "Vector shift amounts must be in the same as their first arg"); 5701 // Verify that the shift amount VT is big enough to hold valid shift 5702 // amounts. This catches things like trying to shift an i1024 value by an 5703 // i8, which is easy to fall into in generic code that uses 5704 // TLI.getShiftAmount(). 5705 assert(N2.getValueType().getScalarSizeInBits() >= 5706 Log2_32_Ceil(VT.getScalarSizeInBits()) && 5707 "Invalid use of small shift amount with oversized value!"); 5708 5709 // Always fold shifts of i1 values so the code generator doesn't need to 5710 // handle them. Since we know the size of the shift has to be less than the 5711 // size of the value, the shift/rotate count is guaranteed to be zero. 5712 if (VT == MVT::i1) 5713 return N1; 5714 if (N2C && N2C->isNullValue()) 5715 return N1; 5716 break; 5717 case ISD::FP_ROUND: 5718 assert(VT.isFloatingPoint() && 5719 N1.getValueType().isFloatingPoint() && 5720 VT.bitsLE(N1.getValueType()) && 5721 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && 5722 "Invalid FP_ROUND!"); 5723 if (N1.getValueType() == VT) return N1; // noop conversion. 5724 break; 5725 case ISD::AssertSext: 5726 case ISD::AssertZext: { 5727 EVT EVT = cast<VTSDNode>(N2)->getVT(); 5728 assert(VT == N1.getValueType() && "Not an inreg extend!"); 5729 assert(VT.isInteger() && EVT.isInteger() && 5730 "Cannot *_EXTEND_INREG FP types"); 5731 assert(!EVT.isVector() && 5732 "AssertSExt/AssertZExt type should be the vector element type " 5733 "rather than the vector type!"); 5734 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!"); 5735 if (VT.getScalarType() == EVT) return N1; // noop assertion. 5736 break; 5737 } 5738 case ISD::SIGN_EXTEND_INREG: { 5739 EVT EVT = cast<VTSDNode>(N2)->getVT(); 5740 assert(VT == N1.getValueType() && "Not an inreg extend!"); 5741 assert(VT.isInteger() && EVT.isInteger() && 5742 "Cannot *_EXTEND_INREG FP types"); 5743 assert(EVT.isVector() == VT.isVector() && 5744 "SIGN_EXTEND_INREG type should be vector iff the operand " 5745 "type is vector!"); 5746 assert((!EVT.isVector() || 5747 EVT.getVectorElementCount() == VT.getVectorElementCount()) && 5748 "Vector element counts must match in SIGN_EXTEND_INREG"); 5749 assert(EVT.bitsLE(VT) && "Not extending!"); 5750 if (EVT == VT) return N1; // Not actually extending 5751 5752 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) { 5753 unsigned FromBits = EVT.getScalarSizeInBits(); 5754 Val <<= Val.getBitWidth() - FromBits; 5755 Val.ashrInPlace(Val.getBitWidth() - FromBits); 5756 return getConstant(Val, DL, ConstantVT); 5757 }; 5758 5759 if (N1C) { 5760 const APInt &Val = N1C->getAPIntValue(); 5761 return SignExtendInReg(Val, VT); 5762 } 5763 5764 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) { 5765 SmallVector<SDValue, 8> Ops; 5766 llvm::EVT OpVT = N1.getOperand(0).getValueType(); 5767 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) { 5768 SDValue Op = N1.getOperand(i); 5769 if (Op.isUndef()) { 5770 Ops.push_back(getUNDEF(OpVT)); 5771 continue; 5772 } 5773 ConstantSDNode *C = cast<ConstantSDNode>(Op); 5774 APInt Val = C->getAPIntValue(); 5775 Ops.push_back(SignExtendInReg(Val, OpVT)); 5776 } 5777 return getBuildVector(VT, DL, Ops); 5778 } 5779 break; 5780 } 5781 case ISD::FP_TO_SINT_SAT: 5782 case ISD::FP_TO_UINT_SAT: { 5783 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() && 5784 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT"); 5785 assert(N1.getValueType().isVector() == VT.isVector() && 5786 "FP_TO_*INT_SAT type should be vector iff the operand type is " 5787 "vector!"); 5788 assert((!VT.isVector() || VT.getVectorNumElements() == 5789 N1.getValueType().getVectorNumElements()) && 5790 "Vector element counts must match in FP_TO_*INT_SAT"); 5791 assert(!cast<VTSDNode>(N2)->getVT().isVector() && 5792 "Type to saturate to must be a scalar."); 5793 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) && 5794 "Not extending!"); 5795 break; 5796 } 5797 case ISD::EXTRACT_VECTOR_ELT: 5798 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && 5799 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \ 5800 element type of the vector."); 5801 5802 // Extract from an undefined value or using an undefined index is undefined. 5803 if (N1.isUndef() || N2.isUndef()) 5804 return getUNDEF(VT); 5805 5806 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length 5807 // vectors. For scalable vectors we will provide appropriate support for 5808 // dealing with arbitrary indices. 5809 if (N2C && N1.getValueType().isFixedLengthVector() && 5810 N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements())) 5811 return getUNDEF(VT); 5812 5813 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 5814 // expanding copies of large vectors from registers. This only works for 5815 // fixed length vectors, since we need to know the exact number of 5816 // elements. 5817 if (N2C && N1.getOperand(0).getValueType().isFixedLengthVector() && 5818 N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0) { 5819 unsigned Factor = 5820 N1.getOperand(0).getValueType().getVectorNumElements(); 5821 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 5822 N1.getOperand(N2C->getZExtValue() / Factor), 5823 getVectorIdxConstant(N2C->getZExtValue() % Factor, DL)); 5824 } 5825 5826 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while 5827 // lowering is expanding large vector constants. 5828 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR || 5829 N1.getOpcode() == ISD::SPLAT_VECTOR)) { 5830 assert((N1.getOpcode() != ISD::BUILD_VECTOR || 5831 N1.getValueType().isFixedLengthVector()) && 5832 "BUILD_VECTOR used for scalable vectors"); 5833 unsigned Index = 5834 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0; 5835 SDValue Elt = N1.getOperand(Index); 5836 5837 if (VT != Elt.getValueType()) 5838 // If the vector element type is not legal, the BUILD_VECTOR operands 5839 // are promoted and implicitly truncated, and the result implicitly 5840 // extended. Make that explicit here. 5841 Elt = getAnyExtOrTrunc(Elt, DL, VT); 5842 5843 return Elt; 5844 } 5845 5846 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 5847 // operations are lowered to scalars. 5848 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 5849 // If the indices are the same, return the inserted element else 5850 // if the indices are known different, extract the element from 5851 // the original vector. 5852 SDValue N1Op2 = N1.getOperand(2); 5853 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2); 5854 5855 if (N1Op2C && N2C) { 5856 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 5857 if (VT == N1.getOperand(1).getValueType()) 5858 return N1.getOperand(1); 5859 return getSExtOrTrunc(N1.getOperand(1), DL, VT); 5860 } 5861 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 5862 } 5863 } 5864 5865 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed 5866 // when vector types are scalarized and v1iX is legal. 5867 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx). 5868 // Here we are completely ignoring the extract element index (N2), 5869 // which is fine for fixed width vectors, since any index other than 0 5870 // is undefined anyway. However, this cannot be ignored for scalable 5871 // vectors - in theory we could support this, but we don't want to do this 5872 // without a profitability check. 5873 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && 5874 N1.getValueType().isFixedLengthVector() && 5875 N1.getValueType().getVectorNumElements() == 1) { 5876 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), 5877 N1.getOperand(1)); 5878 } 5879 break; 5880 case ISD::EXTRACT_ELEMENT: 5881 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 5882 assert(!N1.getValueType().isVector() && !VT.isVector() && 5883 (N1.getValueType().isInteger() == VT.isInteger()) && 5884 N1.getValueType() != VT && 5885 "Wrong types for EXTRACT_ELEMENT!"); 5886 5887 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 5888 // 64-bit integers into 32-bit parts. Instead of building the extract of 5889 // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 5890 if (N1.getOpcode() == ISD::BUILD_PAIR) 5891 return N1.getOperand(N2C->getZExtValue()); 5892 5893 // EXTRACT_ELEMENT of a constant int is also very common. 5894 if (N1C) { 5895 unsigned ElementSize = VT.getSizeInBits(); 5896 unsigned Shift = ElementSize * N2C->getZExtValue(); 5897 const APInt &Val = N1C->getAPIntValue(); 5898 return getConstant(Val.extractBits(ElementSize, Shift), DL, VT); 5899 } 5900 break; 5901 case ISD::EXTRACT_SUBVECTOR: { 5902 EVT N1VT = N1.getValueType(); 5903 assert(VT.isVector() && N1VT.isVector() && 5904 "Extract subvector VTs must be vectors!"); 5905 assert(VT.getVectorElementType() == N1VT.getVectorElementType() && 5906 "Extract subvector VTs must have the same element type!"); 5907 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) && 5908 "Cannot extract a scalable vector from a fixed length vector!"); 5909 assert((VT.isScalableVector() != N1VT.isScalableVector() || 5910 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) && 5911 "Extract subvector must be from larger vector to smaller vector!"); 5912 assert(N2C && "Extract subvector index must be a constant"); 5913 assert((VT.isScalableVector() != N1VT.isScalableVector() || 5914 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <= 5915 N1VT.getVectorMinNumElements()) && 5916 "Extract subvector overflow!"); 5917 assert(N2C->getAPIntValue().getBitWidth() == 5918 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && 5919 "Constant index for EXTRACT_SUBVECTOR has an invalid size"); 5920 5921 // Trivial extraction. 5922 if (VT == N1VT) 5923 return N1; 5924 5925 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF. 5926 if (N1.isUndef()) 5927 return getUNDEF(VT); 5928 5929 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of 5930 // the concat have the same type as the extract. 5931 if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 && 5932 VT == N1.getOperand(0).getValueType()) { 5933 unsigned Factor = VT.getVectorMinNumElements(); 5934 return N1.getOperand(N2C->getZExtValue() / Factor); 5935 } 5936 5937 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created 5938 // during shuffle legalization. 5939 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) && 5940 VT == N1.getOperand(1).getValueType()) 5941 return N1.getOperand(1); 5942 break; 5943 } 5944 } 5945 5946 // Perform trivial constant folding. 5947 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2})) 5948 return SV; 5949 5950 if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2)) 5951 return V; 5952 5953 // Canonicalize an UNDEF to the RHS, even over a constant. 5954 if (N1.isUndef()) { 5955 if (TLI->isCommutativeBinOp(Opcode)) { 5956 std::swap(N1, N2); 5957 } else { 5958 switch (Opcode) { 5959 case ISD::SIGN_EXTEND_INREG: 5960 case ISD::SUB: 5961 return getUNDEF(VT); // fold op(undef, arg2) -> undef 5962 case ISD::UDIV: 5963 case ISD::SDIV: 5964 case ISD::UREM: 5965 case ISD::SREM: 5966 case ISD::SSUBSAT: 5967 case ISD::USUBSAT: 5968 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0 5969 } 5970 } 5971 } 5972 5973 // Fold a bunch of operators when the RHS is undef. 5974 if (N2.isUndef()) { 5975 switch (Opcode) { 5976 case ISD::XOR: 5977 if (N1.isUndef()) 5978 // Handle undef ^ undef -> 0 special case. This is a common 5979 // idiom (misuse). 5980 return getConstant(0, DL, VT); 5981 LLVM_FALLTHROUGH; 5982 case ISD::ADD: 5983 case ISD::SUB: 5984 case ISD::UDIV: 5985 case ISD::SDIV: 5986 case ISD::UREM: 5987 case ISD::SREM: 5988 return getUNDEF(VT); // fold op(arg1, undef) -> undef 5989 case ISD::MUL: 5990 case ISD::AND: 5991 case ISD::SSUBSAT: 5992 case ISD::USUBSAT: 5993 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0 5994 case ISD::OR: 5995 case ISD::SADDSAT: 5996 case ISD::UADDSAT: 5997 return getAllOnesConstant(DL, VT); 5998 } 5999 } 6000 6001 // Memoize this node if possible. 6002 SDNode *N; 6003 SDVTList VTs = getVTList(VT); 6004 SDValue Ops[] = {N1, N2}; 6005 if (VT != MVT::Glue) { 6006 FoldingSetNodeID ID; 6007 AddNodeIDNode(ID, Opcode, VTs, Ops); 6008 void *IP = nullptr; 6009 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 6010 E->intersectFlagsWith(Flags); 6011 return SDValue(E, 0); 6012 } 6013 6014 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 6015 N->setFlags(Flags); 6016 createOperands(N, Ops); 6017 CSEMap.InsertNode(N, IP); 6018 } else { 6019 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 6020 createOperands(N, Ops); 6021 } 6022 6023 InsertNode(N); 6024 SDValue V = SDValue(N, 0); 6025 NewSDValueDbgMsg(V, "Creating new node: ", this); 6026 return V; 6027 } 6028 6029 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 6030 SDValue N1, SDValue N2, SDValue N3) { 6031 SDNodeFlags Flags; 6032 if (Inserter) 6033 Flags = Inserter->getFlags(); 6034 return getNode(Opcode, DL, VT, N1, N2, N3, Flags); 6035 } 6036 6037 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 6038 SDValue N1, SDValue N2, SDValue N3, 6039 const SDNodeFlags Flags) { 6040 assert(N1.getOpcode() != ISD::DELETED_NODE && 6041 N2.getOpcode() != ISD::DELETED_NODE && 6042 N3.getOpcode() != ISD::DELETED_NODE && 6043 "Operand is DELETED_NODE!"); 6044 // Perform various simplifications. 6045 switch (Opcode) { 6046 case ISD::FMA: { 6047 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 6048 assert(N1.getValueType() == VT && N2.getValueType() == VT && 6049 N3.getValueType() == VT && "FMA types must match!"); 6050 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 6051 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 6052 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3); 6053 if (N1CFP && N2CFP && N3CFP) { 6054 APFloat V1 = N1CFP->getValueAPF(); 6055 const APFloat &V2 = N2CFP->getValueAPF(); 6056 const APFloat &V3 = N3CFP->getValueAPF(); 6057 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); 6058 return getConstantFP(V1, DL, VT); 6059 } 6060 break; 6061 } 6062 case ISD::BUILD_VECTOR: { 6063 // Attempt to simplify BUILD_VECTOR. 6064 SDValue Ops[] = {N1, N2, N3}; 6065 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) 6066 return V; 6067 break; 6068 } 6069 case ISD::CONCAT_VECTORS: { 6070 SDValue Ops[] = {N1, N2, N3}; 6071 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) 6072 return V; 6073 break; 6074 } 6075 case ISD::SETCC: { 6076 assert(VT.isInteger() && "SETCC result type must be an integer!"); 6077 assert(N1.getValueType() == N2.getValueType() && 6078 "SETCC operands must have the same type!"); 6079 assert(VT.isVector() == N1.getValueType().isVector() && 6080 "SETCC type should be vector iff the operand type is vector!"); 6081 assert((!VT.isVector() || VT.getVectorElementCount() == 6082 N1.getValueType().getVectorElementCount()) && 6083 "SETCC vector element counts must match!"); 6084 // Use FoldSetCC to simplify SETCC's. 6085 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL)) 6086 return V; 6087 // Vector constant folding. 6088 SDValue Ops[] = {N1, N2, N3}; 6089 if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) { 6090 NewSDValueDbgMsg(V, "New node vector constant folding: ", this); 6091 return V; 6092 } 6093 break; 6094 } 6095 case ISD::SELECT: 6096 case ISD::VSELECT: 6097 if (SDValue V = simplifySelect(N1, N2, N3)) 6098 return V; 6099 break; 6100 case ISD::VECTOR_SHUFFLE: 6101 llvm_unreachable("should use getVectorShuffle constructor!"); 6102 case ISD::INSERT_VECTOR_ELT: { 6103 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3); 6104 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except 6105 // for scalable vectors where we will generate appropriate code to 6106 // deal with out-of-bounds cases correctly. 6107 if (N3C && N1.getValueType().isFixedLengthVector() && 6108 N3C->getZExtValue() >= N1.getValueType().getVectorNumElements()) 6109 return getUNDEF(VT); 6110 6111 // Undefined index can be assumed out-of-bounds, so that's UNDEF too. 6112 if (N3.isUndef()) 6113 return getUNDEF(VT); 6114 6115 // If the inserted element is an UNDEF, just use the input vector. 6116 if (N2.isUndef()) 6117 return N1; 6118 6119 break; 6120 } 6121 case ISD::INSERT_SUBVECTOR: { 6122 // Inserting undef into undef is still undef. 6123 if (N1.isUndef() && N2.isUndef()) 6124 return getUNDEF(VT); 6125 6126 EVT N2VT = N2.getValueType(); 6127 assert(VT == N1.getValueType() && 6128 "Dest and insert subvector source types must match!"); 6129 assert(VT.isVector() && N2VT.isVector() && 6130 "Insert subvector VTs must be vectors!"); 6131 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) && 6132 "Cannot insert a scalable vector into a fixed length vector!"); 6133 assert((VT.isScalableVector() != N2VT.isScalableVector() || 6134 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) && 6135 "Insert subvector must be from smaller vector to larger vector!"); 6136 assert(isa<ConstantSDNode>(N3) && 6137 "Insert subvector index must be constant"); 6138 assert((VT.isScalableVector() != N2VT.isScalableVector() || 6139 (N2VT.getVectorMinNumElements() + 6140 cast<ConstantSDNode>(N3)->getZExtValue()) <= 6141 VT.getVectorMinNumElements()) && 6142 "Insert subvector overflow!"); 6143 assert(cast<ConstantSDNode>(N3)->getAPIntValue().getBitWidth() == 6144 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && 6145 "Constant index for INSERT_SUBVECTOR has an invalid size"); 6146 6147 // Trivial insertion. 6148 if (VT == N2VT) 6149 return N2; 6150 6151 // If this is an insert of an extracted vector into an undef vector, we 6152 // can just use the input to the extract. 6153 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR && 6154 N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT) 6155 return N2.getOperand(0); 6156 break; 6157 } 6158 case ISD::BITCAST: 6159 // Fold bit_convert nodes from a type to themselves. 6160 if (N1.getValueType() == VT) 6161 return N1; 6162 break; 6163 } 6164 6165 // Memoize node if it doesn't produce a flag. 6166 SDNode *N; 6167 SDVTList VTs = getVTList(VT); 6168 SDValue Ops[] = {N1, N2, N3}; 6169 if (VT != MVT::Glue) { 6170 FoldingSetNodeID ID; 6171 AddNodeIDNode(ID, Opcode, VTs, Ops); 6172 void *IP = nullptr; 6173 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 6174 E->intersectFlagsWith(Flags); 6175 return SDValue(E, 0); 6176 } 6177 6178 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 6179 N->setFlags(Flags); 6180 createOperands(N, Ops); 6181 CSEMap.InsertNode(N, IP); 6182 } else { 6183 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 6184 createOperands(N, Ops); 6185 } 6186 6187 InsertNode(N); 6188 SDValue V = SDValue(N, 0); 6189 NewSDValueDbgMsg(V, "Creating new node: ", this); 6190 return V; 6191 } 6192 6193 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 6194 SDValue N1, SDValue N2, SDValue N3, SDValue N4) { 6195 SDValue Ops[] = { N1, N2, N3, N4 }; 6196 return getNode(Opcode, DL, VT, Ops); 6197 } 6198 6199 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 6200 SDValue N1, SDValue N2, SDValue N3, SDValue N4, 6201 SDValue N5) { 6202 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 6203 return getNode(Opcode, DL, VT, Ops); 6204 } 6205 6206 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all 6207 /// the incoming stack arguments to be loaded from the stack. 6208 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 6209 SmallVector<SDValue, 8> ArgChains; 6210 6211 // Include the original chain at the beginning of the list. When this is 6212 // used by target LowerCall hooks, this helps legalize find the 6213 // CALLSEQ_BEGIN node. 6214 ArgChains.push_back(Chain); 6215 6216 // Add a chain value for each stack argument. 6217 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 6218 UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 6219 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 6220 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 6221 if (FI->getIndex() < 0) 6222 ArgChains.push_back(SDValue(L, 1)); 6223 6224 // Build a tokenfactor for all the chains. 6225 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); 6226 } 6227 6228 /// getMemsetValue - Vectorized representation of the memset value 6229 /// operand. 6230 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 6231 const SDLoc &dl) { 6232 assert(!Value.isUndef()); 6233 6234 unsigned NumBits = VT.getScalarSizeInBits(); 6235 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 6236 assert(C->getAPIntValue().getBitWidth() == 8); 6237 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); 6238 if (VT.isInteger()) { 6239 bool IsOpaque = VT.getSizeInBits() > 64 || 6240 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue()); 6241 return DAG.getConstant(Val, dl, VT, false, IsOpaque); 6242 } 6243 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl, 6244 VT); 6245 } 6246 6247 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?"); 6248 EVT IntVT = VT.getScalarType(); 6249 if (!IntVT.isInteger()) 6250 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits()); 6251 6252 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value); 6253 if (NumBits > 8) { 6254 // Use a multiplication with 0x010101... to extend the input to the 6255 // required length. 6256 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); 6257 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value, 6258 DAG.getConstant(Magic, dl, IntVT)); 6259 } 6260 6261 if (VT != Value.getValueType() && !VT.isInteger()) 6262 Value = DAG.getBitcast(VT.getScalarType(), Value); 6263 if (VT != Value.getValueType()) 6264 Value = DAG.getSplatBuildVector(VT, dl, Value); 6265 6266 return Value; 6267 } 6268 6269 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only 6270 /// used when a memcpy is turned into a memset when the source is a constant 6271 /// string ptr. 6272 static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, 6273 const TargetLowering &TLI, 6274 const ConstantDataArraySlice &Slice) { 6275 // Handle vector with all elements zero. 6276 if (Slice.Array == nullptr) { 6277 if (VT.isInteger()) 6278 return DAG.getConstant(0, dl, VT); 6279 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128) 6280 return DAG.getConstantFP(0.0, dl, VT); 6281 if (VT.isVector()) { 6282 unsigned NumElts = VT.getVectorNumElements(); 6283 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 6284 return DAG.getNode(ISD::BITCAST, dl, VT, 6285 DAG.getConstant(0, dl, 6286 EVT::getVectorVT(*DAG.getContext(), 6287 EltVT, NumElts))); 6288 } 6289 llvm_unreachable("Expected type!"); 6290 } 6291 6292 assert(!VT.isVector() && "Can't handle vector type here!"); 6293 unsigned NumVTBits = VT.getSizeInBits(); 6294 unsigned NumVTBytes = NumVTBits / 8; 6295 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length)); 6296 6297 APInt Val(NumVTBits, 0); 6298 if (DAG.getDataLayout().isLittleEndian()) { 6299 for (unsigned i = 0; i != NumBytes; ++i) 6300 Val |= (uint64_t)(unsigned char)Slice[i] << i*8; 6301 } else { 6302 for (unsigned i = 0; i != NumBytes; ++i) 6303 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8; 6304 } 6305 6306 // If the "cost" of materializing the integer immediate is less than the cost 6307 // of a load, then it is cost effective to turn the load into the immediate. 6308 Type *Ty = VT.getTypeForEVT(*DAG.getContext()); 6309 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty)) 6310 return DAG.getConstant(Val, dl, VT); 6311 return SDValue(nullptr, 0); 6312 } 6313 6314 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset, 6315 const SDLoc &DL, 6316 const SDNodeFlags Flags) { 6317 EVT VT = Base.getValueType(); 6318 SDValue Index; 6319 6320 if (Offset.isScalable()) 6321 Index = getVScale(DL, Base.getValueType(), 6322 APInt(Base.getValueSizeInBits().getFixedSize(), 6323 Offset.getKnownMinSize())); 6324 else 6325 Index = getConstant(Offset.getFixedSize(), DL, VT); 6326 6327 return getMemBasePlusOffset(Base, Index, DL, Flags); 6328 } 6329 6330 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset, 6331 const SDLoc &DL, 6332 const SDNodeFlags Flags) { 6333 assert(Offset.getValueType().isInteger()); 6334 EVT BasePtrVT = Ptr.getValueType(); 6335 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags); 6336 } 6337 6338 /// Returns true if memcpy source is constant data. 6339 static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) { 6340 uint64_t SrcDelta = 0; 6341 GlobalAddressSDNode *G = nullptr; 6342 if (Src.getOpcode() == ISD::GlobalAddress) 6343 G = cast<GlobalAddressSDNode>(Src); 6344 else if (Src.getOpcode() == ISD::ADD && 6345 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 6346 Src.getOperand(1).getOpcode() == ISD::Constant) { 6347 G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 6348 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 6349 } 6350 if (!G) 6351 return false; 6352 6353 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8, 6354 SrcDelta + G->getOffset()); 6355 } 6356 6357 static bool shouldLowerMemFuncForSize(const MachineFunction &MF, 6358 SelectionDAG &DAG) { 6359 // On Darwin, -Os means optimize for size without hurting performance, so 6360 // only really optimize for size when -Oz (MinSize) is used. 6361 if (MF.getTarget().getTargetTriple().isOSDarwin()) 6362 return MF.getFunction().hasMinSize(); 6363 return DAG.shouldOptForSize(); 6364 } 6365 6366 static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, 6367 SmallVector<SDValue, 32> &OutChains, unsigned From, 6368 unsigned To, SmallVector<SDValue, 16> &OutLoadChains, 6369 SmallVector<SDValue, 16> &OutStoreChains) { 6370 assert(OutLoadChains.size() && "Missing loads in memcpy inlining"); 6371 assert(OutStoreChains.size() && "Missing stores in memcpy inlining"); 6372 SmallVector<SDValue, 16> GluedLoadChains; 6373 for (unsigned i = From; i < To; ++i) { 6374 OutChains.push_back(OutLoadChains[i]); 6375 GluedLoadChains.push_back(OutLoadChains[i]); 6376 } 6377 6378 // Chain for all loads. 6379 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 6380 GluedLoadChains); 6381 6382 for (unsigned i = From; i < To; ++i) { 6383 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]); 6384 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(), 6385 ST->getBasePtr(), ST->getMemoryVT(), 6386 ST->getMemOperand()); 6387 OutChains.push_back(NewStore); 6388 } 6389 } 6390 6391 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, 6392 SDValue Chain, SDValue Dst, SDValue Src, 6393 uint64_t Size, Align Alignment, 6394 bool isVol, bool AlwaysInline, 6395 MachinePointerInfo DstPtrInfo, 6396 MachinePointerInfo SrcPtrInfo, 6397 const AAMDNodes &AAInfo) { 6398 // Turn a memcpy of undef to nop. 6399 // FIXME: We need to honor volatile even is Src is undef. 6400 if (Src.isUndef()) 6401 return Chain; 6402 6403 // Expand memcpy to a series of load and store ops if the size operand falls 6404 // below a certain threshold. 6405 // TODO: In the AlwaysInline case, if the size is big then generate a loop 6406 // rather than maybe a humongous number of loads and stores. 6407 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 6408 const DataLayout &DL = DAG.getDataLayout(); 6409 LLVMContext &C = *DAG.getContext(); 6410 std::vector<EVT> MemOps; 6411 bool DstAlignCanChange = false; 6412 MachineFunction &MF = DAG.getMachineFunction(); 6413 MachineFrameInfo &MFI = MF.getFrameInfo(); 6414 bool OptSize = shouldLowerMemFuncForSize(MF, DAG); 6415 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 6416 if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) 6417 DstAlignCanChange = true; 6418 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src); 6419 if (!SrcAlign || Alignment > *SrcAlign) 6420 SrcAlign = Alignment; 6421 assert(SrcAlign && "SrcAlign must be set"); 6422 ConstantDataArraySlice Slice; 6423 // If marked as volatile, perform a copy even when marked as constant. 6424 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice); 6425 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr; 6426 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 6427 const MemOp Op = isZeroConstant 6428 ? MemOp::Set(Size, DstAlignCanChange, Alignment, 6429 /*IsZeroMemset*/ true, isVol) 6430 : MemOp::Copy(Size, DstAlignCanChange, Alignment, 6431 *SrcAlign, isVol, CopyFromConstant); 6432 if (!TLI.findOptimalMemOpLowering( 6433 MemOps, Limit, Op, DstPtrInfo.getAddrSpace(), 6434 SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes())) 6435 return SDValue(); 6436 6437 if (DstAlignCanChange) { 6438 Type *Ty = MemOps[0].getTypeForEVT(C); 6439 Align NewAlign = DL.getABITypeAlign(Ty); 6440 6441 // Don't promote to an alignment that would require dynamic stack 6442 // realignment. 6443 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 6444 if (!TRI->hasStackRealignment(MF)) 6445 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign)) 6446 NewAlign = NewAlign / 2; 6447 6448 if (NewAlign > Alignment) { 6449 // Give the stack frame object a larger alignment if needed. 6450 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) 6451 MFI.setObjectAlignment(FI->getIndex(), NewAlign); 6452 Alignment = NewAlign; 6453 } 6454 } 6455 6456 // Prepare AAInfo for loads/stores after lowering this memcpy. 6457 AAMDNodes NewAAInfo = AAInfo; 6458 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; 6459 6460 MachineMemOperand::Flags MMOFlags = 6461 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; 6462 SmallVector<SDValue, 16> OutLoadChains; 6463 SmallVector<SDValue, 16> OutStoreChains; 6464 SmallVector<SDValue, 32> OutChains; 6465 unsigned NumMemOps = MemOps.size(); 6466 uint64_t SrcOff = 0, DstOff = 0; 6467 for (unsigned i = 0; i != NumMemOps; ++i) { 6468 EVT VT = MemOps[i]; 6469 unsigned VTSize = VT.getSizeInBits() / 8; 6470 SDValue Value, Store; 6471 6472 if (VTSize > Size) { 6473 // Issuing an unaligned load / store pair that overlaps with the previous 6474 // pair. Adjust the offset accordingly. 6475 assert(i == NumMemOps-1 && i != 0); 6476 SrcOff -= VTSize - Size; 6477 DstOff -= VTSize - Size; 6478 } 6479 6480 if (CopyFromConstant && 6481 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) { 6482 // It's unlikely a store of a vector immediate can be done in a single 6483 // instruction. It would require a load from a constantpool first. 6484 // We only handle zero vectors here. 6485 // FIXME: Handle other cases where store of vector immediate is done in 6486 // a single instruction. 6487 ConstantDataArraySlice SubSlice; 6488 if (SrcOff < Slice.Length) { 6489 SubSlice = Slice; 6490 SubSlice.move(SrcOff); 6491 } else { 6492 // This is an out-of-bounds access and hence UB. Pretend we read zero. 6493 SubSlice.Array = nullptr; 6494 SubSlice.Offset = 0; 6495 SubSlice.Length = VTSize; 6496 } 6497 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice); 6498 if (Value.getNode()) { 6499 Store = DAG.getStore( 6500 Chain, dl, Value, 6501 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), 6502 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo); 6503 OutChains.push_back(Store); 6504 } 6505 } 6506 6507 if (!Store.getNode()) { 6508 // The type might not be legal for the target. This should only happen 6509 // if the type is smaller than a legal type, as on PPC, so the right 6510 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 6511 // to Load/Store if NVT==VT. 6512 // FIXME does the case above also need this? 6513 EVT NVT = TLI.getTypeToTransformTo(C, VT); 6514 assert(NVT.bitsGE(VT)); 6515 6516 bool isDereferenceable = 6517 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); 6518 MachineMemOperand::Flags SrcMMOFlags = MMOFlags; 6519 if (isDereferenceable) 6520 SrcMMOFlags |= MachineMemOperand::MODereferenceable; 6521 6522 Value = DAG.getExtLoad( 6523 ISD::EXTLOAD, dl, NVT, Chain, 6524 DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), 6525 SrcPtrInfo.getWithOffset(SrcOff), VT, 6526 commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo); 6527 OutLoadChains.push_back(Value.getValue(1)); 6528 6529 Store = DAG.getTruncStore( 6530 Chain, dl, Value, 6531 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), 6532 DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo); 6533 OutStoreChains.push_back(Store); 6534 } 6535 SrcOff += VTSize; 6536 DstOff += VTSize; 6537 Size -= VTSize; 6538 } 6539 6540 unsigned GluedLdStLimit = MaxLdStGlue == 0 ? 6541 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue; 6542 unsigned NumLdStInMemcpy = OutStoreChains.size(); 6543 6544 if (NumLdStInMemcpy) { 6545 // It may be that memcpy might be converted to memset if it's memcpy 6546 // of constants. In such a case, we won't have loads and stores, but 6547 // just stores. In the absence of loads, there is nothing to gang up. 6548 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) { 6549 // If target does not care, just leave as it. 6550 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) { 6551 OutChains.push_back(OutLoadChains[i]); 6552 OutChains.push_back(OutStoreChains[i]); 6553 } 6554 } else { 6555 // Ld/St less than/equal limit set by target. 6556 if (NumLdStInMemcpy <= GluedLdStLimit) { 6557 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0, 6558 NumLdStInMemcpy, OutLoadChains, 6559 OutStoreChains); 6560 } else { 6561 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit; 6562 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit; 6563 unsigned GlueIter = 0; 6564 6565 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) { 6566 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit; 6567 unsigned IndexTo = NumLdStInMemcpy - GlueIter; 6568 6569 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo, 6570 OutLoadChains, OutStoreChains); 6571 GlueIter += GluedLdStLimit; 6572 } 6573 6574 // Residual ld/st. 6575 if (RemainingLdStInMemcpy) { 6576 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0, 6577 RemainingLdStInMemcpy, OutLoadChains, 6578 OutStoreChains); 6579 } 6580 } 6581 } 6582 } 6583 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 6584 } 6585 6586 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, 6587 SDValue Chain, SDValue Dst, SDValue Src, 6588 uint64_t Size, Align Alignment, 6589 bool isVol, bool AlwaysInline, 6590 MachinePointerInfo DstPtrInfo, 6591 MachinePointerInfo SrcPtrInfo, 6592 const AAMDNodes &AAInfo) { 6593 // Turn a memmove of undef to nop. 6594 // FIXME: We need to honor volatile even is Src is undef. 6595 if (Src.isUndef()) 6596 return Chain; 6597 6598 // Expand memmove to a series of load and store ops if the size operand falls 6599 // below a certain threshold. 6600 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 6601 const DataLayout &DL = DAG.getDataLayout(); 6602 LLVMContext &C = *DAG.getContext(); 6603 std::vector<EVT> MemOps; 6604 bool DstAlignCanChange = false; 6605 MachineFunction &MF = DAG.getMachineFunction(); 6606 MachineFrameInfo &MFI = MF.getFrameInfo(); 6607 bool OptSize = shouldLowerMemFuncForSize(MF, DAG); 6608 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 6609 if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) 6610 DstAlignCanChange = true; 6611 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src); 6612 if (!SrcAlign || Alignment > *SrcAlign) 6613 SrcAlign = Alignment; 6614 assert(SrcAlign && "SrcAlign must be set"); 6615 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 6616 if (!TLI.findOptimalMemOpLowering( 6617 MemOps, Limit, 6618 MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign, 6619 /*IsVolatile*/ true), 6620 DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), 6621 MF.getFunction().getAttributes())) 6622 return SDValue(); 6623 6624 if (DstAlignCanChange) { 6625 Type *Ty = MemOps[0].getTypeForEVT(C); 6626 Align NewAlign = DL.getABITypeAlign(Ty); 6627 if (NewAlign > Alignment) { 6628 // Give the stack frame object a larger alignment if needed. 6629 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) 6630 MFI.setObjectAlignment(FI->getIndex(), NewAlign); 6631 Alignment = NewAlign; 6632 } 6633 } 6634 6635 // Prepare AAInfo for loads/stores after lowering this memmove. 6636 AAMDNodes NewAAInfo = AAInfo; 6637 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; 6638 6639 MachineMemOperand::Flags MMOFlags = 6640 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; 6641 uint64_t SrcOff = 0, DstOff = 0; 6642 SmallVector<SDValue, 8> LoadValues; 6643 SmallVector<SDValue, 8> LoadChains; 6644 SmallVector<SDValue, 8> OutChains; 6645 unsigned NumMemOps = MemOps.size(); 6646 for (unsigned i = 0; i < NumMemOps; i++) { 6647 EVT VT = MemOps[i]; 6648 unsigned VTSize = VT.getSizeInBits() / 8; 6649 SDValue Value; 6650 6651 bool isDereferenceable = 6652 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); 6653 MachineMemOperand::Flags SrcMMOFlags = MMOFlags; 6654 if (isDereferenceable) 6655 SrcMMOFlags |= MachineMemOperand::MODereferenceable; 6656 6657 Value = DAG.getLoad( 6658 VT, dl, Chain, 6659 DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), 6660 SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo); 6661 LoadValues.push_back(Value); 6662 LoadChains.push_back(Value.getValue(1)); 6663 SrcOff += VTSize; 6664 } 6665 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 6666 OutChains.clear(); 6667 for (unsigned i = 0; i < NumMemOps; i++) { 6668 EVT VT = MemOps[i]; 6669 unsigned VTSize = VT.getSizeInBits() / 8; 6670 SDValue Store; 6671 6672 Store = DAG.getStore( 6673 Chain, dl, LoadValues[i], 6674 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), 6675 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo); 6676 OutChains.push_back(Store); 6677 DstOff += VTSize; 6678 } 6679 6680 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 6681 } 6682 6683 /// Lower the call to 'memset' intrinsic function into a series of store 6684 /// operations. 6685 /// 6686 /// \param DAG Selection DAG where lowered code is placed. 6687 /// \param dl Link to corresponding IR location. 6688 /// \param Chain Control flow dependency. 6689 /// \param Dst Pointer to destination memory location. 6690 /// \param Src Value of byte to write into the memory. 6691 /// \param Size Number of bytes to write. 6692 /// \param Alignment Alignment of the destination in bytes. 6693 /// \param isVol True if destination is volatile. 6694 /// \param DstPtrInfo IR information on the memory pointer. 6695 /// \returns New head in the control flow, if lowering was successful, empty 6696 /// SDValue otherwise. 6697 /// 6698 /// The function tries to replace 'llvm.memset' intrinsic with several store 6699 /// operations and value calculation code. This is usually profitable for small 6700 /// memory size. 6701 static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, 6702 SDValue Chain, SDValue Dst, SDValue Src, 6703 uint64_t Size, Align Alignment, bool isVol, 6704 MachinePointerInfo DstPtrInfo, 6705 const AAMDNodes &AAInfo) { 6706 // Turn a memset of undef to nop. 6707 // FIXME: We need to honor volatile even is Src is undef. 6708 if (Src.isUndef()) 6709 return Chain; 6710 6711 // Expand memset to a series of load/store ops if the size operand 6712 // falls below a certain threshold. 6713 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 6714 std::vector<EVT> MemOps; 6715 bool DstAlignCanChange = false; 6716 MachineFunction &MF = DAG.getMachineFunction(); 6717 MachineFrameInfo &MFI = MF.getFrameInfo(); 6718 bool OptSize = shouldLowerMemFuncForSize(MF, DAG); 6719 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 6720 if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) 6721 DstAlignCanChange = true; 6722 bool IsZeroVal = 6723 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 6724 if (!TLI.findOptimalMemOpLowering( 6725 MemOps, TLI.getMaxStoresPerMemset(OptSize), 6726 MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol), 6727 DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes())) 6728 return SDValue(); 6729 6730 if (DstAlignCanChange) { 6731 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 6732 Align NewAlign = DAG.getDataLayout().getABITypeAlign(Ty); 6733 if (NewAlign > Alignment) { 6734 // Give the stack frame object a larger alignment if needed. 6735 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) 6736 MFI.setObjectAlignment(FI->getIndex(), NewAlign); 6737 Alignment = NewAlign; 6738 } 6739 } 6740 6741 SmallVector<SDValue, 8> OutChains; 6742 uint64_t DstOff = 0; 6743 unsigned NumMemOps = MemOps.size(); 6744 6745 // Find the largest store and generate the bit pattern for it. 6746 EVT LargestVT = MemOps[0]; 6747 for (unsigned i = 1; i < NumMemOps; i++) 6748 if (MemOps[i].bitsGT(LargestVT)) 6749 LargestVT = MemOps[i]; 6750 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 6751 6752 // Prepare AAInfo for loads/stores after lowering this memset. 6753 AAMDNodes NewAAInfo = AAInfo; 6754 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; 6755 6756 for (unsigned i = 0; i < NumMemOps; i++) { 6757 EVT VT = MemOps[i]; 6758 unsigned VTSize = VT.getSizeInBits() / 8; 6759 if (VTSize > Size) { 6760 // Issuing an unaligned load / store pair that overlaps with the previous 6761 // pair. Adjust the offset accordingly. 6762 assert(i == NumMemOps-1 && i != 0); 6763 DstOff -= VTSize - Size; 6764 } 6765 6766 // If this store is smaller than the largest store see whether we can get 6767 // the smaller value for free with a truncate. 6768 SDValue Value = MemSetValue; 6769 if (VT.bitsLT(LargestVT)) { 6770 if (!LargestVT.isVector() && !VT.isVector() && 6771 TLI.isTruncateFree(LargestVT, VT)) 6772 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 6773 else 6774 Value = getMemsetValue(Src, VT, DAG, dl); 6775 } 6776 assert(Value.getValueType() == VT && "Value with wrong type."); 6777 SDValue Store = DAG.getStore( 6778 Chain, dl, Value, 6779 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), 6780 DstPtrInfo.getWithOffset(DstOff), Alignment, 6781 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone, 6782 NewAAInfo); 6783 OutChains.push_back(Store); 6784 DstOff += VT.getSizeInBits() / 8; 6785 Size -= VTSize; 6786 } 6787 6788 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 6789 } 6790 6791 static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, 6792 unsigned AS) { 6793 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all 6794 // pointer operands can be losslessly bitcasted to pointers of address space 0 6795 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) { 6796 report_fatal_error("cannot lower memory intrinsic in address space " + 6797 Twine(AS)); 6798 } 6799 } 6800 6801 SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, 6802 SDValue Src, SDValue Size, Align Alignment, 6803 bool isVol, bool AlwaysInline, bool isTailCall, 6804 MachinePointerInfo DstPtrInfo, 6805 MachinePointerInfo SrcPtrInfo, 6806 const AAMDNodes &AAInfo) { 6807 // Check to see if we should lower the memcpy to loads and stores first. 6808 // For cases within the target-specified limits, this is the best choice. 6809 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 6810 if (ConstantSize) { 6811 // Memcpy with size zero? Just return the original chain. 6812 if (ConstantSize->isNullValue()) 6813 return Chain; 6814 6815 SDValue Result = getMemcpyLoadsAndStores( 6816 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, 6817 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo); 6818 if (Result.getNode()) 6819 return Result; 6820 } 6821 6822 // Then check to see if we should lower the memcpy with target-specific 6823 // code. If the target chooses to do this, this is the next best. 6824 if (TSI) { 6825 SDValue Result = TSI->EmitTargetCodeForMemcpy( 6826 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, 6827 DstPtrInfo, SrcPtrInfo); 6828 if (Result.getNode()) 6829 return Result; 6830 } 6831 6832 // If we really need inline code and the target declined to provide it, 6833 // use a (potentially long) sequence of loads and stores. 6834 if (AlwaysInline) { 6835 assert(ConstantSize && "AlwaysInline requires a constant size!"); 6836 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 6837 ConstantSize->getZExtValue(), Alignment, 6838 isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo); 6839 } 6840 6841 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); 6842 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); 6843 6844 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 6845 // memcpy is not guaranteed to be safe. libc memcpys aren't required to 6846 // respect volatile, so they may do things like read or write memory 6847 // beyond the given memory regions. But fixing this isn't easy, and most 6848 // people don't care. 6849 6850 // Emit a library call. 6851 TargetLowering::ArgListTy Args; 6852 TargetLowering::ArgListEntry Entry; 6853 Entry.Ty = Type::getInt8PtrTy(*getContext()); 6854 Entry.Node = Dst; Args.push_back(Entry); 6855 Entry.Node = Src; Args.push_back(Entry); 6856 6857 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 6858 Entry.Node = Size; Args.push_back(Entry); 6859 // FIXME: pass in SDLoc 6860 TargetLowering::CallLoweringInfo CLI(*this); 6861 CLI.setDebugLoc(dl) 6862 .setChain(Chain) 6863 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY), 6864 Dst.getValueType().getTypeForEVT(*getContext()), 6865 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), 6866 TLI->getPointerTy(getDataLayout())), 6867 std::move(Args)) 6868 .setDiscardResult() 6869 .setTailCall(isTailCall); 6870 6871 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 6872 return CallResult.second; 6873 } 6874 6875 SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl, 6876 SDValue Dst, unsigned DstAlign, 6877 SDValue Src, unsigned SrcAlign, 6878 SDValue Size, Type *SizeTy, 6879 unsigned ElemSz, bool isTailCall, 6880 MachinePointerInfo DstPtrInfo, 6881 MachinePointerInfo SrcPtrInfo) { 6882 // Emit a library call. 6883 TargetLowering::ArgListTy Args; 6884 TargetLowering::ArgListEntry Entry; 6885 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 6886 Entry.Node = Dst; 6887 Args.push_back(Entry); 6888 6889 Entry.Node = Src; 6890 Args.push_back(Entry); 6891 6892 Entry.Ty = SizeTy; 6893 Entry.Node = Size; 6894 Args.push_back(Entry); 6895 6896 RTLIB::Libcall LibraryCall = 6897 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz); 6898 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) 6899 report_fatal_error("Unsupported element size"); 6900 6901 TargetLowering::CallLoweringInfo CLI(*this); 6902 CLI.setDebugLoc(dl) 6903 .setChain(Chain) 6904 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall), 6905 Type::getVoidTy(*getContext()), 6906 getExternalSymbol(TLI->getLibcallName(LibraryCall), 6907 TLI->getPointerTy(getDataLayout())), 6908 std::move(Args)) 6909 .setDiscardResult() 6910 .setTailCall(isTailCall); 6911 6912 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); 6913 return CallResult.second; 6914 } 6915 6916 SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, 6917 SDValue Src, SDValue Size, Align Alignment, 6918 bool isVol, bool isTailCall, 6919 MachinePointerInfo DstPtrInfo, 6920 MachinePointerInfo SrcPtrInfo, 6921 const AAMDNodes &AAInfo) { 6922 // Check to see if we should lower the memmove to loads and stores first. 6923 // For cases within the target-specified limits, this is the best choice. 6924 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 6925 if (ConstantSize) { 6926 // Memmove with size zero? Just return the original chain. 6927 if (ConstantSize->isNullValue()) 6928 return Chain; 6929 6930 SDValue Result = getMemmoveLoadsAndStores( 6931 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, 6932 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo); 6933 if (Result.getNode()) 6934 return Result; 6935 } 6936 6937 // Then check to see if we should lower the memmove with target-specific 6938 // code. If the target chooses to do this, this is the next best. 6939 if (TSI) { 6940 SDValue Result = 6941 TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, 6942 Alignment, isVol, DstPtrInfo, SrcPtrInfo); 6943 if (Result.getNode()) 6944 return Result; 6945 } 6946 6947 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); 6948 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); 6949 6950 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 6951 // not be safe. See memcpy above for more details. 6952 6953 // Emit a library call. 6954 TargetLowering::ArgListTy Args; 6955 TargetLowering::ArgListEntry Entry; 6956 Entry.Ty = Type::getInt8PtrTy(*getContext()); 6957 Entry.Node = Dst; Args.push_back(Entry); 6958 Entry.Node = Src; Args.push_back(Entry); 6959 6960 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 6961 Entry.Node = Size; Args.push_back(Entry); 6962 // FIXME: pass in SDLoc 6963 TargetLowering::CallLoweringInfo CLI(*this); 6964 CLI.setDebugLoc(dl) 6965 .setChain(Chain) 6966 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE), 6967 Dst.getValueType().getTypeForEVT(*getContext()), 6968 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), 6969 TLI->getPointerTy(getDataLayout())), 6970 std::move(Args)) 6971 .setDiscardResult() 6972 .setTailCall(isTailCall); 6973 6974 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 6975 return CallResult.second; 6976 } 6977 6978 SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl, 6979 SDValue Dst, unsigned DstAlign, 6980 SDValue Src, unsigned SrcAlign, 6981 SDValue Size, Type *SizeTy, 6982 unsigned ElemSz, bool isTailCall, 6983 MachinePointerInfo DstPtrInfo, 6984 MachinePointerInfo SrcPtrInfo) { 6985 // Emit a library call. 6986 TargetLowering::ArgListTy Args; 6987 TargetLowering::ArgListEntry Entry; 6988 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 6989 Entry.Node = Dst; 6990 Args.push_back(Entry); 6991 6992 Entry.Node = Src; 6993 Args.push_back(Entry); 6994 6995 Entry.Ty = SizeTy; 6996 Entry.Node = Size; 6997 Args.push_back(Entry); 6998 6999 RTLIB::Libcall LibraryCall = 7000 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz); 7001 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) 7002 report_fatal_error("Unsupported element size"); 7003 7004 TargetLowering::CallLoweringInfo CLI(*this); 7005 CLI.setDebugLoc(dl) 7006 .setChain(Chain) 7007 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall), 7008 Type::getVoidTy(*getContext()), 7009 getExternalSymbol(TLI->getLibcallName(LibraryCall), 7010 TLI->getPointerTy(getDataLayout())), 7011 std::move(Args)) 7012 .setDiscardResult() 7013 .setTailCall(isTailCall); 7014 7015 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); 7016 return CallResult.second; 7017 } 7018 7019 SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, 7020 SDValue Src, SDValue Size, Align Alignment, 7021 bool isVol, bool isTailCall, 7022 MachinePointerInfo DstPtrInfo, 7023 const AAMDNodes &AAInfo) { 7024 // Check to see if we should lower the memset to stores first. 7025 // For cases within the target-specified limits, this is the best choice. 7026 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 7027 if (ConstantSize) { 7028 // Memset with size zero? Just return the original chain. 7029 if (ConstantSize->isNullValue()) 7030 return Chain; 7031 7032 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src, 7033 ConstantSize->getZExtValue(), Alignment, 7034 isVol, DstPtrInfo, AAInfo); 7035 7036 if (Result.getNode()) 7037 return Result; 7038 } 7039 7040 // Then check to see if we should lower the memset with target-specific 7041 // code. If the target chooses to do this, this is the next best. 7042 if (TSI) { 7043 SDValue Result = TSI->EmitTargetCodeForMemset( 7044 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, DstPtrInfo); 7045 if (Result.getNode()) 7046 return Result; 7047 } 7048 7049 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); 7050 7051 // Emit a library call. 7052 TargetLowering::ArgListTy Args; 7053 TargetLowering::ArgListEntry Entry; 7054 Entry.Node = Dst; Entry.Ty = Type::getInt8PtrTy(*getContext()); 7055 Args.push_back(Entry); 7056 Entry.Node = Src; 7057 Entry.Ty = Src.getValueType().getTypeForEVT(*getContext()); 7058 Args.push_back(Entry); 7059 Entry.Node = Size; 7060 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 7061 Args.push_back(Entry); 7062 7063 // FIXME: pass in SDLoc 7064 TargetLowering::CallLoweringInfo CLI(*this); 7065 CLI.setDebugLoc(dl) 7066 .setChain(Chain) 7067 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET), 7068 Dst.getValueType().getTypeForEVT(*getContext()), 7069 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), 7070 TLI->getPointerTy(getDataLayout())), 7071 std::move(Args)) 7072 .setDiscardResult() 7073 .setTailCall(isTailCall); 7074 7075 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 7076 return CallResult.second; 7077 } 7078 7079 SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl, 7080 SDValue Dst, unsigned DstAlign, 7081 SDValue Value, SDValue Size, Type *SizeTy, 7082 unsigned ElemSz, bool isTailCall, 7083 MachinePointerInfo DstPtrInfo) { 7084 // Emit a library call. 7085 TargetLowering::ArgListTy Args; 7086 TargetLowering::ArgListEntry Entry; 7087 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 7088 Entry.Node = Dst; 7089 Args.push_back(Entry); 7090 7091 Entry.Ty = Type::getInt8Ty(*getContext()); 7092 Entry.Node = Value; 7093 Args.push_back(Entry); 7094 7095 Entry.Ty = SizeTy; 7096 Entry.Node = Size; 7097 Args.push_back(Entry); 7098 7099 RTLIB::Libcall LibraryCall = 7100 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz); 7101 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) 7102 report_fatal_error("Unsupported element size"); 7103 7104 TargetLowering::CallLoweringInfo CLI(*this); 7105 CLI.setDebugLoc(dl) 7106 .setChain(Chain) 7107 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall), 7108 Type::getVoidTy(*getContext()), 7109 getExternalSymbol(TLI->getLibcallName(LibraryCall), 7110 TLI->getPointerTy(getDataLayout())), 7111 std::move(Args)) 7112 .setDiscardResult() 7113 .setTailCall(isTailCall); 7114 7115 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); 7116 return CallResult.second; 7117 } 7118 7119 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 7120 SDVTList VTList, ArrayRef<SDValue> Ops, 7121 MachineMemOperand *MMO) { 7122 FoldingSetNodeID ID; 7123 ID.AddInteger(MemVT.getRawBits()); 7124 AddNodeIDNode(ID, Opcode, VTList, Ops); 7125 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7126 void* IP = nullptr; 7127 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7128 cast<AtomicSDNode>(E)->refineAlignment(MMO); 7129 return SDValue(E, 0); 7130 } 7131 7132 auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), 7133 VTList, MemVT, MMO); 7134 createOperands(N, Ops); 7135 7136 CSEMap.InsertNode(N, IP); 7137 InsertNode(N); 7138 return SDValue(N, 0); 7139 } 7140 7141 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, 7142 EVT MemVT, SDVTList VTs, SDValue Chain, 7143 SDValue Ptr, SDValue Cmp, SDValue Swp, 7144 MachineMemOperand *MMO) { 7145 assert(Opcode == ISD::ATOMIC_CMP_SWAP || 7146 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 7147 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 7148 7149 SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 7150 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); 7151 } 7152 7153 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 7154 SDValue Chain, SDValue Ptr, SDValue Val, 7155 MachineMemOperand *MMO) { 7156 assert((Opcode == ISD::ATOMIC_LOAD_ADD || 7157 Opcode == ISD::ATOMIC_LOAD_SUB || 7158 Opcode == ISD::ATOMIC_LOAD_AND || 7159 Opcode == ISD::ATOMIC_LOAD_CLR || 7160 Opcode == ISD::ATOMIC_LOAD_OR || 7161 Opcode == ISD::ATOMIC_LOAD_XOR || 7162 Opcode == ISD::ATOMIC_LOAD_NAND || 7163 Opcode == ISD::ATOMIC_LOAD_MIN || 7164 Opcode == ISD::ATOMIC_LOAD_MAX || 7165 Opcode == ISD::ATOMIC_LOAD_UMIN || 7166 Opcode == ISD::ATOMIC_LOAD_UMAX || 7167 Opcode == ISD::ATOMIC_LOAD_FADD || 7168 Opcode == ISD::ATOMIC_LOAD_FSUB || 7169 Opcode == ISD::ATOMIC_SWAP || 7170 Opcode == ISD::ATOMIC_STORE) && 7171 "Invalid Atomic Op"); 7172 7173 EVT VT = Val.getValueType(); 7174 7175 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 7176 getVTList(VT, MVT::Other); 7177 SDValue Ops[] = {Chain, Ptr, Val}; 7178 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); 7179 } 7180 7181 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 7182 EVT VT, SDValue Chain, SDValue Ptr, 7183 MachineMemOperand *MMO) { 7184 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 7185 7186 SDVTList VTs = getVTList(VT, MVT::Other); 7187 SDValue Ops[] = {Chain, Ptr}; 7188 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); 7189 } 7190 7191 /// getMergeValues - Create a MERGE_VALUES node from the given operands. 7192 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) { 7193 if (Ops.size() == 1) 7194 return Ops[0]; 7195 7196 SmallVector<EVT, 4> VTs; 7197 VTs.reserve(Ops.size()); 7198 for (const SDValue &Op : Ops) 7199 VTs.push_back(Op.getValueType()); 7200 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops); 7201 } 7202 7203 SDValue SelectionDAG::getMemIntrinsicNode( 7204 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, 7205 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, 7206 MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) { 7207 if (!Size && MemVT.isScalableVector()) 7208 Size = MemoryLocation::UnknownSize; 7209 else if (!Size) 7210 Size = MemVT.getStoreSize(); 7211 7212 MachineFunction &MF = getMachineFunction(); 7213 MachineMemOperand *MMO = 7214 MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo); 7215 7216 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO); 7217 } 7218 7219 SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, 7220 SDVTList VTList, 7221 ArrayRef<SDValue> Ops, EVT MemVT, 7222 MachineMemOperand *MMO) { 7223 assert((Opcode == ISD::INTRINSIC_VOID || 7224 Opcode == ISD::INTRINSIC_W_CHAIN || 7225 Opcode == ISD::PREFETCH || 7226 ((int)Opcode <= std::numeric_limits<int>::max() && 7227 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 7228 "Opcode is not a memory-accessing opcode!"); 7229 7230 // Memoize the node unless it returns a flag. 7231 MemIntrinsicSDNode *N; 7232 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 7233 FoldingSetNodeID ID; 7234 AddNodeIDNode(ID, Opcode, VTList, Ops); 7235 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>( 7236 Opcode, dl.getIROrder(), VTList, MemVT, MMO)); 7237 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7238 void *IP = nullptr; 7239 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7240 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 7241 return SDValue(E, 0); 7242 } 7243 7244 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), 7245 VTList, MemVT, MMO); 7246 createOperands(N, Ops); 7247 7248 CSEMap.InsertNode(N, IP); 7249 } else { 7250 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), 7251 VTList, MemVT, MMO); 7252 createOperands(N, Ops); 7253 } 7254 InsertNode(N); 7255 SDValue V(N, 0); 7256 NewSDValueDbgMsg(V, "Creating new node: ", this); 7257 return V; 7258 } 7259 7260 SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, 7261 SDValue Chain, int FrameIndex, 7262 int64_t Size, int64_t Offset) { 7263 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END; 7264 const auto VTs = getVTList(MVT::Other); 7265 SDValue Ops[2] = { 7266 Chain, 7267 getFrameIndex(FrameIndex, 7268 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()), 7269 true)}; 7270 7271 FoldingSetNodeID ID; 7272 AddNodeIDNode(ID, Opcode, VTs, Ops); 7273 ID.AddInteger(FrameIndex); 7274 ID.AddInteger(Size); 7275 ID.AddInteger(Offset); 7276 void *IP = nullptr; 7277 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 7278 return SDValue(E, 0); 7279 7280 LifetimeSDNode *N = newSDNode<LifetimeSDNode>( 7281 Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset); 7282 createOperands(N, Ops); 7283 CSEMap.InsertNode(N, IP); 7284 InsertNode(N); 7285 SDValue V(N, 0); 7286 NewSDValueDbgMsg(V, "Creating new node: ", this); 7287 return V; 7288 } 7289 7290 SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, 7291 uint64_t Guid, uint64_t Index, 7292 uint32_t Attr) { 7293 const unsigned Opcode = ISD::PSEUDO_PROBE; 7294 const auto VTs = getVTList(MVT::Other); 7295 SDValue Ops[] = {Chain}; 7296 FoldingSetNodeID ID; 7297 AddNodeIDNode(ID, Opcode, VTs, Ops); 7298 ID.AddInteger(Guid); 7299 ID.AddInteger(Index); 7300 void *IP = nullptr; 7301 if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP)) 7302 return SDValue(E, 0); 7303 7304 auto *N = newSDNode<PseudoProbeSDNode>( 7305 Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr); 7306 createOperands(N, Ops); 7307 CSEMap.InsertNode(N, IP); 7308 InsertNode(N); 7309 SDValue V(N, 0); 7310 NewSDValueDbgMsg(V, "Creating new node: ", this); 7311 return V; 7312 } 7313 7314 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 7315 /// MachinePointerInfo record from it. This is particularly useful because the 7316 /// code generator has many cases where it doesn't bother passing in a 7317 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 7318 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, 7319 SelectionDAG &DAG, SDValue Ptr, 7320 int64_t Offset = 0) { 7321 // If this is FI+Offset, we can model it. 7322 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 7323 return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), 7324 FI->getIndex(), Offset); 7325 7326 // If this is (FI+Offset1)+Offset2, we can model it. 7327 if (Ptr.getOpcode() != ISD::ADD || 7328 !isa<ConstantSDNode>(Ptr.getOperand(1)) || 7329 !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 7330 return Info; 7331 7332 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 7333 return MachinePointerInfo::getFixedStack( 7334 DAG.getMachineFunction(), FI, 7335 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 7336 } 7337 7338 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 7339 /// MachinePointerInfo record from it. This is particularly useful because the 7340 /// code generator has many cases where it doesn't bother passing in a 7341 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 7342 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, 7343 SelectionDAG &DAG, SDValue Ptr, 7344 SDValue OffsetOp) { 7345 // If the 'Offset' value isn't a constant, we can't handle this. 7346 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 7347 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue()); 7348 if (OffsetOp.isUndef()) 7349 return InferPointerInfo(Info, DAG, Ptr); 7350 return Info; 7351 } 7352 7353 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 7354 EVT VT, const SDLoc &dl, SDValue Chain, 7355 SDValue Ptr, SDValue Offset, 7356 MachinePointerInfo PtrInfo, EVT MemVT, 7357 Align Alignment, 7358 MachineMemOperand::Flags MMOFlags, 7359 const AAMDNodes &AAInfo, const MDNode *Ranges) { 7360 assert(Chain.getValueType() == MVT::Other && 7361 "Invalid chain type"); 7362 7363 MMOFlags |= MachineMemOperand::MOLoad; 7364 assert((MMOFlags & MachineMemOperand::MOStore) == 0); 7365 // If we don't have a PtrInfo, infer the trivial frame index case to simplify 7366 // clients. 7367 if (PtrInfo.V.isNull()) 7368 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); 7369 7370 uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); 7371 MachineFunction &MF = getMachineFunction(); 7372 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, 7373 Alignment, AAInfo, Ranges); 7374 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 7375 } 7376 7377 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 7378 EVT VT, const SDLoc &dl, SDValue Chain, 7379 SDValue Ptr, SDValue Offset, EVT MemVT, 7380 MachineMemOperand *MMO) { 7381 if (VT == MemVT) { 7382 ExtType = ISD::NON_EXTLOAD; 7383 } else if (ExtType == ISD::NON_EXTLOAD) { 7384 assert(VT == MemVT && "Non-extending load from different memory type!"); 7385 } else { 7386 // Extending load. 7387 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 7388 "Should only be an extending load, not truncating!"); 7389 assert(VT.isInteger() == MemVT.isInteger() && 7390 "Cannot convert from FP to Int or Int -> FP!"); 7391 assert(VT.isVector() == MemVT.isVector() && 7392 "Cannot use an ext load to convert to or from a vector!"); 7393 assert((!VT.isVector() || 7394 VT.getVectorElementCount() == MemVT.getVectorElementCount()) && 7395 "Cannot use an ext load to change the number of vector elements!"); 7396 } 7397 7398 bool Indexed = AM != ISD::UNINDEXED; 7399 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!"); 7400 7401 SDVTList VTs = Indexed ? 7402 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 7403 SDValue Ops[] = { Chain, Ptr, Offset }; 7404 FoldingSetNodeID ID; 7405 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); 7406 ID.AddInteger(MemVT.getRawBits()); 7407 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>( 7408 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO)); 7409 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7410 void *IP = nullptr; 7411 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7412 cast<LoadSDNode>(E)->refineAlignment(MMO); 7413 return SDValue(E, 0); 7414 } 7415 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, 7416 ExtType, MemVT, MMO); 7417 createOperands(N, Ops); 7418 7419 CSEMap.InsertNode(N, IP); 7420 InsertNode(N); 7421 SDValue V(N, 0); 7422 NewSDValueDbgMsg(V, "Creating new node: ", this); 7423 return V; 7424 } 7425 7426 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, 7427 SDValue Ptr, MachinePointerInfo PtrInfo, 7428 MaybeAlign Alignment, 7429 MachineMemOperand::Flags MMOFlags, 7430 const AAMDNodes &AAInfo, const MDNode *Ranges) { 7431 SDValue Undef = getUNDEF(Ptr.getValueType()); 7432 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 7433 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges); 7434 } 7435 7436 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, 7437 SDValue Ptr, MachineMemOperand *MMO) { 7438 SDValue Undef = getUNDEF(Ptr.getValueType()); 7439 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 7440 VT, MMO); 7441 } 7442 7443 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, 7444 EVT VT, SDValue Chain, SDValue Ptr, 7445 MachinePointerInfo PtrInfo, EVT MemVT, 7446 MaybeAlign Alignment, 7447 MachineMemOperand::Flags MMOFlags, 7448 const AAMDNodes &AAInfo) { 7449 SDValue Undef = getUNDEF(Ptr.getValueType()); 7450 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo, 7451 MemVT, Alignment, MMOFlags, AAInfo); 7452 } 7453 7454 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, 7455 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT, 7456 MachineMemOperand *MMO) { 7457 SDValue Undef = getUNDEF(Ptr.getValueType()); 7458 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 7459 MemVT, MMO); 7460 } 7461 7462 SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, 7463 SDValue Base, SDValue Offset, 7464 ISD::MemIndexedMode AM) { 7465 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 7466 assert(LD->getOffset().isUndef() && "Load is already a indexed load!"); 7467 // Don't propagate the invariant or dereferenceable flags. 7468 auto MMOFlags = 7469 LD->getMemOperand()->getFlags() & 7470 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 7471 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 7472 LD->getChain(), Base, Offset, LD->getPointerInfo(), 7473 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo()); 7474 } 7475 7476 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, 7477 SDValue Ptr, MachinePointerInfo PtrInfo, 7478 Align Alignment, 7479 MachineMemOperand::Flags MMOFlags, 7480 const AAMDNodes &AAInfo) { 7481 assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); 7482 7483 MMOFlags |= MachineMemOperand::MOStore; 7484 assert((MMOFlags & MachineMemOperand::MOLoad) == 0); 7485 7486 if (PtrInfo.V.isNull()) 7487 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); 7488 7489 MachineFunction &MF = getMachineFunction(); 7490 uint64_t Size = 7491 MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize()); 7492 MachineMemOperand *MMO = 7493 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo); 7494 return getStore(Chain, dl, Val, Ptr, MMO); 7495 } 7496 7497 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, 7498 SDValue Ptr, MachineMemOperand *MMO) { 7499 assert(Chain.getValueType() == MVT::Other && 7500 "Invalid chain type"); 7501 EVT VT = Val.getValueType(); 7502 SDVTList VTs = getVTList(MVT::Other); 7503 SDValue Undef = getUNDEF(Ptr.getValueType()); 7504 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 7505 FoldingSetNodeID ID; 7506 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); 7507 ID.AddInteger(VT.getRawBits()); 7508 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>( 7509 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO)); 7510 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7511 void *IP = nullptr; 7512 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7513 cast<StoreSDNode>(E)->refineAlignment(MMO); 7514 return SDValue(E, 0); 7515 } 7516 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, 7517 ISD::UNINDEXED, false, VT, MMO); 7518 createOperands(N, Ops); 7519 7520 CSEMap.InsertNode(N, IP); 7521 InsertNode(N); 7522 SDValue V(N, 0); 7523 NewSDValueDbgMsg(V, "Creating new node: ", this); 7524 return V; 7525 } 7526 7527 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, 7528 SDValue Ptr, MachinePointerInfo PtrInfo, 7529 EVT SVT, Align Alignment, 7530 MachineMemOperand::Flags MMOFlags, 7531 const AAMDNodes &AAInfo) { 7532 assert(Chain.getValueType() == MVT::Other && 7533 "Invalid chain type"); 7534 7535 MMOFlags |= MachineMemOperand::MOStore; 7536 assert((MMOFlags & MachineMemOperand::MOLoad) == 0); 7537 7538 if (PtrInfo.V.isNull()) 7539 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); 7540 7541 MachineFunction &MF = getMachineFunction(); 7542 MachineMemOperand *MMO = MF.getMachineMemOperand( 7543 PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), 7544 Alignment, AAInfo); 7545 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 7546 } 7547 7548 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, 7549 SDValue Ptr, EVT SVT, 7550 MachineMemOperand *MMO) { 7551 EVT VT = Val.getValueType(); 7552 7553 assert(Chain.getValueType() == MVT::Other && 7554 "Invalid chain type"); 7555 if (VT == SVT) 7556 return getStore(Chain, dl, Val, Ptr, MMO); 7557 7558 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 7559 "Should only be a truncating store, not extending!"); 7560 assert(VT.isInteger() == SVT.isInteger() && 7561 "Can't do FP-INT conversion!"); 7562 assert(VT.isVector() == SVT.isVector() && 7563 "Cannot use trunc store to convert to or from a vector!"); 7564 assert((!VT.isVector() || 7565 VT.getVectorElementCount() == SVT.getVectorElementCount()) && 7566 "Cannot use trunc store to change the number of vector elements!"); 7567 7568 SDVTList VTs = getVTList(MVT::Other); 7569 SDValue Undef = getUNDEF(Ptr.getValueType()); 7570 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 7571 FoldingSetNodeID ID; 7572 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); 7573 ID.AddInteger(SVT.getRawBits()); 7574 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>( 7575 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO)); 7576 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7577 void *IP = nullptr; 7578 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7579 cast<StoreSDNode>(E)->refineAlignment(MMO); 7580 return SDValue(E, 0); 7581 } 7582 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, 7583 ISD::UNINDEXED, true, SVT, MMO); 7584 createOperands(N, Ops); 7585 7586 CSEMap.InsertNode(N, IP); 7587 InsertNode(N); 7588 SDValue V(N, 0); 7589 NewSDValueDbgMsg(V, "Creating new node: ", this); 7590 return V; 7591 } 7592 7593 SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl, 7594 SDValue Base, SDValue Offset, 7595 ISD::MemIndexedMode AM) { 7596 StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 7597 assert(ST->getOffset().isUndef() && "Store is already a indexed store!"); 7598 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 7599 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 7600 FoldingSetNodeID ID; 7601 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); 7602 ID.AddInteger(ST->getMemoryVT().getRawBits()); 7603 ID.AddInteger(ST->getRawSubclassData()); 7604 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 7605 void *IP = nullptr; 7606 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 7607 return SDValue(E, 0); 7608 7609 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, 7610 ST->isTruncatingStore(), ST->getMemoryVT(), 7611 ST->getMemOperand()); 7612 createOperands(N, Ops); 7613 7614 CSEMap.InsertNode(N, IP); 7615 InsertNode(N); 7616 SDValue V(N, 0); 7617 NewSDValueDbgMsg(V, "Creating new node: ", this); 7618 return V; 7619 } 7620 7621 SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, 7622 SDValue Base, SDValue Offset, SDValue Mask, 7623 SDValue PassThru, EVT MemVT, 7624 MachineMemOperand *MMO, 7625 ISD::MemIndexedMode AM, 7626 ISD::LoadExtType ExtTy, bool isExpanding) { 7627 bool Indexed = AM != ISD::UNINDEXED; 7628 assert((Indexed || Offset.isUndef()) && 7629 "Unindexed masked load with an offset!"); 7630 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other) 7631 : getVTList(VT, MVT::Other); 7632 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru}; 7633 FoldingSetNodeID ID; 7634 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); 7635 ID.AddInteger(MemVT.getRawBits()); 7636 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>( 7637 dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO)); 7638 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7639 void *IP = nullptr; 7640 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7641 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO); 7642 return SDValue(E, 0); 7643 } 7644 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, 7645 AM, ExtTy, isExpanding, MemVT, MMO); 7646 createOperands(N, Ops); 7647 7648 CSEMap.InsertNode(N, IP); 7649 InsertNode(N); 7650 SDValue V(N, 0); 7651 NewSDValueDbgMsg(V, "Creating new node: ", this); 7652 return V; 7653 } 7654 7655 SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, 7656 SDValue Base, SDValue Offset, 7657 ISD::MemIndexedMode AM) { 7658 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad); 7659 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!"); 7660 return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base, 7661 Offset, LD->getMask(), LD->getPassThru(), 7662 LD->getMemoryVT(), LD->getMemOperand(), AM, 7663 LD->getExtensionType(), LD->isExpandingLoad()); 7664 } 7665 7666 SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, 7667 SDValue Val, SDValue Base, SDValue Offset, 7668 SDValue Mask, EVT MemVT, 7669 MachineMemOperand *MMO, 7670 ISD::MemIndexedMode AM, bool IsTruncating, 7671 bool IsCompressing) { 7672 assert(Chain.getValueType() == MVT::Other && 7673 "Invalid chain type"); 7674 bool Indexed = AM != ISD::UNINDEXED; 7675 assert((Indexed || Offset.isUndef()) && 7676 "Unindexed masked store with an offset!"); 7677 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other) 7678 : getVTList(MVT::Other); 7679 SDValue Ops[] = {Chain, Val, Base, Offset, Mask}; 7680 FoldingSetNodeID ID; 7681 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); 7682 ID.AddInteger(MemVT.getRawBits()); 7683 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>( 7684 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO)); 7685 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7686 void *IP = nullptr; 7687 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7688 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO); 7689 return SDValue(E, 0); 7690 } 7691 auto *N = 7692 newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, 7693 IsTruncating, IsCompressing, MemVT, MMO); 7694 createOperands(N, Ops); 7695 7696 CSEMap.InsertNode(N, IP); 7697 InsertNode(N); 7698 SDValue V(N, 0); 7699 NewSDValueDbgMsg(V, "Creating new node: ", this); 7700 return V; 7701 } 7702 7703 SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, 7704 SDValue Base, SDValue Offset, 7705 ISD::MemIndexedMode AM) { 7706 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore); 7707 assert(ST->getOffset().isUndef() && 7708 "Masked store is already a indexed store!"); 7709 return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset, 7710 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(), 7711 AM, ST->isTruncatingStore(), ST->isCompressingStore()); 7712 } 7713 7714 SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, 7715 ArrayRef<SDValue> Ops, 7716 MachineMemOperand *MMO, 7717 ISD::MemIndexType IndexType, 7718 ISD::LoadExtType ExtTy) { 7719 assert(Ops.size() == 6 && "Incompatible number of operands"); 7720 7721 FoldingSetNodeID ID; 7722 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); 7723 ID.AddInteger(MemVT.getRawBits()); 7724 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>( 7725 dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy)); 7726 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7727 void *IP = nullptr; 7728 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7729 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO); 7730 return SDValue(E, 0); 7731 } 7732 7733 IndexType = TLI->getCanonicalIndexType(IndexType, MemVT, Ops[4]); 7734 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), 7735 VTs, MemVT, MMO, IndexType, ExtTy); 7736 createOperands(N, Ops); 7737 7738 assert(N->getPassThru().getValueType() == N->getValueType(0) && 7739 "Incompatible type of the PassThru value in MaskedGatherSDNode"); 7740 assert(N->getMask().getValueType().getVectorElementCount() == 7741 N->getValueType(0).getVectorElementCount() && 7742 "Vector width mismatch between mask and data"); 7743 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() == 7744 N->getValueType(0).getVectorElementCount().isScalable() && 7745 "Scalable flags of index and data do not match"); 7746 assert(ElementCount::isKnownGE( 7747 N->getIndex().getValueType().getVectorElementCount(), 7748 N->getValueType(0).getVectorElementCount()) && 7749 "Vector width mismatch between index and data"); 7750 assert(isa<ConstantSDNode>(N->getScale()) && 7751 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && 7752 "Scale should be a constant power of 2"); 7753 7754 CSEMap.InsertNode(N, IP); 7755 InsertNode(N); 7756 SDValue V(N, 0); 7757 NewSDValueDbgMsg(V, "Creating new node: ", this); 7758 return V; 7759 } 7760 7761 SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, 7762 ArrayRef<SDValue> Ops, 7763 MachineMemOperand *MMO, 7764 ISD::MemIndexType IndexType, 7765 bool IsTrunc) { 7766 assert(Ops.size() == 6 && "Incompatible number of operands"); 7767 7768 FoldingSetNodeID ID; 7769 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); 7770 ID.AddInteger(MemVT.getRawBits()); 7771 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>( 7772 dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc)); 7773 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 7774 void *IP = nullptr; 7775 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 7776 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO); 7777 return SDValue(E, 0); 7778 } 7779 7780 IndexType = TLI->getCanonicalIndexType(IndexType, MemVT, Ops[4]); 7781 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), 7782 VTs, MemVT, MMO, IndexType, IsTrunc); 7783 createOperands(N, Ops); 7784 7785 assert(N->getMask().getValueType().getVectorElementCount() == 7786 N->getValue().getValueType().getVectorElementCount() && 7787 "Vector width mismatch between mask and data"); 7788 assert( 7789 N->getIndex().getValueType().getVectorElementCount().isScalable() == 7790 N->getValue().getValueType().getVectorElementCount().isScalable() && 7791 "Scalable flags of index and data do not match"); 7792 assert(ElementCount::isKnownGE( 7793 N->getIndex().getValueType().getVectorElementCount(), 7794 N->getValue().getValueType().getVectorElementCount()) && 7795 "Vector width mismatch between index and data"); 7796 assert(isa<ConstantSDNode>(N->getScale()) && 7797 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && 7798 "Scale should be a constant power of 2"); 7799 7800 CSEMap.InsertNode(N, IP); 7801 InsertNode(N); 7802 SDValue V(N, 0); 7803 NewSDValueDbgMsg(V, "Creating new node: ", this); 7804 return V; 7805 } 7806 7807 SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) { 7808 // select undef, T, F --> T (if T is a constant), otherwise F 7809 // select, ?, undef, F --> F 7810 // select, ?, T, undef --> T 7811 if (Cond.isUndef()) 7812 return isConstantValueOfAnyType(T) ? T : F; 7813 if (T.isUndef()) 7814 return F; 7815 if (F.isUndef()) 7816 return T; 7817 7818 // select true, T, F --> T 7819 // select false, T, F --> F 7820 if (auto *CondC = dyn_cast<ConstantSDNode>(Cond)) 7821 return CondC->isNullValue() ? F : T; 7822 7823 // TODO: This should simplify VSELECT with constant condition using something 7824 // like this (but check boolean contents to be complete?): 7825 // if (ISD::isBuildVectorAllOnes(Cond.getNode())) 7826 // return T; 7827 // if (ISD::isBuildVectorAllZeros(Cond.getNode())) 7828 // return F; 7829 7830 // select ?, T, T --> T 7831 if (T == F) 7832 return T; 7833 7834 return SDValue(); 7835 } 7836 7837 SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) { 7838 // shift undef, Y --> 0 (can always assume that the undef value is 0) 7839 if (X.isUndef()) 7840 return getConstant(0, SDLoc(X.getNode()), X.getValueType()); 7841 // shift X, undef --> undef (because it may shift by the bitwidth) 7842 if (Y.isUndef()) 7843 return getUNDEF(X.getValueType()); 7844 7845 // shift 0, Y --> 0 7846 // shift X, 0 --> X 7847 if (isNullOrNullSplat(X) || isNullOrNullSplat(Y)) 7848 return X; 7849 7850 // shift X, C >= bitwidth(X) --> undef 7851 // All vector elements must be too big (or undef) to avoid partial undefs. 7852 auto isShiftTooBig = [X](ConstantSDNode *Val) { 7853 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits()); 7854 }; 7855 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true)) 7856 return getUNDEF(X.getValueType()); 7857 7858 return SDValue(); 7859 } 7860 7861 SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, 7862 SDNodeFlags Flags) { 7863 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand 7864 // (an undef operand can be chosen to be Nan/Inf), then the result of this 7865 // operation is poison. That result can be relaxed to undef. 7866 ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true); 7867 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true); 7868 bool HasNan = (XC && XC->getValueAPF().isNaN()) || 7869 (YC && YC->getValueAPF().isNaN()); 7870 bool HasInf = (XC && XC->getValueAPF().isInfinity()) || 7871 (YC && YC->getValueAPF().isInfinity()); 7872 7873 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef())) 7874 return getUNDEF(X.getValueType()); 7875 7876 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef())) 7877 return getUNDEF(X.getValueType()); 7878 7879 if (!YC) 7880 return SDValue(); 7881 7882 // X + -0.0 --> X 7883 if (Opcode == ISD::FADD) 7884 if (YC->getValueAPF().isNegZero()) 7885 return X; 7886 7887 // X - +0.0 --> X 7888 if (Opcode == ISD::FSUB) 7889 if (YC->getValueAPF().isPosZero()) 7890 return X; 7891 7892 // X * 1.0 --> X 7893 // X / 1.0 --> X 7894 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV) 7895 if (YC->getValueAPF().isExactlyValue(1.0)) 7896 return X; 7897 7898 // X * 0.0 --> 0.0 7899 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros()) 7900 if (YC->getValueAPF().isZero()) 7901 return getConstantFP(0.0, SDLoc(Y), Y.getValueType()); 7902 7903 return SDValue(); 7904 } 7905 7906 SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, 7907 SDValue Ptr, SDValue SV, unsigned Align) { 7908 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) }; 7909 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops); 7910 } 7911 7912 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 7913 ArrayRef<SDUse> Ops) { 7914 switch (Ops.size()) { 7915 case 0: return getNode(Opcode, DL, VT); 7916 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0])); 7917 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 7918 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 7919 default: break; 7920 } 7921 7922 // Copy from an SDUse array into an SDValue array for use with 7923 // the regular getNode logic. 7924 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end()); 7925 return getNode(Opcode, DL, VT, NewOps); 7926 } 7927 7928 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 7929 ArrayRef<SDValue> Ops) { 7930 SDNodeFlags Flags; 7931 if (Inserter) 7932 Flags = Inserter->getFlags(); 7933 return getNode(Opcode, DL, VT, Ops, Flags); 7934 } 7935 7936 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 7937 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { 7938 unsigned NumOps = Ops.size(); 7939 switch (NumOps) { 7940 case 0: return getNode(Opcode, DL, VT); 7941 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags); 7942 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags); 7943 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags); 7944 default: break; 7945 } 7946 7947 #ifndef NDEBUG 7948 for (auto &Op : Ops) 7949 assert(Op.getOpcode() != ISD::DELETED_NODE && 7950 "Operand is DELETED_NODE!"); 7951 #endif 7952 7953 switch (Opcode) { 7954 default: break; 7955 case ISD::BUILD_VECTOR: 7956 // Attempt to simplify BUILD_VECTOR. 7957 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) 7958 return V; 7959 break; 7960 case ISD::CONCAT_VECTORS: 7961 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) 7962 return V; 7963 break; 7964 case ISD::SELECT_CC: 7965 assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 7966 assert(Ops[0].getValueType() == Ops[1].getValueType() && 7967 "LHS and RHS of condition must have same type!"); 7968 assert(Ops[2].getValueType() == Ops[3].getValueType() && 7969 "True and False arms of SelectCC must have same type!"); 7970 assert(Ops[2].getValueType() == VT && 7971 "select_cc node must be of same type as true and false value!"); 7972 break; 7973 case ISD::BR_CC: 7974 assert(NumOps == 5 && "BR_CC takes 5 operands!"); 7975 assert(Ops[2].getValueType() == Ops[3].getValueType() && 7976 "LHS/RHS of comparison should match types!"); 7977 break; 7978 } 7979 7980 // Memoize nodes. 7981 SDNode *N; 7982 SDVTList VTs = getVTList(VT); 7983 7984 if (VT != MVT::Glue) { 7985 FoldingSetNodeID ID; 7986 AddNodeIDNode(ID, Opcode, VTs, Ops); 7987 void *IP = nullptr; 7988 7989 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 7990 return SDValue(E, 0); 7991 7992 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 7993 createOperands(N, Ops); 7994 7995 CSEMap.InsertNode(N, IP); 7996 } else { 7997 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 7998 createOperands(N, Ops); 7999 } 8000 8001 N->setFlags(Flags); 8002 InsertNode(N); 8003 SDValue V(N, 0); 8004 NewSDValueDbgMsg(V, "Creating new node: ", this); 8005 return V; 8006 } 8007 8008 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, 8009 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) { 8010 return getNode(Opcode, DL, getVTList(ResultTys), Ops); 8011 } 8012 8013 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 8014 ArrayRef<SDValue> Ops) { 8015 SDNodeFlags Flags; 8016 if (Inserter) 8017 Flags = Inserter->getFlags(); 8018 return getNode(Opcode, DL, VTList, Ops, Flags); 8019 } 8020 8021 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 8022 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { 8023 if (VTList.NumVTs == 1) 8024 return getNode(Opcode, DL, VTList.VTs[0], Ops); 8025 8026 #ifndef NDEBUG 8027 for (auto &Op : Ops) 8028 assert(Op.getOpcode() != ISD::DELETED_NODE && 8029 "Operand is DELETED_NODE!"); 8030 #endif 8031 8032 switch (Opcode) { 8033 case ISD::STRICT_FP_EXTEND: 8034 assert(VTList.NumVTs == 2 && Ops.size() == 2 && 8035 "Invalid STRICT_FP_EXTEND!"); 8036 assert(VTList.VTs[0].isFloatingPoint() && 8037 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!"); 8038 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && 8039 "STRICT_FP_EXTEND result type should be vector iff the operand " 8040 "type is vector!"); 8041 assert((!VTList.VTs[0].isVector() || 8042 VTList.VTs[0].getVectorNumElements() == 8043 Ops[1].getValueType().getVectorNumElements()) && 8044 "Vector element count mismatch!"); 8045 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) && 8046 "Invalid fpext node, dst <= src!"); 8047 break; 8048 case ISD::STRICT_FP_ROUND: 8049 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!"); 8050 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && 8051 "STRICT_FP_ROUND result type should be vector iff the operand " 8052 "type is vector!"); 8053 assert((!VTList.VTs[0].isVector() || 8054 VTList.VTs[0].getVectorNumElements() == 8055 Ops[1].getValueType().getVectorNumElements()) && 8056 "Vector element count mismatch!"); 8057 assert(VTList.VTs[0].isFloatingPoint() && 8058 Ops[1].getValueType().isFloatingPoint() && 8059 VTList.VTs[0].bitsLT(Ops[1].getValueType()) && 8060 isa<ConstantSDNode>(Ops[2]) && 8061 (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || 8062 cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && 8063 "Invalid STRICT_FP_ROUND!"); 8064 break; 8065 #if 0 8066 // FIXME: figure out how to safely handle things like 8067 // int foo(int x) { return 1 << (x & 255); } 8068 // int bar() { return foo(256); } 8069 case ISD::SRA_PARTS: 8070 case ISD::SRL_PARTS: 8071 case ISD::SHL_PARTS: 8072 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 8073 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 8074 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 8075 else if (N3.getOpcode() == ISD::AND) 8076 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 8077 // If the and is only masking out bits that cannot effect the shift, 8078 // eliminate the and. 8079 unsigned NumBits = VT.getScalarSizeInBits()*2; 8080 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 8081 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 8082 } 8083 break; 8084 #endif 8085 } 8086 8087 // Memoize the node unless it returns a flag. 8088 SDNode *N; 8089 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 8090 FoldingSetNodeID ID; 8091 AddNodeIDNode(ID, Opcode, VTList, Ops); 8092 void *IP = nullptr; 8093 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 8094 return SDValue(E, 0); 8095 8096 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); 8097 createOperands(N, Ops); 8098 CSEMap.InsertNode(N, IP); 8099 } else { 8100 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); 8101 createOperands(N, Ops); 8102 } 8103 8104 N->setFlags(Flags); 8105 InsertNode(N); 8106 SDValue V(N, 0); 8107 NewSDValueDbgMsg(V, "Creating new node: ", this); 8108 return V; 8109 } 8110 8111 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, 8112 SDVTList VTList) { 8113 return getNode(Opcode, DL, VTList, None); 8114 } 8115 8116 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 8117 SDValue N1) { 8118 SDValue Ops[] = { N1 }; 8119 return getNode(Opcode, DL, VTList, Ops); 8120 } 8121 8122 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 8123 SDValue N1, SDValue N2) { 8124 SDValue Ops[] = { N1, N2 }; 8125 return getNode(Opcode, DL, VTList, Ops); 8126 } 8127 8128 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 8129 SDValue N1, SDValue N2, SDValue N3) { 8130 SDValue Ops[] = { N1, N2, N3 }; 8131 return getNode(Opcode, DL, VTList, Ops); 8132 } 8133 8134 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 8135 SDValue N1, SDValue N2, SDValue N3, SDValue N4) { 8136 SDValue Ops[] = { N1, N2, N3, N4 }; 8137 return getNode(Opcode, DL, VTList, Ops); 8138 } 8139 8140 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 8141 SDValue N1, SDValue N2, SDValue N3, SDValue N4, 8142 SDValue N5) { 8143 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 8144 return getNode(Opcode, DL, VTList, Ops); 8145 } 8146 8147 SDVTList SelectionDAG::getVTList(EVT VT) { 8148 return makeVTList(SDNode::getValueTypeList(VT), 1); 8149 } 8150 8151 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 8152 FoldingSetNodeID ID; 8153 ID.AddInteger(2U); 8154 ID.AddInteger(VT1.getRawBits()); 8155 ID.AddInteger(VT2.getRawBits()); 8156 8157 void *IP = nullptr; 8158 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 8159 if (!Result) { 8160 EVT *Array = Allocator.Allocate<EVT>(2); 8161 Array[0] = VT1; 8162 Array[1] = VT2; 8163 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); 8164 VTListMap.InsertNode(Result, IP); 8165 } 8166 return Result->getSDVTList(); 8167 } 8168 8169 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 8170 FoldingSetNodeID ID; 8171 ID.AddInteger(3U); 8172 ID.AddInteger(VT1.getRawBits()); 8173 ID.AddInteger(VT2.getRawBits()); 8174 ID.AddInteger(VT3.getRawBits()); 8175 8176 void *IP = nullptr; 8177 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 8178 if (!Result) { 8179 EVT *Array = Allocator.Allocate<EVT>(3); 8180 Array[0] = VT1; 8181 Array[1] = VT2; 8182 Array[2] = VT3; 8183 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); 8184 VTListMap.InsertNode(Result, IP); 8185 } 8186 return Result->getSDVTList(); 8187 } 8188 8189 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 8190 FoldingSetNodeID ID; 8191 ID.AddInteger(4U); 8192 ID.AddInteger(VT1.getRawBits()); 8193 ID.AddInteger(VT2.getRawBits()); 8194 ID.AddInteger(VT3.getRawBits()); 8195 ID.AddInteger(VT4.getRawBits()); 8196 8197 void *IP = nullptr; 8198 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 8199 if (!Result) { 8200 EVT *Array = Allocator.Allocate<EVT>(4); 8201 Array[0] = VT1; 8202 Array[1] = VT2; 8203 Array[2] = VT3; 8204 Array[3] = VT4; 8205 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); 8206 VTListMap.InsertNode(Result, IP); 8207 } 8208 return Result->getSDVTList(); 8209 } 8210 8211 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) { 8212 unsigned NumVTs = VTs.size(); 8213 FoldingSetNodeID ID; 8214 ID.AddInteger(NumVTs); 8215 for (unsigned index = 0; index < NumVTs; index++) { 8216 ID.AddInteger(VTs[index].getRawBits()); 8217 } 8218 8219 void *IP = nullptr; 8220 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 8221 if (!Result) { 8222 EVT *Array = Allocator.Allocate<EVT>(NumVTs); 8223 llvm::copy(VTs, Array); 8224 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); 8225 VTListMap.InsertNode(Result, IP); 8226 } 8227 return Result->getSDVTList(); 8228 } 8229 8230 8231 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the 8232 /// specified operands. If the resultant node already exists in the DAG, 8233 /// this does not modify the specified node, instead it returns the node that 8234 /// already exists. If the resultant node does not exist in the DAG, the 8235 /// input node is returned. As a degenerate case, if you specify the same 8236 /// input operands as the node already has, the input node is returned. 8237 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 8238 assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 8239 8240 // Check to see if there is no change. 8241 if (Op == N->getOperand(0)) return N; 8242 8243 // See if the modified node already exists. 8244 void *InsertPos = nullptr; 8245 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 8246 return Existing; 8247 8248 // Nope it doesn't. Remove the node from its current place in the maps. 8249 if (InsertPos) 8250 if (!RemoveNodeFromCSEMaps(N)) 8251 InsertPos = nullptr; 8252 8253 // Now we update the operands. 8254 N->OperandList[0].set(Op); 8255 8256 updateDivergence(N); 8257 // If this gets put into a CSE map, add it. 8258 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 8259 return N; 8260 } 8261 8262 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 8263 assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 8264 8265 // Check to see if there is no change. 8266 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 8267 return N; // No operands changed, just return the input node. 8268 8269 // See if the modified node already exists. 8270 void *InsertPos = nullptr; 8271 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 8272 return Existing; 8273 8274 // Nope it doesn't. Remove the node from its current place in the maps. 8275 if (InsertPos) 8276 if (!RemoveNodeFromCSEMaps(N)) 8277 InsertPos = nullptr; 8278 8279 // Now we update the operands. 8280 if (N->OperandList[0] != Op1) 8281 N->OperandList[0].set(Op1); 8282 if (N->OperandList[1] != Op2) 8283 N->OperandList[1].set(Op2); 8284 8285 updateDivergence(N); 8286 // If this gets put into a CSE map, add it. 8287 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 8288 return N; 8289 } 8290 8291 SDNode *SelectionDAG:: 8292 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 8293 SDValue Ops[] = { Op1, Op2, Op3 }; 8294 return UpdateNodeOperands(N, Ops); 8295 } 8296 8297 SDNode *SelectionDAG:: 8298 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 8299 SDValue Op3, SDValue Op4) { 8300 SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 8301 return UpdateNodeOperands(N, Ops); 8302 } 8303 8304 SDNode *SelectionDAG:: 8305 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 8306 SDValue Op3, SDValue Op4, SDValue Op5) { 8307 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 8308 return UpdateNodeOperands(N, Ops); 8309 } 8310 8311 SDNode *SelectionDAG:: 8312 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) { 8313 unsigned NumOps = Ops.size(); 8314 assert(N->getNumOperands() == NumOps && 8315 "Update with wrong number of operands"); 8316 8317 // If no operands changed just return the input node. 8318 if (std::equal(Ops.begin(), Ops.end(), N->op_begin())) 8319 return N; 8320 8321 // See if the modified node already exists. 8322 void *InsertPos = nullptr; 8323 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos)) 8324 return Existing; 8325 8326 // Nope it doesn't. Remove the node from its current place in the maps. 8327 if (InsertPos) 8328 if (!RemoveNodeFromCSEMaps(N)) 8329 InsertPos = nullptr; 8330 8331 // Now we update the operands. 8332 for (unsigned i = 0; i != NumOps; ++i) 8333 if (N->OperandList[i] != Ops[i]) 8334 N->OperandList[i].set(Ops[i]); 8335 8336 updateDivergence(N); 8337 // If this gets put into a CSE map, add it. 8338 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 8339 return N; 8340 } 8341 8342 /// DropOperands - Release the operands and set this node to have 8343 /// zero operands. 8344 void SDNode::DropOperands() { 8345 // Unlike the code in MorphNodeTo that does this, we don't need to 8346 // watch for dead nodes here. 8347 for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 8348 SDUse &Use = *I++; 8349 Use.set(SDValue()); 8350 } 8351 } 8352 8353 void SelectionDAG::setNodeMemRefs(MachineSDNode *N, 8354 ArrayRef<MachineMemOperand *> NewMemRefs) { 8355 if (NewMemRefs.empty()) { 8356 N->clearMemRefs(); 8357 return; 8358 } 8359 8360 // Check if we can avoid allocating by storing a single reference directly. 8361 if (NewMemRefs.size() == 1) { 8362 N->MemRefs = NewMemRefs[0]; 8363 N->NumMemRefs = 1; 8364 return; 8365 } 8366 8367 MachineMemOperand **MemRefsBuffer = 8368 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size()); 8369 llvm::copy(NewMemRefs, MemRefsBuffer); 8370 N->MemRefs = MemRefsBuffer; 8371 N->NumMemRefs = static_cast<int>(NewMemRefs.size()); 8372 } 8373 8374 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 8375 /// machine opcode. 8376 /// 8377 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 8378 EVT VT) { 8379 SDVTList VTs = getVTList(VT); 8380 return SelectNodeTo(N, MachineOpc, VTs, None); 8381 } 8382 8383 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 8384 EVT VT, SDValue Op1) { 8385 SDVTList VTs = getVTList(VT); 8386 SDValue Ops[] = { Op1 }; 8387 return SelectNodeTo(N, MachineOpc, VTs, Ops); 8388 } 8389 8390 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 8391 EVT VT, SDValue Op1, 8392 SDValue Op2) { 8393 SDVTList VTs = getVTList(VT); 8394 SDValue Ops[] = { Op1, Op2 }; 8395 return SelectNodeTo(N, MachineOpc, VTs, Ops); 8396 } 8397 8398 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 8399 EVT VT, SDValue Op1, 8400 SDValue Op2, SDValue Op3) { 8401 SDVTList VTs = getVTList(VT); 8402 SDValue Ops[] = { Op1, Op2, Op3 }; 8403 return SelectNodeTo(N, MachineOpc, VTs, Ops); 8404 } 8405 8406 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 8407 EVT VT, ArrayRef<SDValue> Ops) { 8408 SDVTList VTs = getVTList(VT); 8409 return SelectNodeTo(N, MachineOpc, VTs, Ops); 8410 } 8411 8412 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 8413 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) { 8414 SDVTList VTs = getVTList(VT1, VT2); 8415 return SelectNodeTo(N, MachineOpc, VTs, Ops); 8416 } 8417 8418 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 8419 EVT VT1, EVT VT2) { 8420 SDVTList VTs = getVTList(VT1, VT2); 8421 return SelectNodeTo(N, MachineOpc, VTs, None); 8422 } 8423 8424 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 8425 EVT VT1, EVT VT2, EVT VT3, 8426 ArrayRef<SDValue> Ops) { 8427 SDVTList VTs = getVTList(VT1, VT2, VT3); 8428 return SelectNodeTo(N, MachineOpc, VTs, Ops); 8429 } 8430 8431 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 8432 EVT VT1, EVT VT2, 8433 SDValue Op1, SDValue Op2) { 8434 SDVTList VTs = getVTList(VT1, VT2); 8435 SDValue Ops[] = { Op1, Op2 }; 8436 return SelectNodeTo(N, MachineOpc, VTs, Ops); 8437 } 8438 8439 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 8440 SDVTList VTs,ArrayRef<SDValue> Ops) { 8441 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops); 8442 // Reset the NodeID to -1. 8443 New->setNodeId(-1); 8444 if (New != N) { 8445 ReplaceAllUsesWith(N, New); 8446 RemoveDeadNode(N); 8447 } 8448 return New; 8449 } 8450 8451 /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away 8452 /// the line number information on the merged node since it is not possible to 8453 /// preserve the information that operation is associated with multiple lines. 8454 /// This will make the debugger working better at -O0, were there is a higher 8455 /// probability having other instructions associated with that line. 8456 /// 8457 /// For IROrder, we keep the smaller of the two 8458 SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) { 8459 DebugLoc NLoc = N->getDebugLoc(); 8460 if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) { 8461 N->setDebugLoc(DebugLoc()); 8462 } 8463 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); 8464 N->setIROrder(Order); 8465 return N; 8466 } 8467 8468 /// MorphNodeTo - This *mutates* the specified node to have the specified 8469 /// return type, opcode, and operands. 8470 /// 8471 /// Note that MorphNodeTo returns the resultant node. If there is already a 8472 /// node of the specified opcode and operands, it returns that node instead of 8473 /// the current one. Note that the SDLoc need not be the same. 8474 /// 8475 /// Using MorphNodeTo is faster than creating a new node and swapping it in 8476 /// with ReplaceAllUsesWith both because it often avoids allocating a new 8477 /// node, and because it doesn't require CSE recalculation for any of 8478 /// the node's users. 8479 /// 8480 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG. 8481 /// As a consequence it isn't appropriate to use from within the DAG combiner or 8482 /// the legalizer which maintain worklists that would need to be updated when 8483 /// deleting things. 8484 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 8485 SDVTList VTs, ArrayRef<SDValue> Ops) { 8486 // If an identical node already exists, use it. 8487 void *IP = nullptr; 8488 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 8489 FoldingSetNodeID ID; 8490 AddNodeIDNode(ID, Opc, VTs, Ops); 8491 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP)) 8492 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N)); 8493 } 8494 8495 if (!RemoveNodeFromCSEMaps(N)) 8496 IP = nullptr; 8497 8498 // Start the morphing. 8499 N->NodeType = Opc; 8500 N->ValueList = VTs.VTs; 8501 N->NumValues = VTs.NumVTs; 8502 8503 // Clear the operands list, updating used nodes to remove this from their 8504 // use list. Keep track of any operands that become dead as a result. 8505 SmallPtrSet<SDNode*, 16> DeadNodeSet; 8506 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 8507 SDUse &Use = *I++; 8508 SDNode *Used = Use.getNode(); 8509 Use.set(SDValue()); 8510 if (Used->use_empty()) 8511 DeadNodeSet.insert(Used); 8512 } 8513 8514 // For MachineNode, initialize the memory references information. 8515 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) 8516 MN->clearMemRefs(); 8517 8518 // Swap for an appropriately sized array from the recycler. 8519 removeOperands(N); 8520 createOperands(N, Ops); 8521 8522 // Delete any nodes that are still dead after adding the uses for the 8523 // new operands. 8524 if (!DeadNodeSet.empty()) { 8525 SmallVector<SDNode *, 16> DeadNodes; 8526 for (SDNode *N : DeadNodeSet) 8527 if (N->use_empty()) 8528 DeadNodes.push_back(N); 8529 RemoveDeadNodes(DeadNodes); 8530 } 8531 8532 if (IP) 8533 CSEMap.InsertNode(N, IP); // Memoize the new node. 8534 return N; 8535 } 8536 8537 SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { 8538 unsigned OrigOpc = Node->getOpcode(); 8539 unsigned NewOpc; 8540 switch (OrigOpc) { 8541 default: 8542 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!"); 8543 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 8544 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break; 8545 #define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 8546 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break; 8547 #include "llvm/IR/ConstrainedOps.def" 8548 } 8549 8550 assert(Node->getNumValues() == 2 && "Unexpected number of results!"); 8551 8552 // We're taking this node out of the chain, so we need to re-link things. 8553 SDValue InputChain = Node->getOperand(0); 8554 SDValue OutputChain = SDValue(Node, 1); 8555 ReplaceAllUsesOfValueWith(OutputChain, InputChain); 8556 8557 SmallVector<SDValue, 3> Ops; 8558 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) 8559 Ops.push_back(Node->getOperand(i)); 8560 8561 SDVTList VTs = getVTList(Node->getValueType(0)); 8562 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops); 8563 8564 // MorphNodeTo can operate in two ways: if an existing node with the 8565 // specified operands exists, it can just return it. Otherwise, it 8566 // updates the node in place to have the requested operands. 8567 if (Res == Node) { 8568 // If we updated the node in place, reset the node ID. To the isel, 8569 // this should be just like a newly allocated machine node. 8570 Res->setNodeId(-1); 8571 } else { 8572 ReplaceAllUsesWith(Node, Res); 8573 RemoveDeadNode(Node); 8574 } 8575 8576 return Res; 8577 } 8578 8579 /// getMachineNode - These are used for target selectors to create a new node 8580 /// with specified return type(s), MachineInstr opcode, and operands. 8581 /// 8582 /// Note that getMachineNode returns the resultant node. If there is already a 8583 /// node of the specified opcode and operands, it returns that node instead of 8584 /// the current one. 8585 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8586 EVT VT) { 8587 SDVTList VTs = getVTList(VT); 8588 return getMachineNode(Opcode, dl, VTs, None); 8589 } 8590 8591 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8592 EVT VT, SDValue Op1) { 8593 SDVTList VTs = getVTList(VT); 8594 SDValue Ops[] = { Op1 }; 8595 return getMachineNode(Opcode, dl, VTs, Ops); 8596 } 8597 8598 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8599 EVT VT, SDValue Op1, SDValue Op2) { 8600 SDVTList VTs = getVTList(VT); 8601 SDValue Ops[] = { Op1, Op2 }; 8602 return getMachineNode(Opcode, dl, VTs, Ops); 8603 } 8604 8605 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8606 EVT VT, SDValue Op1, SDValue Op2, 8607 SDValue Op3) { 8608 SDVTList VTs = getVTList(VT); 8609 SDValue Ops[] = { Op1, Op2, Op3 }; 8610 return getMachineNode(Opcode, dl, VTs, Ops); 8611 } 8612 8613 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8614 EVT VT, ArrayRef<SDValue> Ops) { 8615 SDVTList VTs = getVTList(VT); 8616 return getMachineNode(Opcode, dl, VTs, Ops); 8617 } 8618 8619 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8620 EVT VT1, EVT VT2, SDValue Op1, 8621 SDValue Op2) { 8622 SDVTList VTs = getVTList(VT1, VT2); 8623 SDValue Ops[] = { Op1, Op2 }; 8624 return getMachineNode(Opcode, dl, VTs, Ops); 8625 } 8626 8627 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8628 EVT VT1, EVT VT2, SDValue Op1, 8629 SDValue Op2, SDValue Op3) { 8630 SDVTList VTs = getVTList(VT1, VT2); 8631 SDValue Ops[] = { Op1, Op2, Op3 }; 8632 return getMachineNode(Opcode, dl, VTs, Ops); 8633 } 8634 8635 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8636 EVT VT1, EVT VT2, 8637 ArrayRef<SDValue> Ops) { 8638 SDVTList VTs = getVTList(VT1, VT2); 8639 return getMachineNode(Opcode, dl, VTs, Ops); 8640 } 8641 8642 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8643 EVT VT1, EVT VT2, EVT VT3, 8644 SDValue Op1, SDValue Op2) { 8645 SDVTList VTs = getVTList(VT1, VT2, VT3); 8646 SDValue Ops[] = { Op1, Op2 }; 8647 return getMachineNode(Opcode, dl, VTs, Ops); 8648 } 8649 8650 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8651 EVT VT1, EVT VT2, EVT VT3, 8652 SDValue Op1, SDValue Op2, 8653 SDValue Op3) { 8654 SDVTList VTs = getVTList(VT1, VT2, VT3); 8655 SDValue Ops[] = { Op1, Op2, Op3 }; 8656 return getMachineNode(Opcode, dl, VTs, Ops); 8657 } 8658 8659 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8660 EVT VT1, EVT VT2, EVT VT3, 8661 ArrayRef<SDValue> Ops) { 8662 SDVTList VTs = getVTList(VT1, VT2, VT3); 8663 return getMachineNode(Opcode, dl, VTs, Ops); 8664 } 8665 8666 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 8667 ArrayRef<EVT> ResultTys, 8668 ArrayRef<SDValue> Ops) { 8669 SDVTList VTs = getVTList(ResultTys); 8670 return getMachineNode(Opcode, dl, VTs, Ops); 8671 } 8672 8673 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL, 8674 SDVTList VTs, 8675 ArrayRef<SDValue> Ops) { 8676 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 8677 MachineSDNode *N; 8678 void *IP = nullptr; 8679 8680 if (DoCSE) { 8681 FoldingSetNodeID ID; 8682 AddNodeIDNode(ID, ~Opcode, VTs, Ops); 8683 IP = nullptr; 8684 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 8685 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL)); 8686 } 8687 } 8688 8689 // Allocate a new MachineSDNode. 8690 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 8691 createOperands(N, Ops); 8692 8693 if (DoCSE) 8694 CSEMap.InsertNode(N, IP); 8695 8696 InsertNode(N); 8697 NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this); 8698 return N; 8699 } 8700 8701 /// getTargetExtractSubreg - A convenience function for creating 8702 /// TargetOpcode::EXTRACT_SUBREG nodes. 8703 SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, 8704 SDValue Operand) { 8705 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); 8706 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 8707 VT, Operand, SRIdxVal); 8708 return SDValue(Subreg, 0); 8709 } 8710 8711 /// getTargetInsertSubreg - A convenience function for creating 8712 /// TargetOpcode::INSERT_SUBREG nodes. 8713 SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, 8714 SDValue Operand, SDValue Subreg) { 8715 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); 8716 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 8717 VT, Operand, Subreg, SRIdxVal); 8718 return SDValue(Result, 0); 8719 } 8720 8721 /// getNodeIfExists - Get the specified node if it's already available, or 8722 /// else return NULL. 8723 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 8724 ArrayRef<SDValue> Ops) { 8725 SDNodeFlags Flags; 8726 if (Inserter) 8727 Flags = Inserter->getFlags(); 8728 return getNodeIfExists(Opcode, VTList, Ops, Flags); 8729 } 8730 8731 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 8732 ArrayRef<SDValue> Ops, 8733 const SDNodeFlags Flags) { 8734 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { 8735 FoldingSetNodeID ID; 8736 AddNodeIDNode(ID, Opcode, VTList, Ops); 8737 void *IP = nullptr; 8738 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) { 8739 E->intersectFlagsWith(Flags); 8740 return E; 8741 } 8742 } 8743 return nullptr; 8744 } 8745 8746 /// doesNodeExist - Check if a node exists without modifying its flags. 8747 bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList, 8748 ArrayRef<SDValue> Ops) { 8749 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { 8750 FoldingSetNodeID ID; 8751 AddNodeIDNode(ID, Opcode, VTList, Ops); 8752 void *IP = nullptr; 8753 if (FindNodeOrInsertPos(ID, SDLoc(), IP)) 8754 return true; 8755 } 8756 return false; 8757 } 8758 8759 /// getDbgValue - Creates a SDDbgValue node. 8760 /// 8761 /// SDNode 8762 SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr, 8763 SDNode *N, unsigned R, bool IsIndirect, 8764 const DebugLoc &DL, unsigned O) { 8765 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 8766 "Expected inlined-at fields to agree"); 8767 return new (DbgInfo->getAlloc()) 8768 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R), 8769 {}, IsIndirect, DL, O, 8770 /*IsVariadic=*/false); 8771 } 8772 8773 /// Constant 8774 SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var, 8775 DIExpression *Expr, 8776 const Value *C, 8777 const DebugLoc &DL, unsigned O) { 8778 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 8779 "Expected inlined-at fields to agree"); 8780 return new (DbgInfo->getAlloc()) 8781 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {}, 8782 /*IsIndirect=*/false, DL, O, 8783 /*IsVariadic=*/false); 8784 } 8785 8786 /// FrameIndex 8787 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, 8788 DIExpression *Expr, unsigned FI, 8789 bool IsIndirect, 8790 const DebugLoc &DL, 8791 unsigned O) { 8792 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 8793 "Expected inlined-at fields to agree"); 8794 return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O); 8795 } 8796 8797 /// FrameIndex with dependencies 8798 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, 8799 DIExpression *Expr, unsigned FI, 8800 ArrayRef<SDNode *> Dependencies, 8801 bool IsIndirect, 8802 const DebugLoc &DL, 8803 unsigned O) { 8804 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 8805 "Expected inlined-at fields to agree"); 8806 return new (DbgInfo->getAlloc()) 8807 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI), 8808 Dependencies, IsIndirect, DL, O, 8809 /*IsVariadic=*/false); 8810 } 8811 8812 /// VReg 8813 SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr, 8814 unsigned VReg, bool IsIndirect, 8815 const DebugLoc &DL, unsigned O) { 8816 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 8817 "Expected inlined-at fields to agree"); 8818 return new (DbgInfo->getAlloc()) 8819 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg), 8820 {}, IsIndirect, DL, O, 8821 /*IsVariadic=*/false); 8822 } 8823 8824 SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr, 8825 ArrayRef<SDDbgOperand> Locs, 8826 ArrayRef<SDNode *> Dependencies, 8827 bool IsIndirect, const DebugLoc &DL, 8828 unsigned O, bool IsVariadic) { 8829 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 8830 "Expected inlined-at fields to agree"); 8831 return new (DbgInfo->getAlloc()) 8832 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect, 8833 DL, O, IsVariadic); 8834 } 8835 8836 void SelectionDAG::transferDbgValues(SDValue From, SDValue To, 8837 unsigned OffsetInBits, unsigned SizeInBits, 8838 bool InvalidateDbg) { 8839 SDNode *FromNode = From.getNode(); 8840 SDNode *ToNode = To.getNode(); 8841 assert(FromNode && ToNode && "Can't modify dbg values"); 8842 8843 // PR35338 8844 // TODO: assert(From != To && "Redundant dbg value transfer"); 8845 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer"); 8846 if (From == To || FromNode == ToNode) 8847 return; 8848 8849 if (!FromNode->getHasDebugValue()) 8850 return; 8851 8852 SDDbgOperand FromLocOp = 8853 SDDbgOperand::fromNode(From.getNode(), From.getResNo()); 8854 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(To.getNode(), To.getResNo()); 8855 8856 SmallVector<SDDbgValue *, 2> ClonedDVs; 8857 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) { 8858 if (Dbg->isInvalidated()) 8859 continue; 8860 8861 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value"); 8862 8863 // Create a new location ops vector that is equal to the old vector, but 8864 // with each instance of FromLocOp replaced with ToLocOp. 8865 bool Changed = false; 8866 auto NewLocOps = Dbg->copyLocationOps(); 8867 std::replace_if( 8868 NewLocOps.begin(), NewLocOps.end(), 8869 [&Changed, FromLocOp](const SDDbgOperand &Op) { 8870 bool Match = Op == FromLocOp; 8871 Changed |= Match; 8872 return Match; 8873 }, 8874 ToLocOp); 8875 // Ignore this SDDbgValue if we didn't find a matching location. 8876 if (!Changed) 8877 continue; 8878 8879 DIVariable *Var = Dbg->getVariable(); 8880 auto *Expr = Dbg->getExpression(); 8881 // If a fragment is requested, update the expression. 8882 if (SizeInBits) { 8883 // When splitting a larger (e.g., sign-extended) value whose 8884 // lower bits are described with an SDDbgValue, do not attempt 8885 // to transfer the SDDbgValue to the upper bits. 8886 if (auto FI = Expr->getFragmentInfo()) 8887 if (OffsetInBits + SizeInBits > FI->SizeInBits) 8888 continue; 8889 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits, 8890 SizeInBits); 8891 if (!Fragment) 8892 continue; 8893 Expr = *Fragment; 8894 } 8895 8896 auto AdditionalDependencies = Dbg->getAdditionalDependencies(); 8897 // Clone the SDDbgValue and move it to To. 8898 SDDbgValue *Clone = getDbgValueList( 8899 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(), 8900 Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()), 8901 Dbg->isVariadic()); 8902 ClonedDVs.push_back(Clone); 8903 8904 if (InvalidateDbg) { 8905 // Invalidate value and indicate the SDDbgValue should not be emitted. 8906 Dbg->setIsInvalidated(); 8907 Dbg->setIsEmitted(); 8908 } 8909 } 8910 8911 for (SDDbgValue *Dbg : ClonedDVs) { 8912 assert(is_contained(Dbg->getSDNodes(), ToNode) && 8913 "Transferred DbgValues should depend on the new SDNode"); 8914 AddDbgValue(Dbg, false); 8915 } 8916 } 8917 8918 void SelectionDAG::salvageDebugInfo(SDNode &N) { 8919 if (!N.getHasDebugValue()) 8920 return; 8921 8922 SmallVector<SDDbgValue *, 2> ClonedDVs; 8923 for (auto DV : GetDbgValues(&N)) { 8924 if (DV->isInvalidated()) 8925 continue; 8926 switch (N.getOpcode()) { 8927 default: 8928 break; 8929 case ISD::ADD: 8930 SDValue N0 = N.getOperand(0); 8931 SDValue N1 = N.getOperand(1); 8932 if (!isConstantIntBuildVectorOrConstantInt(N0) && 8933 isConstantIntBuildVectorOrConstantInt(N1)) { 8934 uint64_t Offset = N.getConstantOperandVal(1); 8935 8936 // Rewrite an ADD constant node into a DIExpression. Since we are 8937 // performing arithmetic to compute the variable's *value* in the 8938 // DIExpression, we need to mark the expression with a 8939 // DW_OP_stack_value. 8940 auto *DIExpr = DV->getExpression(); 8941 auto NewLocOps = DV->copyLocationOps(); 8942 bool Changed = false; 8943 for (size_t i = 0; i < NewLocOps.size(); ++i) { 8944 // We're not given a ResNo to compare against because the whole 8945 // node is going away. We know that any ISD::ADD only has one 8946 // result, so we can assume any node match is using the result. 8947 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE || 8948 NewLocOps[i].getSDNode() != &N) 8949 continue; 8950 NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo()); 8951 SmallVector<uint64_t, 3> ExprOps; 8952 DIExpression::appendOffset(ExprOps, Offset); 8953 DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true); 8954 Changed = true; 8955 } 8956 (void)Changed; 8957 assert(Changed && "Salvage target doesn't use N"); 8958 8959 auto AdditionalDependencies = DV->getAdditionalDependencies(); 8960 SDDbgValue *Clone = getDbgValueList(DV->getVariable(), DIExpr, 8961 NewLocOps, AdditionalDependencies, 8962 DV->isIndirect(), DV->getDebugLoc(), 8963 DV->getOrder(), DV->isVariadic()); 8964 ClonedDVs.push_back(Clone); 8965 DV->setIsInvalidated(); 8966 DV->setIsEmitted(); 8967 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; 8968 N0.getNode()->dumprFull(this); 8969 dbgs() << " into " << *DIExpr << '\n'); 8970 } 8971 } 8972 } 8973 8974 for (SDDbgValue *Dbg : ClonedDVs) { 8975 assert(!Dbg->getSDNodes().empty() && 8976 "Salvaged DbgValue should depend on a new SDNode"); 8977 AddDbgValue(Dbg, false); 8978 } 8979 } 8980 8981 /// Creates a SDDbgLabel node. 8982 SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label, 8983 const DebugLoc &DL, unsigned O) { 8984 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) && 8985 "Expected inlined-at fields to agree"); 8986 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O); 8987 } 8988 8989 namespace { 8990 8991 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 8992 /// pointed to by a use iterator is deleted, increment the use iterator 8993 /// so that it doesn't dangle. 8994 /// 8995 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 8996 SDNode::use_iterator &UI; 8997 SDNode::use_iterator &UE; 8998 8999 void NodeDeleted(SDNode *N, SDNode *E) override { 9000 // Increment the iterator as needed. 9001 while (UI != UE && N == *UI) 9002 ++UI; 9003 } 9004 9005 public: 9006 RAUWUpdateListener(SelectionDAG &d, 9007 SDNode::use_iterator &ui, 9008 SDNode::use_iterator &ue) 9009 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} 9010 }; 9011 9012 } // end anonymous namespace 9013 9014 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 9015 /// This can cause recursive merging of nodes in the DAG. 9016 /// 9017 /// This version assumes From has a single result value. 9018 /// 9019 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { 9020 SDNode *From = FromN.getNode(); 9021 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 9022 "Cannot replace with this method!"); 9023 assert(From != To.getNode() && "Cannot replace uses of with self"); 9024 9025 // Preserve Debug Values 9026 transferDbgValues(FromN, To); 9027 9028 // Iterate over all the existing uses of From. New uses will be added 9029 // to the beginning of the use list, which we avoid visiting. 9030 // This specifically avoids visiting uses of From that arise while the 9031 // replacement is happening, because any such uses would be the result 9032 // of CSE: If an existing node looks like From after one of its operands 9033 // is replaced by To, we don't want to replace of all its users with To 9034 // too. See PR3018 for more info. 9035 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 9036 RAUWUpdateListener Listener(*this, UI, UE); 9037 while (UI != UE) { 9038 SDNode *User = *UI; 9039 9040 // This node is about to morph, remove its old self from the CSE maps. 9041 RemoveNodeFromCSEMaps(User); 9042 9043 // A user can appear in a use list multiple times, and when this 9044 // happens the uses are usually next to each other in the list. 9045 // To help reduce the number of CSE recomputations, process all 9046 // the uses of this user that we can find this way. 9047 do { 9048 SDUse &Use = UI.getUse(); 9049 ++UI; 9050 Use.set(To); 9051 if (To->isDivergent() != From->isDivergent()) 9052 updateDivergence(User); 9053 } while (UI != UE && *UI == User); 9054 // Now that we have modified User, add it back to the CSE maps. If it 9055 // already exists there, recursively merge the results together. 9056 AddModifiedNodeToCSEMaps(User); 9057 } 9058 9059 // If we just RAUW'd the root, take note. 9060 if (FromN == getRoot()) 9061 setRoot(To); 9062 } 9063 9064 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 9065 /// This can cause recursive merging of nodes in the DAG. 9066 /// 9067 /// This version assumes that for each value of From, there is a 9068 /// corresponding value in To in the same position with the same type. 9069 /// 9070 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { 9071 #ifndef NDEBUG 9072 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 9073 assert((!From->hasAnyUseOfValue(i) || 9074 From->getValueType(i) == To->getValueType(i)) && 9075 "Cannot use this version of ReplaceAllUsesWith!"); 9076 #endif 9077 9078 // Handle the trivial case. 9079 if (From == To) 9080 return; 9081 9082 // Preserve Debug Info. Only do this if there's a use. 9083 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 9084 if (From->hasAnyUseOfValue(i)) { 9085 assert((i < To->getNumValues()) && "Invalid To location"); 9086 transferDbgValues(SDValue(From, i), SDValue(To, i)); 9087 } 9088 9089 // Iterate over just the existing users of From. See the comments in 9090 // the ReplaceAllUsesWith above. 9091 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 9092 RAUWUpdateListener Listener(*this, UI, UE); 9093 while (UI != UE) { 9094 SDNode *User = *UI; 9095 9096 // This node is about to morph, remove its old self from the CSE maps. 9097 RemoveNodeFromCSEMaps(User); 9098 9099 // A user can appear in a use list multiple times, and when this 9100 // happens the uses are usually next to each other in the list. 9101 // To help reduce the number of CSE recomputations, process all 9102 // the uses of this user that we can find this way. 9103 do { 9104 SDUse &Use = UI.getUse(); 9105 ++UI; 9106 Use.setNode(To); 9107 if (To->isDivergent() != From->isDivergent()) 9108 updateDivergence(User); 9109 } while (UI != UE && *UI == User); 9110 9111 // Now that we have modified User, add it back to the CSE maps. If it 9112 // already exists there, recursively merge the results together. 9113 AddModifiedNodeToCSEMaps(User); 9114 } 9115 9116 // If we just RAUW'd the root, take note. 9117 if (From == getRoot().getNode()) 9118 setRoot(SDValue(To, getRoot().getResNo())); 9119 } 9120 9121 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 9122 /// This can cause recursive merging of nodes in the DAG. 9123 /// 9124 /// This version can replace From with any result values. To must match the 9125 /// number and types of values returned by From. 9126 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { 9127 if (From->getNumValues() == 1) // Handle the simple case efficiently. 9128 return ReplaceAllUsesWith(SDValue(From, 0), To[0]); 9129 9130 // Preserve Debug Info. 9131 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 9132 transferDbgValues(SDValue(From, i), To[i]); 9133 9134 // Iterate over just the existing users of From. See the comments in 9135 // the ReplaceAllUsesWith above. 9136 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 9137 RAUWUpdateListener Listener(*this, UI, UE); 9138 while (UI != UE) { 9139 SDNode *User = *UI; 9140 9141 // This node is about to morph, remove its old self from the CSE maps. 9142 RemoveNodeFromCSEMaps(User); 9143 9144 // A user can appear in a use list multiple times, and when this happens the 9145 // uses are usually next to each other in the list. To help reduce the 9146 // number of CSE and divergence recomputations, process all the uses of this 9147 // user that we can find this way. 9148 bool To_IsDivergent = false; 9149 do { 9150 SDUse &Use = UI.getUse(); 9151 const SDValue &ToOp = To[Use.getResNo()]; 9152 ++UI; 9153 Use.set(ToOp); 9154 To_IsDivergent |= ToOp->isDivergent(); 9155 } while (UI != UE && *UI == User); 9156 9157 if (To_IsDivergent != From->isDivergent()) 9158 updateDivergence(User); 9159 9160 // Now that we have modified User, add it back to the CSE maps. If it 9161 // already exists there, recursively merge the results together. 9162 AddModifiedNodeToCSEMaps(User); 9163 } 9164 9165 // If we just RAUW'd the root, take note. 9166 if (From == getRoot().getNode()) 9167 setRoot(SDValue(To[getRoot().getResNo()])); 9168 } 9169 9170 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 9171 /// uses of other values produced by From.getNode() alone. The Deleted 9172 /// vector is handled the same way as for ReplaceAllUsesWith. 9173 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ 9174 // Handle the really simple, really trivial case efficiently. 9175 if (From == To) return; 9176 9177 // Handle the simple, trivial, case efficiently. 9178 if (From.getNode()->getNumValues() == 1) { 9179 ReplaceAllUsesWith(From, To); 9180 return; 9181 } 9182 9183 // Preserve Debug Info. 9184 transferDbgValues(From, To); 9185 9186 // Iterate over just the existing users of From. See the comments in 9187 // the ReplaceAllUsesWith above. 9188 SDNode::use_iterator UI = From.getNode()->use_begin(), 9189 UE = From.getNode()->use_end(); 9190 RAUWUpdateListener Listener(*this, UI, UE); 9191 while (UI != UE) { 9192 SDNode *User = *UI; 9193 bool UserRemovedFromCSEMaps = false; 9194 9195 // A user can appear in a use list multiple times, and when this 9196 // happens the uses are usually next to each other in the list. 9197 // To help reduce the number of CSE recomputations, process all 9198 // the uses of this user that we can find this way. 9199 do { 9200 SDUse &Use = UI.getUse(); 9201 9202 // Skip uses of different values from the same node. 9203 if (Use.getResNo() != From.getResNo()) { 9204 ++UI; 9205 continue; 9206 } 9207 9208 // If this node hasn't been modified yet, it's still in the CSE maps, 9209 // so remove its old self from the CSE maps. 9210 if (!UserRemovedFromCSEMaps) { 9211 RemoveNodeFromCSEMaps(User); 9212 UserRemovedFromCSEMaps = true; 9213 } 9214 9215 ++UI; 9216 Use.set(To); 9217 if (To->isDivergent() != From->isDivergent()) 9218 updateDivergence(User); 9219 } while (UI != UE && *UI == User); 9220 // We are iterating over all uses of the From node, so if a use 9221 // doesn't use the specific value, no changes are made. 9222 if (!UserRemovedFromCSEMaps) 9223 continue; 9224 9225 // Now that we have modified User, add it back to the CSE maps. If it 9226 // already exists there, recursively merge the results together. 9227 AddModifiedNodeToCSEMaps(User); 9228 } 9229 9230 // If we just RAUW'd the root, take note. 9231 if (From == getRoot()) 9232 setRoot(To); 9233 } 9234 9235 namespace { 9236 9237 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 9238 /// to record information about a use. 9239 struct UseMemo { 9240 SDNode *User; 9241 unsigned Index; 9242 SDUse *Use; 9243 }; 9244 9245 /// operator< - Sort Memos by User. 9246 bool operator<(const UseMemo &L, const UseMemo &R) { 9247 return (intptr_t)L.User < (intptr_t)R.User; 9248 } 9249 9250 } // end anonymous namespace 9251 9252 bool SelectionDAG::calculateDivergence(SDNode *N) { 9253 if (TLI->isSDNodeAlwaysUniform(N)) { 9254 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) && 9255 "Conflicting divergence information!"); 9256 return false; 9257 } 9258 if (TLI->isSDNodeSourceOfDivergence(N, FLI, DA)) 9259 return true; 9260 for (auto &Op : N->ops()) { 9261 if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent()) 9262 return true; 9263 } 9264 return false; 9265 } 9266 9267 void SelectionDAG::updateDivergence(SDNode *N) { 9268 SmallVector<SDNode *, 16> Worklist(1, N); 9269 do { 9270 N = Worklist.pop_back_val(); 9271 bool IsDivergent = calculateDivergence(N); 9272 if (N->SDNodeBits.IsDivergent != IsDivergent) { 9273 N->SDNodeBits.IsDivergent = IsDivergent; 9274 llvm::append_range(Worklist, N->uses()); 9275 } 9276 } while (!Worklist.empty()); 9277 } 9278 9279 void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) { 9280 DenseMap<SDNode *, unsigned> Degree; 9281 Order.reserve(AllNodes.size()); 9282 for (auto &N : allnodes()) { 9283 unsigned NOps = N.getNumOperands(); 9284 Degree[&N] = NOps; 9285 if (0 == NOps) 9286 Order.push_back(&N); 9287 } 9288 for (size_t I = 0; I != Order.size(); ++I) { 9289 SDNode *N = Order[I]; 9290 for (auto U : N->uses()) { 9291 unsigned &UnsortedOps = Degree[U]; 9292 if (0 == --UnsortedOps) 9293 Order.push_back(U); 9294 } 9295 } 9296 } 9297 9298 #ifndef NDEBUG 9299 void SelectionDAG::VerifyDAGDiverence() { 9300 std::vector<SDNode *> TopoOrder; 9301 CreateTopologicalOrder(TopoOrder); 9302 for (auto *N : TopoOrder) { 9303 assert(calculateDivergence(N) == N->isDivergent() && 9304 "Divergence bit inconsistency detected"); 9305 } 9306 } 9307 #endif 9308 9309 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 9310 /// uses of other values produced by From.getNode() alone. The same value 9311 /// may appear in both the From and To list. The Deleted vector is 9312 /// handled the same way as for ReplaceAllUsesWith. 9313 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 9314 const SDValue *To, 9315 unsigned Num){ 9316 // Handle the simple, trivial case efficiently. 9317 if (Num == 1) 9318 return ReplaceAllUsesOfValueWith(*From, *To); 9319 9320 transferDbgValues(*From, *To); 9321 9322 // Read up all the uses and make records of them. This helps 9323 // processing new uses that are introduced during the 9324 // replacement process. 9325 SmallVector<UseMemo, 4> Uses; 9326 for (unsigned i = 0; i != Num; ++i) { 9327 unsigned FromResNo = From[i].getResNo(); 9328 SDNode *FromNode = From[i].getNode(); 9329 for (SDNode::use_iterator UI = FromNode->use_begin(), 9330 E = FromNode->use_end(); UI != E; ++UI) { 9331 SDUse &Use = UI.getUse(); 9332 if (Use.getResNo() == FromResNo) { 9333 UseMemo Memo = { *UI, i, &Use }; 9334 Uses.push_back(Memo); 9335 } 9336 } 9337 } 9338 9339 // Sort the uses, so that all the uses from a given User are together. 9340 llvm::sort(Uses); 9341 9342 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 9343 UseIndex != UseIndexEnd; ) { 9344 // We know that this user uses some value of From. If it is the right 9345 // value, update it. 9346 SDNode *User = Uses[UseIndex].User; 9347 9348 // This node is about to morph, remove its old self from the CSE maps. 9349 RemoveNodeFromCSEMaps(User); 9350 9351 // The Uses array is sorted, so all the uses for a given User 9352 // are next to each other in the list. 9353 // To help reduce the number of CSE recomputations, process all 9354 // the uses of this user that we can find this way. 9355 do { 9356 unsigned i = Uses[UseIndex].Index; 9357 SDUse &Use = *Uses[UseIndex].Use; 9358 ++UseIndex; 9359 9360 Use.set(To[i]); 9361 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 9362 9363 // Now that we have modified User, add it back to the CSE maps. If it 9364 // already exists there, recursively merge the results together. 9365 AddModifiedNodeToCSEMaps(User); 9366 } 9367 } 9368 9369 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 9370 /// based on their topological order. It returns the maximum id and a vector 9371 /// of the SDNodes* in assigned order by reference. 9372 unsigned SelectionDAG::AssignTopologicalOrder() { 9373 unsigned DAGSize = 0; 9374 9375 // SortedPos tracks the progress of the algorithm. Nodes before it are 9376 // sorted, nodes after it are unsorted. When the algorithm completes 9377 // it is at the end of the list. 9378 allnodes_iterator SortedPos = allnodes_begin(); 9379 9380 // Visit all the nodes. Move nodes with no operands to the front of 9381 // the list immediately. Annotate nodes that do have operands with their 9382 // operand count. Before we do this, the Node Id fields of the nodes 9383 // may contain arbitrary values. After, the Node Id fields for nodes 9384 // before SortedPos will contain the topological sort index, and the 9385 // Node Id fields for nodes At SortedPos and after will contain the 9386 // count of outstanding operands. 9387 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 9388 SDNode *N = &*I++; 9389 checkForCycles(N, this); 9390 unsigned Degree = N->getNumOperands(); 9391 if (Degree == 0) { 9392 // A node with no uses, add it to the result array immediately. 9393 N->setNodeId(DAGSize++); 9394 allnodes_iterator Q(N); 9395 if (Q != SortedPos) 9396 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 9397 assert(SortedPos != AllNodes.end() && "Overran node list"); 9398 ++SortedPos; 9399 } else { 9400 // Temporarily use the Node Id as scratch space for the degree count. 9401 N->setNodeId(Degree); 9402 } 9403 } 9404 9405 // Visit all the nodes. As we iterate, move nodes into sorted order, 9406 // such that by the time the end is reached all nodes will be sorted. 9407 for (SDNode &Node : allnodes()) { 9408 SDNode *N = &Node; 9409 checkForCycles(N, this); 9410 // N is in sorted position, so all its uses have one less operand 9411 // that needs to be sorted. 9412 for (SDNode *P : N->uses()) { 9413 unsigned Degree = P->getNodeId(); 9414 assert(Degree != 0 && "Invalid node degree"); 9415 --Degree; 9416 if (Degree == 0) { 9417 // All of P's operands are sorted, so P may sorted now. 9418 P->setNodeId(DAGSize++); 9419 if (P->getIterator() != SortedPos) 9420 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 9421 assert(SortedPos != AllNodes.end() && "Overran node list"); 9422 ++SortedPos; 9423 } else { 9424 // Update P's outstanding operand count. 9425 P->setNodeId(Degree); 9426 } 9427 } 9428 if (Node.getIterator() == SortedPos) { 9429 #ifndef NDEBUG 9430 allnodes_iterator I(N); 9431 SDNode *S = &*++I; 9432 dbgs() << "Overran sorted position:\n"; 9433 S->dumprFull(this); dbgs() << "\n"; 9434 dbgs() << "Checking if this is due to cycles\n"; 9435 checkForCycles(this, true); 9436 #endif 9437 llvm_unreachable(nullptr); 9438 } 9439 } 9440 9441 assert(SortedPos == AllNodes.end() && 9442 "Topological sort incomplete!"); 9443 assert(AllNodes.front().getOpcode() == ISD::EntryToken && 9444 "First node in topological sort is not the entry token!"); 9445 assert(AllNodes.front().getNodeId() == 0 && 9446 "First node in topological sort has non-zero id!"); 9447 assert(AllNodes.front().getNumOperands() == 0 && 9448 "First node in topological sort has operands!"); 9449 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 9450 "Last node in topologic sort has unexpected id!"); 9451 assert(AllNodes.back().use_empty() && 9452 "Last node in topologic sort has users!"); 9453 assert(DAGSize == allnodes_size() && "Node count mismatch!"); 9454 return DAGSize; 9455 } 9456 9457 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 9458 /// value is produced by SD. 9459 void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) { 9460 for (SDNode *SD : DB->getSDNodes()) { 9461 if (!SD) 9462 continue; 9463 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue()); 9464 SD->setHasDebugValue(true); 9465 } 9466 DbgInfo->add(DB, isParameter); 9467 } 9468 9469 void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); } 9470 9471 SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain, 9472 SDValue NewMemOpChain) { 9473 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node"); 9474 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT"); 9475 // The new memory operation must have the same position as the old load in 9476 // terms of memory dependency. Create a TokenFactor for the old load and new 9477 // memory operation and update uses of the old load's output chain to use that 9478 // TokenFactor. 9479 if (OldChain == NewMemOpChain || OldChain.use_empty()) 9480 return NewMemOpChain; 9481 9482 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other, 9483 OldChain, NewMemOpChain); 9484 ReplaceAllUsesOfValueWith(OldChain, TokenFactor); 9485 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain); 9486 return TokenFactor; 9487 } 9488 9489 SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, 9490 SDValue NewMemOp) { 9491 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node"); 9492 SDValue OldChain = SDValue(OldLoad, 1); 9493 SDValue NewMemOpChain = NewMemOp.getValue(1); 9494 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain); 9495 } 9496 9497 SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op, 9498 Function **OutFunction) { 9499 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol"); 9500 9501 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 9502 auto *Module = MF->getFunction().getParent(); 9503 auto *Function = Module->getFunction(Symbol); 9504 9505 if (OutFunction != nullptr) 9506 *OutFunction = Function; 9507 9508 if (Function != nullptr) { 9509 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace()); 9510 return getGlobalAddress(Function, SDLoc(Op), PtrTy); 9511 } 9512 9513 std::string ErrorStr; 9514 raw_string_ostream ErrorFormatter(ErrorStr); 9515 9516 ErrorFormatter << "Undefined external symbol "; 9517 ErrorFormatter << '"' << Symbol << '"'; 9518 ErrorFormatter.flush(); 9519 9520 report_fatal_error(ErrorStr); 9521 } 9522 9523 //===----------------------------------------------------------------------===// 9524 // SDNode Class 9525 //===----------------------------------------------------------------------===// 9526 9527 bool llvm::isNullConstant(SDValue V) { 9528 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 9529 return Const != nullptr && Const->isNullValue(); 9530 } 9531 9532 bool llvm::isNullFPConstant(SDValue V) { 9533 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V); 9534 return Const != nullptr && Const->isZero() && !Const->isNegative(); 9535 } 9536 9537 bool llvm::isAllOnesConstant(SDValue V) { 9538 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 9539 return Const != nullptr && Const->isAllOnesValue(); 9540 } 9541 9542 bool llvm::isOneConstant(SDValue V) { 9543 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 9544 return Const != nullptr && Const->isOne(); 9545 } 9546 9547 SDValue llvm::peekThroughBitcasts(SDValue V) { 9548 while (V.getOpcode() == ISD::BITCAST) 9549 V = V.getOperand(0); 9550 return V; 9551 } 9552 9553 SDValue llvm::peekThroughOneUseBitcasts(SDValue V) { 9554 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse()) 9555 V = V.getOperand(0); 9556 return V; 9557 } 9558 9559 SDValue llvm::peekThroughExtractSubvectors(SDValue V) { 9560 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR) 9561 V = V.getOperand(0); 9562 return V; 9563 } 9564 9565 bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) { 9566 if (V.getOpcode() != ISD::XOR) 9567 return false; 9568 V = peekThroughBitcasts(V.getOperand(1)); 9569 unsigned NumBits = V.getScalarValueSizeInBits(); 9570 ConstantSDNode *C = 9571 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true); 9572 return C && (C->getAPIntValue().countTrailingOnes() >= NumBits); 9573 } 9574 9575 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs, 9576 bool AllowTruncation) { 9577 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) 9578 return CN; 9579 9580 // SplatVectors can truncate their operands. Ignore that case here unless 9581 // AllowTruncation is set. 9582 if (N->getOpcode() == ISD::SPLAT_VECTOR) { 9583 EVT VecEltVT = N->getValueType(0).getVectorElementType(); 9584 if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 9585 EVT CVT = CN->getValueType(0); 9586 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension"); 9587 if (AllowTruncation || CVT == VecEltVT) 9588 return CN; 9589 } 9590 } 9591 9592 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { 9593 BitVector UndefElements; 9594 ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements); 9595 9596 // BuildVectors can truncate their operands. Ignore that case here unless 9597 // AllowTruncation is set. 9598 if (CN && (UndefElements.none() || AllowUndefs)) { 9599 EVT CVT = CN->getValueType(0); 9600 EVT NSVT = N.getValueType().getScalarType(); 9601 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension"); 9602 if (AllowTruncation || (CVT == NSVT)) 9603 return CN; 9604 } 9605 } 9606 9607 return nullptr; 9608 } 9609 9610 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts, 9611 bool AllowUndefs, 9612 bool AllowTruncation) { 9613 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) 9614 return CN; 9615 9616 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { 9617 BitVector UndefElements; 9618 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements); 9619 9620 // BuildVectors can truncate their operands. Ignore that case here unless 9621 // AllowTruncation is set. 9622 if (CN && (UndefElements.none() || AllowUndefs)) { 9623 EVT CVT = CN->getValueType(0); 9624 EVT NSVT = N.getValueType().getScalarType(); 9625 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension"); 9626 if (AllowTruncation || (CVT == NSVT)) 9627 return CN; 9628 } 9629 } 9630 9631 return nullptr; 9632 } 9633 9634 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) { 9635 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N)) 9636 return CN; 9637 9638 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { 9639 BitVector UndefElements; 9640 ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements); 9641 if (CN && (UndefElements.none() || AllowUndefs)) 9642 return CN; 9643 } 9644 9645 if (N.getOpcode() == ISD::SPLAT_VECTOR) 9646 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0))) 9647 return CN; 9648 9649 return nullptr; 9650 } 9651 9652 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, 9653 const APInt &DemandedElts, 9654 bool AllowUndefs) { 9655 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N)) 9656 return CN; 9657 9658 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { 9659 BitVector UndefElements; 9660 ConstantFPSDNode *CN = 9661 BV->getConstantFPSplatNode(DemandedElts, &UndefElements); 9662 if (CN && (UndefElements.none() || AllowUndefs)) 9663 return CN; 9664 } 9665 9666 return nullptr; 9667 } 9668 9669 bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) { 9670 // TODO: may want to use peekThroughBitcast() here. 9671 ConstantSDNode *C = 9672 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true); 9673 return C && C->isNullValue(); 9674 } 9675 9676 bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) { 9677 // TODO: may want to use peekThroughBitcast() here. 9678 unsigned BitWidth = N.getScalarValueSizeInBits(); 9679 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs); 9680 return C && C->isOne() && C->getValueSizeInBits(0) == BitWidth; 9681 } 9682 9683 bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) { 9684 N = peekThroughBitcasts(N); 9685 unsigned BitWidth = N.getScalarValueSizeInBits(); 9686 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs); 9687 return C && C->isAllOnesValue() && C->getValueSizeInBits(0) == BitWidth; 9688 } 9689 9690 HandleSDNode::~HandleSDNode() { 9691 DropOperands(); 9692 } 9693 9694 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, 9695 const DebugLoc &DL, 9696 const GlobalValue *GA, EVT VT, 9697 int64_t o, unsigned TF) 9698 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 9699 TheGlobal = GA; 9700 } 9701 9702 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, 9703 EVT VT, unsigned SrcAS, 9704 unsigned DestAS) 9705 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)), 9706 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} 9707 9708 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, 9709 SDVTList VTs, EVT memvt, MachineMemOperand *mmo) 9710 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { 9711 MemSDNodeBits.IsVolatile = MMO->isVolatile(); 9712 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); 9713 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable(); 9714 MemSDNodeBits.IsInvariant = MMO->isInvariant(); 9715 9716 // We check here that the size of the memory operand fits within the size of 9717 // the MMO. This is because the MMO might indicate only a possible address 9718 // range instead of specifying the affected memory addresses precisely. 9719 // TODO: Make MachineMemOperands aware of scalable vectors. 9720 assert(memvt.getStoreSize().getKnownMinSize() <= MMO->getSize() && 9721 "Size mismatch!"); 9722 } 9723 9724 /// Profile - Gather unique data for the node. 9725 /// 9726 void SDNode::Profile(FoldingSetNodeID &ID) const { 9727 AddNodeIDNode(ID, this); 9728 } 9729 9730 namespace { 9731 9732 struct EVTArray { 9733 std::vector<EVT> VTs; 9734 9735 EVTArray() { 9736 VTs.reserve(MVT::VALUETYPE_SIZE); 9737 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i) 9738 VTs.push_back(MVT((MVT::SimpleValueType)i)); 9739 } 9740 }; 9741 9742 } // end anonymous namespace 9743 9744 static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs; 9745 static ManagedStatic<EVTArray> SimpleVTArray; 9746 static ManagedStatic<sys::SmartMutex<true>> VTMutex; 9747 9748 /// getValueTypeList - Return a pointer to the specified value type. 9749 /// 9750 const EVT *SDNode::getValueTypeList(EVT VT) { 9751 if (VT.isExtended()) { 9752 sys::SmartScopedLock<true> Lock(*VTMutex); 9753 return &(*EVTs->insert(VT).first); 9754 } 9755 assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!"); 9756 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 9757 } 9758 9759 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 9760 /// indicated value. This method ignores uses of other values defined by this 9761 /// operation. 9762 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 9763 assert(Value < getNumValues() && "Bad value!"); 9764 9765 // TODO: Only iterate over uses of a given value of the node 9766 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 9767 if (UI.getUse().getResNo() == Value) { 9768 if (NUses == 0) 9769 return false; 9770 --NUses; 9771 } 9772 } 9773 9774 // Found exactly the right number of uses? 9775 return NUses == 0; 9776 } 9777 9778 /// hasAnyUseOfValue - Return true if there are any use of the indicated 9779 /// value. This method ignores uses of other values defined by this operation. 9780 bool SDNode::hasAnyUseOfValue(unsigned Value) const { 9781 assert(Value < getNumValues() && "Bad value!"); 9782 9783 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 9784 if (UI.getUse().getResNo() == Value) 9785 return true; 9786 9787 return false; 9788 } 9789 9790 /// isOnlyUserOf - Return true if this node is the only use of N. 9791 bool SDNode::isOnlyUserOf(const SDNode *N) const { 9792 bool Seen = false; 9793 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 9794 SDNode *User = *I; 9795 if (User == this) 9796 Seen = true; 9797 else 9798 return false; 9799 } 9800 9801 return Seen; 9802 } 9803 9804 /// Return true if the only users of N are contained in Nodes. 9805 bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { 9806 bool Seen = false; 9807 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 9808 SDNode *User = *I; 9809 if (llvm::is_contained(Nodes, User)) 9810 Seen = true; 9811 else 9812 return false; 9813 } 9814 9815 return Seen; 9816 } 9817 9818 /// isOperand - Return true if this node is an operand of N. 9819 bool SDValue::isOperandOf(const SDNode *N) const { 9820 return is_contained(N->op_values(), *this); 9821 } 9822 9823 bool SDNode::isOperandOf(const SDNode *N) const { 9824 return any_of(N->op_values(), 9825 [this](SDValue Op) { return this == Op.getNode(); }); 9826 } 9827 9828 /// reachesChainWithoutSideEffects - Return true if this operand (which must 9829 /// be a chain) reaches the specified operand without crossing any 9830 /// side-effecting instructions on any chain path. In practice, this looks 9831 /// through token factors and non-volatile loads. In order to remain efficient, 9832 /// this only looks a couple of nodes in, it does not do an exhaustive search. 9833 /// 9834 /// Note that we only need to examine chains when we're searching for 9835 /// side-effects; SelectionDAG requires that all side-effects are represented 9836 /// by chains, even if another operand would force a specific ordering. This 9837 /// constraint is necessary to allow transformations like splitting loads. 9838 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 9839 unsigned Depth) const { 9840 if (*this == Dest) return true; 9841 9842 // Don't search too deeply, we just want to be able to see through 9843 // TokenFactor's etc. 9844 if (Depth == 0) return false; 9845 9846 // If this is a token factor, all inputs to the TF happen in parallel. 9847 if (getOpcode() == ISD::TokenFactor) { 9848 // First, try a shallow search. 9849 if (is_contained((*this)->ops(), Dest)) { 9850 // We found the chain we want as an operand of this TokenFactor. 9851 // Essentially, we reach the chain without side-effects if we could 9852 // serialize the TokenFactor into a simple chain of operations with 9853 // Dest as the last operation. This is automatically true if the 9854 // chain has one use: there are no other ordering constraints. 9855 // If the chain has more than one use, we give up: some other 9856 // use of Dest might force a side-effect between Dest and the current 9857 // node. 9858 if (Dest.hasOneUse()) 9859 return true; 9860 } 9861 // Next, try a deep search: check whether every operand of the TokenFactor 9862 // reaches Dest. 9863 return llvm::all_of((*this)->ops(), [=](SDValue Op) { 9864 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1); 9865 }); 9866 } 9867 9868 // Loads don't have side effects, look through them. 9869 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 9870 if (Ld->isUnordered()) 9871 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 9872 } 9873 return false; 9874 } 9875 9876 bool SDNode::hasPredecessor(const SDNode *N) const { 9877 SmallPtrSet<const SDNode *, 32> Visited; 9878 SmallVector<const SDNode *, 16> Worklist; 9879 Worklist.push_back(this); 9880 return hasPredecessorHelper(N, Visited, Worklist); 9881 } 9882 9883 void SDNode::intersectFlagsWith(const SDNodeFlags Flags) { 9884 this->Flags.intersectWith(Flags); 9885 } 9886 9887 SDValue 9888 SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, 9889 ArrayRef<ISD::NodeType> CandidateBinOps, 9890 bool AllowPartials) { 9891 // The pattern must end in an extract from index 0. 9892 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT || 9893 !isNullConstant(Extract->getOperand(1))) 9894 return SDValue(); 9895 9896 // Match against one of the candidate binary ops. 9897 SDValue Op = Extract->getOperand(0); 9898 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) { 9899 return Op.getOpcode() == unsigned(BinOp); 9900 })) 9901 return SDValue(); 9902 9903 // Floating-point reductions may require relaxed constraints on the final step 9904 // of the reduction because they may reorder intermediate operations. 9905 unsigned CandidateBinOp = Op.getOpcode(); 9906 if (Op.getValueType().isFloatingPoint()) { 9907 SDNodeFlags Flags = Op->getFlags(); 9908 switch (CandidateBinOp) { 9909 case ISD::FADD: 9910 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation()) 9911 return SDValue(); 9912 break; 9913 default: 9914 llvm_unreachable("Unhandled FP opcode for binop reduction"); 9915 } 9916 } 9917 9918 // Matching failed - attempt to see if we did enough stages that a partial 9919 // reduction from a subvector is possible. 9920 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) { 9921 if (!AllowPartials || !Op) 9922 return SDValue(); 9923 EVT OpVT = Op.getValueType(); 9924 EVT OpSVT = OpVT.getScalarType(); 9925 EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts); 9926 if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0)) 9927 return SDValue(); 9928 BinOp = (ISD::NodeType)CandidateBinOp; 9929 return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op, 9930 getVectorIdxConstant(0, SDLoc(Op))); 9931 }; 9932 9933 // At each stage, we're looking for something that looks like: 9934 // %s = shufflevector <8 x i32> %op, <8 x i32> undef, 9935 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, 9936 // i32 undef, i32 undef, i32 undef, i32 undef> 9937 // %a = binop <8 x i32> %op, %s 9938 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid, 9939 // we expect something like: 9940 // <4,5,6,7,u,u,u,u> 9941 // <2,3,u,u,u,u,u,u> 9942 // <1,u,u,u,u,u,u,u> 9943 // While a partial reduction match would be: 9944 // <2,3,u,u,u,u,u,u> 9945 // <1,u,u,u,u,u,u,u> 9946 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements()); 9947 SDValue PrevOp; 9948 for (unsigned i = 0; i < Stages; ++i) { 9949 unsigned MaskEnd = (1 << i); 9950 9951 if (Op.getOpcode() != CandidateBinOp) 9952 return PartialReduction(PrevOp, MaskEnd); 9953 9954 SDValue Op0 = Op.getOperand(0); 9955 SDValue Op1 = Op.getOperand(1); 9956 9957 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0); 9958 if (Shuffle) { 9959 Op = Op1; 9960 } else { 9961 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1); 9962 Op = Op0; 9963 } 9964 9965 // The first operand of the shuffle should be the same as the other operand 9966 // of the binop. 9967 if (!Shuffle || Shuffle->getOperand(0) != Op) 9968 return PartialReduction(PrevOp, MaskEnd); 9969 9970 // Verify the shuffle has the expected (at this stage of the pyramid) mask. 9971 for (int Index = 0; Index < (int)MaskEnd; ++Index) 9972 if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index)) 9973 return PartialReduction(PrevOp, MaskEnd); 9974 9975 PrevOp = Op; 9976 } 9977 9978 // Handle subvector reductions, which tend to appear after the shuffle 9979 // reduction stages. 9980 while (Op.getOpcode() == CandidateBinOp) { 9981 unsigned NumElts = Op.getValueType().getVectorNumElements(); 9982 SDValue Op0 = Op.getOperand(0); 9983 SDValue Op1 = Op.getOperand(1); 9984 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR || 9985 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR || 9986 Op0.getOperand(0) != Op1.getOperand(0)) 9987 break; 9988 SDValue Src = Op0.getOperand(0); 9989 unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); 9990 if (NumSrcElts != (2 * NumElts)) 9991 break; 9992 if (!(Op0.getConstantOperandAPInt(1) == 0 && 9993 Op1.getConstantOperandAPInt(1) == NumElts) && 9994 !(Op1.getConstantOperandAPInt(1) == 0 && 9995 Op0.getConstantOperandAPInt(1) == NumElts)) 9996 break; 9997 Op = Src; 9998 } 9999 10000 BinOp = (ISD::NodeType)CandidateBinOp; 10001 return Op; 10002 } 10003 10004 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 10005 assert(N->getNumValues() == 1 && 10006 "Can't unroll a vector with multiple results!"); 10007 10008 EVT VT = N->getValueType(0); 10009 unsigned NE = VT.getVectorNumElements(); 10010 EVT EltVT = VT.getVectorElementType(); 10011 SDLoc dl(N); 10012 10013 SmallVector<SDValue, 8> Scalars; 10014 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 10015 10016 // If ResNE is 0, fully unroll the vector op. 10017 if (ResNE == 0) 10018 ResNE = NE; 10019 else if (NE > ResNE) 10020 NE = ResNE; 10021 10022 unsigned i; 10023 for (i= 0; i != NE; ++i) { 10024 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 10025 SDValue Operand = N->getOperand(j); 10026 EVT OperandVT = Operand.getValueType(); 10027 if (OperandVT.isVector()) { 10028 // A vector operand; extract a single element. 10029 EVT OperandEltVT = OperandVT.getVectorElementType(); 10030 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, 10031 Operand, getVectorIdxConstant(i, dl)); 10032 } else { 10033 // A scalar operand; just use it as is. 10034 Operands[j] = Operand; 10035 } 10036 } 10037 10038 switch (N->getOpcode()) { 10039 default: { 10040 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands, 10041 N->getFlags())); 10042 break; 10043 } 10044 case ISD::VSELECT: 10045 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands)); 10046 break; 10047 case ISD::SHL: 10048 case ISD::SRA: 10049 case ISD::SRL: 10050 case ISD::ROTL: 10051 case ISD::ROTR: 10052 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 10053 getShiftAmountOperand(Operands[0].getValueType(), 10054 Operands[1]))); 10055 break; 10056 case ISD::SIGN_EXTEND_INREG: { 10057 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 10058 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 10059 Operands[0], 10060 getValueType(ExtVT))); 10061 } 10062 } 10063 } 10064 10065 for (; i < ResNE; ++i) 10066 Scalars.push_back(getUNDEF(EltVT)); 10067 10068 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE); 10069 return getBuildVector(VecVT, dl, Scalars); 10070 } 10071 10072 std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp( 10073 SDNode *N, unsigned ResNE) { 10074 unsigned Opcode = N->getOpcode(); 10075 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO || 10076 Opcode == ISD::USUBO || Opcode == ISD::SSUBO || 10077 Opcode == ISD::UMULO || Opcode == ISD::SMULO) && 10078 "Expected an overflow opcode"); 10079 10080 EVT ResVT = N->getValueType(0); 10081 EVT OvVT = N->getValueType(1); 10082 EVT ResEltVT = ResVT.getVectorElementType(); 10083 EVT OvEltVT = OvVT.getVectorElementType(); 10084 SDLoc dl(N); 10085 10086 // If ResNE is 0, fully unroll the vector op. 10087 unsigned NE = ResVT.getVectorNumElements(); 10088 if (ResNE == 0) 10089 ResNE = NE; 10090 else if (NE > ResNE) 10091 NE = ResNE; 10092 10093 SmallVector<SDValue, 8> LHSScalars; 10094 SmallVector<SDValue, 8> RHSScalars; 10095 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE); 10096 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE); 10097 10098 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT); 10099 SDVTList VTs = getVTList(ResEltVT, SVT); 10100 SmallVector<SDValue, 8> ResScalars; 10101 SmallVector<SDValue, 8> OvScalars; 10102 for (unsigned i = 0; i < NE; ++i) { 10103 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]); 10104 SDValue Ov = 10105 getSelect(dl, OvEltVT, Res.getValue(1), 10106 getBoolConstant(true, dl, OvEltVT, ResVT), 10107 getConstant(0, dl, OvEltVT)); 10108 10109 ResScalars.push_back(Res); 10110 OvScalars.push_back(Ov); 10111 } 10112 10113 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT)); 10114 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT)); 10115 10116 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE); 10117 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE); 10118 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars), 10119 getBuildVector(NewOvVT, dl, OvScalars)); 10120 } 10121 10122 bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, 10123 LoadSDNode *Base, 10124 unsigned Bytes, 10125 int Dist) const { 10126 if (LD->isVolatile() || Base->isVolatile()) 10127 return false; 10128 // TODO: probably too restrictive for atomics, revisit 10129 if (!LD->isSimple()) 10130 return false; 10131 if (LD->isIndexed() || Base->isIndexed()) 10132 return false; 10133 if (LD->getChain() != Base->getChain()) 10134 return false; 10135 EVT VT = LD->getValueType(0); 10136 if (VT.getSizeInBits() / 8 != Bytes) 10137 return false; 10138 10139 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this); 10140 auto LocDecomp = BaseIndexOffset::match(LD, *this); 10141 10142 int64_t Offset = 0; 10143 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset)) 10144 return (Dist * Bytes == Offset); 10145 return false; 10146 } 10147 10148 /// InferPtrAlignment - Infer alignment of a load / store address. Return None 10149 /// if it cannot be inferred. 10150 MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const { 10151 // If this is a GlobalAddress + cst, return the alignment. 10152 const GlobalValue *GV = nullptr; 10153 int64_t GVOffset = 0; 10154 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 10155 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); 10156 KnownBits Known(PtrWidth); 10157 llvm::computeKnownBits(GV, Known, getDataLayout()); 10158 unsigned AlignBits = Known.countMinTrailingZeros(); 10159 if (AlignBits) 10160 return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset); 10161 } 10162 10163 // If this is a direct reference to a stack slot, use information about the 10164 // stack slot's alignment. 10165 int FrameIdx = INT_MIN; 10166 int64_t FrameOffset = 0; 10167 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 10168 FrameIdx = FI->getIndex(); 10169 } else if (isBaseWithConstantOffset(Ptr) && 10170 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 10171 // Handle FI+Cst 10172 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 10173 FrameOffset = Ptr.getConstantOperandVal(1); 10174 } 10175 10176 if (FrameIdx != INT_MIN) { 10177 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); 10178 return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset); 10179 } 10180 10181 return None; 10182 } 10183 10184 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type 10185 /// which is split (or expanded) into two not necessarily identical pieces. 10186 std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const { 10187 // Currently all types are split in half. 10188 EVT LoVT, HiVT; 10189 if (!VT.isVector()) 10190 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT); 10191 else 10192 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext()); 10193 10194 return std::make_pair(LoVT, HiVT); 10195 } 10196 10197 /// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a 10198 /// type, dependent on an enveloping VT that has been split into two identical 10199 /// pieces. Sets the HiIsEmpty flag when hi type has zero storage size. 10200 std::pair<EVT, EVT> 10201 SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, 10202 bool *HiIsEmpty) const { 10203 EVT EltTp = VT.getVectorElementType(); 10204 // Examples: 10205 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty) 10206 // custom VL=9 with enveloping VL=8/8 yields 8/1 10207 // custom VL=10 with enveloping VL=8/8 yields 8/2 10208 // etc. 10209 ElementCount VTNumElts = VT.getVectorElementCount(); 10210 ElementCount EnvNumElts = EnvVT.getVectorElementCount(); 10211 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() && 10212 "Mixing fixed width and scalable vectors when enveloping a type"); 10213 EVT LoVT, HiVT; 10214 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) { 10215 LoVT = EnvVT; 10216 HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts); 10217 *HiIsEmpty = false; 10218 } else { 10219 // Flag that hi type has zero storage size, but return split envelop type 10220 // (this would be easier if vector types with zero elements were allowed). 10221 LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts); 10222 HiVT = EnvVT; 10223 *HiIsEmpty = true; 10224 } 10225 return std::make_pair(LoVT, HiVT); 10226 } 10227 10228 /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the 10229 /// low/high part. 10230 std::pair<SDValue, SDValue> 10231 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, 10232 const EVT &HiVT) { 10233 assert(LoVT.isScalableVector() == HiVT.isScalableVector() && 10234 LoVT.isScalableVector() == N.getValueType().isScalableVector() && 10235 "Splitting vector with an invalid mixture of fixed and scalable " 10236 "vector types"); 10237 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <= 10238 N.getValueType().getVectorMinNumElements() && 10239 "More vector elements requested than available!"); 10240 SDValue Lo, Hi; 10241 Lo = 10242 getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL)); 10243 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements() 10244 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales 10245 // IDX with the runtime scaling factor of the result vector type. For 10246 // fixed-width result vectors, that runtime scaling factor is 1. 10247 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, 10248 getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL)); 10249 return std::make_pair(Lo, Hi); 10250 } 10251 10252 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR. 10253 SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) { 10254 EVT VT = N.getValueType(); 10255 EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(), 10256 NextPowerOf2(VT.getVectorNumElements())); 10257 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N, 10258 getVectorIdxConstant(0, DL)); 10259 } 10260 10261 void SelectionDAG::ExtractVectorElements(SDValue Op, 10262 SmallVectorImpl<SDValue> &Args, 10263 unsigned Start, unsigned Count, 10264 EVT EltVT) { 10265 EVT VT = Op.getValueType(); 10266 if (Count == 0) 10267 Count = VT.getVectorNumElements(); 10268 if (EltVT == EVT()) 10269 EltVT = VT.getVectorElementType(); 10270 SDLoc SL(Op); 10271 for (unsigned i = Start, e = Start + Count; i != e; ++i) { 10272 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op, 10273 getVectorIdxConstant(i, SL))); 10274 } 10275 } 10276 10277 // getAddressSpace - Return the address space this GlobalAddress belongs to. 10278 unsigned GlobalAddressSDNode::getAddressSpace() const { 10279 return getGlobal()->getType()->getAddressSpace(); 10280 } 10281 10282 Type *ConstantPoolSDNode::getType() const { 10283 if (isMachineConstantPoolEntry()) 10284 return Val.MachineCPVal->getType(); 10285 return Val.ConstVal->getType(); 10286 } 10287 10288 bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, 10289 unsigned &SplatBitSize, 10290 bool &HasAnyUndefs, 10291 unsigned MinSplatBits, 10292 bool IsBigEndian) const { 10293 EVT VT = getValueType(0); 10294 assert(VT.isVector() && "Expected a vector type"); 10295 unsigned VecWidth = VT.getSizeInBits(); 10296 if (MinSplatBits > VecWidth) 10297 return false; 10298 10299 // FIXME: The widths are based on this node's type, but build vectors can 10300 // truncate their operands. 10301 SplatValue = APInt(VecWidth, 0); 10302 SplatUndef = APInt(VecWidth, 0); 10303 10304 // Get the bits. Bits with undefined values (when the corresponding element 10305 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 10306 // in SplatValue. If any of the values are not constant, give up and return 10307 // false. 10308 unsigned int NumOps = getNumOperands(); 10309 assert(NumOps > 0 && "isConstantSplat has 0-size build vector"); 10310 unsigned EltWidth = VT.getScalarSizeInBits(); 10311 10312 for (unsigned j = 0; j < NumOps; ++j) { 10313 unsigned i = IsBigEndian ? NumOps - 1 - j : j; 10314 SDValue OpVal = getOperand(i); 10315 unsigned BitPos = j * EltWidth; 10316 10317 if (OpVal.isUndef()) 10318 SplatUndef.setBits(BitPos, BitPos + EltWidth); 10319 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal)) 10320 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos); 10321 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 10322 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos); 10323 else 10324 return false; 10325 } 10326 10327 // The build_vector is all constants or undefs. Find the smallest element 10328 // size that splats the vector. 10329 HasAnyUndefs = (SplatUndef != 0); 10330 10331 // FIXME: This does not work for vectors with elements less than 8 bits. 10332 while (VecWidth > 8) { 10333 unsigned HalfSize = VecWidth / 2; 10334 APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize); 10335 APInt LowValue = SplatValue.extractBits(HalfSize, 0); 10336 APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize); 10337 APInt LowUndef = SplatUndef.extractBits(HalfSize, 0); 10338 10339 // If the two halves do not match (ignoring undef bits), stop here. 10340 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 10341 MinSplatBits > HalfSize) 10342 break; 10343 10344 SplatValue = HighValue | LowValue; 10345 SplatUndef = HighUndef & LowUndef; 10346 10347 VecWidth = HalfSize; 10348 } 10349 10350 SplatBitSize = VecWidth; 10351 return true; 10352 } 10353 10354 SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts, 10355 BitVector *UndefElements) const { 10356 unsigned NumOps = getNumOperands(); 10357 if (UndefElements) { 10358 UndefElements->clear(); 10359 UndefElements->resize(NumOps); 10360 } 10361 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); 10362 if (!DemandedElts) 10363 return SDValue(); 10364 SDValue Splatted; 10365 for (unsigned i = 0; i != NumOps; ++i) { 10366 if (!DemandedElts[i]) 10367 continue; 10368 SDValue Op = getOperand(i); 10369 if (Op.isUndef()) { 10370 if (UndefElements) 10371 (*UndefElements)[i] = true; 10372 } else if (!Splatted) { 10373 Splatted = Op; 10374 } else if (Splatted != Op) { 10375 return SDValue(); 10376 } 10377 } 10378 10379 if (!Splatted) { 10380 unsigned FirstDemandedIdx = DemandedElts.countTrailingZeros(); 10381 assert(getOperand(FirstDemandedIdx).isUndef() && 10382 "Can only have a splat without a constant for all undefs."); 10383 return getOperand(FirstDemandedIdx); 10384 } 10385 10386 return Splatted; 10387 } 10388 10389 SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { 10390 APInt DemandedElts = APInt::getAllOnesValue(getNumOperands()); 10391 return getSplatValue(DemandedElts, UndefElements); 10392 } 10393 10394 bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts, 10395 SmallVectorImpl<SDValue> &Sequence, 10396 BitVector *UndefElements) const { 10397 unsigned NumOps = getNumOperands(); 10398 Sequence.clear(); 10399 if (UndefElements) { 10400 UndefElements->clear(); 10401 UndefElements->resize(NumOps); 10402 } 10403 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); 10404 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps)) 10405 return false; 10406 10407 // Set the undefs even if we don't find a sequence (like getSplatValue). 10408 if (UndefElements) 10409 for (unsigned I = 0; I != NumOps; ++I) 10410 if (DemandedElts[I] && getOperand(I).isUndef()) 10411 (*UndefElements)[I] = true; 10412 10413 // Iteratively widen the sequence length looking for repetitions. 10414 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) { 10415 Sequence.append(SeqLen, SDValue()); 10416 for (unsigned I = 0; I != NumOps; ++I) { 10417 if (!DemandedElts[I]) 10418 continue; 10419 SDValue &SeqOp = Sequence[I % SeqLen]; 10420 SDValue Op = getOperand(I); 10421 if (Op.isUndef()) { 10422 if (!SeqOp) 10423 SeqOp = Op; 10424 continue; 10425 } 10426 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) { 10427 Sequence.clear(); 10428 break; 10429 } 10430 SeqOp = Op; 10431 } 10432 if (!Sequence.empty()) 10433 return true; 10434 } 10435 10436 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern"); 10437 return false; 10438 } 10439 10440 bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, 10441 BitVector *UndefElements) const { 10442 APInt DemandedElts = APInt::getAllOnesValue(getNumOperands()); 10443 return getRepeatedSequence(DemandedElts, Sequence, UndefElements); 10444 } 10445 10446 ConstantSDNode * 10447 BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts, 10448 BitVector *UndefElements) const { 10449 return dyn_cast_or_null<ConstantSDNode>( 10450 getSplatValue(DemandedElts, UndefElements)); 10451 } 10452 10453 ConstantSDNode * 10454 BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const { 10455 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements)); 10456 } 10457 10458 ConstantFPSDNode * 10459 BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts, 10460 BitVector *UndefElements) const { 10461 return dyn_cast_or_null<ConstantFPSDNode>( 10462 getSplatValue(DemandedElts, UndefElements)); 10463 } 10464 10465 ConstantFPSDNode * 10466 BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const { 10467 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements)); 10468 } 10469 10470 int32_t 10471 BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, 10472 uint32_t BitWidth) const { 10473 if (ConstantFPSDNode *CN = 10474 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) { 10475 bool IsExact; 10476 APSInt IntVal(BitWidth); 10477 const APFloat &APF = CN->getValueAPF(); 10478 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) != 10479 APFloat::opOK || 10480 !IsExact) 10481 return -1; 10482 10483 return IntVal.exactLogBase2(); 10484 } 10485 return -1; 10486 } 10487 10488 bool BuildVectorSDNode::isConstant() const { 10489 for (const SDValue &Op : op_values()) { 10490 unsigned Opc = Op.getOpcode(); 10491 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP) 10492 return false; 10493 } 10494 return true; 10495 } 10496 10497 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 10498 // Find the first non-undef value in the shuffle mask. 10499 unsigned i, e; 10500 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 10501 /* search */; 10502 10503 // If all elements are undefined, this shuffle can be considered a splat 10504 // (although it should eventually get simplified away completely). 10505 if (i == e) 10506 return true; 10507 10508 // Make sure all remaining elements are either undef or the same as the first 10509 // non-undef value. 10510 for (int Idx = Mask[i]; i != e; ++i) 10511 if (Mask[i] >= 0 && Mask[i] != Idx) 10512 return false; 10513 return true; 10514 } 10515 10516 // Returns the SDNode if it is a constant integer BuildVector 10517 // or constant integer. 10518 SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const { 10519 if (isa<ConstantSDNode>(N)) 10520 return N.getNode(); 10521 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) 10522 return N.getNode(); 10523 // Treat a GlobalAddress supporting constant offset folding as a 10524 // constant integer. 10525 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N)) 10526 if (GA->getOpcode() == ISD::GlobalAddress && 10527 TLI->isOffsetFoldingLegal(GA)) 10528 return GA; 10529 if ((N.getOpcode() == ISD::SPLAT_VECTOR) && 10530 isa<ConstantSDNode>(N.getOperand(0))) 10531 return N.getNode(); 10532 return nullptr; 10533 } 10534 10535 // Returns the SDNode if it is a constant float BuildVector 10536 // or constant float. 10537 SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const { 10538 if (isa<ConstantFPSDNode>(N)) 10539 return N.getNode(); 10540 10541 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())) 10542 return N.getNode(); 10543 10544 return nullptr; 10545 } 10546 10547 void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) { 10548 assert(!Node->OperandList && "Node already has operands"); 10549 assert(SDNode::getMaxNumOperands() >= Vals.size() && 10550 "too many operands to fit into SDNode"); 10551 SDUse *Ops = OperandRecycler.allocate( 10552 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator); 10553 10554 bool IsDivergent = false; 10555 for (unsigned I = 0; I != Vals.size(); ++I) { 10556 Ops[I].setUser(Node); 10557 Ops[I].setInitial(Vals[I]); 10558 if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence. 10559 IsDivergent |= Ops[I].getNode()->isDivergent(); 10560 } 10561 Node->NumOperands = Vals.size(); 10562 Node->OperandList = Ops; 10563 if (!TLI->isSDNodeAlwaysUniform(Node)) { 10564 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA); 10565 Node->SDNodeBits.IsDivergent = IsDivergent; 10566 } 10567 checkForCycles(Node); 10568 } 10569 10570 SDValue SelectionDAG::getTokenFactor(const SDLoc &DL, 10571 SmallVectorImpl<SDValue> &Vals) { 10572 size_t Limit = SDNode::getMaxNumOperands(); 10573 while (Vals.size() > Limit) { 10574 unsigned SliceIdx = Vals.size() - Limit; 10575 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit); 10576 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs); 10577 Vals.erase(Vals.begin() + SliceIdx, Vals.end()); 10578 Vals.emplace_back(NewTF); 10579 } 10580 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals); 10581 } 10582 10583 SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL, 10584 EVT VT, SDNodeFlags Flags) { 10585 switch (Opcode) { 10586 default: 10587 return SDValue(); 10588 case ISD::ADD: 10589 case ISD::OR: 10590 case ISD::XOR: 10591 case ISD::UMAX: 10592 return getConstant(0, DL, VT); 10593 case ISD::MUL: 10594 return getConstant(1, DL, VT); 10595 case ISD::AND: 10596 case ISD::UMIN: 10597 return getAllOnesConstant(DL, VT); 10598 case ISD::SMAX: 10599 return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT); 10600 case ISD::SMIN: 10601 return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT); 10602 case ISD::FADD: 10603 return getConstantFP(-0.0, DL, VT); 10604 case ISD::FMUL: 10605 return getConstantFP(1.0, DL, VT); 10606 case ISD::FMINNUM: 10607 case ISD::FMAXNUM: { 10608 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. 10609 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); 10610 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) : 10611 !Flags.hasNoInfs() ? APFloat::getInf(Semantics) : 10612 APFloat::getLargest(Semantics); 10613 if (Opcode == ISD::FMAXNUM) 10614 NeutralAF.changeSign(); 10615 10616 return getConstantFP(NeutralAF, DL, VT); 10617 } 10618 } 10619 } 10620 10621 #ifndef NDEBUG 10622 static void checkForCyclesHelper(const SDNode *N, 10623 SmallPtrSetImpl<const SDNode*> &Visited, 10624 SmallPtrSetImpl<const SDNode*> &Checked, 10625 const llvm::SelectionDAG *DAG) { 10626 // If this node has already been checked, don't check it again. 10627 if (Checked.count(N)) 10628 return; 10629 10630 // If a node has already been visited on this depth-first walk, reject it as 10631 // a cycle. 10632 if (!Visited.insert(N).second) { 10633 errs() << "Detected cycle in SelectionDAG\n"; 10634 dbgs() << "Offending node:\n"; 10635 N->dumprFull(DAG); dbgs() << "\n"; 10636 abort(); 10637 } 10638 10639 for (const SDValue &Op : N->op_values()) 10640 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG); 10641 10642 Checked.insert(N); 10643 Visited.erase(N); 10644 } 10645 #endif 10646 10647 void llvm::checkForCycles(const llvm::SDNode *N, 10648 const llvm::SelectionDAG *DAG, 10649 bool force) { 10650 #ifndef NDEBUG 10651 bool check = force; 10652 #ifdef EXPENSIVE_CHECKS 10653 check = true; 10654 #endif // EXPENSIVE_CHECKS 10655 if (check) { 10656 assert(N && "Checking nonexistent SDNode"); 10657 SmallPtrSet<const SDNode*, 32> visited; 10658 SmallPtrSet<const SDNode*, 32> checked; 10659 checkForCyclesHelper(N, visited, checked, DAG); 10660 } 10661 #endif // !NDEBUG 10662 } 10663 10664 void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) { 10665 checkForCycles(DAG->getRoot().getNode(), DAG, force); 10666 } 10667