1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements integer type expansion and promotion for LegalizeTypes. 10 // Promotion is the act of changing a computation in an illegal type into a 11 // computation in a larger type. For example, implementing i8 arithmetic in an 12 // i32 register (often needed on powerpc). 13 // Expansion is the act of changing a computation in an illegal type into a 14 // computation in two identical registers of a smaller type. For example, 15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit 16 // targets). 17 // 18 //===----------------------------------------------------------------------===// 19 20 #include "LegalizeTypes.h" 21 #include "llvm/IR/DerivedTypes.h" 22 #include "llvm/Support/ErrorHandling.h" 23 #include "llvm/Support/KnownBits.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace llvm; 26 27 #define DEBUG_TYPE "legalize-types" 28 29 //===----------------------------------------------------------------------===// 30 // Integer Result Promotion 31 //===----------------------------------------------------------------------===// 32 33 /// PromoteIntegerResult - This method is called when a result of a node is 34 /// found to be in need of promotion to a larger type. At this point, the node 35 /// may also have invalid operands or may have other results that need 36 /// expansion, we just know that (at least) one result needs promotion. 37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 38 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); 39 dbgs() << "\n"); 40 SDValue Res = SDValue(); 41 42 // See if the target wants to custom expand this node. 43 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 44 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 45 return; 46 } 47 48 switch (N->getOpcode()) { 49 default: 50 #ifndef NDEBUG 51 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 52 N->dump(&DAG); dbgs() << "\n"; 53 #endif 54 llvm_unreachable("Do not know how to promote this operator!"); 55 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 56 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 57 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 58 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 59 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; 60 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 61 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 62 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 63 case ISD::CTLZ_ZERO_UNDEF: 64 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 65 case ISD::PARITY: 66 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; 67 case ISD::CTTZ_ZERO_UNDEF: 68 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 69 case ISD::EXTRACT_VECTOR_ELT: 70 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 71 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 72 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); 73 break; 74 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); 75 break; 76 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 77 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; 78 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 79 case ISD::STRICT_FSETCC: 80 case ISD::STRICT_FSETCCS: 81 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 82 case ISD::SMIN: 83 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; 84 case ISD::UMIN: 85 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; 86 87 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 88 case ISD::SIGN_EXTEND_INREG: 89 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 90 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 91 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 92 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 93 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 94 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 95 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; 96 97 case ISD::EXTRACT_SUBVECTOR: 98 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 99 case ISD::VECTOR_SHUFFLE: 100 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 101 case ISD::INSERT_VECTOR_ELT: 102 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 103 case ISD::BUILD_VECTOR: 104 Res = PromoteIntRes_BUILD_VECTOR(N); break; 105 case ISD::SCALAR_TO_VECTOR: 106 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; 107 case ISD::SPLAT_VECTOR: 108 Res = PromoteIntRes_SPLAT_VECTOR(N); break; 109 case ISD::CONCAT_VECTORS: 110 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 111 112 case ISD::ANY_EXTEND_VECTOR_INREG: 113 case ISD::SIGN_EXTEND_VECTOR_INREG: 114 case ISD::ZERO_EXTEND_VECTOR_INREG: 115 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; 116 117 case ISD::SIGN_EXTEND: 118 case ISD::ZERO_EXTEND: 119 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 120 121 case ISD::STRICT_FP_TO_SINT: 122 case ISD::STRICT_FP_TO_UINT: 123 case ISD::FP_TO_SINT: 124 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 125 126 case ISD::FP_TO_SINT_SAT: 127 case ISD::FP_TO_UINT_SAT: 128 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; 129 130 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break; 131 132 case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break; 133 134 case ISD::AND: 135 case ISD::OR: 136 case ISD::XOR: 137 case ISD::ADD: 138 case ISD::SUB: 139 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 140 141 case ISD::SDIV: 142 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; 143 144 case ISD::UDIV: 145 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; 146 147 case ISD::SADDO: 148 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 149 case ISD::UADDO: 150 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 151 case ISD::SMULO: 152 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 153 154 case ISD::ADDE: 155 case ISD::SUBE: 156 case ISD::ADDCARRY: 157 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break; 158 159 case ISD::SADDO_CARRY: 160 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; 161 162 case ISD::SADDSAT: 163 case ISD::UADDSAT: 164 case ISD::SSUBSAT: 165 case ISD::USUBSAT: 166 case ISD::SSHLSAT: 167 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break; 168 169 case ISD::SMULFIX: 170 case ISD::SMULFIXSAT: 171 case ISD::UMULFIX: 172 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; 173 174 case ISD::SDIVFIX: 175 case ISD::SDIVFIXSAT: 176 case ISD::UDIVFIX: 177 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; 178 179 case ISD::ABS: Res = PromoteIntRes_ABS(N); break; 180 181 case ISD::ATOMIC_LOAD: 182 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 183 184 case ISD::ATOMIC_LOAD_ADD: 185 case ISD::ATOMIC_LOAD_SUB: 186 case ISD::ATOMIC_LOAD_AND: 187 case ISD::ATOMIC_LOAD_CLR: 188 case ISD::ATOMIC_LOAD_OR: 189 case ISD::ATOMIC_LOAD_XOR: 190 case ISD::ATOMIC_LOAD_NAND: 191 case ISD::ATOMIC_LOAD_MIN: 192 case ISD::ATOMIC_LOAD_MAX: 193 case ISD::ATOMIC_LOAD_UMIN: 194 case ISD::ATOMIC_LOAD_UMAX: 195 case ISD::ATOMIC_SWAP: 196 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 197 198 case ISD::ATOMIC_CMP_SWAP: 199 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 200 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 201 break; 202 203 case ISD::VECREDUCE_ADD: 204 case ISD::VECREDUCE_MUL: 205 case ISD::VECREDUCE_AND: 206 case ISD::VECREDUCE_OR: 207 case ISD::VECREDUCE_XOR: 208 case ISD::VECREDUCE_SMAX: 209 case ISD::VECREDUCE_SMIN: 210 case ISD::VECREDUCE_UMAX: 211 case ISD::VECREDUCE_UMIN: 212 Res = PromoteIntRes_VECREDUCE(N); 213 break; 214 215 case ISD::FREEZE: 216 Res = PromoteIntRes_FREEZE(N); 217 break; 218 219 case ISD::ROTL: 220 case ISD::ROTR: 221 Res = PromoteIntRes_Rotate(N); 222 break; 223 224 case ISD::FSHL: 225 case ISD::FSHR: 226 Res = PromoteIntRes_FunnelShift(N); 227 break; 228 } 229 230 // If the result is null then the sub-method took care of registering it. 231 if (Res.getNode()) 232 SetPromotedInteger(SDValue(N, ResNo), Res); 233 } 234 235 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 236 unsigned ResNo) { 237 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 238 return GetPromotedInteger(Op); 239 } 240 241 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 242 // Sign-extend the new bits, and continue the assertion. 243 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 244 return DAG.getNode(ISD::AssertSext, SDLoc(N), 245 Op.getValueType(), Op, N->getOperand(1)); 246 } 247 248 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 249 // Zero the new bits, and continue the assertion. 250 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 251 return DAG.getNode(ISD::AssertZext, SDLoc(N), 252 Op.getValueType(), Op, N->getOperand(1)); 253 } 254 255 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 256 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 257 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 258 N->getMemoryVT(), ResVT, 259 N->getChain(), N->getBasePtr(), 260 N->getMemOperand()); 261 // Legalize the chain result - switch anything that used the old chain to 262 // use the new one. 263 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 264 return Res; 265 } 266 267 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 268 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 269 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 270 N->getMemoryVT(), 271 N->getChain(), N->getBasePtr(), 272 Op2, N->getMemOperand()); 273 // Legalize the chain result - switch anything that used the old chain to 274 // use the new one. 275 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 276 return Res; 277 } 278 279 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 280 unsigned ResNo) { 281 if (ResNo == 1) { 282 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 283 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 284 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 285 286 // Only use the result of getSetCCResultType if it is legal, 287 // otherwise just use the promoted result type (NVT). 288 if (!TLI.isTypeLegal(SVT)) 289 SVT = NVT; 290 291 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 292 SDValue Res = DAG.getAtomicCmpSwap( 293 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 294 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 295 N->getMemOperand()); 296 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 297 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 298 return Res.getValue(1); 299 } 300 301 // Op2 is used for the comparison and thus must be extended according to the 302 // target's atomic operations. Op3 is merely stored and so can be left alone. 303 SDValue Op2 = N->getOperand(2); 304 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 305 switch (TLI.getExtendForAtomicCmpSwapArg()) { 306 case ISD::SIGN_EXTEND: 307 Op2 = SExtPromotedInteger(Op2); 308 break; 309 case ISD::ZERO_EXTEND: 310 Op2 = ZExtPromotedInteger(Op2); 311 break; 312 case ISD::ANY_EXTEND: 313 Op2 = GetPromotedInteger(Op2); 314 break; 315 default: 316 llvm_unreachable("Invalid atomic op extension"); 317 } 318 319 SDVTList VTs = 320 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 321 SDValue Res = DAG.getAtomicCmpSwap( 322 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 323 N->getBasePtr(), Op2, Op3, N->getMemOperand()); 324 // Update the use to N with the newly created Res. 325 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) 326 ReplaceValueWith(SDValue(N, i), Res.getValue(i)); 327 return Res; 328 } 329 330 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 331 SDValue InOp = N->getOperand(0); 332 EVT InVT = InOp.getValueType(); 333 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 334 EVT OutVT = N->getValueType(0); 335 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 336 SDLoc dl(N); 337 338 switch (getTypeAction(InVT)) { 339 case TargetLowering::TypeLegal: 340 break; 341 case TargetLowering::TypePromoteInteger: 342 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 343 // The input promotes to the same size. Convert the promoted value. 344 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 345 break; 346 case TargetLowering::TypeSoftenFloat: 347 // Promote the integer operand by hand. 348 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 349 case TargetLowering::TypeSoftPromoteHalf: 350 // Promote the integer operand by hand. 351 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp)); 352 case TargetLowering::TypePromoteFloat: { 353 // Convert the promoted float by hand. 354 if (!NOutVT.isVector()) 355 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); 356 break; 357 } 358 case TargetLowering::TypeExpandInteger: 359 case TargetLowering::TypeExpandFloat: 360 break; 361 case TargetLowering::TypeScalarizeVector: 362 // Convert the element to an integer and promote it by hand. 363 if (!NOutVT.isVector()) 364 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 365 BitConvertToInteger(GetScalarizedVector(InOp))); 366 break; 367 case TargetLowering::TypeScalarizeScalableVector: 368 report_fatal_error("Scalarization of scalable vectors is not supported."); 369 case TargetLowering::TypeSplitVector: { 370 if (!NOutVT.isVector()) { 371 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 372 // pieces of the input into integers and reassemble in the final type. 373 SDValue Lo, Hi; 374 GetSplitVector(N->getOperand(0), Lo, Hi); 375 Lo = BitConvertToInteger(Lo); 376 Hi = BitConvertToInteger(Hi); 377 378 if (DAG.getDataLayout().isBigEndian()) 379 std::swap(Lo, Hi); 380 381 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 382 EVT::getIntegerVT(*DAG.getContext(), 383 NOutVT.getSizeInBits()), 384 JoinIntegers(Lo, Hi)); 385 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 386 } 387 break; 388 } 389 case TargetLowering::TypeWidenVector: 390 // The input is widened to the same size. Convert to the widened value. 391 // Make sure that the outgoing value is not a vector, because this would 392 // make us bitcast between two vectors which are legalized in different ways. 393 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) { 394 SDValue Res = 395 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 396 397 // For big endian targets we need to shift the casted value or the 398 // interesting bits will end up at the wrong place. 399 if (DAG.getDataLayout().isBigEndian()) { 400 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 401 EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout()); 402 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!"); 403 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res, 404 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy)); 405 } 406 return Res; 407 } 408 // If the output type is also a vector and widening it to the same size 409 // as the widened input type would be a legal type, we can widen the bitcast 410 // and handle the promotion after. 411 if (NOutVT.isVector()) { 412 unsigned WidenInSize = NInVT.getSizeInBits(); 413 unsigned OutSize = OutVT.getSizeInBits(); 414 if (WidenInSize % OutSize == 0) { 415 unsigned Scale = WidenInSize / OutSize; 416 EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(), 417 OutVT.getVectorElementType(), 418 OutVT.getVectorNumElements() * Scale); 419 if (isTypeLegal(WideOutVT)) { 420 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); 421 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, 422 DAG.getVectorIdxConstant(0, dl)); 423 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp); 424 } 425 } 426 } 427 } 428 429 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 430 CreateStackStoreLoad(InOp, OutVT)); 431 } 432 433 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit any shift amount 434 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't. 435 static EVT getShiftAmountTyForConstant(EVT VT, const TargetLowering &TLI, 436 SelectionDAG &DAG) { 437 EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 438 // If any possible shift value won't fit in the prefered type, just use 439 // something safe. It will be legalized when the shift is expanded. 440 if (!ShiftVT.isVector() && 441 ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits())) 442 ShiftVT = MVT::i32; 443 return ShiftVT; 444 } 445 446 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { 447 SDValue V = GetPromotedInteger(N->getOperand(0)); 448 return DAG.getNode(ISD::FREEZE, SDLoc(N), 449 V.getValueType(), V); 450 } 451 452 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 453 SDValue Op = GetPromotedInteger(N->getOperand(0)); 454 EVT OVT = N->getValueType(0); 455 EVT NVT = Op.getValueType(); 456 SDLoc dl(N); 457 458 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 459 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG); 460 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 461 DAG.getConstant(DiffBits, dl, ShiftVT)); 462 } 463 464 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { 465 SDValue Op = GetPromotedInteger(N->getOperand(0)); 466 EVT OVT = N->getValueType(0); 467 EVT NVT = Op.getValueType(); 468 SDLoc dl(N); 469 470 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 471 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG); 472 return DAG.getNode(ISD::SRL, dl, NVT, 473 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), 474 DAG.getConstant(DiffBits, dl, ShiftVT)); 475 } 476 477 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 478 // The pair element type may be legal, or may not promote to the same type as 479 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 480 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 481 TLI.getTypeToTransformTo(*DAG.getContext(), 482 N->getValueType(0)), JoinIntegers(N->getOperand(0), 483 N->getOperand(1))); 484 } 485 486 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 487 EVT VT = N->getValueType(0); 488 // FIXME there is no actual debug info here 489 SDLoc dl(N); 490 // Zero extend things like i1, sign extend everything else. It shouldn't 491 // matter in theory which one we pick, but this tends to give better code? 492 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 493 SDValue Result = DAG.getNode(Opc, dl, 494 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 495 SDValue(N, 0)); 496 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 497 return Result; 498 } 499 500 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 501 // Zero extend to the promoted type and do the count there. 502 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 503 SDLoc dl(N); 504 EVT OVT = N->getValueType(0); 505 EVT NVT = Op.getValueType(); 506 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 507 // Subtract off the extra leading bits in the bigger type. 508 return DAG.getNode( 509 ISD::SUB, dl, NVT, Op, 510 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, 511 NVT)); 512 } 513 514 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { 515 // Zero extend to the promoted type and do the count or parity there. 516 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 517 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op); 518 } 519 520 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 521 SDValue Op = GetPromotedInteger(N->getOperand(0)); 522 EVT OVT = N->getValueType(0); 523 EVT NVT = Op.getValueType(); 524 SDLoc dl(N); 525 if (N->getOpcode() == ISD::CTTZ) { 526 // The count is the same in the promoted type except if the original 527 // value was zero. This can be handled by setting the bit just off 528 // the top of the original type. 529 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), 530 OVT.getScalarSizeInBits()); 531 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); 532 } 533 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 534 } 535 536 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 537 SDLoc dl(N); 538 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 539 540 SDValue Op0 = N->getOperand(0); 541 SDValue Op1 = N->getOperand(1); 542 543 // If the input also needs to be promoted, do that first so we can get a 544 // get a good idea for the output type. 545 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType()) 546 == TargetLowering::TypePromoteInteger) { 547 SDValue In = GetPromotedInteger(Op0); 548 549 // If the new type is larger than NVT, use it. We probably won't need to 550 // promote it again. 551 EVT SVT = In.getValueType().getScalarType(); 552 if (SVT.bitsGE(NVT)) { 553 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1); 554 return DAG.getAnyExtOrTrunc(Ext, dl, NVT); 555 } 556 } 557 558 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1); 559 } 560 561 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 562 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 563 unsigned NewOpc = N->getOpcode(); 564 SDLoc dl(N); 565 566 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 567 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 568 // and SINT conversions are Custom, there is no way to tell which is 569 // preferable. We choose SINT because that's the right thing on PPC.) 570 if (N->getOpcode() == ISD::FP_TO_UINT && 571 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 572 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 573 NewOpc = ISD::FP_TO_SINT; 574 575 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && 576 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) && 577 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) 578 NewOpc = ISD::STRICT_FP_TO_SINT; 579 580 SDValue Res; 581 if (N->isStrictFPOpcode()) { 582 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, 583 {N->getOperand(0), N->getOperand(1)}); 584 // Legalize the chain result - switch anything that used the old chain to 585 // use the new one. 586 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 587 } else 588 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 589 590 // Assert that the converted value fits in the original type. If it doesn't 591 // (eg: because the value being converted is too big), then the result of the 592 // original operation was undefined anyway, so the assert is still correct. 593 // 594 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: 595 // before legalization: fp-to-uint16, 65534. -> 0xfffe 596 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe 597 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT || 598 N->getOpcode() == ISD::STRICT_FP_TO_UINT) ? 599 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, 600 DAG.getValueType(N->getValueType(0).getScalarType())); 601 } 602 603 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { 604 // Promote the result type, while keeping the original width in Op1. 605 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 606 SDLoc dl(N); 607 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), 608 N->getOperand(1)); 609 } 610 611 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) { 612 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 613 SDLoc dl(N); 614 615 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 616 } 617 618 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) { 619 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 620 SDLoc dl(N); 621 622 SDValue Res = 623 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); 624 625 // Legalize the chain result - switch anything that used the old chain to 626 // use the new one. 627 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 628 return Res; 629 } 630 631 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 632 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 633 SDLoc dl(N); 634 635 if (getTypeAction(N->getOperand(0).getValueType()) 636 == TargetLowering::TypePromoteInteger) { 637 SDValue Res = GetPromotedInteger(N->getOperand(0)); 638 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 639 640 // If the result and operand types are the same after promotion, simplify 641 // to an in-register extension. 642 if (NVT == Res.getValueType()) { 643 // The high bits are not guaranteed to be anything. Insert an extend. 644 if (N->getOpcode() == ISD::SIGN_EXTEND) 645 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 646 DAG.getValueType(N->getOperand(0).getValueType())); 647 if (N->getOpcode() == ISD::ZERO_EXTEND) 648 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); 649 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 650 return Res; 651 } 652 } 653 654 // Otherwise, just extend the original operand all the way to the larger type. 655 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 656 } 657 658 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 659 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 660 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 661 ISD::LoadExtType ExtType = 662 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 663 SDLoc dl(N); 664 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 665 N->getMemoryVT(), N->getMemOperand()); 666 667 // Legalize the chain result - switch anything that used the old chain to 668 // use the new one. 669 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 670 return Res; 671 } 672 673 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { 674 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 675 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 676 677 SDLoc dl(N); 678 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), 679 N->getOffset(), N->getMask(), ExtPassThru, 680 N->getMemoryVT(), N->getMemOperand(), 681 N->getAddressingMode(), ISD::EXTLOAD); 682 // Legalize the chain result - switch anything that used the old chain to 683 // use the new one. 684 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 685 return Res; 686 } 687 688 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { 689 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 690 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 691 assert(NVT == ExtPassThru.getValueType() && 692 "Gather result type and the passThru argument type should be the same"); 693 694 ISD::LoadExtType ExtType = N->getExtensionType(); 695 if (ExtType == ISD::NON_EXTLOAD) 696 ExtType = ISD::EXTLOAD; 697 698 SDLoc dl(N); 699 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), 700 N->getIndex(), N->getScale() }; 701 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), 702 N->getMemoryVT(), dl, Ops, 703 N->getMemOperand(), N->getIndexType(), 704 ExtType); 705 // Legalize the chain result - switch anything that used the old chain to 706 // use the new one. 707 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 708 return Res; 709 } 710 711 /// Promote the overflow flag of an overflowing arithmetic node. 712 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 713 // Change the return type of the boolean result while obeying 714 // getSetCCResultType. 715 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 716 EVT VT = N->getValueType(0); 717 EVT SVT = getSetCCResultType(VT); 718 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; 719 unsigned NumOps = N->getNumOperands(); 720 assert(NumOps <= 3 && "Too many operands"); 721 if (NumOps == 3) 722 Ops[2] = N->getOperand(2); 723 724 SDLoc dl(N); 725 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), 726 makeArrayRef(Ops, NumOps)); 727 728 // Modified the sum result - switch anything that used the old sum to use 729 // the new one. 730 ReplaceValueWith(SDValue(N, 0), Res); 731 732 // Convert to the expected type. 733 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); 734 } 735 736 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { 737 // If the promoted type is legal, we can convert this to: 738 // 1. ANY_EXTEND iN to iM 739 // 2. SHL by M-N 740 // 3. [US][ADD|SUB|SHL]SAT 741 // 4. L/ASHR by M-N 742 // Else it is more efficient to convert this to a min and a max 743 // operation in the higher precision arithmetic. 744 SDLoc dl(N); 745 SDValue Op1 = N->getOperand(0); 746 SDValue Op2 = N->getOperand(1); 747 unsigned OldBits = Op1.getScalarValueSizeInBits(); 748 749 unsigned Opcode = N->getOpcode(); 750 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; 751 752 SDValue Op1Promoted, Op2Promoted; 753 if (IsShift) { 754 Op1Promoted = GetPromotedInteger(Op1); 755 Op2Promoted = ZExtPromotedInteger(Op2); 756 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { 757 Op1Promoted = ZExtPromotedInteger(Op1); 758 Op2Promoted = ZExtPromotedInteger(Op2); 759 } else { 760 Op1Promoted = SExtPromotedInteger(Op1); 761 Op2Promoted = SExtPromotedInteger(Op2); 762 } 763 EVT PromotedType = Op1Promoted.getValueType(); 764 unsigned NewBits = PromotedType.getScalarSizeInBits(); 765 766 // Shift cannot use a min/max expansion, we can't detect overflow if all of 767 // the bits have been shifted out. 768 if (IsShift || TLI.isOperationLegalOrCustom(Opcode, PromotedType)) { 769 unsigned ShiftOp; 770 switch (Opcode) { 771 case ISD::SADDSAT: 772 case ISD::SSUBSAT: 773 case ISD::SSHLSAT: 774 ShiftOp = ISD::SRA; 775 break; 776 case ISD::UADDSAT: 777 case ISD::USUBSAT: 778 case ISD::USHLSAT: 779 ShiftOp = ISD::SRL; 780 break; 781 default: 782 llvm_unreachable("Expected opcode to be signed or unsigned saturation " 783 "addition, subtraction or left shift"); 784 } 785 786 unsigned SHLAmount = NewBits - OldBits; 787 EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); 788 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT); 789 Op1Promoted = 790 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); 791 if (!IsShift) 792 Op2Promoted = 793 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); 794 795 SDValue Result = 796 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); 797 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); 798 } else { 799 if (Opcode == ISD::USUBSAT) { 800 SDValue Max = 801 DAG.getNode(ISD::UMAX, dl, PromotedType, Op1Promoted, Op2Promoted); 802 return DAG.getNode(ISD::SUB, dl, PromotedType, Max, Op2Promoted); 803 } 804 805 if (Opcode == ISD::UADDSAT) { 806 APInt MaxVal = APInt::getAllOnesValue(OldBits).zext(NewBits); 807 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 808 SDValue Add = 809 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); 810 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); 811 } 812 813 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; 814 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits); 815 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits); 816 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType); 817 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 818 SDValue Result = 819 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); 820 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); 821 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); 822 return Result; 823 } 824 } 825 826 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { 827 // Can just promote the operands then continue with operation. 828 SDLoc dl(N); 829 SDValue Op1Promoted, Op2Promoted; 830 bool Signed = 831 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; 832 bool Saturating = 833 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; 834 if (Signed) { 835 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 836 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 837 } else { 838 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 839 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 840 } 841 EVT OldType = N->getOperand(0).getValueType(); 842 EVT PromotedType = Op1Promoted.getValueType(); 843 unsigned DiffSize = 844 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); 845 846 if (Saturating) { 847 // Promoting the operand and result values changes the saturation width, 848 // which is extends the values that we clamp to on saturation. This could be 849 // resolved by shifting one of the operands the same amount, which would 850 // also shift the result we compare against, then shifting back. 851 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); 852 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 853 DAG.getConstant(DiffSize, dl, ShiftTy)); 854 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 855 Op2Promoted, N->getOperand(2)); 856 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; 857 return DAG.getNode(ShiftOp, dl, PromotedType, Result, 858 DAG.getConstant(DiffSize, dl, ShiftTy)); 859 } 860 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, 861 N->getOperand(2)); 862 } 863 864 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, 865 unsigned SatW, bool Signed, 866 const TargetLowering &TLI, 867 SelectionDAG &DAG) { 868 EVT VT = V.getValueType(); 869 unsigned VTW = VT.getScalarSizeInBits(); 870 871 if (!Signed) { 872 // Saturate to the unsigned maximum by getting the minimum of V and the 873 // maximum. 874 return DAG.getNode(ISD::UMIN, dl, VT, V, 875 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW), 876 dl, VT)); 877 } 878 879 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the 880 // signed minimum of it and V. 881 V = DAG.getNode(ISD::SMIN, dl, VT, V, 882 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1), 883 dl, VT)); 884 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the 885 // signed maximum of it and V. 886 V = DAG.getNode(ISD::SMAX, dl, VT, V, 887 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1), 888 dl, VT)); 889 return V; 890 } 891 892 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, 893 unsigned Scale, const TargetLowering &TLI, 894 SelectionDAG &DAG, unsigned SatW = 0) { 895 EVT VT = LHS.getValueType(); 896 unsigned VTSize = VT.getScalarSizeInBits(); 897 bool Signed = N->getOpcode() == ISD::SDIVFIX || 898 N->getOpcode() == ISD::SDIVFIXSAT; 899 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 900 N->getOpcode() == ISD::UDIVFIXSAT; 901 902 SDLoc dl(N); 903 // Widen the types by a factor of two. This is guaranteed to expand, since it 904 // will always have enough high bits in the LHS to shift into. 905 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2); 906 if (VT.isVector()) 907 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, 908 VT.getVectorElementCount()); 909 if (Signed) { 910 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT); 911 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT); 912 } else { 913 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT); 914 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT); 915 } 916 917 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, 918 DAG); 919 assert(Res && "Expanding DIVFIX with wide type failed?"); 920 if (Saturating) { 921 // If the caller has told us to saturate at something less, use that width 922 // instead of the type before doubling. However, it cannot be more than 923 // what we just widened! 924 assert(SatW <= VTSize && 925 "Tried to saturate to more than the original type?"); 926 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed, 927 TLI, DAG); 928 } 929 return DAG.getZExtOrTrunc(Res, dl, VT); 930 } 931 932 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { 933 SDLoc dl(N); 934 SDValue Op1Promoted, Op2Promoted; 935 bool Signed = N->getOpcode() == ISD::SDIVFIX || 936 N->getOpcode() == ISD::SDIVFIXSAT; 937 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 938 N->getOpcode() == ISD::UDIVFIXSAT; 939 if (Signed) { 940 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 941 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 942 } else { 943 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 944 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 945 } 946 EVT PromotedType = Op1Promoted.getValueType(); 947 unsigned Scale = N->getConstantOperandVal(2); 948 949 // If the type is already legal and the operation is legal in that type, we 950 // should not early expand. 951 if (TLI.isTypeLegal(PromotedType)) { 952 TargetLowering::LegalizeAction Action = 953 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale); 954 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { 955 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); 956 unsigned Diff = PromotedType.getScalarSizeInBits() - 957 N->getValueType(0).getScalarSizeInBits(); 958 if (Saturating) 959 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 960 DAG.getConstant(Diff, dl, ShiftTy)); 961 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 962 Op2Promoted, N->getOperand(2)); 963 if (Saturating) 964 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res, 965 DAG.getConstant(Diff, dl, ShiftTy)); 966 return Res; 967 } 968 } 969 970 // See if we can perform the division in this type without expanding. 971 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted, 972 Op2Promoted, Scale, DAG)) { 973 if (Saturating) 974 Res = SaturateWidenedDIVFIX(Res, dl, 975 N->getValueType(0).getScalarSizeInBits(), 976 Signed, TLI, DAG); 977 return Res; 978 } 979 // If we cannot, expand it to twice the type width. If we are saturating, give 980 // it the original width as a saturating width so we don't need to emit 981 // two saturations. 982 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG, 983 N->getValueType(0).getScalarSizeInBits()); 984 } 985 986 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 987 if (ResNo == 1) 988 return PromoteIntRes_Overflow(N); 989 990 // The operation overflowed iff the result in the larger type is not the 991 // sign extension of its truncation to the original type. 992 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 993 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 994 EVT OVT = N->getOperand(0).getValueType(); 995 EVT NVT = LHS.getValueType(); 996 SDLoc dl(N); 997 998 // Do the arithmetic in the larger type. 999 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 1000 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1001 1002 // Calculate the overflow flag: sign extend the arithmetic result from 1003 // the original type. 1004 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 1005 DAG.getValueType(OVT)); 1006 // Overflowed if and only if this is not equal to Res. 1007 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1008 1009 // Use the calculated overflow everywhere. 1010 ReplaceValueWith(SDValue(N, 1), Ofl); 1011 1012 return Res; 1013 } 1014 1015 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 1016 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 1017 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 1018 return DAG.getSelect(SDLoc(N), 1019 LHS.getValueType(), N->getOperand(0), LHS, RHS); 1020 } 1021 1022 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { 1023 SDValue Mask = N->getOperand(0); 1024 1025 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 1026 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 1027 return DAG.getNode(ISD::VSELECT, SDLoc(N), 1028 LHS.getValueType(), Mask, LHS, RHS); 1029 } 1030 1031 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 1032 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 1033 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 1034 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 1035 LHS.getValueType(), N->getOperand(0), 1036 N->getOperand(1), LHS, RHS, N->getOperand(4)); 1037 } 1038 1039 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 1040 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 1041 EVT InVT = N->getOperand(OpNo).getValueType(); 1042 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1043 1044 EVT SVT = getSetCCResultType(InVT); 1045 1046 // If we got back a type that needs to be promoted, this likely means the 1047 // the input type also needs to be promoted. So get the promoted type for 1048 // the input and try the query again. 1049 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { 1050 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 1051 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 1052 SVT = getSetCCResultType(InVT); 1053 } else { 1054 // Input type isn't promoted, just use the default promoted type. 1055 SVT = NVT; 1056 } 1057 } 1058 1059 SDLoc dl(N); 1060 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && 1061 "Vector compare must return a vector result!"); 1062 1063 // Get the SETCC result using the canonical SETCC type. 1064 SDValue SetCC; 1065 if (N->isStrictFPOpcode()) { 1066 EVT VTs[] = {SVT, MVT::Other}; 1067 SDValue Opers[] = {N->getOperand(0), N->getOperand(1), 1068 N->getOperand(2), N->getOperand(3)}; 1069 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers); 1070 // Legalize the chain result - switch anything that used the old chain to 1071 // use the new one. 1072 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1)); 1073 } else 1074 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), 1075 N->getOperand(1), N->getOperand(2)); 1076 1077 // Convert to the expected type. 1078 return DAG.getSExtOrTrunc(SetCC, dl, NVT); 1079 } 1080 1081 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 1082 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1083 SDValue RHS = N->getOperand(1); 1084 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1085 RHS = ZExtPromotedInteger(RHS); 1086 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS); 1087 } 1088 1089 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 1090 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1091 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 1092 Op.getValueType(), Op, N->getOperand(1)); 1093 } 1094 1095 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 1096 // The input may have strange things in the top bits of the registers, but 1097 // these operations don't care. They may have weird bits going out, but 1098 // that too is okay if they are integer operations. 1099 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1100 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 1101 return DAG.getNode(N->getOpcode(), SDLoc(N), 1102 LHS.getValueType(), LHS, RHS); 1103 } 1104 1105 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { 1106 // Sign extend the input. 1107 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1108 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1109 return DAG.getNode(N->getOpcode(), SDLoc(N), 1110 LHS.getValueType(), LHS, RHS); 1111 } 1112 1113 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { 1114 // Zero extend the input. 1115 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1116 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1117 return DAG.getNode(N->getOpcode(), SDLoc(N), 1118 LHS.getValueType(), LHS, RHS); 1119 } 1120 1121 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { 1122 // It doesn't matter if we sign extend or zero extend in the inputs. So do 1123 // whatever is best for the target. 1124 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0)); 1125 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1)); 1126 return DAG.getNode(N->getOpcode(), SDLoc(N), 1127 LHS.getValueType(), LHS, RHS); 1128 } 1129 1130 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 1131 // The input value must be properly sign extended. 1132 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1133 SDValue RHS = N->getOperand(1); 1134 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1135 RHS = ZExtPromotedInteger(RHS); 1136 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS); 1137 } 1138 1139 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 1140 // The input value must be properly zero extended. 1141 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1142 SDValue RHS = N->getOperand(1); 1143 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1144 RHS = ZExtPromotedInteger(RHS); 1145 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS); 1146 } 1147 1148 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { 1149 // Lower the rotate to shifts and ORs which can be promoted. 1150 SDValue Res; 1151 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG); 1152 ReplaceValueWith(SDValue(N, 0), Res); 1153 return SDValue(); 1154 } 1155 1156 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { 1157 SDValue Hi = GetPromotedInteger(N->getOperand(0)); 1158 SDValue Lo = GetPromotedInteger(N->getOperand(1)); 1159 SDValue Amount = GetPromotedInteger(N->getOperand(2)); 1160 1161 SDLoc DL(N); 1162 EVT OldVT = N->getOperand(0).getValueType(); 1163 EVT VT = Lo.getValueType(); 1164 unsigned Opcode = N->getOpcode(); 1165 bool IsFSHR = Opcode == ISD::FSHR; 1166 unsigned OldBits = OldVT.getScalarSizeInBits(); 1167 unsigned NewBits = VT.getScalarSizeInBits(); 1168 1169 // Amount has to be interpreted modulo the old bit width. 1170 Amount = 1171 DAG.getNode(ISD::UREM, DL, VT, Amount, DAG.getConstant(OldBits, DL, VT)); 1172 1173 // If the promoted type is twice the size (or more), then we use the 1174 // traditional funnel 'double' shift codegen. This isn't necessary if the 1175 // shift amount is constant. 1176 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. 1177 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). 1178 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amount) && 1179 !TLI.isOperationLegalOrCustom(Opcode, VT)) { 1180 SDValue HiShift = DAG.getConstant(OldBits, DL, VT); 1181 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift); 1182 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT); 1183 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo); 1184 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amount); 1185 if (!IsFSHR) 1186 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift); 1187 return Res; 1188 } 1189 1190 // Shift Lo up to occupy the upper bits of the promoted type. 1191 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, VT); 1192 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset); 1193 1194 // Increase Amount to shift the result into the lower bits of the promoted 1195 // type. 1196 if (IsFSHR) 1197 Amount = DAG.getNode(ISD::ADD, DL, VT, Amount, ShiftOffset); 1198 1199 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amount); 1200 } 1201 1202 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 1203 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1204 SDValue Res; 1205 SDValue InOp = N->getOperand(0); 1206 SDLoc dl(N); 1207 1208 switch (getTypeAction(InOp.getValueType())) { 1209 default: llvm_unreachable("Unknown type action!"); 1210 case TargetLowering::TypeLegal: 1211 case TargetLowering::TypeExpandInteger: 1212 Res = InOp; 1213 break; 1214 case TargetLowering::TypePromoteInteger: 1215 Res = GetPromotedInteger(InOp); 1216 break; 1217 case TargetLowering::TypeSplitVector: { 1218 EVT InVT = InOp.getValueType(); 1219 assert(InVT.isVector() && "Cannot split scalar types"); 1220 unsigned NumElts = InVT.getVectorNumElements(); 1221 assert(NumElts == NVT.getVectorNumElements() && 1222 "Dst and Src must have the same number of elements"); 1223 assert(isPowerOf2_32(NumElts) && 1224 "Promoted vector type must be a power of two"); 1225 1226 SDValue EOp1, EOp2; 1227 GetSplitVector(InOp, EOp1, EOp2); 1228 1229 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 1230 NumElts/2); 1231 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 1232 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 1233 1234 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 1235 } 1236 case TargetLowering::TypeWidenVector: { 1237 SDValue WideInOp = GetWidenedVector(InOp); 1238 1239 // Truncate widened InOp. 1240 unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); 1241 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), 1242 N->getValueType(0).getScalarType(), NumElem); 1243 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); 1244 1245 // Zero extend so that the elements are of same type as those of NVT 1246 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), 1247 NumElem); 1248 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); 1249 1250 // Extract the low NVT subvector. 1251 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl); 1252 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); 1253 } 1254 } 1255 1256 // Truncate to NVT instead of VT 1257 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 1258 } 1259 1260 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 1261 if (ResNo == 1) 1262 return PromoteIntRes_Overflow(N); 1263 1264 // The operation overflowed iff the result in the larger type is not the 1265 // zero extension of its truncation to the original type. 1266 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1267 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1268 EVT OVT = N->getOperand(0).getValueType(); 1269 EVT NVT = LHS.getValueType(); 1270 SDLoc dl(N); 1271 1272 // Do the arithmetic in the larger type. 1273 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 1274 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1275 1276 // Calculate the overflow flag: zero extend the arithmetic result from 1277 // the original type. 1278 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 1279 // Overflowed if and only if this is not equal to Res. 1280 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1281 1282 // Use the calculated overflow everywhere. 1283 ReplaceValueWith(SDValue(N, 1), Ofl); 1284 1285 return Res; 1286 } 1287 1288 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that 1289 // the third operand of ADDE/SUBE nodes is carry flag, which differs from 1290 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean. 1291 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) { 1292 if (ResNo == 1) 1293 return PromoteIntRes_Overflow(N); 1294 1295 // We need to sign-extend the operands so the carry value computed by the 1296 // wide operation will be equivalent to the carry value computed by the 1297 // narrow operation. 1298 // An ADDCARRY can generate carry only if any of the operands has its 1299 // most significant bit set. Sign extension propagates the most significant 1300 // bit into the higher bits which means the extra bit that the narrow 1301 // addition would need (i.e. the carry) will be propagated through the higher 1302 // bits of the wide addition. 1303 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be 1304 // preserved by sign extension. 1305 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1306 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1307 1308 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; 1309 1310 // Do the arithmetic in the wide type. 1311 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), 1312 LHS, RHS, N->getOperand(2)); 1313 1314 // Update the users of the original carry/borrow value. 1315 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 1316 1317 return SDValue(Res.getNode(), 0); 1318 } 1319 1320 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, 1321 unsigned ResNo) { 1322 assert(ResNo == 1 && "Don't know how to promote other results yet."); 1323 return PromoteIntRes_Overflow(N); 1324 } 1325 1326 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { 1327 SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); 1328 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); 1329 } 1330 1331 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 1332 // Promote the overflow bit trivially. 1333 if (ResNo == 1) 1334 return PromoteIntRes_Overflow(N); 1335 1336 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 1337 SDLoc DL(N); 1338 EVT SmallVT = LHS.getValueType(); 1339 1340 // To determine if the result overflowed in a larger type, we extend the 1341 // input to the larger type, do the multiply (checking if it overflows), 1342 // then also check the high bits of the result to see if overflow happened 1343 // there. 1344 if (N->getOpcode() == ISD::SMULO) { 1345 LHS = SExtPromotedInteger(LHS); 1346 RHS = SExtPromotedInteger(RHS); 1347 } else { 1348 LHS = ZExtPromotedInteger(LHS); 1349 RHS = ZExtPromotedInteger(RHS); 1350 } 1351 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 1352 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 1353 1354 // Overflow occurred if it occurred in the larger type, or if the high part 1355 // of the result does not zero/sign-extend the low part. Check this second 1356 // possibility first. 1357 SDValue Overflow; 1358 if (N->getOpcode() == ISD::UMULO) { 1359 // Unsigned overflow occurred if the high part is non-zero. 1360 unsigned Shift = SmallVT.getScalarSizeInBits(); 1361 EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG); 1362 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 1363 DAG.getConstant(Shift, DL, ShiftTy)); 1364 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 1365 DAG.getConstant(0, DL, Hi.getValueType()), 1366 ISD::SETNE); 1367 } else { 1368 // Signed overflow occurred if the high part does not sign extend the low. 1369 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 1370 Mul, DAG.getValueType(SmallVT)); 1371 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 1372 } 1373 1374 // The only other way for overflow to occur is if the multiplication in the 1375 // larger type itself overflowed. 1376 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 1377 SDValue(Mul.getNode(), 1)); 1378 1379 // Use the calculated overflow everywhere. 1380 ReplaceValueWith(SDValue(N, 1), Overflow); 1381 return Mul; 1382 } 1383 1384 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 1385 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 1386 N->getValueType(0))); 1387 } 1388 1389 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { 1390 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1391 1392 APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue(); 1393 return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits())); 1394 } 1395 1396 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 1397 SDValue Chain = N->getOperand(0); // Get the chain. 1398 SDValue Ptr = N->getOperand(1); // Get the pointer. 1399 EVT VT = N->getValueType(0); 1400 SDLoc dl(N); 1401 1402 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 1403 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 1404 // The argument is passed as NumRegs registers of type RegVT. 1405 1406 SmallVector<SDValue, 8> Parts(NumRegs); 1407 for (unsigned i = 0; i < NumRegs; ++i) { 1408 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 1409 N->getConstantOperandVal(3)); 1410 Chain = Parts[i].getValue(1); 1411 } 1412 1413 // Handle endianness of the load. 1414 if (DAG.getDataLayout().isBigEndian()) 1415 std::reverse(Parts.begin(), Parts.end()); 1416 1417 // Assemble the parts in the promoted type. 1418 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1419 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 1420 for (unsigned i = 1; i < NumRegs; ++i) { 1421 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 1422 // Shift it to the right position and "or" it in. 1423 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 1424 DAG.getConstant(i * RegVT.getSizeInBits(), dl, 1425 TLI.getPointerTy(DAG.getDataLayout()))); 1426 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 1427 } 1428 1429 // Modified the chain result - switch anything that used the old chain to 1430 // use the new one. 1431 ReplaceValueWith(SDValue(N, 1), Chain); 1432 1433 return Res; 1434 } 1435 1436 //===----------------------------------------------------------------------===// 1437 // Integer Operand Promotion 1438 //===----------------------------------------------------------------------===// 1439 1440 /// PromoteIntegerOperand - This method is called when the specified operand of 1441 /// the specified node is found to need promotion. At this point, all of the 1442 /// result types of the node are known to be legal, but other operands of the 1443 /// node may need promotion or expansion as well as the specified one. 1444 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 1445 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); 1446 dbgs() << "\n"); 1447 SDValue Res = SDValue(); 1448 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 1449 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 1450 return false; 1451 } 1452 1453 switch (N->getOpcode()) { 1454 default: 1455 #ifndef NDEBUG 1456 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 1457 N->dump(&DAG); dbgs() << "\n"; 1458 #endif 1459 llvm_unreachable("Do not know how to promote this operator's operand!"); 1460 1461 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 1462 case ISD::ATOMIC_STORE: 1463 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 1464 break; 1465 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 1466 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 1467 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 1468 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 1469 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 1470 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 1471 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 1472 case ISD::INSERT_VECTOR_ELT: 1473 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 1474 case ISD::SCALAR_TO_VECTOR: 1475 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 1476 case ISD::SPLAT_VECTOR: 1477 Res = PromoteIntOp_SPLAT_VECTOR(N); break; 1478 case ISD::VSELECT: 1479 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 1480 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 1481 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 1482 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 1483 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 1484 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; 1485 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 1486 OpNo); break; 1487 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), 1488 OpNo); break; 1489 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), 1490 OpNo); break; 1491 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), 1492 OpNo); break; 1493 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), 1494 OpNo); break; 1495 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 1496 case ISD::FP16_TO_FP: 1497 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 1498 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; 1499 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 1500 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; 1501 1502 case ISD::SHL: 1503 case ISD::SRA: 1504 case ISD::SRL: 1505 case ISD::ROTL: 1506 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 1507 1508 case ISD::SADDO_CARRY: 1509 case ISD::SSUBO_CARRY: 1510 case ISD::ADDCARRY: 1511 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break; 1512 1513 case ISD::FRAMEADDR: 1514 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; 1515 1516 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break; 1517 1518 case ISD::SMULFIX: 1519 case ISD::SMULFIXSAT: 1520 case ISD::UMULFIX: 1521 case ISD::UMULFIXSAT: 1522 case ISD::SDIVFIX: 1523 case ISD::SDIVFIXSAT: 1524 case ISD::UDIVFIX: 1525 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; 1526 1527 case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break; 1528 1529 case ISD::VECREDUCE_ADD: 1530 case ISD::VECREDUCE_MUL: 1531 case ISD::VECREDUCE_AND: 1532 case ISD::VECREDUCE_OR: 1533 case ISD::VECREDUCE_XOR: 1534 case ISD::VECREDUCE_SMAX: 1535 case ISD::VECREDUCE_SMIN: 1536 case ISD::VECREDUCE_UMAX: 1537 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; 1538 } 1539 1540 // If the result is null, the sub-method took care of registering results etc. 1541 if (!Res.getNode()) return false; 1542 1543 // If the result is N, the sub-method updated N in place. Tell the legalizer 1544 // core about this. 1545 if (Res.getNode() == N) 1546 return true; 1547 1548 const bool IsStrictFp = N->isStrictFPOpcode(); 1549 assert(Res.getValueType() == N->getValueType(0) && 1550 N->getNumValues() == (IsStrictFp ? 2 : 1) && 1551 "Invalid operand expansion"); 1552 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: "; 1553 Res.dump()); 1554 1555 ReplaceValueWith(SDValue(N, 0), Res); 1556 if (IsStrictFp) 1557 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 1558 1559 return false; 1560 } 1561 1562 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is 1563 /// shared among BR_CC, SELECT_CC, and SETCC handlers. 1564 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 1565 ISD::CondCode CCCode) { 1566 // We have to insert explicit sign or zero extends. Note that we could 1567 // insert sign extends for ALL conditions. For those operations where either 1568 // zero or sign extension would be valid, use SExtOrZExtPromotedInteger 1569 // which will choose the cheapest for the target. 1570 switch (CCCode) { 1571 default: llvm_unreachable("Unknown integer comparison!"); 1572 case ISD::SETEQ: 1573 case ISD::SETNE: { 1574 SDValue OpL = GetPromotedInteger(NewLHS); 1575 SDValue OpR = GetPromotedInteger(NewRHS); 1576 1577 // We would prefer to promote the comparison operand with sign extension. 1578 // If the width of OpL/OpR excluding the duplicated sign bits is no greater 1579 // than the width of NewLHS/NewRH, we can avoid inserting real truncate 1580 // instruction, which is redundant eventually. 1581 unsigned OpLEffectiveBits = 1582 OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1; 1583 unsigned OpREffectiveBits = 1584 OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1; 1585 if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() && 1586 OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) { 1587 NewLHS = OpL; 1588 NewRHS = OpR; 1589 } else { 1590 NewLHS = SExtOrZExtPromotedInteger(NewLHS); 1591 NewRHS = SExtOrZExtPromotedInteger(NewRHS); 1592 } 1593 break; 1594 } 1595 case ISD::SETUGE: 1596 case ISD::SETUGT: 1597 case ISD::SETULE: 1598 case ISD::SETULT: 1599 NewLHS = SExtOrZExtPromotedInteger(NewLHS); 1600 NewRHS = SExtOrZExtPromotedInteger(NewRHS); 1601 break; 1602 case ISD::SETGE: 1603 case ISD::SETGT: 1604 case ISD::SETLT: 1605 case ISD::SETLE: 1606 NewLHS = SExtPromotedInteger(NewLHS); 1607 NewRHS = SExtPromotedInteger(NewRHS); 1608 break; 1609 } 1610 } 1611 1612 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 1613 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1614 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 1615 } 1616 1617 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 1618 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 1619 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 1620 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); 1621 } 1622 1623 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 1624 // This should only occur in unusual situations like bitcasting to an 1625 // x86_fp80, so just turn it into a store+load 1626 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 1627 } 1628 1629 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 1630 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1631 1632 SDValue LHS = N->getOperand(2); 1633 SDValue RHS = N->getOperand(3); 1634 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 1635 1636 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 1637 // legal types. 1638 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1639 N->getOperand(1), LHS, RHS, N->getOperand(4)), 1640 0); 1641 } 1642 1643 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 1644 assert(OpNo == 1 && "only know how to promote condition"); 1645 1646 // Promote all the way up to the canonical SetCC type. 1647 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 1648 1649 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 1650 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 1651 N->getOperand(2)), 0); 1652 } 1653 1654 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 1655 // Since the result type is legal, the operands must promote to it. 1656 EVT OVT = N->getOperand(0).getValueType(); 1657 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 1658 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 1659 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 1660 SDLoc dl(N); 1661 1662 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 1663 DAG.getConstant(OVT.getSizeInBits(), dl, 1664 TLI.getPointerTy(DAG.getDataLayout()))); 1665 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 1666 } 1667 1668 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 1669 // The vector type is legal but the element type is not. This implies 1670 // that the vector is a power-of-two in length and that the element 1671 // type does not have a strange size (eg: it is not i1). 1672 EVT VecVT = N->getValueType(0); 1673 unsigned NumElts = VecVT.getVectorNumElements(); 1674 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 1675 "Legal vector of one illegal element?"); 1676 1677 // Promote the inserted value. The type does not need to match the 1678 // vector element type. Check that any extra bits introduced will be 1679 // truncated away. 1680 assert(N->getOperand(0).getValueSizeInBits() >= 1681 N->getValueType(0).getScalarSizeInBits() && 1682 "Type of inserted value narrower than vector element type!"); 1683 1684 SmallVector<SDValue, 16> NewOps; 1685 for (unsigned i = 0; i < NumElts; ++i) 1686 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 1687 1688 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1689 } 1690 1691 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 1692 unsigned OpNo) { 1693 if (OpNo == 1) { 1694 // Promote the inserted value. This is valid because the type does not 1695 // have to match the vector element type. 1696 1697 // Check that any extra bits introduced will be truncated away. 1698 assert(N->getOperand(1).getValueSizeInBits() >= 1699 N->getValueType(0).getScalarSizeInBits() && 1700 "Type of inserted value narrower than vector element type!"); 1701 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1702 GetPromotedInteger(N->getOperand(1)), 1703 N->getOperand(2)), 1704 0); 1705 } 1706 1707 assert(OpNo == 2 && "Different operand and result vector types?"); 1708 1709 // Promote the index. 1710 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 1711 TLI.getVectorIdxTy(DAG.getDataLayout())); 1712 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1713 N->getOperand(1), Idx), 0); 1714 } 1715 1716 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 1717 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 1718 // the operand in place. 1719 return SDValue(DAG.UpdateNodeOperands(N, 1720 GetPromotedInteger(N->getOperand(0))), 0); 1721 } 1722 1723 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) { 1724 // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the 1725 // operand in place. 1726 return SDValue( 1727 DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0); 1728 } 1729 1730 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 1731 assert(OpNo == 0 && "Only know how to promote the condition!"); 1732 SDValue Cond = N->getOperand(0); 1733 EVT OpTy = N->getOperand(1).getValueType(); 1734 1735 if (N->getOpcode() == ISD::VSELECT) 1736 if (SDValue Res = WidenVSELECTMask(N)) 1737 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 1738 Res, N->getOperand(1), N->getOperand(2)); 1739 1740 // Promote all the way up to the canonical SetCC type. 1741 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 1742 Cond = PromoteTargetBoolean(Cond, OpVT); 1743 1744 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 1745 N->getOperand(2)), 0); 1746 } 1747 1748 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 1749 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1750 1751 SDValue LHS = N->getOperand(0); 1752 SDValue RHS = N->getOperand(1); 1753 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1754 1755 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1756 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1757 N->getOperand(3), N->getOperand(4)), 0); 1758 } 1759 1760 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1761 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1762 1763 SDValue LHS = N->getOperand(0); 1764 SDValue RHS = N->getOperand(1); 1765 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 1766 1767 // The CC (#2) is always legal. 1768 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 1769 } 1770 1771 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 1772 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1773 ZExtPromotedInteger(N->getOperand(1))), 0); 1774 } 1775 1776 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 1777 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1778 SDLoc dl(N); 1779 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1780 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 1781 Op, DAG.getValueType(N->getOperand(0).getValueType())); 1782 } 1783 1784 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 1785 return SDValue(DAG.UpdateNodeOperands(N, 1786 SExtPromotedInteger(N->getOperand(0))), 0); 1787 } 1788 1789 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { 1790 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1791 SExtPromotedInteger(N->getOperand(1))), 0); 1792 } 1793 1794 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 1795 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1796 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 1797 SDLoc dl(N); 1798 1799 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 1800 1801 // Truncate the value and store the result. 1802 return DAG.getTruncStore(Ch, dl, Val, Ptr, 1803 N->getMemoryVT(), N->getMemOperand()); 1804 } 1805 1806 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, 1807 unsigned OpNo) { 1808 1809 SDValue DataOp = N->getValue(); 1810 EVT DataVT = DataOp.getValueType(); 1811 SDValue Mask = N->getMask(); 1812 SDLoc dl(N); 1813 1814 bool TruncateStore = false; 1815 if (OpNo == 4) { 1816 Mask = PromoteTargetBoolean(Mask, DataVT); 1817 // Update in place. 1818 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 1819 NewOps[4] = Mask; 1820 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1821 } else { // Data operand 1822 assert(OpNo == 1 && "Unexpected operand for promotion"); 1823 DataOp = GetPromotedInteger(DataOp); 1824 TruncateStore = true; 1825 } 1826 1827 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), 1828 N->getOffset(), Mask, N->getMemoryVT(), 1829 N->getMemOperand(), N->getAddressingMode(), 1830 TruncateStore, N->isCompressingStore()); 1831 } 1832 1833 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, 1834 unsigned OpNo) { 1835 assert(OpNo == 3 && "Only know how to promote the mask!"); 1836 EVT DataVT = N->getValueType(0); 1837 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1838 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 1839 NewOps[OpNo] = Mask; 1840 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 1841 if (Res == N) 1842 return SDValue(Res, 0); 1843 1844 // Update triggered CSE, do our own replacement since caller can't. 1845 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 1846 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 1847 return SDValue(); 1848 } 1849 1850 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, 1851 unsigned OpNo) { 1852 1853 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1854 if (OpNo == 2) { 1855 // The Mask 1856 EVT DataVT = N->getValueType(0); 1857 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1858 } else if (OpNo == 4) { 1859 // The Index 1860 if (N->isIndexSigned()) 1861 // Need to sign extend the index since the bits will likely be used. 1862 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 1863 else 1864 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 1865 } else 1866 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1867 1868 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 1869 if (Res == N) 1870 return SDValue(Res, 0); 1871 1872 // Update triggered CSE, do our own replacement since caller can't. 1873 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 1874 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 1875 return SDValue(); 1876 } 1877 1878 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, 1879 unsigned OpNo) { 1880 bool TruncateStore = N->isTruncatingStore(); 1881 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1882 if (OpNo == 2) { 1883 // The Mask 1884 EVT DataVT = N->getValue().getValueType(); 1885 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1886 } else if (OpNo == 4) { 1887 // The Index 1888 if (N->isIndexSigned()) 1889 // Need to sign extend the index since the bits will likely be used. 1890 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 1891 else 1892 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 1893 1894 N->setIndexType(TLI.getCanonicalIndexType(N->getIndexType(), 1895 N->getMemoryVT(), NewOps[OpNo])); 1896 } else { 1897 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1898 TruncateStore = true; 1899 } 1900 1901 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), 1902 SDLoc(N), NewOps, N->getMemOperand(), 1903 N->getIndexType(), TruncateStore); 1904 } 1905 1906 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 1907 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1908 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 1909 } 1910 1911 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 1912 return SDValue(DAG.UpdateNodeOperands(N, 1913 ZExtPromotedInteger(N->getOperand(0))), 0); 1914 } 1915 1916 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { 1917 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1918 ZExtPromotedInteger(N->getOperand(1))), 0); 1919 } 1920 1921 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 1922 SDLoc dl(N); 1923 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1924 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1925 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); 1926 } 1927 1928 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { 1929 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1930 1931 SDValue LHS = N->getOperand(0); 1932 SDValue RHS = N->getOperand(1); 1933 SDValue Carry = N->getOperand(2); 1934 SDLoc DL(N); 1935 1936 Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); 1937 1938 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); 1939 } 1940 1941 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { 1942 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 1943 return SDValue( 1944 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); 1945 } 1946 1947 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { 1948 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. 1949 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 1950 return SDValue(DAG.UpdateNodeOperands(N, Op), 0); 1951 } 1952 1953 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) { 1954 assert(OpNo > 1 && "Don't know how to promote this operand!"); 1955 // Promote the rw, locality, and cache type arguments to a supported integer 1956 // width. 1957 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 1958 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3)); 1959 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4)); 1960 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), 1961 Op2, Op3, Op4), 1962 0); 1963 } 1964 1965 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) { 1966 SDValue Op = SExtPromotedInteger(N->getOperand(1)); 1967 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); 1968 } 1969 1970 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { 1971 SDLoc dl(N); 1972 SDValue Op; 1973 switch (N->getOpcode()) { 1974 default: llvm_unreachable("Expected integer vector reduction"); 1975 case ISD::VECREDUCE_ADD: 1976 case ISD::VECREDUCE_MUL: 1977 case ISD::VECREDUCE_AND: 1978 case ISD::VECREDUCE_OR: 1979 case ISD::VECREDUCE_XOR: 1980 Op = GetPromotedInteger(N->getOperand(0)); 1981 break; 1982 case ISD::VECREDUCE_SMAX: 1983 case ISD::VECREDUCE_SMIN: 1984 Op = SExtPromotedInteger(N->getOperand(0)); 1985 break; 1986 case ISD::VECREDUCE_UMAX: 1987 case ISD::VECREDUCE_UMIN: 1988 Op = ZExtPromotedInteger(N->getOperand(0)); 1989 break; 1990 } 1991 1992 EVT EltVT = Op.getValueType().getVectorElementType(); 1993 EVT VT = N->getValueType(0); 1994 if (VT.bitsGE(EltVT)) 1995 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op); 1996 1997 // Result size must be >= element size. If this is not the case after 1998 // promotion, also promote the result type and then truncate. 1999 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op); 2000 return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce); 2001 } 2002 2003 //===----------------------------------------------------------------------===// 2004 // Integer Result Expansion 2005 //===----------------------------------------------------------------------===// 2006 2007 /// ExpandIntegerResult - This method is called when the specified result of the 2008 /// specified node is found to need expansion. At this point, the node may also 2009 /// have invalid operands or may have other results that need promotion, we just 2010 /// know that (at least) one result needs expansion. 2011 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 2012 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); 2013 dbgs() << "\n"); 2014 SDValue Lo, Hi; 2015 Lo = Hi = SDValue(); 2016 2017 // See if the target wants to custom expand this node. 2018 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 2019 return; 2020 2021 switch (N->getOpcode()) { 2022 default: 2023 #ifndef NDEBUG 2024 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 2025 N->dump(&DAG); dbgs() << "\n"; 2026 #endif 2027 report_fatal_error("Do not know how to expand the result of this " 2028 "operator!"); 2029 2030 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 2031 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 2032 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 2033 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 2034 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; 2035 2036 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 2037 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 2038 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 2039 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 2040 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 2041 2042 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 2043 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 2044 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 2045 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; 2046 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 2047 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; 2048 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 2049 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; 2050 case ISD::CTLZ_ZERO_UNDEF: 2051 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 2052 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 2053 case ISD::CTTZ_ZERO_UNDEF: 2054 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 2055 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break; 2056 case ISD::STRICT_FP_TO_SINT: 2057 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 2058 case ISD::STRICT_FP_TO_UINT: 2059 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 2060 case ISD::FP_TO_SINT_SAT: 2061 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; 2062 case ISD::STRICT_LLROUND: 2063 case ISD::STRICT_LLRINT: 2064 case ISD::LLROUND: 2065 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break; 2066 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 2067 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 2068 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; 2069 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 2070 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 2071 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 2072 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 2073 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 2074 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 2075 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 2076 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 2077 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 2078 2079 case ISD::ATOMIC_LOAD_ADD: 2080 case ISD::ATOMIC_LOAD_SUB: 2081 case ISD::ATOMIC_LOAD_AND: 2082 case ISD::ATOMIC_LOAD_CLR: 2083 case ISD::ATOMIC_LOAD_OR: 2084 case ISD::ATOMIC_LOAD_XOR: 2085 case ISD::ATOMIC_LOAD_NAND: 2086 case ISD::ATOMIC_LOAD_MIN: 2087 case ISD::ATOMIC_LOAD_MAX: 2088 case ISD::ATOMIC_LOAD_UMIN: 2089 case ISD::ATOMIC_LOAD_UMAX: 2090 case ISD::ATOMIC_SWAP: 2091 case ISD::ATOMIC_CMP_SWAP: { 2092 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 2093 SplitInteger(Tmp.first, Lo, Hi); 2094 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2095 break; 2096 } 2097 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 2098 AtomicSDNode *AN = cast<AtomicSDNode>(N); 2099 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 2100 SDValue Tmp = DAG.getAtomicCmpSwap( 2101 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 2102 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 2103 AN->getMemOperand()); 2104 2105 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 2106 // success simply by comparing the loaded value against the ingoing 2107 // comparison. 2108 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 2109 N->getOperand(2), ISD::SETEQ); 2110 2111 SplitInteger(Tmp, Lo, Hi); 2112 ReplaceValueWith(SDValue(N, 1), Success); 2113 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 2114 break; 2115 } 2116 2117 case ISD::AND: 2118 case ISD::OR: 2119 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 2120 2121 case ISD::UMAX: 2122 case ISD::SMAX: 2123 case ISD::UMIN: 2124 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; 2125 2126 case ISD::ADD: 2127 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 2128 2129 case ISD::ADDC: 2130 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 2131 2132 case ISD::ADDE: 2133 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 2134 2135 case ISD::ADDCARRY: 2136 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break; 2137 2138 case ISD::SADDO_CARRY: 2139 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; 2140 2141 case ISD::SHL: 2142 case ISD::SRA: 2143 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 2144 2145 case ISD::SADDO: 2146 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 2147 case ISD::UADDO: 2148 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 2149 case ISD::UMULO: 2150 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 2151 2152 case ISD::SADDSAT: 2153 case ISD::UADDSAT: 2154 case ISD::SSUBSAT: 2155 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; 2156 2157 case ISD::SSHLSAT: 2158 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; 2159 2160 case ISD::SMULFIX: 2161 case ISD::SMULFIXSAT: 2162 case ISD::UMULFIX: 2163 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; 2164 2165 case ISD::SDIVFIX: 2166 case ISD::SDIVFIXSAT: 2167 case ISD::UDIVFIX: 2168 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; 2169 2170 case ISD::VECREDUCE_ADD: 2171 case ISD::VECREDUCE_MUL: 2172 case ISD::VECREDUCE_AND: 2173 case ISD::VECREDUCE_OR: 2174 case ISD::VECREDUCE_XOR: 2175 case ISD::VECREDUCE_SMAX: 2176 case ISD::VECREDUCE_SMIN: 2177 case ISD::VECREDUCE_UMAX: 2178 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; 2179 2180 case ISD::ROTL: 2181 case ISD::ROTR: 2182 ExpandIntRes_Rotate(N, Lo, Hi); 2183 break; 2184 2185 case ISD::FSHL: 2186 case ISD::FSHR: 2187 ExpandIntRes_FunnelShift(N, Lo, Hi); 2188 break; 2189 } 2190 2191 // If Lo/Hi is null, the sub-method took care of registering results etc. 2192 if (Lo.getNode()) 2193 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 2194 } 2195 2196 /// Lower an atomic node to the appropriate builtin call. 2197 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 2198 unsigned Opc = Node->getOpcode(); 2199 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2200 AtomicOrdering order = cast<AtomicSDNode>(Node)->getOrdering(); 2201 // Lower to outline atomic libcall if outline atomics enabled, 2202 // or to sync libcall otherwise 2203 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); 2204 EVT RetVT = Node->getValueType(0); 2205 TargetLowering::MakeLibCallOptions CallOptions; 2206 SmallVector<SDValue, 4> Ops; 2207 if (TLI.getLibcallName(LC)) { 2208 Ops.append(Node->op_begin() + 2, Node->op_end()); 2209 Ops.push_back(Node->getOperand(1)); 2210 } else { 2211 LC = RTLIB::getSYNC(Opc, VT); 2212 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2213 "Unexpected atomic op or value type!"); 2214 Ops.append(Node->op_begin() + 1, Node->op_end()); 2215 } 2216 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), 2217 Node->getOperand(0)); 2218 } 2219 2220 /// N is a shift by a value that needs to be expanded, 2221 /// and the shift amount is a constant 'Amt'. Expand the operation. 2222 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, 2223 SDValue &Lo, SDValue &Hi) { 2224 SDLoc DL(N); 2225 // Expand the incoming operand to be shifted, so that we have its parts 2226 SDValue InL, InH; 2227 GetExpandedInteger(N->getOperand(0), InL, InH); 2228 2229 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization 2230 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. 2231 if (!Amt) { 2232 Lo = InL; 2233 Hi = InH; 2234 return; 2235 } 2236 2237 EVT NVT = InL.getValueType(); 2238 unsigned VTBits = N->getValueType(0).getSizeInBits(); 2239 unsigned NVTBits = NVT.getSizeInBits(); 2240 EVT ShTy = N->getOperand(1).getValueType(); 2241 2242 if (N->getOpcode() == ISD::SHL) { 2243 if (Amt.ugt(VTBits)) { 2244 Lo = Hi = DAG.getConstant(0, DL, NVT); 2245 } else if (Amt.ugt(NVTBits)) { 2246 Lo = DAG.getConstant(0, DL, NVT); 2247 Hi = DAG.getNode(ISD::SHL, DL, 2248 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2249 } else if (Amt == NVTBits) { 2250 Lo = DAG.getConstant(0, DL, NVT); 2251 Hi = InL; 2252 } else { 2253 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); 2254 Hi = DAG.getNode(ISD::OR, DL, NVT, 2255 DAG.getNode(ISD::SHL, DL, NVT, InH, 2256 DAG.getConstant(Amt, DL, ShTy)), 2257 DAG.getNode(ISD::SRL, DL, NVT, InL, 2258 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2259 } 2260 return; 2261 } 2262 2263 if (N->getOpcode() == ISD::SRL) { 2264 if (Amt.ugt(VTBits)) { 2265 Lo = Hi = DAG.getConstant(0, DL, NVT); 2266 } else if (Amt.ugt(NVTBits)) { 2267 Lo = DAG.getNode(ISD::SRL, DL, 2268 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2269 Hi = DAG.getConstant(0, DL, NVT); 2270 } else if (Amt == NVTBits) { 2271 Lo = InH; 2272 Hi = DAG.getConstant(0, DL, NVT); 2273 } else { 2274 Lo = DAG.getNode(ISD::OR, DL, NVT, 2275 DAG.getNode(ISD::SRL, DL, NVT, InL, 2276 DAG.getConstant(Amt, DL, ShTy)), 2277 DAG.getNode(ISD::SHL, DL, NVT, InH, 2278 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2279 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2280 } 2281 return; 2282 } 2283 2284 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2285 if (Amt.ugt(VTBits)) { 2286 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2287 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2288 } else if (Amt.ugt(NVTBits)) { 2289 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2290 DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2291 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2292 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2293 } else if (Amt == NVTBits) { 2294 Lo = InH; 2295 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2296 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2297 } else { 2298 Lo = DAG.getNode(ISD::OR, DL, NVT, 2299 DAG.getNode(ISD::SRL, DL, NVT, InL, 2300 DAG.getConstant(Amt, DL, ShTy)), 2301 DAG.getNode(ISD::SHL, DL, NVT, InH, 2302 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2303 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2304 } 2305 } 2306 2307 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 2308 /// this shift based on knowledge of the high bit of the shift amount. If we 2309 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual 2310 /// shift amount. 2311 bool DAGTypeLegalizer:: 2312 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2313 SDValue Amt = N->getOperand(1); 2314 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2315 EVT ShTy = Amt.getValueType(); 2316 unsigned ShBits = ShTy.getScalarSizeInBits(); 2317 unsigned NVTBits = NVT.getScalarSizeInBits(); 2318 assert(isPowerOf2_32(NVTBits) && 2319 "Expanded integer type size not a power of two!"); 2320 SDLoc dl(N); 2321 2322 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 2323 KnownBits Known = DAG.computeKnownBits(N->getOperand(1)); 2324 2325 // If we don't know anything about the high bits, exit. 2326 if (((Known.Zero|Known.One) & HighBitMask) == 0) 2327 return false; 2328 2329 // Get the incoming operand to be shifted. 2330 SDValue InL, InH; 2331 GetExpandedInteger(N->getOperand(0), InL, InH); 2332 2333 // If we know that any of the high bits of the shift amount are one, then we 2334 // can do this as a couple of simple shifts. 2335 if (Known.One.intersects(HighBitMask)) { 2336 // Mask out the high bit, which we know is set. 2337 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 2338 DAG.getConstant(~HighBitMask, dl, ShTy)); 2339 2340 switch (N->getOpcode()) { 2341 default: llvm_unreachable("Unknown shift"); 2342 case ISD::SHL: 2343 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. 2344 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 2345 return true; 2346 case ISD::SRL: 2347 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2348 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 2349 return true; 2350 case ISD::SRA: 2351 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 2352 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2353 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 2354 return true; 2355 } 2356 } 2357 2358 // If we know that all of the high bits of the shift amount are zero, then we 2359 // can do this as a couple of simple shifts. 2360 if (HighBitMask.isSubsetOf(Known.Zero)) { 2361 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 2362 // shift if x is zero. We can use XOR here because x is known to be smaller 2363 // than 32. 2364 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 2365 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2366 2367 unsigned Op1, Op2; 2368 switch (N->getOpcode()) { 2369 default: llvm_unreachable("Unknown shift"); 2370 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 2371 case ISD::SRL: 2372 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 2373 } 2374 2375 // When shifting right the arithmetic for Lo and Hi is swapped. 2376 if (N->getOpcode() != ISD::SHL) 2377 std::swap(InL, InH); 2378 2379 // Use a little trick to get the bits that move from Lo to Hi. First 2380 // shift by one bit. 2381 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); 2382 // Then compute the remaining shift with amount-1. 2383 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 2384 2385 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 2386 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 2387 2388 if (N->getOpcode() != ISD::SHL) 2389 std::swap(Hi, Lo); 2390 return true; 2391 } 2392 2393 return false; 2394 } 2395 2396 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 2397 /// of any size. 2398 bool DAGTypeLegalizer:: 2399 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2400 SDValue Amt = N->getOperand(1); 2401 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2402 EVT ShTy = Amt.getValueType(); 2403 unsigned NVTBits = NVT.getSizeInBits(); 2404 assert(isPowerOf2_32(NVTBits) && 2405 "Expanded integer type size not a power of two!"); 2406 SDLoc dl(N); 2407 2408 // Get the incoming operand to be shifted. 2409 SDValue InL, InH; 2410 GetExpandedInteger(N->getOperand(0), InL, InH); 2411 2412 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); 2413 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 2414 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 2415 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2416 Amt, NVBitsNode, ISD::SETULT); 2417 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2418 Amt, DAG.getConstant(0, dl, ShTy), 2419 ISD::SETEQ); 2420 2421 SDValue LoS, HiS, LoL, HiL; 2422 switch (N->getOpcode()) { 2423 default: llvm_unreachable("Unknown shift"); 2424 case ISD::SHL: 2425 // Short: ShAmt < NVTBits 2426 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 2427 HiS = DAG.getNode(ISD::OR, dl, NVT, 2428 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 2429 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 2430 2431 // Long: ShAmt >= NVTBits 2432 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. 2433 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 2434 2435 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 2436 Hi = DAG.getSelect(dl, NVT, isZero, InH, 2437 DAG.getSelect(dl, NVT, isShort, HiS, HiL)); 2438 return true; 2439 case ISD::SRL: 2440 // Short: ShAmt < NVTBits 2441 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 2442 LoS = DAG.getNode(ISD::OR, dl, NVT, 2443 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2444 // FIXME: If Amt is zero, the following shift generates an undefined result 2445 // on some architectures. 2446 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2447 2448 // Long: ShAmt >= NVTBits 2449 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2450 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2451 2452 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2453 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2454 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2455 return true; 2456 case ISD::SRA: 2457 // Short: ShAmt < NVTBits 2458 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 2459 LoS = DAG.getNode(ISD::OR, dl, NVT, 2460 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2461 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2462 2463 // Long: ShAmt >= NVTBits 2464 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 2465 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2466 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2467 2468 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2469 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2470 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2471 return true; 2472 } 2473 } 2474 2475 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { 2476 2477 switch (Op) { 2478 default: llvm_unreachable("invalid min/max opcode"); 2479 case ISD::SMAX: 2480 return std::make_pair(ISD::SETGT, ISD::UMAX); 2481 case ISD::UMAX: 2482 return std::make_pair(ISD::SETUGT, ISD::UMAX); 2483 case ISD::SMIN: 2484 return std::make_pair(ISD::SETLT, ISD::UMIN); 2485 case ISD::UMIN: 2486 return std::make_pair(ISD::SETULT, ISD::UMIN); 2487 } 2488 } 2489 2490 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, 2491 SDValue &Lo, SDValue &Hi) { 2492 SDLoc DL(N); 2493 ISD::NodeType LoOpc; 2494 ISD::CondCode CondC; 2495 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); 2496 2497 // Expand the subcomponents. 2498 SDValue LHSL, LHSH, RHSL, RHSH; 2499 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2500 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2501 2502 // Value types 2503 EVT NVT = LHSL.getValueType(); 2504 EVT CCT = getSetCCResultType(NVT); 2505 2506 // Hi part is always the same op 2507 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 2508 2509 // We need to know whether to select Lo part that corresponds to 'winning' 2510 // Hi part or if Hi parts are equal. 2511 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); 2512 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); 2513 2514 // Lo part corresponding to the 'winning' Hi part 2515 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); 2516 2517 // Recursed Lo part if Hi parts are equal, this uses unsigned version 2518 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); 2519 2520 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); 2521 } 2522 2523 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 2524 SDValue &Lo, SDValue &Hi) { 2525 SDLoc dl(N); 2526 // Expand the subcomponents. 2527 SDValue LHSL, LHSH, RHSL, RHSH; 2528 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2529 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2530 2531 EVT NVT = LHSL.getValueType(); 2532 SDValue LoOps[2] = { LHSL, RHSL }; 2533 SDValue HiOps[3] = { LHSH, RHSH }; 2534 2535 bool HasOpCarry = TLI.isOperationLegalOrCustom( 2536 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY, 2537 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2538 if (HasOpCarry) { 2539 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 2540 if (N->getOpcode() == ISD::ADD) { 2541 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 2542 HiOps[2] = Lo.getValue(1); 2543 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps); 2544 } else { 2545 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 2546 HiOps[2] = Lo.getValue(1); 2547 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps); 2548 } 2549 return; 2550 } 2551 2552 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 2553 // them. TODO: Teach operation legalization how to expand unsupported 2554 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 2555 // a carry of type MVT::Glue, but there doesn't seem to be any way to 2556 // generate a value of this type in the expanded code sequence. 2557 bool hasCarry = 2558 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 2559 ISD::ADDC : ISD::SUBC, 2560 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2561 2562 if (hasCarry) { 2563 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 2564 if (N->getOpcode() == ISD::ADD) { 2565 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 2566 HiOps[2] = Lo.getValue(1); 2567 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 2568 } else { 2569 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 2570 HiOps[2] = Lo.getValue(1); 2571 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 2572 } 2573 return; 2574 } 2575 2576 bool hasOVF = 2577 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 2578 ISD::UADDO : ISD::USUBO, 2579 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2580 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); 2581 2582 if (hasOVF) { 2583 EVT OvfVT = getSetCCResultType(NVT); 2584 SDVTList VTList = DAG.getVTList(NVT, OvfVT); 2585 int RevOpc; 2586 if (N->getOpcode() == ISD::ADD) { 2587 RevOpc = ISD::SUB; 2588 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 2589 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 2590 } else { 2591 RevOpc = ISD::ADD; 2592 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 2593 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 2594 } 2595 SDValue OVF = Lo.getValue(1); 2596 2597 switch (BoolType) { 2598 case TargetLoweringBase::UndefinedBooleanContent: 2599 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); 2600 LLVM_FALLTHROUGH; 2601 case TargetLoweringBase::ZeroOrOneBooleanContent: 2602 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); 2603 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); 2604 break; 2605 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 2606 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); 2607 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); 2608 } 2609 return; 2610 } 2611 2612 if (N->getOpcode() == ISD::ADD) { 2613 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 2614 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 2615 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 2616 ISD::SETULT); 2617 2618 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) { 2619 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT); 2620 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); 2621 return; 2622 } 2623 2624 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1, 2625 DAG.getConstant(1, dl, NVT), 2626 DAG.getConstant(0, dl, NVT)); 2627 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1], 2628 ISD::SETULT); 2629 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2, 2630 DAG.getConstant(1, dl, NVT), Carry1); 2631 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 2632 } else { 2633 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 2634 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 2635 SDValue Cmp = 2636 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 2637 LoOps[0], LoOps[1], ISD::SETULT); 2638 2639 SDValue Borrow; 2640 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 2641 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); 2642 else 2643 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 2644 DAG.getConstant(0, dl, NVT)); 2645 2646 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 2647 } 2648 } 2649 2650 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 2651 SDValue &Lo, SDValue &Hi) { 2652 // Expand the subcomponents. 2653 SDValue LHSL, LHSH, RHSL, RHSH; 2654 SDLoc dl(N); 2655 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2656 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2657 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 2658 SDValue LoOps[2] = { LHSL, RHSL }; 2659 SDValue HiOps[3] = { LHSH, RHSH }; 2660 2661 if (N->getOpcode() == ISD::ADDC) { 2662 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 2663 HiOps[2] = Lo.getValue(1); 2664 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 2665 } else { 2666 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 2667 HiOps[2] = Lo.getValue(1); 2668 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 2669 } 2670 2671 // Legalized the flag result - switch anything that used the old flag to 2672 // use the new one. 2673 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2674 } 2675 2676 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 2677 SDValue &Lo, SDValue &Hi) { 2678 // Expand the subcomponents. 2679 SDValue LHSL, LHSH, RHSL, RHSH; 2680 SDLoc dl(N); 2681 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2682 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2683 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 2684 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 2685 SDValue HiOps[3] = { LHSH, RHSH }; 2686 2687 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2688 HiOps[2] = Lo.getValue(1); 2689 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 2690 2691 // Legalized the flag result - switch anything that used the old flag to 2692 // use the new one. 2693 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2694 } 2695 2696 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 2697 SDValue &Lo, SDValue &Hi) { 2698 SDValue LHS = N->getOperand(0); 2699 SDValue RHS = N->getOperand(1); 2700 SDLoc dl(N); 2701 2702 SDValue Ovf; 2703 2704 unsigned CarryOp, NoCarryOp; 2705 ISD::CondCode Cond; 2706 switch(N->getOpcode()) { 2707 case ISD::UADDO: 2708 CarryOp = ISD::ADDCARRY; 2709 NoCarryOp = ISD::ADD; 2710 Cond = ISD::SETULT; 2711 break; 2712 case ISD::USUBO: 2713 CarryOp = ISD::SUBCARRY; 2714 NoCarryOp = ISD::SUB; 2715 Cond = ISD::SETUGT; 2716 break; 2717 default: 2718 llvm_unreachable("Node has unexpected Opcode"); 2719 } 2720 2721 bool HasCarryOp = TLI.isOperationLegalOrCustom( 2722 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 2723 2724 if (HasCarryOp) { 2725 // Expand the subcomponents. 2726 SDValue LHSL, LHSH, RHSL, RHSH; 2727 GetExpandedInteger(LHS, LHSL, LHSH); 2728 GetExpandedInteger(RHS, RHSL, RHSH); 2729 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 2730 SDValue LoOps[2] = { LHSL, RHSL }; 2731 SDValue HiOps[3] = { LHSH, RHSH }; 2732 2733 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2734 HiOps[2] = Lo.getValue(1); 2735 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps); 2736 2737 Ovf = Hi.getValue(1); 2738 } else { 2739 // Expand the result by simply replacing it with the equivalent 2740 // non-overflow-checking operation. 2741 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS); 2742 SplitInteger(Sum, Lo, Hi); 2743 2744 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 2745 // overflows iff a - b > a. 2746 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); 2747 } 2748 2749 // Legalized the flag result - switch anything that used the old flag to 2750 // use the new one. 2751 ReplaceValueWith(SDValue(N, 1), Ovf); 2752 } 2753 2754 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N, 2755 SDValue &Lo, SDValue &Hi) { 2756 // Expand the subcomponents. 2757 SDValue LHSL, LHSH, RHSL, RHSH; 2758 SDLoc dl(N); 2759 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2760 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2761 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 2762 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 2763 SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; 2764 2765 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2766 HiOps[2] = Lo.getValue(1); 2767 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 2768 2769 // Legalized the flag result - switch anything that used the old flag to 2770 // use the new one. 2771 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2772 } 2773 2774 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, 2775 SDValue &Lo, SDValue &Hi) { 2776 // Expand the subcomponents. 2777 SDValue LHSL, LHSH, RHSL, RHSH; 2778 SDLoc dl(N); 2779 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2780 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2781 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 2782 2783 // We need to use an unsigned carry op for the lo part. 2784 unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY 2785 : ISD::SUBCARRY; 2786 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) }); 2787 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 2788 2789 // Legalized the flag result - switch anything that used the old flag to 2790 // use the new one. 2791 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2792 } 2793 2794 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 2795 SDValue &Lo, SDValue &Hi) { 2796 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2797 SDLoc dl(N); 2798 SDValue Op = N->getOperand(0); 2799 if (Op.getValueType().bitsLE(NVT)) { 2800 // The low part is any extension of the input (which degenerates to a copy). 2801 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 2802 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 2803 } else { 2804 // For example, extension of an i48 to an i64. The operand type necessarily 2805 // promotes to the result type, so will end up being expanded too. 2806 assert(getTypeAction(Op.getValueType()) == 2807 TargetLowering::TypePromoteInteger && 2808 "Only know how to promote this result!"); 2809 SDValue Res = GetPromotedInteger(Op); 2810 assert(Res.getValueType() == N->getValueType(0) && 2811 "Operand over promoted?"); 2812 // Split the promoted operand. This will simplify when it is expanded. 2813 SplitInteger(Res, Lo, Hi); 2814 } 2815 } 2816 2817 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 2818 SDValue &Lo, SDValue &Hi) { 2819 SDLoc dl(N); 2820 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2821 EVT NVT = Lo.getValueType(); 2822 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2823 unsigned NVTBits = NVT.getSizeInBits(); 2824 unsigned EVTBits = EVT.getSizeInBits(); 2825 2826 if (NVTBits < EVTBits) { 2827 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 2828 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2829 EVTBits - NVTBits))); 2830 } else { 2831 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 2832 // The high part replicates the sign bit of Lo, make it explicit. 2833 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2834 DAG.getConstant(NVTBits - 1, dl, 2835 TLI.getPointerTy(DAG.getDataLayout()))); 2836 } 2837 } 2838 2839 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 2840 SDValue &Lo, SDValue &Hi) { 2841 SDLoc dl(N); 2842 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2843 EVT NVT = Lo.getValueType(); 2844 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2845 unsigned NVTBits = NVT.getSizeInBits(); 2846 unsigned EVTBits = EVT.getSizeInBits(); 2847 2848 if (NVTBits < EVTBits) { 2849 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 2850 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2851 EVTBits - NVTBits))); 2852 } else { 2853 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 2854 // The high part must be zero, make it explicit. 2855 Hi = DAG.getConstant(0, dl, NVT); 2856 } 2857 } 2858 2859 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, 2860 SDValue &Lo, SDValue &Hi) { 2861 SDLoc dl(N); 2862 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 2863 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); 2864 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); 2865 } 2866 2867 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 2868 SDValue &Lo, SDValue &Hi) { 2869 SDLoc dl(N); 2870 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 2871 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 2872 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 2873 } 2874 2875 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, 2876 SDValue &Hi) { 2877 SDLoc dl(N); 2878 // parity(HiLo) -> parity(Lo^Hi) 2879 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2880 EVT NVT = Lo.getValueType(); 2881 Lo = 2882 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi)); 2883 Hi = DAG.getConstant(0, dl, NVT); 2884 } 2885 2886 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 2887 SDValue &Lo, SDValue &Hi) { 2888 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2889 unsigned NBitWidth = NVT.getSizeInBits(); 2890 auto Constant = cast<ConstantSDNode>(N); 2891 const APInt &Cst = Constant->getAPIntValue(); 2892 bool IsTarget = Constant->isTargetOpcode(); 2893 bool IsOpaque = Constant->isOpaque(); 2894 SDLoc dl(N); 2895 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); 2896 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, 2897 IsOpaque); 2898 } 2899 2900 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { 2901 SDLoc dl(N); 2902 2903 SDValue N0 = N->getOperand(0); 2904 GetExpandedInteger(N0, Lo, Hi); 2905 EVT NVT = Lo.getValueType(); 2906 2907 // If we have ADDCARRY, use the expanded form of the sra+add+xor sequence we 2908 // use in LegalizeDAG. The ADD part of the expansion is based on 2909 // ExpandIntRes_ADDSUB which also uses ADDCARRY/UADDO after checking that 2910 // ADDCARRY is LegalOrCustom. Each of the pieces here can be further expanded 2911 // if needed. Shift expansion has a special case for filling with sign bits 2912 // so that we will only end up with one SRA. 2913 bool HasAddCarry = TLI.isOperationLegalOrCustom( 2914 ISD::ADDCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2915 if (HasAddCarry) { 2916 EVT ShiftAmtTy = getShiftAmountTyForConstant(NVT, TLI, DAG); 2917 SDValue Sign = 2918 DAG.getNode(ISD::SRA, dl, NVT, Hi, 2919 DAG.getConstant(NVT.getSizeInBits() - 1, dl, ShiftAmtTy)); 2920 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 2921 Lo = DAG.getNode(ISD::UADDO, dl, VTList, Lo, Sign); 2922 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Hi, Sign, Lo.getValue(1)); 2923 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign); 2924 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign); 2925 return; 2926 } 2927 2928 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) 2929 EVT VT = N->getValueType(0); 2930 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, 2931 DAG.getConstant(0, dl, VT), N0); 2932 SDValue NegLo, NegHi; 2933 SplitInteger(Neg, NegLo, NegHi); 2934 2935 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), 2936 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT); 2937 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); 2938 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); 2939 } 2940 2941 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 2942 SDValue &Lo, SDValue &Hi) { 2943 SDLoc dl(N); 2944 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 2945 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2946 EVT NVT = Lo.getValueType(); 2947 2948 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 2949 DAG.getConstant(0, dl, NVT), ISD::SETNE); 2950 2951 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 2952 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 2953 2954 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 2955 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 2956 DAG.getConstant(NVT.getSizeInBits(), dl, 2957 NVT))); 2958 Hi = DAG.getConstant(0, dl, NVT); 2959 } 2960 2961 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 2962 SDValue &Lo, SDValue &Hi) { 2963 SDLoc dl(N); 2964 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 2965 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2966 EVT NVT = Lo.getValueType(); 2967 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 2968 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 2969 Hi = DAG.getConstant(0, dl, NVT); 2970 } 2971 2972 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 2973 SDValue &Lo, SDValue &Hi) { 2974 SDLoc dl(N); 2975 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 2976 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2977 EVT NVT = Lo.getValueType(); 2978 2979 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 2980 DAG.getConstant(0, dl, NVT), ISD::SETNE); 2981 2982 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 2983 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 2984 2985 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 2986 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 2987 DAG.getConstant(NVT.getSizeInBits(), dl, 2988 NVT))); 2989 Hi = DAG.getConstant(0, dl, NVT); 2990 } 2991 2992 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo, 2993 SDValue &Hi) { 2994 SDLoc dl(N); 2995 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2996 unsigned NBitWidth = NVT.getSizeInBits(); 2997 2998 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 2999 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0)); 3000 SDValue Chain = Lo.getValue(1); 3001 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS 3002 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3003 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy)); 3004 3005 // Legalize the chain result - switch anything that used the old chain to 3006 // use the new one. 3007 ReplaceValueWith(SDValue(N, 1), Chain); 3008 } 3009 3010 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 3011 SDValue &Hi) { 3012 SDLoc dl(N); 3013 EVT VT = N->getValueType(0); 3014 3015 bool IsStrict = N->isStrictFPOpcode(); 3016 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3017 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3018 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3019 Op = GetPromotedFloat(Op); 3020 3021 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3022 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3023 Op = GetSoftPromotedHalf(Op); 3024 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); 3025 } 3026 3027 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 3028 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 3029 TargetLowering::MakeLibCallOptions CallOptions; 3030 CallOptions.setSExt(true); 3031 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3032 CallOptions, dl, Chain); 3033 SplitInteger(Tmp.first, Lo, Hi); 3034 3035 if (IsStrict) 3036 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3037 } 3038 3039 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 3040 SDValue &Hi) { 3041 SDLoc dl(N); 3042 EVT VT = N->getValueType(0); 3043 3044 bool IsStrict = N->isStrictFPOpcode(); 3045 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3046 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3047 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3048 Op = GetPromotedFloat(Op); 3049 3050 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3051 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3052 Op = GetSoftPromotedHalf(Op); 3053 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); 3054 } 3055 3056 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 3057 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 3058 TargetLowering::MakeLibCallOptions CallOptions; 3059 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3060 CallOptions, dl, Chain); 3061 SplitInteger(Tmp.first, Lo, Hi); 3062 3063 if (IsStrict) 3064 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3065 } 3066 3067 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, 3068 SDValue &Hi) { 3069 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 3070 SplitInteger(Res, Lo, Hi); 3071 } 3072 3073 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo, 3074 SDValue &Hi) { 3075 SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); 3076 3077 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && 3078 "Input type needs to be promoted!"); 3079 3080 EVT VT = Op.getValueType(); 3081 3082 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3083 if (N->getOpcode() == ISD::LLROUND || 3084 N->getOpcode() == ISD::STRICT_LLROUND) { 3085 if (VT == MVT::f32) 3086 LC = RTLIB::LLROUND_F32; 3087 else if (VT == MVT::f64) 3088 LC = RTLIB::LLROUND_F64; 3089 else if (VT == MVT::f80) 3090 LC = RTLIB::LLROUND_F80; 3091 else if (VT == MVT::f128) 3092 LC = RTLIB::LLROUND_F128; 3093 else if (VT == MVT::ppcf128) 3094 LC = RTLIB::LLROUND_PPCF128; 3095 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); 3096 } else if (N->getOpcode() == ISD::LLRINT || 3097 N->getOpcode() == ISD::STRICT_LLRINT) { 3098 if (VT == MVT::f32) 3099 LC = RTLIB::LLRINT_F32; 3100 else if (VT == MVT::f64) 3101 LC = RTLIB::LLRINT_F64; 3102 else if (VT == MVT::f80) 3103 LC = RTLIB::LLRINT_F80; 3104 else if (VT == MVT::f128) 3105 LC = RTLIB::LLRINT_F128; 3106 else if (VT == MVT::ppcf128) 3107 LC = RTLIB::LLRINT_PPCF128; 3108 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); 3109 } else 3110 llvm_unreachable("Unexpected opcode!"); 3111 3112 SDLoc dl(N); 3113 EVT RetVT = N->getValueType(0); 3114 SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue(); 3115 3116 TargetLowering::MakeLibCallOptions CallOptions; 3117 CallOptions.setSExt(true); 3118 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, 3119 Op, CallOptions, dl, 3120 Chain); 3121 SplitInteger(Tmp.first, Lo, Hi); 3122 3123 if (N->isStrictFPOpcode()) 3124 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3125 } 3126 3127 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 3128 SDValue &Lo, SDValue &Hi) { 3129 if (N->isAtomic()) { 3130 // It's typical to have larger CAS than atomic load instructions. 3131 SDLoc dl(N); 3132 EVT VT = N->getMemoryVT(); 3133 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 3134 SDValue Zero = DAG.getConstant(0, dl, VT); 3135 SDValue Swap = DAG.getAtomicCmpSwap( 3136 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 3137 VT, VTs, N->getOperand(0), 3138 N->getOperand(1), Zero, Zero, N->getMemOperand()); 3139 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 3140 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 3141 return; 3142 } 3143 3144 if (ISD::isNormalLoad(N)) { 3145 ExpandRes_NormalLoad(N, Lo, Hi); 3146 return; 3147 } 3148 3149 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 3150 3151 EVT VT = N->getValueType(0); 3152 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3153 SDValue Ch = N->getChain(); 3154 SDValue Ptr = N->getBasePtr(); 3155 ISD::LoadExtType ExtType = N->getExtensionType(); 3156 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 3157 AAMDNodes AAInfo = N->getAAInfo(); 3158 SDLoc dl(N); 3159 3160 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 3161 3162 if (N->getMemoryVT().bitsLE(NVT)) { 3163 EVT MemVT = N->getMemoryVT(); 3164 3165 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, 3166 N->getOriginalAlign(), MMOFlags, AAInfo); 3167 3168 // Remember the chain. 3169 Ch = Lo.getValue(1); 3170 3171 if (ExtType == ISD::SEXTLOAD) { 3172 // The high part is obtained by SRA'ing all but one of the bits of the 3173 // lo part. 3174 unsigned LoSize = Lo.getValueSizeInBits(); 3175 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3176 DAG.getConstant(LoSize - 1, dl, 3177 TLI.getPointerTy(DAG.getDataLayout()))); 3178 } else if (ExtType == ISD::ZEXTLOAD) { 3179 // The high part is just a zero. 3180 Hi = DAG.getConstant(0, dl, NVT); 3181 } else { 3182 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 3183 // The high part is undefined. 3184 Hi = DAG.getUNDEF(NVT); 3185 } 3186 } else if (DAG.getDataLayout().isLittleEndian()) { 3187 // Little-endian - low bits are at low addresses. 3188 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 3189 N->getOriginalAlign(), MMOFlags, AAInfo); 3190 3191 unsigned ExcessBits = 3192 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 3193 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 3194 3195 // Increment the pointer to the other half. 3196 unsigned IncrementSize = NVT.getSizeInBits()/8; 3197 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 3198 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 3199 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 3200 N->getOriginalAlign(), MMOFlags, AAInfo); 3201 3202 // Build a factor node to remember that this load is independent of the 3203 // other one. 3204 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3205 Hi.getValue(1)); 3206 } else { 3207 // Big-endian - high bits are at low addresses. Favor aligned loads at 3208 // the cost of some bit-fiddling. 3209 EVT MemVT = N->getMemoryVT(); 3210 unsigned EBytes = MemVT.getStoreSize(); 3211 unsigned IncrementSize = NVT.getSizeInBits()/8; 3212 unsigned ExcessBits = (EBytes - IncrementSize)*8; 3213 3214 // Load both the high bits and maybe some of the low bits. 3215 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 3216 EVT::getIntegerVT(*DAG.getContext(), 3217 MemVT.getSizeInBits() - ExcessBits), 3218 N->getOriginalAlign(), MMOFlags, AAInfo); 3219 3220 // Increment the pointer to the other half. 3221 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 3222 // Load the rest of the low bits. 3223 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 3224 N->getPointerInfo().getWithOffset(IncrementSize), 3225 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 3226 N->getOriginalAlign(), MMOFlags, AAInfo); 3227 3228 // Build a factor node to remember that this load is independent of the 3229 // other one. 3230 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3231 Hi.getValue(1)); 3232 3233 if (ExcessBits < NVT.getSizeInBits()) { 3234 // Transfer low bits from the bottom of Hi to the top of Lo. 3235 Lo = DAG.getNode( 3236 ISD::OR, dl, NVT, Lo, 3237 DAG.getNode(ISD::SHL, dl, NVT, Hi, 3238 DAG.getConstant(ExcessBits, dl, 3239 TLI.getPointerTy(DAG.getDataLayout())))); 3240 // Move high bits to the right position in Hi. 3241 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, 3242 Hi, 3243 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 3244 TLI.getPointerTy(DAG.getDataLayout()))); 3245 } 3246 } 3247 3248 // Legalize the chain result - switch anything that used the old chain to 3249 // use the new one. 3250 ReplaceValueWith(SDValue(N, 1), Ch); 3251 } 3252 3253 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 3254 SDValue &Lo, SDValue &Hi) { 3255 SDLoc dl(N); 3256 SDValue LL, LH, RL, RH; 3257 GetExpandedInteger(N->getOperand(0), LL, LH); 3258 GetExpandedInteger(N->getOperand(1), RL, RH); 3259 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 3260 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 3261 } 3262 3263 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 3264 SDValue &Lo, SDValue &Hi) { 3265 EVT VT = N->getValueType(0); 3266 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3267 SDLoc dl(N); 3268 3269 SDValue LL, LH, RL, RH; 3270 GetExpandedInteger(N->getOperand(0), LL, LH); 3271 GetExpandedInteger(N->getOperand(1), RL, RH); 3272 3273 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, 3274 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3275 LL, LH, RL, RH)) 3276 return; 3277 3278 // If nothing else, we can make a libcall. 3279 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3280 if (VT == MVT::i16) 3281 LC = RTLIB::MUL_I16; 3282 else if (VT == MVT::i32) 3283 LC = RTLIB::MUL_I32; 3284 else if (VT == MVT::i64) 3285 LC = RTLIB::MUL_I64; 3286 else if (VT == MVT::i128) 3287 LC = RTLIB::MUL_I128; 3288 3289 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 3290 // We'll expand the multiplication by brute force because we have no other 3291 // options. This is a trivially-generalized version of the code from 3292 // Hacker's Delight (itself derived from Knuth's Algorithm M from section 3293 // 4.3.1). 3294 unsigned Bits = NVT.getSizeInBits(); 3295 unsigned HalfBits = Bits >> 1; 3296 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, 3297 NVT); 3298 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); 3299 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); 3300 3301 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); 3302 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); 3303 3304 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 3305 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) { 3306 // The type from TLI is too small to fit the shift amount we want. 3307 // Override it with i32. The shift will have to be legalized. 3308 ShiftAmtTy = MVT::i32; 3309 } 3310 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy); 3311 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); 3312 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); 3313 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); 3314 3315 SDValue U = DAG.getNode(ISD::ADD, dl, NVT, 3316 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); 3317 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); 3318 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); 3319 3320 SDValue V = DAG.getNode(ISD::ADD, dl, NVT, 3321 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); 3322 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); 3323 3324 SDValue W = DAG.getNode(ISD::ADD, dl, NVT, 3325 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), 3326 DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); 3327 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, 3328 DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); 3329 3330 Hi = DAG.getNode(ISD::ADD, dl, NVT, W, 3331 DAG.getNode(ISD::ADD, dl, NVT, 3332 DAG.getNode(ISD::MUL, dl, NVT, RH, LL), 3333 DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); 3334 return; 3335 } 3336 3337 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3338 TargetLowering::MakeLibCallOptions CallOptions; 3339 CallOptions.setSExt(true); 3340 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, 3341 Lo, Hi); 3342 } 3343 3344 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, 3345 SDValue &Hi) { 3346 SDLoc DL(N); 3347 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3348 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); 3349 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); 3350 Lo = R.getValue(0); 3351 Hi = R.getValue(1); 3352 ReplaceValueWith(SDValue(N, 1), R.getValue(2)); 3353 } 3354 3355 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, 3356 SDValue &Hi) { 3357 SDValue Result = TLI.expandAddSubSat(N, DAG); 3358 SplitInteger(Result, Lo, Hi); 3359 } 3360 3361 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, 3362 SDValue &Hi) { 3363 SDValue Result = TLI.expandShlSat(N, DAG); 3364 SplitInteger(Result, Lo, Hi); 3365 } 3366 3367 /// This performs an expansion of the integer result for a fixed point 3368 /// multiplication. The default expansion performs rounding down towards 3369 /// negative infinity, though targets that do care about rounding should specify 3370 /// a target hook for rounding and provide their own expansion or lowering of 3371 /// fixed point multiplication to be consistent with rounding. 3372 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, 3373 SDValue &Hi) { 3374 SDLoc dl(N); 3375 EVT VT = N->getValueType(0); 3376 unsigned VTSize = VT.getScalarSizeInBits(); 3377 SDValue LHS = N->getOperand(0); 3378 SDValue RHS = N->getOperand(1); 3379 uint64_t Scale = N->getConstantOperandVal(2); 3380 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || 3381 N->getOpcode() == ISD::UMULFIXSAT); 3382 bool Signed = (N->getOpcode() == ISD::SMULFIX || 3383 N->getOpcode() == ISD::SMULFIXSAT); 3384 3385 // Handle special case when scale is equal to zero. 3386 if (!Scale) { 3387 SDValue Result; 3388 if (!Saturating) { 3389 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 3390 } else { 3391 EVT BoolVT = getSetCCResultType(VT); 3392 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; 3393 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); 3394 SDValue Product = Result.getValue(0); 3395 SDValue Overflow = Result.getValue(1); 3396 if (Signed) { 3397 APInt MinVal = APInt::getSignedMinValue(VTSize); 3398 APInt MaxVal = APInt::getSignedMaxValue(VTSize); 3399 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); 3400 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 3401 SDValue Zero = DAG.getConstant(0, dl, VT); 3402 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT); 3403 Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin); 3404 Result = DAG.getSelect(dl, VT, Overflow, Result, Product); 3405 } else { 3406 // For unsigned multiplication, we only need to check the max since we 3407 // can't really overflow towards zero. 3408 APInt MaxVal = APInt::getMaxValue(VTSize); 3409 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 3410 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product); 3411 } 3412 } 3413 SplitInteger(Result, Lo, Hi); 3414 return; 3415 } 3416 3417 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will 3418 // cover for unhandled cases below, while still being valid for UMULFIX[SAT]. 3419 assert(Scale <= VTSize && "Scale can't be larger than the value type size."); 3420 3421 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3422 SDValue LL, LH, RL, RH; 3423 GetExpandedInteger(LHS, LL, LH); 3424 GetExpandedInteger(RHS, RL, RH); 3425 SmallVector<SDValue, 4> Result; 3426 3427 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; 3428 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, 3429 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3430 LL, LH, RL, RH)) { 3431 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); 3432 return; 3433 } 3434 3435 unsigned NVTSize = NVT.getScalarSizeInBits(); 3436 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " 3437 "the size of the current value type"); 3438 EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 3439 3440 // After getting the multiplication result in 4 parts, we need to perform a 3441 // shift right by the amount of the scale to get the result in that scale. 3442 // 3443 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in 3444 // 128 bits that are cut into 4 32-bit parts: 3445 // 3446 // HH HL LH LL 3447 // |---32---|---32---|---32---|---32---| 3448 // 128 96 64 32 0 3449 // 3450 // |------VTSize-----| 3451 // 3452 // |NVTSize-| 3453 // 3454 // The resulting Lo and Hi would normally be in LL and LH after the shift. But 3455 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift 3456 // amount and get Lo and Hi using two funnel shifts. Or for the special case 3457 // when Scale is a multiple of NVTSize we can just pick the result without 3458 // shifting. 3459 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. 3460 if (Scale % NVTSize) { 3461 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy); 3462 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0], 3463 ShiftAmount); 3464 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1], 3465 ShiftAmount); 3466 } else { 3467 Lo = Result[Part0]; 3468 Hi = Result[Part0 + 1]; 3469 } 3470 3471 // Unless saturation is requested we are done. The result is in <Hi,Lo>. 3472 if (!Saturating) 3473 return; 3474 3475 // Can not overflow when there is no integer part. 3476 if (Scale == VTSize) 3477 return; 3478 3479 // To handle saturation we must check for overflow in the multiplication. 3480 // 3481 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) 3482 // aren't all zeroes. 3483 // 3484 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) 3485 // aren't all ones or all zeroes. 3486 // 3487 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the 3488 // highest bit of HH determines saturation direction in the event of signed 3489 // saturation. 3490 3491 SDValue ResultHL = Result[2]; 3492 SDValue ResultHH = Result[3]; 3493 3494 SDValue SatMax, SatMin; 3495 SDValue NVTZero = DAG.getConstant(0, dl, NVT); 3496 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); 3497 EVT BoolNVT = getSetCCResultType(NVT); 3498 3499 if (!Signed) { 3500 if (Scale < NVTSize) { 3501 // Overflow happened if ((HH | (HL >> Scale)) != 0). 3502 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 3503 DAG.getConstant(Scale, dl, ShiftTy)); 3504 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH); 3505 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE); 3506 } else if (Scale == NVTSize) { 3507 // Overflow happened if (HH != 0). 3508 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE); 3509 } else if (Scale < VTSize) { 3510 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0). 3511 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 3512 DAG.getConstant(Scale - NVTSize, dl, 3513 ShiftTy)); 3514 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE); 3515 } else 3516 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" 3517 "(and saturation can't happen with Scale==VTSize)."); 3518 3519 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi); 3520 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo); 3521 return; 3522 } 3523 3524 if (Scale < NVTSize) { 3525 // The number of overflow bits we can check are VTSize - Scale + 1 (we 3526 // include the sign bit). If these top bits are > 0, then we overflowed past 3527 // the max value. If these top bits are < -1, then we overflowed past the 3528 // min value. Otherwise, we did not overflow. 3529 unsigned OverflowBits = VTSize - Scale + 1; 3530 assert(OverflowBits <= VTSize && OverflowBits > NVTSize && 3531 "Extent of overflow bits must start within HL"); 3532 SDValue HLHiMask = DAG.getConstant( 3533 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); 3534 SDValue HLLoMask = DAG.getConstant( 3535 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); 3536 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). 3537 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 3538 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 3539 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); 3540 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 3541 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT)); 3542 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). 3543 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 3544 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 3545 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); 3546 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 3547 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT)); 3548 } else if (Scale == NVTSize) { 3549 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). 3550 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 3551 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 3552 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); 3553 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 3554 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg)); 3555 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). 3556 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 3557 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 3558 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); 3559 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 3560 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos)); 3561 } else if (Scale < VTSize) { 3562 // This is similar to the case when we saturate if Scale < NVTSize, but we 3563 // only need to check HH. 3564 unsigned OverflowBits = VTSize - Scale + 1; 3565 SDValue HHHiMask = DAG.getConstant( 3566 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); 3567 SDValue HHLoMask = DAG.getConstant( 3568 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); 3569 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); 3570 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); 3571 } else 3572 llvm_unreachable("Illegal scale for signed fixed point mul."); 3573 3574 // Saturate to signed maximum. 3575 APInt MaxHi = APInt::getSignedMaxValue(NVTSize); 3576 APInt MaxLo = APInt::getAllOnesValue(NVTSize); 3577 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi); 3578 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo); 3579 // Saturate to signed minimum. 3580 APInt MinHi = APInt::getSignedMinValue(NVTSize); 3581 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi); 3582 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); 3583 } 3584 3585 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, 3586 SDValue &Hi) { 3587 SDLoc dl(N); 3588 // Try expanding in the existing type first. 3589 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0), 3590 N->getOperand(1), 3591 N->getConstantOperandVal(2), DAG); 3592 3593 if (!Res) 3594 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1), 3595 N->getConstantOperandVal(2), TLI, DAG); 3596 SplitInteger(Res, Lo, Hi); 3597 } 3598 3599 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 3600 SDValue &Lo, SDValue &Hi) { 3601 SDValue LHS = Node->getOperand(0); 3602 SDValue RHS = Node->getOperand(1); 3603 SDLoc dl(Node); 3604 3605 SDValue Ovf; 3606 3607 unsigned CarryOp; 3608 switch(Node->getOpcode()) { 3609 default: llvm_unreachable("Node has unexpected Opcode"); 3610 case ISD::SADDO: CarryOp = ISD::SADDO_CARRY; break; 3611 case ISD::SSUBO: CarryOp = ISD::SSUBO_CARRY; break; 3612 } 3613 3614 bool HasCarryOp = TLI.isOperationLegalOrCustom( 3615 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 3616 3617 if (HasCarryOp) { 3618 // Expand the subcomponents. 3619 SDValue LHSL, LHSH, RHSL, RHSH; 3620 GetExpandedInteger(LHS, LHSL, LHSH); 3621 GetExpandedInteger(RHS, RHSL, RHSH); 3622 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1)); 3623 3624 Lo = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 3625 ISD::UADDO : ISD::USUBO, dl, VTList, { LHSL, RHSL }); 3626 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 3627 3628 Ovf = Hi.getValue(1); 3629 } else { 3630 // Expand the result by simply replacing it with the equivalent 3631 // non-overflow-checking operation. 3632 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 3633 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3634 LHS, RHS); 3635 SplitInteger(Sum, Lo, Hi); 3636 3637 // Compute the overflow. 3638 // 3639 // LHSSign -> LHS >= 0 3640 // RHSSign -> RHS >= 0 3641 // SumSign -> Sum >= 0 3642 // 3643 // Add: 3644 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 3645 // Sub: 3646 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 3647 // 3648 EVT OType = Node->getValueType(1); 3649 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); 3650 3651 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 3652 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 3653 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 3654 Node->getOpcode() == ISD::SADDO ? 3655 ISD::SETEQ : ISD::SETNE); 3656 3657 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 3658 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 3659 3660 Ovf = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 3661 } 3662 3663 // Use the calculated overflow everywhere. 3664 ReplaceValueWith(SDValue(Node, 1), Ovf); 3665 } 3666 3667 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 3668 SDValue &Lo, SDValue &Hi) { 3669 EVT VT = N->getValueType(0); 3670 SDLoc dl(N); 3671 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3672 3673 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 3674 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 3675 SplitInteger(Res.getValue(0), Lo, Hi); 3676 return; 3677 } 3678 3679 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3680 if (VT == MVT::i16) 3681 LC = RTLIB::SDIV_I16; 3682 else if (VT == MVT::i32) 3683 LC = RTLIB::SDIV_I32; 3684 else if (VT == MVT::i64) 3685 LC = RTLIB::SDIV_I64; 3686 else if (VT == MVT::i128) 3687 LC = RTLIB::SDIV_I128; 3688 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 3689 3690 TargetLowering::MakeLibCallOptions CallOptions; 3691 CallOptions.setSExt(true); 3692 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3693 } 3694 3695 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 3696 SDValue &Lo, SDValue &Hi) { 3697 EVT VT = N->getValueType(0); 3698 SDLoc dl(N); 3699 3700 // If we can emit an efficient shift operation, do so now. Check to see if 3701 // the RHS is a constant. 3702 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 3703 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); 3704 3705 // If we can determine that the high bit of the shift is zero or one, even if 3706 // the low bits are variable, emit this shift in an optimized form. 3707 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 3708 return; 3709 3710 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 3711 unsigned PartsOpc; 3712 if (N->getOpcode() == ISD::SHL) { 3713 PartsOpc = ISD::SHL_PARTS; 3714 } else if (N->getOpcode() == ISD::SRL) { 3715 PartsOpc = ISD::SRL_PARTS; 3716 } else { 3717 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 3718 PartsOpc = ISD::SRA_PARTS; 3719 } 3720 3721 // Next check to see if the target supports this SHL_PARTS operation or if it 3722 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for 3723 // size, but create a libcall instead. 3724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3725 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 3726 const bool LegalOrCustom = 3727 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 3728 Action == TargetLowering::Custom; 3729 3730 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) { 3731 // Expand the subcomponents. 3732 SDValue LHSL, LHSH; 3733 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3734 EVT VT = LHSL.getValueType(); 3735 3736 // If the shift amount operand is coming from a vector legalization it may 3737 // have an illegal type. Fix that first by casting the operand, otherwise 3738 // the new SHL_PARTS operation would need further legalization. 3739 SDValue ShiftOp = N->getOperand(1); 3740 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 3741 assert(ShiftTy.getScalarSizeInBits() >= 3742 Log2_32_Ceil(VT.getScalarSizeInBits()) && 3743 "ShiftAmountTy is too small to cover the range of this type!"); 3744 if (ShiftOp.getValueType() != ShiftTy) 3745 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 3746 3747 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 3748 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 3749 Hi = Lo.getValue(1); 3750 return; 3751 } 3752 3753 // Otherwise, emit a libcall. 3754 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3755 bool isSigned; 3756 if (N->getOpcode() == ISD::SHL) { 3757 isSigned = false; /*sign irrelevant*/ 3758 if (VT == MVT::i16) 3759 LC = RTLIB::SHL_I16; 3760 else if (VT == MVT::i32) 3761 LC = RTLIB::SHL_I32; 3762 else if (VT == MVT::i64) 3763 LC = RTLIB::SHL_I64; 3764 else if (VT == MVT::i128) 3765 LC = RTLIB::SHL_I128; 3766 } else if (N->getOpcode() == ISD::SRL) { 3767 isSigned = false; 3768 if (VT == MVT::i16) 3769 LC = RTLIB::SRL_I16; 3770 else if (VT == MVT::i32) 3771 LC = RTLIB::SRL_I32; 3772 else if (VT == MVT::i64) 3773 LC = RTLIB::SRL_I64; 3774 else if (VT == MVT::i128) 3775 LC = RTLIB::SRL_I128; 3776 } else { 3777 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 3778 isSigned = true; 3779 if (VT == MVT::i16) 3780 LC = RTLIB::SRA_I16; 3781 else if (VT == MVT::i32) 3782 LC = RTLIB::SRA_I32; 3783 else if (VT == MVT::i64) 3784 LC = RTLIB::SRA_I64; 3785 else if (VT == MVT::i128) 3786 LC = RTLIB::SRA_I128; 3787 } 3788 3789 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 3790 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3791 TargetLowering::MakeLibCallOptions CallOptions; 3792 CallOptions.setSExt(isSigned); 3793 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3794 return; 3795 } 3796 3797 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 3798 llvm_unreachable("Unsupported shift!"); 3799 } 3800 3801 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 3802 SDValue &Lo, SDValue &Hi) { 3803 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3804 SDLoc dl(N); 3805 SDValue Op = N->getOperand(0); 3806 if (Op.getValueType().bitsLE(NVT)) { 3807 // The low part is sign extension of the input (degenerates to a copy). 3808 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 3809 // The high part is obtained by SRA'ing all but one of the bits of low part. 3810 unsigned LoSize = NVT.getSizeInBits(); 3811 Hi = DAG.getNode( 3812 ISD::SRA, dl, NVT, Lo, 3813 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); 3814 } else { 3815 // For example, extension of an i48 to an i64. The operand type necessarily 3816 // promotes to the result type, so will end up being expanded too. 3817 assert(getTypeAction(Op.getValueType()) == 3818 TargetLowering::TypePromoteInteger && 3819 "Only know how to promote this result!"); 3820 SDValue Res = GetPromotedInteger(Op); 3821 assert(Res.getValueType() == N->getValueType(0) && 3822 "Operand over promoted?"); 3823 // Split the promoted operand. This will simplify when it is expanded. 3824 SplitInteger(Res, Lo, Hi); 3825 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 3826 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 3827 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3828 ExcessBits))); 3829 } 3830 } 3831 3832 void DAGTypeLegalizer:: 3833 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 3834 SDLoc dl(N); 3835 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3836 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3837 3838 if (EVT.bitsLE(Lo.getValueType())) { 3839 // sext_inreg the low part if needed. 3840 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 3841 N->getOperand(1)); 3842 3843 // The high part gets the sign extension from the lo-part. This handles 3844 // things like sextinreg V:i64 from i8. 3845 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 3846 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, 3847 TLI.getPointerTy(DAG.getDataLayout()))); 3848 } else { 3849 // For example, extension of an i48 to an i64. Leave the low part alone, 3850 // sext_inreg the high part. 3851 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); 3852 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 3853 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3854 ExcessBits))); 3855 } 3856 } 3857 3858 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 3859 SDValue &Lo, SDValue &Hi) { 3860 EVT VT = N->getValueType(0); 3861 SDLoc dl(N); 3862 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3863 3864 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 3865 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 3866 SplitInteger(Res.getValue(1), Lo, Hi); 3867 return; 3868 } 3869 3870 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3871 if (VT == MVT::i16) 3872 LC = RTLIB::SREM_I16; 3873 else if (VT == MVT::i32) 3874 LC = RTLIB::SREM_I32; 3875 else if (VT == MVT::i64) 3876 LC = RTLIB::SREM_I64; 3877 else if (VT == MVT::i128) 3878 LC = RTLIB::SREM_I128; 3879 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 3880 3881 TargetLowering::MakeLibCallOptions CallOptions; 3882 CallOptions.setSExt(true); 3883 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3884 } 3885 3886 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 3887 SDValue &Lo, SDValue &Hi) { 3888 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3889 SDLoc dl(N); 3890 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 3891 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), 3892 N->getOperand(0), 3893 DAG.getConstant(NVT.getSizeInBits(), dl, 3894 TLI.getPointerTy(DAG.getDataLayout()))); 3895 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 3896 } 3897 3898 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 3899 SDValue &Lo, SDValue &Hi) { 3900 EVT VT = N->getValueType(0); 3901 SDLoc dl(N); 3902 3903 if (N->getOpcode() == ISD::UMULO) { 3904 // This section expands the operation into the following sequence of 3905 // instructions. `iNh` here refers to a type which has half the bit width of 3906 // the type the original operation operated on. 3907 // 3908 // %0 = %LHS.HI != 0 && %RHS.HI != 0 3909 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) 3910 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) 3911 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) 3912 // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh 3913 // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 ) 3914 // 3915 // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 } 3916 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 3917 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; 3918 SplitInteger(LHS, LHSLow, LHSHigh); 3919 SplitInteger(RHS, RHSLow, RHSHigh); 3920 EVT HalfVT = LHSLow.getValueType() 3921 , BitVT = N->getValueType(1); 3922 SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT); 3923 SDVTList VTFullAddO = DAG.getVTList(VT, BitVT); 3924 3925 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT); 3926 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT, 3927 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE), 3928 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE)); 3929 3930 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow); 3931 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1)); 3932 SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero, 3933 One.getValue(0)); 3934 3935 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow); 3936 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1)); 3937 SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero, 3938 Two.getValue(0)); 3939 3940 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not 3941 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this 3942 // operation recursively legalized?). 3943 // 3944 // Many backends understand this pattern and will convert into LOHI 3945 // themselves, if applicable. 3946 SDValue Three = DAG.getNode(ISD::MUL, dl, VT, 3947 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow), 3948 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow)); 3949 SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh); 3950 SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four); 3951 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1)); 3952 SplitInteger(Five, Lo, Hi); 3953 ReplaceValueWith(SDValue(N, 1), Overflow); 3954 return; 3955 } 3956 3957 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 3958 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 3959 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 3960 3961 // Replace this with a libcall that will check overflow. 3962 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3963 if (VT == MVT::i32) 3964 LC = RTLIB::MULO_I32; 3965 else if (VT == MVT::i64) 3966 LC = RTLIB::MULO_I64; 3967 else if (VT == MVT::i128) 3968 LC = RTLIB::MULO_I128; 3969 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); 3970 3971 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 3972 // Temporary for the overflow value, default it to zero. 3973 SDValue Chain = 3974 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, 3975 MachinePointerInfo()); 3976 3977 TargetLowering::ArgListTy Args; 3978 TargetLowering::ArgListEntry Entry; 3979 for (const SDValue &Op : N->op_values()) { 3980 EVT ArgVT = Op.getValueType(); 3981 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 3982 Entry.Node = Op; 3983 Entry.Ty = ArgTy; 3984 Entry.IsSExt = true; 3985 Entry.IsZExt = false; 3986 Args.push_back(Entry); 3987 } 3988 3989 // Also pass the address of the overflow check. 3990 Entry.Node = Temp; 3991 Entry.Ty = PtrTy->getPointerTo(); 3992 Entry.IsSExt = true; 3993 Entry.IsZExt = false; 3994 Args.push_back(Entry); 3995 3996 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 3997 3998 TargetLowering::CallLoweringInfo CLI(DAG); 3999 CLI.setDebugLoc(dl) 4000 .setChain(Chain) 4001 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) 4002 .setSExtResult(); 4003 4004 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 4005 4006 SplitInteger(CallInfo.first, Lo, Hi); 4007 SDValue Temp2 = 4008 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); 4009 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 4010 DAG.getConstant(0, dl, PtrVT), 4011 ISD::SETNE); 4012 // Use the overflow from the libcall everywhere. 4013 ReplaceValueWith(SDValue(N, 1), Ofl); 4014 } 4015 4016 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 4017 SDValue &Lo, SDValue &Hi) { 4018 EVT VT = N->getValueType(0); 4019 SDLoc dl(N); 4020 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4021 4022 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4023 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4024 SplitInteger(Res.getValue(0), Lo, Hi); 4025 return; 4026 } 4027 4028 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4029 if (VT == MVT::i16) 4030 LC = RTLIB::UDIV_I16; 4031 else if (VT == MVT::i32) 4032 LC = RTLIB::UDIV_I32; 4033 else if (VT == MVT::i64) 4034 LC = RTLIB::UDIV_I64; 4035 else if (VT == MVT::i128) 4036 LC = RTLIB::UDIV_I128; 4037 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 4038 4039 TargetLowering::MakeLibCallOptions CallOptions; 4040 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4041 } 4042 4043 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 4044 SDValue &Lo, SDValue &Hi) { 4045 EVT VT = N->getValueType(0); 4046 SDLoc dl(N); 4047 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4048 4049 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4050 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4051 SplitInteger(Res.getValue(1), Lo, Hi); 4052 return; 4053 } 4054 4055 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4056 if (VT == MVT::i16) 4057 LC = RTLIB::UREM_I16; 4058 else if (VT == MVT::i32) 4059 LC = RTLIB::UREM_I32; 4060 else if (VT == MVT::i64) 4061 LC = RTLIB::UREM_I64; 4062 else if (VT == MVT::i128) 4063 LC = RTLIB::UREM_I128; 4064 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 4065 4066 TargetLowering::MakeLibCallOptions CallOptions; 4067 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4068 } 4069 4070 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 4071 SDValue &Lo, SDValue &Hi) { 4072 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4073 SDLoc dl(N); 4074 SDValue Op = N->getOperand(0); 4075 if (Op.getValueType().bitsLE(NVT)) { 4076 // The low part is zero extension of the input (degenerates to a copy). 4077 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 4078 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. 4079 } else { 4080 // For example, extension of an i48 to an i64. The operand type necessarily 4081 // promotes to the result type, so will end up being expanded too. 4082 assert(getTypeAction(Op.getValueType()) == 4083 TargetLowering::TypePromoteInteger && 4084 "Only know how to promote this result!"); 4085 SDValue Res = GetPromotedInteger(Op); 4086 assert(Res.getValueType() == N->getValueType(0) && 4087 "Operand over promoted?"); 4088 // Split the promoted operand. This will simplify when it is expanded. 4089 SplitInteger(Res, Lo, Hi); 4090 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 4091 Hi = DAG.getZeroExtendInReg(Hi, dl, 4092 EVT::getIntegerVT(*DAG.getContext(), 4093 ExcessBits)); 4094 } 4095 } 4096 4097 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 4098 SDValue &Lo, SDValue &Hi) { 4099 SDLoc dl(N); 4100 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 4101 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 4102 SDValue Zero = DAG.getConstant(0, dl, VT); 4103 SDValue Swap = DAG.getAtomicCmpSwap( 4104 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 4105 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 4106 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); 4107 4108 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 4109 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 4110 } 4111 4112 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, 4113 SDValue &Lo, SDValue &Hi) { 4114 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate 4115 // both halves independently. 4116 SDValue Res = TLI.expandVecReduce(N, DAG); 4117 SplitInteger(Res, Lo, Hi); 4118 } 4119 4120 void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, 4121 SDValue &Lo, SDValue &Hi) { 4122 // Lower the rotate to shifts and ORs which can be expanded. 4123 SDValue Res; 4124 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG); 4125 SplitInteger(Res, Lo, Hi); 4126 } 4127 4128 void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, 4129 SDValue &Lo, SDValue &Hi) { 4130 // Lower the funnel shift to shifts and ORs which can be expanded. 4131 SDValue Res; 4132 TLI.expandFunnelShift(N, Res, DAG); 4133 SplitInteger(Res, Lo, Hi); 4134 } 4135 4136 //===----------------------------------------------------------------------===// 4137 // Integer Operand Expansion 4138 //===----------------------------------------------------------------------===// 4139 4140 /// ExpandIntegerOperand - This method is called when the specified operand of 4141 /// the specified node is found to need expansion. At this point, all of the 4142 /// result types of the node are known to be legal, but other operands of the 4143 /// node may need promotion or expansion as well as the specified one. 4144 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 4145 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); 4146 dbgs() << "\n"); 4147 SDValue Res = SDValue(); 4148 4149 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 4150 return false; 4151 4152 switch (N->getOpcode()) { 4153 default: 4154 #ifndef NDEBUG 4155 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 4156 N->dump(&DAG); dbgs() << "\n"; 4157 #endif 4158 report_fatal_error("Do not know how to expand this operator's operand!"); 4159 4160 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 4161 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 4162 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 4163 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 4164 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 4165 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 4166 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 4167 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 4168 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; 4169 case ISD::STRICT_SINT_TO_FP: 4170 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 4171 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 4172 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 4173 case ISD::STRICT_UINT_TO_FP: 4174 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 4175 4176 case ISD::SHL: 4177 case ISD::SRA: 4178 case ISD::SRL: 4179 case ISD::ROTL: 4180 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 4181 case ISD::RETURNADDR: 4182 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 4183 4184 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 4185 } 4186 4187 // If the result is null, the sub-method took care of registering results etc. 4188 if (!Res.getNode()) return false; 4189 4190 // If the result is N, the sub-method updated N in place. Tell the legalizer 4191 // core about this. 4192 if (Res.getNode() == N) 4193 return true; 4194 4195 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 4196 "Invalid operand expansion"); 4197 4198 ReplaceValueWith(SDValue(N, 0), Res); 4199 return false; 4200 } 4201 4202 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 4203 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 4204 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 4205 SDValue &NewRHS, 4206 ISD::CondCode &CCCode, 4207 const SDLoc &dl) { 4208 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 4209 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 4210 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 4211 4212 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 4213 if (RHSLo == RHSHi) { 4214 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 4215 if (RHSCST->isAllOnesValue()) { 4216 // Equality comparison to -1. 4217 NewLHS = DAG.getNode(ISD::AND, dl, 4218 LHSLo.getValueType(), LHSLo, LHSHi); 4219 NewRHS = RHSLo; 4220 return; 4221 } 4222 } 4223 } 4224 4225 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 4226 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 4227 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 4228 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); 4229 return; 4230 } 4231 4232 // If this is a comparison of the sign bit, just look at the top part. 4233 // X > -1, x < 0 4234 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 4235 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 4236 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 4237 NewLHS = LHSHi; 4238 NewRHS = RHSHi; 4239 return; 4240 } 4241 4242 // FIXME: This generated code sucks. 4243 ISD::CondCode LowCC; 4244 switch (CCCode) { 4245 default: llvm_unreachable("Unknown integer setcc!"); 4246 case ISD::SETLT: 4247 case ISD::SETULT: LowCC = ISD::SETULT; break; 4248 case ISD::SETGT: 4249 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 4250 case ISD::SETLE: 4251 case ISD::SETULE: LowCC = ISD::SETULE; break; 4252 case ISD::SETGE: 4253 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 4254 } 4255 4256 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison 4257 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands 4258 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; 4259 4260 // NOTE: on targets without efficient SELECT of bools, we can always use 4261 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 4262 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 4263 nullptr); 4264 SDValue LoCmp, HiCmp; 4265 if (TLI.isTypeLegal(LHSLo.getValueType()) && 4266 TLI.isTypeLegal(RHSLo.getValueType())) 4267 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, 4268 RHSLo, LowCC, false, DagCombineInfo, dl); 4269 if (!LoCmp.getNode()) 4270 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 4271 RHSLo, LowCC); 4272 if (TLI.isTypeLegal(LHSHi.getValueType()) && 4273 TLI.isTypeLegal(RHSHi.getValueType())) 4274 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, 4275 RHSHi, CCCode, false, DagCombineInfo, dl); 4276 if (!HiCmp.getNode()) 4277 HiCmp = 4278 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), 4279 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 4280 4281 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); 4282 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); 4283 4284 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 4285 CCCode == ISD::SETUGE || CCCode == ISD::SETULE); 4286 4287 if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) || 4288 (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) || 4289 (LoCmpC && LoCmpC->isNullValue())))) { 4290 // For LE / GE, if high part is known false, ignore the low part. 4291 // For LT / GT: if low part is known false, return the high part. 4292 // if high part is known true, ignore the low part. 4293 NewLHS = HiCmp; 4294 NewRHS = SDValue(); 4295 return; 4296 } 4297 4298 if (LHSHi == RHSHi) { 4299 // Comparing the low bits is enough. 4300 NewLHS = LoCmp; 4301 NewRHS = SDValue(); 4302 return; 4303 } 4304 4305 // Lower with SETCCCARRY if the target supports it. 4306 EVT HiVT = LHSHi.getValueType(); 4307 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); 4308 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); 4309 4310 // FIXME: Make all targets support this, then remove the other lowering. 4311 if (HasSETCCCARRY) { 4312 // SETCCCARRY can detect < and >= directly. For > and <=, flip 4313 // operands and condition code. 4314 bool FlipOperands = false; 4315 switch (CCCode) { 4316 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; 4317 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; 4318 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; 4319 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; 4320 default: break; 4321 } 4322 if (FlipOperands) { 4323 std::swap(LHSLo, RHSLo); 4324 std::swap(LHSHi, RHSHi); 4325 } 4326 // Perform a wide subtraction, feeding the carry from the low part into 4327 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high 4328 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is 4329 // zero or positive iff LHS >= RHS. 4330 EVT LoVT = LHSLo.getValueType(); 4331 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT)); 4332 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo); 4333 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT), 4334 LHSHi, RHSHi, LowCmp.getValue(1), 4335 DAG.getCondCode(CCCode)); 4336 NewLHS = Res; 4337 NewRHS = SDValue(); 4338 return; 4339 } 4340 4341 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, 4342 false, DagCombineInfo, dl); 4343 if (!NewLHS.getNode()) 4344 NewLHS = 4345 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); 4346 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); 4347 NewRHS = SDValue(); 4348 } 4349 4350 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 4351 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 4352 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 4353 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 4354 4355 // If ExpandSetCCOperands returned a scalar, we need to compare the result 4356 // against zero to select between true and false values. 4357 if (!NewRHS.getNode()) { 4358 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 4359 CCCode = ISD::SETNE; 4360 } 4361 4362 // Update N to have the operands specified. 4363 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 4364 DAG.getCondCode(CCCode), NewLHS, NewRHS, 4365 N->getOperand(4)), 0); 4366 } 4367 4368 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 4369 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 4370 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 4371 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 4372 4373 // If ExpandSetCCOperands returned a scalar, we need to compare the result 4374 // against zero to select between true and false values. 4375 if (!NewRHS.getNode()) { 4376 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 4377 CCCode = ISD::SETNE; 4378 } 4379 4380 // Update N to have the operands specified. 4381 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 4382 N->getOperand(2), N->getOperand(3), 4383 DAG.getCondCode(CCCode)), 0); 4384 } 4385 4386 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 4387 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 4388 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 4389 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 4390 4391 // If ExpandSetCCOperands returned a scalar, use it. 4392 if (!NewRHS.getNode()) { 4393 assert(NewLHS.getValueType() == N->getValueType(0) && 4394 "Unexpected setcc expansion!"); 4395 return NewLHS; 4396 } 4397 4398 // Otherwise, update N to have the operands specified. 4399 return SDValue( 4400 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); 4401 } 4402 4403 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { 4404 SDValue LHS = N->getOperand(0); 4405 SDValue RHS = N->getOperand(1); 4406 SDValue Carry = N->getOperand(2); 4407 SDValue Cond = N->getOperand(3); 4408 SDLoc dl = SDLoc(N); 4409 4410 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 4411 GetExpandedInteger(LHS, LHSLo, LHSHi); 4412 GetExpandedInteger(RHS, RHSLo, RHSHi); 4413 4414 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high. 4415 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); 4416 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry); 4417 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, 4418 LowCmp.getValue(1), Cond); 4419 } 4420 4421 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 4422 // The value being shifted is legal, but the shift amount is too big. 4423 // It follows that either the result of the shift is undefined, or the 4424 // upper half of the shift amount is zero. Just use the lower half. 4425 SDValue Lo, Hi; 4426 GetExpandedInteger(N->getOperand(1), Lo, Hi); 4427 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 4428 } 4429 4430 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 4431 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 4432 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 4433 // constant to valid type. 4434 SDValue Lo, Hi; 4435 GetExpandedInteger(N->getOperand(0), Lo, Hi); 4436 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 4437 } 4438 4439 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 4440 bool IsStrict = N->isStrictFPOpcode(); 4441 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 4442 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 4443 EVT DstVT = N->getValueType(0); 4444 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 4445 assert(LC != RTLIB::UNKNOWN_LIBCALL && 4446 "Don't know how to expand this SINT_TO_FP!"); 4447 TargetLowering::MakeLibCallOptions CallOptions; 4448 CallOptions.setSExt(true); 4449 std::pair<SDValue, SDValue> Tmp = 4450 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 4451 4452 if (!IsStrict) 4453 return Tmp.first; 4454 4455 ReplaceValueWith(SDValue(N, 1), Tmp.second); 4456 ReplaceValueWith(SDValue(N, 0), Tmp.first); 4457 return SDValue(); 4458 } 4459 4460 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 4461 if (N->isAtomic()) { 4462 // It's typical to have larger CAS than atomic store instructions. 4463 SDLoc dl(N); 4464 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 4465 N->getMemoryVT(), 4466 N->getOperand(0), N->getOperand(2), 4467 N->getOperand(1), 4468 N->getMemOperand()); 4469 return Swap.getValue(1); 4470 } 4471 if (ISD::isNormalStore(N)) 4472 return ExpandOp_NormalStore(N, OpNo); 4473 4474 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 4475 assert(OpNo == 1 && "Can only expand the stored value so far"); 4476 4477 EVT VT = N->getOperand(1).getValueType(); 4478 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4479 SDValue Ch = N->getChain(); 4480 SDValue Ptr = N->getBasePtr(); 4481 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 4482 AAMDNodes AAInfo = N->getAAInfo(); 4483 SDLoc dl(N); 4484 SDValue Lo, Hi; 4485 4486 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 4487 4488 if (N->getMemoryVT().bitsLE(NVT)) { 4489 GetExpandedInteger(N->getValue(), Lo, Hi); 4490 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 4491 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags, 4492 AAInfo); 4493 } 4494 4495 if (DAG.getDataLayout().isLittleEndian()) { 4496 // Little-endian - low bits are at low addresses. 4497 GetExpandedInteger(N->getValue(), Lo, Hi); 4498 4499 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 4500 N->getOriginalAlign(), MMOFlags, AAInfo); 4501 4502 unsigned ExcessBits = 4503 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 4504 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 4505 4506 // Increment the pointer to the other half. 4507 unsigned IncrementSize = NVT.getSizeInBits()/8; 4508 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 4509 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 4510 N->getPointerInfo().getWithOffset(IncrementSize), 4511 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo); 4512 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 4513 } 4514 4515 // Big-endian - high bits are at low addresses. Favor aligned stores at 4516 // the cost of some bit-fiddling. 4517 GetExpandedInteger(N->getValue(), Lo, Hi); 4518 4519 EVT ExtVT = N->getMemoryVT(); 4520 unsigned EBytes = ExtVT.getStoreSize(); 4521 unsigned IncrementSize = NVT.getSizeInBits()/8; 4522 unsigned ExcessBits = (EBytes - IncrementSize)*8; 4523 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 4524 ExtVT.getSizeInBits() - ExcessBits); 4525 4526 if (ExcessBits < NVT.getSizeInBits()) { 4527 // Transfer high bits from the top of Lo to the bottom of Hi. 4528 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 4529 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 4530 TLI.getPointerTy(DAG.getDataLayout()))); 4531 Hi = DAG.getNode( 4532 ISD::OR, dl, NVT, Hi, 4533 DAG.getNode(ISD::SRL, dl, NVT, Lo, 4534 DAG.getConstant(ExcessBits, dl, 4535 TLI.getPointerTy(DAG.getDataLayout())))); 4536 } 4537 4538 // Store both the high bits and maybe some of the low bits. 4539 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, 4540 N->getOriginalAlign(), MMOFlags, AAInfo); 4541 4542 // Increment the pointer to the other half. 4543 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 4544 // Store the lowest ExcessBits bits in the second half. 4545 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 4546 N->getPointerInfo().getWithOffset(IncrementSize), 4547 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 4548 N->getOriginalAlign(), MMOFlags, AAInfo); 4549 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 4550 } 4551 4552 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 4553 SDValue InL, InH; 4554 GetExpandedInteger(N->getOperand(0), InL, InH); 4555 // Just truncate the low part of the source. 4556 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 4557 } 4558 4559 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 4560 bool IsStrict = N->isStrictFPOpcode(); 4561 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 4562 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 4563 EVT DstVT = N->getValueType(0); 4564 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT); 4565 assert(LC != RTLIB::UNKNOWN_LIBCALL && 4566 "Don't know how to expand this UINT_TO_FP!"); 4567 TargetLowering::MakeLibCallOptions CallOptions; 4568 CallOptions.setSExt(true); 4569 std::pair<SDValue, SDValue> Tmp = 4570 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 4571 4572 if (!IsStrict) 4573 return Tmp.first; 4574 4575 ReplaceValueWith(SDValue(N, 1), Tmp.second); 4576 ReplaceValueWith(SDValue(N, 0), Tmp.first); 4577 return SDValue(); 4578 } 4579 4580 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 4581 SDLoc dl(N); 4582 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 4583 cast<AtomicSDNode>(N)->getMemoryVT(), 4584 N->getOperand(0), 4585 N->getOperand(1), N->getOperand(2), 4586 cast<AtomicSDNode>(N)->getMemOperand()); 4587 return Swap.getValue(1); 4588 } 4589 4590 4591 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 4592 4593 EVT OutVT = N->getValueType(0); 4594 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4595 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4596 EVT NOutVTElem = NOutVT.getVectorElementType(); 4597 4598 SDLoc dl(N); 4599 SDValue BaseIdx = N->getOperand(1); 4600 4601 // TODO: We may be able to use this for types other than scalable 4602 // vectors and fix those tests that expect BUILD_VECTOR to be used 4603 if (OutVT.isScalableVector()) { 4604 SDValue InOp0 = N->getOperand(0); 4605 EVT InVT = InOp0.getValueType(); 4606 4607 // Promote operands and see if this is handled by target lowering, 4608 // Otherwise, use the BUILD_VECTOR approach below 4609 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 4610 // Collect the (promoted) operands 4611 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx }; 4612 4613 EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); 4614 assert(PromEltVT.bitsLE(NOutVTElem) && 4615 "Promoted operand has an element type greater than result"); 4616 4617 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT); 4618 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops); 4619 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); 4620 } 4621 } 4622 4623 if (OutVT.isScalableVector()) 4624 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR"); 4625 4626 SDValue InOp0 = N->getOperand(0); 4627 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) 4628 InOp0 = GetPromotedInteger(N->getOperand(0)); 4629 4630 EVT InVT = InOp0.getValueType(); 4631 4632 unsigned OutNumElems = OutVT.getVectorNumElements(); 4633 SmallVector<SDValue, 8> Ops; 4634 Ops.reserve(OutNumElems); 4635 for (unsigned i = 0; i != OutNumElems; ++i) { 4636 4637 // Extract the element from the original vector. 4638 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 4639 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); 4640 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 4641 InVT.getVectorElementType(), N->getOperand(0), Index); 4642 4643 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); 4644 // Insert the converted element to the new vector. 4645 Ops.push_back(Op); 4646 } 4647 4648 return DAG.getBuildVector(NOutVT, dl, Ops); 4649 } 4650 4651 4652 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 4653 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 4654 EVT VT = N->getValueType(0); 4655 SDLoc dl(N); 4656 4657 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); 4658 4659 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4660 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 4661 EVT OutVT = V0.getValueType(); 4662 4663 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); 4664 } 4665 4666 4667 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 4668 EVT OutVT = N->getValueType(0); 4669 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4670 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4671 unsigned NumElems = N->getNumOperands(); 4672 EVT NOutVTElem = NOutVT.getVectorElementType(); 4673 4674 SDLoc dl(N); 4675 4676 SmallVector<SDValue, 8> Ops; 4677 Ops.reserve(NumElems); 4678 for (unsigned i = 0; i != NumElems; ++i) { 4679 SDValue Op; 4680 // BUILD_VECTOR integer operand types are allowed to be larger than the 4681 // result's element type. This may still be true after the promotion. For 4682 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 4683 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 4684 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem)) 4685 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); 4686 else 4687 Op = N->getOperand(i); 4688 Ops.push_back(Op); 4689 } 4690 4691 return DAG.getBuildVector(NOutVT, dl, Ops); 4692 } 4693 4694 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { 4695 4696 SDLoc dl(N); 4697 4698 assert(!N->getOperand(0).getValueType().isVector() && 4699 "Input must be a scalar"); 4700 4701 EVT OutVT = N->getValueType(0); 4702 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4703 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4704 EVT NOutVTElem = NOutVT.getVectorElementType(); 4705 4706 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); 4707 4708 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); 4709 } 4710 4711 SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) { 4712 SDLoc dl(N); 4713 4714 SDValue SplatVal = N->getOperand(0); 4715 4716 assert(!SplatVal.getValueType().isVector() && "Input must be a scalar"); 4717 4718 EVT OutVT = N->getValueType(0); 4719 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4720 assert(NOutVT.isVector() && "Type must be promoted to a vector type"); 4721 EVT NOutElemVT = NOutVT.getVectorElementType(); 4722 4723 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal); 4724 4725 return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op); 4726 } 4727 4728 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 4729 SDLoc dl(N); 4730 4731 EVT OutVT = N->getValueType(0); 4732 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4733 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4734 4735 EVT OutElemTy = NOutVT.getVectorElementType(); 4736 4737 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 4738 unsigned NumOutElem = NOutVT.getVectorNumElements(); 4739 unsigned NumOperands = N->getNumOperands(); 4740 assert(NumElem * NumOperands == NumOutElem && 4741 "Unexpected number of elements"); 4742 4743 // Take the elements from the first vector. 4744 SmallVector<SDValue, 8> Ops(NumOutElem); 4745 for (unsigned i = 0; i < NumOperands; ++i) { 4746 SDValue Op = N->getOperand(i); 4747 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) 4748 Op = GetPromotedInteger(Op); 4749 EVT SclrTy = Op.getValueType().getVectorElementType(); 4750 assert(NumElem == Op.getValueType().getVectorNumElements() && 4751 "Unexpected number of elements"); 4752 4753 for (unsigned j = 0; j < NumElem; ++j) { 4754 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, 4755 DAG.getVectorIdxConstant(j, dl)); 4756 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); 4757 } 4758 } 4759 4760 return DAG.getBuildVector(NOutVT, dl, Ops); 4761 } 4762 4763 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { 4764 EVT VT = N->getValueType(0); 4765 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4766 assert(NVT.isVector() && "This type must be promoted to a vector type"); 4767 4768 SDLoc dl(N); 4769 4770 // For operands whose TypeAction is to promote, extend the promoted node 4771 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion 4772 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to 4773 // type.. 4774 if (getTypeAction(N->getOperand(0).getValueType()) 4775 == TargetLowering::TypePromoteInteger) { 4776 SDValue Promoted; 4777 4778 switch(N->getOpcode()) { 4779 case ISD::SIGN_EXTEND_VECTOR_INREG: 4780 Promoted = SExtPromotedInteger(N->getOperand(0)); 4781 break; 4782 case ISD::ZERO_EXTEND_VECTOR_INREG: 4783 Promoted = ZExtPromotedInteger(N->getOperand(0)); 4784 break; 4785 case ISD::ANY_EXTEND_VECTOR_INREG: 4786 Promoted = GetPromotedInteger(N->getOperand(0)); 4787 break; 4788 default: 4789 llvm_unreachable("Node has unexpected Opcode"); 4790 } 4791 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); 4792 } 4793 4794 // Directly extend to the appropriate transform-to type. 4795 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 4796 } 4797 4798 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 4799 EVT OutVT = N->getValueType(0); 4800 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4801 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4802 4803 EVT NOutVTElem = NOutVT.getVectorElementType(); 4804 4805 SDLoc dl(N); 4806 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4807 4808 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 4809 NOutVTElem, N->getOperand(1)); 4810 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 4811 V0, ConvElem, N->getOperand(2)); 4812 } 4813 4814 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { 4815 // The VECREDUCE result size may be larger than the element size, so 4816 // we can simply change the result type. 4817 SDLoc dl(N); 4818 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4819 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 4820 } 4821 4822 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 4823 SDLoc dl(N); 4824 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4825 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, 4826 TLI.getVectorIdxTy(DAG.getDataLayout())); 4827 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 4828 V0->getValueType(0).getScalarType(), V0, V1); 4829 4830 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 4831 // element types. If this is the case then we need to expand the outgoing 4832 // value and not truncate it. 4833 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 4834 } 4835 4836 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { 4837 SDLoc dl(N); 4838 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4839 MVT InVT = V0.getValueType().getSimpleVT(); 4840 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), 4841 N->getValueType(0).getVectorNumElements()); 4842 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); 4843 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 4844 } 4845 4846 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 4847 SDLoc dl(N); 4848 4849 EVT ResVT = N->getValueType(0); 4850 unsigned NumElems = N->getNumOperands(); 4851 4852 if (ResVT.isScalableVector()) { 4853 SDValue ResVec = DAG.getUNDEF(ResVT); 4854 4855 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { 4856 SDValue Op = N->getOperand(OpIdx); 4857 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); 4858 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op, 4859 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl)); 4860 } 4861 4862 return ResVec; 4863 } 4864 4865 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 4866 4867 SmallVector<SDValue, 8> NewOps; 4868 NewOps.reserve(NumElems); 4869 4870 // For each incoming vector 4871 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 4872 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 4873 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 4874 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 4875 4876 for (unsigned i=0; i<NumElem; ++i) { 4877 // Extract element from incoming vector 4878 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, 4879 DAG.getVectorIdxConstant(i, dl)); 4880 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 4881 NewOps.push_back(Tr); 4882 } 4883 } 4884 4885 return DAG.getBuildVector(N->getValueType(0), dl, NewOps); 4886 } 4887