1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements integer type expansion and promotion for LegalizeTypes. 10 // Promotion is the act of changing a computation in an illegal type into a 11 // computation in a larger type. For example, implementing i8 arithmetic in an 12 // i32 register (often needed on powerpc). 13 // Expansion is the act of changing a computation in an illegal type into a 14 // computation in two identical registers of a smaller type. For example, 15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit 16 // targets). 17 // 18 //===----------------------------------------------------------------------===// 19 20 #include "LegalizeTypes.h" 21 #include "llvm/IR/DerivedTypes.h" 22 #include "llvm/Support/ErrorHandling.h" 23 #include "llvm/Support/KnownBits.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace llvm; 26 27 #define DEBUG_TYPE "legalize-types" 28 29 //===----------------------------------------------------------------------===// 30 // Integer Result Promotion 31 //===----------------------------------------------------------------------===// 32 33 /// PromoteIntegerResult - This method is called when a result of a node is 34 /// found to be in need of promotion to a larger type. At this point, the node 35 /// may also have invalid operands or may have other results that need 36 /// expansion, we just know that (at least) one result needs promotion. 37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 38 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); 39 dbgs() << "\n"); 40 SDValue Res = SDValue(); 41 42 // See if the target wants to custom expand this node. 43 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 44 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 45 return; 46 } 47 48 switch (N->getOpcode()) { 49 default: 50 #ifndef NDEBUG 51 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 52 N->dump(&DAG); dbgs() << "\n"; 53 #endif 54 llvm_unreachable("Do not know how to promote this operator!"); 55 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 56 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 57 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 58 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 59 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; 60 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 61 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 62 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 63 case ISD::CTLZ_ZERO_UNDEF: 64 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 65 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; 66 case ISD::CTTZ_ZERO_UNDEF: 67 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 68 case ISD::EXTRACT_VECTOR_ELT: 69 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 70 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 71 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); 72 break; 73 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); 74 break; 75 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 76 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; 77 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 78 case ISD::STRICT_FSETCC: 79 case ISD::STRICT_FSETCCS: 80 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 81 case ISD::SMIN: 82 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; 83 case ISD::UMIN: 84 case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break; 85 86 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 87 case ISD::SIGN_EXTEND_INREG: 88 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 89 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 90 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 91 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 92 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 93 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 94 95 case ISD::EXTRACT_SUBVECTOR: 96 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 97 case ISD::VECTOR_SHUFFLE: 98 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 99 case ISD::INSERT_VECTOR_ELT: 100 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 101 case ISD::BUILD_VECTOR: 102 Res = PromoteIntRes_BUILD_VECTOR(N); break; 103 case ISD::SCALAR_TO_VECTOR: 104 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; 105 case ISD::SPLAT_VECTOR: 106 Res = PromoteIntRes_SPLAT_VECTOR(N); break; 107 case ISD::CONCAT_VECTORS: 108 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 109 110 case ISD::ANY_EXTEND_VECTOR_INREG: 111 case ISD::SIGN_EXTEND_VECTOR_INREG: 112 case ISD::ZERO_EXTEND_VECTOR_INREG: 113 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; 114 115 case ISD::SIGN_EXTEND: 116 case ISD::ZERO_EXTEND: 117 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 118 119 case ISD::STRICT_FP_TO_SINT: 120 case ISD::STRICT_FP_TO_UINT: 121 case ISD::FP_TO_SINT: 122 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 123 124 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break; 125 126 case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break; 127 128 case ISD::AND: 129 case ISD::OR: 130 case ISD::XOR: 131 case ISD::ADD: 132 case ISD::SUB: 133 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 134 135 case ISD::SDIV: 136 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; 137 138 case ISD::UDIV: 139 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; 140 141 case ISD::SADDO: 142 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 143 case ISD::UADDO: 144 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 145 case ISD::SMULO: 146 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 147 148 case ISD::ADDE: 149 case ISD::SUBE: 150 case ISD::ADDCARRY: 151 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break; 152 153 case ISD::SADDSAT: 154 case ISD::UADDSAT: 155 case ISD::SSUBSAT: 156 case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break; 157 158 case ISD::SMULFIX: 159 case ISD::SMULFIXSAT: 160 case ISD::UMULFIX: 161 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; 162 163 case ISD::SDIVFIX: 164 case ISD::UDIVFIX: Res = PromoteIntRes_DIVFIX(N); break; 165 166 case ISD::ABS: Res = PromoteIntRes_ABS(N); break; 167 168 case ISD::ATOMIC_LOAD: 169 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 170 171 case ISD::ATOMIC_LOAD_ADD: 172 case ISD::ATOMIC_LOAD_SUB: 173 case ISD::ATOMIC_LOAD_AND: 174 case ISD::ATOMIC_LOAD_CLR: 175 case ISD::ATOMIC_LOAD_OR: 176 case ISD::ATOMIC_LOAD_XOR: 177 case ISD::ATOMIC_LOAD_NAND: 178 case ISD::ATOMIC_LOAD_MIN: 179 case ISD::ATOMIC_LOAD_MAX: 180 case ISD::ATOMIC_LOAD_UMIN: 181 case ISD::ATOMIC_LOAD_UMAX: 182 case ISD::ATOMIC_SWAP: 183 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 184 185 case ISD::ATOMIC_CMP_SWAP: 186 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 187 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 188 break; 189 190 case ISD::VECREDUCE_ADD: 191 case ISD::VECREDUCE_MUL: 192 case ISD::VECREDUCE_AND: 193 case ISD::VECREDUCE_OR: 194 case ISD::VECREDUCE_XOR: 195 case ISD::VECREDUCE_SMAX: 196 case ISD::VECREDUCE_SMIN: 197 case ISD::VECREDUCE_UMAX: 198 case ISD::VECREDUCE_UMIN: 199 Res = PromoteIntRes_VECREDUCE(N); 200 break; 201 } 202 203 // If the result is null then the sub-method took care of registering it. 204 if (Res.getNode()) 205 SetPromotedInteger(SDValue(N, ResNo), Res); 206 } 207 208 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 209 unsigned ResNo) { 210 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 211 return GetPromotedInteger(Op); 212 } 213 214 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 215 // Sign-extend the new bits, and continue the assertion. 216 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 217 return DAG.getNode(ISD::AssertSext, SDLoc(N), 218 Op.getValueType(), Op, N->getOperand(1)); 219 } 220 221 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 222 // Zero the new bits, and continue the assertion. 223 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 224 return DAG.getNode(ISD::AssertZext, SDLoc(N), 225 Op.getValueType(), Op, N->getOperand(1)); 226 } 227 228 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 229 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 230 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 231 N->getMemoryVT(), ResVT, 232 N->getChain(), N->getBasePtr(), 233 N->getMemOperand()); 234 // Legalize the chain result - switch anything that used the old chain to 235 // use the new one. 236 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 237 return Res; 238 } 239 240 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 241 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 242 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 243 N->getMemoryVT(), 244 N->getChain(), N->getBasePtr(), 245 Op2, N->getMemOperand()); 246 // Legalize the chain result - switch anything that used the old chain to 247 // use the new one. 248 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 249 return Res; 250 } 251 252 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 253 unsigned ResNo) { 254 if (ResNo == 1) { 255 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 256 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 257 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 258 259 // Only use the result of getSetCCResultType if it is legal, 260 // otherwise just use the promoted result type (NVT). 261 if (!TLI.isTypeLegal(SVT)) 262 SVT = NVT; 263 264 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 265 SDValue Res = DAG.getAtomicCmpSwap( 266 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 267 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 268 N->getMemOperand()); 269 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 270 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 271 return Res.getValue(1); 272 } 273 274 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 275 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 276 SDVTList VTs = 277 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 278 SDValue Res = DAG.getAtomicCmpSwap( 279 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 280 N->getBasePtr(), Op2, Op3, N->getMemOperand()); 281 // Update the use to N with the newly created Res. 282 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) 283 ReplaceValueWith(SDValue(N, i), Res.getValue(i)); 284 return Res; 285 } 286 287 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 288 SDValue InOp = N->getOperand(0); 289 EVT InVT = InOp.getValueType(); 290 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 291 EVT OutVT = N->getValueType(0); 292 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 293 SDLoc dl(N); 294 295 switch (getTypeAction(InVT)) { 296 case TargetLowering::TypeLegal: 297 break; 298 case TargetLowering::TypePromoteInteger: 299 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 300 // The input promotes to the same size. Convert the promoted value. 301 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 302 break; 303 case TargetLowering::TypeSoftenFloat: 304 // Promote the integer operand by hand. 305 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 306 case TargetLowering::TypePromoteFloat: { 307 // Convert the promoted float by hand. 308 if (!NOutVT.isVector()) 309 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); 310 break; 311 } 312 case TargetLowering::TypeExpandInteger: 313 case TargetLowering::TypeExpandFloat: 314 break; 315 case TargetLowering::TypeScalarizeVector: 316 // Convert the element to an integer and promote it by hand. 317 if (!NOutVT.isVector()) 318 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 319 BitConvertToInteger(GetScalarizedVector(InOp))); 320 break; 321 case TargetLowering::TypeSplitVector: { 322 if (!NOutVT.isVector()) { 323 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 324 // pieces of the input into integers and reassemble in the final type. 325 SDValue Lo, Hi; 326 GetSplitVector(N->getOperand(0), Lo, Hi); 327 Lo = BitConvertToInteger(Lo); 328 Hi = BitConvertToInteger(Hi); 329 330 if (DAG.getDataLayout().isBigEndian()) 331 std::swap(Lo, Hi); 332 333 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 334 EVT::getIntegerVT(*DAG.getContext(), 335 NOutVT.getSizeInBits()), 336 JoinIntegers(Lo, Hi)); 337 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 338 } 339 break; 340 } 341 case TargetLowering::TypeWidenVector: 342 // The input is widened to the same size. Convert to the widened value. 343 // Make sure that the outgoing value is not a vector, because this would 344 // make us bitcast between two vectors which are legalized in different ways. 345 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) { 346 SDValue Res = 347 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 348 349 // For big endian targets we need to shift the casted value or the 350 // interesting bits will end up at the wrong place. 351 if (DAG.getDataLayout().isBigEndian()) { 352 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 353 EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout()); 354 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!"); 355 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res, 356 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy)); 357 } 358 return Res; 359 } 360 // If the output type is also a vector and widening it to the same size 361 // as the widened input type would be a legal type, we can widen the bitcast 362 // and handle the promotion after. 363 if (NOutVT.isVector()) { 364 unsigned WidenInSize = NInVT.getSizeInBits(); 365 unsigned OutSize = OutVT.getSizeInBits(); 366 if (WidenInSize % OutSize == 0) { 367 unsigned Scale = WidenInSize / OutSize; 368 EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(), 369 OutVT.getVectorElementType(), 370 OutVT.getVectorNumElements() * Scale); 371 if (isTypeLegal(WideOutVT)) { 372 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); 373 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); 374 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, 375 DAG.getConstant(0, dl, IdxTy)); 376 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp); 377 } 378 } 379 } 380 } 381 382 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 383 CreateStackStoreLoad(InOp, OutVT)); 384 } 385 386 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit any shift amount 387 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't. 388 static EVT getShiftAmountTyForConstant(EVT VT, const TargetLowering &TLI, 389 SelectionDAG &DAG) { 390 EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 391 // If any possible shift value won't fit in the prefered type, just use 392 // something safe. It will be legalized when the shift is expanded. 393 if (!ShiftVT.isVector() && 394 ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits())) 395 ShiftVT = MVT::i32; 396 return ShiftVT; 397 } 398 399 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 400 SDValue Op = GetPromotedInteger(N->getOperand(0)); 401 EVT OVT = N->getValueType(0); 402 EVT NVT = Op.getValueType(); 403 SDLoc dl(N); 404 405 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 406 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG); 407 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 408 DAG.getConstant(DiffBits, dl, ShiftVT)); 409 } 410 411 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { 412 SDValue Op = GetPromotedInteger(N->getOperand(0)); 413 EVT OVT = N->getValueType(0); 414 EVT NVT = Op.getValueType(); 415 SDLoc dl(N); 416 417 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 418 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG); 419 return DAG.getNode(ISD::SRL, dl, NVT, 420 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), 421 DAG.getConstant(DiffBits, dl, ShiftVT)); 422 } 423 424 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 425 // The pair element type may be legal, or may not promote to the same type as 426 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 427 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 428 TLI.getTypeToTransformTo(*DAG.getContext(), 429 N->getValueType(0)), JoinIntegers(N->getOperand(0), 430 N->getOperand(1))); 431 } 432 433 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 434 EVT VT = N->getValueType(0); 435 // FIXME there is no actual debug info here 436 SDLoc dl(N); 437 // Zero extend things like i1, sign extend everything else. It shouldn't 438 // matter in theory which one we pick, but this tends to give better code? 439 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 440 SDValue Result = DAG.getNode(Opc, dl, 441 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 442 SDValue(N, 0)); 443 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 444 return Result; 445 } 446 447 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 448 // Zero extend to the promoted type and do the count there. 449 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 450 SDLoc dl(N); 451 EVT OVT = N->getValueType(0); 452 EVT NVT = Op.getValueType(); 453 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 454 // Subtract off the extra leading bits in the bigger type. 455 return DAG.getNode( 456 ISD::SUB, dl, NVT, Op, 457 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, 458 NVT)); 459 } 460 461 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 462 // Zero extend to the promoted type and do the count there. 463 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 464 return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op); 465 } 466 467 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 468 SDValue Op = GetPromotedInteger(N->getOperand(0)); 469 EVT OVT = N->getValueType(0); 470 EVT NVT = Op.getValueType(); 471 SDLoc dl(N); 472 if (N->getOpcode() == ISD::CTTZ) { 473 // The count is the same in the promoted type except if the original 474 // value was zero. This can be handled by setting the bit just off 475 // the top of the original type. 476 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), 477 OVT.getScalarSizeInBits()); 478 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); 479 } 480 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 481 } 482 483 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 484 SDLoc dl(N); 485 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 486 487 SDValue Op0 = N->getOperand(0); 488 SDValue Op1 = N->getOperand(1); 489 490 // If the input also needs to be promoted, do that first so we can get a 491 // get a good idea for the output type. 492 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType()) 493 == TargetLowering::TypePromoteInteger) { 494 SDValue In = GetPromotedInteger(Op0); 495 496 // If the new type is larger than NVT, use it. We probably won't need to 497 // promote it again. 498 EVT SVT = In.getValueType().getScalarType(); 499 if (SVT.bitsGE(NVT)) { 500 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1); 501 return DAG.getAnyExtOrTrunc(Ext, dl, NVT); 502 } 503 } 504 505 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1); 506 } 507 508 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 509 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 510 unsigned NewOpc = N->getOpcode(); 511 SDLoc dl(N); 512 513 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 514 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 515 // and SINT conversions are Custom, there is no way to tell which is 516 // preferable. We choose SINT because that's the right thing on PPC.) 517 if (N->getOpcode() == ISD::FP_TO_UINT && 518 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 519 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 520 NewOpc = ISD::FP_TO_SINT; 521 522 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && 523 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) && 524 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) 525 NewOpc = ISD::STRICT_FP_TO_SINT; 526 527 SDValue Res; 528 if (N->isStrictFPOpcode()) { 529 Res = DAG.getNode(NewOpc, dl, { NVT, MVT::Other }, 530 { N->getOperand(0), N->getOperand(1) }); 531 // Legalize the chain result - switch anything that used the old chain to 532 // use the new one. 533 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 534 } else 535 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 536 537 // Assert that the converted value fits in the original type. If it doesn't 538 // (eg: because the value being converted is too big), then the result of the 539 // original operation was undefined anyway, so the assert is still correct. 540 // 541 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: 542 // before legalization: fp-to-uint16, 65534. -> 0xfffe 543 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe 544 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT || 545 N->getOpcode() == ISD::STRICT_FP_TO_UINT) ? 546 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, 547 DAG.getValueType(N->getValueType(0).getScalarType())); 548 } 549 550 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) { 551 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 552 SDLoc dl(N); 553 554 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 555 } 556 557 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) { 558 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 559 SDLoc dl(N); 560 561 return DAG.getNode(N->getOpcode(), dl, NVT); 562 } 563 564 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 565 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 566 SDLoc dl(N); 567 568 if (getTypeAction(N->getOperand(0).getValueType()) 569 == TargetLowering::TypePromoteInteger) { 570 SDValue Res = GetPromotedInteger(N->getOperand(0)); 571 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 572 573 // If the result and operand types are the same after promotion, simplify 574 // to an in-register extension. 575 if (NVT == Res.getValueType()) { 576 // The high bits are not guaranteed to be anything. Insert an extend. 577 if (N->getOpcode() == ISD::SIGN_EXTEND) 578 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 579 DAG.getValueType(N->getOperand(0).getValueType())); 580 if (N->getOpcode() == ISD::ZERO_EXTEND) 581 return DAG.getZeroExtendInReg(Res, dl, 582 N->getOperand(0).getValueType().getScalarType()); 583 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 584 return Res; 585 } 586 } 587 588 // Otherwise, just extend the original operand all the way to the larger type. 589 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 590 } 591 592 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 593 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 594 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 595 ISD::LoadExtType ExtType = 596 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 597 SDLoc dl(N); 598 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 599 N->getMemoryVT(), N->getMemOperand()); 600 601 // Legalize the chain result - switch anything that used the old chain to 602 // use the new one. 603 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 604 return Res; 605 } 606 607 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { 608 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 609 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 610 611 SDLoc dl(N); 612 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), 613 N->getOffset(), N->getMask(), ExtPassThru, 614 N->getMemoryVT(), N->getMemOperand(), 615 N->getAddressingMode(), ISD::EXTLOAD); 616 // Legalize the chain result - switch anything that used the old chain to 617 // use the new one. 618 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 619 return Res; 620 } 621 622 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { 623 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 624 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 625 assert(NVT == ExtPassThru.getValueType() && 626 "Gather result type and the passThru argument type should be the same"); 627 628 SDLoc dl(N); 629 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), 630 N->getIndex(), N->getScale() }; 631 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), 632 N->getMemoryVT(), dl, Ops, 633 N->getMemOperand(), N->getIndexType()); 634 // Legalize the chain result - switch anything that used the old chain to 635 // use the new one. 636 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 637 return Res; 638 } 639 640 /// Promote the overflow flag of an overflowing arithmetic node. 641 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 642 // Change the return type of the boolean result while obeying 643 // getSetCCResultType. 644 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 645 EVT VT = N->getValueType(0); 646 EVT SVT = getSetCCResultType(VT); 647 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; 648 unsigned NumOps = N->getNumOperands(); 649 assert(NumOps <= 3 && "Too many operands"); 650 if (NumOps == 3) 651 Ops[2] = N->getOperand(2); 652 653 SDLoc dl(N); 654 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), 655 makeArrayRef(Ops, NumOps)); 656 657 // Modified the sum result - switch anything that used the old sum to use 658 // the new one. 659 ReplaceValueWith(SDValue(N, 0), Res); 660 661 // Convert to the expected type. 662 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); 663 } 664 665 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) { 666 // If the promoted type is legal, we can convert this to: 667 // 1. ANY_EXTEND iN to iM 668 // 2. SHL by M-N 669 // 3. [US][ADD|SUB]SAT 670 // 4. L/ASHR by M-N 671 // Else it is more efficient to convert this to a min and a max 672 // operation in the higher precision arithmetic. 673 SDLoc dl(N); 674 SDValue Op1 = N->getOperand(0); 675 SDValue Op2 = N->getOperand(1); 676 unsigned OldBits = Op1.getScalarValueSizeInBits(); 677 678 unsigned Opcode = N->getOpcode(); 679 680 SDValue Op1Promoted, Op2Promoted; 681 if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { 682 Op1Promoted = ZExtPromotedInteger(Op1); 683 Op2Promoted = ZExtPromotedInteger(Op2); 684 } else { 685 Op1Promoted = SExtPromotedInteger(Op1); 686 Op2Promoted = SExtPromotedInteger(Op2); 687 } 688 EVT PromotedType = Op1Promoted.getValueType(); 689 unsigned NewBits = PromotedType.getScalarSizeInBits(); 690 691 if (TLI.isOperationLegalOrCustom(Opcode, PromotedType)) { 692 unsigned ShiftOp; 693 switch (Opcode) { 694 case ISD::SADDSAT: 695 case ISD::SSUBSAT: 696 ShiftOp = ISD::SRA; 697 break; 698 case ISD::UADDSAT: 699 case ISD::USUBSAT: 700 ShiftOp = ISD::SRL; 701 break; 702 default: 703 llvm_unreachable("Expected opcode to be signed or unsigned saturation " 704 "addition or subtraction"); 705 } 706 707 unsigned SHLAmount = NewBits - OldBits; 708 EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); 709 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT); 710 Op1Promoted = 711 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); 712 Op2Promoted = 713 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); 714 715 SDValue Result = 716 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); 717 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); 718 } else { 719 if (Opcode == ISD::USUBSAT) { 720 SDValue Max = 721 DAG.getNode(ISD::UMAX, dl, PromotedType, Op1Promoted, Op2Promoted); 722 return DAG.getNode(ISD::SUB, dl, PromotedType, Max, Op2Promoted); 723 } 724 725 if (Opcode == ISD::UADDSAT) { 726 APInt MaxVal = APInt::getAllOnesValue(OldBits).zext(NewBits); 727 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 728 SDValue Add = 729 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); 730 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); 731 } 732 733 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; 734 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits); 735 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits); 736 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType); 737 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 738 SDValue Result = 739 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); 740 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); 741 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); 742 return Result; 743 } 744 } 745 746 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { 747 // Can just promote the operands then continue with operation. 748 SDLoc dl(N); 749 SDValue Op1Promoted, Op2Promoted; 750 bool Signed = 751 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; 752 bool Saturating = 753 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; 754 if (Signed) { 755 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 756 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 757 } else { 758 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 759 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 760 } 761 EVT OldType = N->getOperand(0).getValueType(); 762 EVT PromotedType = Op1Promoted.getValueType(); 763 unsigned DiffSize = 764 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); 765 766 if (Saturating) { 767 // Promoting the operand and result values changes the saturation width, 768 // which is extends the values that we clamp to on saturation. This could be 769 // resolved by shifting one of the operands the same amount, which would 770 // also shift the result we compare against, then shifting back. 771 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); 772 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 773 DAG.getConstant(DiffSize, dl, ShiftTy)); 774 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 775 Op2Promoted, N->getOperand(2)); 776 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; 777 return DAG.getNode(ShiftOp, dl, PromotedType, Result, 778 DAG.getConstant(DiffSize, dl, ShiftTy)); 779 } 780 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, 781 N->getOperand(2)); 782 } 783 784 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, 785 unsigned Scale, const TargetLowering &TLI, 786 SelectionDAG &DAG) { 787 EVT VT = LHS.getValueType(); 788 bool Signed = N->getOpcode() == ISD::SDIVFIX; 789 790 SDLoc dl(N); 791 // See if we can perform the division in this type without widening. 792 if (SDValue V = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, 793 DAG)) 794 return V; 795 796 // If that didn't work, double the type width and try again. That must work, 797 // or something is wrong. 798 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), 799 VT.getScalarSizeInBits() * 2); 800 if (Signed) { 801 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT); 802 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT); 803 } else { 804 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT); 805 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT); 806 } 807 808 // TODO: Saturation. 809 810 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, 811 DAG); 812 assert(Res && "Expanding DIVFIX with wide type failed?"); 813 return DAG.getZExtOrTrunc(Res, dl, VT); 814 } 815 816 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { 817 SDLoc dl(N); 818 SDValue Op1Promoted, Op2Promoted; 819 bool Signed = N->getOpcode() == ISD::SDIVFIX; 820 if (Signed) { 821 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 822 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 823 } else { 824 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 825 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 826 } 827 EVT PromotedType = Op1Promoted.getValueType(); 828 unsigned Scale = N->getConstantOperandVal(2); 829 830 SDValue Res; 831 // If the type is already legal and the operation is legal in that type, we 832 // should not early expand. 833 if (TLI.isTypeLegal(PromotedType)) { 834 TargetLowering::LegalizeAction Action = 835 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale); 836 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) 837 Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 838 Op2Promoted, N->getOperand(2)); 839 } 840 841 if (!Res) 842 Res = earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG); 843 844 // TODO: Saturation. 845 846 return Res; 847 } 848 849 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 850 if (ResNo == 1) 851 return PromoteIntRes_Overflow(N); 852 853 // The operation overflowed iff the result in the larger type is not the 854 // sign extension of its truncation to the original type. 855 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 856 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 857 EVT OVT = N->getOperand(0).getValueType(); 858 EVT NVT = LHS.getValueType(); 859 SDLoc dl(N); 860 861 // Do the arithmetic in the larger type. 862 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 863 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 864 865 // Calculate the overflow flag: sign extend the arithmetic result from 866 // the original type. 867 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 868 DAG.getValueType(OVT)); 869 // Overflowed if and only if this is not equal to Res. 870 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 871 872 // Use the calculated overflow everywhere. 873 ReplaceValueWith(SDValue(N, 1), Ofl); 874 875 return Res; 876 } 877 878 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 879 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 880 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 881 return DAG.getSelect(SDLoc(N), 882 LHS.getValueType(), N->getOperand(0), LHS, RHS); 883 } 884 885 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { 886 SDValue Mask = N->getOperand(0); 887 888 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 889 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 890 return DAG.getNode(ISD::VSELECT, SDLoc(N), 891 LHS.getValueType(), Mask, LHS, RHS); 892 } 893 894 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 895 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 896 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 897 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 898 LHS.getValueType(), N->getOperand(0), 899 N->getOperand(1), LHS, RHS, N->getOperand(4)); 900 } 901 902 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 903 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 904 EVT InVT = N->getOperand(OpNo).getValueType(); 905 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 906 907 EVT SVT = getSetCCResultType(InVT); 908 909 // If we got back a type that needs to be promoted, this likely means the 910 // the input type also needs to be promoted. So get the promoted type for 911 // the input and try the query again. 912 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { 913 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 914 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 915 SVT = getSetCCResultType(InVT); 916 } else { 917 // Input type isn't promoted, just use the default promoted type. 918 SVT = NVT; 919 } 920 } 921 922 SDLoc dl(N); 923 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && 924 "Vector compare must return a vector result!"); 925 926 // Get the SETCC result using the canonical SETCC type. 927 SDValue SetCC; 928 if (N->isStrictFPOpcode()) { 929 EVT VTs[] = {SVT, MVT::Other}; 930 SDValue Opers[] = {N->getOperand(0), N->getOperand(1), 931 N->getOperand(2), N->getOperand(3)}; 932 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers); 933 // Legalize the chain result - switch anything that used the old chain to 934 // use the new one. 935 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1)); 936 } else 937 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), 938 N->getOperand(1), N->getOperand(2)); 939 940 // Convert to the expected type. 941 return DAG.getSExtOrTrunc(SetCC, dl, NVT); 942 } 943 944 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 945 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 946 SDValue RHS = N->getOperand(1); 947 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 948 RHS = ZExtPromotedInteger(RHS); 949 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS); 950 } 951 952 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 953 SDValue Op = GetPromotedInteger(N->getOperand(0)); 954 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 955 Op.getValueType(), Op, N->getOperand(1)); 956 } 957 958 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 959 // The input may have strange things in the top bits of the registers, but 960 // these operations don't care. They may have weird bits going out, but 961 // that too is okay if they are integer operations. 962 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 963 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 964 return DAG.getNode(N->getOpcode(), SDLoc(N), 965 LHS.getValueType(), LHS, RHS); 966 } 967 968 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { 969 // Sign extend the input. 970 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 971 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 972 return DAG.getNode(N->getOpcode(), SDLoc(N), 973 LHS.getValueType(), LHS, RHS); 974 } 975 976 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { 977 // Zero extend the input. 978 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 979 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 980 return DAG.getNode(N->getOpcode(), SDLoc(N), 981 LHS.getValueType(), LHS, RHS); 982 } 983 984 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 985 // The input value must be properly sign extended. 986 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 987 SDValue RHS = N->getOperand(1); 988 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 989 RHS = ZExtPromotedInteger(RHS); 990 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS); 991 } 992 993 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 994 // The input value must be properly zero extended. 995 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 996 SDValue RHS = N->getOperand(1); 997 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 998 RHS = ZExtPromotedInteger(RHS); 999 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS); 1000 } 1001 1002 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 1003 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1004 SDValue Res; 1005 SDValue InOp = N->getOperand(0); 1006 SDLoc dl(N); 1007 1008 switch (getTypeAction(InOp.getValueType())) { 1009 default: llvm_unreachable("Unknown type action!"); 1010 case TargetLowering::TypeLegal: 1011 case TargetLowering::TypeExpandInteger: 1012 Res = InOp; 1013 break; 1014 case TargetLowering::TypePromoteInteger: 1015 Res = GetPromotedInteger(InOp); 1016 break; 1017 case TargetLowering::TypeSplitVector: { 1018 EVT InVT = InOp.getValueType(); 1019 assert(InVT.isVector() && "Cannot split scalar types"); 1020 unsigned NumElts = InVT.getVectorNumElements(); 1021 assert(NumElts == NVT.getVectorNumElements() && 1022 "Dst and Src must have the same number of elements"); 1023 assert(isPowerOf2_32(NumElts) && 1024 "Promoted vector type must be a power of two"); 1025 1026 SDValue EOp1, EOp2; 1027 GetSplitVector(InOp, EOp1, EOp2); 1028 1029 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 1030 NumElts/2); 1031 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 1032 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 1033 1034 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 1035 } 1036 case TargetLowering::TypeWidenVector: { 1037 SDValue WideInOp = GetWidenedVector(InOp); 1038 1039 // Truncate widened InOp. 1040 unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); 1041 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), 1042 N->getValueType(0).getScalarType(), NumElem); 1043 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); 1044 1045 // Zero extend so that the elements are of same type as those of NVT 1046 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), 1047 NumElem); 1048 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); 1049 1050 // Extract the low NVT subvector. 1051 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); 1052 SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy); 1053 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); 1054 } 1055 } 1056 1057 // Truncate to NVT instead of VT 1058 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 1059 } 1060 1061 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 1062 if (ResNo == 1) 1063 return PromoteIntRes_Overflow(N); 1064 1065 // The operation overflowed iff the result in the larger type is not the 1066 // zero extension of its truncation to the original type. 1067 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1068 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1069 EVT OVT = N->getOperand(0).getValueType(); 1070 EVT NVT = LHS.getValueType(); 1071 SDLoc dl(N); 1072 1073 // Do the arithmetic in the larger type. 1074 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 1075 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1076 1077 // Calculate the overflow flag: zero extend the arithmetic result from 1078 // the original type. 1079 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType()); 1080 // Overflowed if and only if this is not equal to Res. 1081 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1082 1083 // Use the calculated overflow everywhere. 1084 ReplaceValueWith(SDValue(N, 1), Ofl); 1085 1086 return Res; 1087 } 1088 1089 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that 1090 // the third operand of ADDE/SUBE nodes is carry flag, which differs from 1091 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean. 1092 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) { 1093 if (ResNo == 1) 1094 return PromoteIntRes_Overflow(N); 1095 1096 // We need to sign-extend the operands so the carry value computed by the 1097 // wide operation will be equivalent to the carry value computed by the 1098 // narrow operation. 1099 // An ADDCARRY can generate carry only if any of the operands has its 1100 // most significant bit set. Sign extension propagates the most significant 1101 // bit into the higher bits which means the extra bit that the narrow 1102 // addition would need (i.e. the carry) will be propagated through the higher 1103 // bits of the wide addition. 1104 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be 1105 // preserved by sign extension. 1106 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1107 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1108 1109 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; 1110 1111 // Do the arithmetic in the wide type. 1112 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), 1113 LHS, RHS, N->getOperand(2)); 1114 1115 // Update the users of the original carry/borrow value. 1116 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 1117 1118 return SDValue(Res.getNode(), 0); 1119 } 1120 1121 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { 1122 SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); 1123 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); 1124 } 1125 1126 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 1127 // Promote the overflow bit trivially. 1128 if (ResNo == 1) 1129 return PromoteIntRes_Overflow(N); 1130 1131 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 1132 SDLoc DL(N); 1133 EVT SmallVT = LHS.getValueType(); 1134 1135 // To determine if the result overflowed in a larger type, we extend the 1136 // input to the larger type, do the multiply (checking if it overflows), 1137 // then also check the high bits of the result to see if overflow happened 1138 // there. 1139 if (N->getOpcode() == ISD::SMULO) { 1140 LHS = SExtPromotedInteger(LHS); 1141 RHS = SExtPromotedInteger(RHS); 1142 } else { 1143 LHS = ZExtPromotedInteger(LHS); 1144 RHS = ZExtPromotedInteger(RHS); 1145 } 1146 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 1147 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 1148 1149 // Overflow occurred if it occurred in the larger type, or if the high part 1150 // of the result does not zero/sign-extend the low part. Check this second 1151 // possibility first. 1152 SDValue Overflow; 1153 if (N->getOpcode() == ISD::UMULO) { 1154 // Unsigned overflow occurred if the high part is non-zero. 1155 unsigned Shift = SmallVT.getScalarSizeInBits(); 1156 EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG); 1157 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 1158 DAG.getConstant(Shift, DL, ShiftTy)); 1159 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 1160 DAG.getConstant(0, DL, Hi.getValueType()), 1161 ISD::SETNE); 1162 } else { 1163 // Signed overflow occurred if the high part does not sign extend the low. 1164 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 1165 Mul, DAG.getValueType(SmallVT)); 1166 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 1167 } 1168 1169 // The only other way for overflow to occur is if the multiplication in the 1170 // larger type itself overflowed. 1171 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 1172 SDValue(Mul.getNode(), 1)); 1173 1174 // Use the calculated overflow everywhere. 1175 ReplaceValueWith(SDValue(N, 1), Overflow); 1176 return Mul; 1177 } 1178 1179 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 1180 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 1181 N->getValueType(0))); 1182 } 1183 1184 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 1185 SDValue Chain = N->getOperand(0); // Get the chain. 1186 SDValue Ptr = N->getOperand(1); // Get the pointer. 1187 EVT VT = N->getValueType(0); 1188 SDLoc dl(N); 1189 1190 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 1191 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 1192 // The argument is passed as NumRegs registers of type RegVT. 1193 1194 SmallVector<SDValue, 8> Parts(NumRegs); 1195 for (unsigned i = 0; i < NumRegs; ++i) { 1196 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 1197 N->getConstantOperandVal(3)); 1198 Chain = Parts[i].getValue(1); 1199 } 1200 1201 // Handle endianness of the load. 1202 if (DAG.getDataLayout().isBigEndian()) 1203 std::reverse(Parts.begin(), Parts.end()); 1204 1205 // Assemble the parts in the promoted type. 1206 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1207 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 1208 for (unsigned i = 1; i < NumRegs; ++i) { 1209 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 1210 // Shift it to the right position and "or" it in. 1211 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 1212 DAG.getConstant(i * RegVT.getSizeInBits(), dl, 1213 TLI.getPointerTy(DAG.getDataLayout()))); 1214 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 1215 } 1216 1217 // Modified the chain result - switch anything that used the old chain to 1218 // use the new one. 1219 ReplaceValueWith(SDValue(N, 1), Chain); 1220 1221 return Res; 1222 } 1223 1224 //===----------------------------------------------------------------------===// 1225 // Integer Operand Promotion 1226 //===----------------------------------------------------------------------===// 1227 1228 /// PromoteIntegerOperand - This method is called when the specified operand of 1229 /// the specified node is found to need promotion. At this point, all of the 1230 /// result types of the node are known to be legal, but other operands of the 1231 /// node may need promotion or expansion as well as the specified one. 1232 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 1233 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); 1234 dbgs() << "\n"); 1235 SDValue Res = SDValue(); 1236 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 1237 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 1238 return false; 1239 } 1240 1241 switch (N->getOpcode()) { 1242 default: 1243 #ifndef NDEBUG 1244 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 1245 N->dump(&DAG); dbgs() << "\n"; 1246 #endif 1247 llvm_unreachable("Do not know how to promote this operator's operand!"); 1248 1249 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 1250 case ISD::ATOMIC_STORE: 1251 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 1252 break; 1253 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 1254 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 1255 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 1256 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 1257 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 1258 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 1259 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 1260 case ISD::INSERT_VECTOR_ELT: 1261 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 1262 case ISD::SCALAR_TO_VECTOR: 1263 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 1264 case ISD::SPLAT_VECTOR: 1265 Res = PromoteIntOp_SPLAT_VECTOR(N); break; 1266 case ISD::VSELECT: 1267 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 1268 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 1269 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 1270 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 1271 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 1272 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; 1273 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 1274 OpNo); break; 1275 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), 1276 OpNo); break; 1277 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), 1278 OpNo); break; 1279 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), 1280 OpNo); break; 1281 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), 1282 OpNo); break; 1283 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 1284 case ISD::FP16_TO_FP: 1285 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 1286 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; 1287 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 1288 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; 1289 1290 case ISD::SHL: 1291 case ISD::SRA: 1292 case ISD::SRL: 1293 case ISD::ROTL: 1294 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 1295 1296 case ISD::ADDCARRY: 1297 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break; 1298 1299 case ISD::FRAMEADDR: 1300 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; 1301 1302 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break; 1303 1304 case ISD::SMULFIX: 1305 case ISD::SMULFIXSAT: 1306 case ISD::UMULFIX: 1307 case ISD::UMULFIXSAT: 1308 case ISD::SDIVFIX: 1309 case ISD::UDIVFIX: Res = PromoteIntOp_FIX(N); break; 1310 1311 case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break; 1312 1313 case ISD::VECREDUCE_ADD: 1314 case ISD::VECREDUCE_MUL: 1315 case ISD::VECREDUCE_AND: 1316 case ISD::VECREDUCE_OR: 1317 case ISD::VECREDUCE_XOR: 1318 case ISD::VECREDUCE_SMAX: 1319 case ISD::VECREDUCE_SMIN: 1320 case ISD::VECREDUCE_UMAX: 1321 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; 1322 } 1323 1324 // If the result is null, the sub-method took care of registering results etc. 1325 if (!Res.getNode()) return false; 1326 1327 // If the result is N, the sub-method updated N in place. Tell the legalizer 1328 // core about this. 1329 if (Res.getNode() == N) 1330 return true; 1331 1332 const bool IsStrictFp = N->isStrictFPOpcode(); 1333 assert(Res.getValueType() == N->getValueType(0) && 1334 N->getNumValues() == (IsStrictFp ? 2 : 1) && 1335 "Invalid operand expansion"); 1336 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: "; 1337 Res.dump()); 1338 1339 ReplaceValueWith(SDValue(N, 0), Res); 1340 if (IsStrictFp) 1341 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 1342 1343 return false; 1344 } 1345 1346 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is 1347 /// shared among BR_CC, SELECT_CC, and SETCC handlers. 1348 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 1349 ISD::CondCode CCCode) { 1350 // We have to insert explicit sign or zero extends. Note that we could 1351 // insert sign extends for ALL conditions. For those operations where either 1352 // zero or sign extension would be valid, use SExtOrZExtPromotedInteger 1353 // which will choose the cheapest for the target. 1354 switch (CCCode) { 1355 default: llvm_unreachable("Unknown integer comparison!"); 1356 case ISD::SETEQ: 1357 case ISD::SETNE: { 1358 SDValue OpL = GetPromotedInteger(NewLHS); 1359 SDValue OpR = GetPromotedInteger(NewRHS); 1360 1361 // We would prefer to promote the comparison operand with sign extension. 1362 // If the width of OpL/OpR excluding the duplicated sign bits is no greater 1363 // than the width of NewLHS/NewRH, we can avoid inserting real truncate 1364 // instruction, which is redundant eventually. 1365 unsigned OpLEffectiveBits = 1366 OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1; 1367 unsigned OpREffectiveBits = 1368 OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1; 1369 if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() && 1370 OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) { 1371 NewLHS = OpL; 1372 NewRHS = OpR; 1373 } else { 1374 NewLHS = SExtOrZExtPromotedInteger(NewLHS); 1375 NewRHS = SExtOrZExtPromotedInteger(NewRHS); 1376 } 1377 break; 1378 } 1379 case ISD::SETUGE: 1380 case ISD::SETUGT: 1381 case ISD::SETULE: 1382 case ISD::SETULT: 1383 NewLHS = SExtOrZExtPromotedInteger(NewLHS); 1384 NewRHS = SExtOrZExtPromotedInteger(NewRHS); 1385 break; 1386 case ISD::SETGE: 1387 case ISD::SETGT: 1388 case ISD::SETLT: 1389 case ISD::SETLE: 1390 NewLHS = SExtPromotedInteger(NewLHS); 1391 NewRHS = SExtPromotedInteger(NewRHS); 1392 break; 1393 } 1394 } 1395 1396 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 1397 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1398 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 1399 } 1400 1401 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 1402 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 1403 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 1404 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); 1405 } 1406 1407 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 1408 // This should only occur in unusual situations like bitcasting to an 1409 // x86_fp80, so just turn it into a store+load 1410 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 1411 } 1412 1413 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 1414 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1415 1416 SDValue LHS = N->getOperand(2); 1417 SDValue RHS = N->getOperand(3); 1418 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 1419 1420 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 1421 // legal types. 1422 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1423 N->getOperand(1), LHS, RHS, N->getOperand(4)), 1424 0); 1425 } 1426 1427 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 1428 assert(OpNo == 1 && "only know how to promote condition"); 1429 1430 // Promote all the way up to the canonical SetCC type. 1431 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 1432 1433 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 1434 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 1435 N->getOperand(2)), 0); 1436 } 1437 1438 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 1439 // Since the result type is legal, the operands must promote to it. 1440 EVT OVT = N->getOperand(0).getValueType(); 1441 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 1442 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 1443 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 1444 SDLoc dl(N); 1445 1446 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 1447 DAG.getConstant(OVT.getSizeInBits(), dl, 1448 TLI.getPointerTy(DAG.getDataLayout()))); 1449 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 1450 } 1451 1452 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 1453 // The vector type is legal but the element type is not. This implies 1454 // that the vector is a power-of-two in length and that the element 1455 // type does not have a strange size (eg: it is not i1). 1456 EVT VecVT = N->getValueType(0); 1457 unsigned NumElts = VecVT.getVectorNumElements(); 1458 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 1459 "Legal vector of one illegal element?"); 1460 1461 // Promote the inserted value. The type does not need to match the 1462 // vector element type. Check that any extra bits introduced will be 1463 // truncated away. 1464 assert(N->getOperand(0).getValueSizeInBits() >= 1465 N->getValueType(0).getScalarSizeInBits() && 1466 "Type of inserted value narrower than vector element type!"); 1467 1468 SmallVector<SDValue, 16> NewOps; 1469 for (unsigned i = 0; i < NumElts; ++i) 1470 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 1471 1472 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1473 } 1474 1475 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 1476 unsigned OpNo) { 1477 if (OpNo == 1) { 1478 // Promote the inserted value. This is valid because the type does not 1479 // have to match the vector element type. 1480 1481 // Check that any extra bits introduced will be truncated away. 1482 assert(N->getOperand(1).getValueSizeInBits() >= 1483 N->getValueType(0).getScalarSizeInBits() && 1484 "Type of inserted value narrower than vector element type!"); 1485 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1486 GetPromotedInteger(N->getOperand(1)), 1487 N->getOperand(2)), 1488 0); 1489 } 1490 1491 assert(OpNo == 2 && "Different operand and result vector types?"); 1492 1493 // Promote the index. 1494 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 1495 TLI.getVectorIdxTy(DAG.getDataLayout())); 1496 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1497 N->getOperand(1), Idx), 0); 1498 } 1499 1500 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 1501 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 1502 // the operand in place. 1503 return SDValue(DAG.UpdateNodeOperands(N, 1504 GetPromotedInteger(N->getOperand(0))), 0); 1505 } 1506 1507 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) { 1508 // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the 1509 // operand in place. 1510 return SDValue( 1511 DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0); 1512 } 1513 1514 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 1515 assert(OpNo == 0 && "Only know how to promote the condition!"); 1516 SDValue Cond = N->getOperand(0); 1517 EVT OpTy = N->getOperand(1).getValueType(); 1518 1519 if (N->getOpcode() == ISD::VSELECT) 1520 if (SDValue Res = WidenVSELECTAndMask(N)) 1521 return Res; 1522 1523 // Promote all the way up to the canonical SetCC type. 1524 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 1525 Cond = PromoteTargetBoolean(Cond, OpVT); 1526 1527 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 1528 N->getOperand(2)), 0); 1529 } 1530 1531 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 1532 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1533 1534 SDValue LHS = N->getOperand(0); 1535 SDValue RHS = N->getOperand(1); 1536 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1537 1538 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1539 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1540 N->getOperand(3), N->getOperand(4)), 0); 1541 } 1542 1543 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1544 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1545 1546 SDValue LHS = N->getOperand(0); 1547 SDValue RHS = N->getOperand(1); 1548 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 1549 1550 // The CC (#2) is always legal. 1551 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 1552 } 1553 1554 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 1555 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1556 ZExtPromotedInteger(N->getOperand(1))), 0); 1557 } 1558 1559 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 1560 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1561 SDLoc dl(N); 1562 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1563 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 1564 Op, DAG.getValueType(N->getOperand(0).getValueType())); 1565 } 1566 1567 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 1568 return SDValue(DAG.UpdateNodeOperands(N, 1569 SExtPromotedInteger(N->getOperand(0))), 0); 1570 } 1571 1572 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { 1573 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1574 SExtPromotedInteger(N->getOperand(1))), 0); 1575 } 1576 1577 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 1578 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1579 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 1580 SDLoc dl(N); 1581 1582 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 1583 1584 // Truncate the value and store the result. 1585 return DAG.getTruncStore(Ch, dl, Val, Ptr, 1586 N->getMemoryVT(), N->getMemOperand()); 1587 } 1588 1589 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, 1590 unsigned OpNo) { 1591 1592 SDValue DataOp = N->getValue(); 1593 EVT DataVT = DataOp.getValueType(); 1594 SDValue Mask = N->getMask(); 1595 SDLoc dl(N); 1596 1597 bool TruncateStore = false; 1598 if (OpNo == 4) { 1599 Mask = PromoteTargetBoolean(Mask, DataVT); 1600 // Update in place. 1601 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 1602 NewOps[4] = Mask; 1603 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1604 } else { // Data operand 1605 assert(OpNo == 1 && "Unexpected operand for promotion"); 1606 DataOp = GetPromotedInteger(DataOp); 1607 TruncateStore = true; 1608 } 1609 1610 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), 1611 N->getOffset(), Mask, N->getMemoryVT(), 1612 N->getMemOperand(), N->getAddressingMode(), 1613 TruncateStore, N->isCompressingStore()); 1614 } 1615 1616 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, 1617 unsigned OpNo) { 1618 assert(OpNo == 3 && "Only know how to promote the mask!"); 1619 EVT DataVT = N->getValueType(0); 1620 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1621 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 1622 NewOps[OpNo] = Mask; 1623 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1624 } 1625 1626 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, 1627 unsigned OpNo) { 1628 1629 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1630 if (OpNo == 2) { 1631 // The Mask 1632 EVT DataVT = N->getValueType(0); 1633 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1634 } else if (OpNo == 4) { 1635 // The Index 1636 if (N->isIndexSigned()) 1637 // Need to sign extend the index since the bits will likely be used. 1638 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 1639 else 1640 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 1641 } else 1642 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1643 1644 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1645 } 1646 1647 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, 1648 unsigned OpNo) { 1649 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1650 if (OpNo == 2) { 1651 // The Mask 1652 EVT DataVT = N->getValue().getValueType(); 1653 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1654 } else if (OpNo == 4) { 1655 // The Index 1656 if (N->isIndexSigned()) 1657 // Need to sign extend the index since the bits will likely be used. 1658 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 1659 else 1660 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 1661 } else 1662 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1663 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1664 } 1665 1666 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 1667 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1668 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 1669 } 1670 1671 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 1672 return SDValue(DAG.UpdateNodeOperands(N, 1673 ZExtPromotedInteger(N->getOperand(0))), 0); 1674 } 1675 1676 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { 1677 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1678 ZExtPromotedInteger(N->getOperand(1))), 0); 1679 } 1680 1681 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 1682 SDLoc dl(N); 1683 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1684 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1685 return DAG.getZeroExtendInReg(Op, dl, 1686 N->getOperand(0).getValueType().getScalarType()); 1687 } 1688 1689 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { 1690 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1691 1692 SDValue LHS = N->getOperand(0); 1693 SDValue RHS = N->getOperand(1); 1694 SDValue Carry = N->getOperand(2); 1695 SDLoc DL(N); 1696 1697 Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); 1698 1699 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); 1700 } 1701 1702 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { 1703 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 1704 return SDValue( 1705 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); 1706 } 1707 1708 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { 1709 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. 1710 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 1711 return SDValue(DAG.UpdateNodeOperands(N, Op), 0); 1712 } 1713 1714 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) { 1715 assert(OpNo > 1 && "Don't know how to promote this operand!"); 1716 // Promote the rw, locality, and cache type arguments to a supported integer 1717 // width. 1718 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 1719 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3)); 1720 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4)); 1721 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), 1722 Op2, Op3, Op4), 1723 0); 1724 } 1725 1726 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) { 1727 SDValue Op = SExtPromotedInteger(N->getOperand(1)); 1728 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); 1729 } 1730 1731 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { 1732 SDLoc dl(N); 1733 SDValue Op; 1734 switch (N->getOpcode()) { 1735 default: llvm_unreachable("Expected integer vector reduction"); 1736 case ISD::VECREDUCE_ADD: 1737 case ISD::VECREDUCE_MUL: 1738 case ISD::VECREDUCE_AND: 1739 case ISD::VECREDUCE_OR: 1740 case ISD::VECREDUCE_XOR: 1741 Op = GetPromotedInteger(N->getOperand(0)); 1742 break; 1743 case ISD::VECREDUCE_SMAX: 1744 case ISD::VECREDUCE_SMIN: 1745 Op = SExtPromotedInteger(N->getOperand(0)); 1746 break; 1747 case ISD::VECREDUCE_UMAX: 1748 case ISD::VECREDUCE_UMIN: 1749 Op = ZExtPromotedInteger(N->getOperand(0)); 1750 break; 1751 } 1752 1753 EVT EltVT = Op.getValueType().getVectorElementType(); 1754 EVT VT = N->getValueType(0); 1755 if (VT.bitsGE(EltVT)) 1756 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op); 1757 1758 // Result size must be >= element size. If this is not the case after 1759 // promotion, also promote the result type and then truncate. 1760 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op); 1761 return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce); 1762 } 1763 1764 //===----------------------------------------------------------------------===// 1765 // Integer Result Expansion 1766 //===----------------------------------------------------------------------===// 1767 1768 /// ExpandIntegerResult - This method is called when the specified result of the 1769 /// specified node is found to need expansion. At this point, the node may also 1770 /// have invalid operands or may have other results that need promotion, we just 1771 /// know that (at least) one result needs expansion. 1772 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 1773 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); 1774 dbgs() << "\n"); 1775 SDValue Lo, Hi; 1776 Lo = Hi = SDValue(); 1777 1778 // See if the target wants to custom expand this node. 1779 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1780 return; 1781 1782 switch (N->getOpcode()) { 1783 default: 1784 #ifndef NDEBUG 1785 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 1786 N->dump(&DAG); dbgs() << "\n"; 1787 #endif 1788 report_fatal_error("Do not know how to expand the result of this " 1789 "operator!"); 1790 1791 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1792 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 1793 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1794 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1795 1796 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1797 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1798 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1799 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1800 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1801 1802 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 1803 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 1804 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 1805 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; 1806 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 1807 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 1808 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; 1809 case ISD::CTLZ_ZERO_UNDEF: 1810 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 1811 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 1812 case ISD::CTTZ_ZERO_UNDEF: 1813 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 1814 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break; 1815 case ISD::STRICT_FP_TO_SINT: 1816 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 1817 case ISD::STRICT_FP_TO_UINT: 1818 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 1819 case ISD::STRICT_LLROUND: 1820 case ISD::STRICT_LLRINT: 1821 case ISD::LLROUND: 1822 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break; 1823 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 1824 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 1825 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; 1826 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 1827 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 1828 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 1829 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 1830 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 1831 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 1832 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 1833 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 1834 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 1835 1836 case ISD::ATOMIC_LOAD_ADD: 1837 case ISD::ATOMIC_LOAD_SUB: 1838 case ISD::ATOMIC_LOAD_AND: 1839 case ISD::ATOMIC_LOAD_CLR: 1840 case ISD::ATOMIC_LOAD_OR: 1841 case ISD::ATOMIC_LOAD_XOR: 1842 case ISD::ATOMIC_LOAD_NAND: 1843 case ISD::ATOMIC_LOAD_MIN: 1844 case ISD::ATOMIC_LOAD_MAX: 1845 case ISD::ATOMIC_LOAD_UMIN: 1846 case ISD::ATOMIC_LOAD_UMAX: 1847 case ISD::ATOMIC_SWAP: 1848 case ISD::ATOMIC_CMP_SWAP: { 1849 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 1850 SplitInteger(Tmp.first, Lo, Hi); 1851 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1852 break; 1853 } 1854 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 1855 AtomicSDNode *AN = cast<AtomicSDNode>(N); 1856 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 1857 SDValue Tmp = DAG.getAtomicCmpSwap( 1858 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 1859 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 1860 AN->getMemOperand()); 1861 1862 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 1863 // success simply by comparing the loaded value against the ingoing 1864 // comparison. 1865 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 1866 N->getOperand(2), ISD::SETEQ); 1867 1868 SplitInteger(Tmp, Lo, Hi); 1869 ReplaceValueWith(SDValue(N, 1), Success); 1870 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 1871 break; 1872 } 1873 1874 case ISD::AND: 1875 case ISD::OR: 1876 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 1877 1878 case ISD::UMAX: 1879 case ISD::SMAX: 1880 case ISD::UMIN: 1881 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; 1882 1883 case ISD::ADD: 1884 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 1885 1886 case ISD::ADDC: 1887 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 1888 1889 case ISD::ADDE: 1890 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 1891 1892 case ISD::ADDCARRY: 1893 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break; 1894 1895 case ISD::SHL: 1896 case ISD::SRA: 1897 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 1898 1899 case ISD::SADDO: 1900 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 1901 case ISD::UADDO: 1902 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 1903 case ISD::UMULO: 1904 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 1905 1906 case ISD::SADDSAT: 1907 case ISD::UADDSAT: 1908 case ISD::SSUBSAT: 1909 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; 1910 1911 case ISD::SMULFIX: 1912 case ISD::SMULFIXSAT: 1913 case ISD::UMULFIX: 1914 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; 1915 1916 case ISD::SDIVFIX: 1917 case ISD::UDIVFIX: ExpandIntRes_DIVFIX(N, Lo, Hi); break; 1918 1919 case ISD::VECREDUCE_ADD: 1920 case ISD::VECREDUCE_MUL: 1921 case ISD::VECREDUCE_AND: 1922 case ISD::VECREDUCE_OR: 1923 case ISD::VECREDUCE_XOR: 1924 case ISD::VECREDUCE_SMAX: 1925 case ISD::VECREDUCE_SMIN: 1926 case ISD::VECREDUCE_UMAX: 1927 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; 1928 } 1929 1930 // If Lo/Hi is null, the sub-method took care of registering results etc. 1931 if (Lo.getNode()) 1932 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 1933 } 1934 1935 /// Lower an atomic node to the appropriate builtin call. 1936 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 1937 unsigned Opc = Node->getOpcode(); 1938 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 1939 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); 1940 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); 1941 1942 EVT RetVT = Node->getValueType(0); 1943 SmallVector<SDValue, 4> Ops(Node->op_begin() + 1, Node->op_end()); 1944 TargetLowering::MakeLibCallOptions CallOptions; 1945 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), 1946 Node->getOperand(0)); 1947 } 1948 1949 /// N is a shift by a value that needs to be expanded, 1950 /// and the shift amount is a constant 'Amt'. Expand the operation. 1951 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, 1952 SDValue &Lo, SDValue &Hi) { 1953 SDLoc DL(N); 1954 // Expand the incoming operand to be shifted, so that we have its parts 1955 SDValue InL, InH; 1956 GetExpandedInteger(N->getOperand(0), InL, InH); 1957 1958 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization 1959 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. 1960 if (!Amt) { 1961 Lo = InL; 1962 Hi = InH; 1963 return; 1964 } 1965 1966 EVT NVT = InL.getValueType(); 1967 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1968 unsigned NVTBits = NVT.getSizeInBits(); 1969 EVT ShTy = N->getOperand(1).getValueType(); 1970 1971 if (N->getOpcode() == ISD::SHL) { 1972 if (Amt.ugt(VTBits)) { 1973 Lo = Hi = DAG.getConstant(0, DL, NVT); 1974 } else if (Amt.ugt(NVTBits)) { 1975 Lo = DAG.getConstant(0, DL, NVT); 1976 Hi = DAG.getNode(ISD::SHL, DL, 1977 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1978 } else if (Amt == NVTBits) { 1979 Lo = DAG.getConstant(0, DL, NVT); 1980 Hi = InL; 1981 } else { 1982 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); 1983 Hi = DAG.getNode(ISD::OR, DL, NVT, 1984 DAG.getNode(ISD::SHL, DL, NVT, InH, 1985 DAG.getConstant(Amt, DL, ShTy)), 1986 DAG.getNode(ISD::SRL, DL, NVT, InL, 1987 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 1988 } 1989 return; 1990 } 1991 1992 if (N->getOpcode() == ISD::SRL) { 1993 if (Amt.ugt(VTBits)) { 1994 Lo = Hi = DAG.getConstant(0, DL, NVT); 1995 } else if (Amt.ugt(NVTBits)) { 1996 Lo = DAG.getNode(ISD::SRL, DL, 1997 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1998 Hi = DAG.getConstant(0, DL, NVT); 1999 } else if (Amt == NVTBits) { 2000 Lo = InH; 2001 Hi = DAG.getConstant(0, DL, NVT); 2002 } else { 2003 Lo = DAG.getNode(ISD::OR, DL, NVT, 2004 DAG.getNode(ISD::SRL, DL, NVT, InL, 2005 DAG.getConstant(Amt, DL, ShTy)), 2006 DAG.getNode(ISD::SHL, DL, NVT, InH, 2007 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2008 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2009 } 2010 return; 2011 } 2012 2013 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2014 if (Amt.ugt(VTBits)) { 2015 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2016 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2017 } else if (Amt.ugt(NVTBits)) { 2018 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2019 DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2020 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2021 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2022 } else if (Amt == NVTBits) { 2023 Lo = InH; 2024 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2025 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2026 } else { 2027 Lo = DAG.getNode(ISD::OR, DL, NVT, 2028 DAG.getNode(ISD::SRL, DL, NVT, InL, 2029 DAG.getConstant(Amt, DL, ShTy)), 2030 DAG.getNode(ISD::SHL, DL, NVT, InH, 2031 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2032 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2033 } 2034 } 2035 2036 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 2037 /// this shift based on knowledge of the high bit of the shift amount. If we 2038 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual 2039 /// shift amount. 2040 bool DAGTypeLegalizer:: 2041 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2042 SDValue Amt = N->getOperand(1); 2043 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2044 EVT ShTy = Amt.getValueType(); 2045 unsigned ShBits = ShTy.getScalarSizeInBits(); 2046 unsigned NVTBits = NVT.getScalarSizeInBits(); 2047 assert(isPowerOf2_32(NVTBits) && 2048 "Expanded integer type size not a power of two!"); 2049 SDLoc dl(N); 2050 2051 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 2052 KnownBits Known = DAG.computeKnownBits(N->getOperand(1)); 2053 2054 // If we don't know anything about the high bits, exit. 2055 if (((Known.Zero|Known.One) & HighBitMask) == 0) 2056 return false; 2057 2058 // Get the incoming operand to be shifted. 2059 SDValue InL, InH; 2060 GetExpandedInteger(N->getOperand(0), InL, InH); 2061 2062 // If we know that any of the high bits of the shift amount are one, then we 2063 // can do this as a couple of simple shifts. 2064 if (Known.One.intersects(HighBitMask)) { 2065 // Mask out the high bit, which we know is set. 2066 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 2067 DAG.getConstant(~HighBitMask, dl, ShTy)); 2068 2069 switch (N->getOpcode()) { 2070 default: llvm_unreachable("Unknown shift"); 2071 case ISD::SHL: 2072 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. 2073 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 2074 return true; 2075 case ISD::SRL: 2076 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2077 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 2078 return true; 2079 case ISD::SRA: 2080 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 2081 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2082 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 2083 return true; 2084 } 2085 } 2086 2087 // If we know that all of the high bits of the shift amount are zero, then we 2088 // can do this as a couple of simple shifts. 2089 if (HighBitMask.isSubsetOf(Known.Zero)) { 2090 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 2091 // shift if x is zero. We can use XOR here because x is known to be smaller 2092 // than 32. 2093 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 2094 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2095 2096 unsigned Op1, Op2; 2097 switch (N->getOpcode()) { 2098 default: llvm_unreachable("Unknown shift"); 2099 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 2100 case ISD::SRL: 2101 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 2102 } 2103 2104 // When shifting right the arithmetic for Lo and Hi is swapped. 2105 if (N->getOpcode() != ISD::SHL) 2106 std::swap(InL, InH); 2107 2108 // Use a little trick to get the bits that move from Lo to Hi. First 2109 // shift by one bit. 2110 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); 2111 // Then compute the remaining shift with amount-1. 2112 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 2113 2114 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 2115 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 2116 2117 if (N->getOpcode() != ISD::SHL) 2118 std::swap(Hi, Lo); 2119 return true; 2120 } 2121 2122 return false; 2123 } 2124 2125 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 2126 /// of any size. 2127 bool DAGTypeLegalizer:: 2128 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2129 SDValue Amt = N->getOperand(1); 2130 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2131 EVT ShTy = Amt.getValueType(); 2132 unsigned NVTBits = NVT.getSizeInBits(); 2133 assert(isPowerOf2_32(NVTBits) && 2134 "Expanded integer type size not a power of two!"); 2135 SDLoc dl(N); 2136 2137 // Get the incoming operand to be shifted. 2138 SDValue InL, InH; 2139 GetExpandedInteger(N->getOperand(0), InL, InH); 2140 2141 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); 2142 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 2143 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 2144 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2145 Amt, NVBitsNode, ISD::SETULT); 2146 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2147 Amt, DAG.getConstant(0, dl, ShTy), 2148 ISD::SETEQ); 2149 2150 SDValue LoS, HiS, LoL, HiL; 2151 switch (N->getOpcode()) { 2152 default: llvm_unreachable("Unknown shift"); 2153 case ISD::SHL: 2154 // Short: ShAmt < NVTBits 2155 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 2156 HiS = DAG.getNode(ISD::OR, dl, NVT, 2157 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 2158 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 2159 2160 // Long: ShAmt >= NVTBits 2161 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. 2162 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 2163 2164 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 2165 Hi = DAG.getSelect(dl, NVT, isZero, InH, 2166 DAG.getSelect(dl, NVT, isShort, HiS, HiL)); 2167 return true; 2168 case ISD::SRL: 2169 // Short: ShAmt < NVTBits 2170 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 2171 LoS = DAG.getNode(ISD::OR, dl, NVT, 2172 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2173 // FIXME: If Amt is zero, the following shift generates an undefined result 2174 // on some architectures. 2175 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2176 2177 // Long: ShAmt >= NVTBits 2178 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2179 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2180 2181 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2182 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2183 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2184 return true; 2185 case ISD::SRA: 2186 // Short: ShAmt < NVTBits 2187 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 2188 LoS = DAG.getNode(ISD::OR, dl, NVT, 2189 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2190 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2191 2192 // Long: ShAmt >= NVTBits 2193 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 2194 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2195 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2196 2197 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2198 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2199 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2200 return true; 2201 } 2202 } 2203 2204 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { 2205 2206 switch (Op) { 2207 default: llvm_unreachable("invalid min/max opcode"); 2208 case ISD::SMAX: 2209 return std::make_pair(ISD::SETGT, ISD::UMAX); 2210 case ISD::UMAX: 2211 return std::make_pair(ISD::SETUGT, ISD::UMAX); 2212 case ISD::SMIN: 2213 return std::make_pair(ISD::SETLT, ISD::UMIN); 2214 case ISD::UMIN: 2215 return std::make_pair(ISD::SETULT, ISD::UMIN); 2216 } 2217 } 2218 2219 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, 2220 SDValue &Lo, SDValue &Hi) { 2221 SDLoc DL(N); 2222 ISD::NodeType LoOpc; 2223 ISD::CondCode CondC; 2224 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); 2225 2226 // Expand the subcomponents. 2227 SDValue LHSL, LHSH, RHSL, RHSH; 2228 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2229 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2230 2231 // Value types 2232 EVT NVT = LHSL.getValueType(); 2233 EVT CCT = getSetCCResultType(NVT); 2234 2235 // Hi part is always the same op 2236 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 2237 2238 // We need to know whether to select Lo part that corresponds to 'winning' 2239 // Hi part or if Hi parts are equal. 2240 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); 2241 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); 2242 2243 // Lo part corresponding to the 'winning' Hi part 2244 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); 2245 2246 // Recursed Lo part if Hi parts are equal, this uses unsigned version 2247 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); 2248 2249 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); 2250 } 2251 2252 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 2253 SDValue &Lo, SDValue &Hi) { 2254 SDLoc dl(N); 2255 // Expand the subcomponents. 2256 SDValue LHSL, LHSH, RHSL, RHSH; 2257 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2258 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2259 2260 EVT NVT = LHSL.getValueType(); 2261 SDValue LoOps[2] = { LHSL, RHSL }; 2262 SDValue HiOps[3] = { LHSH, RHSH }; 2263 2264 bool HasOpCarry = TLI.isOperationLegalOrCustom( 2265 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY, 2266 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2267 if (HasOpCarry) { 2268 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 2269 if (N->getOpcode() == ISD::ADD) { 2270 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 2271 HiOps[2] = Lo.getValue(1); 2272 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps); 2273 } else { 2274 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 2275 HiOps[2] = Lo.getValue(1); 2276 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps); 2277 } 2278 return; 2279 } 2280 2281 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 2282 // them. TODO: Teach operation legalization how to expand unsupported 2283 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 2284 // a carry of type MVT::Glue, but there doesn't seem to be any way to 2285 // generate a value of this type in the expanded code sequence. 2286 bool hasCarry = 2287 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 2288 ISD::ADDC : ISD::SUBC, 2289 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2290 2291 if (hasCarry) { 2292 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 2293 if (N->getOpcode() == ISD::ADD) { 2294 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 2295 HiOps[2] = Lo.getValue(1); 2296 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 2297 } else { 2298 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 2299 HiOps[2] = Lo.getValue(1); 2300 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 2301 } 2302 return; 2303 } 2304 2305 bool hasOVF = 2306 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 2307 ISD::UADDO : ISD::USUBO, 2308 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2309 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); 2310 2311 if (hasOVF) { 2312 EVT OvfVT = getSetCCResultType(NVT); 2313 SDVTList VTList = DAG.getVTList(NVT, OvfVT); 2314 int RevOpc; 2315 if (N->getOpcode() == ISD::ADD) { 2316 RevOpc = ISD::SUB; 2317 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 2318 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 2319 } else { 2320 RevOpc = ISD::ADD; 2321 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 2322 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 2323 } 2324 SDValue OVF = Lo.getValue(1); 2325 2326 switch (BoolType) { 2327 case TargetLoweringBase::UndefinedBooleanContent: 2328 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); 2329 LLVM_FALLTHROUGH; 2330 case TargetLoweringBase::ZeroOrOneBooleanContent: 2331 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); 2332 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); 2333 break; 2334 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 2335 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); 2336 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); 2337 } 2338 return; 2339 } 2340 2341 if (N->getOpcode() == ISD::ADD) { 2342 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 2343 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 2344 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 2345 ISD::SETULT); 2346 2347 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) { 2348 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT); 2349 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); 2350 return; 2351 } 2352 2353 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1, 2354 DAG.getConstant(1, dl, NVT), 2355 DAG.getConstant(0, dl, NVT)); 2356 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1], 2357 ISD::SETULT); 2358 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2, 2359 DAG.getConstant(1, dl, NVT), Carry1); 2360 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 2361 } else { 2362 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 2363 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 2364 SDValue Cmp = 2365 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 2366 LoOps[0], LoOps[1], ISD::SETULT); 2367 2368 SDValue Borrow; 2369 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 2370 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); 2371 else 2372 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 2373 DAG.getConstant(0, dl, NVT)); 2374 2375 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 2376 } 2377 } 2378 2379 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 2380 SDValue &Lo, SDValue &Hi) { 2381 // Expand the subcomponents. 2382 SDValue LHSL, LHSH, RHSL, RHSH; 2383 SDLoc dl(N); 2384 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2385 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2386 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 2387 SDValue LoOps[2] = { LHSL, RHSL }; 2388 SDValue HiOps[3] = { LHSH, RHSH }; 2389 2390 if (N->getOpcode() == ISD::ADDC) { 2391 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 2392 HiOps[2] = Lo.getValue(1); 2393 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 2394 } else { 2395 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 2396 HiOps[2] = Lo.getValue(1); 2397 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 2398 } 2399 2400 // Legalized the flag result - switch anything that used the old flag to 2401 // use the new one. 2402 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2403 } 2404 2405 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 2406 SDValue &Lo, SDValue &Hi) { 2407 // Expand the subcomponents. 2408 SDValue LHSL, LHSH, RHSL, RHSH; 2409 SDLoc dl(N); 2410 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2411 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2412 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 2413 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 2414 SDValue HiOps[3] = { LHSH, RHSH }; 2415 2416 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2417 HiOps[2] = Lo.getValue(1); 2418 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 2419 2420 // Legalized the flag result - switch anything that used the old flag to 2421 // use the new one. 2422 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2423 } 2424 2425 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 2426 SDValue &Lo, SDValue &Hi) { 2427 SDValue LHS = N->getOperand(0); 2428 SDValue RHS = N->getOperand(1); 2429 SDLoc dl(N); 2430 2431 SDValue Ovf; 2432 2433 unsigned CarryOp, NoCarryOp; 2434 ISD::CondCode Cond; 2435 switch(N->getOpcode()) { 2436 case ISD::UADDO: 2437 CarryOp = ISD::ADDCARRY; 2438 NoCarryOp = ISD::ADD; 2439 Cond = ISD::SETULT; 2440 break; 2441 case ISD::USUBO: 2442 CarryOp = ISD::SUBCARRY; 2443 NoCarryOp = ISD::SUB; 2444 Cond = ISD::SETUGT; 2445 break; 2446 default: 2447 llvm_unreachable("Node has unexpected Opcode"); 2448 } 2449 2450 bool HasCarryOp = TLI.isOperationLegalOrCustom( 2451 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 2452 2453 if (HasCarryOp) { 2454 // Expand the subcomponents. 2455 SDValue LHSL, LHSH, RHSL, RHSH; 2456 GetExpandedInteger(LHS, LHSL, LHSH); 2457 GetExpandedInteger(RHS, RHSL, RHSH); 2458 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 2459 SDValue LoOps[2] = { LHSL, RHSL }; 2460 SDValue HiOps[3] = { LHSH, RHSH }; 2461 2462 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2463 HiOps[2] = Lo.getValue(1); 2464 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps); 2465 2466 Ovf = Hi.getValue(1); 2467 } else { 2468 // Expand the result by simply replacing it with the equivalent 2469 // non-overflow-checking operation. 2470 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS); 2471 SplitInteger(Sum, Lo, Hi); 2472 2473 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 2474 // overflows iff a - b > a. 2475 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); 2476 } 2477 2478 // Legalized the flag result - switch anything that used the old flag to 2479 // use the new one. 2480 ReplaceValueWith(SDValue(N, 1), Ovf); 2481 } 2482 2483 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N, 2484 SDValue &Lo, SDValue &Hi) { 2485 // Expand the subcomponents. 2486 SDValue LHSL, LHSH, RHSL, RHSH; 2487 SDLoc dl(N); 2488 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2489 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2490 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 2491 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 2492 SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; 2493 2494 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2495 HiOps[2] = Lo.getValue(1); 2496 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 2497 2498 // Legalized the flag result - switch anything that used the old flag to 2499 // use the new one. 2500 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2501 } 2502 2503 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 2504 SDValue &Lo, SDValue &Hi) { 2505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2506 SDLoc dl(N); 2507 SDValue Op = N->getOperand(0); 2508 if (Op.getValueType().bitsLE(NVT)) { 2509 // The low part is any extension of the input (which degenerates to a copy). 2510 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 2511 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 2512 } else { 2513 // For example, extension of an i48 to an i64. The operand type necessarily 2514 // promotes to the result type, so will end up being expanded too. 2515 assert(getTypeAction(Op.getValueType()) == 2516 TargetLowering::TypePromoteInteger && 2517 "Only know how to promote this result!"); 2518 SDValue Res = GetPromotedInteger(Op); 2519 assert(Res.getValueType() == N->getValueType(0) && 2520 "Operand over promoted?"); 2521 // Split the promoted operand. This will simplify when it is expanded. 2522 SplitInteger(Res, Lo, Hi); 2523 } 2524 } 2525 2526 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 2527 SDValue &Lo, SDValue &Hi) { 2528 SDLoc dl(N); 2529 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2530 EVT NVT = Lo.getValueType(); 2531 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2532 unsigned NVTBits = NVT.getSizeInBits(); 2533 unsigned EVTBits = EVT.getSizeInBits(); 2534 2535 if (NVTBits < EVTBits) { 2536 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 2537 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2538 EVTBits - NVTBits))); 2539 } else { 2540 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 2541 // The high part replicates the sign bit of Lo, make it explicit. 2542 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2543 DAG.getConstant(NVTBits - 1, dl, 2544 TLI.getPointerTy(DAG.getDataLayout()))); 2545 } 2546 } 2547 2548 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 2549 SDValue &Lo, SDValue &Hi) { 2550 SDLoc dl(N); 2551 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2552 EVT NVT = Lo.getValueType(); 2553 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2554 unsigned NVTBits = NVT.getSizeInBits(); 2555 unsigned EVTBits = EVT.getSizeInBits(); 2556 2557 if (NVTBits < EVTBits) { 2558 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 2559 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2560 EVTBits - NVTBits))); 2561 } else { 2562 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 2563 // The high part must be zero, make it explicit. 2564 Hi = DAG.getConstant(0, dl, NVT); 2565 } 2566 } 2567 2568 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, 2569 SDValue &Lo, SDValue &Hi) { 2570 SDLoc dl(N); 2571 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 2572 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); 2573 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); 2574 } 2575 2576 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 2577 SDValue &Lo, SDValue &Hi) { 2578 SDLoc dl(N); 2579 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 2580 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 2581 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 2582 } 2583 2584 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 2585 SDValue &Lo, SDValue &Hi) { 2586 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2587 unsigned NBitWidth = NVT.getSizeInBits(); 2588 auto Constant = cast<ConstantSDNode>(N); 2589 const APInt &Cst = Constant->getAPIntValue(); 2590 bool IsTarget = Constant->isTargetOpcode(); 2591 bool IsOpaque = Constant->isOpaque(); 2592 SDLoc dl(N); 2593 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); 2594 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, 2595 IsOpaque); 2596 } 2597 2598 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { 2599 SDLoc dl(N); 2600 2601 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) 2602 EVT VT = N->getValueType(0); 2603 SDValue N0 = N->getOperand(0); 2604 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, 2605 DAG.getConstant(0, dl, VT), N0); 2606 SDValue NegLo, NegHi; 2607 SplitInteger(Neg, NegLo, NegHi); 2608 2609 GetExpandedInteger(N0, Lo, Hi); 2610 EVT NVT = Lo.getValueType(); 2611 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), 2612 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT); 2613 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); 2614 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); 2615 } 2616 2617 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 2618 SDValue &Lo, SDValue &Hi) { 2619 SDLoc dl(N); 2620 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 2621 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2622 EVT NVT = Lo.getValueType(); 2623 2624 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 2625 DAG.getConstant(0, dl, NVT), ISD::SETNE); 2626 2627 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 2628 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 2629 2630 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 2631 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 2632 DAG.getConstant(NVT.getSizeInBits(), dl, 2633 NVT))); 2634 Hi = DAG.getConstant(0, dl, NVT); 2635 } 2636 2637 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 2638 SDValue &Lo, SDValue &Hi) { 2639 SDLoc dl(N); 2640 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 2641 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2642 EVT NVT = Lo.getValueType(); 2643 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 2644 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 2645 Hi = DAG.getConstant(0, dl, NVT); 2646 } 2647 2648 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 2649 SDValue &Lo, SDValue &Hi) { 2650 SDLoc dl(N); 2651 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 2652 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2653 EVT NVT = Lo.getValueType(); 2654 2655 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 2656 DAG.getConstant(0, dl, NVT), ISD::SETNE); 2657 2658 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 2659 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 2660 2661 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 2662 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 2663 DAG.getConstant(NVT.getSizeInBits(), dl, 2664 NVT))); 2665 Hi = DAG.getConstant(0, dl, NVT); 2666 } 2667 2668 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo, 2669 SDValue &Hi) { 2670 SDLoc dl(N); 2671 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2672 unsigned NBitWidth = NVT.getSizeInBits(); 2673 2674 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 2675 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT); 2676 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS 2677 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2678 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy)); 2679 } 2680 2681 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 2682 SDValue &Hi) { 2683 SDLoc dl(N); 2684 EVT VT = N->getValueType(0); 2685 2686 bool IsStrict = N->isStrictFPOpcode(); 2687 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2688 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 2689 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 2690 Op = GetPromotedFloat(Op); 2691 2692 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 2693 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 2694 TargetLowering::MakeLibCallOptions CallOptions; 2695 CallOptions.setSExt(true); 2696 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 2697 CallOptions, dl, Chain); 2698 SplitInteger(Tmp.first, Lo, Hi); 2699 2700 if (IsStrict) 2701 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2702 } 2703 2704 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 2705 SDValue &Hi) { 2706 SDLoc dl(N); 2707 EVT VT = N->getValueType(0); 2708 2709 bool IsStrict = N->isStrictFPOpcode(); 2710 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2711 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 2712 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 2713 Op = GetPromotedFloat(Op); 2714 2715 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 2716 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 2717 TargetLowering::MakeLibCallOptions CallOptions; 2718 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 2719 CallOptions, dl, Chain); 2720 SplitInteger(Tmp.first, Lo, Hi); 2721 2722 if (IsStrict) 2723 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2724 } 2725 2726 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo, 2727 SDValue &Hi) { 2728 SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); 2729 2730 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && 2731 "Input type needs to be promoted!"); 2732 2733 EVT VT = Op.getValueType(); 2734 2735 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2736 if (N->getOpcode() == ISD::LLROUND || 2737 N->getOpcode() == ISD::STRICT_LLROUND) { 2738 if (VT == MVT::f32) 2739 LC = RTLIB::LLROUND_F32; 2740 else if (VT == MVT::f64) 2741 LC = RTLIB::LLROUND_F64; 2742 else if (VT == MVT::f80) 2743 LC = RTLIB::LLROUND_F80; 2744 else if (VT == MVT::f128) 2745 LC = RTLIB::LLROUND_F128; 2746 else if (VT == MVT::ppcf128) 2747 LC = RTLIB::LLROUND_PPCF128; 2748 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); 2749 } else if (N->getOpcode() == ISD::LLRINT || 2750 N->getOpcode() == ISD::STRICT_LLRINT) { 2751 if (VT == MVT::f32) 2752 LC = RTLIB::LLRINT_F32; 2753 else if (VT == MVT::f64) 2754 LC = RTLIB::LLRINT_F64; 2755 else if (VT == MVT::f80) 2756 LC = RTLIB::LLRINT_F80; 2757 else if (VT == MVT::f128) 2758 LC = RTLIB::LLRINT_F128; 2759 else if (VT == MVT::ppcf128) 2760 LC = RTLIB::LLRINT_PPCF128; 2761 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); 2762 } else 2763 llvm_unreachable("Unexpected opcode!"); 2764 2765 SDLoc dl(N); 2766 EVT RetVT = N->getValueType(0); 2767 SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue(); 2768 2769 TargetLowering::MakeLibCallOptions CallOptions; 2770 CallOptions.setSExt(true); 2771 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, 2772 Op, CallOptions, dl, 2773 Chain); 2774 SplitInteger(Tmp.first, Lo, Hi); 2775 2776 if (N->isStrictFPOpcode()) 2777 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2778 } 2779 2780 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 2781 SDValue &Lo, SDValue &Hi) { 2782 if (N->isAtomic()) { 2783 // It's typical to have larger CAS than atomic load instructions. 2784 SDLoc dl(N); 2785 EVT VT = N->getMemoryVT(); 2786 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 2787 SDValue Zero = DAG.getConstant(0, dl, VT); 2788 SDValue Swap = DAG.getAtomicCmpSwap( 2789 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 2790 VT, VTs, N->getOperand(0), 2791 N->getOperand(1), Zero, Zero, N->getMemOperand()); 2792 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 2793 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 2794 return; 2795 } 2796 2797 if (ISD::isNormalLoad(N)) { 2798 ExpandRes_NormalLoad(N, Lo, Hi); 2799 return; 2800 } 2801 2802 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 2803 2804 EVT VT = N->getValueType(0); 2805 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2806 SDValue Ch = N->getChain(); 2807 SDValue Ptr = N->getBasePtr(); 2808 ISD::LoadExtType ExtType = N->getExtensionType(); 2809 unsigned Alignment = N->getAlignment(); 2810 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 2811 AAMDNodes AAInfo = N->getAAInfo(); 2812 SDLoc dl(N); 2813 2814 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2815 2816 if (N->getMemoryVT().bitsLE(NVT)) { 2817 EVT MemVT = N->getMemoryVT(); 2818 2819 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, 2820 Alignment, MMOFlags, AAInfo); 2821 2822 // Remember the chain. 2823 Ch = Lo.getValue(1); 2824 2825 if (ExtType == ISD::SEXTLOAD) { 2826 // The high part is obtained by SRA'ing all but one of the bits of the 2827 // lo part. 2828 unsigned LoSize = Lo.getValueSizeInBits(); 2829 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2830 DAG.getConstant(LoSize - 1, dl, 2831 TLI.getPointerTy(DAG.getDataLayout()))); 2832 } else if (ExtType == ISD::ZEXTLOAD) { 2833 // The high part is just a zero. 2834 Hi = DAG.getConstant(0, dl, NVT); 2835 } else { 2836 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 2837 // The high part is undefined. 2838 Hi = DAG.getUNDEF(NVT); 2839 } 2840 } else if (DAG.getDataLayout().isLittleEndian()) { 2841 // Little-endian - low bits are at low addresses. 2842 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags, 2843 AAInfo); 2844 2845 unsigned ExcessBits = 2846 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 2847 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 2848 2849 // Increment the pointer to the other half. 2850 unsigned IncrementSize = NVT.getSizeInBits()/8; 2851 Ptr = DAG.getMemBasePlusOffset(Ptr, IncrementSize, dl); 2852 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 2853 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 2854 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 2855 2856 // Build a factor node to remember that this load is independent of the 2857 // other one. 2858 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2859 Hi.getValue(1)); 2860 } else { 2861 // Big-endian - high bits are at low addresses. Favor aligned loads at 2862 // the cost of some bit-fiddling. 2863 EVT MemVT = N->getMemoryVT(); 2864 unsigned EBytes = MemVT.getStoreSize(); 2865 unsigned IncrementSize = NVT.getSizeInBits()/8; 2866 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2867 2868 // Load both the high bits and maybe some of the low bits. 2869 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 2870 EVT::getIntegerVT(*DAG.getContext(), 2871 MemVT.getSizeInBits() - ExcessBits), 2872 Alignment, MMOFlags, AAInfo); 2873 2874 // Increment the pointer to the other half. 2875 Ptr = DAG.getMemBasePlusOffset(Ptr, IncrementSize, dl); 2876 // Load the rest of the low bits. 2877 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 2878 N->getPointerInfo().getWithOffset(IncrementSize), 2879 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 2880 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 2881 2882 // Build a factor node to remember that this load is independent of the 2883 // other one. 2884 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2885 Hi.getValue(1)); 2886 2887 if (ExcessBits < NVT.getSizeInBits()) { 2888 // Transfer low bits from the bottom of Hi to the top of Lo. 2889 Lo = DAG.getNode( 2890 ISD::OR, dl, NVT, Lo, 2891 DAG.getNode(ISD::SHL, dl, NVT, Hi, 2892 DAG.getConstant(ExcessBits, dl, 2893 TLI.getPointerTy(DAG.getDataLayout())))); 2894 // Move high bits to the right position in Hi. 2895 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, 2896 Hi, 2897 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 2898 TLI.getPointerTy(DAG.getDataLayout()))); 2899 } 2900 } 2901 2902 // Legalize the chain result - switch anything that used the old chain to 2903 // use the new one. 2904 ReplaceValueWith(SDValue(N, 1), Ch); 2905 } 2906 2907 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 2908 SDValue &Lo, SDValue &Hi) { 2909 SDLoc dl(N); 2910 SDValue LL, LH, RL, RH; 2911 GetExpandedInteger(N->getOperand(0), LL, LH); 2912 GetExpandedInteger(N->getOperand(1), RL, RH); 2913 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 2914 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 2915 } 2916 2917 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 2918 SDValue &Lo, SDValue &Hi) { 2919 EVT VT = N->getValueType(0); 2920 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2921 SDLoc dl(N); 2922 2923 SDValue LL, LH, RL, RH; 2924 GetExpandedInteger(N->getOperand(0), LL, LH); 2925 GetExpandedInteger(N->getOperand(1), RL, RH); 2926 2927 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, 2928 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 2929 LL, LH, RL, RH)) 2930 return; 2931 2932 // If nothing else, we can make a libcall. 2933 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2934 if (VT == MVT::i16) 2935 LC = RTLIB::MUL_I16; 2936 else if (VT == MVT::i32) 2937 LC = RTLIB::MUL_I32; 2938 else if (VT == MVT::i64) 2939 LC = RTLIB::MUL_I64; 2940 else if (VT == MVT::i128) 2941 LC = RTLIB::MUL_I128; 2942 2943 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 2944 // We'll expand the multiplication by brute force because we have no other 2945 // options. This is a trivially-generalized version of the code from 2946 // Hacker's Delight (itself derived from Knuth's Algorithm M from section 2947 // 4.3.1). 2948 unsigned Bits = NVT.getSizeInBits(); 2949 unsigned HalfBits = Bits >> 1; 2950 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, 2951 NVT); 2952 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); 2953 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); 2954 2955 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); 2956 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); 2957 2958 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 2959 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) { 2960 // The type from TLI is too small to fit the shift amount we want. 2961 // Override it with i32. The shift will have to be legalized. 2962 ShiftAmtTy = MVT::i32; 2963 } 2964 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy); 2965 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); 2966 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); 2967 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); 2968 2969 SDValue U = DAG.getNode(ISD::ADD, dl, NVT, 2970 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); 2971 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); 2972 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); 2973 2974 SDValue V = DAG.getNode(ISD::ADD, dl, NVT, 2975 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); 2976 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); 2977 2978 SDValue W = DAG.getNode(ISD::ADD, dl, NVT, 2979 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), 2980 DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); 2981 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, 2982 DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); 2983 2984 Hi = DAG.getNode(ISD::ADD, dl, NVT, W, 2985 DAG.getNode(ISD::ADD, dl, NVT, 2986 DAG.getNode(ISD::MUL, dl, NVT, RH, LL), 2987 DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); 2988 return; 2989 } 2990 2991 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2992 TargetLowering::MakeLibCallOptions CallOptions; 2993 CallOptions.setSExt(true); 2994 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, 2995 Lo, Hi); 2996 } 2997 2998 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, 2999 SDValue &Hi) { 3000 SDLoc DL(N); 3001 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3002 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); 3003 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); 3004 Lo = R.getValue(0); 3005 Hi = R.getValue(1); 3006 ReplaceValueWith(SDValue(N, 1), R.getValue(2)); 3007 } 3008 3009 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, 3010 SDValue &Hi) { 3011 SDValue Result = TLI.expandAddSubSat(N, DAG); 3012 SplitInteger(Result, Lo, Hi); 3013 } 3014 3015 /// This performs an expansion of the integer result for a fixed point 3016 /// multiplication. The default expansion performs rounding down towards 3017 /// negative infinity, though targets that do care about rounding should specify 3018 /// a target hook for rounding and provide their own expansion or lowering of 3019 /// fixed point multiplication to be consistent with rounding. 3020 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, 3021 SDValue &Hi) { 3022 SDLoc dl(N); 3023 EVT VT = N->getValueType(0); 3024 unsigned VTSize = VT.getScalarSizeInBits(); 3025 SDValue LHS = N->getOperand(0); 3026 SDValue RHS = N->getOperand(1); 3027 uint64_t Scale = N->getConstantOperandVal(2); 3028 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || 3029 N->getOpcode() == ISD::UMULFIXSAT); 3030 bool Signed = (N->getOpcode() == ISD::SMULFIX || 3031 N->getOpcode() == ISD::SMULFIXSAT); 3032 3033 // Handle special case when scale is equal to zero. 3034 if (!Scale) { 3035 SDValue Result; 3036 if (!Saturating) { 3037 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 3038 } else { 3039 EVT BoolVT = getSetCCResultType(VT); 3040 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; 3041 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); 3042 SDValue Product = Result.getValue(0); 3043 SDValue Overflow = Result.getValue(1); 3044 if (Signed) { 3045 APInt MinVal = APInt::getSignedMinValue(VTSize); 3046 APInt MaxVal = APInt::getSignedMaxValue(VTSize); 3047 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); 3048 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 3049 SDValue Zero = DAG.getConstant(0, dl, VT); 3050 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT); 3051 Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin); 3052 Result = DAG.getSelect(dl, VT, Overflow, Result, Product); 3053 } else { 3054 // For unsigned multiplication, we only need to check the max since we 3055 // can't really overflow towards zero. 3056 APInt MaxVal = APInt::getMaxValue(VTSize); 3057 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 3058 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product); 3059 } 3060 } 3061 SplitInteger(Result, Lo, Hi); 3062 return; 3063 } 3064 3065 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will 3066 // cover for unhandled cases below, while still being valid for UMULFIX[SAT]. 3067 assert(Scale <= VTSize && "Scale can't be larger than the value type size."); 3068 3069 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3070 SDValue LL, LH, RL, RH; 3071 GetExpandedInteger(LHS, LL, LH); 3072 GetExpandedInteger(RHS, RL, RH); 3073 SmallVector<SDValue, 4> Result; 3074 3075 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; 3076 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, 3077 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3078 LL, LH, RL, RH)) { 3079 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); 3080 return; 3081 } 3082 3083 unsigned NVTSize = NVT.getScalarSizeInBits(); 3084 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " 3085 "the size of the current value type"); 3086 EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 3087 3088 // After getting the multiplication result in 4 parts, we need to perform a 3089 // shift right by the amount of the scale to get the result in that scale. 3090 // 3091 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in 3092 // 128 bits that are cut into 4 32-bit parts: 3093 // 3094 // HH HL LH LL 3095 // |---32---|---32---|---32---|---32---| 3096 // 128 96 64 32 0 3097 // 3098 // |------VTSize-----| 3099 // 3100 // |NVTSize-| 3101 // 3102 // The resulting Lo and Hi would normally be in LL and LH after the shift. But 3103 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift 3104 // amount and get Lo and Hi using two funnel shifts. Or for the special case 3105 // when Scale is a multiple of NVTSize we can just pick the result without 3106 // shifting. 3107 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. 3108 if (Scale % NVTSize) { 3109 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy); 3110 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0], 3111 ShiftAmount); 3112 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1], 3113 ShiftAmount); 3114 } else { 3115 Lo = Result[Part0]; 3116 Hi = Result[Part0 + 1]; 3117 } 3118 3119 // Unless saturation is requested we are done. The result is in <Hi,Lo>. 3120 if (!Saturating) 3121 return; 3122 3123 // Can not overflow when there is no integer part. 3124 if (Scale == VTSize) 3125 return; 3126 3127 // To handle saturation we must check for overflow in the multiplication. 3128 // 3129 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) 3130 // aren't all zeroes. 3131 // 3132 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) 3133 // aren't all ones or all zeroes. 3134 // 3135 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the 3136 // highest bit of HH determines saturation direction in the event of signed 3137 // saturation. 3138 3139 SDValue ResultHL = Result[2]; 3140 SDValue ResultHH = Result[3]; 3141 3142 SDValue SatMax, SatMin; 3143 SDValue NVTZero = DAG.getConstant(0, dl, NVT); 3144 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); 3145 EVT BoolNVT = getSetCCResultType(NVT); 3146 3147 if (!Signed) { 3148 if (Scale < NVTSize) { 3149 // Overflow happened if ((HH | (HL >> Scale)) != 0). 3150 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 3151 DAG.getConstant(Scale, dl, ShiftTy)); 3152 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH); 3153 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE); 3154 } else if (Scale == NVTSize) { 3155 // Overflow happened if (HH != 0). 3156 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE); 3157 } else if (Scale < VTSize) { 3158 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0). 3159 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 3160 DAG.getConstant(Scale - NVTSize, dl, 3161 ShiftTy)); 3162 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE); 3163 } else 3164 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" 3165 "(and saturation can't happen with Scale==VTSize)."); 3166 3167 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi); 3168 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo); 3169 return; 3170 } 3171 3172 if (Scale < NVTSize) { 3173 // The number of overflow bits we can check are VTSize - Scale + 1 (we 3174 // include the sign bit). If these top bits are > 0, then we overflowed past 3175 // the max value. If these top bits are < -1, then we overflowed past the 3176 // min value. Otherwise, we did not overflow. 3177 unsigned OverflowBits = VTSize - Scale + 1; 3178 assert(OverflowBits <= VTSize && OverflowBits > NVTSize && 3179 "Extent of overflow bits must start within HL"); 3180 SDValue HLHiMask = DAG.getConstant( 3181 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); 3182 SDValue HLLoMask = DAG.getConstant( 3183 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); 3184 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). 3185 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 3186 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 3187 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); 3188 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 3189 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT)); 3190 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). 3191 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 3192 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 3193 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); 3194 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 3195 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT)); 3196 } else if (Scale == NVTSize) { 3197 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). 3198 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 3199 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 3200 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); 3201 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 3202 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg)); 3203 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). 3204 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 3205 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 3206 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); 3207 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 3208 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos)); 3209 } else if (Scale < VTSize) { 3210 // This is similar to the case when we saturate if Scale < NVTSize, but we 3211 // only need to check HH. 3212 unsigned OverflowBits = VTSize - Scale + 1; 3213 SDValue HHHiMask = DAG.getConstant( 3214 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); 3215 SDValue HHLoMask = DAG.getConstant( 3216 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); 3217 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); 3218 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); 3219 } else 3220 llvm_unreachable("Illegal scale for signed fixed point mul."); 3221 3222 // Saturate to signed maximum. 3223 APInt MaxHi = APInt::getSignedMaxValue(NVTSize); 3224 APInt MaxLo = APInt::getAllOnesValue(NVTSize); 3225 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi); 3226 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo); 3227 // Saturate to signed minimum. 3228 APInt MinHi = APInt::getSignedMinValue(NVTSize); 3229 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi); 3230 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); 3231 } 3232 3233 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, 3234 SDValue &Hi) { 3235 SDValue Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1), 3236 N->getConstantOperandVal(2), TLI, DAG); 3237 SplitInteger(Res, Lo, Hi); 3238 } 3239 3240 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 3241 SDValue &Lo, SDValue &Hi) { 3242 SDValue LHS = Node->getOperand(0); 3243 SDValue RHS = Node->getOperand(1); 3244 SDLoc dl(Node); 3245 3246 // Expand the result by simply replacing it with the equivalent 3247 // non-overflow-checking operation. 3248 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 3249 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3250 LHS, RHS); 3251 SplitInteger(Sum, Lo, Hi); 3252 3253 // Compute the overflow. 3254 // 3255 // LHSSign -> LHS >= 0 3256 // RHSSign -> RHS >= 0 3257 // SumSign -> Sum >= 0 3258 // 3259 // Add: 3260 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 3261 // Sub: 3262 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 3263 // 3264 EVT OType = Node->getValueType(1); 3265 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); 3266 3267 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 3268 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 3269 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 3270 Node->getOpcode() == ISD::SADDO ? 3271 ISD::SETEQ : ISD::SETNE); 3272 3273 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 3274 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 3275 3276 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 3277 3278 // Use the calculated overflow everywhere. 3279 ReplaceValueWith(SDValue(Node, 1), Cmp); 3280 } 3281 3282 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 3283 SDValue &Lo, SDValue &Hi) { 3284 EVT VT = N->getValueType(0); 3285 SDLoc dl(N); 3286 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3287 3288 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 3289 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 3290 SplitInteger(Res.getValue(0), Lo, Hi); 3291 return; 3292 } 3293 3294 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3295 if (VT == MVT::i16) 3296 LC = RTLIB::SDIV_I16; 3297 else if (VT == MVT::i32) 3298 LC = RTLIB::SDIV_I32; 3299 else if (VT == MVT::i64) 3300 LC = RTLIB::SDIV_I64; 3301 else if (VT == MVT::i128) 3302 LC = RTLIB::SDIV_I128; 3303 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 3304 3305 TargetLowering::MakeLibCallOptions CallOptions; 3306 CallOptions.setSExt(true); 3307 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3308 } 3309 3310 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 3311 SDValue &Lo, SDValue &Hi) { 3312 EVT VT = N->getValueType(0); 3313 SDLoc dl(N); 3314 3315 // If we can emit an efficient shift operation, do so now. Check to see if 3316 // the RHS is a constant. 3317 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 3318 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); 3319 3320 // If we can determine that the high bit of the shift is zero or one, even if 3321 // the low bits are variable, emit this shift in an optimized form. 3322 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 3323 return; 3324 3325 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 3326 unsigned PartsOpc; 3327 if (N->getOpcode() == ISD::SHL) { 3328 PartsOpc = ISD::SHL_PARTS; 3329 } else if (N->getOpcode() == ISD::SRL) { 3330 PartsOpc = ISD::SRL_PARTS; 3331 } else { 3332 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 3333 PartsOpc = ISD::SRA_PARTS; 3334 } 3335 3336 // Next check to see if the target supports this SHL_PARTS operation or if it 3337 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for 3338 // size, but create a libcall instead. 3339 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3340 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 3341 const bool LegalOrCustom = 3342 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 3343 Action == TargetLowering::Custom; 3344 3345 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) { 3346 // Expand the subcomponents. 3347 SDValue LHSL, LHSH; 3348 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3349 EVT VT = LHSL.getValueType(); 3350 3351 // If the shift amount operand is coming from a vector legalization it may 3352 // have an illegal type. Fix that first by casting the operand, otherwise 3353 // the new SHL_PARTS operation would need further legalization. 3354 SDValue ShiftOp = N->getOperand(1); 3355 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 3356 assert(ShiftTy.getScalarSizeInBits() >= 3357 Log2_32_Ceil(VT.getScalarSizeInBits()) && 3358 "ShiftAmountTy is too small to cover the range of this type!"); 3359 if (ShiftOp.getValueType() != ShiftTy) 3360 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 3361 3362 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 3363 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 3364 Hi = Lo.getValue(1); 3365 return; 3366 } 3367 3368 // Otherwise, emit a libcall. 3369 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3370 bool isSigned; 3371 if (N->getOpcode() == ISD::SHL) { 3372 isSigned = false; /*sign irrelevant*/ 3373 if (VT == MVT::i16) 3374 LC = RTLIB::SHL_I16; 3375 else if (VT == MVT::i32) 3376 LC = RTLIB::SHL_I32; 3377 else if (VT == MVT::i64) 3378 LC = RTLIB::SHL_I64; 3379 else if (VT == MVT::i128) 3380 LC = RTLIB::SHL_I128; 3381 } else if (N->getOpcode() == ISD::SRL) { 3382 isSigned = false; 3383 if (VT == MVT::i16) 3384 LC = RTLIB::SRL_I16; 3385 else if (VT == MVT::i32) 3386 LC = RTLIB::SRL_I32; 3387 else if (VT == MVT::i64) 3388 LC = RTLIB::SRL_I64; 3389 else if (VT == MVT::i128) 3390 LC = RTLIB::SRL_I128; 3391 } else { 3392 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 3393 isSigned = true; 3394 if (VT == MVT::i16) 3395 LC = RTLIB::SRA_I16; 3396 else if (VT == MVT::i32) 3397 LC = RTLIB::SRA_I32; 3398 else if (VT == MVT::i64) 3399 LC = RTLIB::SRA_I64; 3400 else if (VT == MVT::i128) 3401 LC = RTLIB::SRA_I128; 3402 } 3403 3404 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 3405 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3406 TargetLowering::MakeLibCallOptions CallOptions; 3407 CallOptions.setSExt(isSigned); 3408 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3409 return; 3410 } 3411 3412 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 3413 llvm_unreachable("Unsupported shift!"); 3414 } 3415 3416 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 3417 SDValue &Lo, SDValue &Hi) { 3418 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3419 SDLoc dl(N); 3420 SDValue Op = N->getOperand(0); 3421 if (Op.getValueType().bitsLE(NVT)) { 3422 // The low part is sign extension of the input (degenerates to a copy). 3423 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 3424 // The high part is obtained by SRA'ing all but one of the bits of low part. 3425 unsigned LoSize = NVT.getSizeInBits(); 3426 Hi = DAG.getNode( 3427 ISD::SRA, dl, NVT, Lo, 3428 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); 3429 } else { 3430 // For example, extension of an i48 to an i64. The operand type necessarily 3431 // promotes to the result type, so will end up being expanded too. 3432 assert(getTypeAction(Op.getValueType()) == 3433 TargetLowering::TypePromoteInteger && 3434 "Only know how to promote this result!"); 3435 SDValue Res = GetPromotedInteger(Op); 3436 assert(Res.getValueType() == N->getValueType(0) && 3437 "Operand over promoted?"); 3438 // Split the promoted operand. This will simplify when it is expanded. 3439 SplitInteger(Res, Lo, Hi); 3440 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 3441 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 3442 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3443 ExcessBits))); 3444 } 3445 } 3446 3447 void DAGTypeLegalizer:: 3448 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 3449 SDLoc dl(N); 3450 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3451 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3452 3453 if (EVT.bitsLE(Lo.getValueType())) { 3454 // sext_inreg the low part if needed. 3455 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 3456 N->getOperand(1)); 3457 3458 // The high part gets the sign extension from the lo-part. This handles 3459 // things like sextinreg V:i64 from i8. 3460 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 3461 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, 3462 TLI.getPointerTy(DAG.getDataLayout()))); 3463 } else { 3464 // For example, extension of an i48 to an i64. Leave the low part alone, 3465 // sext_inreg the high part. 3466 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); 3467 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 3468 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3469 ExcessBits))); 3470 } 3471 } 3472 3473 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 3474 SDValue &Lo, SDValue &Hi) { 3475 EVT VT = N->getValueType(0); 3476 SDLoc dl(N); 3477 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3478 3479 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 3480 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 3481 SplitInteger(Res.getValue(1), Lo, Hi); 3482 return; 3483 } 3484 3485 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3486 if (VT == MVT::i16) 3487 LC = RTLIB::SREM_I16; 3488 else if (VT == MVT::i32) 3489 LC = RTLIB::SREM_I32; 3490 else if (VT == MVT::i64) 3491 LC = RTLIB::SREM_I64; 3492 else if (VT == MVT::i128) 3493 LC = RTLIB::SREM_I128; 3494 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 3495 3496 TargetLowering::MakeLibCallOptions CallOptions; 3497 CallOptions.setSExt(true); 3498 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3499 } 3500 3501 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 3502 SDValue &Lo, SDValue &Hi) { 3503 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3504 SDLoc dl(N); 3505 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 3506 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), 3507 N->getOperand(0), 3508 DAG.getConstant(NVT.getSizeInBits(), dl, 3509 TLI.getPointerTy(DAG.getDataLayout()))); 3510 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 3511 } 3512 3513 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 3514 SDValue &Lo, SDValue &Hi) { 3515 EVT VT = N->getValueType(0); 3516 SDLoc dl(N); 3517 3518 if (N->getOpcode() == ISD::UMULO) { 3519 // This section expands the operation into the following sequence of 3520 // instructions. `iNh` here refers to a type which has half the bit width of 3521 // the type the original operation operated on. 3522 // 3523 // %0 = %LHS.HI != 0 && %RHS.HI != 0 3524 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) 3525 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) 3526 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) 3527 // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh 3528 // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 ) 3529 // 3530 // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 } 3531 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 3532 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; 3533 SplitInteger(LHS, LHSLow, LHSHigh); 3534 SplitInteger(RHS, RHSLow, RHSHigh); 3535 EVT HalfVT = LHSLow.getValueType() 3536 , BitVT = N->getValueType(1); 3537 SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT); 3538 SDVTList VTFullAddO = DAG.getVTList(VT, BitVT); 3539 3540 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT); 3541 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT, 3542 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE), 3543 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE)); 3544 3545 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow); 3546 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1)); 3547 SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero, 3548 One.getValue(0)); 3549 3550 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow); 3551 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1)); 3552 SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero, 3553 Two.getValue(0)); 3554 3555 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not 3556 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this 3557 // operation recursively legalized?). 3558 // 3559 // Many backends understand this pattern and will convert into LOHI 3560 // themselves, if applicable. 3561 SDValue Three = DAG.getNode(ISD::MUL, dl, VT, 3562 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow), 3563 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow)); 3564 SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh); 3565 SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four); 3566 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1)); 3567 SplitInteger(Five, Lo, Hi); 3568 ReplaceValueWith(SDValue(N, 1), Overflow); 3569 return; 3570 } 3571 3572 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 3573 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 3574 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 3575 3576 // Replace this with a libcall that will check overflow. 3577 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3578 if (VT == MVT::i32) 3579 LC = RTLIB::MULO_I32; 3580 else if (VT == MVT::i64) 3581 LC = RTLIB::MULO_I64; 3582 else if (VT == MVT::i128) 3583 LC = RTLIB::MULO_I128; 3584 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); 3585 3586 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 3587 // Temporary for the overflow value, default it to zero. 3588 SDValue Chain = 3589 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, 3590 MachinePointerInfo()); 3591 3592 TargetLowering::ArgListTy Args; 3593 TargetLowering::ArgListEntry Entry; 3594 for (const SDValue &Op : N->op_values()) { 3595 EVT ArgVT = Op.getValueType(); 3596 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 3597 Entry.Node = Op; 3598 Entry.Ty = ArgTy; 3599 Entry.IsSExt = true; 3600 Entry.IsZExt = false; 3601 Args.push_back(Entry); 3602 } 3603 3604 // Also pass the address of the overflow check. 3605 Entry.Node = Temp; 3606 Entry.Ty = PtrTy->getPointerTo(); 3607 Entry.IsSExt = true; 3608 Entry.IsZExt = false; 3609 Args.push_back(Entry); 3610 3611 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 3612 3613 TargetLowering::CallLoweringInfo CLI(DAG); 3614 CLI.setDebugLoc(dl) 3615 .setChain(Chain) 3616 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) 3617 .setSExtResult(); 3618 3619 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 3620 3621 SplitInteger(CallInfo.first, Lo, Hi); 3622 SDValue Temp2 = 3623 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); 3624 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 3625 DAG.getConstant(0, dl, PtrVT), 3626 ISD::SETNE); 3627 // Use the overflow from the libcall everywhere. 3628 ReplaceValueWith(SDValue(N, 1), Ofl); 3629 } 3630 3631 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 3632 SDValue &Lo, SDValue &Hi) { 3633 EVT VT = N->getValueType(0); 3634 SDLoc dl(N); 3635 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3636 3637 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 3638 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 3639 SplitInteger(Res.getValue(0), Lo, Hi); 3640 return; 3641 } 3642 3643 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3644 if (VT == MVT::i16) 3645 LC = RTLIB::UDIV_I16; 3646 else if (VT == MVT::i32) 3647 LC = RTLIB::UDIV_I32; 3648 else if (VT == MVT::i64) 3649 LC = RTLIB::UDIV_I64; 3650 else if (VT == MVT::i128) 3651 LC = RTLIB::UDIV_I128; 3652 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 3653 3654 TargetLowering::MakeLibCallOptions CallOptions; 3655 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3656 } 3657 3658 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 3659 SDValue &Lo, SDValue &Hi) { 3660 EVT VT = N->getValueType(0); 3661 SDLoc dl(N); 3662 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3663 3664 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 3665 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 3666 SplitInteger(Res.getValue(1), Lo, Hi); 3667 return; 3668 } 3669 3670 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3671 if (VT == MVT::i16) 3672 LC = RTLIB::UREM_I16; 3673 else if (VT == MVT::i32) 3674 LC = RTLIB::UREM_I32; 3675 else if (VT == MVT::i64) 3676 LC = RTLIB::UREM_I64; 3677 else if (VT == MVT::i128) 3678 LC = RTLIB::UREM_I128; 3679 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 3680 3681 TargetLowering::MakeLibCallOptions CallOptions; 3682 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3683 } 3684 3685 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 3686 SDValue &Lo, SDValue &Hi) { 3687 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3688 SDLoc dl(N); 3689 SDValue Op = N->getOperand(0); 3690 if (Op.getValueType().bitsLE(NVT)) { 3691 // The low part is zero extension of the input (degenerates to a copy). 3692 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 3693 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. 3694 } else { 3695 // For example, extension of an i48 to an i64. The operand type necessarily 3696 // promotes to the result type, so will end up being expanded too. 3697 assert(getTypeAction(Op.getValueType()) == 3698 TargetLowering::TypePromoteInteger && 3699 "Only know how to promote this result!"); 3700 SDValue Res = GetPromotedInteger(Op); 3701 assert(Res.getValueType() == N->getValueType(0) && 3702 "Operand over promoted?"); 3703 // Split the promoted operand. This will simplify when it is expanded. 3704 SplitInteger(Res, Lo, Hi); 3705 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 3706 Hi = DAG.getZeroExtendInReg(Hi, dl, 3707 EVT::getIntegerVT(*DAG.getContext(), 3708 ExcessBits)); 3709 } 3710 } 3711 3712 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 3713 SDValue &Lo, SDValue &Hi) { 3714 SDLoc dl(N); 3715 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 3716 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 3717 SDValue Zero = DAG.getConstant(0, dl, VT); 3718 SDValue Swap = DAG.getAtomicCmpSwap( 3719 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 3720 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 3721 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); 3722 3723 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 3724 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 3725 } 3726 3727 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, 3728 SDValue &Lo, SDValue &Hi) { 3729 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate 3730 // both halves independently. 3731 SDValue Res = TLI.expandVecReduce(N, DAG); 3732 SplitInteger(Res, Lo, Hi); 3733 } 3734 3735 //===----------------------------------------------------------------------===// 3736 // Integer Operand Expansion 3737 //===----------------------------------------------------------------------===// 3738 3739 /// ExpandIntegerOperand - This method is called when the specified operand of 3740 /// the specified node is found to need expansion. At this point, all of the 3741 /// result types of the node are known to be legal, but other operands of the 3742 /// node may need promotion or expansion as well as the specified one. 3743 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 3744 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); 3745 dbgs() << "\n"); 3746 SDValue Res = SDValue(); 3747 3748 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 3749 return false; 3750 3751 switch (N->getOpcode()) { 3752 default: 3753 #ifndef NDEBUG 3754 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 3755 N->dump(&DAG); dbgs() << "\n"; 3756 #endif 3757 report_fatal_error("Do not know how to expand this operator's operand!"); 3758 3759 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 3760 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 3761 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 3762 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 3763 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 3764 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 3765 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 3766 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 3767 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; 3768 case ISD::STRICT_SINT_TO_FP: 3769 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 3770 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 3771 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 3772 case ISD::STRICT_UINT_TO_FP: 3773 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 3774 3775 case ISD::SHL: 3776 case ISD::SRA: 3777 case ISD::SRL: 3778 case ISD::ROTL: 3779 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 3780 case ISD::RETURNADDR: 3781 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 3782 3783 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 3784 } 3785 3786 // If the result is null, the sub-method took care of registering results etc. 3787 if (!Res.getNode()) return false; 3788 3789 // If the result is N, the sub-method updated N in place. Tell the legalizer 3790 // core about this. 3791 if (Res.getNode() == N) 3792 return true; 3793 3794 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 3795 "Invalid operand expansion"); 3796 3797 ReplaceValueWith(SDValue(N, 0), Res); 3798 return false; 3799 } 3800 3801 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 3802 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 3803 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 3804 SDValue &NewRHS, 3805 ISD::CondCode &CCCode, 3806 const SDLoc &dl) { 3807 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 3808 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 3809 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 3810 3811 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 3812 if (RHSLo == RHSHi) { 3813 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 3814 if (RHSCST->isAllOnesValue()) { 3815 // Equality comparison to -1. 3816 NewLHS = DAG.getNode(ISD::AND, dl, 3817 LHSLo.getValueType(), LHSLo, LHSHi); 3818 NewRHS = RHSLo; 3819 return; 3820 } 3821 } 3822 } 3823 3824 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 3825 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 3826 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 3827 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); 3828 return; 3829 } 3830 3831 // If this is a comparison of the sign bit, just look at the top part. 3832 // X > -1, x < 0 3833 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 3834 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 3835 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 3836 NewLHS = LHSHi; 3837 NewRHS = RHSHi; 3838 return; 3839 } 3840 3841 // FIXME: This generated code sucks. 3842 ISD::CondCode LowCC; 3843 switch (CCCode) { 3844 default: llvm_unreachable("Unknown integer setcc!"); 3845 case ISD::SETLT: 3846 case ISD::SETULT: LowCC = ISD::SETULT; break; 3847 case ISD::SETGT: 3848 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 3849 case ISD::SETLE: 3850 case ISD::SETULE: LowCC = ISD::SETULE; break; 3851 case ISD::SETGE: 3852 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 3853 } 3854 3855 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison 3856 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands 3857 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; 3858 3859 // NOTE: on targets without efficient SELECT of bools, we can always use 3860 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 3861 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 3862 nullptr); 3863 SDValue LoCmp, HiCmp; 3864 if (TLI.isTypeLegal(LHSLo.getValueType()) && 3865 TLI.isTypeLegal(RHSLo.getValueType())) 3866 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, 3867 RHSLo, LowCC, false, DagCombineInfo, dl); 3868 if (!LoCmp.getNode()) 3869 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 3870 RHSLo, LowCC); 3871 if (TLI.isTypeLegal(LHSHi.getValueType()) && 3872 TLI.isTypeLegal(RHSHi.getValueType())) 3873 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, 3874 RHSHi, CCCode, false, DagCombineInfo, dl); 3875 if (!HiCmp.getNode()) 3876 HiCmp = 3877 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), 3878 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 3879 3880 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); 3881 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); 3882 3883 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 3884 CCCode == ISD::SETUGE || CCCode == ISD::SETULE); 3885 3886 if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) || 3887 (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) || 3888 (LoCmpC && LoCmpC->isNullValue())))) { 3889 // For LE / GE, if high part is known false, ignore the low part. 3890 // For LT / GT: if low part is known false, return the high part. 3891 // if high part is known true, ignore the low part. 3892 NewLHS = HiCmp; 3893 NewRHS = SDValue(); 3894 return; 3895 } 3896 3897 if (LHSHi == RHSHi) { 3898 // Comparing the low bits is enough. 3899 NewLHS = LoCmp; 3900 NewRHS = SDValue(); 3901 return; 3902 } 3903 3904 // Lower with SETCCCARRY if the target supports it. 3905 EVT HiVT = LHSHi.getValueType(); 3906 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); 3907 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); 3908 3909 // FIXME: Make all targets support this, then remove the other lowering. 3910 if (HasSETCCCARRY) { 3911 // SETCCCARRY can detect < and >= directly. For > and <=, flip 3912 // operands and condition code. 3913 bool FlipOperands = false; 3914 switch (CCCode) { 3915 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; 3916 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; 3917 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; 3918 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; 3919 default: break; 3920 } 3921 if (FlipOperands) { 3922 std::swap(LHSLo, RHSLo); 3923 std::swap(LHSHi, RHSHi); 3924 } 3925 // Perform a wide subtraction, feeding the carry from the low part into 3926 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high 3927 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is 3928 // zero or positive iff LHS >= RHS. 3929 EVT LoVT = LHSLo.getValueType(); 3930 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT)); 3931 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo); 3932 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT), 3933 LHSHi, RHSHi, LowCmp.getValue(1), 3934 DAG.getCondCode(CCCode)); 3935 NewLHS = Res; 3936 NewRHS = SDValue(); 3937 return; 3938 } 3939 3940 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, 3941 false, DagCombineInfo, dl); 3942 if (!NewLHS.getNode()) 3943 NewLHS = 3944 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); 3945 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); 3946 NewRHS = SDValue(); 3947 } 3948 3949 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 3950 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 3951 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 3952 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 3953 3954 // If ExpandSetCCOperands returned a scalar, we need to compare the result 3955 // against zero to select between true and false values. 3956 if (!NewRHS.getNode()) { 3957 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 3958 CCCode = ISD::SETNE; 3959 } 3960 3961 // Update N to have the operands specified. 3962 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 3963 DAG.getCondCode(CCCode), NewLHS, NewRHS, 3964 N->getOperand(4)), 0); 3965 } 3966 3967 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 3968 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 3969 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 3970 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 3971 3972 // If ExpandSetCCOperands returned a scalar, we need to compare the result 3973 // against zero to select between true and false values. 3974 if (!NewRHS.getNode()) { 3975 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 3976 CCCode = ISD::SETNE; 3977 } 3978 3979 // Update N to have the operands specified. 3980 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 3981 N->getOperand(2), N->getOperand(3), 3982 DAG.getCondCode(CCCode)), 0); 3983 } 3984 3985 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 3986 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 3987 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 3988 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 3989 3990 // If ExpandSetCCOperands returned a scalar, use it. 3991 if (!NewRHS.getNode()) { 3992 assert(NewLHS.getValueType() == N->getValueType(0) && 3993 "Unexpected setcc expansion!"); 3994 return NewLHS; 3995 } 3996 3997 // Otherwise, update N to have the operands specified. 3998 return SDValue( 3999 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); 4000 } 4001 4002 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { 4003 SDValue LHS = N->getOperand(0); 4004 SDValue RHS = N->getOperand(1); 4005 SDValue Carry = N->getOperand(2); 4006 SDValue Cond = N->getOperand(3); 4007 SDLoc dl = SDLoc(N); 4008 4009 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 4010 GetExpandedInteger(LHS, LHSLo, LHSHi); 4011 GetExpandedInteger(RHS, RHSLo, RHSHi); 4012 4013 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high. 4014 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); 4015 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry); 4016 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, 4017 LowCmp.getValue(1), Cond); 4018 } 4019 4020 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 4021 // The value being shifted is legal, but the shift amount is too big. 4022 // It follows that either the result of the shift is undefined, or the 4023 // upper half of the shift amount is zero. Just use the lower half. 4024 SDValue Lo, Hi; 4025 GetExpandedInteger(N->getOperand(1), Lo, Hi); 4026 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 4027 } 4028 4029 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 4030 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 4031 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 4032 // constant to valid type. 4033 SDValue Lo, Hi; 4034 GetExpandedInteger(N->getOperand(0), Lo, Hi); 4035 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 4036 } 4037 4038 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 4039 bool IsStrict = N->isStrictFPOpcode(); 4040 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 4041 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 4042 EVT DstVT = N->getValueType(0); 4043 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 4044 assert(LC != RTLIB::UNKNOWN_LIBCALL && 4045 "Don't know how to expand this SINT_TO_FP!"); 4046 TargetLowering::MakeLibCallOptions CallOptions; 4047 CallOptions.setSExt(true); 4048 std::pair<SDValue, SDValue> Tmp = 4049 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 4050 4051 if (!IsStrict) 4052 return Tmp.first; 4053 4054 ReplaceValueWith(SDValue(N, 1), Tmp.second); 4055 ReplaceValueWith(SDValue(N, 0), Tmp.first); 4056 return SDValue(); 4057 } 4058 4059 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 4060 if (N->isAtomic()) { 4061 // It's typical to have larger CAS than atomic store instructions. 4062 SDLoc dl(N); 4063 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 4064 N->getMemoryVT(), 4065 N->getOperand(0), N->getOperand(2), 4066 N->getOperand(1), 4067 N->getMemOperand()); 4068 return Swap.getValue(1); 4069 } 4070 if (ISD::isNormalStore(N)) 4071 return ExpandOp_NormalStore(N, OpNo); 4072 4073 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 4074 assert(OpNo == 1 && "Can only expand the stored value so far"); 4075 4076 EVT VT = N->getOperand(1).getValueType(); 4077 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4078 SDValue Ch = N->getChain(); 4079 SDValue Ptr = N->getBasePtr(); 4080 unsigned Alignment = N->getAlignment(); 4081 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 4082 AAMDNodes AAInfo = N->getAAInfo(); 4083 SDLoc dl(N); 4084 SDValue Lo, Hi; 4085 4086 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 4087 4088 if (N->getMemoryVT().bitsLE(NVT)) { 4089 GetExpandedInteger(N->getValue(), Lo, Hi); 4090 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 4091 N->getMemoryVT(), Alignment, MMOFlags, AAInfo); 4092 } 4093 4094 if (DAG.getDataLayout().isLittleEndian()) { 4095 // Little-endian - low bits are at low addresses. 4096 GetExpandedInteger(N->getValue(), Lo, Hi); 4097 4098 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags, 4099 AAInfo); 4100 4101 unsigned ExcessBits = 4102 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 4103 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 4104 4105 // Increment the pointer to the other half. 4106 unsigned IncrementSize = NVT.getSizeInBits()/8; 4107 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize); 4108 Hi = DAG.getTruncStore( 4109 Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 4110 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 4111 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 4112 } 4113 4114 // Big-endian - high bits are at low addresses. Favor aligned stores at 4115 // the cost of some bit-fiddling. 4116 GetExpandedInteger(N->getValue(), Lo, Hi); 4117 4118 EVT ExtVT = N->getMemoryVT(); 4119 unsigned EBytes = ExtVT.getStoreSize(); 4120 unsigned IncrementSize = NVT.getSizeInBits()/8; 4121 unsigned ExcessBits = (EBytes - IncrementSize)*8; 4122 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 4123 ExtVT.getSizeInBits() - ExcessBits); 4124 4125 if (ExcessBits < NVT.getSizeInBits()) { 4126 // Transfer high bits from the top of Lo to the bottom of Hi. 4127 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 4128 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 4129 TLI.getPointerTy(DAG.getDataLayout()))); 4130 Hi = DAG.getNode( 4131 ISD::OR, dl, NVT, Hi, 4132 DAG.getNode(ISD::SRL, dl, NVT, Lo, 4133 DAG.getConstant(ExcessBits, dl, 4134 TLI.getPointerTy(DAG.getDataLayout())))); 4135 } 4136 4137 // Store both the high bits and maybe some of the low bits. 4138 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment, 4139 MMOFlags, AAInfo); 4140 4141 // Increment the pointer to the other half. 4142 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize); 4143 // Store the lowest ExcessBits bits in the second half. 4144 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 4145 N->getPointerInfo().getWithOffset(IncrementSize), 4146 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 4147 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 4148 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 4149 } 4150 4151 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 4152 SDValue InL, InH; 4153 GetExpandedInteger(N->getOperand(0), InL, InH); 4154 // Just truncate the low part of the source. 4155 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 4156 } 4157 4158 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 4159 bool IsStrict = N->isStrictFPOpcode(); 4160 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 4161 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 4162 EVT DstVT = N->getValueType(0); 4163 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT); 4164 assert(LC != RTLIB::UNKNOWN_LIBCALL && 4165 "Don't know how to expand this UINT_TO_FP!"); 4166 TargetLowering::MakeLibCallOptions CallOptions; 4167 CallOptions.setSExt(true); 4168 std::pair<SDValue, SDValue> Tmp = 4169 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 4170 4171 if (!IsStrict) 4172 return Tmp.first; 4173 4174 ReplaceValueWith(SDValue(N, 1), Tmp.second); 4175 ReplaceValueWith(SDValue(N, 0), Tmp.first); 4176 return SDValue(); 4177 } 4178 4179 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 4180 SDLoc dl(N); 4181 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 4182 cast<AtomicSDNode>(N)->getMemoryVT(), 4183 N->getOperand(0), 4184 N->getOperand(1), N->getOperand(2), 4185 cast<AtomicSDNode>(N)->getMemOperand()); 4186 return Swap.getValue(1); 4187 } 4188 4189 4190 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 4191 4192 EVT OutVT = N->getValueType(0); 4193 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4194 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4195 unsigned OutNumElems = OutVT.getVectorNumElements(); 4196 EVT NOutVTElem = NOutVT.getVectorElementType(); 4197 4198 SDLoc dl(N); 4199 SDValue BaseIdx = N->getOperand(1); 4200 4201 SDValue InOp0 = N->getOperand(0); 4202 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) 4203 InOp0 = GetPromotedInteger(N->getOperand(0)); 4204 4205 EVT InVT = InOp0.getValueType(); 4206 4207 SmallVector<SDValue, 8> Ops; 4208 Ops.reserve(OutNumElems); 4209 for (unsigned i = 0; i != OutNumElems; ++i) { 4210 4211 // Extract the element from the original vector. 4212 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 4213 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); 4214 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 4215 InVT.getVectorElementType(), N->getOperand(0), Index); 4216 4217 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); 4218 // Insert the converted element to the new vector. 4219 Ops.push_back(Op); 4220 } 4221 4222 return DAG.getBuildVector(NOutVT, dl, Ops); 4223 } 4224 4225 4226 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 4227 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 4228 EVT VT = N->getValueType(0); 4229 SDLoc dl(N); 4230 4231 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); 4232 4233 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4234 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 4235 EVT OutVT = V0.getValueType(); 4236 4237 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); 4238 } 4239 4240 4241 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 4242 EVT OutVT = N->getValueType(0); 4243 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4244 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4245 unsigned NumElems = N->getNumOperands(); 4246 EVT NOutVTElem = NOutVT.getVectorElementType(); 4247 4248 SDLoc dl(N); 4249 4250 SmallVector<SDValue, 8> Ops; 4251 Ops.reserve(NumElems); 4252 for (unsigned i = 0; i != NumElems; ++i) { 4253 SDValue Op; 4254 // BUILD_VECTOR integer operand types are allowed to be larger than the 4255 // result's element type. This may still be true after the promotion. For 4256 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 4257 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 4258 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem)) 4259 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); 4260 else 4261 Op = N->getOperand(i); 4262 Ops.push_back(Op); 4263 } 4264 4265 return DAG.getBuildVector(NOutVT, dl, Ops); 4266 } 4267 4268 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { 4269 4270 SDLoc dl(N); 4271 4272 assert(!N->getOperand(0).getValueType().isVector() && 4273 "Input must be a scalar"); 4274 4275 EVT OutVT = N->getValueType(0); 4276 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4277 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4278 EVT NOutVTElem = NOutVT.getVectorElementType(); 4279 4280 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); 4281 4282 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); 4283 } 4284 4285 SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) { 4286 SDLoc dl(N); 4287 4288 SDValue SplatVal = N->getOperand(0); 4289 4290 assert(!SplatVal.getValueType().isVector() && "Input must be a scalar"); 4291 4292 EVT OutVT = N->getValueType(0); 4293 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4294 assert(NOutVT.isVector() && "Type must be promoted to a vector type"); 4295 EVT NOutElemVT = NOutVT.getVectorElementType(); 4296 4297 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal); 4298 4299 return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op); 4300 } 4301 4302 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 4303 SDLoc dl(N); 4304 4305 EVT OutVT = N->getValueType(0); 4306 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4307 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4308 4309 EVT OutElemTy = NOutVT.getVectorElementType(); 4310 4311 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 4312 unsigned NumOutElem = NOutVT.getVectorNumElements(); 4313 unsigned NumOperands = N->getNumOperands(); 4314 assert(NumElem * NumOperands == NumOutElem && 4315 "Unexpected number of elements"); 4316 4317 // Take the elements from the first vector. 4318 SmallVector<SDValue, 8> Ops(NumOutElem); 4319 for (unsigned i = 0; i < NumOperands; ++i) { 4320 SDValue Op = N->getOperand(i); 4321 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) 4322 Op = GetPromotedInteger(Op); 4323 EVT SclrTy = Op.getValueType().getVectorElementType(); 4324 assert(NumElem == Op.getValueType().getVectorNumElements() && 4325 "Unexpected number of elements"); 4326 4327 for (unsigned j = 0; j < NumElem; ++j) { 4328 SDValue Ext = DAG.getNode( 4329 ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, 4330 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4331 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); 4332 } 4333 } 4334 4335 return DAG.getBuildVector(NOutVT, dl, Ops); 4336 } 4337 4338 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { 4339 EVT VT = N->getValueType(0); 4340 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4341 assert(NVT.isVector() && "This type must be promoted to a vector type"); 4342 4343 SDLoc dl(N); 4344 4345 // For operands whose TypeAction is to promote, extend the promoted node 4346 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion 4347 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to 4348 // type.. 4349 if (getTypeAction(N->getOperand(0).getValueType()) 4350 == TargetLowering::TypePromoteInteger) { 4351 SDValue Promoted; 4352 4353 switch(N->getOpcode()) { 4354 case ISD::SIGN_EXTEND_VECTOR_INREG: 4355 Promoted = SExtPromotedInteger(N->getOperand(0)); 4356 break; 4357 case ISD::ZERO_EXTEND_VECTOR_INREG: 4358 Promoted = ZExtPromotedInteger(N->getOperand(0)); 4359 break; 4360 case ISD::ANY_EXTEND_VECTOR_INREG: 4361 Promoted = GetPromotedInteger(N->getOperand(0)); 4362 break; 4363 default: 4364 llvm_unreachable("Node has unexpected Opcode"); 4365 } 4366 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); 4367 } 4368 4369 // Directly extend to the appropriate transform-to type. 4370 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 4371 } 4372 4373 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 4374 EVT OutVT = N->getValueType(0); 4375 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4376 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4377 4378 EVT NOutVTElem = NOutVT.getVectorElementType(); 4379 4380 SDLoc dl(N); 4381 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4382 4383 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 4384 NOutVTElem, N->getOperand(1)); 4385 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 4386 V0, ConvElem, N->getOperand(2)); 4387 } 4388 4389 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { 4390 // The VECREDUCE result size may be larger than the element size, so 4391 // we can simply change the result type. 4392 SDLoc dl(N); 4393 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4394 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 4395 } 4396 4397 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 4398 SDLoc dl(N); 4399 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4400 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, 4401 TLI.getVectorIdxTy(DAG.getDataLayout())); 4402 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 4403 V0->getValueType(0).getScalarType(), V0, V1); 4404 4405 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 4406 // element types. If this is the case then we need to expand the outgoing 4407 // value and not truncate it. 4408 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 4409 } 4410 4411 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { 4412 SDLoc dl(N); 4413 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4414 MVT InVT = V0.getValueType().getSimpleVT(); 4415 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), 4416 N->getValueType(0).getVectorNumElements()); 4417 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); 4418 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 4419 } 4420 4421 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 4422 SDLoc dl(N); 4423 unsigned NumElems = N->getNumOperands(); 4424 4425 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 4426 4427 SmallVector<SDValue, 8> NewOps; 4428 NewOps.reserve(NumElems); 4429 4430 // For each incoming vector 4431 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 4432 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 4433 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 4434 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 4435 4436 for (unsigned i=0; i<NumElem; ++i) { 4437 // Extract element from incoming vector 4438 SDValue Ex = DAG.getNode( 4439 ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, 4440 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4441 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 4442 NewOps.push_back(Tr); 4443 } 4444 } 4445 4446 return DAG.getBuildVector(N->getValueType(0), dl, NewOps); 4447 } 4448