1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements integer type expansion and promotion for LegalizeTypes. 10 // Promotion is the act of changing a computation in an illegal type into a 11 // computation in a larger type. For example, implementing i8 arithmetic in an 12 // i32 register (often needed on powerpc). 13 // Expansion is the act of changing a computation in an illegal type into a 14 // computation in two identical registers of a smaller type. For example, 15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit 16 // targets). 17 // 18 //===----------------------------------------------------------------------===// 19 20 #include "LegalizeTypes.h" 21 #include "llvm/Analysis/TargetLibraryInfo.h" 22 #include "llvm/IR/DerivedTypes.h" 23 #include "llvm/Support/ErrorHandling.h" 24 #include "llvm/Support/KnownBits.h" 25 #include "llvm/Support/raw_ostream.h" 26 using namespace llvm; 27 28 #define DEBUG_TYPE "legalize-types" 29 30 //===----------------------------------------------------------------------===// 31 // Integer Result Promotion 32 //===----------------------------------------------------------------------===// 33 34 /// PromoteIntegerResult - This method is called when a result of a node is 35 /// found to be in need of promotion to a larger type. At this point, the node 36 /// may also have invalid operands or may have other results that need 37 /// expansion, we just know that (at least) one result needs promotion. 38 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 39 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); 40 dbgs() << "\n"); 41 SDValue Res = SDValue(); 42 43 // See if the target wants to custom expand this node. 44 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 45 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 46 return; 47 } 48 49 switch (N->getOpcode()) { 50 default: 51 #ifndef NDEBUG 52 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 53 N->dump(&DAG); dbgs() << "\n"; 54 #endif 55 llvm_unreachable("Do not know how to promote this operator!"); 56 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 57 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 58 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 59 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 60 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; 61 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 62 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 63 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 64 case ISD::CTLZ_ZERO_UNDEF: 65 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 66 case ISD::PARITY: 67 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; 68 case ISD::CTTZ_ZERO_UNDEF: 69 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 70 case ISD::EXTRACT_VECTOR_ELT: 71 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 72 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 73 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); 74 break; 75 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); 76 break; 77 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 78 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; 79 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 80 case ISD::STRICT_FSETCC: 81 case ISD::STRICT_FSETCCS: 82 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 83 case ISD::SMIN: 84 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; 85 case ISD::UMIN: 86 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; 87 88 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 89 case ISD::SIGN_EXTEND_INREG: 90 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 91 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 92 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 93 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 94 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 95 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 96 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; 97 98 case ISD::EXTRACT_SUBVECTOR: 99 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 100 case ISD::INSERT_SUBVECTOR: 101 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break; 102 case ISD::VECTOR_REVERSE: 103 Res = PromoteIntRes_VECTOR_REVERSE(N); break; 104 case ISD::VECTOR_SHUFFLE: 105 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 106 case ISD::VECTOR_SPLICE: 107 Res = PromoteIntRes_VECTOR_SPLICE(N); break; 108 case ISD::INSERT_VECTOR_ELT: 109 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 110 case ISD::BUILD_VECTOR: 111 Res = PromoteIntRes_BUILD_VECTOR(N); break; 112 case ISD::SCALAR_TO_VECTOR: 113 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; 114 case ISD::SPLAT_VECTOR: 115 Res = PromoteIntRes_SPLAT_VECTOR(N); break; 116 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break; 117 case ISD::CONCAT_VECTORS: 118 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 119 120 case ISD::ANY_EXTEND_VECTOR_INREG: 121 case ISD::SIGN_EXTEND_VECTOR_INREG: 122 case ISD::ZERO_EXTEND_VECTOR_INREG: 123 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; 124 125 case ISD::SIGN_EXTEND: 126 case ISD::ZERO_EXTEND: 127 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 128 129 case ISD::STRICT_FP_TO_SINT: 130 case ISD::STRICT_FP_TO_UINT: 131 case ISD::FP_TO_SINT: 132 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 133 134 case ISD::FP_TO_SINT_SAT: 135 case ISD::FP_TO_UINT_SAT: 136 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; 137 138 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break; 139 140 case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break; 141 142 case ISD::AND: 143 case ISD::OR: 144 case ISD::XOR: 145 case ISD::ADD: 146 case ISD::SUB: 147 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 148 149 case ISD::SDIV: 150 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; 151 152 case ISD::UDIV: 153 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; 154 155 case ISD::SADDO: 156 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 157 case ISD::UADDO: 158 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 159 case ISD::SMULO: 160 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 161 162 case ISD::ADDE: 163 case ISD::SUBE: 164 case ISD::ADDCARRY: 165 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break; 166 167 case ISD::SADDO_CARRY: 168 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; 169 170 case ISD::SADDSAT: 171 case ISD::UADDSAT: 172 case ISD::SSUBSAT: 173 case ISD::USUBSAT: 174 case ISD::SSHLSAT: 175 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break; 176 177 case ISD::SMULFIX: 178 case ISD::SMULFIXSAT: 179 case ISD::UMULFIX: 180 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; 181 182 case ISD::SDIVFIX: 183 case ISD::SDIVFIXSAT: 184 case ISD::UDIVFIX: 185 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; 186 187 case ISD::ABS: Res = PromoteIntRes_ABS(N); break; 188 189 case ISD::ATOMIC_LOAD: 190 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 191 192 case ISD::ATOMIC_LOAD_ADD: 193 case ISD::ATOMIC_LOAD_SUB: 194 case ISD::ATOMIC_LOAD_AND: 195 case ISD::ATOMIC_LOAD_CLR: 196 case ISD::ATOMIC_LOAD_OR: 197 case ISD::ATOMIC_LOAD_XOR: 198 case ISD::ATOMIC_LOAD_NAND: 199 case ISD::ATOMIC_LOAD_MIN: 200 case ISD::ATOMIC_LOAD_MAX: 201 case ISD::ATOMIC_LOAD_UMIN: 202 case ISD::ATOMIC_LOAD_UMAX: 203 case ISD::ATOMIC_SWAP: 204 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 205 206 case ISD::ATOMIC_CMP_SWAP: 207 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 208 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 209 break; 210 211 case ISD::VECREDUCE_ADD: 212 case ISD::VECREDUCE_MUL: 213 case ISD::VECREDUCE_AND: 214 case ISD::VECREDUCE_OR: 215 case ISD::VECREDUCE_XOR: 216 case ISD::VECREDUCE_SMAX: 217 case ISD::VECREDUCE_SMIN: 218 case ISD::VECREDUCE_UMAX: 219 case ISD::VECREDUCE_UMIN: 220 Res = PromoteIntRes_VECREDUCE(N); 221 break; 222 223 case ISD::FREEZE: 224 Res = PromoteIntRes_FREEZE(N); 225 break; 226 227 case ISD::ROTL: 228 case ISD::ROTR: 229 Res = PromoteIntRes_Rotate(N); 230 break; 231 232 case ISD::FSHL: 233 case ISD::FSHR: 234 Res = PromoteIntRes_FunnelShift(N); 235 break; 236 } 237 238 // If the result is null then the sub-method took care of registering it. 239 if (Res.getNode()) 240 SetPromotedInteger(SDValue(N, ResNo), Res); 241 } 242 243 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 244 unsigned ResNo) { 245 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 246 return GetPromotedInteger(Op); 247 } 248 249 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 250 // Sign-extend the new bits, and continue the assertion. 251 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 252 return DAG.getNode(ISD::AssertSext, SDLoc(N), 253 Op.getValueType(), Op, N->getOperand(1)); 254 } 255 256 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 257 // Zero the new bits, and continue the assertion. 258 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 259 return DAG.getNode(ISD::AssertZext, SDLoc(N), 260 Op.getValueType(), Op, N->getOperand(1)); 261 } 262 263 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 264 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 265 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 266 N->getMemoryVT(), ResVT, 267 N->getChain(), N->getBasePtr(), 268 N->getMemOperand()); 269 // Legalize the chain result - switch anything that used the old chain to 270 // use the new one. 271 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 272 return Res; 273 } 274 275 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 276 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 277 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 278 N->getMemoryVT(), 279 N->getChain(), N->getBasePtr(), 280 Op2, N->getMemOperand()); 281 // Legalize the chain result - switch anything that used the old chain to 282 // use the new one. 283 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 284 return Res; 285 } 286 287 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 288 unsigned ResNo) { 289 if (ResNo == 1) { 290 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 291 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 292 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 293 294 // Only use the result of getSetCCResultType if it is legal, 295 // otherwise just use the promoted result type (NVT). 296 if (!TLI.isTypeLegal(SVT)) 297 SVT = NVT; 298 299 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 300 SDValue Res = DAG.getAtomicCmpSwap( 301 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 302 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 303 N->getMemOperand()); 304 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 305 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 306 return Res.getValue(1); 307 } 308 309 // Op2 is used for the comparison and thus must be extended according to the 310 // target's atomic operations. Op3 is merely stored and so can be left alone. 311 SDValue Op2 = N->getOperand(2); 312 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 313 switch (TLI.getExtendForAtomicCmpSwapArg()) { 314 case ISD::SIGN_EXTEND: 315 Op2 = SExtPromotedInteger(Op2); 316 break; 317 case ISD::ZERO_EXTEND: 318 Op2 = ZExtPromotedInteger(Op2); 319 break; 320 case ISD::ANY_EXTEND: 321 Op2 = GetPromotedInteger(Op2); 322 break; 323 default: 324 llvm_unreachable("Invalid atomic op extension"); 325 } 326 327 SDVTList VTs = 328 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 329 SDValue Res = DAG.getAtomicCmpSwap( 330 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 331 N->getBasePtr(), Op2, Op3, N->getMemOperand()); 332 // Update the use to N with the newly created Res. 333 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) 334 ReplaceValueWith(SDValue(N, i), Res.getValue(i)); 335 return Res; 336 } 337 338 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 339 SDValue InOp = N->getOperand(0); 340 EVT InVT = InOp.getValueType(); 341 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 342 EVT OutVT = N->getValueType(0); 343 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 344 SDLoc dl(N); 345 346 switch (getTypeAction(InVT)) { 347 case TargetLowering::TypeLegal: 348 break; 349 case TargetLowering::TypePromoteInteger: 350 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 351 // The input promotes to the same size. Convert the promoted value. 352 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 353 break; 354 case TargetLowering::TypeSoftenFloat: 355 // Promote the integer operand by hand. 356 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 357 case TargetLowering::TypeSoftPromoteHalf: 358 // Promote the integer operand by hand. 359 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp)); 360 case TargetLowering::TypePromoteFloat: { 361 // Convert the promoted float by hand. 362 if (!NOutVT.isVector()) 363 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); 364 break; 365 } 366 case TargetLowering::TypeExpandInteger: 367 case TargetLowering::TypeExpandFloat: 368 break; 369 case TargetLowering::TypeScalarizeVector: 370 // Convert the element to an integer and promote it by hand. 371 if (!NOutVT.isVector()) 372 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 373 BitConvertToInteger(GetScalarizedVector(InOp))); 374 break; 375 case TargetLowering::TypeScalarizeScalableVector: 376 report_fatal_error("Scalarization of scalable vectors is not supported."); 377 case TargetLowering::TypeSplitVector: { 378 if (!NOutVT.isVector()) { 379 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 380 // pieces of the input into integers and reassemble in the final type. 381 SDValue Lo, Hi; 382 GetSplitVector(N->getOperand(0), Lo, Hi); 383 Lo = BitConvertToInteger(Lo); 384 Hi = BitConvertToInteger(Hi); 385 386 if (DAG.getDataLayout().isBigEndian()) 387 std::swap(Lo, Hi); 388 389 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 390 EVT::getIntegerVT(*DAG.getContext(), 391 NOutVT.getSizeInBits()), 392 JoinIntegers(Lo, Hi)); 393 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 394 } 395 break; 396 } 397 case TargetLowering::TypeWidenVector: 398 // The input is widened to the same size. Convert to the widened value. 399 // Make sure that the outgoing value is not a vector, because this would 400 // make us bitcast between two vectors which are legalized in different ways. 401 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) { 402 SDValue Res = 403 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 404 405 // For big endian targets we need to shift the casted value or the 406 // interesting bits will end up at the wrong place. 407 if (DAG.getDataLayout().isBigEndian()) { 408 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 409 EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout()); 410 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!"); 411 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res, 412 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy)); 413 } 414 return Res; 415 } 416 // If the output type is also a vector and widening it to the same size 417 // as the widened input type would be a legal type, we can widen the bitcast 418 // and handle the promotion after. 419 if (NOutVT.isVector()) { 420 unsigned WidenInSize = NInVT.getSizeInBits(); 421 unsigned OutSize = OutVT.getSizeInBits(); 422 if (WidenInSize % OutSize == 0) { 423 unsigned Scale = WidenInSize / OutSize; 424 EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(), 425 OutVT.getVectorElementType(), 426 OutVT.getVectorNumElements() * Scale); 427 if (isTypeLegal(WideOutVT)) { 428 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); 429 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, 430 DAG.getVectorIdxConstant(0, dl)); 431 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp); 432 } 433 } 434 } 435 } 436 437 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 438 CreateStackStoreLoad(InOp, OutVT)); 439 } 440 441 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit any shift amount 442 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't. 443 static EVT getShiftAmountTyForConstant(EVT VT, const TargetLowering &TLI, 444 SelectionDAG &DAG) { 445 EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 446 // If any possible shift value won't fit in the prefered type, just use 447 // something safe. It will be legalized when the shift is expanded. 448 if (!ShiftVT.isVector() && 449 ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits())) 450 ShiftVT = MVT::i32; 451 return ShiftVT; 452 } 453 454 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { 455 SDValue V = GetPromotedInteger(N->getOperand(0)); 456 return DAG.getNode(ISD::FREEZE, SDLoc(N), 457 V.getValueType(), V); 458 } 459 460 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 461 SDValue Op = GetPromotedInteger(N->getOperand(0)); 462 EVT OVT = N->getValueType(0); 463 EVT NVT = Op.getValueType(); 464 SDLoc dl(N); 465 466 // If the larger BSWAP isn't supported by the target, try to expand now. 467 // If we expand later we'll end up with more operations since we lost the 468 // original type. We only do this for scalars since we have a shuffle 469 // based lowering for vectors in LegalizeVectorOps. 470 if (!OVT.isVector() && 471 !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) { 472 if (SDValue Res = TLI.expandBSWAP(N, DAG)) 473 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); 474 } 475 476 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 477 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG); 478 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 479 DAG.getConstant(DiffBits, dl, ShiftVT)); 480 } 481 482 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { 483 SDValue Op = GetPromotedInteger(N->getOperand(0)); 484 EVT OVT = N->getValueType(0); 485 EVT NVT = Op.getValueType(); 486 SDLoc dl(N); 487 488 // If the larger BITREVERSE isn't supported by the target, try to expand now. 489 // If we expand later we'll end up with more operations since we lost the 490 // original type. We only do this for scalars since we have a shuffle 491 // based lowering for vectors in LegalizeVectorOps. 492 if (!OVT.isVector() && OVT.isSimple() && 493 !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) { 494 if (SDValue Res = TLI.expandBITREVERSE(N, DAG)) 495 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); 496 } 497 498 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 499 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG); 500 return DAG.getNode(ISD::SRL, dl, NVT, 501 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), 502 DAG.getConstant(DiffBits, dl, ShiftVT)); 503 } 504 505 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 506 // The pair element type may be legal, or may not promote to the same type as 507 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 508 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 509 TLI.getTypeToTransformTo(*DAG.getContext(), 510 N->getValueType(0)), JoinIntegers(N->getOperand(0), 511 N->getOperand(1))); 512 } 513 514 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 515 EVT VT = N->getValueType(0); 516 // FIXME there is no actual debug info here 517 SDLoc dl(N); 518 // Zero extend things like i1, sign extend everything else. It shouldn't 519 // matter in theory which one we pick, but this tends to give better code? 520 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 521 SDValue Result = DAG.getNode(Opc, dl, 522 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 523 SDValue(N, 0)); 524 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 525 return Result; 526 } 527 528 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 529 // Zero extend to the promoted type and do the count there. 530 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 531 SDLoc dl(N); 532 EVT OVT = N->getValueType(0); 533 EVT NVT = Op.getValueType(); 534 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 535 // Subtract off the extra leading bits in the bigger type. 536 return DAG.getNode( 537 ISD::SUB, dl, NVT, Op, 538 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, 539 NVT)); 540 } 541 542 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { 543 // Zero extend to the promoted type and do the count or parity there. 544 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 545 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op); 546 } 547 548 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 549 SDValue Op = GetPromotedInteger(N->getOperand(0)); 550 EVT OVT = N->getValueType(0); 551 EVT NVT = Op.getValueType(); 552 SDLoc dl(N); 553 if (N->getOpcode() == ISD::CTTZ) { 554 // The count is the same in the promoted type except if the original 555 // value was zero. This can be handled by setting the bit just off 556 // the top of the original type. 557 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), 558 OVT.getScalarSizeInBits()); 559 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); 560 } 561 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 562 } 563 564 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 565 SDLoc dl(N); 566 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 567 568 SDValue Op0 = N->getOperand(0); 569 SDValue Op1 = N->getOperand(1); 570 571 // If the input also needs to be promoted, do that first so we can get a 572 // get a good idea for the output type. 573 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType()) 574 == TargetLowering::TypePromoteInteger) { 575 SDValue In = GetPromotedInteger(Op0); 576 577 // If the new type is larger than NVT, use it. We probably won't need to 578 // promote it again. 579 EVT SVT = In.getValueType().getScalarType(); 580 if (SVT.bitsGE(NVT)) { 581 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1); 582 return DAG.getAnyExtOrTrunc(Ext, dl, NVT); 583 } 584 } 585 586 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1); 587 } 588 589 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 590 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 591 unsigned NewOpc = N->getOpcode(); 592 SDLoc dl(N); 593 594 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 595 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 596 // and SINT conversions are Custom, there is no way to tell which is 597 // preferable. We choose SINT because that's the right thing on PPC.) 598 if (N->getOpcode() == ISD::FP_TO_UINT && 599 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 600 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 601 NewOpc = ISD::FP_TO_SINT; 602 603 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && 604 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) && 605 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) 606 NewOpc = ISD::STRICT_FP_TO_SINT; 607 608 SDValue Res; 609 if (N->isStrictFPOpcode()) { 610 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, 611 {N->getOperand(0), N->getOperand(1)}); 612 // Legalize the chain result - switch anything that used the old chain to 613 // use the new one. 614 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 615 } else 616 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 617 618 // Assert that the converted value fits in the original type. If it doesn't 619 // (eg: because the value being converted is too big), then the result of the 620 // original operation was undefined anyway, so the assert is still correct. 621 // 622 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: 623 // before legalization: fp-to-uint16, 65534. -> 0xfffe 624 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe 625 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT || 626 N->getOpcode() == ISD::STRICT_FP_TO_UINT) ? 627 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, 628 DAG.getValueType(N->getValueType(0).getScalarType())); 629 } 630 631 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { 632 // Promote the result type, while keeping the original width in Op1. 633 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 634 SDLoc dl(N); 635 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), 636 N->getOperand(1)); 637 } 638 639 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) { 640 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 641 SDLoc dl(N); 642 643 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 644 } 645 646 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) { 647 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 648 SDLoc dl(N); 649 650 SDValue Res = 651 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); 652 653 // Legalize the chain result - switch anything that used the old chain to 654 // use the new one. 655 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 656 return Res; 657 } 658 659 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 660 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 661 SDLoc dl(N); 662 663 if (getTypeAction(N->getOperand(0).getValueType()) 664 == TargetLowering::TypePromoteInteger) { 665 SDValue Res = GetPromotedInteger(N->getOperand(0)); 666 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 667 668 // If the result and operand types are the same after promotion, simplify 669 // to an in-register extension. 670 if (NVT == Res.getValueType()) { 671 // The high bits are not guaranteed to be anything. Insert an extend. 672 if (N->getOpcode() == ISD::SIGN_EXTEND) 673 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 674 DAG.getValueType(N->getOperand(0).getValueType())); 675 if (N->getOpcode() == ISD::ZERO_EXTEND) 676 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); 677 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 678 return Res; 679 } 680 } 681 682 // Otherwise, just extend the original operand all the way to the larger type. 683 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 684 } 685 686 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 687 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 688 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 689 ISD::LoadExtType ExtType = 690 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 691 SDLoc dl(N); 692 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 693 N->getMemoryVT(), N->getMemOperand()); 694 695 // Legalize the chain result - switch anything that used the old chain to 696 // use the new one. 697 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 698 return Res; 699 } 700 701 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { 702 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 703 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 704 705 SDLoc dl(N); 706 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), 707 N->getOffset(), N->getMask(), ExtPassThru, 708 N->getMemoryVT(), N->getMemOperand(), 709 N->getAddressingMode(), ISD::EXTLOAD); 710 // Legalize the chain result - switch anything that used the old chain to 711 // use the new one. 712 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 713 return Res; 714 } 715 716 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { 717 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 718 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 719 assert(NVT == ExtPassThru.getValueType() && 720 "Gather result type and the passThru argument type should be the same"); 721 722 ISD::LoadExtType ExtType = N->getExtensionType(); 723 if (ExtType == ISD::NON_EXTLOAD) 724 ExtType = ISD::EXTLOAD; 725 726 SDLoc dl(N); 727 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), 728 N->getIndex(), N->getScale() }; 729 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), 730 N->getMemoryVT(), dl, Ops, 731 N->getMemOperand(), N->getIndexType(), 732 ExtType); 733 // Legalize the chain result - switch anything that used the old chain to 734 // use the new one. 735 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 736 return Res; 737 } 738 739 /// Promote the overflow flag of an overflowing arithmetic node. 740 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 741 // Change the return type of the boolean result while obeying 742 // getSetCCResultType. 743 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 744 EVT VT = N->getValueType(0); 745 EVT SVT = getSetCCResultType(VT); 746 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; 747 unsigned NumOps = N->getNumOperands(); 748 assert(NumOps <= 3 && "Too many operands"); 749 if (NumOps == 3) 750 Ops[2] = N->getOperand(2); 751 752 SDLoc dl(N); 753 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), 754 makeArrayRef(Ops, NumOps)); 755 756 // Modified the sum result - switch anything that used the old sum to use 757 // the new one. 758 ReplaceValueWith(SDValue(N, 0), Res); 759 760 // Convert to the expected type. 761 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); 762 } 763 764 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { 765 // If the promoted type is legal, we can convert this to: 766 // 1. ANY_EXTEND iN to iM 767 // 2. SHL by M-N 768 // 3. [US][ADD|SUB|SHL]SAT 769 // 4. L/ASHR by M-N 770 // Else it is more efficient to convert this to a min and a max 771 // operation in the higher precision arithmetic. 772 SDLoc dl(N); 773 SDValue Op1 = N->getOperand(0); 774 SDValue Op2 = N->getOperand(1); 775 unsigned OldBits = Op1.getScalarValueSizeInBits(); 776 777 unsigned Opcode = N->getOpcode(); 778 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; 779 780 SDValue Op1Promoted, Op2Promoted; 781 if (IsShift) { 782 Op1Promoted = GetPromotedInteger(Op1); 783 Op2Promoted = ZExtPromotedInteger(Op2); 784 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { 785 Op1Promoted = ZExtPromotedInteger(Op1); 786 Op2Promoted = ZExtPromotedInteger(Op2); 787 } else { 788 Op1Promoted = SExtPromotedInteger(Op1); 789 Op2Promoted = SExtPromotedInteger(Op2); 790 } 791 EVT PromotedType = Op1Promoted.getValueType(); 792 unsigned NewBits = PromotedType.getScalarSizeInBits(); 793 794 if (Opcode == ISD::UADDSAT) { 795 APInt MaxVal = APInt::getAllOnesValue(OldBits).zext(NewBits); 796 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 797 SDValue Add = 798 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); 799 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); 800 } 801 802 // USUBSAT can always be promoted as long as we have zero-extended the args. 803 if (Opcode == ISD::USUBSAT) 804 return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted, 805 Op2Promoted); 806 807 // Shift cannot use a min/max expansion, we can't detect overflow if all of 808 // the bits have been shifted out. 809 if (IsShift || TLI.isOperationLegalOrCustom(Opcode, PromotedType)) { 810 unsigned ShiftOp; 811 switch (Opcode) { 812 case ISD::SADDSAT: 813 case ISD::SSUBSAT: 814 case ISD::SSHLSAT: 815 ShiftOp = ISD::SRA; 816 break; 817 case ISD::USHLSAT: 818 ShiftOp = ISD::SRL; 819 break; 820 default: 821 llvm_unreachable("Expected opcode to be signed or unsigned saturation " 822 "addition, subtraction or left shift"); 823 } 824 825 unsigned SHLAmount = NewBits - OldBits; 826 EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); 827 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT); 828 Op1Promoted = 829 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); 830 if (!IsShift) 831 Op2Promoted = 832 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); 833 834 SDValue Result = 835 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); 836 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); 837 } 838 839 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; 840 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits); 841 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits); 842 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType); 843 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 844 SDValue Result = 845 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); 846 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); 847 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); 848 return Result; 849 } 850 851 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { 852 // Can just promote the operands then continue with operation. 853 SDLoc dl(N); 854 SDValue Op1Promoted, Op2Promoted; 855 bool Signed = 856 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; 857 bool Saturating = 858 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; 859 if (Signed) { 860 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 861 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 862 } else { 863 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 864 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 865 } 866 EVT OldType = N->getOperand(0).getValueType(); 867 EVT PromotedType = Op1Promoted.getValueType(); 868 unsigned DiffSize = 869 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); 870 871 if (Saturating) { 872 // Promoting the operand and result values changes the saturation width, 873 // which is extends the values that we clamp to on saturation. This could be 874 // resolved by shifting one of the operands the same amount, which would 875 // also shift the result we compare against, then shifting back. 876 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); 877 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 878 DAG.getConstant(DiffSize, dl, ShiftTy)); 879 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 880 Op2Promoted, N->getOperand(2)); 881 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; 882 return DAG.getNode(ShiftOp, dl, PromotedType, Result, 883 DAG.getConstant(DiffSize, dl, ShiftTy)); 884 } 885 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, 886 N->getOperand(2)); 887 } 888 889 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, 890 unsigned SatW, bool Signed, 891 const TargetLowering &TLI, 892 SelectionDAG &DAG) { 893 EVT VT = V.getValueType(); 894 unsigned VTW = VT.getScalarSizeInBits(); 895 896 if (!Signed) { 897 // Saturate to the unsigned maximum by getting the minimum of V and the 898 // maximum. 899 return DAG.getNode(ISD::UMIN, dl, VT, V, 900 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW), 901 dl, VT)); 902 } 903 904 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the 905 // signed minimum of it and V. 906 V = DAG.getNode(ISD::SMIN, dl, VT, V, 907 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1), 908 dl, VT)); 909 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the 910 // signed maximum of it and V. 911 V = DAG.getNode(ISD::SMAX, dl, VT, V, 912 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1), 913 dl, VT)); 914 return V; 915 } 916 917 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, 918 unsigned Scale, const TargetLowering &TLI, 919 SelectionDAG &DAG, unsigned SatW = 0) { 920 EVT VT = LHS.getValueType(); 921 unsigned VTSize = VT.getScalarSizeInBits(); 922 bool Signed = N->getOpcode() == ISD::SDIVFIX || 923 N->getOpcode() == ISD::SDIVFIXSAT; 924 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 925 N->getOpcode() == ISD::UDIVFIXSAT; 926 927 SDLoc dl(N); 928 // Widen the types by a factor of two. This is guaranteed to expand, since it 929 // will always have enough high bits in the LHS to shift into. 930 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2); 931 if (VT.isVector()) 932 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, 933 VT.getVectorElementCount()); 934 if (Signed) { 935 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT); 936 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT); 937 } else { 938 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT); 939 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT); 940 } 941 942 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, 943 DAG); 944 assert(Res && "Expanding DIVFIX with wide type failed?"); 945 if (Saturating) { 946 // If the caller has told us to saturate at something less, use that width 947 // instead of the type before doubling. However, it cannot be more than 948 // what we just widened! 949 assert(SatW <= VTSize && 950 "Tried to saturate to more than the original type?"); 951 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed, 952 TLI, DAG); 953 } 954 return DAG.getZExtOrTrunc(Res, dl, VT); 955 } 956 957 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { 958 SDLoc dl(N); 959 SDValue Op1Promoted, Op2Promoted; 960 bool Signed = N->getOpcode() == ISD::SDIVFIX || 961 N->getOpcode() == ISD::SDIVFIXSAT; 962 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 963 N->getOpcode() == ISD::UDIVFIXSAT; 964 if (Signed) { 965 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 966 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 967 } else { 968 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 969 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 970 } 971 EVT PromotedType = Op1Promoted.getValueType(); 972 unsigned Scale = N->getConstantOperandVal(2); 973 974 // If the type is already legal and the operation is legal in that type, we 975 // should not early expand. 976 if (TLI.isTypeLegal(PromotedType)) { 977 TargetLowering::LegalizeAction Action = 978 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale); 979 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { 980 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); 981 unsigned Diff = PromotedType.getScalarSizeInBits() - 982 N->getValueType(0).getScalarSizeInBits(); 983 if (Saturating) 984 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 985 DAG.getConstant(Diff, dl, ShiftTy)); 986 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 987 Op2Promoted, N->getOperand(2)); 988 if (Saturating) 989 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res, 990 DAG.getConstant(Diff, dl, ShiftTy)); 991 return Res; 992 } 993 } 994 995 // See if we can perform the division in this type without expanding. 996 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted, 997 Op2Promoted, Scale, DAG)) { 998 if (Saturating) 999 Res = SaturateWidenedDIVFIX(Res, dl, 1000 N->getValueType(0).getScalarSizeInBits(), 1001 Signed, TLI, DAG); 1002 return Res; 1003 } 1004 // If we cannot, expand it to twice the type width. If we are saturating, give 1005 // it the original width as a saturating width so we don't need to emit 1006 // two saturations. 1007 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG, 1008 N->getValueType(0).getScalarSizeInBits()); 1009 } 1010 1011 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 1012 if (ResNo == 1) 1013 return PromoteIntRes_Overflow(N); 1014 1015 // The operation overflowed iff the result in the larger type is not the 1016 // sign extension of its truncation to the original type. 1017 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1018 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1019 EVT OVT = N->getOperand(0).getValueType(); 1020 EVT NVT = LHS.getValueType(); 1021 SDLoc dl(N); 1022 1023 // Do the arithmetic in the larger type. 1024 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 1025 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1026 1027 // Calculate the overflow flag: sign extend the arithmetic result from 1028 // the original type. 1029 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 1030 DAG.getValueType(OVT)); 1031 // Overflowed if and only if this is not equal to Res. 1032 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1033 1034 // Use the calculated overflow everywhere. 1035 ReplaceValueWith(SDValue(N, 1), Ofl); 1036 1037 return Res; 1038 } 1039 1040 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 1041 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 1042 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 1043 return DAG.getSelect(SDLoc(N), 1044 LHS.getValueType(), N->getOperand(0), LHS, RHS); 1045 } 1046 1047 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { 1048 SDValue Mask = N->getOperand(0); 1049 1050 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 1051 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 1052 return DAG.getNode(ISD::VSELECT, SDLoc(N), 1053 LHS.getValueType(), Mask, LHS, RHS); 1054 } 1055 1056 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 1057 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 1058 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 1059 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 1060 LHS.getValueType(), N->getOperand(0), 1061 N->getOperand(1), LHS, RHS, N->getOperand(4)); 1062 } 1063 1064 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 1065 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 1066 EVT InVT = N->getOperand(OpNo).getValueType(); 1067 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1068 1069 EVT SVT = getSetCCResultType(InVT); 1070 1071 // If we got back a type that needs to be promoted, this likely means the 1072 // the input type also needs to be promoted. So get the promoted type for 1073 // the input and try the query again. 1074 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { 1075 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 1076 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 1077 SVT = getSetCCResultType(InVT); 1078 } else { 1079 // Input type isn't promoted, just use the default promoted type. 1080 SVT = NVT; 1081 } 1082 } 1083 1084 SDLoc dl(N); 1085 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && 1086 "Vector compare must return a vector result!"); 1087 1088 // Get the SETCC result using the canonical SETCC type. 1089 SDValue SetCC; 1090 if (N->isStrictFPOpcode()) { 1091 EVT VTs[] = {SVT, MVT::Other}; 1092 SDValue Opers[] = {N->getOperand(0), N->getOperand(1), 1093 N->getOperand(2), N->getOperand(3)}; 1094 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers); 1095 // Legalize the chain result - switch anything that used the old chain to 1096 // use the new one. 1097 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1)); 1098 } else 1099 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), 1100 N->getOperand(1), N->getOperand(2)); 1101 1102 // Convert to the expected type. 1103 return DAG.getSExtOrTrunc(SetCC, dl, NVT); 1104 } 1105 1106 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 1107 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1108 SDValue RHS = N->getOperand(1); 1109 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1110 RHS = ZExtPromotedInteger(RHS); 1111 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS); 1112 } 1113 1114 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 1115 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1116 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 1117 Op.getValueType(), Op, N->getOperand(1)); 1118 } 1119 1120 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 1121 // The input may have strange things in the top bits of the registers, but 1122 // these operations don't care. They may have weird bits going out, but 1123 // that too is okay if they are integer operations. 1124 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1125 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 1126 return DAG.getNode(N->getOpcode(), SDLoc(N), 1127 LHS.getValueType(), LHS, RHS); 1128 } 1129 1130 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { 1131 // Sign extend the input. 1132 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1133 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1134 return DAG.getNode(N->getOpcode(), SDLoc(N), 1135 LHS.getValueType(), LHS, RHS); 1136 } 1137 1138 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { 1139 // Zero extend the input. 1140 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1141 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1142 return DAG.getNode(N->getOpcode(), SDLoc(N), 1143 LHS.getValueType(), LHS, RHS); 1144 } 1145 1146 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { 1147 // It doesn't matter if we sign extend or zero extend in the inputs. So do 1148 // whatever is best for the target. 1149 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0)); 1150 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1)); 1151 return DAG.getNode(N->getOpcode(), SDLoc(N), 1152 LHS.getValueType(), LHS, RHS); 1153 } 1154 1155 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 1156 // The input value must be properly sign extended. 1157 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1158 SDValue RHS = N->getOperand(1); 1159 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1160 RHS = ZExtPromotedInteger(RHS); 1161 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS); 1162 } 1163 1164 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 1165 // The input value must be properly zero extended. 1166 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1167 SDValue RHS = N->getOperand(1); 1168 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1169 RHS = ZExtPromotedInteger(RHS); 1170 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS); 1171 } 1172 1173 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { 1174 // Lower the rotate to shifts and ORs which can be promoted. 1175 SDValue Res; 1176 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG); 1177 ReplaceValueWith(SDValue(N, 0), Res); 1178 return SDValue(); 1179 } 1180 1181 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { 1182 SDValue Hi = GetPromotedInteger(N->getOperand(0)); 1183 SDValue Lo = GetPromotedInteger(N->getOperand(1)); 1184 SDValue Amount = GetPromotedInteger(N->getOperand(2)); 1185 1186 SDLoc DL(N); 1187 EVT OldVT = N->getOperand(0).getValueType(); 1188 EVT VT = Lo.getValueType(); 1189 unsigned Opcode = N->getOpcode(); 1190 bool IsFSHR = Opcode == ISD::FSHR; 1191 unsigned OldBits = OldVT.getScalarSizeInBits(); 1192 unsigned NewBits = VT.getScalarSizeInBits(); 1193 1194 // Amount has to be interpreted modulo the old bit width. 1195 Amount = 1196 DAG.getNode(ISD::UREM, DL, VT, Amount, DAG.getConstant(OldBits, DL, VT)); 1197 1198 // If the promoted type is twice the size (or more), then we use the 1199 // traditional funnel 'double' shift codegen. This isn't necessary if the 1200 // shift amount is constant. 1201 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. 1202 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). 1203 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amount) && 1204 !TLI.isOperationLegalOrCustom(Opcode, VT)) { 1205 SDValue HiShift = DAG.getConstant(OldBits, DL, VT); 1206 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift); 1207 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT); 1208 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo); 1209 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amount); 1210 if (!IsFSHR) 1211 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift); 1212 return Res; 1213 } 1214 1215 // Shift Lo up to occupy the upper bits of the promoted type. 1216 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, VT); 1217 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset); 1218 1219 // Increase Amount to shift the result into the lower bits of the promoted 1220 // type. 1221 if (IsFSHR) 1222 Amount = DAG.getNode(ISD::ADD, DL, VT, Amount, ShiftOffset); 1223 1224 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amount); 1225 } 1226 1227 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 1228 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1229 SDValue Res; 1230 SDValue InOp = N->getOperand(0); 1231 SDLoc dl(N); 1232 1233 switch (getTypeAction(InOp.getValueType())) { 1234 default: llvm_unreachable("Unknown type action!"); 1235 case TargetLowering::TypeLegal: 1236 case TargetLowering::TypeExpandInteger: 1237 Res = InOp; 1238 break; 1239 case TargetLowering::TypePromoteInteger: 1240 Res = GetPromotedInteger(InOp); 1241 break; 1242 case TargetLowering::TypeSplitVector: { 1243 EVT InVT = InOp.getValueType(); 1244 assert(InVT.isVector() && "Cannot split scalar types"); 1245 ElementCount NumElts = InVT.getVectorElementCount(); 1246 assert(NumElts == NVT.getVectorElementCount() && 1247 "Dst and Src must have the same number of elements"); 1248 assert(isPowerOf2_32(NumElts.getKnownMinValue()) && 1249 "Promoted vector type must be a power of two"); 1250 1251 SDValue EOp1, EOp2; 1252 GetSplitVector(InOp, EOp1, EOp2); 1253 1254 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 1255 NumElts.divideCoefficientBy(2)); 1256 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 1257 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 1258 1259 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 1260 } 1261 case TargetLowering::TypeWidenVector: { 1262 SDValue WideInOp = GetWidenedVector(InOp); 1263 1264 // Truncate widened InOp. 1265 unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); 1266 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), 1267 N->getValueType(0).getScalarType(), NumElem); 1268 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); 1269 1270 // Zero extend so that the elements are of same type as those of NVT 1271 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), 1272 NumElem); 1273 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); 1274 1275 // Extract the low NVT subvector. 1276 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl); 1277 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); 1278 } 1279 } 1280 1281 // Truncate to NVT instead of VT 1282 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 1283 } 1284 1285 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 1286 if (ResNo == 1) 1287 return PromoteIntRes_Overflow(N); 1288 1289 // The operation overflowed iff the result in the larger type is not the 1290 // zero extension of its truncation to the original type. 1291 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1292 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1293 EVT OVT = N->getOperand(0).getValueType(); 1294 EVT NVT = LHS.getValueType(); 1295 SDLoc dl(N); 1296 1297 // Do the arithmetic in the larger type. 1298 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 1299 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1300 1301 // Calculate the overflow flag: zero extend the arithmetic result from 1302 // the original type. 1303 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 1304 // Overflowed if and only if this is not equal to Res. 1305 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1306 1307 // Use the calculated overflow everywhere. 1308 ReplaceValueWith(SDValue(N, 1), Ofl); 1309 1310 return Res; 1311 } 1312 1313 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that 1314 // the third operand of ADDE/SUBE nodes is carry flag, which differs from 1315 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean. 1316 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) { 1317 if (ResNo == 1) 1318 return PromoteIntRes_Overflow(N); 1319 1320 // We need to sign-extend the operands so the carry value computed by the 1321 // wide operation will be equivalent to the carry value computed by the 1322 // narrow operation. 1323 // An ADDCARRY can generate carry only if any of the operands has its 1324 // most significant bit set. Sign extension propagates the most significant 1325 // bit into the higher bits which means the extra bit that the narrow 1326 // addition would need (i.e. the carry) will be propagated through the higher 1327 // bits of the wide addition. 1328 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be 1329 // preserved by sign extension. 1330 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1331 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1332 1333 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; 1334 1335 // Do the arithmetic in the wide type. 1336 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), 1337 LHS, RHS, N->getOperand(2)); 1338 1339 // Update the users of the original carry/borrow value. 1340 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 1341 1342 return SDValue(Res.getNode(), 0); 1343 } 1344 1345 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, 1346 unsigned ResNo) { 1347 assert(ResNo == 1 && "Don't know how to promote other results yet."); 1348 return PromoteIntRes_Overflow(N); 1349 } 1350 1351 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { 1352 SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); 1353 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); 1354 } 1355 1356 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 1357 // Promote the overflow bit trivially. 1358 if (ResNo == 1) 1359 return PromoteIntRes_Overflow(N); 1360 1361 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 1362 SDLoc DL(N); 1363 EVT SmallVT = LHS.getValueType(); 1364 1365 // To determine if the result overflowed in a larger type, we extend the 1366 // input to the larger type, do the multiply (checking if it overflows), 1367 // then also check the high bits of the result to see if overflow happened 1368 // there. 1369 if (N->getOpcode() == ISD::SMULO) { 1370 LHS = SExtPromotedInteger(LHS); 1371 RHS = SExtPromotedInteger(RHS); 1372 } else { 1373 LHS = ZExtPromotedInteger(LHS); 1374 RHS = ZExtPromotedInteger(RHS); 1375 } 1376 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 1377 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 1378 1379 // Overflow occurred if it occurred in the larger type, or if the high part 1380 // of the result does not zero/sign-extend the low part. Check this second 1381 // possibility first. 1382 SDValue Overflow; 1383 if (N->getOpcode() == ISD::UMULO) { 1384 // Unsigned overflow occurred if the high part is non-zero. 1385 unsigned Shift = SmallVT.getScalarSizeInBits(); 1386 EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG); 1387 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 1388 DAG.getConstant(Shift, DL, ShiftTy)); 1389 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 1390 DAG.getConstant(0, DL, Hi.getValueType()), 1391 ISD::SETNE); 1392 } else { 1393 // Signed overflow occurred if the high part does not sign extend the low. 1394 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 1395 Mul, DAG.getValueType(SmallVT)); 1396 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 1397 } 1398 1399 // The only other way for overflow to occur is if the multiplication in the 1400 // larger type itself overflowed. 1401 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 1402 SDValue(Mul.getNode(), 1)); 1403 1404 // Use the calculated overflow everywhere. 1405 ReplaceValueWith(SDValue(N, 1), Overflow); 1406 return Mul; 1407 } 1408 1409 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 1410 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 1411 N->getValueType(0))); 1412 } 1413 1414 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { 1415 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1416 1417 APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue(); 1418 return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits())); 1419 } 1420 1421 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 1422 SDValue Chain = N->getOperand(0); // Get the chain. 1423 SDValue Ptr = N->getOperand(1); // Get the pointer. 1424 EVT VT = N->getValueType(0); 1425 SDLoc dl(N); 1426 1427 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 1428 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 1429 // The argument is passed as NumRegs registers of type RegVT. 1430 1431 SmallVector<SDValue, 8> Parts(NumRegs); 1432 for (unsigned i = 0; i < NumRegs; ++i) { 1433 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 1434 N->getConstantOperandVal(3)); 1435 Chain = Parts[i].getValue(1); 1436 } 1437 1438 // Handle endianness of the load. 1439 if (DAG.getDataLayout().isBigEndian()) 1440 std::reverse(Parts.begin(), Parts.end()); 1441 1442 // Assemble the parts in the promoted type. 1443 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1444 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 1445 for (unsigned i = 1; i < NumRegs; ++i) { 1446 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 1447 // Shift it to the right position and "or" it in. 1448 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 1449 DAG.getConstant(i * RegVT.getSizeInBits(), dl, 1450 TLI.getPointerTy(DAG.getDataLayout()))); 1451 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 1452 } 1453 1454 // Modified the chain result - switch anything that used the old chain to 1455 // use the new one. 1456 ReplaceValueWith(SDValue(N, 1), Chain); 1457 1458 return Res; 1459 } 1460 1461 //===----------------------------------------------------------------------===// 1462 // Integer Operand Promotion 1463 //===----------------------------------------------------------------------===// 1464 1465 /// PromoteIntegerOperand - This method is called when the specified operand of 1466 /// the specified node is found to need promotion. At this point, all of the 1467 /// result types of the node are known to be legal, but other operands of the 1468 /// node may need promotion or expansion as well as the specified one. 1469 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 1470 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); 1471 dbgs() << "\n"); 1472 SDValue Res = SDValue(); 1473 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 1474 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 1475 return false; 1476 } 1477 1478 switch (N->getOpcode()) { 1479 default: 1480 #ifndef NDEBUG 1481 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 1482 N->dump(&DAG); dbgs() << "\n"; 1483 #endif 1484 llvm_unreachable("Do not know how to promote this operator's operand!"); 1485 1486 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 1487 case ISD::ATOMIC_STORE: 1488 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 1489 break; 1490 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 1491 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 1492 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 1493 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 1494 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 1495 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 1496 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 1497 case ISD::INSERT_VECTOR_ELT: 1498 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 1499 case ISD::SCALAR_TO_VECTOR: 1500 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 1501 case ISD::SPLAT_VECTOR: 1502 Res = PromoteIntOp_SPLAT_VECTOR(N); break; 1503 case ISD::VSELECT: 1504 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 1505 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 1506 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 1507 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 1508 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 1509 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; 1510 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 1511 OpNo); break; 1512 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), 1513 OpNo); break; 1514 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), 1515 OpNo); break; 1516 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), 1517 OpNo); break; 1518 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), 1519 OpNo); break; 1520 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 1521 case ISD::FP16_TO_FP: 1522 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 1523 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; 1524 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 1525 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; 1526 1527 case ISD::SHL: 1528 case ISD::SRA: 1529 case ISD::SRL: 1530 case ISD::ROTL: 1531 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 1532 1533 case ISD::SADDO_CARRY: 1534 case ISD::SSUBO_CARRY: 1535 case ISD::ADDCARRY: 1536 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break; 1537 1538 case ISD::FRAMEADDR: 1539 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; 1540 1541 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break; 1542 1543 case ISD::SMULFIX: 1544 case ISD::SMULFIXSAT: 1545 case ISD::UMULFIX: 1546 case ISD::UMULFIXSAT: 1547 case ISD::SDIVFIX: 1548 case ISD::SDIVFIXSAT: 1549 case ISD::UDIVFIX: 1550 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; 1551 1552 case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break; 1553 1554 case ISD::VECREDUCE_ADD: 1555 case ISD::VECREDUCE_MUL: 1556 case ISD::VECREDUCE_AND: 1557 case ISD::VECREDUCE_OR: 1558 case ISD::VECREDUCE_XOR: 1559 case ISD::VECREDUCE_SMAX: 1560 case ISD::VECREDUCE_SMIN: 1561 case ISD::VECREDUCE_UMAX: 1562 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; 1563 1564 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; 1565 } 1566 1567 // If the result is null, the sub-method took care of registering results etc. 1568 if (!Res.getNode()) return false; 1569 1570 // If the result is N, the sub-method updated N in place. Tell the legalizer 1571 // core about this. 1572 if (Res.getNode() == N) 1573 return true; 1574 1575 const bool IsStrictFp = N->isStrictFPOpcode(); 1576 assert(Res.getValueType() == N->getValueType(0) && 1577 N->getNumValues() == (IsStrictFp ? 2 : 1) && 1578 "Invalid operand expansion"); 1579 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: "; 1580 Res.dump()); 1581 1582 ReplaceValueWith(SDValue(N, 0), Res); 1583 if (IsStrictFp) 1584 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 1585 1586 return false; 1587 } 1588 1589 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is 1590 /// shared among BR_CC, SELECT_CC, and SETCC handlers. 1591 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 1592 ISD::CondCode CCCode) { 1593 // We have to insert explicit sign or zero extends. Note that we could 1594 // insert sign extends for ALL conditions. For those operations where either 1595 // zero or sign extension would be valid, use SExtOrZExtPromotedInteger 1596 // which will choose the cheapest for the target. 1597 switch (CCCode) { 1598 default: llvm_unreachable("Unknown integer comparison!"); 1599 case ISD::SETEQ: 1600 case ISD::SETNE: { 1601 SDValue OpL = GetPromotedInteger(NewLHS); 1602 SDValue OpR = GetPromotedInteger(NewRHS); 1603 1604 // We would prefer to promote the comparison operand with sign extension. 1605 // If the width of OpL/OpR excluding the duplicated sign bits is no greater 1606 // than the width of NewLHS/NewRH, we can avoid inserting real truncate 1607 // instruction, which is redundant eventually. 1608 unsigned OpLEffectiveBits = 1609 OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1; 1610 unsigned OpREffectiveBits = 1611 OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1; 1612 if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() && 1613 OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) { 1614 NewLHS = OpL; 1615 NewRHS = OpR; 1616 } else { 1617 NewLHS = SExtOrZExtPromotedInteger(NewLHS); 1618 NewRHS = SExtOrZExtPromotedInteger(NewRHS); 1619 } 1620 break; 1621 } 1622 case ISD::SETUGE: 1623 case ISD::SETUGT: 1624 case ISD::SETULE: 1625 case ISD::SETULT: 1626 NewLHS = SExtOrZExtPromotedInteger(NewLHS); 1627 NewRHS = SExtOrZExtPromotedInteger(NewRHS); 1628 break; 1629 case ISD::SETGE: 1630 case ISD::SETGT: 1631 case ISD::SETLT: 1632 case ISD::SETLE: 1633 NewLHS = SExtPromotedInteger(NewLHS); 1634 NewRHS = SExtPromotedInteger(NewRHS); 1635 break; 1636 } 1637 } 1638 1639 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 1640 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1641 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 1642 } 1643 1644 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 1645 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 1646 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 1647 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); 1648 } 1649 1650 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 1651 // This should only occur in unusual situations like bitcasting to an 1652 // x86_fp80, so just turn it into a store+load 1653 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 1654 } 1655 1656 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 1657 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1658 1659 SDValue LHS = N->getOperand(2); 1660 SDValue RHS = N->getOperand(3); 1661 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 1662 1663 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 1664 // legal types. 1665 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1666 N->getOperand(1), LHS, RHS, N->getOperand(4)), 1667 0); 1668 } 1669 1670 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 1671 assert(OpNo == 1 && "only know how to promote condition"); 1672 1673 // Promote all the way up to the canonical SetCC type. 1674 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 1675 1676 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 1677 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 1678 N->getOperand(2)), 0); 1679 } 1680 1681 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 1682 // Since the result type is legal, the operands must promote to it. 1683 EVT OVT = N->getOperand(0).getValueType(); 1684 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 1685 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 1686 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 1687 SDLoc dl(N); 1688 1689 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 1690 DAG.getConstant(OVT.getSizeInBits(), dl, 1691 TLI.getPointerTy(DAG.getDataLayout()))); 1692 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 1693 } 1694 1695 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 1696 // The vector type is legal but the element type is not. This implies 1697 // that the vector is a power-of-two in length and that the element 1698 // type does not have a strange size (eg: it is not i1). 1699 EVT VecVT = N->getValueType(0); 1700 unsigned NumElts = VecVT.getVectorNumElements(); 1701 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 1702 "Legal vector of one illegal element?"); 1703 1704 // Promote the inserted value. The type does not need to match the 1705 // vector element type. Check that any extra bits introduced will be 1706 // truncated away. 1707 assert(N->getOperand(0).getValueSizeInBits() >= 1708 N->getValueType(0).getScalarSizeInBits() && 1709 "Type of inserted value narrower than vector element type!"); 1710 1711 SmallVector<SDValue, 16> NewOps; 1712 for (unsigned i = 0; i < NumElts; ++i) 1713 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 1714 1715 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1716 } 1717 1718 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 1719 unsigned OpNo) { 1720 if (OpNo == 1) { 1721 // Promote the inserted value. This is valid because the type does not 1722 // have to match the vector element type. 1723 1724 // Check that any extra bits introduced will be truncated away. 1725 assert(N->getOperand(1).getValueSizeInBits() >= 1726 N->getValueType(0).getScalarSizeInBits() && 1727 "Type of inserted value narrower than vector element type!"); 1728 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1729 GetPromotedInteger(N->getOperand(1)), 1730 N->getOperand(2)), 1731 0); 1732 } 1733 1734 assert(OpNo == 2 && "Different operand and result vector types?"); 1735 1736 // Promote the index. 1737 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 1738 TLI.getVectorIdxTy(DAG.getDataLayout())); 1739 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1740 N->getOperand(1), Idx), 0); 1741 } 1742 1743 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 1744 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 1745 // the operand in place. 1746 return SDValue(DAG.UpdateNodeOperands(N, 1747 GetPromotedInteger(N->getOperand(0))), 0); 1748 } 1749 1750 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) { 1751 // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the 1752 // operand in place. 1753 return SDValue( 1754 DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0); 1755 } 1756 1757 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 1758 assert(OpNo == 0 && "Only know how to promote the condition!"); 1759 SDValue Cond = N->getOperand(0); 1760 EVT OpTy = N->getOperand(1).getValueType(); 1761 1762 if (N->getOpcode() == ISD::VSELECT) 1763 if (SDValue Res = WidenVSELECTMask(N)) 1764 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 1765 Res, N->getOperand(1), N->getOperand(2)); 1766 1767 // Promote all the way up to the canonical SetCC type. 1768 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 1769 Cond = PromoteTargetBoolean(Cond, OpVT); 1770 1771 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 1772 N->getOperand(2)), 0); 1773 } 1774 1775 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 1776 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1777 1778 SDValue LHS = N->getOperand(0); 1779 SDValue RHS = N->getOperand(1); 1780 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1781 1782 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1783 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1784 N->getOperand(3), N->getOperand(4)), 0); 1785 } 1786 1787 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1788 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1789 1790 SDValue LHS = N->getOperand(0); 1791 SDValue RHS = N->getOperand(1); 1792 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 1793 1794 // The CC (#2) is always legal. 1795 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 1796 } 1797 1798 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 1799 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1800 ZExtPromotedInteger(N->getOperand(1))), 0); 1801 } 1802 1803 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 1804 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1805 SDLoc dl(N); 1806 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1807 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 1808 Op, DAG.getValueType(N->getOperand(0).getValueType())); 1809 } 1810 1811 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 1812 return SDValue(DAG.UpdateNodeOperands(N, 1813 SExtPromotedInteger(N->getOperand(0))), 0); 1814 } 1815 1816 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { 1817 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1818 SExtPromotedInteger(N->getOperand(1))), 0); 1819 } 1820 1821 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 1822 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1823 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 1824 SDLoc dl(N); 1825 1826 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 1827 1828 // Truncate the value and store the result. 1829 return DAG.getTruncStore(Ch, dl, Val, Ptr, 1830 N->getMemoryVT(), N->getMemOperand()); 1831 } 1832 1833 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, 1834 unsigned OpNo) { 1835 1836 SDValue DataOp = N->getValue(); 1837 EVT DataVT = DataOp.getValueType(); 1838 SDValue Mask = N->getMask(); 1839 SDLoc dl(N); 1840 1841 bool TruncateStore = false; 1842 if (OpNo == 4) { 1843 Mask = PromoteTargetBoolean(Mask, DataVT); 1844 // Update in place. 1845 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 1846 NewOps[4] = Mask; 1847 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1848 } else { // Data operand 1849 assert(OpNo == 1 && "Unexpected operand for promotion"); 1850 DataOp = GetPromotedInteger(DataOp); 1851 TruncateStore = true; 1852 } 1853 1854 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), 1855 N->getOffset(), Mask, N->getMemoryVT(), 1856 N->getMemOperand(), N->getAddressingMode(), 1857 TruncateStore, N->isCompressingStore()); 1858 } 1859 1860 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, 1861 unsigned OpNo) { 1862 assert(OpNo == 3 && "Only know how to promote the mask!"); 1863 EVT DataVT = N->getValueType(0); 1864 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1865 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 1866 NewOps[OpNo] = Mask; 1867 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 1868 if (Res == N) 1869 return SDValue(Res, 0); 1870 1871 // Update triggered CSE, do our own replacement since caller can't. 1872 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 1873 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 1874 return SDValue(); 1875 } 1876 1877 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, 1878 unsigned OpNo) { 1879 1880 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1881 if (OpNo == 2) { 1882 // The Mask 1883 EVT DataVT = N->getValueType(0); 1884 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1885 } else if (OpNo == 4) { 1886 // The Index 1887 if (N->isIndexSigned()) 1888 // Need to sign extend the index since the bits will likely be used. 1889 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 1890 else 1891 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 1892 } else 1893 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1894 1895 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 1896 if (Res == N) 1897 return SDValue(Res, 0); 1898 1899 // Update triggered CSE, do our own replacement since caller can't. 1900 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 1901 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 1902 return SDValue(); 1903 } 1904 1905 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, 1906 unsigned OpNo) { 1907 bool TruncateStore = N->isTruncatingStore(); 1908 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1909 if (OpNo == 2) { 1910 // The Mask 1911 EVT DataVT = N->getValue().getValueType(); 1912 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1913 } else if (OpNo == 4) { 1914 // The Index 1915 if (N->isIndexSigned()) 1916 // Need to sign extend the index since the bits will likely be used. 1917 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 1918 else 1919 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 1920 1921 N->setIndexType(TLI.getCanonicalIndexType(N->getIndexType(), 1922 N->getMemoryVT(), NewOps[OpNo])); 1923 } else { 1924 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1925 TruncateStore = true; 1926 } 1927 1928 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), 1929 SDLoc(N), NewOps, N->getMemOperand(), 1930 N->getIndexType(), TruncateStore); 1931 } 1932 1933 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 1934 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1935 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 1936 } 1937 1938 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 1939 return SDValue(DAG.UpdateNodeOperands(N, 1940 ZExtPromotedInteger(N->getOperand(0))), 0); 1941 } 1942 1943 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { 1944 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1945 ZExtPromotedInteger(N->getOperand(1))), 0); 1946 } 1947 1948 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 1949 SDLoc dl(N); 1950 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1951 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1952 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); 1953 } 1954 1955 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { 1956 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1957 1958 SDValue LHS = N->getOperand(0); 1959 SDValue RHS = N->getOperand(1); 1960 SDValue Carry = N->getOperand(2); 1961 SDLoc DL(N); 1962 1963 Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); 1964 1965 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); 1966 } 1967 1968 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { 1969 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 1970 return SDValue( 1971 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); 1972 } 1973 1974 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { 1975 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. 1976 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 1977 return SDValue(DAG.UpdateNodeOperands(N, Op), 0); 1978 } 1979 1980 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) { 1981 assert(OpNo > 1 && "Don't know how to promote this operand!"); 1982 // Promote the rw, locality, and cache type arguments to a supported integer 1983 // width. 1984 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 1985 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3)); 1986 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4)); 1987 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), 1988 Op2, Op3, Op4), 1989 0); 1990 } 1991 1992 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) { 1993 // FIXME: Support for promotion of STRICT_FPOWI is not implemented yet. 1994 assert(N->getOpcode() == ISD::FPOWI && "No STRICT_FPOWI support here yet."); 1995 1996 // The integer operand is the last operand in FPOWI (so the result and 1997 // floating point operand is already type legalized). 1998 1999 // We can't just promote the exponent type in FPOWI, since we want to lower 2000 // the node to a libcall and we if we promote to a type larger than 2001 // sizeof(int) the libcall might not be according to the targets ABI. Instead 2002 // we rewrite to a libcall here directly, letting makeLibCall handle promotion 2003 // if the target accepts it according to shouldSignExtendTypeInLibCall. 2004 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0)); 2005 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); 2006 if (!TLI.getLibcallName(LC)) { 2007 // Some targets don't have a powi libcall; use pow instead. 2008 // FIXME: Implement this if some target needs it. 2009 DAG.getContext()->emitError("Don't know how to promote fpowi to fpow"); 2010 return DAG.getUNDEF(N->getValueType(0)); 2011 } 2012 // The exponent should fit in a sizeof(int) type for the libcall to be valid. 2013 assert(DAG.getLibInfo().getIntSize() == 2014 N->getOperand(1).getValueType().getSizeInBits() && 2015 "POWI exponent should match with sizeof(int) when doing the libcall."); 2016 TargetLowering::MakeLibCallOptions CallOptions; 2017 CallOptions.setSExt(true); 2018 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2019 std::pair<SDValue, SDValue> Tmp = 2020 TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, 2021 CallOptions, SDLoc(N), SDValue()); 2022 ReplaceValueWith(SDValue(N, 0), Tmp.first); 2023 return SDValue(); 2024 } 2025 2026 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { 2027 SDLoc dl(N); 2028 SDValue Op; 2029 switch (N->getOpcode()) { 2030 default: llvm_unreachable("Expected integer vector reduction"); 2031 case ISD::VECREDUCE_ADD: 2032 case ISD::VECREDUCE_MUL: 2033 case ISD::VECREDUCE_AND: 2034 case ISD::VECREDUCE_OR: 2035 case ISD::VECREDUCE_XOR: 2036 Op = GetPromotedInteger(N->getOperand(0)); 2037 break; 2038 case ISD::VECREDUCE_SMAX: 2039 case ISD::VECREDUCE_SMIN: 2040 Op = SExtPromotedInteger(N->getOperand(0)); 2041 break; 2042 case ISD::VECREDUCE_UMAX: 2043 case ISD::VECREDUCE_UMIN: 2044 Op = ZExtPromotedInteger(N->getOperand(0)); 2045 break; 2046 } 2047 2048 EVT EltVT = Op.getValueType().getVectorElementType(); 2049 EVT VT = N->getValueType(0); 2050 if (VT.bitsGE(EltVT)) 2051 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op); 2052 2053 // Result size must be >= element size. If this is not the case after 2054 // promotion, also promote the result type and then truncate. 2055 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op); 2056 return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce); 2057 } 2058 2059 SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) { 2060 SDValue Op = ZExtPromotedInteger(N->getOperand(1)); 2061 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); 2062 } 2063 2064 //===----------------------------------------------------------------------===// 2065 // Integer Result Expansion 2066 //===----------------------------------------------------------------------===// 2067 2068 /// ExpandIntegerResult - This method is called when the specified result of the 2069 /// specified node is found to need expansion. At this point, the node may also 2070 /// have invalid operands or may have other results that need promotion, we just 2071 /// know that (at least) one result needs expansion. 2072 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 2073 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); 2074 dbgs() << "\n"); 2075 SDValue Lo, Hi; 2076 Lo = Hi = SDValue(); 2077 2078 // See if the target wants to custom expand this node. 2079 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 2080 return; 2081 2082 switch (N->getOpcode()) { 2083 default: 2084 #ifndef NDEBUG 2085 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 2086 N->dump(&DAG); dbgs() << "\n"; 2087 #endif 2088 report_fatal_error("Do not know how to expand the result of this " 2089 "operator!"); 2090 2091 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 2092 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 2093 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 2094 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 2095 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; 2096 2097 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 2098 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 2099 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 2100 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 2101 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 2102 2103 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 2104 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 2105 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 2106 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; 2107 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 2108 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; 2109 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 2110 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; 2111 case ISD::CTLZ_ZERO_UNDEF: 2112 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 2113 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 2114 case ISD::CTTZ_ZERO_UNDEF: 2115 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 2116 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break; 2117 case ISD::STRICT_FP_TO_SINT: 2118 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 2119 case ISD::STRICT_FP_TO_UINT: 2120 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 2121 case ISD::FP_TO_SINT_SAT: 2122 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; 2123 case ISD::STRICT_LLROUND: 2124 case ISD::STRICT_LLRINT: 2125 case ISD::LLROUND: 2126 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break; 2127 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 2128 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 2129 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; 2130 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 2131 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 2132 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 2133 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 2134 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 2135 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 2136 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 2137 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 2138 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 2139 2140 case ISD::ATOMIC_LOAD_ADD: 2141 case ISD::ATOMIC_LOAD_SUB: 2142 case ISD::ATOMIC_LOAD_AND: 2143 case ISD::ATOMIC_LOAD_CLR: 2144 case ISD::ATOMIC_LOAD_OR: 2145 case ISD::ATOMIC_LOAD_XOR: 2146 case ISD::ATOMIC_LOAD_NAND: 2147 case ISD::ATOMIC_LOAD_MIN: 2148 case ISD::ATOMIC_LOAD_MAX: 2149 case ISD::ATOMIC_LOAD_UMIN: 2150 case ISD::ATOMIC_LOAD_UMAX: 2151 case ISD::ATOMIC_SWAP: 2152 case ISD::ATOMIC_CMP_SWAP: { 2153 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 2154 SplitInteger(Tmp.first, Lo, Hi); 2155 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2156 break; 2157 } 2158 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 2159 AtomicSDNode *AN = cast<AtomicSDNode>(N); 2160 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 2161 SDValue Tmp = DAG.getAtomicCmpSwap( 2162 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 2163 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 2164 AN->getMemOperand()); 2165 2166 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 2167 // success simply by comparing the loaded value against the ingoing 2168 // comparison. 2169 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 2170 N->getOperand(2), ISD::SETEQ); 2171 2172 SplitInteger(Tmp, Lo, Hi); 2173 ReplaceValueWith(SDValue(N, 1), Success); 2174 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 2175 break; 2176 } 2177 2178 case ISD::AND: 2179 case ISD::OR: 2180 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 2181 2182 case ISD::UMAX: 2183 case ISD::SMAX: 2184 case ISD::UMIN: 2185 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; 2186 2187 case ISD::ADD: 2188 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 2189 2190 case ISD::ADDC: 2191 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 2192 2193 case ISD::ADDE: 2194 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 2195 2196 case ISD::ADDCARRY: 2197 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break; 2198 2199 case ISD::SADDO_CARRY: 2200 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; 2201 2202 case ISD::SHL: 2203 case ISD::SRA: 2204 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 2205 2206 case ISD::SADDO: 2207 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 2208 case ISD::UADDO: 2209 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 2210 case ISD::UMULO: 2211 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 2212 2213 case ISD::SADDSAT: 2214 case ISD::UADDSAT: 2215 case ISD::SSUBSAT: 2216 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; 2217 2218 case ISD::SSHLSAT: 2219 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; 2220 2221 case ISD::SMULFIX: 2222 case ISD::SMULFIXSAT: 2223 case ISD::UMULFIX: 2224 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; 2225 2226 case ISD::SDIVFIX: 2227 case ISD::SDIVFIXSAT: 2228 case ISD::UDIVFIX: 2229 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; 2230 2231 case ISD::VECREDUCE_ADD: 2232 case ISD::VECREDUCE_MUL: 2233 case ISD::VECREDUCE_AND: 2234 case ISD::VECREDUCE_OR: 2235 case ISD::VECREDUCE_XOR: 2236 case ISD::VECREDUCE_SMAX: 2237 case ISD::VECREDUCE_SMIN: 2238 case ISD::VECREDUCE_UMAX: 2239 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; 2240 2241 case ISD::ROTL: 2242 case ISD::ROTR: 2243 ExpandIntRes_Rotate(N, Lo, Hi); 2244 break; 2245 2246 case ISD::FSHL: 2247 case ISD::FSHR: 2248 ExpandIntRes_FunnelShift(N, Lo, Hi); 2249 break; 2250 2251 case ISD::VSCALE: 2252 ExpandIntRes_VSCALE(N, Lo, Hi); 2253 break; 2254 } 2255 2256 // If Lo/Hi is null, the sub-method took care of registering results etc. 2257 if (Lo.getNode()) 2258 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 2259 } 2260 2261 /// Lower an atomic node to the appropriate builtin call. 2262 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 2263 unsigned Opc = Node->getOpcode(); 2264 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2265 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering(); 2266 // Lower to outline atomic libcall if outline atomics enabled, 2267 // or to sync libcall otherwise 2268 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); 2269 EVT RetVT = Node->getValueType(0); 2270 TargetLowering::MakeLibCallOptions CallOptions; 2271 SmallVector<SDValue, 4> Ops; 2272 if (TLI.getLibcallName(LC)) { 2273 Ops.append(Node->op_begin() + 2, Node->op_end()); 2274 Ops.push_back(Node->getOperand(1)); 2275 } else { 2276 LC = RTLIB::getSYNC(Opc, VT); 2277 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2278 "Unexpected atomic op or value type!"); 2279 Ops.append(Node->op_begin() + 1, Node->op_end()); 2280 } 2281 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), 2282 Node->getOperand(0)); 2283 } 2284 2285 /// N is a shift by a value that needs to be expanded, 2286 /// and the shift amount is a constant 'Amt'. Expand the operation. 2287 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, 2288 SDValue &Lo, SDValue &Hi) { 2289 SDLoc DL(N); 2290 // Expand the incoming operand to be shifted, so that we have its parts 2291 SDValue InL, InH; 2292 GetExpandedInteger(N->getOperand(0), InL, InH); 2293 2294 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization 2295 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. 2296 if (!Amt) { 2297 Lo = InL; 2298 Hi = InH; 2299 return; 2300 } 2301 2302 EVT NVT = InL.getValueType(); 2303 unsigned VTBits = N->getValueType(0).getSizeInBits(); 2304 unsigned NVTBits = NVT.getSizeInBits(); 2305 EVT ShTy = N->getOperand(1).getValueType(); 2306 2307 if (N->getOpcode() == ISD::SHL) { 2308 if (Amt.ugt(VTBits)) { 2309 Lo = Hi = DAG.getConstant(0, DL, NVT); 2310 } else if (Amt.ugt(NVTBits)) { 2311 Lo = DAG.getConstant(0, DL, NVT); 2312 Hi = DAG.getNode(ISD::SHL, DL, 2313 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2314 } else if (Amt == NVTBits) { 2315 Lo = DAG.getConstant(0, DL, NVT); 2316 Hi = InL; 2317 } else { 2318 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); 2319 Hi = DAG.getNode(ISD::OR, DL, NVT, 2320 DAG.getNode(ISD::SHL, DL, NVT, InH, 2321 DAG.getConstant(Amt, DL, ShTy)), 2322 DAG.getNode(ISD::SRL, DL, NVT, InL, 2323 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2324 } 2325 return; 2326 } 2327 2328 if (N->getOpcode() == ISD::SRL) { 2329 if (Amt.ugt(VTBits)) { 2330 Lo = Hi = DAG.getConstant(0, DL, NVT); 2331 } else if (Amt.ugt(NVTBits)) { 2332 Lo = DAG.getNode(ISD::SRL, DL, 2333 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2334 Hi = DAG.getConstant(0, DL, NVT); 2335 } else if (Amt == NVTBits) { 2336 Lo = InH; 2337 Hi = DAG.getConstant(0, DL, NVT); 2338 } else { 2339 Lo = DAG.getNode(ISD::OR, DL, NVT, 2340 DAG.getNode(ISD::SRL, DL, NVT, InL, 2341 DAG.getConstant(Amt, DL, ShTy)), 2342 DAG.getNode(ISD::SHL, DL, NVT, InH, 2343 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2344 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2345 } 2346 return; 2347 } 2348 2349 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2350 if (Amt.ugt(VTBits)) { 2351 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2352 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2353 } else if (Amt.ugt(NVTBits)) { 2354 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2355 DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2356 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2357 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2358 } else if (Amt == NVTBits) { 2359 Lo = InH; 2360 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2361 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2362 } else { 2363 Lo = DAG.getNode(ISD::OR, DL, NVT, 2364 DAG.getNode(ISD::SRL, DL, NVT, InL, 2365 DAG.getConstant(Amt, DL, ShTy)), 2366 DAG.getNode(ISD::SHL, DL, NVT, InH, 2367 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2368 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2369 } 2370 } 2371 2372 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 2373 /// this shift based on knowledge of the high bit of the shift amount. If we 2374 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual 2375 /// shift amount. 2376 bool DAGTypeLegalizer:: 2377 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2378 SDValue Amt = N->getOperand(1); 2379 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2380 EVT ShTy = Amt.getValueType(); 2381 unsigned ShBits = ShTy.getScalarSizeInBits(); 2382 unsigned NVTBits = NVT.getScalarSizeInBits(); 2383 assert(isPowerOf2_32(NVTBits) && 2384 "Expanded integer type size not a power of two!"); 2385 SDLoc dl(N); 2386 2387 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 2388 KnownBits Known = DAG.computeKnownBits(N->getOperand(1)); 2389 2390 // If we don't know anything about the high bits, exit. 2391 if (((Known.Zero|Known.One) & HighBitMask) == 0) 2392 return false; 2393 2394 // Get the incoming operand to be shifted. 2395 SDValue InL, InH; 2396 GetExpandedInteger(N->getOperand(0), InL, InH); 2397 2398 // If we know that any of the high bits of the shift amount are one, then we 2399 // can do this as a couple of simple shifts. 2400 if (Known.One.intersects(HighBitMask)) { 2401 // Mask out the high bit, which we know is set. 2402 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 2403 DAG.getConstant(~HighBitMask, dl, ShTy)); 2404 2405 switch (N->getOpcode()) { 2406 default: llvm_unreachable("Unknown shift"); 2407 case ISD::SHL: 2408 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. 2409 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 2410 return true; 2411 case ISD::SRL: 2412 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2413 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 2414 return true; 2415 case ISD::SRA: 2416 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 2417 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2418 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 2419 return true; 2420 } 2421 } 2422 2423 // If we know that all of the high bits of the shift amount are zero, then we 2424 // can do this as a couple of simple shifts. 2425 if (HighBitMask.isSubsetOf(Known.Zero)) { 2426 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 2427 // shift if x is zero. We can use XOR here because x is known to be smaller 2428 // than 32. 2429 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 2430 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2431 2432 unsigned Op1, Op2; 2433 switch (N->getOpcode()) { 2434 default: llvm_unreachable("Unknown shift"); 2435 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 2436 case ISD::SRL: 2437 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 2438 } 2439 2440 // When shifting right the arithmetic for Lo and Hi is swapped. 2441 if (N->getOpcode() != ISD::SHL) 2442 std::swap(InL, InH); 2443 2444 // Use a little trick to get the bits that move from Lo to Hi. First 2445 // shift by one bit. 2446 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); 2447 // Then compute the remaining shift with amount-1. 2448 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 2449 2450 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 2451 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 2452 2453 if (N->getOpcode() != ISD::SHL) 2454 std::swap(Hi, Lo); 2455 return true; 2456 } 2457 2458 return false; 2459 } 2460 2461 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 2462 /// of any size. 2463 bool DAGTypeLegalizer:: 2464 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2465 SDValue Amt = N->getOperand(1); 2466 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2467 EVT ShTy = Amt.getValueType(); 2468 unsigned NVTBits = NVT.getSizeInBits(); 2469 assert(isPowerOf2_32(NVTBits) && 2470 "Expanded integer type size not a power of two!"); 2471 SDLoc dl(N); 2472 2473 // Get the incoming operand to be shifted. 2474 SDValue InL, InH; 2475 GetExpandedInteger(N->getOperand(0), InL, InH); 2476 2477 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); 2478 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 2479 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 2480 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2481 Amt, NVBitsNode, ISD::SETULT); 2482 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2483 Amt, DAG.getConstant(0, dl, ShTy), 2484 ISD::SETEQ); 2485 2486 SDValue LoS, HiS, LoL, HiL; 2487 switch (N->getOpcode()) { 2488 default: llvm_unreachable("Unknown shift"); 2489 case ISD::SHL: 2490 // Short: ShAmt < NVTBits 2491 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 2492 HiS = DAG.getNode(ISD::OR, dl, NVT, 2493 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 2494 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 2495 2496 // Long: ShAmt >= NVTBits 2497 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. 2498 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 2499 2500 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 2501 Hi = DAG.getSelect(dl, NVT, isZero, InH, 2502 DAG.getSelect(dl, NVT, isShort, HiS, HiL)); 2503 return true; 2504 case ISD::SRL: 2505 // Short: ShAmt < NVTBits 2506 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 2507 LoS = DAG.getNode(ISD::OR, dl, NVT, 2508 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2509 // FIXME: If Amt is zero, the following shift generates an undefined result 2510 // on some architectures. 2511 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2512 2513 // Long: ShAmt >= NVTBits 2514 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2515 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2516 2517 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2518 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2519 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2520 return true; 2521 case ISD::SRA: 2522 // Short: ShAmt < NVTBits 2523 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 2524 LoS = DAG.getNode(ISD::OR, dl, NVT, 2525 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2526 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2527 2528 // Long: ShAmt >= NVTBits 2529 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 2530 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2531 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2532 2533 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2534 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2535 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2536 return true; 2537 } 2538 } 2539 2540 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { 2541 2542 switch (Op) { 2543 default: llvm_unreachable("invalid min/max opcode"); 2544 case ISD::SMAX: 2545 return std::make_pair(ISD::SETGT, ISD::UMAX); 2546 case ISD::UMAX: 2547 return std::make_pair(ISD::SETUGT, ISD::UMAX); 2548 case ISD::SMIN: 2549 return std::make_pair(ISD::SETLT, ISD::UMIN); 2550 case ISD::UMIN: 2551 return std::make_pair(ISD::SETULT, ISD::UMIN); 2552 } 2553 } 2554 2555 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, 2556 SDValue &Lo, SDValue &Hi) { 2557 SDLoc DL(N); 2558 ISD::NodeType LoOpc; 2559 ISD::CondCode CondC; 2560 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); 2561 2562 // Expand the subcomponents. 2563 SDValue LHSL, LHSH, RHSL, RHSH; 2564 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2565 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2566 2567 // Value types 2568 EVT NVT = LHSL.getValueType(); 2569 EVT CCT = getSetCCResultType(NVT); 2570 2571 // Hi part is always the same op 2572 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 2573 2574 // We need to know whether to select Lo part that corresponds to 'winning' 2575 // Hi part or if Hi parts are equal. 2576 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); 2577 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); 2578 2579 // Lo part corresponding to the 'winning' Hi part 2580 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); 2581 2582 // Recursed Lo part if Hi parts are equal, this uses unsigned version 2583 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); 2584 2585 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); 2586 } 2587 2588 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 2589 SDValue &Lo, SDValue &Hi) { 2590 SDLoc dl(N); 2591 // Expand the subcomponents. 2592 SDValue LHSL, LHSH, RHSL, RHSH; 2593 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2594 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2595 2596 EVT NVT = LHSL.getValueType(); 2597 SDValue LoOps[2] = { LHSL, RHSL }; 2598 SDValue HiOps[3] = { LHSH, RHSH }; 2599 2600 bool HasOpCarry = TLI.isOperationLegalOrCustom( 2601 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY, 2602 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2603 if (HasOpCarry) { 2604 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 2605 if (N->getOpcode() == ISD::ADD) { 2606 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 2607 HiOps[2] = Lo.getValue(1); 2608 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps); 2609 } else { 2610 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 2611 HiOps[2] = Lo.getValue(1); 2612 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps); 2613 } 2614 return; 2615 } 2616 2617 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 2618 // them. TODO: Teach operation legalization how to expand unsupported 2619 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 2620 // a carry of type MVT::Glue, but there doesn't seem to be any way to 2621 // generate a value of this type in the expanded code sequence. 2622 bool hasCarry = 2623 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 2624 ISD::ADDC : ISD::SUBC, 2625 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2626 2627 if (hasCarry) { 2628 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 2629 if (N->getOpcode() == ISD::ADD) { 2630 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 2631 HiOps[2] = Lo.getValue(1); 2632 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 2633 } else { 2634 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 2635 HiOps[2] = Lo.getValue(1); 2636 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 2637 } 2638 return; 2639 } 2640 2641 bool hasOVF = 2642 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 2643 ISD::UADDO : ISD::USUBO, 2644 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2645 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); 2646 2647 if (hasOVF) { 2648 EVT OvfVT = getSetCCResultType(NVT); 2649 SDVTList VTList = DAG.getVTList(NVT, OvfVT); 2650 int RevOpc; 2651 if (N->getOpcode() == ISD::ADD) { 2652 RevOpc = ISD::SUB; 2653 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 2654 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 2655 } else { 2656 RevOpc = ISD::ADD; 2657 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 2658 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 2659 } 2660 SDValue OVF = Lo.getValue(1); 2661 2662 switch (BoolType) { 2663 case TargetLoweringBase::UndefinedBooleanContent: 2664 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); 2665 LLVM_FALLTHROUGH; 2666 case TargetLoweringBase::ZeroOrOneBooleanContent: 2667 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); 2668 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); 2669 break; 2670 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 2671 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); 2672 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); 2673 } 2674 return; 2675 } 2676 2677 if (N->getOpcode() == ISD::ADD) { 2678 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 2679 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 2680 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 2681 ISD::SETULT); 2682 2683 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) { 2684 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT); 2685 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); 2686 return; 2687 } 2688 2689 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1, 2690 DAG.getConstant(1, dl, NVT), 2691 DAG.getConstant(0, dl, NVT)); 2692 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1], 2693 ISD::SETULT); 2694 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2, 2695 DAG.getConstant(1, dl, NVT), Carry1); 2696 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 2697 } else { 2698 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 2699 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 2700 SDValue Cmp = 2701 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 2702 LoOps[0], LoOps[1], ISD::SETULT); 2703 2704 SDValue Borrow; 2705 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 2706 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); 2707 else 2708 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 2709 DAG.getConstant(0, dl, NVT)); 2710 2711 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 2712 } 2713 } 2714 2715 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 2716 SDValue &Lo, SDValue &Hi) { 2717 // Expand the subcomponents. 2718 SDValue LHSL, LHSH, RHSL, RHSH; 2719 SDLoc dl(N); 2720 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2721 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2722 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 2723 SDValue LoOps[2] = { LHSL, RHSL }; 2724 SDValue HiOps[3] = { LHSH, RHSH }; 2725 2726 if (N->getOpcode() == ISD::ADDC) { 2727 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 2728 HiOps[2] = Lo.getValue(1); 2729 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 2730 } else { 2731 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 2732 HiOps[2] = Lo.getValue(1); 2733 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 2734 } 2735 2736 // Legalized the flag result - switch anything that used the old flag to 2737 // use the new one. 2738 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2739 } 2740 2741 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 2742 SDValue &Lo, SDValue &Hi) { 2743 // Expand the subcomponents. 2744 SDValue LHSL, LHSH, RHSL, RHSH; 2745 SDLoc dl(N); 2746 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2747 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2748 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 2749 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 2750 SDValue HiOps[3] = { LHSH, RHSH }; 2751 2752 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2753 HiOps[2] = Lo.getValue(1); 2754 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 2755 2756 // Legalized the flag result - switch anything that used the old flag to 2757 // use the new one. 2758 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2759 } 2760 2761 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 2762 SDValue &Lo, SDValue &Hi) { 2763 SDValue LHS = N->getOperand(0); 2764 SDValue RHS = N->getOperand(1); 2765 SDLoc dl(N); 2766 2767 SDValue Ovf; 2768 2769 unsigned CarryOp, NoCarryOp; 2770 ISD::CondCode Cond; 2771 switch(N->getOpcode()) { 2772 case ISD::UADDO: 2773 CarryOp = ISD::ADDCARRY; 2774 NoCarryOp = ISD::ADD; 2775 Cond = ISD::SETULT; 2776 break; 2777 case ISD::USUBO: 2778 CarryOp = ISD::SUBCARRY; 2779 NoCarryOp = ISD::SUB; 2780 Cond = ISD::SETUGT; 2781 break; 2782 default: 2783 llvm_unreachable("Node has unexpected Opcode"); 2784 } 2785 2786 bool HasCarryOp = TLI.isOperationLegalOrCustom( 2787 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 2788 2789 if (HasCarryOp) { 2790 // Expand the subcomponents. 2791 SDValue LHSL, LHSH, RHSL, RHSH; 2792 GetExpandedInteger(LHS, LHSL, LHSH); 2793 GetExpandedInteger(RHS, RHSL, RHSH); 2794 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 2795 SDValue LoOps[2] = { LHSL, RHSL }; 2796 SDValue HiOps[3] = { LHSH, RHSH }; 2797 2798 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2799 HiOps[2] = Lo.getValue(1); 2800 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps); 2801 2802 Ovf = Hi.getValue(1); 2803 } else { 2804 // Expand the result by simply replacing it with the equivalent 2805 // non-overflow-checking operation. 2806 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS); 2807 SplitInteger(Sum, Lo, Hi); 2808 2809 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 2810 // overflows iff a - b > a. 2811 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); 2812 } 2813 2814 // Legalized the flag result - switch anything that used the old flag to 2815 // use the new one. 2816 ReplaceValueWith(SDValue(N, 1), Ovf); 2817 } 2818 2819 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N, 2820 SDValue &Lo, SDValue &Hi) { 2821 // Expand the subcomponents. 2822 SDValue LHSL, LHSH, RHSL, RHSH; 2823 SDLoc dl(N); 2824 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2825 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2826 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 2827 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 2828 SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; 2829 2830 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2831 HiOps[2] = Lo.getValue(1); 2832 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 2833 2834 // Legalized the flag result - switch anything that used the old flag to 2835 // use the new one. 2836 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2837 } 2838 2839 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, 2840 SDValue &Lo, SDValue &Hi) { 2841 // Expand the subcomponents. 2842 SDValue LHSL, LHSH, RHSL, RHSH; 2843 SDLoc dl(N); 2844 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2845 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2846 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 2847 2848 // We need to use an unsigned carry op for the lo part. 2849 unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY 2850 : ISD::SUBCARRY; 2851 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) }); 2852 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 2853 2854 // Legalized the flag result - switch anything that used the old flag to 2855 // use the new one. 2856 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2857 } 2858 2859 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 2860 SDValue &Lo, SDValue &Hi) { 2861 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2862 SDLoc dl(N); 2863 SDValue Op = N->getOperand(0); 2864 if (Op.getValueType().bitsLE(NVT)) { 2865 // The low part is any extension of the input (which degenerates to a copy). 2866 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 2867 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 2868 } else { 2869 // For example, extension of an i48 to an i64. The operand type necessarily 2870 // promotes to the result type, so will end up being expanded too. 2871 assert(getTypeAction(Op.getValueType()) == 2872 TargetLowering::TypePromoteInteger && 2873 "Only know how to promote this result!"); 2874 SDValue Res = GetPromotedInteger(Op); 2875 assert(Res.getValueType() == N->getValueType(0) && 2876 "Operand over promoted?"); 2877 // Split the promoted operand. This will simplify when it is expanded. 2878 SplitInteger(Res, Lo, Hi); 2879 } 2880 } 2881 2882 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 2883 SDValue &Lo, SDValue &Hi) { 2884 SDLoc dl(N); 2885 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2886 EVT NVT = Lo.getValueType(); 2887 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2888 unsigned NVTBits = NVT.getSizeInBits(); 2889 unsigned EVTBits = EVT.getSizeInBits(); 2890 2891 if (NVTBits < EVTBits) { 2892 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 2893 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2894 EVTBits - NVTBits))); 2895 } else { 2896 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 2897 // The high part replicates the sign bit of Lo, make it explicit. 2898 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2899 DAG.getConstant(NVTBits - 1, dl, 2900 TLI.getPointerTy(DAG.getDataLayout()))); 2901 } 2902 } 2903 2904 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 2905 SDValue &Lo, SDValue &Hi) { 2906 SDLoc dl(N); 2907 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2908 EVT NVT = Lo.getValueType(); 2909 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2910 unsigned NVTBits = NVT.getSizeInBits(); 2911 unsigned EVTBits = EVT.getSizeInBits(); 2912 2913 if (NVTBits < EVTBits) { 2914 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 2915 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2916 EVTBits - NVTBits))); 2917 } else { 2918 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 2919 // The high part must be zero, make it explicit. 2920 Hi = DAG.getConstant(0, dl, NVT); 2921 } 2922 } 2923 2924 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, 2925 SDValue &Lo, SDValue &Hi) { 2926 SDLoc dl(N); 2927 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 2928 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); 2929 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); 2930 } 2931 2932 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 2933 SDValue &Lo, SDValue &Hi) { 2934 SDLoc dl(N); 2935 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 2936 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 2937 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 2938 } 2939 2940 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, 2941 SDValue &Hi) { 2942 SDLoc dl(N); 2943 // parity(HiLo) -> parity(Lo^Hi) 2944 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2945 EVT NVT = Lo.getValueType(); 2946 Lo = 2947 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi)); 2948 Hi = DAG.getConstant(0, dl, NVT); 2949 } 2950 2951 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 2952 SDValue &Lo, SDValue &Hi) { 2953 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2954 unsigned NBitWidth = NVT.getSizeInBits(); 2955 auto Constant = cast<ConstantSDNode>(N); 2956 const APInt &Cst = Constant->getAPIntValue(); 2957 bool IsTarget = Constant->isTargetOpcode(); 2958 bool IsOpaque = Constant->isOpaque(); 2959 SDLoc dl(N); 2960 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); 2961 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, 2962 IsOpaque); 2963 } 2964 2965 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { 2966 SDLoc dl(N); 2967 2968 SDValue N0 = N->getOperand(0); 2969 GetExpandedInteger(N0, Lo, Hi); 2970 EVT NVT = Lo.getValueType(); 2971 2972 // If we have ADDCARRY, use the expanded form of the sra+add+xor sequence we 2973 // use in LegalizeDAG. The ADD part of the expansion is based on 2974 // ExpandIntRes_ADDSUB which also uses ADDCARRY/UADDO after checking that 2975 // ADDCARRY is LegalOrCustom. Each of the pieces here can be further expanded 2976 // if needed. Shift expansion has a special case for filling with sign bits 2977 // so that we will only end up with one SRA. 2978 bool HasAddCarry = TLI.isOperationLegalOrCustom( 2979 ISD::ADDCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2980 if (HasAddCarry) { 2981 EVT ShiftAmtTy = getShiftAmountTyForConstant(NVT, TLI, DAG); 2982 SDValue Sign = 2983 DAG.getNode(ISD::SRA, dl, NVT, Hi, 2984 DAG.getConstant(NVT.getSizeInBits() - 1, dl, ShiftAmtTy)); 2985 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 2986 Lo = DAG.getNode(ISD::UADDO, dl, VTList, Lo, Sign); 2987 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Hi, Sign, Lo.getValue(1)); 2988 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign); 2989 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign); 2990 return; 2991 } 2992 2993 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) 2994 EVT VT = N->getValueType(0); 2995 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, 2996 DAG.getConstant(0, dl, VT), N0); 2997 SDValue NegLo, NegHi; 2998 SplitInteger(Neg, NegLo, NegHi); 2999 3000 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), 3001 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT); 3002 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); 3003 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); 3004 } 3005 3006 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 3007 SDValue &Lo, SDValue &Hi) { 3008 SDLoc dl(N); 3009 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 3010 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3011 EVT NVT = Lo.getValueType(); 3012 3013 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 3014 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3015 3016 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 3017 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 3018 3019 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 3020 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 3021 DAG.getConstant(NVT.getSizeInBits(), dl, 3022 NVT))); 3023 Hi = DAG.getConstant(0, dl, NVT); 3024 } 3025 3026 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 3027 SDValue &Lo, SDValue &Hi) { 3028 SDLoc dl(N); 3029 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 3030 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3031 EVT NVT = Lo.getValueType(); 3032 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 3033 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 3034 Hi = DAG.getConstant(0, dl, NVT); 3035 } 3036 3037 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 3038 SDValue &Lo, SDValue &Hi) { 3039 SDLoc dl(N); 3040 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 3041 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3042 EVT NVT = Lo.getValueType(); 3043 3044 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 3045 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3046 3047 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 3048 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 3049 3050 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 3051 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 3052 DAG.getConstant(NVT.getSizeInBits(), dl, 3053 NVT))); 3054 Hi = DAG.getConstant(0, dl, NVT); 3055 } 3056 3057 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo, 3058 SDValue &Hi) { 3059 SDLoc dl(N); 3060 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3061 unsigned NBitWidth = NVT.getSizeInBits(); 3062 3063 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 3064 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0)); 3065 SDValue Chain = Lo.getValue(1); 3066 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS 3067 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3068 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy)); 3069 3070 // Legalize the chain result - switch anything that used the old chain to 3071 // use the new one. 3072 ReplaceValueWith(SDValue(N, 1), Chain); 3073 } 3074 3075 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 3076 SDValue &Hi) { 3077 SDLoc dl(N); 3078 EVT VT = N->getValueType(0); 3079 3080 bool IsStrict = N->isStrictFPOpcode(); 3081 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3082 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3083 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3084 Op = GetPromotedFloat(Op); 3085 3086 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3087 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3088 Op = GetSoftPromotedHalf(Op); 3089 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); 3090 } 3091 3092 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 3093 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 3094 TargetLowering::MakeLibCallOptions CallOptions; 3095 CallOptions.setSExt(true); 3096 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3097 CallOptions, dl, Chain); 3098 SplitInteger(Tmp.first, Lo, Hi); 3099 3100 if (IsStrict) 3101 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3102 } 3103 3104 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 3105 SDValue &Hi) { 3106 SDLoc dl(N); 3107 EVT VT = N->getValueType(0); 3108 3109 bool IsStrict = N->isStrictFPOpcode(); 3110 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3111 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3112 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3113 Op = GetPromotedFloat(Op); 3114 3115 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3116 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3117 Op = GetSoftPromotedHalf(Op); 3118 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); 3119 } 3120 3121 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 3122 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 3123 TargetLowering::MakeLibCallOptions CallOptions; 3124 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3125 CallOptions, dl, Chain); 3126 SplitInteger(Tmp.first, Lo, Hi); 3127 3128 if (IsStrict) 3129 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3130 } 3131 3132 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, 3133 SDValue &Hi) { 3134 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 3135 SplitInteger(Res, Lo, Hi); 3136 } 3137 3138 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo, 3139 SDValue &Hi) { 3140 SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); 3141 3142 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && 3143 "Input type needs to be promoted!"); 3144 3145 EVT VT = Op.getValueType(); 3146 3147 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3148 if (N->getOpcode() == ISD::LLROUND || 3149 N->getOpcode() == ISD::STRICT_LLROUND) { 3150 if (VT == MVT::f32) 3151 LC = RTLIB::LLROUND_F32; 3152 else if (VT == MVT::f64) 3153 LC = RTLIB::LLROUND_F64; 3154 else if (VT == MVT::f80) 3155 LC = RTLIB::LLROUND_F80; 3156 else if (VT == MVT::f128) 3157 LC = RTLIB::LLROUND_F128; 3158 else if (VT == MVT::ppcf128) 3159 LC = RTLIB::LLROUND_PPCF128; 3160 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); 3161 } else if (N->getOpcode() == ISD::LLRINT || 3162 N->getOpcode() == ISD::STRICT_LLRINT) { 3163 if (VT == MVT::f32) 3164 LC = RTLIB::LLRINT_F32; 3165 else if (VT == MVT::f64) 3166 LC = RTLIB::LLRINT_F64; 3167 else if (VT == MVT::f80) 3168 LC = RTLIB::LLRINT_F80; 3169 else if (VT == MVT::f128) 3170 LC = RTLIB::LLRINT_F128; 3171 else if (VT == MVT::ppcf128) 3172 LC = RTLIB::LLRINT_PPCF128; 3173 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); 3174 } else 3175 llvm_unreachable("Unexpected opcode!"); 3176 3177 SDLoc dl(N); 3178 EVT RetVT = N->getValueType(0); 3179 SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue(); 3180 3181 TargetLowering::MakeLibCallOptions CallOptions; 3182 CallOptions.setSExt(true); 3183 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, 3184 Op, CallOptions, dl, 3185 Chain); 3186 SplitInteger(Tmp.first, Lo, Hi); 3187 3188 if (N->isStrictFPOpcode()) 3189 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3190 } 3191 3192 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 3193 SDValue &Lo, SDValue &Hi) { 3194 if (N->isAtomic()) { 3195 // It's typical to have larger CAS than atomic load instructions. 3196 SDLoc dl(N); 3197 EVT VT = N->getMemoryVT(); 3198 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 3199 SDValue Zero = DAG.getConstant(0, dl, VT); 3200 SDValue Swap = DAG.getAtomicCmpSwap( 3201 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 3202 VT, VTs, N->getOperand(0), 3203 N->getOperand(1), Zero, Zero, N->getMemOperand()); 3204 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 3205 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 3206 return; 3207 } 3208 3209 if (ISD::isNormalLoad(N)) { 3210 ExpandRes_NormalLoad(N, Lo, Hi); 3211 return; 3212 } 3213 3214 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 3215 3216 EVT VT = N->getValueType(0); 3217 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3218 SDValue Ch = N->getChain(); 3219 SDValue Ptr = N->getBasePtr(); 3220 ISD::LoadExtType ExtType = N->getExtensionType(); 3221 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 3222 AAMDNodes AAInfo = N->getAAInfo(); 3223 SDLoc dl(N); 3224 3225 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 3226 3227 if (N->getMemoryVT().bitsLE(NVT)) { 3228 EVT MemVT = N->getMemoryVT(); 3229 3230 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, 3231 N->getOriginalAlign(), MMOFlags, AAInfo); 3232 3233 // Remember the chain. 3234 Ch = Lo.getValue(1); 3235 3236 if (ExtType == ISD::SEXTLOAD) { 3237 // The high part is obtained by SRA'ing all but one of the bits of the 3238 // lo part. 3239 unsigned LoSize = Lo.getValueSizeInBits(); 3240 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3241 DAG.getConstant(LoSize - 1, dl, 3242 TLI.getPointerTy(DAG.getDataLayout()))); 3243 } else if (ExtType == ISD::ZEXTLOAD) { 3244 // The high part is just a zero. 3245 Hi = DAG.getConstant(0, dl, NVT); 3246 } else { 3247 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 3248 // The high part is undefined. 3249 Hi = DAG.getUNDEF(NVT); 3250 } 3251 } else if (DAG.getDataLayout().isLittleEndian()) { 3252 // Little-endian - low bits are at low addresses. 3253 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 3254 N->getOriginalAlign(), MMOFlags, AAInfo); 3255 3256 unsigned ExcessBits = 3257 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 3258 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 3259 3260 // Increment the pointer to the other half. 3261 unsigned IncrementSize = NVT.getSizeInBits()/8; 3262 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 3263 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 3264 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 3265 N->getOriginalAlign(), MMOFlags, AAInfo); 3266 3267 // Build a factor node to remember that this load is independent of the 3268 // other one. 3269 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3270 Hi.getValue(1)); 3271 } else { 3272 // Big-endian - high bits are at low addresses. Favor aligned loads at 3273 // the cost of some bit-fiddling. 3274 EVT MemVT = N->getMemoryVT(); 3275 unsigned EBytes = MemVT.getStoreSize(); 3276 unsigned IncrementSize = NVT.getSizeInBits()/8; 3277 unsigned ExcessBits = (EBytes - IncrementSize)*8; 3278 3279 // Load both the high bits and maybe some of the low bits. 3280 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 3281 EVT::getIntegerVT(*DAG.getContext(), 3282 MemVT.getSizeInBits() - ExcessBits), 3283 N->getOriginalAlign(), MMOFlags, AAInfo); 3284 3285 // Increment the pointer to the other half. 3286 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 3287 // Load the rest of the low bits. 3288 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 3289 N->getPointerInfo().getWithOffset(IncrementSize), 3290 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 3291 N->getOriginalAlign(), MMOFlags, AAInfo); 3292 3293 // Build a factor node to remember that this load is independent of the 3294 // other one. 3295 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3296 Hi.getValue(1)); 3297 3298 if (ExcessBits < NVT.getSizeInBits()) { 3299 // Transfer low bits from the bottom of Hi to the top of Lo. 3300 Lo = DAG.getNode( 3301 ISD::OR, dl, NVT, Lo, 3302 DAG.getNode(ISD::SHL, dl, NVT, Hi, 3303 DAG.getConstant(ExcessBits, dl, 3304 TLI.getPointerTy(DAG.getDataLayout())))); 3305 // Move high bits to the right position in Hi. 3306 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, 3307 Hi, 3308 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 3309 TLI.getPointerTy(DAG.getDataLayout()))); 3310 } 3311 } 3312 3313 // Legalize the chain result - switch anything that used the old chain to 3314 // use the new one. 3315 ReplaceValueWith(SDValue(N, 1), Ch); 3316 } 3317 3318 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 3319 SDValue &Lo, SDValue &Hi) { 3320 SDLoc dl(N); 3321 SDValue LL, LH, RL, RH; 3322 GetExpandedInteger(N->getOperand(0), LL, LH); 3323 GetExpandedInteger(N->getOperand(1), RL, RH); 3324 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 3325 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 3326 } 3327 3328 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 3329 SDValue &Lo, SDValue &Hi) { 3330 EVT VT = N->getValueType(0); 3331 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3332 SDLoc dl(N); 3333 3334 SDValue LL, LH, RL, RH; 3335 GetExpandedInteger(N->getOperand(0), LL, LH); 3336 GetExpandedInteger(N->getOperand(1), RL, RH); 3337 3338 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, 3339 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3340 LL, LH, RL, RH)) 3341 return; 3342 3343 // If nothing else, we can make a libcall. 3344 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3345 if (VT == MVT::i16) 3346 LC = RTLIB::MUL_I16; 3347 else if (VT == MVT::i32) 3348 LC = RTLIB::MUL_I32; 3349 else if (VT == MVT::i64) 3350 LC = RTLIB::MUL_I64; 3351 else if (VT == MVT::i128) 3352 LC = RTLIB::MUL_I128; 3353 3354 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 3355 // We'll expand the multiplication by brute force because we have no other 3356 // options. This is a trivially-generalized version of the code from 3357 // Hacker's Delight (itself derived from Knuth's Algorithm M from section 3358 // 4.3.1). 3359 unsigned Bits = NVT.getSizeInBits(); 3360 unsigned HalfBits = Bits >> 1; 3361 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, 3362 NVT); 3363 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); 3364 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); 3365 3366 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); 3367 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); 3368 3369 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 3370 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) { 3371 // The type from TLI is too small to fit the shift amount we want. 3372 // Override it with i32. The shift will have to be legalized. 3373 ShiftAmtTy = MVT::i32; 3374 } 3375 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy); 3376 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); 3377 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); 3378 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); 3379 3380 SDValue U = DAG.getNode(ISD::ADD, dl, NVT, 3381 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); 3382 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); 3383 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); 3384 3385 SDValue V = DAG.getNode(ISD::ADD, dl, NVT, 3386 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); 3387 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); 3388 3389 SDValue W = DAG.getNode(ISD::ADD, dl, NVT, 3390 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), 3391 DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); 3392 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, 3393 DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); 3394 3395 Hi = DAG.getNode(ISD::ADD, dl, NVT, W, 3396 DAG.getNode(ISD::ADD, dl, NVT, 3397 DAG.getNode(ISD::MUL, dl, NVT, RH, LL), 3398 DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); 3399 return; 3400 } 3401 3402 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3403 TargetLowering::MakeLibCallOptions CallOptions; 3404 CallOptions.setSExt(true); 3405 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, 3406 Lo, Hi); 3407 } 3408 3409 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, 3410 SDValue &Hi) { 3411 SDLoc DL(N); 3412 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3413 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); 3414 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); 3415 Lo = R.getValue(0); 3416 Hi = R.getValue(1); 3417 ReplaceValueWith(SDValue(N, 1), R.getValue(2)); 3418 } 3419 3420 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, 3421 SDValue &Hi) { 3422 SDValue Result = TLI.expandAddSubSat(N, DAG); 3423 SplitInteger(Result, Lo, Hi); 3424 } 3425 3426 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, 3427 SDValue &Hi) { 3428 SDValue Result = TLI.expandShlSat(N, DAG); 3429 SplitInteger(Result, Lo, Hi); 3430 } 3431 3432 /// This performs an expansion of the integer result for a fixed point 3433 /// multiplication. The default expansion performs rounding down towards 3434 /// negative infinity, though targets that do care about rounding should specify 3435 /// a target hook for rounding and provide their own expansion or lowering of 3436 /// fixed point multiplication to be consistent with rounding. 3437 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, 3438 SDValue &Hi) { 3439 SDLoc dl(N); 3440 EVT VT = N->getValueType(0); 3441 unsigned VTSize = VT.getScalarSizeInBits(); 3442 SDValue LHS = N->getOperand(0); 3443 SDValue RHS = N->getOperand(1); 3444 uint64_t Scale = N->getConstantOperandVal(2); 3445 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || 3446 N->getOpcode() == ISD::UMULFIXSAT); 3447 bool Signed = (N->getOpcode() == ISD::SMULFIX || 3448 N->getOpcode() == ISD::SMULFIXSAT); 3449 3450 // Handle special case when scale is equal to zero. 3451 if (!Scale) { 3452 SDValue Result; 3453 if (!Saturating) { 3454 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 3455 } else { 3456 EVT BoolVT = getSetCCResultType(VT); 3457 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; 3458 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); 3459 SDValue Product = Result.getValue(0); 3460 SDValue Overflow = Result.getValue(1); 3461 if (Signed) { 3462 APInt MinVal = APInt::getSignedMinValue(VTSize); 3463 APInt MaxVal = APInt::getSignedMaxValue(VTSize); 3464 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); 3465 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 3466 SDValue Zero = DAG.getConstant(0, dl, VT); 3467 // Xor the inputs, if resulting sign bit is 0 the product will be 3468 // positive, else negative. 3469 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); 3470 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); 3471 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); 3472 Result = DAG.getSelect(dl, VT, Overflow, Result, Product); 3473 } else { 3474 // For unsigned multiplication, we only need to check the max since we 3475 // can't really overflow towards zero. 3476 APInt MaxVal = APInt::getMaxValue(VTSize); 3477 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 3478 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product); 3479 } 3480 } 3481 SplitInteger(Result, Lo, Hi); 3482 return; 3483 } 3484 3485 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will 3486 // cover for unhandled cases below, while still being valid for UMULFIX[SAT]. 3487 assert(Scale <= VTSize && "Scale can't be larger than the value type size."); 3488 3489 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3490 SDValue LL, LH, RL, RH; 3491 GetExpandedInteger(LHS, LL, LH); 3492 GetExpandedInteger(RHS, RL, RH); 3493 SmallVector<SDValue, 4> Result; 3494 3495 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; 3496 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, 3497 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3498 LL, LH, RL, RH)) { 3499 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); 3500 return; 3501 } 3502 3503 unsigned NVTSize = NVT.getScalarSizeInBits(); 3504 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " 3505 "the size of the current value type"); 3506 EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 3507 3508 // After getting the multiplication result in 4 parts, we need to perform a 3509 // shift right by the amount of the scale to get the result in that scale. 3510 // 3511 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in 3512 // 128 bits that are cut into 4 32-bit parts: 3513 // 3514 // HH HL LH LL 3515 // |---32---|---32---|---32---|---32---| 3516 // 128 96 64 32 0 3517 // 3518 // |------VTSize-----| 3519 // 3520 // |NVTSize-| 3521 // 3522 // The resulting Lo and Hi would normally be in LL and LH after the shift. But 3523 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift 3524 // amount and get Lo and Hi using two funnel shifts. Or for the special case 3525 // when Scale is a multiple of NVTSize we can just pick the result without 3526 // shifting. 3527 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. 3528 if (Scale % NVTSize) { 3529 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy); 3530 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0], 3531 ShiftAmount); 3532 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1], 3533 ShiftAmount); 3534 } else { 3535 Lo = Result[Part0]; 3536 Hi = Result[Part0 + 1]; 3537 } 3538 3539 // Unless saturation is requested we are done. The result is in <Hi,Lo>. 3540 if (!Saturating) 3541 return; 3542 3543 // Can not overflow when there is no integer part. 3544 if (Scale == VTSize) 3545 return; 3546 3547 // To handle saturation we must check for overflow in the multiplication. 3548 // 3549 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) 3550 // aren't all zeroes. 3551 // 3552 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) 3553 // aren't all ones or all zeroes. 3554 // 3555 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the 3556 // highest bit of HH determines saturation direction in the event of signed 3557 // saturation. 3558 3559 SDValue ResultHL = Result[2]; 3560 SDValue ResultHH = Result[3]; 3561 3562 SDValue SatMax, SatMin; 3563 SDValue NVTZero = DAG.getConstant(0, dl, NVT); 3564 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); 3565 EVT BoolNVT = getSetCCResultType(NVT); 3566 3567 if (!Signed) { 3568 if (Scale < NVTSize) { 3569 // Overflow happened if ((HH | (HL >> Scale)) != 0). 3570 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 3571 DAG.getConstant(Scale, dl, ShiftTy)); 3572 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH); 3573 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE); 3574 } else if (Scale == NVTSize) { 3575 // Overflow happened if (HH != 0). 3576 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE); 3577 } else if (Scale < VTSize) { 3578 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0). 3579 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 3580 DAG.getConstant(Scale - NVTSize, dl, 3581 ShiftTy)); 3582 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE); 3583 } else 3584 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" 3585 "(and saturation can't happen with Scale==VTSize)."); 3586 3587 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi); 3588 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo); 3589 return; 3590 } 3591 3592 if (Scale < NVTSize) { 3593 // The number of overflow bits we can check are VTSize - Scale + 1 (we 3594 // include the sign bit). If these top bits are > 0, then we overflowed past 3595 // the max value. If these top bits are < -1, then we overflowed past the 3596 // min value. Otherwise, we did not overflow. 3597 unsigned OverflowBits = VTSize - Scale + 1; 3598 assert(OverflowBits <= VTSize && OverflowBits > NVTSize && 3599 "Extent of overflow bits must start within HL"); 3600 SDValue HLHiMask = DAG.getConstant( 3601 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); 3602 SDValue HLLoMask = DAG.getConstant( 3603 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); 3604 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). 3605 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 3606 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 3607 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); 3608 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 3609 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT)); 3610 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). 3611 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 3612 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 3613 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); 3614 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 3615 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT)); 3616 } else if (Scale == NVTSize) { 3617 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). 3618 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 3619 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 3620 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); 3621 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 3622 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg)); 3623 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). 3624 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 3625 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 3626 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); 3627 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 3628 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos)); 3629 } else if (Scale < VTSize) { 3630 // This is similar to the case when we saturate if Scale < NVTSize, but we 3631 // only need to check HH. 3632 unsigned OverflowBits = VTSize - Scale + 1; 3633 SDValue HHHiMask = DAG.getConstant( 3634 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); 3635 SDValue HHLoMask = DAG.getConstant( 3636 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); 3637 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); 3638 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); 3639 } else 3640 llvm_unreachable("Illegal scale for signed fixed point mul."); 3641 3642 // Saturate to signed maximum. 3643 APInt MaxHi = APInt::getSignedMaxValue(NVTSize); 3644 APInt MaxLo = APInt::getAllOnesValue(NVTSize); 3645 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi); 3646 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo); 3647 // Saturate to signed minimum. 3648 APInt MinHi = APInt::getSignedMinValue(NVTSize); 3649 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi); 3650 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); 3651 } 3652 3653 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, 3654 SDValue &Hi) { 3655 SDLoc dl(N); 3656 // Try expanding in the existing type first. 3657 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0), 3658 N->getOperand(1), 3659 N->getConstantOperandVal(2), DAG); 3660 3661 if (!Res) 3662 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1), 3663 N->getConstantOperandVal(2), TLI, DAG); 3664 SplitInteger(Res, Lo, Hi); 3665 } 3666 3667 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 3668 SDValue &Lo, SDValue &Hi) { 3669 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && 3670 "Node has unexpected Opcode"); 3671 SDValue LHS = Node->getOperand(0); 3672 SDValue RHS = Node->getOperand(1); 3673 SDLoc dl(Node); 3674 3675 SDValue Ovf; 3676 3677 bool IsAdd = Node->getOpcode() == ISD::SADDO; 3678 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY; 3679 3680 bool HasCarryOp = TLI.isOperationLegalOrCustom( 3681 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 3682 3683 if (HasCarryOp) { 3684 // Expand the subcomponents. 3685 SDValue LHSL, LHSH, RHSL, RHSH; 3686 GetExpandedInteger(LHS, LHSL, LHSH); 3687 GetExpandedInteger(RHS, RHSL, RHSH); 3688 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1)); 3689 3690 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL}); 3691 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 3692 3693 Ovf = Hi.getValue(1); 3694 } else { 3695 // Expand the result by simply replacing it with the equivalent 3696 // non-overflow-checking operation. 3697 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 3698 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3699 LHS, RHS); 3700 SplitInteger(Sum, Lo, Hi); 3701 3702 // Compute the overflow. 3703 // 3704 // LHSSign -> LHS < 0 3705 // RHSSign -> RHS < 0 3706 // SumSign -> Sum < 0 3707 // 3708 // Add: 3709 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 3710 // Sub: 3711 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 3712 // 3713 // To get better codegen we can rewrite this by doing bitwise math on 3714 // the integers and extract the final sign bit at the end. So the 3715 // above becomes: 3716 // 3717 // Add: 3718 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0 3719 // Sub: 3720 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0 3721 // 3722 // NOTE: This is different than the expansion we do in expandSADDSUBO 3723 // because it is more costly to determine the RHS is > 0 for SSUBO with the 3724 // integers split. 3725 EVT VT = LHS.getValueType(); 3726 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); 3727 if (IsAdd) 3728 SignsMatch = DAG.getNOT(dl, SignsMatch, VT); 3729 3730 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum); 3731 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE); 3732 EVT OType = Node->getValueType(1); 3733 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT); 3734 } 3735 3736 // Use the calculated overflow everywhere. 3737 ReplaceValueWith(SDValue(Node, 1), Ovf); 3738 } 3739 3740 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 3741 SDValue &Lo, SDValue &Hi) { 3742 EVT VT = N->getValueType(0); 3743 SDLoc dl(N); 3744 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3745 3746 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 3747 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 3748 SplitInteger(Res.getValue(0), Lo, Hi); 3749 return; 3750 } 3751 3752 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3753 if (VT == MVT::i16) 3754 LC = RTLIB::SDIV_I16; 3755 else if (VT == MVT::i32) 3756 LC = RTLIB::SDIV_I32; 3757 else if (VT == MVT::i64) 3758 LC = RTLIB::SDIV_I64; 3759 else if (VT == MVT::i128) 3760 LC = RTLIB::SDIV_I128; 3761 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 3762 3763 TargetLowering::MakeLibCallOptions CallOptions; 3764 CallOptions.setSExt(true); 3765 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3766 } 3767 3768 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 3769 SDValue &Lo, SDValue &Hi) { 3770 EVT VT = N->getValueType(0); 3771 SDLoc dl(N); 3772 3773 // If we can emit an efficient shift operation, do so now. Check to see if 3774 // the RHS is a constant. 3775 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 3776 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); 3777 3778 // If we can determine that the high bit of the shift is zero or one, even if 3779 // the low bits are variable, emit this shift in an optimized form. 3780 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 3781 return; 3782 3783 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 3784 unsigned PartsOpc; 3785 if (N->getOpcode() == ISD::SHL) { 3786 PartsOpc = ISD::SHL_PARTS; 3787 } else if (N->getOpcode() == ISD::SRL) { 3788 PartsOpc = ISD::SRL_PARTS; 3789 } else { 3790 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 3791 PartsOpc = ISD::SRA_PARTS; 3792 } 3793 3794 // Next check to see if the target supports this SHL_PARTS operation or if it 3795 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for 3796 // size, but create a libcall instead. 3797 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3798 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 3799 const bool LegalOrCustom = 3800 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 3801 Action == TargetLowering::Custom; 3802 3803 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) { 3804 // Expand the subcomponents. 3805 SDValue LHSL, LHSH; 3806 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3807 EVT VT = LHSL.getValueType(); 3808 3809 // If the shift amount operand is coming from a vector legalization it may 3810 // have an illegal type. Fix that first by casting the operand, otherwise 3811 // the new SHL_PARTS operation would need further legalization. 3812 SDValue ShiftOp = N->getOperand(1); 3813 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 3814 assert(ShiftTy.getScalarSizeInBits() >= 3815 Log2_32_Ceil(VT.getScalarSizeInBits()) && 3816 "ShiftAmountTy is too small to cover the range of this type!"); 3817 if (ShiftOp.getValueType() != ShiftTy) 3818 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 3819 3820 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 3821 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 3822 Hi = Lo.getValue(1); 3823 return; 3824 } 3825 3826 // Otherwise, emit a libcall. 3827 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3828 bool isSigned; 3829 if (N->getOpcode() == ISD::SHL) { 3830 isSigned = false; /*sign irrelevant*/ 3831 if (VT == MVT::i16) 3832 LC = RTLIB::SHL_I16; 3833 else if (VT == MVT::i32) 3834 LC = RTLIB::SHL_I32; 3835 else if (VT == MVT::i64) 3836 LC = RTLIB::SHL_I64; 3837 else if (VT == MVT::i128) 3838 LC = RTLIB::SHL_I128; 3839 } else if (N->getOpcode() == ISD::SRL) { 3840 isSigned = false; 3841 if (VT == MVT::i16) 3842 LC = RTLIB::SRL_I16; 3843 else if (VT == MVT::i32) 3844 LC = RTLIB::SRL_I32; 3845 else if (VT == MVT::i64) 3846 LC = RTLIB::SRL_I64; 3847 else if (VT == MVT::i128) 3848 LC = RTLIB::SRL_I128; 3849 } else { 3850 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 3851 isSigned = true; 3852 if (VT == MVT::i16) 3853 LC = RTLIB::SRA_I16; 3854 else if (VT == MVT::i32) 3855 LC = RTLIB::SRA_I32; 3856 else if (VT == MVT::i64) 3857 LC = RTLIB::SRA_I64; 3858 else if (VT == MVT::i128) 3859 LC = RTLIB::SRA_I128; 3860 } 3861 3862 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 3863 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3864 TargetLowering::MakeLibCallOptions CallOptions; 3865 CallOptions.setSExt(isSigned); 3866 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3867 return; 3868 } 3869 3870 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 3871 llvm_unreachable("Unsupported shift!"); 3872 } 3873 3874 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 3875 SDValue &Lo, SDValue &Hi) { 3876 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3877 SDLoc dl(N); 3878 SDValue Op = N->getOperand(0); 3879 if (Op.getValueType().bitsLE(NVT)) { 3880 // The low part is sign extension of the input (degenerates to a copy). 3881 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 3882 // The high part is obtained by SRA'ing all but one of the bits of low part. 3883 unsigned LoSize = NVT.getSizeInBits(); 3884 Hi = DAG.getNode( 3885 ISD::SRA, dl, NVT, Lo, 3886 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); 3887 } else { 3888 // For example, extension of an i48 to an i64. The operand type necessarily 3889 // promotes to the result type, so will end up being expanded too. 3890 assert(getTypeAction(Op.getValueType()) == 3891 TargetLowering::TypePromoteInteger && 3892 "Only know how to promote this result!"); 3893 SDValue Res = GetPromotedInteger(Op); 3894 assert(Res.getValueType() == N->getValueType(0) && 3895 "Operand over promoted?"); 3896 // Split the promoted operand. This will simplify when it is expanded. 3897 SplitInteger(Res, Lo, Hi); 3898 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 3899 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 3900 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3901 ExcessBits))); 3902 } 3903 } 3904 3905 void DAGTypeLegalizer:: 3906 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 3907 SDLoc dl(N); 3908 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3909 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3910 3911 if (EVT.bitsLE(Lo.getValueType())) { 3912 // sext_inreg the low part if needed. 3913 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 3914 N->getOperand(1)); 3915 3916 // The high part gets the sign extension from the lo-part. This handles 3917 // things like sextinreg V:i64 from i8. 3918 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 3919 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, 3920 TLI.getPointerTy(DAG.getDataLayout()))); 3921 } else { 3922 // For example, extension of an i48 to an i64. Leave the low part alone, 3923 // sext_inreg the high part. 3924 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); 3925 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 3926 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3927 ExcessBits))); 3928 } 3929 } 3930 3931 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 3932 SDValue &Lo, SDValue &Hi) { 3933 EVT VT = N->getValueType(0); 3934 SDLoc dl(N); 3935 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3936 3937 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 3938 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 3939 SplitInteger(Res.getValue(1), Lo, Hi); 3940 return; 3941 } 3942 3943 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3944 if (VT == MVT::i16) 3945 LC = RTLIB::SREM_I16; 3946 else if (VT == MVT::i32) 3947 LC = RTLIB::SREM_I32; 3948 else if (VT == MVT::i64) 3949 LC = RTLIB::SREM_I64; 3950 else if (VT == MVT::i128) 3951 LC = RTLIB::SREM_I128; 3952 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 3953 3954 TargetLowering::MakeLibCallOptions CallOptions; 3955 CallOptions.setSExt(true); 3956 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 3957 } 3958 3959 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 3960 SDValue &Lo, SDValue &Hi) { 3961 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3962 SDLoc dl(N); 3963 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 3964 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), 3965 N->getOperand(0), 3966 DAG.getConstant(NVT.getSizeInBits(), dl, 3967 TLI.getPointerTy(DAG.getDataLayout()))); 3968 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 3969 } 3970 3971 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 3972 SDValue &Lo, SDValue &Hi) { 3973 EVT VT = N->getValueType(0); 3974 SDLoc dl(N); 3975 3976 if (N->getOpcode() == ISD::UMULO) { 3977 // This section expands the operation into the following sequence of 3978 // instructions. `iNh` here refers to a type which has half the bit width of 3979 // the type the original operation operated on. 3980 // 3981 // %0 = %LHS.HI != 0 && %RHS.HI != 0 3982 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) 3983 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) 3984 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) 3985 // %4 = add iNh %1.0, %2.0 as iN 3986 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH) 3987 // 3988 // %lo = %3.LO 3989 // %hi = %5.0 3990 // %ovf = %0 || %1.1 || %2.1 || %5.1 3991 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 3992 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; 3993 GetExpandedInteger(LHS, LHSLow, LHSHigh); 3994 GetExpandedInteger(RHS, RHSLow, RHSHigh); 3995 EVT HalfVT = LHSLow.getValueType(); 3996 EVT BitVT = N->getValueType(1); 3997 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT); 3998 3999 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT); 4000 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT, 4001 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE), 4002 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE)); 4003 4004 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow); 4005 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1)); 4006 4007 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow); 4008 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1)); 4009 4010 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two); 4011 4012 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not 4013 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this 4014 // operation recursively legalized?). 4015 // 4016 // Many backends understand this pattern and will convert into LOHI 4017 // themselves, if applicable. 4018 SDValue Three = DAG.getNode(ISD::MUL, dl, VT, 4019 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow), 4020 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow)); 4021 SplitInteger(Three, Lo, Hi); 4022 4023 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum); 4024 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1)); 4025 ReplaceValueWith(SDValue(N, 1), Overflow); 4026 return; 4027 } 4028 4029 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 4030 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 4031 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 4032 4033 // Replace this with a libcall that will check overflow. 4034 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4035 if (VT == MVT::i32) 4036 LC = RTLIB::MULO_I32; 4037 else if (VT == MVT::i64) 4038 LC = RTLIB::MULO_I64; 4039 else if (VT == MVT::i128) 4040 LC = RTLIB::MULO_I128; 4041 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); 4042 4043 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 4044 // Temporary for the overflow value, default it to zero. 4045 SDValue Chain = 4046 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, 4047 MachinePointerInfo()); 4048 4049 TargetLowering::ArgListTy Args; 4050 TargetLowering::ArgListEntry Entry; 4051 for (const SDValue &Op : N->op_values()) { 4052 EVT ArgVT = Op.getValueType(); 4053 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 4054 Entry.Node = Op; 4055 Entry.Ty = ArgTy; 4056 Entry.IsSExt = true; 4057 Entry.IsZExt = false; 4058 Args.push_back(Entry); 4059 } 4060 4061 // Also pass the address of the overflow check. 4062 Entry.Node = Temp; 4063 Entry.Ty = PtrTy->getPointerTo(); 4064 Entry.IsSExt = true; 4065 Entry.IsZExt = false; 4066 Args.push_back(Entry); 4067 4068 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 4069 4070 TargetLowering::CallLoweringInfo CLI(DAG); 4071 CLI.setDebugLoc(dl) 4072 .setChain(Chain) 4073 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) 4074 .setSExtResult(); 4075 4076 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 4077 4078 SplitInteger(CallInfo.first, Lo, Hi); 4079 SDValue Temp2 = 4080 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); 4081 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 4082 DAG.getConstant(0, dl, PtrVT), 4083 ISD::SETNE); 4084 // Use the overflow from the libcall everywhere. 4085 ReplaceValueWith(SDValue(N, 1), Ofl); 4086 } 4087 4088 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 4089 SDValue &Lo, SDValue &Hi) { 4090 EVT VT = N->getValueType(0); 4091 SDLoc dl(N); 4092 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4093 4094 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4095 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4096 SplitInteger(Res.getValue(0), Lo, Hi); 4097 return; 4098 } 4099 4100 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4101 if (VT == MVT::i16) 4102 LC = RTLIB::UDIV_I16; 4103 else if (VT == MVT::i32) 4104 LC = RTLIB::UDIV_I32; 4105 else if (VT == MVT::i64) 4106 LC = RTLIB::UDIV_I64; 4107 else if (VT == MVT::i128) 4108 LC = RTLIB::UDIV_I128; 4109 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 4110 4111 TargetLowering::MakeLibCallOptions CallOptions; 4112 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4113 } 4114 4115 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 4116 SDValue &Lo, SDValue &Hi) { 4117 EVT VT = N->getValueType(0); 4118 SDLoc dl(N); 4119 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4120 4121 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4122 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4123 SplitInteger(Res.getValue(1), Lo, Hi); 4124 return; 4125 } 4126 4127 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4128 if (VT == MVT::i16) 4129 LC = RTLIB::UREM_I16; 4130 else if (VT == MVT::i32) 4131 LC = RTLIB::UREM_I32; 4132 else if (VT == MVT::i64) 4133 LC = RTLIB::UREM_I64; 4134 else if (VT == MVT::i128) 4135 LC = RTLIB::UREM_I128; 4136 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 4137 4138 TargetLowering::MakeLibCallOptions CallOptions; 4139 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4140 } 4141 4142 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 4143 SDValue &Lo, SDValue &Hi) { 4144 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4145 SDLoc dl(N); 4146 SDValue Op = N->getOperand(0); 4147 if (Op.getValueType().bitsLE(NVT)) { 4148 // The low part is zero extension of the input (degenerates to a copy). 4149 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 4150 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. 4151 } else { 4152 // For example, extension of an i48 to an i64. The operand type necessarily 4153 // promotes to the result type, so will end up being expanded too. 4154 assert(getTypeAction(Op.getValueType()) == 4155 TargetLowering::TypePromoteInteger && 4156 "Only know how to promote this result!"); 4157 SDValue Res = GetPromotedInteger(Op); 4158 assert(Res.getValueType() == N->getValueType(0) && 4159 "Operand over promoted?"); 4160 // Split the promoted operand. This will simplify when it is expanded. 4161 SplitInteger(Res, Lo, Hi); 4162 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 4163 Hi = DAG.getZeroExtendInReg(Hi, dl, 4164 EVT::getIntegerVT(*DAG.getContext(), 4165 ExcessBits)); 4166 } 4167 } 4168 4169 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 4170 SDValue &Lo, SDValue &Hi) { 4171 SDLoc dl(N); 4172 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 4173 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 4174 SDValue Zero = DAG.getConstant(0, dl, VT); 4175 SDValue Swap = DAG.getAtomicCmpSwap( 4176 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 4177 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 4178 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); 4179 4180 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 4181 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 4182 } 4183 4184 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, 4185 SDValue &Lo, SDValue &Hi) { 4186 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate 4187 // both halves independently. 4188 SDValue Res = TLI.expandVecReduce(N, DAG); 4189 SplitInteger(Res, Lo, Hi); 4190 } 4191 4192 void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, 4193 SDValue &Lo, SDValue &Hi) { 4194 // Lower the rotate to shifts and ORs which can be expanded. 4195 SDValue Res; 4196 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG); 4197 SplitInteger(Res, Lo, Hi); 4198 } 4199 4200 void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, 4201 SDValue &Lo, SDValue &Hi) { 4202 // Lower the funnel shift to shifts and ORs which can be expanded. 4203 SDValue Res; 4204 TLI.expandFunnelShift(N, Res, DAG); 4205 SplitInteger(Res, Lo, Hi); 4206 } 4207 4208 void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, 4209 SDValue &Hi) { 4210 EVT VT = N->getValueType(0); 4211 EVT HalfVT = 4212 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2); 4213 SDLoc dl(N); 4214 4215 // We assume VSCALE(1) fits into a legal integer. 4216 APInt One(HalfVT.getSizeInBits(), 1); 4217 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One); 4218 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase); 4219 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0)); 4220 SplitInteger(Res, Lo, Hi); 4221 } 4222 4223 //===----------------------------------------------------------------------===// 4224 // Integer Operand Expansion 4225 //===----------------------------------------------------------------------===// 4226 4227 /// ExpandIntegerOperand - This method is called when the specified operand of 4228 /// the specified node is found to need expansion. At this point, all of the 4229 /// result types of the node are known to be legal, but other operands of the 4230 /// node may need promotion or expansion as well as the specified one. 4231 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 4232 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); 4233 dbgs() << "\n"); 4234 SDValue Res = SDValue(); 4235 4236 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 4237 return false; 4238 4239 switch (N->getOpcode()) { 4240 default: 4241 #ifndef NDEBUG 4242 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 4243 N->dump(&DAG); dbgs() << "\n"; 4244 #endif 4245 report_fatal_error("Do not know how to expand this operator's operand!"); 4246 4247 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 4248 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 4249 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 4250 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 4251 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 4252 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 4253 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break; 4254 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 4255 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 4256 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; 4257 case ISD::STRICT_SINT_TO_FP: 4258 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 4259 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 4260 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 4261 case ISD::STRICT_UINT_TO_FP: 4262 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 4263 4264 case ISD::SHL: 4265 case ISD::SRA: 4266 case ISD::SRL: 4267 case ISD::ROTL: 4268 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 4269 case ISD::RETURNADDR: 4270 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 4271 4272 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 4273 } 4274 4275 // If the result is null, the sub-method took care of registering results etc. 4276 if (!Res.getNode()) return false; 4277 4278 // If the result is N, the sub-method updated N in place. Tell the legalizer 4279 // core about this. 4280 if (Res.getNode() == N) 4281 return true; 4282 4283 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 4284 "Invalid operand expansion"); 4285 4286 ReplaceValueWith(SDValue(N, 0), Res); 4287 return false; 4288 } 4289 4290 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 4291 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 4292 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 4293 SDValue &NewRHS, 4294 ISD::CondCode &CCCode, 4295 const SDLoc &dl) { 4296 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 4297 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 4298 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 4299 4300 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 4301 if (RHSLo == RHSHi) { 4302 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 4303 if (RHSCST->isAllOnesValue()) { 4304 // Equality comparison to -1. 4305 NewLHS = DAG.getNode(ISD::AND, dl, 4306 LHSLo.getValueType(), LHSLo, LHSHi); 4307 NewRHS = RHSLo; 4308 return; 4309 } 4310 } 4311 } 4312 4313 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 4314 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 4315 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 4316 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); 4317 return; 4318 } 4319 4320 // If this is a comparison of the sign bit, just look at the top part. 4321 // X > -1, x < 0 4322 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 4323 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 4324 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 4325 NewLHS = LHSHi; 4326 NewRHS = RHSHi; 4327 return; 4328 } 4329 4330 // FIXME: This generated code sucks. 4331 ISD::CondCode LowCC; 4332 switch (CCCode) { 4333 default: llvm_unreachable("Unknown integer setcc!"); 4334 case ISD::SETLT: 4335 case ISD::SETULT: LowCC = ISD::SETULT; break; 4336 case ISD::SETGT: 4337 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 4338 case ISD::SETLE: 4339 case ISD::SETULE: LowCC = ISD::SETULE; break; 4340 case ISD::SETGE: 4341 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 4342 } 4343 4344 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison 4345 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands 4346 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; 4347 4348 // NOTE: on targets without efficient SELECT of bools, we can always use 4349 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 4350 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 4351 nullptr); 4352 SDValue LoCmp, HiCmp; 4353 if (TLI.isTypeLegal(LHSLo.getValueType()) && 4354 TLI.isTypeLegal(RHSLo.getValueType())) 4355 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, 4356 RHSLo, LowCC, false, DagCombineInfo, dl); 4357 if (!LoCmp.getNode()) 4358 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 4359 RHSLo, LowCC); 4360 if (TLI.isTypeLegal(LHSHi.getValueType()) && 4361 TLI.isTypeLegal(RHSHi.getValueType())) 4362 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, 4363 RHSHi, CCCode, false, DagCombineInfo, dl); 4364 if (!HiCmp.getNode()) 4365 HiCmp = 4366 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), 4367 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 4368 4369 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); 4370 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); 4371 4372 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 4373 CCCode == ISD::SETUGE || CCCode == ISD::SETULE); 4374 4375 if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) || 4376 (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) || 4377 (LoCmpC && LoCmpC->isNullValue())))) { 4378 // For LE / GE, if high part is known false, ignore the low part. 4379 // For LT / GT: if low part is known false, return the high part. 4380 // if high part is known true, ignore the low part. 4381 NewLHS = HiCmp; 4382 NewRHS = SDValue(); 4383 return; 4384 } 4385 4386 if (LHSHi == RHSHi) { 4387 // Comparing the low bits is enough. 4388 NewLHS = LoCmp; 4389 NewRHS = SDValue(); 4390 return; 4391 } 4392 4393 // Lower with SETCCCARRY if the target supports it. 4394 EVT HiVT = LHSHi.getValueType(); 4395 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); 4396 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); 4397 4398 // FIXME: Make all targets support this, then remove the other lowering. 4399 if (HasSETCCCARRY) { 4400 // SETCCCARRY can detect < and >= directly. For > and <=, flip 4401 // operands and condition code. 4402 bool FlipOperands = false; 4403 switch (CCCode) { 4404 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; 4405 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; 4406 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; 4407 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; 4408 default: break; 4409 } 4410 if (FlipOperands) { 4411 std::swap(LHSLo, RHSLo); 4412 std::swap(LHSHi, RHSHi); 4413 } 4414 // Perform a wide subtraction, feeding the carry from the low part into 4415 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high 4416 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is 4417 // zero or positive iff LHS >= RHS. 4418 EVT LoVT = LHSLo.getValueType(); 4419 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT)); 4420 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo); 4421 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT), 4422 LHSHi, RHSHi, LowCmp.getValue(1), 4423 DAG.getCondCode(CCCode)); 4424 NewLHS = Res; 4425 NewRHS = SDValue(); 4426 return; 4427 } 4428 4429 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, 4430 false, DagCombineInfo, dl); 4431 if (!NewLHS.getNode()) 4432 NewLHS = 4433 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); 4434 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); 4435 NewRHS = SDValue(); 4436 } 4437 4438 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 4439 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 4440 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 4441 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 4442 4443 // If ExpandSetCCOperands returned a scalar, we need to compare the result 4444 // against zero to select between true and false values. 4445 if (!NewRHS.getNode()) { 4446 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 4447 CCCode = ISD::SETNE; 4448 } 4449 4450 // Update N to have the operands specified. 4451 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 4452 DAG.getCondCode(CCCode), NewLHS, NewRHS, 4453 N->getOperand(4)), 0); 4454 } 4455 4456 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 4457 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 4458 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 4459 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 4460 4461 // If ExpandSetCCOperands returned a scalar, we need to compare the result 4462 // against zero to select between true and false values. 4463 if (!NewRHS.getNode()) { 4464 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 4465 CCCode = ISD::SETNE; 4466 } 4467 4468 // Update N to have the operands specified. 4469 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 4470 N->getOperand(2), N->getOperand(3), 4471 DAG.getCondCode(CCCode)), 0); 4472 } 4473 4474 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 4475 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 4476 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 4477 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 4478 4479 // If ExpandSetCCOperands returned a scalar, use it. 4480 if (!NewRHS.getNode()) { 4481 assert(NewLHS.getValueType() == N->getValueType(0) && 4482 "Unexpected setcc expansion!"); 4483 return NewLHS; 4484 } 4485 4486 // Otherwise, update N to have the operands specified. 4487 return SDValue( 4488 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); 4489 } 4490 4491 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { 4492 SDValue LHS = N->getOperand(0); 4493 SDValue RHS = N->getOperand(1); 4494 SDValue Carry = N->getOperand(2); 4495 SDValue Cond = N->getOperand(3); 4496 SDLoc dl = SDLoc(N); 4497 4498 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 4499 GetExpandedInteger(LHS, LHSLo, LHSHi); 4500 GetExpandedInteger(RHS, RHSLo, RHSHi); 4501 4502 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high. 4503 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); 4504 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry); 4505 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, 4506 LowCmp.getValue(1), Cond); 4507 } 4508 4509 SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) { 4510 // Split the operand and replace with SPLAT_VECTOR_PARTS. 4511 SDValue Lo, Hi; 4512 GetExpandedInteger(N->getOperand(0), Lo, Hi); 4513 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo, 4514 Hi); 4515 } 4516 4517 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 4518 // The value being shifted is legal, but the shift amount is too big. 4519 // It follows that either the result of the shift is undefined, or the 4520 // upper half of the shift amount is zero. Just use the lower half. 4521 SDValue Lo, Hi; 4522 GetExpandedInteger(N->getOperand(1), Lo, Hi); 4523 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 4524 } 4525 4526 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 4527 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 4528 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 4529 // constant to valid type. 4530 SDValue Lo, Hi; 4531 GetExpandedInteger(N->getOperand(0), Lo, Hi); 4532 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 4533 } 4534 4535 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 4536 bool IsStrict = N->isStrictFPOpcode(); 4537 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 4538 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 4539 EVT DstVT = N->getValueType(0); 4540 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 4541 assert(LC != RTLIB::UNKNOWN_LIBCALL && 4542 "Don't know how to expand this SINT_TO_FP!"); 4543 TargetLowering::MakeLibCallOptions CallOptions; 4544 CallOptions.setSExt(true); 4545 std::pair<SDValue, SDValue> Tmp = 4546 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 4547 4548 if (!IsStrict) 4549 return Tmp.first; 4550 4551 ReplaceValueWith(SDValue(N, 1), Tmp.second); 4552 ReplaceValueWith(SDValue(N, 0), Tmp.first); 4553 return SDValue(); 4554 } 4555 4556 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 4557 if (N->isAtomic()) { 4558 // It's typical to have larger CAS than atomic store instructions. 4559 SDLoc dl(N); 4560 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 4561 N->getMemoryVT(), 4562 N->getOperand(0), N->getOperand(2), 4563 N->getOperand(1), 4564 N->getMemOperand()); 4565 return Swap.getValue(1); 4566 } 4567 if (ISD::isNormalStore(N)) 4568 return ExpandOp_NormalStore(N, OpNo); 4569 4570 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 4571 assert(OpNo == 1 && "Can only expand the stored value so far"); 4572 4573 EVT VT = N->getOperand(1).getValueType(); 4574 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4575 SDValue Ch = N->getChain(); 4576 SDValue Ptr = N->getBasePtr(); 4577 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 4578 AAMDNodes AAInfo = N->getAAInfo(); 4579 SDLoc dl(N); 4580 SDValue Lo, Hi; 4581 4582 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 4583 4584 if (N->getMemoryVT().bitsLE(NVT)) { 4585 GetExpandedInteger(N->getValue(), Lo, Hi); 4586 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 4587 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags, 4588 AAInfo); 4589 } 4590 4591 if (DAG.getDataLayout().isLittleEndian()) { 4592 // Little-endian - low bits are at low addresses. 4593 GetExpandedInteger(N->getValue(), Lo, Hi); 4594 4595 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 4596 N->getOriginalAlign(), MMOFlags, AAInfo); 4597 4598 unsigned ExcessBits = 4599 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 4600 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 4601 4602 // Increment the pointer to the other half. 4603 unsigned IncrementSize = NVT.getSizeInBits()/8; 4604 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 4605 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 4606 N->getPointerInfo().getWithOffset(IncrementSize), 4607 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo); 4608 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 4609 } 4610 4611 // Big-endian - high bits are at low addresses. Favor aligned stores at 4612 // the cost of some bit-fiddling. 4613 GetExpandedInteger(N->getValue(), Lo, Hi); 4614 4615 EVT ExtVT = N->getMemoryVT(); 4616 unsigned EBytes = ExtVT.getStoreSize(); 4617 unsigned IncrementSize = NVT.getSizeInBits()/8; 4618 unsigned ExcessBits = (EBytes - IncrementSize)*8; 4619 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 4620 ExtVT.getSizeInBits() - ExcessBits); 4621 4622 if (ExcessBits < NVT.getSizeInBits()) { 4623 // Transfer high bits from the top of Lo to the bottom of Hi. 4624 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 4625 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 4626 TLI.getPointerTy(DAG.getDataLayout()))); 4627 Hi = DAG.getNode( 4628 ISD::OR, dl, NVT, Hi, 4629 DAG.getNode(ISD::SRL, dl, NVT, Lo, 4630 DAG.getConstant(ExcessBits, dl, 4631 TLI.getPointerTy(DAG.getDataLayout())))); 4632 } 4633 4634 // Store both the high bits and maybe some of the low bits. 4635 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, 4636 N->getOriginalAlign(), MMOFlags, AAInfo); 4637 4638 // Increment the pointer to the other half. 4639 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 4640 // Store the lowest ExcessBits bits in the second half. 4641 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 4642 N->getPointerInfo().getWithOffset(IncrementSize), 4643 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 4644 N->getOriginalAlign(), MMOFlags, AAInfo); 4645 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 4646 } 4647 4648 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 4649 SDValue InL, InH; 4650 GetExpandedInteger(N->getOperand(0), InL, InH); 4651 // Just truncate the low part of the source. 4652 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 4653 } 4654 4655 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 4656 bool IsStrict = N->isStrictFPOpcode(); 4657 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 4658 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 4659 EVT DstVT = N->getValueType(0); 4660 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT); 4661 assert(LC != RTLIB::UNKNOWN_LIBCALL && 4662 "Don't know how to expand this UINT_TO_FP!"); 4663 TargetLowering::MakeLibCallOptions CallOptions; 4664 CallOptions.setSExt(true); 4665 std::pair<SDValue, SDValue> Tmp = 4666 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 4667 4668 if (!IsStrict) 4669 return Tmp.first; 4670 4671 ReplaceValueWith(SDValue(N, 1), Tmp.second); 4672 ReplaceValueWith(SDValue(N, 0), Tmp.first); 4673 return SDValue(); 4674 } 4675 4676 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 4677 SDLoc dl(N); 4678 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 4679 cast<AtomicSDNode>(N)->getMemoryVT(), 4680 N->getOperand(0), 4681 N->getOperand(1), N->getOperand(2), 4682 cast<AtomicSDNode>(N)->getMemOperand()); 4683 return Swap.getValue(1); 4684 } 4685 4686 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { 4687 SDLoc dl(N); 4688 4689 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4690 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 4691 EVT OutVT = V0.getValueType(); 4692 4693 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2)); 4694 } 4695 4696 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 4697 4698 EVT OutVT = N->getValueType(0); 4699 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4700 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4701 EVT NOutVTElem = NOutVT.getVectorElementType(); 4702 4703 SDLoc dl(N); 4704 SDValue BaseIdx = N->getOperand(1); 4705 4706 // TODO: We may be able to use this for types other than scalable 4707 // vectors and fix those tests that expect BUILD_VECTOR to be used 4708 if (OutVT.isScalableVector()) { 4709 SDValue InOp0 = N->getOperand(0); 4710 EVT InVT = InOp0.getValueType(); 4711 4712 // Promote operands and see if this is handled by target lowering, 4713 // Otherwise, use the BUILD_VECTOR approach below 4714 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 4715 // Collect the (promoted) operands 4716 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx }; 4717 4718 EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); 4719 assert(PromEltVT.bitsLE(NOutVTElem) && 4720 "Promoted operand has an element type greater than result"); 4721 4722 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT); 4723 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops); 4724 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); 4725 } 4726 } 4727 4728 if (OutVT.isScalableVector()) 4729 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR"); 4730 4731 SDValue InOp0 = N->getOperand(0); 4732 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) 4733 InOp0 = GetPromotedInteger(N->getOperand(0)); 4734 4735 EVT InVT = InOp0.getValueType(); 4736 4737 unsigned OutNumElems = OutVT.getVectorNumElements(); 4738 SmallVector<SDValue, 8> Ops; 4739 Ops.reserve(OutNumElems); 4740 for (unsigned i = 0; i != OutNumElems; ++i) { 4741 4742 // Extract the element from the original vector. 4743 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 4744 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); 4745 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 4746 InVT.getVectorElementType(), N->getOperand(0), Index); 4747 4748 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); 4749 // Insert the converted element to the new vector. 4750 Ops.push_back(Op); 4751 } 4752 4753 return DAG.getBuildVector(NOutVT, dl, Ops); 4754 } 4755 4756 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) { 4757 EVT OutVT = N->getValueType(0); 4758 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4759 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4760 4761 SDLoc dl(N); 4762 SDValue Vec = N->getOperand(0); 4763 SDValue SubVec = N->getOperand(1); 4764 SDValue Idx = N->getOperand(2); 4765 4766 EVT SubVecVT = SubVec.getValueType(); 4767 EVT NSubVT = 4768 EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(), 4769 SubVecVT.getVectorElementCount()); 4770 4771 Vec = GetPromotedInteger(Vec); 4772 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec); 4773 4774 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx); 4775 } 4776 4777 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) { 4778 SDLoc dl(N); 4779 4780 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4781 EVT OutVT = V0.getValueType(); 4782 4783 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0); 4784 } 4785 4786 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 4787 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 4788 EVT VT = N->getValueType(0); 4789 SDLoc dl(N); 4790 4791 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); 4792 4793 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4794 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 4795 EVT OutVT = V0.getValueType(); 4796 4797 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); 4798 } 4799 4800 4801 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 4802 EVT OutVT = N->getValueType(0); 4803 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4804 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4805 unsigned NumElems = N->getNumOperands(); 4806 EVT NOutVTElem = NOutVT.getVectorElementType(); 4807 4808 SDLoc dl(N); 4809 4810 SmallVector<SDValue, 8> Ops; 4811 Ops.reserve(NumElems); 4812 for (unsigned i = 0; i != NumElems; ++i) { 4813 SDValue Op; 4814 // BUILD_VECTOR integer operand types are allowed to be larger than the 4815 // result's element type. This may still be true after the promotion. For 4816 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 4817 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 4818 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem)) 4819 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); 4820 else 4821 Op = N->getOperand(i); 4822 Ops.push_back(Op); 4823 } 4824 4825 return DAG.getBuildVector(NOutVT, dl, Ops); 4826 } 4827 4828 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { 4829 4830 SDLoc dl(N); 4831 4832 assert(!N->getOperand(0).getValueType().isVector() && 4833 "Input must be a scalar"); 4834 4835 EVT OutVT = N->getValueType(0); 4836 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4837 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4838 EVT NOutVTElem = NOutVT.getVectorElementType(); 4839 4840 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); 4841 4842 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); 4843 } 4844 4845 SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) { 4846 SDLoc dl(N); 4847 4848 SDValue SplatVal = N->getOperand(0); 4849 4850 assert(!SplatVal.getValueType().isVector() && "Input must be a scalar"); 4851 4852 EVT OutVT = N->getValueType(0); 4853 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4854 assert(NOutVT.isVector() && "Type must be promoted to a vector type"); 4855 EVT NOutElemVT = NOutVT.getVectorElementType(); 4856 4857 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal); 4858 4859 return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op); 4860 } 4861 4862 SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) { 4863 SDLoc dl(N); 4864 EVT OutVT = N->getValueType(0); 4865 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4866 assert(NOutVT.isVector() && "Type must be promoted to a vector type"); 4867 APInt StepVal = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue(); 4868 return DAG.getStepVector(dl, NOutVT, 4869 StepVal.sext(NOutVT.getScalarSizeInBits())); 4870 } 4871 4872 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 4873 SDLoc dl(N); 4874 4875 EVT OutVT = N->getValueType(0); 4876 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4877 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4878 4879 EVT OutElemTy = NOutVT.getVectorElementType(); 4880 4881 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 4882 unsigned NumOutElem = NOutVT.getVectorNumElements(); 4883 unsigned NumOperands = N->getNumOperands(); 4884 assert(NumElem * NumOperands == NumOutElem && 4885 "Unexpected number of elements"); 4886 4887 // Take the elements from the first vector. 4888 SmallVector<SDValue, 8> Ops(NumOutElem); 4889 for (unsigned i = 0; i < NumOperands; ++i) { 4890 SDValue Op = N->getOperand(i); 4891 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) 4892 Op = GetPromotedInteger(Op); 4893 EVT SclrTy = Op.getValueType().getVectorElementType(); 4894 assert(NumElem == Op.getValueType().getVectorNumElements() && 4895 "Unexpected number of elements"); 4896 4897 for (unsigned j = 0; j < NumElem; ++j) { 4898 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, 4899 DAG.getVectorIdxConstant(j, dl)); 4900 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); 4901 } 4902 } 4903 4904 return DAG.getBuildVector(NOutVT, dl, Ops); 4905 } 4906 4907 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { 4908 EVT VT = N->getValueType(0); 4909 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4910 assert(NVT.isVector() && "This type must be promoted to a vector type"); 4911 4912 SDLoc dl(N); 4913 4914 // For operands whose TypeAction is to promote, extend the promoted node 4915 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion 4916 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to 4917 // type.. 4918 if (getTypeAction(N->getOperand(0).getValueType()) 4919 == TargetLowering::TypePromoteInteger) { 4920 SDValue Promoted; 4921 4922 switch(N->getOpcode()) { 4923 case ISD::SIGN_EXTEND_VECTOR_INREG: 4924 Promoted = SExtPromotedInteger(N->getOperand(0)); 4925 break; 4926 case ISD::ZERO_EXTEND_VECTOR_INREG: 4927 Promoted = ZExtPromotedInteger(N->getOperand(0)); 4928 break; 4929 case ISD::ANY_EXTEND_VECTOR_INREG: 4930 Promoted = GetPromotedInteger(N->getOperand(0)); 4931 break; 4932 default: 4933 llvm_unreachable("Node has unexpected Opcode"); 4934 } 4935 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); 4936 } 4937 4938 // Directly extend to the appropriate transform-to type. 4939 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 4940 } 4941 4942 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 4943 EVT OutVT = N->getValueType(0); 4944 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 4945 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 4946 4947 EVT NOutVTElem = NOutVT.getVectorElementType(); 4948 4949 SDLoc dl(N); 4950 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4951 4952 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 4953 NOutVTElem, N->getOperand(1)); 4954 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 4955 V0, ConvElem, N->getOperand(2)); 4956 } 4957 4958 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { 4959 // The VECREDUCE result size may be larger than the element size, so 4960 // we can simply change the result type. 4961 SDLoc dl(N); 4962 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4963 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 4964 } 4965 4966 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 4967 SDLoc dl(N); 4968 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4969 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, 4970 TLI.getVectorIdxTy(DAG.getDataLayout())); 4971 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 4972 V0->getValueType(0).getScalarType(), V0, V1); 4973 4974 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 4975 // element types. If this is the case then we need to expand the outgoing 4976 // value and not truncate it. 4977 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 4978 } 4979 4980 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { 4981 SDLoc dl(N); 4982 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 4983 MVT InVT = V0.getValueType().getSimpleVT(); 4984 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), 4985 N->getValueType(0).getVectorNumElements()); 4986 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); 4987 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 4988 } 4989 4990 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 4991 SDLoc dl(N); 4992 4993 EVT ResVT = N->getValueType(0); 4994 unsigned NumElems = N->getNumOperands(); 4995 4996 if (ResVT.isScalableVector()) { 4997 SDValue ResVec = DAG.getUNDEF(ResVT); 4998 4999 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { 5000 SDValue Op = N->getOperand(OpIdx); 5001 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); 5002 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op, 5003 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl)); 5004 } 5005 5006 return ResVec; 5007 } 5008 5009 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 5010 5011 SmallVector<SDValue, 8> NewOps; 5012 NewOps.reserve(NumElems); 5013 5014 // For each incoming vector 5015 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 5016 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 5017 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 5018 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 5019 5020 for (unsigned i=0; i<NumElem; ++i) { 5021 // Extract element from incoming vector 5022 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, 5023 DAG.getVectorIdxConstant(i, dl)); 5024 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 5025 NewOps.push_back(Tr); 5026 } 5027 } 5028 5029 return DAG.getBuildVector(N->getValueType(0), dl, NewOps); 5030 } 5031