1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements integer type expansion and promotion for LegalizeTypes. 10 // Promotion is the act of changing a computation in an illegal type into a 11 // computation in a larger type. For example, implementing i8 arithmetic in an 12 // i32 register (often needed on powerpc). 13 // Expansion is the act of changing a computation in an illegal type into a 14 // computation in two identical registers of a smaller type. For example, 15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit 16 // targets). 17 // 18 //===----------------------------------------------------------------------===// 19 20 #include "LegalizeTypes.h" 21 #include "llvm/Analysis/TargetLibraryInfo.h" 22 #include "llvm/CodeGen/StackMaps.h" 23 #include "llvm/IR/DerivedTypes.h" 24 #include "llvm/Support/ErrorHandling.h" 25 #include "llvm/Support/KnownBits.h" 26 #include "llvm/Support/raw_ostream.h" 27 #include <algorithm> 28 using namespace llvm; 29 30 #define DEBUG_TYPE "legalize-types" 31 32 //===----------------------------------------------------------------------===// 33 // Integer Result Promotion 34 //===----------------------------------------------------------------------===// 35 36 /// PromoteIntegerResult - This method is called when a result of a node is 37 /// found to be in need of promotion to a larger type. At this point, the node 38 /// may also have invalid operands or may have other results that need 39 /// expansion, we just know that (at least) one result needs promotion. 40 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 41 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); 42 dbgs() << "\n"); 43 SDValue Res = SDValue(); 44 45 // See if the target wants to custom expand this node. 46 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 47 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 48 return; 49 } 50 51 switch (N->getOpcode()) { 52 default: 53 #ifndef NDEBUG 54 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 55 N->dump(&DAG); dbgs() << "\n"; 56 #endif 57 llvm_unreachable("Do not know how to promote this operator!"); 58 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 59 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 60 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 61 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 62 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; 63 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 64 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 65 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 66 case ISD::CTLZ_ZERO_UNDEF: 67 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 68 case ISD::PARITY: 69 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; 70 case ISD::CTTZ_ZERO_UNDEF: 71 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 72 case ISD::EXTRACT_VECTOR_ELT: 73 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 74 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 75 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); 76 break; 77 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); 78 break; 79 case ISD::SELECT: 80 case ISD::VSELECT: 81 case ISD::VP_SELECT: 82 case ISD::VP_MERGE: 83 Res = PromoteIntRes_Select(N); 84 break; 85 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 86 case ISD::STRICT_FSETCC: 87 case ISD::STRICT_FSETCCS: 88 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 89 case ISD::SMIN: 90 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; 91 case ISD::UMIN: 92 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; 93 94 case ISD::SHL: 95 case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break; 96 case ISD::SIGN_EXTEND_INREG: 97 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 98 case ISD::SRA: 99 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break; 100 case ISD::SRL: 101 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break; 102 case ISD::VP_TRUNCATE: 103 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 104 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 105 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 106 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; 107 108 case ISD::EXTRACT_SUBVECTOR: 109 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 110 case ISD::INSERT_SUBVECTOR: 111 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break; 112 case ISD::VECTOR_REVERSE: 113 Res = PromoteIntRes_VECTOR_REVERSE(N); break; 114 case ISD::VECTOR_SHUFFLE: 115 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 116 case ISD::VECTOR_SPLICE: 117 Res = PromoteIntRes_VECTOR_SPLICE(N); break; 118 case ISD::INSERT_VECTOR_ELT: 119 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 120 case ISD::BUILD_VECTOR: 121 Res = PromoteIntRes_BUILD_VECTOR(N); 122 break; 123 case ISD::SPLAT_VECTOR: 124 case ISD::SCALAR_TO_VECTOR: 125 Res = PromoteIntRes_ScalarOp(N); 126 break; 127 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break; 128 case ISD::CONCAT_VECTORS: 129 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 130 131 case ISD::ANY_EXTEND_VECTOR_INREG: 132 case ISD::SIGN_EXTEND_VECTOR_INREG: 133 case ISD::ZERO_EXTEND_VECTOR_INREG: 134 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; 135 136 case ISD::SIGN_EXTEND: 137 case ISD::ZERO_EXTEND: 138 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 139 140 case ISD::VP_FPTOSI: 141 case ISD::VP_FPTOUI: 142 case ISD::STRICT_FP_TO_SINT: 143 case ISD::STRICT_FP_TO_UINT: 144 case ISD::FP_TO_SINT: 145 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 146 147 case ISD::FP_TO_SINT_SAT: 148 case ISD::FP_TO_UINT_SAT: 149 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; 150 151 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break; 152 153 case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break; 154 155 case ISD::AND: 156 case ISD::OR: 157 case ISD::XOR: 158 case ISD::ADD: 159 case ISD::SUB: 160 case ISD::MUL: 161 case ISD::VP_AND: 162 case ISD::VP_OR: 163 case ISD::VP_XOR: 164 case ISD::VP_ADD: 165 case ISD::VP_SUB: 166 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 167 168 case ISD::SDIV: 169 case ISD::SREM: 170 case ISD::VP_SDIV: 171 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; 172 173 case ISD::UDIV: 174 case ISD::UREM: 175 case ISD::VP_UDIV: 176 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; 177 178 case ISD::SADDO: 179 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 180 case ISD::UADDO: 181 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 182 case ISD::SMULO: 183 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 184 185 case ISD::ADDE: 186 case ISD::SUBE: 187 case ISD::ADDCARRY: 188 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break; 189 190 case ISD::SADDO_CARRY: 191 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; 192 193 case ISD::SADDSAT: 194 case ISD::UADDSAT: 195 case ISD::SSUBSAT: 196 case ISD::USUBSAT: 197 case ISD::SSHLSAT: 198 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break; 199 200 case ISD::SMULFIX: 201 case ISD::SMULFIXSAT: 202 case ISD::UMULFIX: 203 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; 204 205 case ISD::SDIVFIX: 206 case ISD::SDIVFIXSAT: 207 case ISD::UDIVFIX: 208 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; 209 210 case ISD::ABS: Res = PromoteIntRes_ABS(N); break; 211 212 case ISD::ATOMIC_LOAD: 213 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 214 215 case ISD::ATOMIC_LOAD_ADD: 216 case ISD::ATOMIC_LOAD_SUB: 217 case ISD::ATOMIC_LOAD_AND: 218 case ISD::ATOMIC_LOAD_CLR: 219 case ISD::ATOMIC_LOAD_OR: 220 case ISD::ATOMIC_LOAD_XOR: 221 case ISD::ATOMIC_LOAD_NAND: 222 case ISD::ATOMIC_LOAD_MIN: 223 case ISD::ATOMIC_LOAD_MAX: 224 case ISD::ATOMIC_LOAD_UMIN: 225 case ISD::ATOMIC_LOAD_UMAX: 226 case ISD::ATOMIC_SWAP: 227 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 228 229 case ISD::ATOMIC_CMP_SWAP: 230 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 231 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 232 break; 233 234 case ISD::VECREDUCE_ADD: 235 case ISD::VECREDUCE_MUL: 236 case ISD::VECREDUCE_AND: 237 case ISD::VECREDUCE_OR: 238 case ISD::VECREDUCE_XOR: 239 case ISD::VECREDUCE_SMAX: 240 case ISD::VECREDUCE_SMIN: 241 case ISD::VECREDUCE_UMAX: 242 case ISD::VECREDUCE_UMIN: 243 Res = PromoteIntRes_VECREDUCE(N); 244 break; 245 246 case ISD::VP_REDUCE_ADD: 247 case ISD::VP_REDUCE_MUL: 248 case ISD::VP_REDUCE_AND: 249 case ISD::VP_REDUCE_OR: 250 case ISD::VP_REDUCE_XOR: 251 case ISD::VP_REDUCE_SMAX: 252 case ISD::VP_REDUCE_SMIN: 253 case ISD::VP_REDUCE_UMAX: 254 case ISD::VP_REDUCE_UMIN: 255 Res = PromoteIntRes_VP_REDUCE(N); 256 break; 257 258 case ISD::FREEZE: 259 Res = PromoteIntRes_FREEZE(N); 260 break; 261 262 case ISD::ROTL: 263 case ISD::ROTR: 264 Res = PromoteIntRes_Rotate(N); 265 break; 266 267 case ISD::FSHL: 268 case ISD::FSHR: 269 Res = PromoteIntRes_FunnelShift(N); 270 break; 271 272 case ISD::IS_FPCLASS: 273 Res = PromoteIntRes_IS_FPCLASS(N); 274 break; 275 } 276 277 // If the result is null then the sub-method took care of registering it. 278 if (Res.getNode()) 279 SetPromotedInteger(SDValue(N, ResNo), Res); 280 } 281 282 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 283 unsigned ResNo) { 284 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 285 return GetPromotedInteger(Op); 286 } 287 288 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 289 // Sign-extend the new bits, and continue the assertion. 290 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 291 return DAG.getNode(ISD::AssertSext, SDLoc(N), 292 Op.getValueType(), Op, N->getOperand(1)); 293 } 294 295 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 296 // Zero the new bits, and continue the assertion. 297 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 298 return DAG.getNode(ISD::AssertZext, SDLoc(N), 299 Op.getValueType(), Op, N->getOperand(1)); 300 } 301 302 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 303 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 304 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 305 N->getMemoryVT(), ResVT, 306 N->getChain(), N->getBasePtr(), 307 N->getMemOperand()); 308 // Legalize the chain result - switch anything that used the old chain to 309 // use the new one. 310 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 311 return Res; 312 } 313 314 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 315 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 316 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 317 N->getMemoryVT(), 318 N->getChain(), N->getBasePtr(), 319 Op2, N->getMemOperand()); 320 // Legalize the chain result - switch anything that used the old chain to 321 // use the new one. 322 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 323 return Res; 324 } 325 326 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 327 unsigned ResNo) { 328 if (ResNo == 1) { 329 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 330 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 331 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 332 333 // Only use the result of getSetCCResultType if it is legal, 334 // otherwise just use the promoted result type (NVT). 335 if (!TLI.isTypeLegal(SVT)) 336 SVT = NVT; 337 338 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 339 SDValue Res = DAG.getAtomicCmpSwap( 340 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 341 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 342 N->getMemOperand()); 343 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 344 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 345 return Res.getValue(1); 346 } 347 348 // Op2 is used for the comparison and thus must be extended according to the 349 // target's atomic operations. Op3 is merely stored and so can be left alone. 350 SDValue Op2 = N->getOperand(2); 351 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 352 switch (TLI.getExtendForAtomicCmpSwapArg()) { 353 case ISD::SIGN_EXTEND: 354 Op2 = SExtPromotedInteger(Op2); 355 break; 356 case ISD::ZERO_EXTEND: 357 Op2 = ZExtPromotedInteger(Op2); 358 break; 359 case ISD::ANY_EXTEND: 360 Op2 = GetPromotedInteger(Op2); 361 break; 362 default: 363 llvm_unreachable("Invalid atomic op extension"); 364 } 365 366 SDVTList VTs = 367 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 368 SDValue Res = DAG.getAtomicCmpSwap( 369 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 370 N->getBasePtr(), Op2, Op3, N->getMemOperand()); 371 // Update the use to N with the newly created Res. 372 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) 373 ReplaceValueWith(SDValue(N, i), Res.getValue(i)); 374 return Res; 375 } 376 377 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 378 SDValue InOp = N->getOperand(0); 379 EVT InVT = InOp.getValueType(); 380 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 381 EVT OutVT = N->getValueType(0); 382 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 383 SDLoc dl(N); 384 385 switch (getTypeAction(InVT)) { 386 case TargetLowering::TypeLegal: 387 break; 388 case TargetLowering::TypePromoteInteger: 389 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 390 // The input promotes to the same size. Convert the promoted value. 391 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 392 break; 393 case TargetLowering::TypeSoftenFloat: 394 // Promote the integer operand by hand. 395 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 396 case TargetLowering::TypeSoftPromoteHalf: 397 // Promote the integer operand by hand. 398 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp)); 399 case TargetLowering::TypePromoteFloat: { 400 // Convert the promoted float by hand. 401 if (!NOutVT.isVector()) 402 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); 403 break; 404 } 405 case TargetLowering::TypeExpandInteger: 406 case TargetLowering::TypeExpandFloat: 407 break; 408 case TargetLowering::TypeScalarizeVector: 409 // Convert the element to an integer and promote it by hand. 410 if (!NOutVT.isVector()) 411 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 412 BitConvertToInteger(GetScalarizedVector(InOp))); 413 break; 414 case TargetLowering::TypeScalarizeScalableVector: 415 report_fatal_error("Scalarization of scalable vectors is not supported."); 416 case TargetLowering::TypeSplitVector: { 417 if (!NOutVT.isVector()) { 418 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 419 // pieces of the input into integers and reassemble in the final type. 420 SDValue Lo, Hi; 421 GetSplitVector(N->getOperand(0), Lo, Hi); 422 Lo = BitConvertToInteger(Lo); 423 Hi = BitConvertToInteger(Hi); 424 425 if (DAG.getDataLayout().isBigEndian()) 426 std::swap(Lo, Hi); 427 428 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 429 EVT::getIntegerVT(*DAG.getContext(), 430 NOutVT.getSizeInBits()), 431 JoinIntegers(Lo, Hi)); 432 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 433 } 434 break; 435 } 436 case TargetLowering::TypeWidenVector: 437 // The input is widened to the same size. Convert to the widened value. 438 // Make sure that the outgoing value is not a vector, because this would 439 // make us bitcast between two vectors which are legalized in different ways. 440 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) { 441 SDValue Res = 442 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 443 444 // For big endian targets we need to shift the casted value or the 445 // interesting bits will end up at the wrong place. 446 if (DAG.getDataLayout().isBigEndian()) { 447 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 448 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!"); 449 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res, 450 DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl)); 451 } 452 return Res; 453 } 454 // If the output type is also a vector and widening it to the same size 455 // as the widened input type would be a legal type, we can widen the bitcast 456 // and handle the promotion after. 457 if (NOutVT.isVector()) { 458 TypeSize WidenInSize = NInVT.getSizeInBits(); 459 TypeSize OutSize = OutVT.getSizeInBits(); 460 if (WidenInSize.hasKnownScalarFactor(OutSize)) { 461 unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize); 462 EVT WideOutVT = 463 EVT::getVectorVT(*DAG.getContext(), OutVT.getVectorElementType(), 464 OutVT.getVectorElementCount() * Scale); 465 if (isTypeLegal(WideOutVT)) { 466 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); 467 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, 468 DAG.getVectorIdxConstant(0, dl)); 469 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp); 470 } 471 } 472 } 473 } 474 475 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 476 CreateStackStoreLoad(InOp, OutVT)); 477 } 478 479 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { 480 SDValue V = GetPromotedInteger(N->getOperand(0)); 481 return DAG.getNode(ISD::FREEZE, SDLoc(N), 482 V.getValueType(), V); 483 } 484 485 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 486 SDValue Op = GetPromotedInteger(N->getOperand(0)); 487 EVT OVT = N->getValueType(0); 488 EVT NVT = Op.getValueType(); 489 SDLoc dl(N); 490 491 // If the larger BSWAP isn't supported by the target, try to expand now. 492 // If we expand later we'll end up with more operations since we lost the 493 // original type. We only do this for scalars since we have a shuffle 494 // based lowering for vectors in LegalizeVectorOps. 495 if (!OVT.isVector() && 496 !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) { 497 if (SDValue Res = TLI.expandBSWAP(N, DAG)) 498 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); 499 } 500 501 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 502 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 503 DAG.getShiftAmountConstant(DiffBits, NVT, dl)); 504 } 505 506 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { 507 SDValue Op = GetPromotedInteger(N->getOperand(0)); 508 EVT OVT = N->getValueType(0); 509 EVT NVT = Op.getValueType(); 510 SDLoc dl(N); 511 512 // If the larger BITREVERSE isn't supported by the target, try to expand now. 513 // If we expand later we'll end up with more operations since we lost the 514 // original type. We only do this for scalars since we have a shuffle 515 // based lowering for vectors in LegalizeVectorOps. 516 if (!OVT.isVector() && OVT.isSimple() && 517 !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) { 518 if (SDValue Res = TLI.expandBITREVERSE(N, DAG)) 519 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); 520 } 521 522 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 523 return DAG.getNode(ISD::SRL, dl, NVT, 524 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), 525 DAG.getShiftAmountConstant(DiffBits, NVT, dl)); 526 } 527 528 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 529 // The pair element type may be legal, or may not promote to the same type as 530 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 531 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 532 TLI.getTypeToTransformTo(*DAG.getContext(), 533 N->getValueType(0)), JoinIntegers(N->getOperand(0), 534 N->getOperand(1))); 535 } 536 537 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 538 EVT VT = N->getValueType(0); 539 // FIXME there is no actual debug info here 540 SDLoc dl(N); 541 // Zero extend things like i1, sign extend everything else. It shouldn't 542 // matter in theory which one we pick, but this tends to give better code? 543 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 544 SDValue Result = DAG.getNode(Opc, dl, 545 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 546 SDValue(N, 0)); 547 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 548 return Result; 549 } 550 551 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 552 EVT OVT = N->getValueType(0); 553 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 554 SDLoc dl(N); 555 556 // If the larger CTLZ isn't supported by the target, try to expand now. 557 // If we expand later we'll end up with more operations since we lost the 558 // original type. 559 if (!OVT.isVector() && TLI.isTypeLegal(NVT) && 560 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ, NVT) && 561 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ_ZERO_UNDEF, NVT)) { 562 if (SDValue Result = TLI.expandCTLZ(N, DAG)) { 563 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); 564 return Result; 565 } 566 } 567 568 // Zero extend to the promoted type and do the count there. 569 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 570 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 571 // Subtract off the extra leading bits in the bigger type. 572 return DAG.getNode( 573 ISD::SUB, dl, NVT, Op, 574 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, 575 NVT)); 576 } 577 578 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { 579 EVT OVT = N->getValueType(0); 580 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 581 582 // If the larger CTPOP isn't supported by the target, try to expand now. 583 // If we expand later we'll end up with more operations since we lost the 584 // original type. 585 // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to 586 // TargetLowering. 587 if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) && 588 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTPOP, NVT)) { 589 if (SDValue Result = TLI.expandCTPOP(N, DAG)) { 590 Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result); 591 return Result; 592 } 593 } 594 595 // Zero extend to the promoted type and do the count or parity there. 596 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 597 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op); 598 } 599 600 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 601 SDValue Op = GetPromotedInteger(N->getOperand(0)); 602 EVT OVT = N->getValueType(0); 603 EVT NVT = Op.getValueType(); 604 SDLoc dl(N); 605 606 // If the larger CTTZ isn't supported by the target, try to expand now. 607 // If we expand later we'll end up with more operations since we lost the 608 // original type. Don't expand if we can use CTPOP or CTLZ expansion on the 609 // larger type. 610 if (!OVT.isVector() && TLI.isTypeLegal(NVT) && 611 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ, NVT) && 612 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ_ZERO_UNDEF, NVT) && 613 !TLI.isOperationLegal(ISD::CTPOP, NVT) && 614 !TLI.isOperationLegal(ISD::CTLZ, NVT)) { 615 if (SDValue Result = TLI.expandCTTZ(N, DAG)) { 616 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); 617 return Result; 618 } 619 } 620 621 if (N->getOpcode() == ISD::CTTZ) { 622 // The count is the same in the promoted type except if the original 623 // value was zero. This can be handled by setting the bit just off 624 // the top of the original type. 625 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), 626 OVT.getScalarSizeInBits()); 627 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); 628 } 629 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 630 } 631 632 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 633 SDLoc dl(N); 634 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 635 636 SDValue Op0 = N->getOperand(0); 637 SDValue Op1 = N->getOperand(1); 638 639 // If the input also needs to be promoted, do that first so we can get a 640 // get a good idea for the output type. 641 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType()) 642 == TargetLowering::TypePromoteInteger) { 643 SDValue In = GetPromotedInteger(Op0); 644 645 // If the new type is larger than NVT, use it. We probably won't need to 646 // promote it again. 647 EVT SVT = In.getValueType().getScalarType(); 648 if (SVT.bitsGE(NVT)) { 649 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1); 650 return DAG.getAnyExtOrTrunc(Ext, dl, NVT); 651 } 652 } 653 654 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1); 655 } 656 657 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 658 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 659 unsigned NewOpc = N->getOpcode(); 660 SDLoc dl(N); 661 662 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 663 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 664 // and SINT conversions are Custom, there is no way to tell which is 665 // preferable. We choose SINT because that's the right thing on PPC.) 666 if (N->getOpcode() == ISD::FP_TO_UINT && 667 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 668 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 669 NewOpc = ISD::FP_TO_SINT; 670 671 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && 672 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) && 673 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) 674 NewOpc = ISD::STRICT_FP_TO_SINT; 675 676 if (N->getOpcode() == ISD::VP_FPTOUI && 677 !TLI.isOperationLegal(ISD::VP_FPTOUI, NVT) && 678 TLI.isOperationLegalOrCustom(ISD::VP_FPTOSI, NVT)) 679 NewOpc = ISD::VP_FPTOSI; 680 681 SDValue Res; 682 if (N->isStrictFPOpcode()) { 683 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, 684 {N->getOperand(0), N->getOperand(1)}); 685 // Legalize the chain result - switch anything that used the old chain to 686 // use the new one. 687 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 688 } else if (NewOpc == ISD::VP_FPTOSI || NewOpc == ISD::VP_FPTOUI) { 689 Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1), 690 N->getOperand(2)}); 691 } else { 692 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 693 } 694 695 // Assert that the converted value fits in the original type. If it doesn't 696 // (eg: because the value being converted is too big), then the result of the 697 // original operation was undefined anyway, so the assert is still correct. 698 // 699 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: 700 // before legalization: fp-to-uint16, 65534. -> 0xfffe 701 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe 702 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT || 703 N->getOpcode() == ISD::STRICT_FP_TO_UINT || 704 N->getOpcode() == ISD::VP_FPTOUI) 705 ? ISD::AssertZext 706 : ISD::AssertSext, 707 dl, NVT, Res, 708 DAG.getValueType(N->getValueType(0).getScalarType())); 709 } 710 711 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { 712 // Promote the result type, while keeping the original width in Op1. 713 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 714 SDLoc dl(N); 715 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), 716 N->getOperand(1)); 717 } 718 719 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) { 720 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 721 SDLoc dl(N); 722 723 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 724 } 725 726 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) { 727 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 728 SDLoc dl(N); 729 730 SDValue Res = 731 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); 732 733 // Legalize the chain result - switch anything that used the old chain to 734 // use the new one. 735 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 736 return Res; 737 } 738 739 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 740 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 741 SDLoc dl(N); 742 743 if (getTypeAction(N->getOperand(0).getValueType()) 744 == TargetLowering::TypePromoteInteger) { 745 SDValue Res = GetPromotedInteger(N->getOperand(0)); 746 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 747 748 // If the result and operand types are the same after promotion, simplify 749 // to an in-register extension. 750 if (NVT == Res.getValueType()) { 751 // The high bits are not guaranteed to be anything. Insert an extend. 752 if (N->getOpcode() == ISD::SIGN_EXTEND) 753 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 754 DAG.getValueType(N->getOperand(0).getValueType())); 755 if (N->getOpcode() == ISD::ZERO_EXTEND) 756 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); 757 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 758 return Res; 759 } 760 } 761 762 // Otherwise, just extend the original operand all the way to the larger type. 763 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 764 } 765 766 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 767 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 768 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 769 ISD::LoadExtType ExtType = 770 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 771 SDLoc dl(N); 772 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 773 N->getMemoryVT(), N->getMemOperand()); 774 775 // Legalize the chain result - switch anything that used the old chain to 776 // use the new one. 777 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 778 return Res; 779 } 780 781 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { 782 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 783 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 784 785 ISD::LoadExtType ExtType = N->getExtensionType(); 786 if (ExtType == ISD::NON_EXTLOAD) 787 ExtType = ISD::EXTLOAD; 788 789 SDLoc dl(N); 790 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), 791 N->getOffset(), N->getMask(), ExtPassThru, 792 N->getMemoryVT(), N->getMemOperand(), 793 N->getAddressingMode(), ExtType, 794 N->isExpandingLoad()); 795 // Legalize the chain result - switch anything that used the old chain to 796 // use the new one. 797 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 798 return Res; 799 } 800 801 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { 802 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 803 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 804 assert(NVT == ExtPassThru.getValueType() && 805 "Gather result type and the passThru argument type should be the same"); 806 807 ISD::LoadExtType ExtType = N->getExtensionType(); 808 if (ExtType == ISD::NON_EXTLOAD) 809 ExtType = ISD::EXTLOAD; 810 811 SDLoc dl(N); 812 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), 813 N->getIndex(), N->getScale() }; 814 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), 815 N->getMemoryVT(), dl, Ops, 816 N->getMemOperand(), N->getIndexType(), 817 ExtType); 818 // Legalize the chain result - switch anything that used the old chain to 819 // use the new one. 820 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 821 return Res; 822 } 823 824 /// Promote the overflow flag of an overflowing arithmetic node. 825 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 826 // Change the return type of the boolean result while obeying 827 // getSetCCResultType. 828 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 829 EVT VT = N->getValueType(0); 830 EVT SVT = getSetCCResultType(VT); 831 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; 832 unsigned NumOps = N->getNumOperands(); 833 assert(NumOps <= 3 && "Too many operands"); 834 if (NumOps == 3) 835 Ops[2] = N->getOperand(2); 836 837 SDLoc dl(N); 838 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), 839 makeArrayRef(Ops, NumOps)); 840 841 // Modified the sum result - switch anything that used the old sum to use 842 // the new one. 843 ReplaceValueWith(SDValue(N, 0), Res); 844 845 // Convert to the expected type. 846 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); 847 } 848 849 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { 850 // If the promoted type is legal, we can convert this to: 851 // 1. ANY_EXTEND iN to iM 852 // 2. SHL by M-N 853 // 3. [US][ADD|SUB|SHL]SAT 854 // 4. L/ASHR by M-N 855 // Else it is more efficient to convert this to a min and a max 856 // operation in the higher precision arithmetic. 857 SDLoc dl(N); 858 SDValue Op1 = N->getOperand(0); 859 SDValue Op2 = N->getOperand(1); 860 unsigned OldBits = Op1.getScalarValueSizeInBits(); 861 862 unsigned Opcode = N->getOpcode(); 863 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; 864 865 SDValue Op1Promoted, Op2Promoted; 866 if (IsShift) { 867 Op1Promoted = GetPromotedInteger(Op1); 868 Op2Promoted = ZExtPromotedInteger(Op2); 869 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { 870 Op1Promoted = ZExtPromotedInteger(Op1); 871 Op2Promoted = ZExtPromotedInteger(Op2); 872 } else { 873 Op1Promoted = SExtPromotedInteger(Op1); 874 Op2Promoted = SExtPromotedInteger(Op2); 875 } 876 EVT PromotedType = Op1Promoted.getValueType(); 877 unsigned NewBits = PromotedType.getScalarSizeInBits(); 878 879 if (Opcode == ISD::UADDSAT) { 880 APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits); 881 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 882 SDValue Add = 883 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); 884 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); 885 } 886 887 // USUBSAT can always be promoted as long as we have zero-extended the args. 888 if (Opcode == ISD::USUBSAT) 889 return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted, 890 Op2Promoted); 891 892 // Shift cannot use a min/max expansion, we can't detect overflow if all of 893 // the bits have been shifted out. 894 if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) { 895 unsigned ShiftOp; 896 switch (Opcode) { 897 case ISD::SADDSAT: 898 case ISD::SSUBSAT: 899 case ISD::SSHLSAT: 900 ShiftOp = ISD::SRA; 901 break; 902 case ISD::USHLSAT: 903 ShiftOp = ISD::SRL; 904 break; 905 default: 906 llvm_unreachable("Expected opcode to be signed or unsigned saturation " 907 "addition, subtraction or left shift"); 908 } 909 910 unsigned SHLAmount = NewBits - OldBits; 911 SDValue ShiftAmount = 912 DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl); 913 Op1Promoted = 914 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); 915 if (!IsShift) 916 Op2Promoted = 917 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); 918 919 SDValue Result = 920 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); 921 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); 922 } 923 924 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; 925 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits); 926 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits); 927 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType); 928 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 929 SDValue Result = 930 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); 931 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); 932 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); 933 return Result; 934 } 935 936 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { 937 // Can just promote the operands then continue with operation. 938 SDLoc dl(N); 939 SDValue Op1Promoted, Op2Promoted; 940 bool Signed = 941 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; 942 bool Saturating = 943 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; 944 if (Signed) { 945 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 946 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 947 } else { 948 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 949 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 950 } 951 EVT OldType = N->getOperand(0).getValueType(); 952 EVT PromotedType = Op1Promoted.getValueType(); 953 unsigned DiffSize = 954 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); 955 956 if (Saturating) { 957 // Promoting the operand and result values changes the saturation width, 958 // which is extends the values that we clamp to on saturation. This could be 959 // resolved by shifting one of the operands the same amount, which would 960 // also shift the result we compare against, then shifting back. 961 Op1Promoted = 962 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 963 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); 964 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 965 Op2Promoted, N->getOperand(2)); 966 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; 967 return DAG.getNode(ShiftOp, dl, PromotedType, Result, 968 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); 969 } 970 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, 971 N->getOperand(2)); 972 } 973 974 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, 975 unsigned SatW, bool Signed, 976 const TargetLowering &TLI, 977 SelectionDAG &DAG) { 978 EVT VT = V.getValueType(); 979 unsigned VTW = VT.getScalarSizeInBits(); 980 981 if (!Signed) { 982 // Saturate to the unsigned maximum by getting the minimum of V and the 983 // maximum. 984 return DAG.getNode(ISD::UMIN, dl, VT, V, 985 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW), 986 dl, VT)); 987 } 988 989 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the 990 // signed minimum of it and V. 991 V = DAG.getNode(ISD::SMIN, dl, VT, V, 992 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1), 993 dl, VT)); 994 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the 995 // signed maximum of it and V. 996 V = DAG.getNode(ISD::SMAX, dl, VT, V, 997 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1), 998 dl, VT)); 999 return V; 1000 } 1001 1002 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, 1003 unsigned Scale, const TargetLowering &TLI, 1004 SelectionDAG &DAG, unsigned SatW = 0) { 1005 EVT VT = LHS.getValueType(); 1006 unsigned VTSize = VT.getScalarSizeInBits(); 1007 bool Signed = N->getOpcode() == ISD::SDIVFIX || 1008 N->getOpcode() == ISD::SDIVFIXSAT; 1009 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 1010 N->getOpcode() == ISD::UDIVFIXSAT; 1011 1012 SDLoc dl(N); 1013 // Widen the types by a factor of two. This is guaranteed to expand, since it 1014 // will always have enough high bits in the LHS to shift into. 1015 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2); 1016 if (VT.isVector()) 1017 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, 1018 VT.getVectorElementCount()); 1019 if (Signed) { 1020 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT); 1021 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT); 1022 } else { 1023 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT); 1024 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT); 1025 } 1026 1027 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, 1028 DAG); 1029 assert(Res && "Expanding DIVFIX with wide type failed?"); 1030 if (Saturating) { 1031 // If the caller has told us to saturate at something less, use that width 1032 // instead of the type before doubling. However, it cannot be more than 1033 // what we just widened! 1034 assert(SatW <= VTSize && 1035 "Tried to saturate to more than the original type?"); 1036 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed, 1037 TLI, DAG); 1038 } 1039 return DAG.getZExtOrTrunc(Res, dl, VT); 1040 } 1041 1042 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { 1043 SDLoc dl(N); 1044 SDValue Op1Promoted, Op2Promoted; 1045 bool Signed = N->getOpcode() == ISD::SDIVFIX || 1046 N->getOpcode() == ISD::SDIVFIXSAT; 1047 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 1048 N->getOpcode() == ISD::UDIVFIXSAT; 1049 if (Signed) { 1050 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 1051 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 1052 } else { 1053 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 1054 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 1055 } 1056 EVT PromotedType = Op1Promoted.getValueType(); 1057 unsigned Scale = N->getConstantOperandVal(2); 1058 1059 // If the type is already legal and the operation is legal in that type, we 1060 // should not early expand. 1061 if (TLI.isTypeLegal(PromotedType)) { 1062 TargetLowering::LegalizeAction Action = 1063 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale); 1064 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { 1065 unsigned Diff = PromotedType.getScalarSizeInBits() - 1066 N->getValueType(0).getScalarSizeInBits(); 1067 if (Saturating) 1068 Op1Promoted = 1069 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 1070 DAG.getShiftAmountConstant(Diff, PromotedType, dl)); 1071 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 1072 Op2Promoted, N->getOperand(2)); 1073 if (Saturating) 1074 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res, 1075 DAG.getShiftAmountConstant(Diff, PromotedType, dl)); 1076 return Res; 1077 } 1078 } 1079 1080 // See if we can perform the division in this type without expanding. 1081 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted, 1082 Op2Promoted, Scale, DAG)) { 1083 if (Saturating) 1084 Res = SaturateWidenedDIVFIX(Res, dl, 1085 N->getValueType(0).getScalarSizeInBits(), 1086 Signed, TLI, DAG); 1087 return Res; 1088 } 1089 // If we cannot, expand it to twice the type width. If we are saturating, give 1090 // it the original width as a saturating width so we don't need to emit 1091 // two saturations. 1092 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG, 1093 N->getValueType(0).getScalarSizeInBits()); 1094 } 1095 1096 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 1097 if (ResNo == 1) 1098 return PromoteIntRes_Overflow(N); 1099 1100 // The operation overflowed iff the result in the larger type is not the 1101 // sign extension of its truncation to the original type. 1102 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1103 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1104 EVT OVT = N->getOperand(0).getValueType(); 1105 EVT NVT = LHS.getValueType(); 1106 SDLoc dl(N); 1107 1108 // Do the arithmetic in the larger type. 1109 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 1110 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1111 1112 // Calculate the overflow flag: sign extend the arithmetic result from 1113 // the original type. 1114 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 1115 DAG.getValueType(OVT)); 1116 // Overflowed if and only if this is not equal to Res. 1117 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1118 1119 // Use the calculated overflow everywhere. 1120 ReplaceValueWith(SDValue(N, 1), Ofl); 1121 1122 return Res; 1123 } 1124 1125 SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) { 1126 SDValue Mask = N->getOperand(0); 1127 1128 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 1129 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 1130 1131 unsigned Opcode = N->getOpcode(); 1132 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) 1133 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS, 1134 N->getOperand(3)); 1135 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS); 1136 } 1137 1138 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 1139 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 1140 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 1141 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 1142 LHS.getValueType(), N->getOperand(0), 1143 N->getOperand(1), LHS, RHS, N->getOperand(4)); 1144 } 1145 1146 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 1147 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 1148 EVT InVT = N->getOperand(OpNo).getValueType(); 1149 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1150 1151 EVT SVT = getSetCCResultType(InVT); 1152 1153 // If we got back a type that needs to be promoted, this likely means the 1154 // the input type also needs to be promoted. So get the promoted type for 1155 // the input and try the query again. 1156 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { 1157 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 1158 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 1159 SVT = getSetCCResultType(InVT); 1160 } else { 1161 // Input type isn't promoted, just use the default promoted type. 1162 SVT = NVT; 1163 } 1164 } 1165 1166 SDLoc dl(N); 1167 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && 1168 "Vector compare must return a vector result!"); 1169 1170 // Get the SETCC result using the canonical SETCC type. 1171 SDValue SetCC; 1172 if (N->isStrictFPOpcode()) { 1173 EVT VTs[] = {SVT, MVT::Other}; 1174 SDValue Opers[] = {N->getOperand(0), N->getOperand(1), 1175 N->getOperand(2), N->getOperand(3)}; 1176 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers); 1177 // Legalize the chain result - switch anything that used the old chain to 1178 // use the new one. 1179 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1)); 1180 } else 1181 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), 1182 N->getOperand(1), N->getOperand(2)); 1183 1184 // Convert to the expected type. 1185 return DAG.getSExtOrTrunc(SetCC, dl, NVT); 1186 } 1187 1188 SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) { 1189 SDLoc DL(N); 1190 SDValue Arg = N->getOperand(0); 1191 SDValue Test = N->getOperand(1); 1192 EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1193 return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test); 1194 } 1195 1196 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 1197 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1198 SDValue RHS = N->getOperand(1); 1199 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1200 RHS = ZExtPromotedInteger(RHS); 1201 if (N->getOpcode() != ISD::VP_SHL) 1202 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1203 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1204 N->getOperand(2), N->getOperand(3)); 1205 } 1206 1207 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 1208 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1209 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 1210 Op.getValueType(), Op, N->getOperand(1)); 1211 } 1212 1213 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 1214 // The input may have strange things in the top bits of the registers, but 1215 // these operations don't care. They may have weird bits going out, but 1216 // that too is okay if they are integer operations. 1217 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1218 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 1219 if (N->getNumOperands() == 2) 1220 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1221 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1222 assert(N->isVPOpcode() && "Expected VP opcode"); 1223 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1224 N->getOperand(2), N->getOperand(3)); 1225 } 1226 1227 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { 1228 // Sign extend the input. 1229 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1230 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1231 if (N->getNumOperands() == 2) 1232 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1233 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1234 assert(N->isVPOpcode() && "Expected VP opcode"); 1235 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1236 N->getOperand(2), N->getOperand(3)); 1237 } 1238 1239 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { 1240 // Zero extend the input. 1241 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1242 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1243 if (N->getNumOperands() == 2) 1244 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1245 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1246 assert(N->isVPOpcode() && "Expected VP opcode"); 1247 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1248 N->getOperand(2), N->getOperand(3)); 1249 } 1250 1251 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { 1252 // It doesn't matter if we sign extend or zero extend in the inputs. So do 1253 // whatever is best for the target. 1254 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0)); 1255 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1)); 1256 return DAG.getNode(N->getOpcode(), SDLoc(N), 1257 LHS.getValueType(), LHS, RHS); 1258 } 1259 1260 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 1261 // The input value must be properly sign extended. 1262 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1263 SDValue RHS = N->getOperand(1); 1264 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1265 RHS = ZExtPromotedInteger(RHS); 1266 if (N->getOpcode() != ISD::VP_ASHR) 1267 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1268 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1269 N->getOperand(2), N->getOperand(3)); 1270 } 1271 1272 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 1273 // The input value must be properly zero extended. 1274 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1275 SDValue RHS = N->getOperand(1); 1276 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1277 RHS = ZExtPromotedInteger(RHS); 1278 if (N->getOpcode() != ISD::VP_LSHR) 1279 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1280 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1281 N->getOperand(2), N->getOperand(3)); 1282 } 1283 1284 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { 1285 // Lower the rotate to shifts and ORs which can be promoted. 1286 SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG); 1287 ReplaceValueWith(SDValue(N, 0), Res); 1288 return SDValue(); 1289 } 1290 1291 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { 1292 SDValue Hi = GetPromotedInteger(N->getOperand(0)); 1293 SDValue Lo = GetPromotedInteger(N->getOperand(1)); 1294 SDValue Amt = N->getOperand(2); 1295 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger) 1296 Amt = ZExtPromotedInteger(Amt); 1297 EVT AmtVT = Amt.getValueType(); 1298 1299 SDLoc DL(N); 1300 EVT OldVT = N->getOperand(0).getValueType(); 1301 EVT VT = Lo.getValueType(); 1302 unsigned Opcode = N->getOpcode(); 1303 bool IsFSHR = Opcode == ISD::FSHR; 1304 unsigned OldBits = OldVT.getScalarSizeInBits(); 1305 unsigned NewBits = VT.getScalarSizeInBits(); 1306 1307 // Amount has to be interpreted modulo the old bit width. 1308 Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt, 1309 DAG.getConstant(OldBits, DL, AmtVT)); 1310 1311 // If the promoted type is twice the size (or more), then we use the 1312 // traditional funnel 'double' shift codegen. This isn't necessary if the 1313 // shift amount is constant. 1314 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. 1315 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). 1316 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) && 1317 !TLI.isOperationLegalOrCustom(Opcode, VT)) { 1318 SDValue HiShift = DAG.getConstant(OldBits, DL, VT); 1319 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift); 1320 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT); 1321 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo); 1322 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt); 1323 if (!IsFSHR) 1324 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift); 1325 return Res; 1326 } 1327 1328 // Shift Lo up to occupy the upper bits of the promoted type. 1329 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT); 1330 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset); 1331 1332 // Increase Amount to shift the result into the lower bits of the promoted 1333 // type. 1334 if (IsFSHR) 1335 Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset); 1336 1337 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt); 1338 } 1339 1340 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 1341 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1342 SDValue Res; 1343 SDValue InOp = N->getOperand(0); 1344 SDLoc dl(N); 1345 1346 switch (getTypeAction(InOp.getValueType())) { 1347 default: llvm_unreachable("Unknown type action!"); 1348 case TargetLowering::TypeLegal: 1349 case TargetLowering::TypeExpandInteger: 1350 Res = InOp; 1351 break; 1352 case TargetLowering::TypePromoteInteger: 1353 Res = GetPromotedInteger(InOp); 1354 break; 1355 case TargetLowering::TypeSplitVector: { 1356 EVT InVT = InOp.getValueType(); 1357 assert(InVT.isVector() && "Cannot split scalar types"); 1358 ElementCount NumElts = InVT.getVectorElementCount(); 1359 assert(NumElts == NVT.getVectorElementCount() && 1360 "Dst and Src must have the same number of elements"); 1361 assert(isPowerOf2_32(NumElts.getKnownMinValue()) && 1362 "Promoted vector type must be a power of two"); 1363 1364 SDValue EOp1, EOp2; 1365 GetSplitVector(InOp, EOp1, EOp2); 1366 1367 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 1368 NumElts.divideCoefficientBy(2)); 1369 if (N->getOpcode() == ISD::TRUNCATE) { 1370 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 1371 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 1372 } else { 1373 assert(N->getOpcode() == ISD::VP_TRUNCATE && 1374 "Expected VP_TRUNCATE opcode"); 1375 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 1376 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 1377 std::tie(EVLLo, EVLHi) = 1378 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); 1379 EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo); 1380 EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi); 1381 } 1382 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 1383 } 1384 // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some 1385 // targets. 1386 case TargetLowering::TypeWidenVector: { 1387 SDValue WideInOp = GetWidenedVector(InOp); 1388 1389 // Truncate widened InOp. 1390 unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); 1391 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), 1392 N->getValueType(0).getScalarType(), NumElem); 1393 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); 1394 1395 // Zero extend so that the elements are of same type as those of NVT 1396 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), 1397 NumElem); 1398 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); 1399 1400 // Extract the low NVT subvector. 1401 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl); 1402 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); 1403 } 1404 } 1405 1406 // Truncate to NVT instead of VT 1407 if (N->getOpcode() == ISD::VP_TRUNCATE) 1408 return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1), 1409 N->getOperand(2)); 1410 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 1411 } 1412 1413 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 1414 if (ResNo == 1) 1415 return PromoteIntRes_Overflow(N); 1416 1417 // The operation overflowed iff the result in the larger type is not the 1418 // zero extension of its truncation to the original type. 1419 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1420 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1421 EVT OVT = N->getOperand(0).getValueType(); 1422 EVT NVT = LHS.getValueType(); 1423 SDLoc dl(N); 1424 1425 // Do the arithmetic in the larger type. 1426 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 1427 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1428 1429 // Calculate the overflow flag: zero extend the arithmetic result from 1430 // the original type. 1431 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 1432 // Overflowed if and only if this is not equal to Res. 1433 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1434 1435 // Use the calculated overflow everywhere. 1436 ReplaceValueWith(SDValue(N, 1), Ofl); 1437 1438 return Res; 1439 } 1440 1441 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that 1442 // the third operand of ADDE/SUBE nodes is carry flag, which differs from 1443 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean. 1444 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) { 1445 if (ResNo == 1) 1446 return PromoteIntRes_Overflow(N); 1447 1448 // We need to sign-extend the operands so the carry value computed by the 1449 // wide operation will be equivalent to the carry value computed by the 1450 // narrow operation. 1451 // An ADDCARRY can generate carry only if any of the operands has its 1452 // most significant bit set. Sign extension propagates the most significant 1453 // bit into the higher bits which means the extra bit that the narrow 1454 // addition would need (i.e. the carry) will be propagated through the higher 1455 // bits of the wide addition. 1456 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be 1457 // preserved by sign extension. 1458 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1459 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1460 1461 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; 1462 1463 // Do the arithmetic in the wide type. 1464 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), 1465 LHS, RHS, N->getOperand(2)); 1466 1467 // Update the users of the original carry/borrow value. 1468 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 1469 1470 return SDValue(Res.getNode(), 0); 1471 } 1472 1473 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, 1474 unsigned ResNo) { 1475 assert(ResNo == 1 && "Don't know how to promote other results yet."); 1476 return PromoteIntRes_Overflow(N); 1477 } 1478 1479 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { 1480 EVT OVT = N->getValueType(0); 1481 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 1482 1483 // If a larger ABS or SMAX isn't supported by the target, try to expand now. 1484 // If we expand later we'll end up sign extending more than just the sra input 1485 // in sra+xor+sub expansion. 1486 if (!OVT.isVector() && 1487 !TLI.isOperationLegalOrCustomOrPromote(ISD::ABS, NVT) && 1488 !TLI.isOperationLegal(ISD::SMAX, NVT)) { 1489 if (SDValue Res = TLI.expandABS(N, DAG)) 1490 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res); 1491 } 1492 1493 SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); 1494 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); 1495 } 1496 1497 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 1498 // Promote the overflow bit trivially. 1499 if (ResNo == 1) 1500 return PromoteIntRes_Overflow(N); 1501 1502 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 1503 SDLoc DL(N); 1504 EVT SmallVT = LHS.getValueType(); 1505 1506 // To determine if the result overflowed in a larger type, we extend the 1507 // input to the larger type, do the multiply (checking if it overflows), 1508 // then also check the high bits of the result to see if overflow happened 1509 // there. 1510 if (N->getOpcode() == ISD::SMULO) { 1511 LHS = SExtPromotedInteger(LHS); 1512 RHS = SExtPromotedInteger(RHS); 1513 } else { 1514 LHS = ZExtPromotedInteger(LHS); 1515 RHS = ZExtPromotedInteger(RHS); 1516 } 1517 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 1518 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 1519 1520 // Overflow occurred if it occurred in the larger type, or if the high part 1521 // of the result does not zero/sign-extend the low part. Check this second 1522 // possibility first. 1523 SDValue Overflow; 1524 if (N->getOpcode() == ISD::UMULO) { 1525 // Unsigned overflow occurred if the high part is non-zero. 1526 unsigned Shift = SmallVT.getScalarSizeInBits(); 1527 SDValue Hi = 1528 DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 1529 DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL)); 1530 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 1531 DAG.getConstant(0, DL, Hi.getValueType()), 1532 ISD::SETNE); 1533 } else { 1534 // Signed overflow occurred if the high part does not sign extend the low. 1535 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 1536 Mul, DAG.getValueType(SmallVT)); 1537 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 1538 } 1539 1540 // The only other way for overflow to occur is if the multiplication in the 1541 // larger type itself overflowed. 1542 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 1543 SDValue(Mul.getNode(), 1)); 1544 1545 // Use the calculated overflow everywhere. 1546 ReplaceValueWith(SDValue(N, 1), Overflow); 1547 return Mul; 1548 } 1549 1550 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 1551 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 1552 N->getValueType(0))); 1553 } 1554 1555 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { 1556 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1557 1558 APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue(); 1559 return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits())); 1560 } 1561 1562 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 1563 SDValue Chain = N->getOperand(0); // Get the chain. 1564 SDValue Ptr = N->getOperand(1); // Get the pointer. 1565 EVT VT = N->getValueType(0); 1566 SDLoc dl(N); 1567 1568 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 1569 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 1570 // The argument is passed as NumRegs registers of type RegVT. 1571 1572 SmallVector<SDValue, 8> Parts(NumRegs); 1573 for (unsigned i = 0; i < NumRegs; ++i) { 1574 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 1575 N->getConstantOperandVal(3)); 1576 Chain = Parts[i].getValue(1); 1577 } 1578 1579 // Handle endianness of the load. 1580 if (DAG.getDataLayout().isBigEndian()) 1581 std::reverse(Parts.begin(), Parts.end()); 1582 1583 // Assemble the parts in the promoted type. 1584 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1585 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 1586 for (unsigned i = 1; i < NumRegs; ++i) { 1587 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 1588 // Shift it to the right position and "or" it in. 1589 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 1590 DAG.getConstant(i * RegVT.getSizeInBits(), dl, 1591 TLI.getPointerTy(DAG.getDataLayout()))); 1592 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 1593 } 1594 1595 // Modified the chain result - switch anything that used the old chain to 1596 // use the new one. 1597 ReplaceValueWith(SDValue(N, 1), Chain); 1598 1599 return Res; 1600 } 1601 1602 //===----------------------------------------------------------------------===// 1603 // Integer Operand Promotion 1604 //===----------------------------------------------------------------------===// 1605 1606 /// PromoteIntegerOperand - This method is called when the specified operand of 1607 /// the specified node is found to need promotion. At this point, all of the 1608 /// result types of the node are known to be legal, but other operands of the 1609 /// node may need promotion or expansion as well as the specified one. 1610 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 1611 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); 1612 dbgs() << "\n"); 1613 SDValue Res = SDValue(); 1614 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 1615 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 1616 return false; 1617 } 1618 1619 switch (N->getOpcode()) { 1620 default: 1621 #ifndef NDEBUG 1622 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 1623 N->dump(&DAG); dbgs() << "\n"; 1624 #endif 1625 llvm_unreachable("Do not know how to promote this operator's operand!"); 1626 1627 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 1628 case ISD::ATOMIC_STORE: 1629 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 1630 break; 1631 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 1632 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 1633 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 1634 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 1635 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 1636 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 1637 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 1638 case ISD::INSERT_VECTOR_ELT: 1639 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo); 1640 break; 1641 case ISD::SPLAT_VECTOR: 1642 case ISD::SCALAR_TO_VECTOR: 1643 Res = PromoteIntOp_ScalarOp(N); 1644 break; 1645 case ISD::VSELECT: 1646 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 1647 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 1648 case ISD::VP_SETCC: 1649 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 1650 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 1651 case ISD::VP_SITOFP: 1652 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 1653 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; 1654 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 1655 OpNo); break; 1656 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), 1657 OpNo); break; 1658 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), 1659 OpNo); break; 1660 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), 1661 OpNo); break; 1662 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), 1663 OpNo); break; 1664 case ISD::VP_TRUNCATE: 1665 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 1666 case ISD::FP16_TO_FP: 1667 case ISD::VP_UITOFP: 1668 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 1669 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; 1670 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 1671 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; 1672 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break; 1673 1674 case ISD::SHL: 1675 case ISD::SRA: 1676 case ISD::SRL: 1677 case ISD::ROTL: 1678 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 1679 1680 case ISD::FSHL: 1681 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break; 1682 1683 case ISD::SADDO_CARRY: 1684 case ISD::SSUBO_CARRY: 1685 case ISD::ADDCARRY: 1686 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break; 1687 1688 case ISD::FRAMEADDR: 1689 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; 1690 1691 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break; 1692 1693 case ISD::SMULFIX: 1694 case ISD::SMULFIXSAT: 1695 case ISD::UMULFIX: 1696 case ISD::UMULFIXSAT: 1697 case ISD::SDIVFIX: 1698 case ISD::SDIVFIXSAT: 1699 case ISD::UDIVFIX: 1700 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; 1701 1702 case ISD::FPOWI: 1703 case ISD::STRICT_FPOWI: Res = PromoteIntOp_FPOWI(N); break; 1704 1705 case ISD::VECREDUCE_ADD: 1706 case ISD::VECREDUCE_MUL: 1707 case ISD::VECREDUCE_AND: 1708 case ISD::VECREDUCE_OR: 1709 case ISD::VECREDUCE_XOR: 1710 case ISD::VECREDUCE_SMAX: 1711 case ISD::VECREDUCE_SMIN: 1712 case ISD::VECREDUCE_UMAX: 1713 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; 1714 case ISD::VP_REDUCE_ADD: 1715 case ISD::VP_REDUCE_MUL: 1716 case ISD::VP_REDUCE_AND: 1717 case ISD::VP_REDUCE_OR: 1718 case ISD::VP_REDUCE_XOR: 1719 case ISD::VP_REDUCE_SMAX: 1720 case ISD::VP_REDUCE_SMIN: 1721 case ISD::VP_REDUCE_UMAX: 1722 case ISD::VP_REDUCE_UMIN: 1723 Res = PromoteIntOp_VP_REDUCE(N, OpNo); 1724 break; 1725 1726 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; 1727 case ISD::STACKMAP: 1728 Res = PromoteIntOp_STACKMAP(N, OpNo); 1729 break; 1730 case ISD::PATCHPOINT: 1731 Res = PromoteIntOp_PATCHPOINT(N, OpNo); 1732 break; 1733 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 1734 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 1735 Res = PromoteIntOp_VP_STRIDED(N, OpNo); 1736 break; 1737 } 1738 1739 // If the result is null, the sub-method took care of registering results etc. 1740 if (!Res.getNode()) return false; 1741 1742 // If the result is N, the sub-method updated N in place. Tell the legalizer 1743 // core about this. 1744 if (Res.getNode() == N) 1745 return true; 1746 1747 const bool IsStrictFp = N->isStrictFPOpcode(); 1748 assert(Res.getValueType() == N->getValueType(0) && 1749 N->getNumValues() == (IsStrictFp ? 2 : 1) && 1750 "Invalid operand expansion"); 1751 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: "; 1752 Res.dump()); 1753 1754 ReplaceValueWith(SDValue(N, 0), Res); 1755 if (IsStrictFp) 1756 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 1757 1758 return false; 1759 } 1760 1761 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is 1762 /// shared among BR_CC, SELECT_CC, and SETCC handlers. 1763 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS, 1764 ISD::CondCode CCCode) { 1765 // We have to insert explicit sign or zero extends. Note that we could 1766 // insert sign extends for ALL conditions. For those operations where either 1767 // zero or sign extension would be valid, we ask the target which extension 1768 // it would prefer. 1769 1770 // Signed comparisons always require sign extension. 1771 if (ISD::isSignedIntSetCC(CCCode)) { 1772 LHS = SExtPromotedInteger(LHS); 1773 RHS = SExtPromotedInteger(RHS); 1774 return; 1775 } 1776 1777 assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) && 1778 "Unknown integer comparison!"); 1779 1780 SDValue OpL = GetPromotedInteger(LHS); 1781 SDValue OpR = GetPromotedInteger(RHS); 1782 1783 if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) { 1784 // The target would prefer to promote the comparison operand with sign 1785 // extension. Honor that unless the promoted values are already zero 1786 // extended. 1787 unsigned OpLEffectiveBits = 1788 DAG.computeKnownBits(OpL).countMaxActiveBits(); 1789 unsigned OpREffectiveBits = 1790 DAG.computeKnownBits(OpR).countMaxActiveBits(); 1791 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && 1792 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { 1793 LHS = OpL; 1794 RHS = OpR; 1795 return; 1796 } 1797 1798 // The promoted values aren't zero extended, use a sext_inreg. 1799 LHS = SExtPromotedInteger(LHS); 1800 RHS = SExtPromotedInteger(RHS); 1801 return; 1802 } 1803 1804 // Prefer to promote the comparison operand with zero extension. 1805 1806 // If the width of OpL/OpR excluding the duplicated sign bits is no greater 1807 // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation 1808 // that we might not be able to remove. 1809 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL); 1810 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR); 1811 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && 1812 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { 1813 LHS = OpL; 1814 RHS = OpR; 1815 return; 1816 } 1817 1818 // Otherwise, use zext_inreg. 1819 LHS = ZExtPromotedInteger(LHS); 1820 RHS = ZExtPromotedInteger(RHS); 1821 } 1822 1823 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 1824 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1825 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 1826 } 1827 1828 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 1829 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 1830 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 1831 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); 1832 } 1833 1834 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 1835 // This should only occur in unusual situations like bitcasting to an 1836 // x86_fp80, so just turn it into a store+load 1837 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 1838 } 1839 1840 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 1841 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1842 1843 SDValue LHS = N->getOperand(2); 1844 SDValue RHS = N->getOperand(3); 1845 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 1846 1847 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 1848 // legal types. 1849 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1850 N->getOperand(1), LHS, RHS, N->getOperand(4)), 1851 0); 1852 } 1853 1854 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 1855 assert(OpNo == 1 && "only know how to promote condition"); 1856 1857 // Promote all the way up to the canonical SetCC type. 1858 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 1859 1860 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 1861 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 1862 N->getOperand(2)), 0); 1863 } 1864 1865 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 1866 // Since the result type is legal, the operands must promote to it. 1867 EVT OVT = N->getOperand(0).getValueType(); 1868 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 1869 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 1870 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 1871 SDLoc dl(N); 1872 1873 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 1874 DAG.getConstant(OVT.getSizeInBits(), dl, 1875 TLI.getPointerTy(DAG.getDataLayout()))); 1876 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 1877 } 1878 1879 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 1880 // The vector type is legal but the element type is not. This implies 1881 // that the vector is a power-of-two in length and that the element 1882 // type does not have a strange size (eg: it is not i1). 1883 EVT VecVT = N->getValueType(0); 1884 unsigned NumElts = VecVT.getVectorNumElements(); 1885 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 1886 "Legal vector of one illegal element?"); 1887 1888 // Promote the inserted value. The type does not need to match the 1889 // vector element type. Check that any extra bits introduced will be 1890 // truncated away. 1891 assert(N->getOperand(0).getValueSizeInBits() >= 1892 N->getValueType(0).getScalarSizeInBits() && 1893 "Type of inserted value narrower than vector element type!"); 1894 1895 SmallVector<SDValue, 16> NewOps; 1896 for (unsigned i = 0; i < NumElts; ++i) 1897 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 1898 1899 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1900 } 1901 1902 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 1903 unsigned OpNo) { 1904 if (OpNo == 1) { 1905 // Promote the inserted value. This is valid because the type does not 1906 // have to match the vector element type. 1907 1908 // Check that any extra bits introduced will be truncated away. 1909 assert(N->getOperand(1).getValueSizeInBits() >= 1910 N->getValueType(0).getScalarSizeInBits() && 1911 "Type of inserted value narrower than vector element type!"); 1912 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1913 GetPromotedInteger(N->getOperand(1)), 1914 N->getOperand(2)), 1915 0); 1916 } 1917 1918 assert(OpNo == 2 && "Different operand and result vector types?"); 1919 1920 // Promote the index. 1921 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 1922 TLI.getVectorIdxTy(DAG.getDataLayout())); 1923 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1924 N->getOperand(1), Idx), 0); 1925 } 1926 1927 SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) { 1928 // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated, 1929 // so just promote the operand in place. 1930 return SDValue(DAG.UpdateNodeOperands(N, 1931 GetPromotedInteger(N->getOperand(0))), 0); 1932 } 1933 1934 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 1935 assert(OpNo == 0 && "Only know how to promote the condition!"); 1936 SDValue Cond = N->getOperand(0); 1937 EVT OpTy = N->getOperand(1).getValueType(); 1938 1939 if (N->getOpcode() == ISD::VSELECT) 1940 if (SDValue Res = WidenVSELECTMask(N)) 1941 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 1942 Res, N->getOperand(1), N->getOperand(2)); 1943 1944 // Promote all the way up to the canonical SetCC type. 1945 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 1946 Cond = PromoteTargetBoolean(Cond, OpVT); 1947 1948 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 1949 N->getOperand(2)), 0); 1950 } 1951 1952 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 1953 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1954 1955 SDValue LHS = N->getOperand(0); 1956 SDValue RHS = N->getOperand(1); 1957 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1958 1959 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1960 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1961 N->getOperand(3), N->getOperand(4)), 0); 1962 } 1963 1964 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1965 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1966 1967 SDValue LHS = N->getOperand(0); 1968 SDValue RHS = N->getOperand(1); 1969 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 1970 1971 // The CC (#2) is always legal. 1972 if (N->getOpcode() == ISD::SETCC) 1973 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 1974 1975 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode"); 1976 1977 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1978 N->getOperand(3), N->getOperand(4)), 1979 0); 1980 } 1981 1982 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 1983 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1984 ZExtPromotedInteger(N->getOperand(1))), 0); 1985 } 1986 1987 SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) { 1988 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), 1989 ZExtPromotedInteger(N->getOperand(2))), 0); 1990 } 1991 1992 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 1993 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1994 SDLoc dl(N); 1995 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1996 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 1997 Op, DAG.getValueType(N->getOperand(0).getValueType())); 1998 } 1999 2000 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 2001 if (N->getOpcode() == ISD::VP_SITOFP) 2002 return SDValue(DAG.UpdateNodeOperands(N, 2003 SExtPromotedInteger(N->getOperand(0)), 2004 N->getOperand(1), N->getOperand(2)), 2005 0); 2006 return SDValue(DAG.UpdateNodeOperands(N, 2007 SExtPromotedInteger(N->getOperand(0))), 0); 2008 } 2009 2010 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { 2011 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2012 SExtPromotedInteger(N->getOperand(1))), 0); 2013 } 2014 2015 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 2016 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2017 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 2018 SDLoc dl(N); 2019 2020 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 2021 2022 // Truncate the value and store the result. 2023 return DAG.getTruncStore(Ch, dl, Val, Ptr, 2024 N->getMemoryVT(), N->getMemOperand()); 2025 } 2026 2027 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, 2028 unsigned OpNo) { 2029 SDValue DataOp = N->getValue(); 2030 SDValue Mask = N->getMask(); 2031 2032 if (OpNo == 4) { 2033 // The Mask. Update in place. 2034 EVT DataVT = DataOp.getValueType(); 2035 Mask = PromoteTargetBoolean(Mask, DataVT); 2036 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2037 NewOps[4] = Mask; 2038 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2039 } 2040 2041 assert(OpNo == 1 && "Unexpected operand for promotion"); 2042 DataOp = GetPromotedInteger(DataOp); 2043 2044 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(), 2045 N->getOffset(), Mask, N->getMemoryVT(), 2046 N->getMemOperand(), N->getAddressingMode(), 2047 /*IsTruncating*/ true, N->isCompressingStore()); 2048 } 2049 2050 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, 2051 unsigned OpNo) { 2052 assert(OpNo == 3 && "Only know how to promote the mask!"); 2053 EVT DataVT = N->getValueType(0); 2054 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2055 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2056 NewOps[OpNo] = Mask; 2057 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 2058 if (Res == N) 2059 return SDValue(Res, 0); 2060 2061 // Update triggered CSE, do our own replacement since caller can't. 2062 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 2063 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 2064 return SDValue(); 2065 } 2066 2067 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, 2068 unsigned OpNo) { 2069 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 2070 2071 if (OpNo == 2) { 2072 // The Mask 2073 EVT DataVT = N->getValueType(0); 2074 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2075 } else if (OpNo == 4) { 2076 // The Index 2077 if (N->isIndexSigned()) 2078 // Need to sign extend the index since the bits will likely be used. 2079 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2080 else 2081 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 2082 } else 2083 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 2084 2085 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 2086 if (Res == N) 2087 return SDValue(Res, 0); 2088 2089 // Update triggered CSE, do our own replacement since caller can't. 2090 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 2091 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 2092 return SDValue(); 2093 } 2094 2095 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, 2096 unsigned OpNo) { 2097 bool TruncateStore = N->isTruncatingStore(); 2098 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 2099 2100 if (OpNo == 2) { 2101 // The Mask 2102 EVT DataVT = N->getValue().getValueType(); 2103 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2104 } else if (OpNo == 4) { 2105 // The Index 2106 if (N->isIndexSigned()) 2107 // Need to sign extend the index since the bits will likely be used. 2108 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2109 else 2110 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 2111 } else { 2112 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 2113 TruncateStore = true; 2114 } 2115 2116 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), 2117 SDLoc(N), NewOps, N->getMemOperand(), 2118 N->getIndexType(), TruncateStore); 2119 } 2120 2121 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 2122 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2123 if (N->getOpcode() == ISD::VP_TRUNCATE) 2124 return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op, 2125 N->getOperand(1), N->getOperand(2)); 2126 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 2127 } 2128 2129 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 2130 if (N->getOpcode() == ISD::VP_UITOFP) 2131 return SDValue(DAG.UpdateNodeOperands(N, 2132 ZExtPromotedInteger(N->getOperand(0)), 2133 N->getOperand(1), N->getOperand(2)), 2134 0); 2135 return SDValue(DAG.UpdateNodeOperands(N, 2136 ZExtPromotedInteger(N->getOperand(0))), 0); 2137 } 2138 2139 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { 2140 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2141 ZExtPromotedInteger(N->getOperand(1))), 0); 2142 } 2143 2144 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 2145 SDLoc dl(N); 2146 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2147 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 2148 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); 2149 } 2150 2151 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { 2152 assert(OpNo == 2 && "Don't know how to promote this operand!"); 2153 2154 SDValue LHS = N->getOperand(0); 2155 SDValue RHS = N->getOperand(1); 2156 SDValue Carry = N->getOperand(2); 2157 SDLoc DL(N); 2158 2159 Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); 2160 2161 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); 2162 } 2163 2164 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { 2165 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 2166 return SDValue( 2167 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); 2168 } 2169 2170 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { 2171 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. 2172 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 2173 return SDValue(DAG.UpdateNodeOperands(N, Op), 0); 2174 } 2175 2176 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) { 2177 assert(OpNo > 1 && "Don't know how to promote this operand!"); 2178 // Promote the rw, locality, and cache type arguments to a supported integer 2179 // width. 2180 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 2181 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3)); 2182 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4)); 2183 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), 2184 Op2, Op3, Op4), 2185 0); 2186 } 2187 2188 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) { 2189 bool IsStrict = N->isStrictFPOpcode(); 2190 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2191 2192 // The integer operand is the last operand in FPOWI (so the result and 2193 // floating point operand is already type legalized). 2194 2195 // We can't just promote the exponent type in FPOWI, since we want to lower 2196 // the node to a libcall and we if we promote to a type larger than 2197 // sizeof(int) the libcall might not be according to the targets ABI. Instead 2198 // we rewrite to a libcall here directly, letting makeLibCall handle promotion 2199 // if the target accepts it according to shouldSignExtendTypeInLibCall. 2200 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0)); 2201 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); 2202 if (!TLI.getLibcallName(LC)) { 2203 // Some targets don't have a powi libcall; use pow instead. 2204 // FIXME: Implement this if some target needs it. 2205 DAG.getContext()->emitError("Don't know how to promote fpowi to fpow"); 2206 return DAG.getUNDEF(N->getValueType(0)); 2207 } 2208 unsigned OpOffset = IsStrict ? 1 : 0; 2209 // The exponent should fit in a sizeof(int) type for the libcall to be valid. 2210 assert(DAG.getLibInfo().getIntSize() == 2211 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && 2212 "POWI exponent should match with sizeof(int) when doing the libcall."); 2213 TargetLowering::MakeLibCallOptions CallOptions; 2214 CallOptions.setSExt(true); 2215 SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)}; 2216 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall( 2217 DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain); 2218 ReplaceValueWith(SDValue(N, 0), Tmp.first); 2219 if (IsStrict) 2220 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2221 return SDValue(); 2222 } 2223 2224 static unsigned getExtendForIntVecReduction(SDNode *N) { 2225 switch (N->getOpcode()) { 2226 default: 2227 llvm_unreachable("Expected integer vector reduction"); 2228 case ISD::VECREDUCE_ADD: 2229 case ISD::VECREDUCE_MUL: 2230 case ISD::VECREDUCE_AND: 2231 case ISD::VECREDUCE_OR: 2232 case ISD::VECREDUCE_XOR: 2233 case ISD::VP_REDUCE_ADD: 2234 case ISD::VP_REDUCE_MUL: 2235 case ISD::VP_REDUCE_AND: 2236 case ISD::VP_REDUCE_OR: 2237 case ISD::VP_REDUCE_XOR: 2238 return ISD::ANY_EXTEND; 2239 case ISD::VECREDUCE_SMAX: 2240 case ISD::VECREDUCE_SMIN: 2241 case ISD::VP_REDUCE_SMAX: 2242 case ISD::VP_REDUCE_SMIN: 2243 return ISD::SIGN_EXTEND; 2244 case ISD::VECREDUCE_UMAX: 2245 case ISD::VECREDUCE_UMIN: 2246 case ISD::VP_REDUCE_UMAX: 2247 case ISD::VP_REDUCE_UMIN: 2248 return ISD::ZERO_EXTEND; 2249 } 2250 } 2251 2252 SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) { 2253 switch (getExtendForIntVecReduction(N)) { 2254 default: 2255 llvm_unreachable("Impossible extension kind for integer reduction"); 2256 case ISD::ANY_EXTEND: 2257 return GetPromotedInteger(V); 2258 case ISD::SIGN_EXTEND: 2259 return SExtPromotedInteger(V); 2260 case ISD::ZERO_EXTEND: 2261 return ZExtPromotedInteger(V); 2262 } 2263 } 2264 2265 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { 2266 SDLoc dl(N); 2267 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0)); 2268 2269 EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType(); 2270 EVT InVT = Op.getValueType(); 2271 EVT EltVT = InVT.getVectorElementType(); 2272 EVT ResVT = N->getValueType(0); 2273 unsigned Opcode = N->getOpcode(); 2274 2275 // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if 2276 // vecreduce_xor is not legal 2277 if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 && 2278 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_XOR, InVT) && 2279 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_ADD, InVT)) 2280 Opcode = ISD::VECREDUCE_ADD; 2281 2282 // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if 2283 // vecreduce_or is not legal 2284 else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 && 2285 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_OR, InVT) && 2286 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMAX, InVT)) 2287 Opcode = ISD::VECREDUCE_UMAX; 2288 2289 // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if 2290 // vecreduce_and is not legal 2291 else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 && 2292 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_AND, InVT) && 2293 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMIN, InVT)) 2294 Opcode = ISD::VECREDUCE_UMIN; 2295 2296 if (ResVT.bitsGE(EltVT)) 2297 return DAG.getNode(Opcode, SDLoc(N), ResVT, Op); 2298 2299 // Result size must be >= element size. If this is not the case after 2300 // promotion, also promote the result type and then truncate. 2301 SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op); 2302 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce); 2303 } 2304 2305 SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) { 2306 SDLoc DL(N); 2307 SDValue Op = N->getOperand(OpNo); 2308 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2309 2310 if (OpNo == 2) { // Mask 2311 // Update in place. 2312 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType()); 2313 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2314 } 2315 2316 assert(OpNo == 1 && "Unexpected operand for promotion"); 2317 2318 Op = PromoteIntOpVectorReduction(N, Op); 2319 2320 NewOps[OpNo] = Op; 2321 2322 EVT VT = N->getValueType(0); 2323 EVT EltVT = Op.getValueType().getScalarType(); 2324 2325 if (VT.bitsGE(EltVT)) 2326 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps); 2327 2328 // Result size must be >= element/start-value size. If this is not the case 2329 // after promotion, also promote both the start value and result type and 2330 // then truncate. 2331 NewOps[0] = 2332 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0)); 2333 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps); 2334 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce); 2335 } 2336 2337 SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) { 2338 SDValue Op = ZExtPromotedInteger(N->getOperand(1)); 2339 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); 2340 } 2341 2342 SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) { 2343 assert(OpNo > 1); // Because the first two arguments are guaranteed legal. 2344 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 2345 SDValue Operand = N->getOperand(OpNo); 2346 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); 2347 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); 2348 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2349 } 2350 2351 SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { 2352 assert(OpNo >= 7); 2353 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 2354 SDValue Operand = N->getOperand(OpNo); 2355 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); 2356 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); 2357 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2358 } 2359 2360 SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { 2361 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || 2362 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); 2363 2364 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); 2365 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2366 2367 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2368 } 2369 2370 //===----------------------------------------------------------------------===// 2371 // Integer Result Expansion 2372 //===----------------------------------------------------------------------===// 2373 2374 /// ExpandIntegerResult - This method is called when the specified result of the 2375 /// specified node is found to need expansion. At this point, the node may also 2376 /// have invalid operands or may have other results that need promotion, we just 2377 /// know that (at least) one result needs expansion. 2378 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 2379 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); 2380 dbgs() << "\n"); 2381 SDValue Lo, Hi; 2382 Lo = Hi = SDValue(); 2383 2384 // See if the target wants to custom expand this node. 2385 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 2386 return; 2387 2388 switch (N->getOpcode()) { 2389 default: 2390 #ifndef NDEBUG 2391 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 2392 N->dump(&DAG); dbgs() << "\n"; 2393 #endif 2394 report_fatal_error("Do not know how to expand the result of this " 2395 "operator!"); 2396 2397 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break; 2398 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 2399 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; 2400 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 2401 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 2402 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; 2403 2404 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 2405 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 2406 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 2407 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 2408 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 2409 2410 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 2411 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 2412 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 2413 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; 2414 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 2415 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; 2416 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 2417 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; 2418 case ISD::CTLZ_ZERO_UNDEF: 2419 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 2420 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 2421 case ISD::CTTZ_ZERO_UNDEF: 2422 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 2423 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break; 2424 case ISD::STRICT_FP_TO_SINT: 2425 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 2426 case ISD::STRICT_FP_TO_UINT: 2427 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 2428 case ISD::FP_TO_SINT_SAT: 2429 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; 2430 case ISD::STRICT_LLROUND: 2431 case ISD::STRICT_LLRINT: 2432 case ISD::LLROUND: 2433 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break; 2434 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 2435 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 2436 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; 2437 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 2438 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 2439 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 2440 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 2441 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 2442 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 2443 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 2444 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 2445 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 2446 2447 case ISD::ATOMIC_LOAD_ADD: 2448 case ISD::ATOMIC_LOAD_SUB: 2449 case ISD::ATOMIC_LOAD_AND: 2450 case ISD::ATOMIC_LOAD_CLR: 2451 case ISD::ATOMIC_LOAD_OR: 2452 case ISD::ATOMIC_LOAD_XOR: 2453 case ISD::ATOMIC_LOAD_NAND: 2454 case ISD::ATOMIC_LOAD_MIN: 2455 case ISD::ATOMIC_LOAD_MAX: 2456 case ISD::ATOMIC_LOAD_UMIN: 2457 case ISD::ATOMIC_LOAD_UMAX: 2458 case ISD::ATOMIC_SWAP: 2459 case ISD::ATOMIC_CMP_SWAP: { 2460 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 2461 SplitInteger(Tmp.first, Lo, Hi); 2462 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2463 break; 2464 } 2465 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 2466 AtomicSDNode *AN = cast<AtomicSDNode>(N); 2467 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 2468 SDValue Tmp = DAG.getAtomicCmpSwap( 2469 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 2470 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 2471 AN->getMemOperand()); 2472 2473 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 2474 // success simply by comparing the loaded value against the ingoing 2475 // comparison. 2476 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 2477 N->getOperand(2), ISD::SETEQ); 2478 2479 SplitInteger(Tmp, Lo, Hi); 2480 ReplaceValueWith(SDValue(N, 1), Success); 2481 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 2482 break; 2483 } 2484 2485 case ISD::AND: 2486 case ISD::OR: 2487 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 2488 2489 case ISD::UMAX: 2490 case ISD::SMAX: 2491 case ISD::UMIN: 2492 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; 2493 2494 case ISD::ADD: 2495 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 2496 2497 case ISD::ADDC: 2498 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 2499 2500 case ISD::ADDE: 2501 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 2502 2503 case ISD::ADDCARRY: 2504 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break; 2505 2506 case ISD::SADDO_CARRY: 2507 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; 2508 2509 case ISD::SHL: 2510 case ISD::SRA: 2511 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 2512 2513 case ISD::SADDO: 2514 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 2515 case ISD::UADDO: 2516 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 2517 case ISD::UMULO: 2518 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 2519 2520 case ISD::SADDSAT: 2521 case ISD::UADDSAT: 2522 case ISD::SSUBSAT: 2523 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; 2524 2525 case ISD::SSHLSAT: 2526 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; 2527 2528 case ISD::SMULFIX: 2529 case ISD::SMULFIXSAT: 2530 case ISD::UMULFIX: 2531 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; 2532 2533 case ISD::SDIVFIX: 2534 case ISD::SDIVFIXSAT: 2535 case ISD::UDIVFIX: 2536 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; 2537 2538 case ISD::VECREDUCE_ADD: 2539 case ISD::VECREDUCE_MUL: 2540 case ISD::VECREDUCE_AND: 2541 case ISD::VECREDUCE_OR: 2542 case ISD::VECREDUCE_XOR: 2543 case ISD::VECREDUCE_SMAX: 2544 case ISD::VECREDUCE_SMIN: 2545 case ISD::VECREDUCE_UMAX: 2546 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; 2547 2548 case ISD::ROTL: 2549 case ISD::ROTR: 2550 ExpandIntRes_Rotate(N, Lo, Hi); 2551 break; 2552 2553 case ISD::FSHL: 2554 case ISD::FSHR: 2555 ExpandIntRes_FunnelShift(N, Lo, Hi); 2556 break; 2557 2558 case ISD::VSCALE: 2559 ExpandIntRes_VSCALE(N, Lo, Hi); 2560 break; 2561 } 2562 2563 // If Lo/Hi is null, the sub-method took care of registering results etc. 2564 if (Lo.getNode()) 2565 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 2566 } 2567 2568 /// Lower an atomic node to the appropriate builtin call. 2569 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 2570 unsigned Opc = Node->getOpcode(); 2571 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2572 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering(); 2573 // Lower to outline atomic libcall if outline atomics enabled, 2574 // or to sync libcall otherwise 2575 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); 2576 EVT RetVT = Node->getValueType(0); 2577 TargetLowering::MakeLibCallOptions CallOptions; 2578 SmallVector<SDValue, 4> Ops; 2579 if (TLI.getLibcallName(LC)) { 2580 Ops.append(Node->op_begin() + 2, Node->op_end()); 2581 Ops.push_back(Node->getOperand(1)); 2582 } else { 2583 LC = RTLIB::getSYNC(Opc, VT); 2584 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2585 "Unexpected atomic op or value type!"); 2586 Ops.append(Node->op_begin() + 1, Node->op_end()); 2587 } 2588 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), 2589 Node->getOperand(0)); 2590 } 2591 2592 /// N is a shift by a value that needs to be expanded, 2593 /// and the shift amount is a constant 'Amt'. Expand the operation. 2594 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, 2595 SDValue &Lo, SDValue &Hi) { 2596 SDLoc DL(N); 2597 // Expand the incoming operand to be shifted, so that we have its parts 2598 SDValue InL, InH; 2599 GetExpandedInteger(N->getOperand(0), InL, InH); 2600 2601 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization 2602 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. 2603 if (!Amt) { 2604 Lo = InL; 2605 Hi = InH; 2606 return; 2607 } 2608 2609 EVT NVT = InL.getValueType(); 2610 unsigned VTBits = N->getValueType(0).getSizeInBits(); 2611 unsigned NVTBits = NVT.getSizeInBits(); 2612 EVT ShTy = N->getOperand(1).getValueType(); 2613 2614 if (N->getOpcode() == ISD::SHL) { 2615 if (Amt.uge(VTBits)) { 2616 Lo = Hi = DAG.getConstant(0, DL, NVT); 2617 } else if (Amt.ugt(NVTBits)) { 2618 Lo = DAG.getConstant(0, DL, NVT); 2619 Hi = DAG.getNode(ISD::SHL, DL, 2620 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2621 } else if (Amt == NVTBits) { 2622 Lo = DAG.getConstant(0, DL, NVT); 2623 Hi = InL; 2624 } else { 2625 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); 2626 Hi = DAG.getNode(ISD::OR, DL, NVT, 2627 DAG.getNode(ISD::SHL, DL, NVT, InH, 2628 DAG.getConstant(Amt, DL, ShTy)), 2629 DAG.getNode(ISD::SRL, DL, NVT, InL, 2630 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2631 } 2632 return; 2633 } 2634 2635 if (N->getOpcode() == ISD::SRL) { 2636 if (Amt.uge(VTBits)) { 2637 Lo = Hi = DAG.getConstant(0, DL, NVT); 2638 } else if (Amt.ugt(NVTBits)) { 2639 Lo = DAG.getNode(ISD::SRL, DL, 2640 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2641 Hi = DAG.getConstant(0, DL, NVT); 2642 } else if (Amt == NVTBits) { 2643 Lo = InH; 2644 Hi = DAG.getConstant(0, DL, NVT); 2645 } else { 2646 Lo = DAG.getNode(ISD::OR, DL, NVT, 2647 DAG.getNode(ISD::SRL, DL, NVT, InL, 2648 DAG.getConstant(Amt, DL, ShTy)), 2649 DAG.getNode(ISD::SHL, DL, NVT, InH, 2650 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2651 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2652 } 2653 return; 2654 } 2655 2656 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2657 if (Amt.uge(VTBits)) { 2658 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2659 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2660 } else if (Amt.ugt(NVTBits)) { 2661 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2662 DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2663 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2664 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2665 } else if (Amt == NVTBits) { 2666 Lo = InH; 2667 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2668 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2669 } else { 2670 Lo = DAG.getNode(ISD::OR, DL, NVT, 2671 DAG.getNode(ISD::SRL, DL, NVT, InL, 2672 DAG.getConstant(Amt, DL, ShTy)), 2673 DAG.getNode(ISD::SHL, DL, NVT, InH, 2674 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2675 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2676 } 2677 } 2678 2679 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 2680 /// this shift based on knowledge of the high bit of the shift amount. If we 2681 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual 2682 /// shift amount. 2683 bool DAGTypeLegalizer:: 2684 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2685 SDValue Amt = N->getOperand(1); 2686 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2687 EVT ShTy = Amt.getValueType(); 2688 unsigned ShBits = ShTy.getScalarSizeInBits(); 2689 unsigned NVTBits = NVT.getScalarSizeInBits(); 2690 assert(isPowerOf2_32(NVTBits) && 2691 "Expanded integer type size not a power of two!"); 2692 SDLoc dl(N); 2693 2694 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 2695 KnownBits Known = DAG.computeKnownBits(N->getOperand(1)); 2696 2697 // If we don't know anything about the high bits, exit. 2698 if (((Known.Zero|Known.One) & HighBitMask) == 0) 2699 return false; 2700 2701 // Get the incoming operand to be shifted. 2702 SDValue InL, InH; 2703 GetExpandedInteger(N->getOperand(0), InL, InH); 2704 2705 // If we know that any of the high bits of the shift amount are one, then we 2706 // can do this as a couple of simple shifts. 2707 if (Known.One.intersects(HighBitMask)) { 2708 // Mask out the high bit, which we know is set. 2709 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 2710 DAG.getConstant(~HighBitMask, dl, ShTy)); 2711 2712 switch (N->getOpcode()) { 2713 default: llvm_unreachable("Unknown shift"); 2714 case ISD::SHL: 2715 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. 2716 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 2717 return true; 2718 case ISD::SRL: 2719 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2720 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 2721 return true; 2722 case ISD::SRA: 2723 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 2724 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2725 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 2726 return true; 2727 } 2728 } 2729 2730 // If we know that all of the high bits of the shift amount are zero, then we 2731 // can do this as a couple of simple shifts. 2732 if (HighBitMask.isSubsetOf(Known.Zero)) { 2733 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 2734 // shift if x is zero. We can use XOR here because x is known to be smaller 2735 // than 32. 2736 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 2737 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2738 2739 unsigned Op1, Op2; 2740 switch (N->getOpcode()) { 2741 default: llvm_unreachable("Unknown shift"); 2742 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 2743 case ISD::SRL: 2744 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 2745 } 2746 2747 // When shifting right the arithmetic for Lo and Hi is swapped. 2748 if (N->getOpcode() != ISD::SHL) 2749 std::swap(InL, InH); 2750 2751 // Use a little trick to get the bits that move from Lo to Hi. First 2752 // shift by one bit. 2753 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); 2754 // Then compute the remaining shift with amount-1. 2755 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 2756 2757 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 2758 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 2759 2760 if (N->getOpcode() != ISD::SHL) 2761 std::swap(Hi, Lo); 2762 return true; 2763 } 2764 2765 return false; 2766 } 2767 2768 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 2769 /// of any size. 2770 bool DAGTypeLegalizer:: 2771 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2772 SDValue Amt = N->getOperand(1); 2773 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2774 EVT ShTy = Amt.getValueType(); 2775 unsigned NVTBits = NVT.getSizeInBits(); 2776 assert(isPowerOf2_32(NVTBits) && 2777 "Expanded integer type size not a power of two!"); 2778 SDLoc dl(N); 2779 2780 // Get the incoming operand to be shifted. 2781 SDValue InL, InH; 2782 GetExpandedInteger(N->getOperand(0), InL, InH); 2783 2784 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); 2785 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 2786 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 2787 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2788 Amt, NVBitsNode, ISD::SETULT); 2789 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2790 Amt, DAG.getConstant(0, dl, ShTy), 2791 ISD::SETEQ); 2792 2793 SDValue LoS, HiS, LoL, HiL; 2794 switch (N->getOpcode()) { 2795 default: llvm_unreachable("Unknown shift"); 2796 case ISD::SHL: 2797 // Short: ShAmt < NVTBits 2798 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 2799 HiS = DAG.getNode(ISD::OR, dl, NVT, 2800 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 2801 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 2802 2803 // Long: ShAmt >= NVTBits 2804 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. 2805 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 2806 2807 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 2808 Hi = DAG.getSelect(dl, NVT, isZero, InH, 2809 DAG.getSelect(dl, NVT, isShort, HiS, HiL)); 2810 return true; 2811 case ISD::SRL: 2812 // Short: ShAmt < NVTBits 2813 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 2814 LoS = DAG.getNode(ISD::OR, dl, NVT, 2815 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2816 // FIXME: If Amt is zero, the following shift generates an undefined result 2817 // on some architectures. 2818 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2819 2820 // Long: ShAmt >= NVTBits 2821 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2822 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2823 2824 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2825 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2826 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2827 return true; 2828 case ISD::SRA: 2829 // Short: ShAmt < NVTBits 2830 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 2831 LoS = DAG.getNode(ISD::OR, dl, NVT, 2832 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2833 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2834 2835 // Long: ShAmt >= NVTBits 2836 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 2837 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2838 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2839 2840 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2841 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2842 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2843 return true; 2844 } 2845 } 2846 2847 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { 2848 2849 switch (Op) { 2850 default: llvm_unreachable("invalid min/max opcode"); 2851 case ISD::SMAX: 2852 return std::make_pair(ISD::SETGT, ISD::UMAX); 2853 case ISD::UMAX: 2854 return std::make_pair(ISD::SETUGT, ISD::UMAX); 2855 case ISD::SMIN: 2856 return std::make_pair(ISD::SETLT, ISD::UMIN); 2857 case ISD::UMIN: 2858 return std::make_pair(ISD::SETULT, ISD::UMIN); 2859 } 2860 } 2861 2862 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, 2863 SDValue &Lo, SDValue &Hi) { 2864 SDLoc DL(N); 2865 ISD::NodeType LoOpc; 2866 ISD::CondCode CondC; 2867 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); 2868 2869 // Expand the subcomponents. 2870 SDValue LHSL, LHSH, RHSL, RHSH; 2871 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2872 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2873 2874 // Value types 2875 EVT NVT = LHSL.getValueType(); 2876 EVT CCT = getSetCCResultType(NVT); 2877 2878 // Hi part is always the same op 2879 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 2880 2881 // We need to know whether to select Lo part that corresponds to 'winning' 2882 // Hi part or if Hi parts are equal. 2883 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); 2884 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); 2885 2886 // Lo part corresponding to the 'winning' Hi part 2887 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); 2888 2889 // Recursed Lo part if Hi parts are equal, this uses unsigned version 2890 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); 2891 2892 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); 2893 } 2894 2895 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 2896 SDValue &Lo, SDValue &Hi) { 2897 SDLoc dl(N); 2898 // Expand the subcomponents. 2899 SDValue LHSL, LHSH, RHSL, RHSH; 2900 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2901 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2902 2903 EVT NVT = LHSL.getValueType(); 2904 SDValue LoOps[2] = { LHSL, RHSL }; 2905 SDValue HiOps[3] = { LHSH, RHSH }; 2906 2907 bool HasOpCarry = TLI.isOperationLegalOrCustom( 2908 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY, 2909 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2910 if (HasOpCarry) { 2911 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 2912 if (N->getOpcode() == ISD::ADD) { 2913 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 2914 HiOps[2] = Lo.getValue(1); 2915 Hi = DAG.computeKnownBits(HiOps[2]).isZero() 2916 ? DAG.getNode(ISD::UADDO, dl, VTList, makeArrayRef(HiOps, 2)) 2917 : DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps); 2918 } else { 2919 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 2920 HiOps[2] = Lo.getValue(1); 2921 Hi = DAG.computeKnownBits(HiOps[2]).isZero() 2922 ? DAG.getNode(ISD::USUBO, dl, VTList, makeArrayRef(HiOps, 2)) 2923 : DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps); 2924 } 2925 return; 2926 } 2927 2928 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 2929 // them. TODO: Teach operation legalization how to expand unsupported 2930 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 2931 // a carry of type MVT::Glue, but there doesn't seem to be any way to 2932 // generate a value of this type in the expanded code sequence. 2933 bool hasCarry = 2934 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 2935 ISD::ADDC : ISD::SUBC, 2936 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2937 2938 if (hasCarry) { 2939 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 2940 if (N->getOpcode() == ISD::ADD) { 2941 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 2942 HiOps[2] = Lo.getValue(1); 2943 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 2944 } else { 2945 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 2946 HiOps[2] = Lo.getValue(1); 2947 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 2948 } 2949 return; 2950 } 2951 2952 bool hasOVF = 2953 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 2954 ISD::UADDO : ISD::USUBO, 2955 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2956 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); 2957 2958 if (hasOVF) { 2959 EVT OvfVT = getSetCCResultType(NVT); 2960 SDVTList VTList = DAG.getVTList(NVT, OvfVT); 2961 int RevOpc; 2962 if (N->getOpcode() == ISD::ADD) { 2963 RevOpc = ISD::SUB; 2964 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 2965 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 2966 } else { 2967 RevOpc = ISD::ADD; 2968 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 2969 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 2970 } 2971 SDValue OVF = Lo.getValue(1); 2972 2973 switch (BoolType) { 2974 case TargetLoweringBase::UndefinedBooleanContent: 2975 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); 2976 LLVM_FALLTHROUGH; 2977 case TargetLoweringBase::ZeroOrOneBooleanContent: 2978 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); 2979 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); 2980 break; 2981 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 2982 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); 2983 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); 2984 } 2985 return; 2986 } 2987 2988 if (N->getOpcode() == ISD::ADD) { 2989 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 2990 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 2991 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 2992 ISD::SETULT); 2993 2994 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) { 2995 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT); 2996 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); 2997 return; 2998 } 2999 3000 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1, 3001 DAG.getConstant(1, dl, NVT), 3002 DAG.getConstant(0, dl, NVT)); 3003 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1], 3004 ISD::SETULT); 3005 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2, 3006 DAG.getConstant(1, dl, NVT), Carry1); 3007 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 3008 } else { 3009 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 3010 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 3011 SDValue Cmp = 3012 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 3013 LoOps[0], LoOps[1], ISD::SETULT); 3014 3015 SDValue Borrow; 3016 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 3017 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); 3018 else 3019 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 3020 DAG.getConstant(0, dl, NVT)); 3021 3022 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 3023 } 3024 } 3025 3026 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 3027 SDValue &Lo, SDValue &Hi) { 3028 // Expand the subcomponents. 3029 SDValue LHSL, LHSH, RHSL, RHSH; 3030 SDLoc dl(N); 3031 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3032 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3033 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 3034 SDValue LoOps[2] = { LHSL, RHSL }; 3035 SDValue HiOps[3] = { LHSH, RHSH }; 3036 3037 if (N->getOpcode() == ISD::ADDC) { 3038 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 3039 HiOps[2] = Lo.getValue(1); 3040 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 3041 } else { 3042 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 3043 HiOps[2] = Lo.getValue(1); 3044 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 3045 } 3046 3047 // Legalized the flag result - switch anything that used the old flag to 3048 // use the new one. 3049 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3050 } 3051 3052 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 3053 SDValue &Lo, SDValue &Hi) { 3054 // Expand the subcomponents. 3055 SDValue LHSL, LHSH, RHSL, RHSH; 3056 SDLoc dl(N); 3057 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3058 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3059 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 3060 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 3061 SDValue HiOps[3] = { LHSH, RHSH }; 3062 3063 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3064 HiOps[2] = Lo.getValue(1); 3065 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 3066 3067 // Legalized the flag result - switch anything that used the old flag to 3068 // use the new one. 3069 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3070 } 3071 3072 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 3073 SDValue &Lo, SDValue &Hi) { 3074 SDValue LHS = N->getOperand(0); 3075 SDValue RHS = N->getOperand(1); 3076 SDLoc dl(N); 3077 3078 SDValue Ovf; 3079 3080 unsigned CarryOp, NoCarryOp; 3081 ISD::CondCode Cond; 3082 switch(N->getOpcode()) { 3083 case ISD::UADDO: 3084 CarryOp = ISD::ADDCARRY; 3085 NoCarryOp = ISD::ADD; 3086 Cond = ISD::SETULT; 3087 break; 3088 case ISD::USUBO: 3089 CarryOp = ISD::SUBCARRY; 3090 NoCarryOp = ISD::SUB; 3091 Cond = ISD::SETUGT; 3092 break; 3093 default: 3094 llvm_unreachable("Node has unexpected Opcode"); 3095 } 3096 3097 bool HasCarryOp = TLI.isOperationLegalOrCustom( 3098 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 3099 3100 if (HasCarryOp) { 3101 // Expand the subcomponents. 3102 SDValue LHSL, LHSH, RHSL, RHSH; 3103 GetExpandedInteger(LHS, LHSL, LHSH); 3104 GetExpandedInteger(RHS, RHSL, RHSH); 3105 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3106 SDValue LoOps[2] = { LHSL, RHSL }; 3107 SDValue HiOps[3] = { LHSH, RHSH }; 3108 3109 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3110 HiOps[2] = Lo.getValue(1); 3111 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps); 3112 3113 Ovf = Hi.getValue(1); 3114 } else { 3115 // Expand the result by simply replacing it with the equivalent 3116 // non-overflow-checking operation. 3117 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS); 3118 SplitInteger(Sum, Lo, Hi); 3119 3120 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 3121 // overflows iff a - b > a. 3122 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); 3123 } 3124 3125 // Legalized the flag result - switch anything that used the old flag to 3126 // use the new one. 3127 ReplaceValueWith(SDValue(N, 1), Ovf); 3128 } 3129 3130 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N, 3131 SDValue &Lo, SDValue &Hi) { 3132 // Expand the subcomponents. 3133 SDValue LHSL, LHSH, RHSL, RHSH; 3134 SDLoc dl(N); 3135 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3136 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3137 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3138 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 3139 SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; 3140 3141 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3142 HiOps[2] = Lo.getValue(1); 3143 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 3144 3145 // Legalized the flag result - switch anything that used the old flag to 3146 // use the new one. 3147 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3148 } 3149 3150 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, 3151 SDValue &Lo, SDValue &Hi) { 3152 // Expand the subcomponents. 3153 SDValue LHSL, LHSH, RHSL, RHSH; 3154 SDLoc dl(N); 3155 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3156 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3157 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3158 3159 // We need to use an unsigned carry op for the lo part. 3160 unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY 3161 : ISD::SUBCARRY; 3162 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) }); 3163 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 3164 3165 // Legalized the flag result - switch anything that used the old flag to 3166 // use the new one. 3167 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3168 } 3169 3170 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 3171 SDValue &Lo, SDValue &Hi) { 3172 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3173 SDLoc dl(N); 3174 SDValue Op = N->getOperand(0); 3175 if (Op.getValueType().bitsLE(NVT)) { 3176 // The low part is any extension of the input (which degenerates to a copy). 3177 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 3178 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 3179 } else { 3180 // For example, extension of an i48 to an i64. The operand type necessarily 3181 // promotes to the result type, so will end up being expanded too. 3182 assert(getTypeAction(Op.getValueType()) == 3183 TargetLowering::TypePromoteInteger && 3184 "Only know how to promote this result!"); 3185 SDValue Res = GetPromotedInteger(Op); 3186 assert(Res.getValueType() == N->getValueType(0) && 3187 "Operand over promoted?"); 3188 // Split the promoted operand. This will simplify when it is expanded. 3189 SplitInteger(Res, Lo, Hi); 3190 } 3191 } 3192 3193 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 3194 SDValue &Lo, SDValue &Hi) { 3195 SDLoc dl(N); 3196 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3197 EVT NVT = Lo.getValueType(); 3198 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3199 unsigned NVTBits = NVT.getSizeInBits(); 3200 unsigned EVTBits = EVT.getSizeInBits(); 3201 3202 if (NVTBits < EVTBits) { 3203 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 3204 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3205 EVTBits - NVTBits))); 3206 } else { 3207 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 3208 // The high part replicates the sign bit of Lo, make it explicit. 3209 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3210 DAG.getConstant(NVTBits - 1, dl, 3211 TLI.getPointerTy(DAG.getDataLayout()))); 3212 } 3213 } 3214 3215 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 3216 SDValue &Lo, SDValue &Hi) { 3217 SDLoc dl(N); 3218 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3219 EVT NVT = Lo.getValueType(); 3220 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3221 unsigned NVTBits = NVT.getSizeInBits(); 3222 unsigned EVTBits = EVT.getSizeInBits(); 3223 3224 if (NVTBits < EVTBits) { 3225 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 3226 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3227 EVTBits - NVTBits))); 3228 } else { 3229 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 3230 // The high part must be zero, make it explicit. 3231 Hi = DAG.getConstant(0, dl, NVT); 3232 } 3233 } 3234 3235 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, 3236 SDValue &Lo, SDValue &Hi) { 3237 SDLoc dl(N); 3238 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 3239 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); 3240 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); 3241 } 3242 3243 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 3244 SDValue &Lo, SDValue &Hi) { 3245 SDLoc dl(N); 3246 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 3247 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 3248 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 3249 } 3250 3251 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, 3252 SDValue &Hi) { 3253 SDLoc dl(N); 3254 // parity(HiLo) -> parity(Lo^Hi) 3255 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3256 EVT NVT = Lo.getValueType(); 3257 Lo = 3258 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi)); 3259 Hi = DAG.getConstant(0, dl, NVT); 3260 } 3261 3262 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 3263 SDValue &Lo, SDValue &Hi) { 3264 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3265 unsigned NBitWidth = NVT.getSizeInBits(); 3266 auto Constant = cast<ConstantSDNode>(N); 3267 const APInt &Cst = Constant->getAPIntValue(); 3268 bool IsTarget = Constant->isTargetOpcode(); 3269 bool IsOpaque = Constant->isOpaque(); 3270 SDLoc dl(N); 3271 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); 3272 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, 3273 IsOpaque); 3274 } 3275 3276 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { 3277 SDLoc dl(N); 3278 3279 SDValue N0 = N->getOperand(0); 3280 GetExpandedInteger(N0, Lo, Hi); 3281 EVT NVT = Lo.getValueType(); 3282 3283 // If we have SUBCARRY, use the expanded form of the sra+xor+sub sequence we 3284 // use in LegalizeDAG. The SUB part of the expansion is based on 3285 // ExpandIntRes_ADDSUB which also uses SUBCARRY/USUBO after checking that 3286 // SUBCARRY is LegalOrCustom. Each of the pieces here can be further expanded 3287 // if needed. Shift expansion has a special case for filling with sign bits 3288 // so that we will only end up with one SRA. 3289 bool HasSubCarry = TLI.isOperationLegalOrCustom( 3290 ISD::SUBCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 3291 if (HasSubCarry) { 3292 SDValue Sign = DAG.getNode( 3293 ISD::SRA, dl, NVT, Hi, 3294 DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl)); 3295 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 3296 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign); 3297 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign); 3298 Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign); 3299 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, Hi, Sign, Lo.getValue(1)); 3300 return; 3301 } 3302 3303 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) 3304 EVT VT = N->getValueType(0); 3305 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, 3306 DAG.getConstant(0, dl, VT), N0); 3307 SDValue NegLo, NegHi; 3308 SplitInteger(Neg, NegLo, NegHi); 3309 3310 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 3311 DAG.getConstant(0, dl, NVT), ISD::SETLT); 3312 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); 3313 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); 3314 } 3315 3316 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 3317 SDValue &Lo, SDValue &Hi) { 3318 SDLoc dl(N); 3319 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 3320 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3321 EVT NVT = Lo.getValueType(); 3322 3323 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 3324 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3325 3326 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 3327 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 3328 3329 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 3330 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 3331 DAG.getConstant(NVT.getSizeInBits(), dl, 3332 NVT))); 3333 Hi = DAG.getConstant(0, dl, NVT); 3334 } 3335 3336 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 3337 SDValue &Lo, SDValue &Hi) { 3338 SDLoc dl(N); 3339 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 3340 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3341 EVT NVT = Lo.getValueType(); 3342 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 3343 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 3344 Hi = DAG.getConstant(0, dl, NVT); 3345 } 3346 3347 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 3348 SDValue &Lo, SDValue &Hi) { 3349 SDLoc dl(N); 3350 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 3351 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3352 EVT NVT = Lo.getValueType(); 3353 3354 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 3355 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3356 3357 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 3358 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 3359 3360 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 3361 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 3362 DAG.getConstant(NVT.getSizeInBits(), dl, 3363 NVT))); 3364 Hi = DAG.getConstant(0, dl, NVT); 3365 } 3366 3367 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo, 3368 SDValue &Hi) { 3369 SDLoc dl(N); 3370 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3371 unsigned NBitWidth = NVT.getSizeInBits(); 3372 3373 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0)); 3374 SDValue Chain = Lo.getValue(1); 3375 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS 3376 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3377 DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl)); 3378 3379 // Legalize the chain result - switch anything that used the old chain to 3380 // use the new one. 3381 ReplaceValueWith(SDValue(N, 1), Chain); 3382 } 3383 3384 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 3385 SDValue &Hi) { 3386 SDLoc dl(N); 3387 EVT VT = N->getValueType(0); 3388 3389 bool IsStrict = N->isStrictFPOpcode(); 3390 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3391 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3392 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3393 Op = GetPromotedFloat(Op); 3394 3395 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3396 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3397 Op = GetSoftPromotedHalf(Op); 3398 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); 3399 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op); 3400 SplitInteger(Op, Lo, Hi); 3401 return; 3402 } 3403 3404 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 3405 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 3406 TargetLowering::MakeLibCallOptions CallOptions; 3407 CallOptions.setSExt(true); 3408 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3409 CallOptions, dl, Chain); 3410 SplitInteger(Tmp.first, Lo, Hi); 3411 3412 if (IsStrict) 3413 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3414 } 3415 3416 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 3417 SDValue &Hi) { 3418 SDLoc dl(N); 3419 EVT VT = N->getValueType(0); 3420 3421 bool IsStrict = N->isStrictFPOpcode(); 3422 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3423 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3424 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3425 Op = GetPromotedFloat(Op); 3426 3427 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3428 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3429 Op = GetSoftPromotedHalf(Op); 3430 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); 3431 Op = DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op); 3432 SplitInteger(Op, Lo, Hi); 3433 return; 3434 } 3435 3436 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 3437 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 3438 TargetLowering::MakeLibCallOptions CallOptions; 3439 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3440 CallOptions, dl, Chain); 3441 SplitInteger(Tmp.first, Lo, Hi); 3442 3443 if (IsStrict) 3444 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3445 } 3446 3447 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, 3448 SDValue &Hi) { 3449 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 3450 SplitInteger(Res, Lo, Hi); 3451 } 3452 3453 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo, 3454 SDValue &Hi) { 3455 SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); 3456 3457 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && 3458 "Input type needs to be promoted!"); 3459 3460 EVT VT = Op.getValueType(); 3461 3462 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3463 if (N->getOpcode() == ISD::LLROUND || 3464 N->getOpcode() == ISD::STRICT_LLROUND) { 3465 if (VT == MVT::f32) 3466 LC = RTLIB::LLROUND_F32; 3467 else if (VT == MVT::f64) 3468 LC = RTLIB::LLROUND_F64; 3469 else if (VT == MVT::f80) 3470 LC = RTLIB::LLROUND_F80; 3471 else if (VT == MVT::f128) 3472 LC = RTLIB::LLROUND_F128; 3473 else if (VT == MVT::ppcf128) 3474 LC = RTLIB::LLROUND_PPCF128; 3475 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); 3476 } else if (N->getOpcode() == ISD::LLRINT || 3477 N->getOpcode() == ISD::STRICT_LLRINT) { 3478 if (VT == MVT::f32) 3479 LC = RTLIB::LLRINT_F32; 3480 else if (VT == MVT::f64) 3481 LC = RTLIB::LLRINT_F64; 3482 else if (VT == MVT::f80) 3483 LC = RTLIB::LLRINT_F80; 3484 else if (VT == MVT::f128) 3485 LC = RTLIB::LLRINT_F128; 3486 else if (VT == MVT::ppcf128) 3487 LC = RTLIB::LLRINT_PPCF128; 3488 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); 3489 } else 3490 llvm_unreachable("Unexpected opcode!"); 3491 3492 SDLoc dl(N); 3493 EVT RetVT = N->getValueType(0); 3494 SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue(); 3495 3496 TargetLowering::MakeLibCallOptions CallOptions; 3497 CallOptions.setSExt(true); 3498 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, 3499 Op, CallOptions, dl, 3500 Chain); 3501 SplitInteger(Tmp.first, Lo, Hi); 3502 3503 if (N->isStrictFPOpcode()) 3504 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3505 } 3506 3507 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 3508 SDValue &Lo, SDValue &Hi) { 3509 if (N->isAtomic()) { 3510 // It's typical to have larger CAS than atomic load instructions. 3511 SDLoc dl(N); 3512 EVT VT = N->getMemoryVT(); 3513 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 3514 SDValue Zero = DAG.getConstant(0, dl, VT); 3515 SDValue Swap = DAG.getAtomicCmpSwap( 3516 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 3517 VT, VTs, N->getOperand(0), 3518 N->getOperand(1), Zero, Zero, N->getMemOperand()); 3519 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 3520 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 3521 return; 3522 } 3523 3524 if (ISD::isNormalLoad(N)) { 3525 ExpandRes_NormalLoad(N, Lo, Hi); 3526 return; 3527 } 3528 3529 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 3530 3531 EVT VT = N->getValueType(0); 3532 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3533 SDValue Ch = N->getChain(); 3534 SDValue Ptr = N->getBasePtr(); 3535 ISD::LoadExtType ExtType = N->getExtensionType(); 3536 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 3537 AAMDNodes AAInfo = N->getAAInfo(); 3538 SDLoc dl(N); 3539 3540 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 3541 3542 if (N->getMemoryVT().bitsLE(NVT)) { 3543 EVT MemVT = N->getMemoryVT(); 3544 3545 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, 3546 N->getOriginalAlign(), MMOFlags, AAInfo); 3547 3548 // Remember the chain. 3549 Ch = Lo.getValue(1); 3550 3551 if (ExtType == ISD::SEXTLOAD) { 3552 // The high part is obtained by SRA'ing all but one of the bits of the 3553 // lo part. 3554 unsigned LoSize = Lo.getValueSizeInBits(); 3555 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3556 DAG.getConstant(LoSize - 1, dl, 3557 TLI.getPointerTy(DAG.getDataLayout()))); 3558 } else if (ExtType == ISD::ZEXTLOAD) { 3559 // The high part is just a zero. 3560 Hi = DAG.getConstant(0, dl, NVT); 3561 } else { 3562 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 3563 // The high part is undefined. 3564 Hi = DAG.getUNDEF(NVT); 3565 } 3566 } else if (DAG.getDataLayout().isLittleEndian()) { 3567 // Little-endian - low bits are at low addresses. 3568 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 3569 N->getOriginalAlign(), MMOFlags, AAInfo); 3570 3571 unsigned ExcessBits = 3572 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 3573 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 3574 3575 // Increment the pointer to the other half. 3576 unsigned IncrementSize = NVT.getSizeInBits()/8; 3577 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 3578 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 3579 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 3580 N->getOriginalAlign(), MMOFlags, AAInfo); 3581 3582 // Build a factor node to remember that this load is independent of the 3583 // other one. 3584 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3585 Hi.getValue(1)); 3586 } else { 3587 // Big-endian - high bits are at low addresses. Favor aligned loads at 3588 // the cost of some bit-fiddling. 3589 EVT MemVT = N->getMemoryVT(); 3590 unsigned EBytes = MemVT.getStoreSize(); 3591 unsigned IncrementSize = NVT.getSizeInBits()/8; 3592 unsigned ExcessBits = (EBytes - IncrementSize)*8; 3593 3594 // Load both the high bits and maybe some of the low bits. 3595 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 3596 EVT::getIntegerVT(*DAG.getContext(), 3597 MemVT.getSizeInBits() - ExcessBits), 3598 N->getOriginalAlign(), MMOFlags, AAInfo); 3599 3600 // Increment the pointer to the other half. 3601 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 3602 // Load the rest of the low bits. 3603 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 3604 N->getPointerInfo().getWithOffset(IncrementSize), 3605 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 3606 N->getOriginalAlign(), MMOFlags, AAInfo); 3607 3608 // Build a factor node to remember that this load is independent of the 3609 // other one. 3610 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3611 Hi.getValue(1)); 3612 3613 if (ExcessBits < NVT.getSizeInBits()) { 3614 // Transfer low bits from the bottom of Hi to the top of Lo. 3615 Lo = DAG.getNode( 3616 ISD::OR, dl, NVT, Lo, 3617 DAG.getNode(ISD::SHL, dl, NVT, Hi, 3618 DAG.getConstant(ExcessBits, dl, 3619 TLI.getPointerTy(DAG.getDataLayout())))); 3620 // Move high bits to the right position in Hi. 3621 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, 3622 Hi, 3623 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 3624 TLI.getPointerTy(DAG.getDataLayout()))); 3625 } 3626 } 3627 3628 // Legalize the chain result - switch anything that used the old chain to 3629 // use the new one. 3630 ReplaceValueWith(SDValue(N, 1), Ch); 3631 } 3632 3633 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 3634 SDValue &Lo, SDValue &Hi) { 3635 SDLoc dl(N); 3636 SDValue LL, LH, RL, RH; 3637 GetExpandedInteger(N->getOperand(0), LL, LH); 3638 GetExpandedInteger(N->getOperand(1), RL, RH); 3639 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 3640 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 3641 } 3642 3643 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 3644 SDValue &Lo, SDValue &Hi) { 3645 EVT VT = N->getValueType(0); 3646 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3647 SDLoc dl(N); 3648 3649 SDValue LL, LH, RL, RH; 3650 GetExpandedInteger(N->getOperand(0), LL, LH); 3651 GetExpandedInteger(N->getOperand(1), RL, RH); 3652 3653 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, 3654 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3655 LL, LH, RL, RH)) 3656 return; 3657 3658 // If nothing else, we can make a libcall. 3659 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3660 if (VT == MVT::i16) 3661 LC = RTLIB::MUL_I16; 3662 else if (VT == MVT::i32) 3663 LC = RTLIB::MUL_I32; 3664 else if (VT == MVT::i64) 3665 LC = RTLIB::MUL_I64; 3666 else if (VT == MVT::i128) 3667 LC = RTLIB::MUL_I128; 3668 3669 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 3670 // We'll expand the multiplication by brute force because we have no other 3671 // options. This is a trivially-generalized version of the code from 3672 // Hacker's Delight (itself derived from Knuth's Algorithm M from section 3673 // 4.3.1). 3674 unsigned Bits = NVT.getSizeInBits(); 3675 unsigned HalfBits = Bits >> 1; 3676 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, 3677 NVT); 3678 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); 3679 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); 3680 3681 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); 3682 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); 3683 3684 SDValue Shift = DAG.getShiftAmountConstant(HalfBits, NVT, dl); 3685 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); 3686 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); 3687 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); 3688 3689 SDValue U = DAG.getNode(ISD::ADD, dl, NVT, 3690 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); 3691 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); 3692 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); 3693 3694 SDValue V = DAG.getNode(ISD::ADD, dl, NVT, 3695 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); 3696 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); 3697 3698 SDValue W = DAG.getNode(ISD::ADD, dl, NVT, 3699 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), 3700 DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); 3701 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, 3702 DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); 3703 3704 Hi = DAG.getNode(ISD::ADD, dl, NVT, W, 3705 DAG.getNode(ISD::ADD, dl, NVT, 3706 DAG.getNode(ISD::MUL, dl, NVT, RH, LL), 3707 DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); 3708 return; 3709 } 3710 3711 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3712 TargetLowering::MakeLibCallOptions CallOptions; 3713 CallOptions.setSExt(true); 3714 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, 3715 Lo, Hi); 3716 } 3717 3718 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, 3719 SDValue &Hi) { 3720 SDLoc DL(N); 3721 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3722 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); 3723 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); 3724 Lo = R.getValue(0); 3725 Hi = R.getValue(1); 3726 ReplaceValueWith(SDValue(N, 1), R.getValue(2)); 3727 } 3728 3729 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, 3730 SDValue &Hi) { 3731 SDValue Result = TLI.expandAddSubSat(N, DAG); 3732 SplitInteger(Result, Lo, Hi); 3733 } 3734 3735 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, 3736 SDValue &Hi) { 3737 SDValue Result = TLI.expandShlSat(N, DAG); 3738 SplitInteger(Result, Lo, Hi); 3739 } 3740 3741 /// This performs an expansion of the integer result for a fixed point 3742 /// multiplication. The default expansion performs rounding down towards 3743 /// negative infinity, though targets that do care about rounding should specify 3744 /// a target hook for rounding and provide their own expansion or lowering of 3745 /// fixed point multiplication to be consistent with rounding. 3746 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, 3747 SDValue &Hi) { 3748 SDLoc dl(N); 3749 EVT VT = N->getValueType(0); 3750 unsigned VTSize = VT.getScalarSizeInBits(); 3751 SDValue LHS = N->getOperand(0); 3752 SDValue RHS = N->getOperand(1); 3753 uint64_t Scale = N->getConstantOperandVal(2); 3754 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || 3755 N->getOpcode() == ISD::UMULFIXSAT); 3756 bool Signed = (N->getOpcode() == ISD::SMULFIX || 3757 N->getOpcode() == ISD::SMULFIXSAT); 3758 3759 // Handle special case when scale is equal to zero. 3760 if (!Scale) { 3761 SDValue Result; 3762 if (!Saturating) { 3763 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 3764 } else { 3765 EVT BoolVT = getSetCCResultType(VT); 3766 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; 3767 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); 3768 SDValue Product = Result.getValue(0); 3769 SDValue Overflow = Result.getValue(1); 3770 if (Signed) { 3771 APInt MinVal = APInt::getSignedMinValue(VTSize); 3772 APInt MaxVal = APInt::getSignedMaxValue(VTSize); 3773 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); 3774 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 3775 SDValue Zero = DAG.getConstant(0, dl, VT); 3776 // Xor the inputs, if resulting sign bit is 0 the product will be 3777 // positive, else negative. 3778 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); 3779 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); 3780 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); 3781 Result = DAG.getSelect(dl, VT, Overflow, Result, Product); 3782 } else { 3783 // For unsigned multiplication, we only need to check the max since we 3784 // can't really overflow towards zero. 3785 APInt MaxVal = APInt::getMaxValue(VTSize); 3786 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 3787 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product); 3788 } 3789 } 3790 SplitInteger(Result, Lo, Hi); 3791 return; 3792 } 3793 3794 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will 3795 // cover for unhandled cases below, while still being valid for UMULFIX[SAT]. 3796 assert(Scale <= VTSize && "Scale can't be larger than the value type size."); 3797 3798 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3799 SDValue LL, LH, RL, RH; 3800 GetExpandedInteger(LHS, LL, LH); 3801 GetExpandedInteger(RHS, RL, RH); 3802 SmallVector<SDValue, 4> Result; 3803 3804 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; 3805 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, 3806 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3807 LL, LH, RL, RH)) { 3808 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); 3809 return; 3810 } 3811 3812 unsigned NVTSize = NVT.getScalarSizeInBits(); 3813 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " 3814 "the size of the current value type"); 3815 3816 // After getting the multiplication result in 4 parts, we need to perform a 3817 // shift right by the amount of the scale to get the result in that scale. 3818 // 3819 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in 3820 // 128 bits that are cut into 4 32-bit parts: 3821 // 3822 // HH HL LH LL 3823 // |---32---|---32---|---32---|---32---| 3824 // 128 96 64 32 0 3825 // 3826 // |------VTSize-----| 3827 // 3828 // |NVTSize-| 3829 // 3830 // The resulting Lo and Hi would normally be in LL and LH after the shift. But 3831 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift 3832 // amount and get Lo and Hi using two funnel shifts. Or for the special case 3833 // when Scale is a multiple of NVTSize we can just pick the result without 3834 // shifting. 3835 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. 3836 if (Scale % NVTSize) { 3837 SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl); 3838 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0], 3839 ShiftAmount); 3840 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1], 3841 ShiftAmount); 3842 } else { 3843 Lo = Result[Part0]; 3844 Hi = Result[Part0 + 1]; 3845 } 3846 3847 // Unless saturation is requested we are done. The result is in <Hi,Lo>. 3848 if (!Saturating) 3849 return; 3850 3851 // Can not overflow when there is no integer part. 3852 if (Scale == VTSize) 3853 return; 3854 3855 // To handle saturation we must check for overflow in the multiplication. 3856 // 3857 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) 3858 // aren't all zeroes. 3859 // 3860 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) 3861 // aren't all ones or all zeroes. 3862 // 3863 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the 3864 // highest bit of HH determines saturation direction in the event of signed 3865 // saturation. 3866 3867 SDValue ResultHL = Result[2]; 3868 SDValue ResultHH = Result[3]; 3869 3870 SDValue SatMax, SatMin; 3871 SDValue NVTZero = DAG.getConstant(0, dl, NVT); 3872 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); 3873 EVT BoolNVT = getSetCCResultType(NVT); 3874 3875 if (!Signed) { 3876 if (Scale < NVTSize) { 3877 // Overflow happened if ((HH | (HL >> Scale)) != 0). 3878 SDValue HLAdjusted = 3879 DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 3880 DAG.getShiftAmountConstant(Scale, NVT, dl)); 3881 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH); 3882 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE); 3883 } else if (Scale == NVTSize) { 3884 // Overflow happened if (HH != 0). 3885 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE); 3886 } else if (Scale < VTSize) { 3887 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0). 3888 SDValue HLAdjusted = 3889 DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 3890 DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl)); 3891 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE); 3892 } else 3893 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" 3894 "(and saturation can't happen with Scale==VTSize)."); 3895 3896 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi); 3897 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo); 3898 return; 3899 } 3900 3901 if (Scale < NVTSize) { 3902 // The number of overflow bits we can check are VTSize - Scale + 1 (we 3903 // include the sign bit). If these top bits are > 0, then we overflowed past 3904 // the max value. If these top bits are < -1, then we overflowed past the 3905 // min value. Otherwise, we did not overflow. 3906 unsigned OverflowBits = VTSize - Scale + 1; 3907 assert(OverflowBits <= VTSize && OverflowBits > NVTSize && 3908 "Extent of overflow bits must start within HL"); 3909 SDValue HLHiMask = DAG.getConstant( 3910 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); 3911 SDValue HLLoMask = DAG.getConstant( 3912 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); 3913 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). 3914 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 3915 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 3916 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); 3917 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 3918 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT)); 3919 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). 3920 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 3921 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 3922 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); 3923 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 3924 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT)); 3925 } else if (Scale == NVTSize) { 3926 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). 3927 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 3928 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 3929 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); 3930 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 3931 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg)); 3932 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). 3933 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 3934 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 3935 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); 3936 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 3937 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos)); 3938 } else if (Scale < VTSize) { 3939 // This is similar to the case when we saturate if Scale < NVTSize, but we 3940 // only need to check HH. 3941 unsigned OverflowBits = VTSize - Scale + 1; 3942 SDValue HHHiMask = DAG.getConstant( 3943 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); 3944 SDValue HHLoMask = DAG.getConstant( 3945 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); 3946 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); 3947 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); 3948 } else 3949 llvm_unreachable("Illegal scale for signed fixed point mul."); 3950 3951 // Saturate to signed maximum. 3952 APInt MaxHi = APInt::getSignedMaxValue(NVTSize); 3953 APInt MaxLo = APInt::getAllOnes(NVTSize); 3954 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi); 3955 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo); 3956 // Saturate to signed minimum. 3957 APInt MinHi = APInt::getSignedMinValue(NVTSize); 3958 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi); 3959 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); 3960 } 3961 3962 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, 3963 SDValue &Hi) { 3964 SDLoc dl(N); 3965 // Try expanding in the existing type first. 3966 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0), 3967 N->getOperand(1), 3968 N->getConstantOperandVal(2), DAG); 3969 3970 if (!Res) 3971 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1), 3972 N->getConstantOperandVal(2), TLI, DAG); 3973 SplitInteger(Res, Lo, Hi); 3974 } 3975 3976 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 3977 SDValue &Lo, SDValue &Hi) { 3978 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && 3979 "Node has unexpected Opcode"); 3980 SDValue LHS = Node->getOperand(0); 3981 SDValue RHS = Node->getOperand(1); 3982 SDLoc dl(Node); 3983 3984 SDValue Ovf; 3985 3986 bool IsAdd = Node->getOpcode() == ISD::SADDO; 3987 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY; 3988 3989 bool HasCarryOp = TLI.isOperationLegalOrCustom( 3990 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 3991 3992 if (HasCarryOp) { 3993 // Expand the subcomponents. 3994 SDValue LHSL, LHSH, RHSL, RHSH; 3995 GetExpandedInteger(LHS, LHSL, LHSH); 3996 GetExpandedInteger(RHS, RHSL, RHSH); 3997 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1)); 3998 3999 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL}); 4000 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 4001 4002 Ovf = Hi.getValue(1); 4003 } else { 4004 // Expand the result by simply replacing it with the equivalent 4005 // non-overflow-checking operation. 4006 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 4007 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 4008 LHS, RHS); 4009 SplitInteger(Sum, Lo, Hi); 4010 4011 // Compute the overflow. 4012 // 4013 // LHSSign -> LHS < 0 4014 // RHSSign -> RHS < 0 4015 // SumSign -> Sum < 0 4016 // 4017 // Add: 4018 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 4019 // Sub: 4020 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 4021 // 4022 // To get better codegen we can rewrite this by doing bitwise math on 4023 // the integers and extract the final sign bit at the end. So the 4024 // above becomes: 4025 // 4026 // Add: 4027 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0 4028 // Sub: 4029 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0 4030 // 4031 // NOTE: This is different than the expansion we do in expandSADDSUBO 4032 // because it is more costly to determine the RHS is > 0 for SSUBO with the 4033 // integers split. 4034 EVT VT = LHS.getValueType(); 4035 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); 4036 if (IsAdd) 4037 SignsMatch = DAG.getNOT(dl, SignsMatch, VT); 4038 4039 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum); 4040 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE); 4041 EVT OType = Node->getValueType(1); 4042 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT); 4043 } 4044 4045 // Use the calculated overflow everywhere. 4046 ReplaceValueWith(SDValue(Node, 1), Ovf); 4047 } 4048 4049 // Emit a call to __udivei4 and friends which require 4050 // the arguments be based on the stack 4051 // and extra argument that contains the number of bits of the operands. 4052 // Returns the result of the call operation. 4053 static SDValue ExpandExtIntRes_DIVREM(const TargetLowering &TLI, 4054 const RTLIB::Libcall &LC, 4055 SelectionDAG &DAG, SDNode *N, 4056 const SDLoc &DL, const EVT &VT) { 4057 4058 SDValue InChain = DAG.getEntryNode(); 4059 4060 TargetLowering::ArgListTy Args; 4061 TargetLowering::ArgListEntry Entry; 4062 4063 // The signature of __udivei4 is 4064 // void __udivei4(unsigned int *quo, unsigned int *a, unsigned int *b, 4065 // unsigned int bits) 4066 EVT ArgVT = N->op_begin()->getValueType(); 4067 assert(ArgVT.isInteger() && ArgVT.getSizeInBits() > 128 && 4068 "Unexpected argument type for lowering"); 4069 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 4070 4071 SDValue Output = DAG.CreateStackTemporary(ArgVT); 4072 Entry.Node = Output; 4073 Entry.Ty = ArgTy->getPointerTo(); 4074 Entry.IsSExt = false; 4075 Entry.IsZExt = false; 4076 Args.push_back(Entry); 4077 4078 for (const llvm::SDUse &Op : N->ops()) { 4079 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT); 4080 InChain = DAG.getStore(InChain, DL, Op, StackPtr, MachinePointerInfo()); 4081 Entry.Node = StackPtr; 4082 Entry.Ty = ArgTy->getPointerTo(); 4083 Entry.IsSExt = false; 4084 Entry.IsZExt = false; 4085 Args.push_back(Entry); 4086 } 4087 4088 int Bits = N->getOperand(0) 4089 .getValueType() 4090 .getTypeForEVT(*DAG.getContext()) 4091 ->getIntegerBitWidth(); 4092 Entry.Node = DAG.getConstant(Bits, DL, TLI.getPointerTy(DAG.getDataLayout())); 4093 Entry.Ty = Type::getInt32Ty(*DAG.getContext()); 4094 Entry.IsSExt = false; 4095 Entry.IsZExt = true; 4096 Args.push_back(Entry); 4097 4098 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 4099 TLI.getPointerTy(DAG.getDataLayout())); 4100 4101 TargetLowering::CallLoweringInfo CLI(DAG); 4102 CLI.setDebugLoc(DL) 4103 .setChain(InChain) 4104 .setLibCallee(TLI.getLibcallCallingConv(LC), 4105 Type::getVoidTy(*DAG.getContext()), Callee, std::move(Args)) 4106 .setDiscardResult(); 4107 4108 SDValue Chain = TLI.LowerCallTo(CLI).second; 4109 4110 return DAG.getLoad(ArgVT, DL, Chain, Output, MachinePointerInfo()); 4111 } 4112 4113 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 4114 SDValue &Lo, SDValue &Hi) { 4115 EVT VT = N->getValueType(0); 4116 SDLoc dl(N); 4117 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4118 4119 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 4120 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4121 SplitInteger(Res.getValue(0), Lo, Hi); 4122 return; 4123 } 4124 4125 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4126 if (VT == MVT::i16) 4127 LC = RTLIB::SDIV_I16; 4128 else if (VT == MVT::i32) 4129 LC = RTLIB::SDIV_I32; 4130 else if (VT == MVT::i64) 4131 LC = RTLIB::SDIV_I64; 4132 else if (VT == MVT::i128) 4133 LC = RTLIB::SDIV_I128; 4134 4135 else { 4136 SDValue Result = 4137 ExpandExtIntRes_DIVREM(TLI, RTLIB::SDIV_IEXT, DAG, N, dl, VT); 4138 SplitInteger(Result, Lo, Hi); 4139 return; 4140 } 4141 4142 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 4143 4144 TargetLowering::MakeLibCallOptions CallOptions; 4145 CallOptions.setSExt(true); 4146 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4147 } 4148 4149 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 4150 SDValue &Lo, SDValue &Hi) { 4151 EVT VT = N->getValueType(0); 4152 SDLoc dl(N); 4153 4154 // If we can emit an efficient shift operation, do so now. Check to see if 4155 // the RHS is a constant. 4156 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 4157 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); 4158 4159 // If we can determine that the high bit of the shift is zero or one, even if 4160 // the low bits are variable, emit this shift in an optimized form. 4161 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 4162 return; 4163 4164 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 4165 unsigned PartsOpc; 4166 if (N->getOpcode() == ISD::SHL) { 4167 PartsOpc = ISD::SHL_PARTS; 4168 } else if (N->getOpcode() == ISD::SRL) { 4169 PartsOpc = ISD::SRL_PARTS; 4170 } else { 4171 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 4172 PartsOpc = ISD::SRA_PARTS; 4173 } 4174 4175 // Next check to see if the target supports this SHL_PARTS operation or if it 4176 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for 4177 // size, but create a libcall instead. 4178 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4179 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 4180 const bool LegalOrCustom = 4181 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 4182 Action == TargetLowering::Custom; 4183 4184 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) { 4185 // Expand the subcomponents. 4186 SDValue LHSL, LHSH; 4187 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 4188 EVT VT = LHSL.getValueType(); 4189 4190 // If the shift amount operand is coming from a vector legalization it may 4191 // have an illegal type. Fix that first by casting the operand, otherwise 4192 // the new SHL_PARTS operation would need further legalization. 4193 SDValue ShiftOp = N->getOperand(1); 4194 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 4195 if (ShiftOp.getValueType() != ShiftTy) 4196 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 4197 4198 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 4199 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 4200 Hi = Lo.getValue(1); 4201 return; 4202 } 4203 4204 // Otherwise, emit a libcall. 4205 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4206 bool isSigned; 4207 if (N->getOpcode() == ISD::SHL) { 4208 isSigned = false; /*sign irrelevant*/ 4209 if (VT == MVT::i16) 4210 LC = RTLIB::SHL_I16; 4211 else if (VT == MVT::i32) 4212 LC = RTLIB::SHL_I32; 4213 else if (VT == MVT::i64) 4214 LC = RTLIB::SHL_I64; 4215 else if (VT == MVT::i128) 4216 LC = RTLIB::SHL_I128; 4217 } else if (N->getOpcode() == ISD::SRL) { 4218 isSigned = false; 4219 if (VT == MVT::i16) 4220 LC = RTLIB::SRL_I16; 4221 else if (VT == MVT::i32) 4222 LC = RTLIB::SRL_I32; 4223 else if (VT == MVT::i64) 4224 LC = RTLIB::SRL_I64; 4225 else if (VT == MVT::i128) 4226 LC = RTLIB::SRL_I128; 4227 } else { 4228 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 4229 isSigned = true; 4230 if (VT == MVT::i16) 4231 LC = RTLIB::SRA_I16; 4232 else if (VT == MVT::i32) 4233 LC = RTLIB::SRA_I32; 4234 else if (VT == MVT::i64) 4235 LC = RTLIB::SRA_I64; 4236 else if (VT == MVT::i128) 4237 LC = RTLIB::SRA_I128; 4238 } 4239 4240 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 4241 EVT ShAmtTy = 4242 EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize()); 4243 SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy); 4244 SDValue Ops[2] = {N->getOperand(0), ShAmt}; 4245 TargetLowering::MakeLibCallOptions CallOptions; 4246 CallOptions.setSExt(isSigned); 4247 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4248 return; 4249 } 4250 4251 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 4252 llvm_unreachable("Unsupported shift!"); 4253 } 4254 4255 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 4256 SDValue &Lo, SDValue &Hi) { 4257 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4258 SDLoc dl(N); 4259 SDValue Op = N->getOperand(0); 4260 if (Op.getValueType().bitsLE(NVT)) { 4261 // The low part is sign extension of the input (degenerates to a copy). 4262 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 4263 // The high part is obtained by SRA'ing all but one of the bits of low part. 4264 unsigned LoSize = NVT.getSizeInBits(); 4265 Hi = DAG.getNode( 4266 ISD::SRA, dl, NVT, Lo, 4267 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); 4268 } else { 4269 // For example, extension of an i48 to an i64. The operand type necessarily 4270 // promotes to the result type, so will end up being expanded too. 4271 assert(getTypeAction(Op.getValueType()) == 4272 TargetLowering::TypePromoteInteger && 4273 "Only know how to promote this result!"); 4274 SDValue Res = GetPromotedInteger(Op); 4275 assert(Res.getValueType() == N->getValueType(0) && 4276 "Operand over promoted?"); 4277 // Split the promoted operand. This will simplify when it is expanded. 4278 SplitInteger(Res, Lo, Hi); 4279 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 4280 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 4281 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 4282 ExcessBits))); 4283 } 4284 } 4285 4286 void DAGTypeLegalizer:: 4287 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 4288 SDLoc dl(N); 4289 GetExpandedInteger(N->getOperand(0), Lo, Hi); 4290 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 4291 4292 if (EVT.bitsLE(Lo.getValueType())) { 4293 // sext_inreg the low part if needed. 4294 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 4295 N->getOperand(1)); 4296 4297 // The high part gets the sign extension from the lo-part. This handles 4298 // things like sextinreg V:i64 from i8. 4299 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 4300 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, 4301 TLI.getPointerTy(DAG.getDataLayout()))); 4302 } else { 4303 // For example, extension of an i48 to an i64. Leave the low part alone, 4304 // sext_inreg the high part. 4305 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); 4306 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 4307 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 4308 ExcessBits))); 4309 } 4310 } 4311 4312 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 4313 SDValue &Lo, SDValue &Hi) { 4314 EVT VT = N->getValueType(0); 4315 SDLoc dl(N); 4316 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4317 4318 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 4319 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4320 SplitInteger(Res.getValue(1), Lo, Hi); 4321 return; 4322 } 4323 4324 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4325 if (VT == MVT::i16) 4326 LC = RTLIB::SREM_I16; 4327 else if (VT == MVT::i32) 4328 LC = RTLIB::SREM_I32; 4329 else if (VT == MVT::i64) 4330 LC = RTLIB::SREM_I64; 4331 else if (VT == MVT::i128) 4332 LC = RTLIB::SREM_I128; 4333 4334 else { 4335 SDValue Result = 4336 ExpandExtIntRes_DIVREM(TLI, RTLIB::SREM_IEXT, DAG, N, dl, VT); 4337 SplitInteger(Result, Lo, Hi); 4338 return; 4339 } 4340 4341 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 4342 4343 TargetLowering::MakeLibCallOptions CallOptions; 4344 CallOptions.setSExt(true); 4345 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4346 } 4347 4348 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 4349 SDValue &Lo, SDValue &Hi) { 4350 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4351 SDLoc dl(N); 4352 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 4353 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), 4354 N->getOperand(0), 4355 DAG.getConstant(NVT.getSizeInBits(), dl, 4356 TLI.getPointerTy(DAG.getDataLayout()))); 4357 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 4358 } 4359 4360 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 4361 SDValue &Lo, SDValue &Hi) { 4362 EVT VT = N->getValueType(0); 4363 SDLoc dl(N); 4364 4365 if (N->getOpcode() == ISD::UMULO) { 4366 // This section expands the operation into the following sequence of 4367 // instructions. `iNh` here refers to a type which has half the bit width of 4368 // the type the original operation operated on. 4369 // 4370 // %0 = %LHS.HI != 0 && %RHS.HI != 0 4371 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) 4372 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) 4373 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) 4374 // %4 = add iNh %1.0, %2.0 as iN 4375 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH) 4376 // 4377 // %lo = %3.LO 4378 // %hi = %5.0 4379 // %ovf = %0 || %1.1 || %2.1 || %5.1 4380 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 4381 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; 4382 GetExpandedInteger(LHS, LHSLow, LHSHigh); 4383 GetExpandedInteger(RHS, RHSLow, RHSHigh); 4384 EVT HalfVT = LHSLow.getValueType(); 4385 EVT BitVT = N->getValueType(1); 4386 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT); 4387 4388 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT); 4389 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT, 4390 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE), 4391 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE)); 4392 4393 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow); 4394 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1)); 4395 4396 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow); 4397 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1)); 4398 4399 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two); 4400 4401 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not 4402 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this 4403 // operation recursively legalized?). 4404 // 4405 // Many backends understand this pattern and will convert into LOHI 4406 // themselves, if applicable. 4407 SDValue Three = DAG.getNode(ISD::MUL, dl, VT, 4408 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow), 4409 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow)); 4410 SplitInteger(Three, Lo, Hi); 4411 4412 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum); 4413 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1)); 4414 ReplaceValueWith(SDValue(N, 1), Overflow); 4415 return; 4416 } 4417 4418 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 4419 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 4420 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 4421 4422 // Replace this with a libcall that will check overflow. 4423 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4424 if (VT == MVT::i32) 4425 LC = RTLIB::MULO_I32; 4426 else if (VT == MVT::i64) 4427 LC = RTLIB::MULO_I64; 4428 else if (VT == MVT::i128) 4429 LC = RTLIB::MULO_I128; 4430 4431 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 4432 // FIXME: This is not an optimal expansion, but better than crashing. 4433 EVT WideVT = 4434 EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2); 4435 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0)); 4436 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1)); 4437 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); 4438 SDValue MulLo, MulHi; 4439 SplitInteger(Mul, MulLo, MulHi); 4440 SDValue SRA = 4441 DAG.getNode(ISD::SRA, dl, VT, MulLo, 4442 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT)); 4443 SDValue Overflow = 4444 DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE); 4445 SplitInteger(MulLo, Lo, Hi); 4446 ReplaceValueWith(SDValue(N, 1), Overflow); 4447 return; 4448 } 4449 4450 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 4451 // Temporary for the overflow value, default it to zero. 4452 SDValue Chain = 4453 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, 4454 MachinePointerInfo()); 4455 4456 TargetLowering::ArgListTy Args; 4457 TargetLowering::ArgListEntry Entry; 4458 for (const SDValue &Op : N->op_values()) { 4459 EVT ArgVT = Op.getValueType(); 4460 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 4461 Entry.Node = Op; 4462 Entry.Ty = ArgTy; 4463 Entry.IsSExt = true; 4464 Entry.IsZExt = false; 4465 Args.push_back(Entry); 4466 } 4467 4468 // Also pass the address of the overflow check. 4469 Entry.Node = Temp; 4470 Entry.Ty = PtrTy->getPointerTo(); 4471 Entry.IsSExt = true; 4472 Entry.IsZExt = false; 4473 Args.push_back(Entry); 4474 4475 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 4476 4477 TargetLowering::CallLoweringInfo CLI(DAG); 4478 CLI.setDebugLoc(dl) 4479 .setChain(Chain) 4480 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) 4481 .setSExtResult(); 4482 4483 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 4484 4485 SplitInteger(CallInfo.first, Lo, Hi); 4486 SDValue Temp2 = 4487 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); 4488 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 4489 DAG.getConstant(0, dl, PtrVT), 4490 ISD::SETNE); 4491 // Use the overflow from the libcall everywhere. 4492 ReplaceValueWith(SDValue(N, 1), Ofl); 4493 } 4494 4495 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 4496 SDValue &Lo, SDValue &Hi) { 4497 EVT VT = N->getValueType(0); 4498 SDLoc dl(N); 4499 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4500 4501 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4502 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4503 SplitInteger(Res.getValue(0), Lo, Hi); 4504 return; 4505 } 4506 4507 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4508 if (VT == MVT::i16) 4509 LC = RTLIB::UDIV_I16; 4510 else if (VT == MVT::i32) 4511 LC = RTLIB::UDIV_I32; 4512 else if (VT == MVT::i64) 4513 LC = RTLIB::UDIV_I64; 4514 else if (VT == MVT::i128) 4515 LC = RTLIB::UDIV_I128; 4516 4517 else { 4518 SDValue Result = 4519 ExpandExtIntRes_DIVREM(TLI, RTLIB::UDIV_IEXT, DAG, N, dl, VT); 4520 SplitInteger(Result, Lo, Hi); 4521 return; 4522 } 4523 4524 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 4525 4526 TargetLowering::MakeLibCallOptions CallOptions; 4527 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4528 } 4529 4530 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 4531 SDValue &Lo, SDValue &Hi) { 4532 EVT VT = N->getValueType(0); 4533 SDLoc dl(N); 4534 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4535 4536 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4537 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4538 SplitInteger(Res.getValue(1), Lo, Hi); 4539 return; 4540 } 4541 4542 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4543 if (VT == MVT::i16) 4544 LC = RTLIB::UREM_I16; 4545 else if (VT == MVT::i32) 4546 LC = RTLIB::UREM_I32; 4547 else if (VT == MVT::i64) 4548 LC = RTLIB::UREM_I64; 4549 else if (VT == MVT::i128) 4550 LC = RTLIB::UREM_I128; 4551 4552 else { 4553 SDValue Result = 4554 ExpandExtIntRes_DIVREM(TLI, RTLIB::UREM_IEXT, DAG, N, dl, VT); 4555 SplitInteger(Result, Lo, Hi); 4556 return; 4557 } 4558 4559 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 4560 4561 TargetLowering::MakeLibCallOptions CallOptions; 4562 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4563 } 4564 4565 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 4566 SDValue &Lo, SDValue &Hi) { 4567 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4568 SDLoc dl(N); 4569 SDValue Op = N->getOperand(0); 4570 if (Op.getValueType().bitsLE(NVT)) { 4571 // The low part is zero extension of the input (degenerates to a copy). 4572 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 4573 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. 4574 } else { 4575 // For example, extension of an i48 to an i64. The operand type necessarily 4576 // promotes to the result type, so will end up being expanded too. 4577 assert(getTypeAction(Op.getValueType()) == 4578 TargetLowering::TypePromoteInteger && 4579 "Only know how to promote this result!"); 4580 SDValue Res = GetPromotedInteger(Op); 4581 assert(Res.getValueType() == N->getValueType(0) && 4582 "Operand over promoted?"); 4583 // Split the promoted operand. This will simplify when it is expanded. 4584 SplitInteger(Res, Lo, Hi); 4585 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 4586 Hi = DAG.getZeroExtendInReg(Hi, dl, 4587 EVT::getIntegerVT(*DAG.getContext(), 4588 ExcessBits)); 4589 } 4590 } 4591 4592 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 4593 SDValue &Lo, SDValue &Hi) { 4594 SDLoc dl(N); 4595 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 4596 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 4597 SDValue Zero = DAG.getConstant(0, dl, VT); 4598 SDValue Swap = DAG.getAtomicCmpSwap( 4599 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 4600 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 4601 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); 4602 4603 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 4604 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 4605 } 4606 4607 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, 4608 SDValue &Lo, SDValue &Hi) { 4609 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate 4610 // both halves independently. 4611 SDValue Res = TLI.expandVecReduce(N, DAG); 4612 SplitInteger(Res, Lo, Hi); 4613 } 4614 4615 void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, 4616 SDValue &Lo, SDValue &Hi) { 4617 // Delegate to funnel-shift expansion. 4618 SDLoc DL(N); 4619 unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR; 4620 SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0), 4621 N->getOperand(0), N->getOperand(1)); 4622 SplitInteger(Res, Lo, Hi); 4623 } 4624 4625 void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo, 4626 SDValue &Hi) { 4627 // Values numbered from least significant to most significant. 4628 SDValue In1, In2, In3, In4; 4629 GetExpandedInteger(N->getOperand(0), In3, In4); 4630 GetExpandedInteger(N->getOperand(1), In1, In2); 4631 EVT HalfVT = In1.getValueType(); 4632 4633 SDLoc DL(N); 4634 unsigned Opc = N->getOpcode(); 4635 SDValue ShAmt = N->getOperand(2); 4636 EVT ShAmtVT = ShAmt.getValueType(); 4637 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT); 4638 4639 // If the shift amount is at least half the bitwidth, swap the inputs. 4640 unsigned HalfVTBits = HalfVT.getScalarSizeInBits(); 4641 SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt, 4642 DAG.getConstant(HalfVTBits, DL, ShAmtVT)); 4643 SDValue Cond = 4644 DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT), 4645 Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ); 4646 4647 // Expand to a pair of funnel shifts. 4648 EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout()); 4649 SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT); 4650 4651 SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2); 4652 SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3); 4653 SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4); 4654 Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt); 4655 Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt); 4656 } 4657 4658 void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, 4659 SDValue &Hi) { 4660 EVT VT = N->getValueType(0); 4661 EVT HalfVT = 4662 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2); 4663 SDLoc dl(N); 4664 4665 // We assume VSCALE(1) fits into a legal integer. 4666 APInt One(HalfVT.getSizeInBits(), 1); 4667 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One); 4668 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase); 4669 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0)); 4670 SplitInteger(Res, Lo, Hi); 4671 } 4672 4673 //===----------------------------------------------------------------------===// 4674 // Integer Operand Expansion 4675 //===----------------------------------------------------------------------===// 4676 4677 /// ExpandIntegerOperand - This method is called when the specified operand of 4678 /// the specified node is found to need expansion. At this point, all of the 4679 /// result types of the node are known to be legal, but other operands of the 4680 /// node may need promotion or expansion as well as the specified one. 4681 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 4682 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); 4683 dbgs() << "\n"); 4684 SDValue Res = SDValue(); 4685 4686 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 4687 return false; 4688 4689 switch (N->getOpcode()) { 4690 default: 4691 #ifndef NDEBUG 4692 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 4693 N->dump(&DAG); dbgs() << "\n"; 4694 #endif 4695 report_fatal_error("Do not know how to expand this operator's operand!"); 4696 4697 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 4698 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 4699 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 4700 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 4701 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 4702 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 4703 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break; 4704 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 4705 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 4706 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; 4707 case ISD::STRICT_SINT_TO_FP: 4708 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 4709 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 4710 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 4711 case ISD::STRICT_UINT_TO_FP: 4712 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 4713 4714 case ISD::SHL: 4715 case ISD::SRA: 4716 case ISD::SRL: 4717 case ISD::ROTL: 4718 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 4719 case ISD::RETURNADDR: 4720 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 4721 4722 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 4723 case ISD::STACKMAP: 4724 Res = ExpandIntOp_STACKMAP(N, OpNo); 4725 break; 4726 case ISD::PATCHPOINT: 4727 Res = ExpandIntOp_PATCHPOINT(N, OpNo); 4728 break; 4729 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 4730 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 4731 Res = ExpandIntOp_VP_STRIDED(N, OpNo); 4732 break; 4733 } 4734 4735 // If the result is null, the sub-method took care of registering results etc. 4736 if (!Res.getNode()) return false; 4737 4738 // If the result is N, the sub-method updated N in place. Tell the legalizer 4739 // core about this. 4740 if (Res.getNode() == N) 4741 return true; 4742 4743 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 4744 "Invalid operand expansion"); 4745 4746 ReplaceValueWith(SDValue(N, 0), Res); 4747 return false; 4748 } 4749 4750 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 4751 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 4752 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 4753 SDValue &NewRHS, 4754 ISD::CondCode &CCCode, 4755 const SDLoc &dl) { 4756 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 4757 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 4758 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 4759 4760 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 4761 if (RHSLo == RHSHi) { 4762 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 4763 if (RHSCST->isAllOnes()) { 4764 // Equality comparison to -1. 4765 NewLHS = DAG.getNode(ISD::AND, dl, 4766 LHSLo.getValueType(), LHSLo, LHSHi); 4767 NewRHS = RHSLo; 4768 return; 4769 } 4770 } 4771 } 4772 4773 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 4774 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 4775 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 4776 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); 4777 return; 4778 } 4779 4780 // If this is a comparison of the sign bit, just look at the top part. 4781 // X > -1, x < 0 4782 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 4783 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0 4784 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1 4785 NewLHS = LHSHi; 4786 NewRHS = RHSHi; 4787 return; 4788 } 4789 4790 // FIXME: This generated code sucks. 4791 ISD::CondCode LowCC; 4792 switch (CCCode) { 4793 default: llvm_unreachable("Unknown integer setcc!"); 4794 case ISD::SETLT: 4795 case ISD::SETULT: LowCC = ISD::SETULT; break; 4796 case ISD::SETGT: 4797 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 4798 case ISD::SETLE: 4799 case ISD::SETULE: LowCC = ISD::SETULE; break; 4800 case ISD::SETGE: 4801 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 4802 } 4803 4804 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison 4805 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands 4806 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; 4807 4808 // NOTE: on targets without efficient SELECT of bools, we can always use 4809 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 4810 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 4811 nullptr); 4812 SDValue LoCmp, HiCmp; 4813 if (TLI.isTypeLegal(LHSLo.getValueType()) && 4814 TLI.isTypeLegal(RHSLo.getValueType())) 4815 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, 4816 RHSLo, LowCC, false, DagCombineInfo, dl); 4817 if (!LoCmp.getNode()) 4818 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 4819 RHSLo, LowCC); 4820 if (TLI.isTypeLegal(LHSHi.getValueType()) && 4821 TLI.isTypeLegal(RHSHi.getValueType())) 4822 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, 4823 RHSHi, CCCode, false, DagCombineInfo, dl); 4824 if (!HiCmp.getNode()) 4825 HiCmp = 4826 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), 4827 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 4828 4829 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); 4830 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); 4831 4832 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 4833 CCCode == ISD::SETUGE || CCCode == ISD::SETULE); 4834 4835 // FIXME: Is the HiCmpC->isOne() here correct for 4836 // ZeroOrNegativeOneBooleanContent. 4837 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) || 4838 (!EqAllowed && 4839 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) { 4840 // For LE / GE, if high part is known false, ignore the low part. 4841 // For LT / GT: if low part is known false, return the high part. 4842 // if high part is known true, ignore the low part. 4843 NewLHS = HiCmp; 4844 NewRHS = SDValue(); 4845 return; 4846 } 4847 4848 if (LHSHi == RHSHi) { 4849 // Comparing the low bits is enough. 4850 NewLHS = LoCmp; 4851 NewRHS = SDValue(); 4852 return; 4853 } 4854 4855 // Lower with SETCCCARRY if the target supports it. 4856 EVT HiVT = LHSHi.getValueType(); 4857 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); 4858 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); 4859 4860 // FIXME: Make all targets support this, then remove the other lowering. 4861 if (HasSETCCCARRY) { 4862 // SETCCCARRY can detect < and >= directly. For > and <=, flip 4863 // operands and condition code. 4864 bool FlipOperands = false; 4865 switch (CCCode) { 4866 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; 4867 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; 4868 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; 4869 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; 4870 default: break; 4871 } 4872 if (FlipOperands) { 4873 std::swap(LHSLo, RHSLo); 4874 std::swap(LHSHi, RHSHi); 4875 } 4876 // Perform a wide subtraction, feeding the carry from the low part into 4877 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high 4878 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is 4879 // zero or positive iff LHS >= RHS. 4880 EVT LoVT = LHSLo.getValueType(); 4881 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT)); 4882 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo); 4883 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT), 4884 LHSHi, RHSHi, LowCmp.getValue(1), 4885 DAG.getCondCode(CCCode)); 4886 NewLHS = Res; 4887 NewRHS = SDValue(); 4888 return; 4889 } 4890 4891 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, 4892 false, DagCombineInfo, dl); 4893 if (!NewLHS.getNode()) 4894 NewLHS = 4895 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); 4896 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); 4897 NewRHS = SDValue(); 4898 } 4899 4900 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 4901 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 4902 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 4903 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 4904 4905 // If ExpandSetCCOperands returned a scalar, we need to compare the result 4906 // against zero to select between true and false values. 4907 if (!NewRHS.getNode()) { 4908 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 4909 CCCode = ISD::SETNE; 4910 } 4911 4912 // Update N to have the operands specified. 4913 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 4914 DAG.getCondCode(CCCode), NewLHS, NewRHS, 4915 N->getOperand(4)), 0); 4916 } 4917 4918 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 4919 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 4920 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 4921 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 4922 4923 // If ExpandSetCCOperands returned a scalar, we need to compare the result 4924 // against zero to select between true and false values. 4925 if (!NewRHS.getNode()) { 4926 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 4927 CCCode = ISD::SETNE; 4928 } 4929 4930 // Update N to have the operands specified. 4931 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 4932 N->getOperand(2), N->getOperand(3), 4933 DAG.getCondCode(CCCode)), 0); 4934 } 4935 4936 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 4937 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 4938 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 4939 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 4940 4941 // If ExpandSetCCOperands returned a scalar, use it. 4942 if (!NewRHS.getNode()) { 4943 assert(NewLHS.getValueType() == N->getValueType(0) && 4944 "Unexpected setcc expansion!"); 4945 return NewLHS; 4946 } 4947 4948 // Otherwise, update N to have the operands specified. 4949 return SDValue( 4950 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); 4951 } 4952 4953 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { 4954 SDValue LHS = N->getOperand(0); 4955 SDValue RHS = N->getOperand(1); 4956 SDValue Carry = N->getOperand(2); 4957 SDValue Cond = N->getOperand(3); 4958 SDLoc dl = SDLoc(N); 4959 4960 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 4961 GetExpandedInteger(LHS, LHSLo, LHSHi); 4962 GetExpandedInteger(RHS, RHSLo, RHSHi); 4963 4964 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high. 4965 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); 4966 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry); 4967 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, 4968 LowCmp.getValue(1), Cond); 4969 } 4970 4971 SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) { 4972 // Split the operand and replace with SPLAT_VECTOR_PARTS. 4973 SDValue Lo, Hi; 4974 GetExpandedInteger(N->getOperand(0), Lo, Hi); 4975 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo, 4976 Hi); 4977 } 4978 4979 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 4980 // The value being shifted is legal, but the shift amount is too big. 4981 // It follows that either the result of the shift is undefined, or the 4982 // upper half of the shift amount is zero. Just use the lower half. 4983 SDValue Lo, Hi; 4984 GetExpandedInteger(N->getOperand(1), Lo, Hi); 4985 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 4986 } 4987 4988 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 4989 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 4990 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 4991 // constant to valid type. 4992 SDValue Lo, Hi; 4993 GetExpandedInteger(N->getOperand(0), Lo, Hi); 4994 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 4995 } 4996 4997 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 4998 bool IsStrict = N->isStrictFPOpcode(); 4999 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 5000 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 5001 EVT DstVT = N->getValueType(0); 5002 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 5003 assert(LC != RTLIB::UNKNOWN_LIBCALL && 5004 "Don't know how to expand this SINT_TO_FP!"); 5005 TargetLowering::MakeLibCallOptions CallOptions; 5006 CallOptions.setSExt(true); 5007 std::pair<SDValue, SDValue> Tmp = 5008 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 5009 5010 if (!IsStrict) 5011 return Tmp.first; 5012 5013 ReplaceValueWith(SDValue(N, 1), Tmp.second); 5014 ReplaceValueWith(SDValue(N, 0), Tmp.first); 5015 return SDValue(); 5016 } 5017 5018 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 5019 if (N->isAtomic()) { 5020 // It's typical to have larger CAS than atomic store instructions. 5021 SDLoc dl(N); 5022 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 5023 N->getMemoryVT(), 5024 N->getOperand(0), N->getOperand(2), 5025 N->getOperand(1), 5026 N->getMemOperand()); 5027 return Swap.getValue(1); 5028 } 5029 if (ISD::isNormalStore(N)) 5030 return ExpandOp_NormalStore(N, OpNo); 5031 5032 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 5033 assert(OpNo == 1 && "Can only expand the stored value so far"); 5034 5035 EVT VT = N->getOperand(1).getValueType(); 5036 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5037 SDValue Ch = N->getChain(); 5038 SDValue Ptr = N->getBasePtr(); 5039 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 5040 AAMDNodes AAInfo = N->getAAInfo(); 5041 SDLoc dl(N); 5042 SDValue Lo, Hi; 5043 5044 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 5045 5046 if (N->getMemoryVT().bitsLE(NVT)) { 5047 GetExpandedInteger(N->getValue(), Lo, Hi); 5048 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 5049 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags, 5050 AAInfo); 5051 } 5052 5053 if (DAG.getDataLayout().isLittleEndian()) { 5054 // Little-endian - low bits are at low addresses. 5055 GetExpandedInteger(N->getValue(), Lo, Hi); 5056 5057 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 5058 N->getOriginalAlign(), MMOFlags, AAInfo); 5059 5060 unsigned ExcessBits = 5061 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 5062 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 5063 5064 // Increment the pointer to the other half. 5065 unsigned IncrementSize = NVT.getSizeInBits()/8; 5066 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 5067 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 5068 N->getPointerInfo().getWithOffset(IncrementSize), 5069 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo); 5070 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 5071 } 5072 5073 // Big-endian - high bits are at low addresses. Favor aligned stores at 5074 // the cost of some bit-fiddling. 5075 GetExpandedInteger(N->getValue(), Lo, Hi); 5076 5077 EVT ExtVT = N->getMemoryVT(); 5078 unsigned EBytes = ExtVT.getStoreSize(); 5079 unsigned IncrementSize = NVT.getSizeInBits()/8; 5080 unsigned ExcessBits = (EBytes - IncrementSize)*8; 5081 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 5082 ExtVT.getSizeInBits() - ExcessBits); 5083 5084 if (ExcessBits < NVT.getSizeInBits()) { 5085 // Transfer high bits from the top of Lo to the bottom of Hi. 5086 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 5087 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 5088 TLI.getPointerTy(DAG.getDataLayout()))); 5089 Hi = DAG.getNode( 5090 ISD::OR, dl, NVT, Hi, 5091 DAG.getNode(ISD::SRL, dl, NVT, Lo, 5092 DAG.getConstant(ExcessBits, dl, 5093 TLI.getPointerTy(DAG.getDataLayout())))); 5094 } 5095 5096 // Store both the high bits and maybe some of the low bits. 5097 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, 5098 N->getOriginalAlign(), MMOFlags, AAInfo); 5099 5100 // Increment the pointer to the other half. 5101 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 5102 // Store the lowest ExcessBits bits in the second half. 5103 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 5104 N->getPointerInfo().getWithOffset(IncrementSize), 5105 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 5106 N->getOriginalAlign(), MMOFlags, AAInfo); 5107 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 5108 } 5109 5110 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 5111 SDValue InL, InH; 5112 GetExpandedInteger(N->getOperand(0), InL, InH); 5113 // Just truncate the low part of the source. 5114 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 5115 } 5116 5117 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 5118 bool IsStrict = N->isStrictFPOpcode(); 5119 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 5120 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 5121 EVT DstVT = N->getValueType(0); 5122 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT); 5123 assert(LC != RTLIB::UNKNOWN_LIBCALL && 5124 "Don't know how to expand this UINT_TO_FP!"); 5125 TargetLowering::MakeLibCallOptions CallOptions; 5126 CallOptions.setSExt(true); 5127 std::pair<SDValue, SDValue> Tmp = 5128 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 5129 5130 if (!IsStrict) 5131 return Tmp.first; 5132 5133 ReplaceValueWith(SDValue(N, 1), Tmp.second); 5134 ReplaceValueWith(SDValue(N, 0), Tmp.first); 5135 return SDValue(); 5136 } 5137 5138 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 5139 SDLoc dl(N); 5140 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 5141 cast<AtomicSDNode>(N)->getMemoryVT(), 5142 N->getOperand(0), 5143 N->getOperand(1), N->getOperand(2), 5144 cast<AtomicSDNode>(N)->getMemOperand()); 5145 return Swap.getValue(1); 5146 } 5147 5148 SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { 5149 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || 5150 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); 5151 5152 SDValue Hi; // The upper half is dropped out. 5153 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); 5154 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo], Hi); 5155 5156 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 5157 } 5158 5159 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { 5160 SDLoc dl(N); 5161 5162 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5163 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5164 EVT OutVT = V0.getValueType(); 5165 5166 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2)); 5167 } 5168 5169 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 5170 5171 EVT OutVT = N->getValueType(0); 5172 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5173 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5174 EVT NOutVTElem = NOutVT.getVectorElementType(); 5175 5176 SDLoc dl(N); 5177 SDValue BaseIdx = N->getOperand(1); 5178 5179 // TODO: We may be able to use this for types other than scalable 5180 // vectors and fix those tests that expect BUILD_VECTOR to be used 5181 if (OutVT.isScalableVector()) { 5182 SDValue InOp0 = N->getOperand(0); 5183 EVT InVT = InOp0.getValueType(); 5184 5185 // Try and extract from a smaller type so that it eventually falls 5186 // into the promotion code below. 5187 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector || 5188 getTypeAction(InVT) == TargetLowering::TypeLegal) { 5189 EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext()); 5190 unsigned NElts = NInVT.getVectorMinNumElements(); 5191 uint64_t IdxVal = cast<ConstantSDNode>(BaseIdx)->getZExtValue(); 5192 5193 SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0, 5194 DAG.getConstant(alignDown(IdxVal, NElts), dl, 5195 BaseIdx.getValueType())); 5196 SDValue Step2 = DAG.getNode( 5197 ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1, 5198 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType())); 5199 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2); 5200 } 5201 5202 // Try and extract from a widened type. 5203 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 5204 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx}; 5205 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops); 5206 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); 5207 } 5208 5209 // Promote operands and see if this is handled by target lowering, 5210 // Otherwise, use the BUILD_VECTOR approach below 5211 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 5212 // Collect the (promoted) operands 5213 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx }; 5214 5215 EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); 5216 assert(PromEltVT.bitsLE(NOutVTElem) && 5217 "Promoted operand has an element type greater than result"); 5218 5219 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT); 5220 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops); 5221 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); 5222 } 5223 } 5224 5225 if (OutVT.isScalableVector()) 5226 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR"); 5227 5228 SDValue InOp0 = N->getOperand(0); 5229 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) 5230 InOp0 = GetPromotedInteger(N->getOperand(0)); 5231 5232 EVT InVT = InOp0.getValueType(); 5233 5234 unsigned OutNumElems = OutVT.getVectorNumElements(); 5235 SmallVector<SDValue, 8> Ops; 5236 Ops.reserve(OutNumElems); 5237 for (unsigned i = 0; i != OutNumElems; ++i) { 5238 5239 // Extract the element from the original vector. 5240 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 5241 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); 5242 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 5243 InVT.getVectorElementType(), N->getOperand(0), Index); 5244 5245 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); 5246 // Insert the converted element to the new vector. 5247 Ops.push_back(Op); 5248 } 5249 5250 return DAG.getBuildVector(NOutVT, dl, Ops); 5251 } 5252 5253 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) { 5254 EVT OutVT = N->getValueType(0); 5255 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5256 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5257 5258 SDLoc dl(N); 5259 SDValue Vec = N->getOperand(0); 5260 SDValue SubVec = N->getOperand(1); 5261 SDValue Idx = N->getOperand(2); 5262 5263 EVT SubVecVT = SubVec.getValueType(); 5264 EVT NSubVT = 5265 EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(), 5266 SubVecVT.getVectorElementCount()); 5267 5268 Vec = GetPromotedInteger(Vec); 5269 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec); 5270 5271 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx); 5272 } 5273 5274 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) { 5275 SDLoc dl(N); 5276 5277 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5278 EVT OutVT = V0.getValueType(); 5279 5280 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0); 5281 } 5282 5283 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 5284 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 5285 EVT VT = N->getValueType(0); 5286 SDLoc dl(N); 5287 5288 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); 5289 5290 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5291 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5292 EVT OutVT = V0.getValueType(); 5293 5294 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); 5295 } 5296 5297 5298 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 5299 EVT OutVT = N->getValueType(0); 5300 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5301 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5302 unsigned NumElems = N->getNumOperands(); 5303 EVT NOutVTElem = NOutVT.getVectorElementType(); 5304 TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(NOutVT); 5305 unsigned NOutExtOpc = TargetLowering::getExtendForContent(NOutBoolType); 5306 SDLoc dl(N); 5307 5308 SmallVector<SDValue, 8> Ops; 5309 Ops.reserve(NumElems); 5310 for (unsigned i = 0; i != NumElems; ++i) { 5311 SDValue Op = N->getOperand(i); 5312 EVT OpVT = Op.getValueType(); 5313 // BUILD_VECTOR integer operand types are allowed to be larger than the 5314 // result's element type. This may still be true after the promotion. For 5315 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 5316 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 5317 if (OpVT.bitsLT(NOutVTElem)) { 5318 unsigned ExtOpc = ISD::ANY_EXTEND; 5319 // Attempt to extend constant bool vectors to match target's BooleanContent. 5320 // While not necessary, this improves chances of the constant correctly 5321 // folding with compare results (e.g. for NOT patterns). 5322 if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant) 5323 ExtOpc = NOutExtOpc; 5324 Op = DAG.getNode(ExtOpc, dl, NOutVTElem, Op); 5325 } 5326 Ops.push_back(Op); 5327 } 5328 5329 return DAG.getBuildVector(NOutVT, dl, Ops); 5330 } 5331 5332 SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) { 5333 5334 SDLoc dl(N); 5335 5336 assert(!N->getOperand(0).getValueType().isVector() && 5337 "Input must be a scalar"); 5338 5339 EVT OutVT = N->getValueType(0); 5340 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5341 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5342 EVT NOutElemVT = NOutVT.getVectorElementType(); 5343 5344 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0)); 5345 5346 return DAG.getNode(N->getOpcode(), dl, NOutVT, Op); 5347 } 5348 5349 SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) { 5350 SDLoc dl(N); 5351 EVT OutVT = N->getValueType(0); 5352 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5353 assert(NOutVT.isScalableVector() && 5354 "Type must be promoted to a scalable vector type"); 5355 APInt StepVal = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue(); 5356 return DAG.getStepVector(dl, NOutVT, 5357 StepVal.sext(NOutVT.getScalarSizeInBits())); 5358 } 5359 5360 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 5361 SDLoc dl(N); 5362 5363 EVT OutVT = N->getValueType(0); 5364 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5365 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5366 5367 unsigned NumOperands = N->getNumOperands(); 5368 unsigned NumOutElem = NOutVT.getVectorMinNumElements(); 5369 EVT OutElemTy = NOutVT.getVectorElementType(); 5370 if (OutVT.isScalableVector()) { 5371 // Find the largest promoted element type for each of the operands. 5372 SDUse *MaxSizedValue = std::max_element( 5373 N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) { 5374 EVT AVT = A.getValueType().getVectorElementType(); 5375 EVT BVT = B.getValueType().getVectorElementType(); 5376 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits(); 5377 }); 5378 EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType(); 5379 5380 // Then promote all vectors to the largest element type. 5381 SmallVector<SDValue, 8> Ops; 5382 for (unsigned I = 0; I < NumOperands; ++I) { 5383 SDValue Op = N->getOperand(I); 5384 EVT OpVT = Op.getValueType(); 5385 if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger) 5386 Op = GetPromotedInteger(Op); 5387 else 5388 assert(getTypeAction(OpVT) == TargetLowering::TypeLegal && 5389 "Unhandled legalization type"); 5390 5391 if (OpVT.getVectorElementType().getScalarSizeInBits() < 5392 MaxElementVT.getScalarSizeInBits()) 5393 Op = DAG.getAnyExtOrTrunc(Op, dl, 5394 OpVT.changeVectorElementType(MaxElementVT)); 5395 Ops.push_back(Op); 5396 } 5397 5398 // Do the CONCAT on the promoted type and finally truncate to (the promoted) 5399 // NOutVT. 5400 return DAG.getAnyExtOrTrunc( 5401 DAG.getNode(ISD::CONCAT_VECTORS, dl, 5402 OutVT.changeVectorElementType(MaxElementVT), Ops), 5403 dl, NOutVT); 5404 } 5405 5406 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 5407 assert(NumElem * NumOperands == NumOutElem && 5408 "Unexpected number of elements"); 5409 5410 // Take the elements from the first vector. 5411 SmallVector<SDValue, 8> Ops(NumOutElem); 5412 for (unsigned i = 0; i < NumOperands; ++i) { 5413 SDValue Op = N->getOperand(i); 5414 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) 5415 Op = GetPromotedInteger(Op); 5416 EVT SclrTy = Op.getValueType().getVectorElementType(); 5417 assert(NumElem == Op.getValueType().getVectorNumElements() && 5418 "Unexpected number of elements"); 5419 5420 for (unsigned j = 0; j < NumElem; ++j) { 5421 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, 5422 DAG.getVectorIdxConstant(j, dl)); 5423 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); 5424 } 5425 } 5426 5427 return DAG.getBuildVector(NOutVT, dl, Ops); 5428 } 5429 5430 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { 5431 EVT VT = N->getValueType(0); 5432 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5433 assert(NVT.isVector() && "This type must be promoted to a vector type"); 5434 5435 SDLoc dl(N); 5436 5437 // For operands whose TypeAction is to promote, extend the promoted node 5438 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion 5439 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to 5440 // type.. 5441 if (getTypeAction(N->getOperand(0).getValueType()) 5442 == TargetLowering::TypePromoteInteger) { 5443 SDValue Promoted; 5444 5445 switch(N->getOpcode()) { 5446 case ISD::SIGN_EXTEND_VECTOR_INREG: 5447 Promoted = SExtPromotedInteger(N->getOperand(0)); 5448 break; 5449 case ISD::ZERO_EXTEND_VECTOR_INREG: 5450 Promoted = ZExtPromotedInteger(N->getOperand(0)); 5451 break; 5452 case ISD::ANY_EXTEND_VECTOR_INREG: 5453 Promoted = GetPromotedInteger(N->getOperand(0)); 5454 break; 5455 default: 5456 llvm_unreachable("Node has unexpected Opcode"); 5457 } 5458 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); 5459 } 5460 5461 // Directly extend to the appropriate transform-to type. 5462 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 5463 } 5464 5465 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 5466 EVT OutVT = N->getValueType(0); 5467 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5468 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5469 5470 EVT NOutVTElem = NOutVT.getVectorElementType(); 5471 5472 SDLoc dl(N); 5473 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5474 5475 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 5476 NOutVTElem, N->getOperand(1)); 5477 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 5478 V0, ConvElem, N->getOperand(2)); 5479 } 5480 5481 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { 5482 // The VECREDUCE result size may be larger than the element size, so 5483 // we can simply change the result type. 5484 SDLoc dl(N); 5485 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5486 return DAG.getNode(N->getOpcode(), dl, NVT, N->ops()); 5487 } 5488 5489 SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) { 5490 // The VP_REDUCE result size may be larger than the element size, so we can 5491 // simply change the result type. However the start value and result must be 5492 // the same. 5493 SDLoc DL(N); 5494 SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0)); 5495 return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start, 5496 N->getOperand(1), N->getOperand(2), N->getOperand(3)); 5497 } 5498 5499 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 5500 SDLoc dl(N); 5501 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5502 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, 5503 TLI.getVectorIdxTy(DAG.getDataLayout())); 5504 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 5505 V0->getValueType(0).getScalarType(), V0, V1); 5506 5507 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 5508 // element types. If this is the case then we need to expand the outgoing 5509 // value and not truncate it. 5510 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 5511 } 5512 5513 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) { 5514 SDLoc dl(N); 5515 // The result type is equal to the first input operand's type, so the 5516 // type that needs promoting must be the second source vector. 5517 SDValue V0 = N->getOperand(0); 5518 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5519 SDValue Idx = N->getOperand(2); 5520 EVT PromVT = EVT::getVectorVT(*DAG.getContext(), 5521 V1.getValueType().getVectorElementType(), 5522 V0.getValueType().getVectorElementCount()); 5523 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT); 5524 SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx); 5525 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 5526 } 5527 5528 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { 5529 SDLoc dl(N); 5530 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5531 MVT InVT = V0.getValueType().getSimpleVT(); 5532 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), 5533 N->getValueType(0).getVectorNumElements()); 5534 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); 5535 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 5536 } 5537 5538 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 5539 SDLoc dl(N); 5540 5541 EVT ResVT = N->getValueType(0); 5542 unsigned NumElems = N->getNumOperands(); 5543 5544 if (ResVT.isScalableVector()) { 5545 SDValue ResVec = DAG.getUNDEF(ResVT); 5546 5547 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { 5548 SDValue Op = N->getOperand(OpIdx); 5549 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); 5550 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op, 5551 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl)); 5552 } 5553 5554 return ResVec; 5555 } 5556 5557 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 5558 5559 SmallVector<SDValue, 8> NewOps; 5560 NewOps.reserve(NumElems); 5561 5562 // For each incoming vector 5563 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 5564 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 5565 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 5566 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 5567 5568 for (unsigned i=0; i<NumElem; ++i) { 5569 // Extract element from incoming vector 5570 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, 5571 DAG.getVectorIdxConstant(i, dl)); 5572 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 5573 NewOps.push_back(Tr); 5574 } 5575 } 5576 5577 return DAG.getBuildVector(N->getValueType(0), dl, NewOps); 5578 } 5579 5580 SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) { 5581 assert(OpNo > 1); 5582 SDValue Op = N->getOperand(OpNo); 5583 5584 // FIXME: Non-constant operands are not yet handled: 5585 // - https://github.com/llvm/llvm-project/issues/26431 5586 // - https://github.com/llvm/llvm-project/issues/55957 5587 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op); 5588 if (!CN) 5589 return SDValue(); 5590 5591 // Copy operands before the one being expanded. 5592 SmallVector<SDValue> NewOps; 5593 for (unsigned I = 0; I < OpNo; I++) 5594 NewOps.push_back(N->getOperand(I)); 5595 5596 EVT Ty = Op.getValueType(); 5597 SDLoc DL = SDLoc(N); 5598 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { 5599 NewOps.push_back( 5600 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); 5601 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); 5602 } else { 5603 // FIXME: https://github.com/llvm/llvm-project/issues/55609 5604 return SDValue(); 5605 } 5606 5607 // Copy remaining operands. 5608 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) 5609 NewOps.push_back(N->getOperand(I)); 5610 5611 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); 5612 5613 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 5614 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 5615 5616 return SDValue(); // Signal that we have replaced the node already. 5617 } 5618 5619 SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { 5620 assert(OpNo >= 7); 5621 SDValue Op = N->getOperand(OpNo); 5622 5623 // FIXME: Non-constant operands are not yet handled: 5624 // - https://github.com/llvm/llvm-project/issues/26431 5625 // - https://github.com/llvm/llvm-project/issues/55957 5626 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op); 5627 if (!CN) 5628 return SDValue(); 5629 5630 // Copy operands before the one being expanded. 5631 SmallVector<SDValue> NewOps; 5632 for (unsigned I = 0; I < OpNo; I++) 5633 NewOps.push_back(N->getOperand(I)); 5634 5635 EVT Ty = Op.getValueType(); 5636 SDLoc DL = SDLoc(N); 5637 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { 5638 NewOps.push_back( 5639 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); 5640 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); 5641 } else { 5642 // FIXME: https://github.com/llvm/llvm-project/issues/55609 5643 return SDValue(); 5644 } 5645 5646 // Copy remaining operands. 5647 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) 5648 NewOps.push_back(N->getOperand(I)); 5649 5650 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); 5651 5652 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 5653 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 5654 5655 return SDValue(); // Signal that we have replaced the node already. 5656 } 5657