1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements integer type expansion and promotion for LegalizeTypes. 10 // Promotion is the act of changing a computation in an illegal type into a 11 // computation in a larger type. For example, implementing i8 arithmetic in an 12 // i32 register (often needed on powerpc). 13 // Expansion is the act of changing a computation in an illegal type into a 14 // computation in two identical registers of a smaller type. For example, 15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit 16 // targets). 17 // 18 //===----------------------------------------------------------------------===// 19 20 #include "LegalizeTypes.h" 21 #include "llvm/Analysis/TargetLibraryInfo.h" 22 #include "llvm/CodeGen/StackMaps.h" 23 #include "llvm/CodeGen/TargetLowering.h" 24 #include "llvm/IR/DerivedTypes.h" 25 #include "llvm/Support/ErrorHandling.h" 26 #include "llvm/Support/KnownBits.h" 27 #include "llvm/Support/raw_ostream.h" 28 #include <algorithm> 29 using namespace llvm; 30 31 #define DEBUG_TYPE "legalize-types" 32 33 //===----------------------------------------------------------------------===// 34 // Integer Result Promotion 35 //===----------------------------------------------------------------------===// 36 37 /// PromoteIntegerResult - This method is called when a result of a node is 38 /// found to be in need of promotion to a larger type. At this point, the node 39 /// may also have invalid operands or may have other results that need 40 /// expansion, we just know that (at least) one result needs promotion. 41 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 42 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); 43 dbgs() << "\n"); 44 SDValue Res = SDValue(); 45 46 // See if the target wants to custom expand this node. 47 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 48 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 49 return; 50 } 51 52 switch (N->getOpcode()) { 53 default: 54 #ifndef NDEBUG 55 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 56 N->dump(&DAG); dbgs() << "\n"; 57 #endif 58 report_fatal_error("Do not know how to promote this operator!"); 59 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 60 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 61 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 62 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 63 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; 64 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 65 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 66 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 67 case ISD::CTLZ_ZERO_UNDEF: 68 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 69 case ISD::PARITY: 70 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; 71 case ISD::CTTZ_ZERO_UNDEF: 72 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 73 case ISD::EXTRACT_VECTOR_ELT: 74 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 75 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 76 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); 77 break; 78 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); 79 break; 80 case ISD::SELECT: 81 case ISD::VSELECT: 82 case ISD::VP_SELECT: 83 case ISD::VP_MERGE: 84 Res = PromoteIntRes_Select(N); 85 break; 86 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 87 case ISD::STRICT_FSETCC: 88 case ISD::STRICT_FSETCCS: 89 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 90 case ISD::SMIN: 91 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; 92 case ISD::UMIN: 93 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; 94 95 case ISD::SHL: 96 case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break; 97 case ISD::SIGN_EXTEND_INREG: 98 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 99 case ISD::SRA: 100 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break; 101 case ISD::SRL: 102 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break; 103 case ISD::VP_TRUNCATE: 104 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 105 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 106 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 107 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; 108 109 case ISD::EXTRACT_SUBVECTOR: 110 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 111 case ISD::INSERT_SUBVECTOR: 112 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break; 113 case ISD::VECTOR_REVERSE: 114 Res = PromoteIntRes_VECTOR_REVERSE(N); break; 115 case ISD::VECTOR_SHUFFLE: 116 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 117 case ISD::VECTOR_SPLICE: 118 Res = PromoteIntRes_VECTOR_SPLICE(N); break; 119 case ISD::VECTOR_INTERLEAVE: 120 case ISD::VECTOR_DEINTERLEAVE: 121 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N); 122 return; 123 case ISD::INSERT_VECTOR_ELT: 124 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 125 case ISD::BUILD_VECTOR: 126 Res = PromoteIntRes_BUILD_VECTOR(N); 127 break; 128 case ISD::SPLAT_VECTOR: 129 case ISD::SCALAR_TO_VECTOR: 130 Res = PromoteIntRes_ScalarOp(N); 131 break; 132 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break; 133 case ISD::CONCAT_VECTORS: 134 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 135 136 case ISD::ANY_EXTEND_VECTOR_INREG: 137 case ISD::SIGN_EXTEND_VECTOR_INREG: 138 case ISD::ZERO_EXTEND_VECTOR_INREG: 139 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; 140 141 case ISD::SIGN_EXTEND: 142 case ISD::VP_SIGN_EXTEND: 143 case ISD::ZERO_EXTEND: 144 case ISD::VP_ZERO_EXTEND: 145 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 146 147 case ISD::VP_FP_TO_SINT: 148 case ISD::VP_FP_TO_UINT: 149 case ISD::STRICT_FP_TO_SINT: 150 case ISD::STRICT_FP_TO_UINT: 151 case ISD::FP_TO_SINT: 152 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 153 154 case ISD::FP_TO_SINT_SAT: 155 case ISD::FP_TO_UINT_SAT: 156 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; 157 158 case ISD::FP_TO_BF16: 159 case ISD::FP_TO_FP16: 160 Res = PromoteIntRes_FP_TO_FP16_BF16(N); 161 break; 162 163 case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break; 164 165 case ISD::AND: 166 case ISD::OR: 167 case ISD::XOR: 168 case ISD::ADD: 169 case ISD::SUB: 170 case ISD::MUL: 171 case ISD::VP_AND: 172 case ISD::VP_OR: 173 case ISD::VP_XOR: 174 case ISD::VP_ADD: 175 case ISD::VP_SUB: 176 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 177 178 case ISD::VP_SMIN: 179 case ISD::VP_SMAX: 180 case ISD::SDIV: 181 case ISD::SREM: 182 case ISD::VP_SDIV: 183 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; 184 185 case ISD::VP_UMIN: 186 case ISD::VP_UMAX: 187 case ISD::UDIV: 188 case ISD::UREM: 189 case ISD::VP_UDIV: 190 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; 191 192 case ISD::SADDO: 193 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 194 case ISD::UADDO: 195 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 196 case ISD::SMULO: 197 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 198 199 case ISD::ADDE: 200 case ISD::SUBE: 201 case ISD::UADDO_CARRY: 202 case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break; 203 204 case ISD::SADDO_CARRY: 205 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; 206 207 case ISD::SADDSAT: 208 case ISD::UADDSAT: 209 case ISD::SSUBSAT: 210 case ISD::USUBSAT: 211 case ISD::SSHLSAT: 212 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break; 213 214 case ISD::SMULFIX: 215 case ISD::SMULFIXSAT: 216 case ISD::UMULFIX: 217 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; 218 219 case ISD::SDIVFIX: 220 case ISD::SDIVFIXSAT: 221 case ISD::UDIVFIX: 222 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; 223 224 case ISD::ABS: Res = PromoteIntRes_ABS(N); break; 225 226 case ISD::ATOMIC_LOAD: 227 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 228 229 case ISD::ATOMIC_LOAD_ADD: 230 case ISD::ATOMIC_LOAD_SUB: 231 case ISD::ATOMIC_LOAD_AND: 232 case ISD::ATOMIC_LOAD_CLR: 233 case ISD::ATOMIC_LOAD_OR: 234 case ISD::ATOMIC_LOAD_XOR: 235 case ISD::ATOMIC_LOAD_NAND: 236 case ISD::ATOMIC_LOAD_MIN: 237 case ISD::ATOMIC_LOAD_MAX: 238 case ISD::ATOMIC_LOAD_UMIN: 239 case ISD::ATOMIC_LOAD_UMAX: 240 case ISD::ATOMIC_SWAP: 241 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 242 243 case ISD::ATOMIC_CMP_SWAP: 244 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 245 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 246 break; 247 248 case ISD::VECREDUCE_ADD: 249 case ISD::VECREDUCE_MUL: 250 case ISD::VECREDUCE_AND: 251 case ISD::VECREDUCE_OR: 252 case ISD::VECREDUCE_XOR: 253 case ISD::VECREDUCE_SMAX: 254 case ISD::VECREDUCE_SMIN: 255 case ISD::VECREDUCE_UMAX: 256 case ISD::VECREDUCE_UMIN: 257 Res = PromoteIntRes_VECREDUCE(N); 258 break; 259 260 case ISD::VP_REDUCE_ADD: 261 case ISD::VP_REDUCE_MUL: 262 case ISD::VP_REDUCE_AND: 263 case ISD::VP_REDUCE_OR: 264 case ISD::VP_REDUCE_XOR: 265 case ISD::VP_REDUCE_SMAX: 266 case ISD::VP_REDUCE_SMIN: 267 case ISD::VP_REDUCE_UMAX: 268 case ISD::VP_REDUCE_UMIN: 269 Res = PromoteIntRes_VP_REDUCE(N); 270 break; 271 272 case ISD::FREEZE: 273 Res = PromoteIntRes_FREEZE(N); 274 break; 275 276 case ISD::ROTL: 277 case ISD::ROTR: 278 Res = PromoteIntRes_Rotate(N); 279 break; 280 281 case ISD::FSHL: 282 case ISD::FSHR: 283 Res = PromoteIntRes_FunnelShift(N); 284 break; 285 286 case ISD::IS_FPCLASS: 287 Res = PromoteIntRes_IS_FPCLASS(N); 288 break; 289 case ISD::FFREXP: 290 Res = PromoteIntRes_FFREXP(N); 291 break; 292 } 293 294 // If the result is null then the sub-method took care of registering it. 295 if (Res.getNode()) 296 SetPromotedInteger(SDValue(N, ResNo), Res); 297 } 298 299 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 300 unsigned ResNo) { 301 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 302 return GetPromotedInteger(Op); 303 } 304 305 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 306 // Sign-extend the new bits, and continue the assertion. 307 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 308 return DAG.getNode(ISD::AssertSext, SDLoc(N), 309 Op.getValueType(), Op, N->getOperand(1)); 310 } 311 312 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 313 // Zero the new bits, and continue the assertion. 314 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 315 return DAG.getNode(ISD::AssertZext, SDLoc(N), 316 Op.getValueType(), Op, N->getOperand(1)); 317 } 318 319 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 320 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 321 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 322 N->getMemoryVT(), ResVT, 323 N->getChain(), N->getBasePtr(), 324 N->getMemOperand()); 325 // Legalize the chain result - switch anything that used the old chain to 326 // use the new one. 327 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 328 return Res; 329 } 330 331 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 332 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 333 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 334 N->getMemoryVT(), 335 N->getChain(), N->getBasePtr(), 336 Op2, N->getMemOperand()); 337 // Legalize the chain result - switch anything that used the old chain to 338 // use the new one. 339 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 340 return Res; 341 } 342 343 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 344 unsigned ResNo) { 345 if (ResNo == 1) { 346 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 347 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 348 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 349 350 // Only use the result of getSetCCResultType if it is legal, 351 // otherwise just use the promoted result type (NVT). 352 if (!TLI.isTypeLegal(SVT)) 353 SVT = NVT; 354 355 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 356 SDValue Res = DAG.getAtomicCmpSwap( 357 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 358 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 359 N->getMemOperand()); 360 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 361 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 362 return Res.getValue(1); 363 } 364 365 // Op2 is used for the comparison and thus must be extended according to the 366 // target's atomic operations. Op3 is merely stored and so can be left alone. 367 SDValue Op2 = N->getOperand(2); 368 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 369 switch (TLI.getExtendForAtomicCmpSwapArg()) { 370 case ISD::SIGN_EXTEND: 371 Op2 = SExtPromotedInteger(Op2); 372 break; 373 case ISD::ZERO_EXTEND: 374 Op2 = ZExtPromotedInteger(Op2); 375 break; 376 case ISD::ANY_EXTEND: 377 Op2 = GetPromotedInteger(Op2); 378 break; 379 default: 380 llvm_unreachable("Invalid atomic op extension"); 381 } 382 383 SDVTList VTs = 384 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 385 SDValue Res = DAG.getAtomicCmpSwap( 386 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 387 N->getBasePtr(), Op2, Op3, N->getMemOperand()); 388 // Update the use to N with the newly created Res. 389 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) 390 ReplaceValueWith(SDValue(N, i), Res.getValue(i)); 391 return Res; 392 } 393 394 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 395 SDValue InOp = N->getOperand(0); 396 EVT InVT = InOp.getValueType(); 397 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 398 EVT OutVT = N->getValueType(0); 399 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 400 SDLoc dl(N); 401 402 switch (getTypeAction(InVT)) { 403 case TargetLowering::TypeLegal: 404 break; 405 case TargetLowering::TypePromoteInteger: 406 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 407 // The input promotes to the same size. Convert the promoted value. 408 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 409 break; 410 case TargetLowering::TypeSoftenFloat: 411 // Promote the integer operand by hand. 412 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 413 case TargetLowering::TypeSoftPromoteHalf: 414 // Promote the integer operand by hand. 415 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp)); 416 case TargetLowering::TypePromoteFloat: { 417 // Convert the promoted float by hand. 418 if (!NOutVT.isVector()) 419 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); 420 break; 421 } 422 case TargetLowering::TypeExpandInteger: 423 case TargetLowering::TypeExpandFloat: 424 break; 425 case TargetLowering::TypeScalarizeVector: 426 // Convert the element to an integer and promote it by hand. 427 if (!NOutVT.isVector()) 428 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 429 BitConvertToInteger(GetScalarizedVector(InOp))); 430 break; 431 case TargetLowering::TypeScalarizeScalableVector: 432 report_fatal_error("Scalarization of scalable vectors is not supported."); 433 case TargetLowering::TypeSplitVector: { 434 if (!NOutVT.isVector()) { 435 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 436 // pieces of the input into integers and reassemble in the final type. 437 SDValue Lo, Hi; 438 GetSplitVector(N->getOperand(0), Lo, Hi); 439 Lo = BitConvertToInteger(Lo); 440 Hi = BitConvertToInteger(Hi); 441 442 if (DAG.getDataLayout().isBigEndian()) 443 std::swap(Lo, Hi); 444 445 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 446 EVT::getIntegerVT(*DAG.getContext(), 447 NOutVT.getSizeInBits()), 448 JoinIntegers(Lo, Hi)); 449 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 450 } 451 break; 452 } 453 case TargetLowering::TypeWidenVector: 454 // The input is widened to the same size. Convert to the widened value. 455 // Make sure that the outgoing value is not a vector, because this would 456 // make us bitcast between two vectors which are legalized in different ways. 457 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) { 458 SDValue Res = 459 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 460 461 // For big endian targets we need to shift the casted value or the 462 // interesting bits will end up at the wrong place. 463 if (DAG.getDataLayout().isBigEndian()) { 464 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 465 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!"); 466 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res, 467 DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl)); 468 } 469 return Res; 470 } 471 // If the output type is also a vector and widening it to the same size 472 // as the widened input type would be a legal type, we can widen the bitcast 473 // and handle the promotion after. 474 if (NOutVT.isVector()) { 475 TypeSize WidenInSize = NInVT.getSizeInBits(); 476 TypeSize OutSize = OutVT.getSizeInBits(); 477 if (WidenInSize.hasKnownScalarFactor(OutSize)) { 478 unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize); 479 EVT WideOutVT = 480 EVT::getVectorVT(*DAG.getContext(), OutVT.getVectorElementType(), 481 OutVT.getVectorElementCount() * Scale); 482 if (isTypeLegal(WideOutVT)) { 483 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); 484 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, 485 DAG.getVectorIdxConstant(0, dl)); 486 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp); 487 } 488 } 489 } 490 } 491 492 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 493 CreateStackStoreLoad(InOp, OutVT)); 494 } 495 496 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { 497 SDValue V = GetPromotedInteger(N->getOperand(0)); 498 return DAG.getNode(ISD::FREEZE, SDLoc(N), 499 V.getValueType(), V); 500 } 501 502 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 503 SDValue Op = GetPromotedInteger(N->getOperand(0)); 504 EVT OVT = N->getValueType(0); 505 EVT NVT = Op.getValueType(); 506 SDLoc dl(N); 507 508 // If the larger BSWAP isn't supported by the target, try to expand now. 509 // If we expand later we'll end up with more operations since we lost the 510 // original type. We only do this for scalars since we have a shuffle 511 // based lowering for vectors in LegalizeVectorOps. 512 if (!OVT.isVector() && 513 !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) { 514 if (SDValue Res = TLI.expandBSWAP(N, DAG)) 515 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); 516 } 517 518 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 519 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 520 DAG.getShiftAmountConstant(DiffBits, NVT, dl)); 521 } 522 523 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { 524 SDValue Op = GetPromotedInteger(N->getOperand(0)); 525 EVT OVT = N->getValueType(0); 526 EVT NVT = Op.getValueType(); 527 SDLoc dl(N); 528 529 // If the larger BITREVERSE isn't supported by the target, try to expand now. 530 // If we expand later we'll end up with more operations since we lost the 531 // original type. We only do this for scalars since we have a shuffle 532 // based lowering for vectors in LegalizeVectorOps. 533 if (!OVT.isVector() && OVT.isSimple() && 534 !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) { 535 if (SDValue Res = TLI.expandBITREVERSE(N, DAG)) 536 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); 537 } 538 539 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 540 return DAG.getNode(ISD::SRL, dl, NVT, 541 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), 542 DAG.getShiftAmountConstant(DiffBits, NVT, dl)); 543 } 544 545 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 546 // The pair element type may be legal, or may not promote to the same type as 547 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 548 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 549 TLI.getTypeToTransformTo(*DAG.getContext(), 550 N->getValueType(0)), JoinIntegers(N->getOperand(0), 551 N->getOperand(1))); 552 } 553 554 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 555 EVT VT = N->getValueType(0); 556 // FIXME there is no actual debug info here 557 SDLoc dl(N); 558 // Zero extend things like i1, sign extend everything else. It shouldn't 559 // matter in theory which one we pick, but this tends to give better code? 560 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 561 SDValue Result = DAG.getNode(Opc, dl, 562 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 563 SDValue(N, 0)); 564 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 565 return Result; 566 } 567 568 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 569 EVT OVT = N->getValueType(0); 570 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 571 SDLoc dl(N); 572 573 // If the larger CTLZ isn't supported by the target, try to expand now. 574 // If we expand later we'll end up with more operations since we lost the 575 // original type. 576 if (!OVT.isVector() && TLI.isTypeLegal(NVT) && 577 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ, NVT) && 578 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ_ZERO_UNDEF, NVT)) { 579 if (SDValue Result = TLI.expandCTLZ(N, DAG)) { 580 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); 581 return Result; 582 } 583 } 584 585 // Zero extend to the promoted type and do the count there. 586 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 587 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 588 // Subtract off the extra leading bits in the bigger type. 589 return DAG.getNode( 590 ISD::SUB, dl, NVT, Op, 591 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, 592 NVT)); 593 } 594 595 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { 596 EVT OVT = N->getValueType(0); 597 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 598 599 // If the larger CTPOP isn't supported by the target, try to expand now. 600 // If we expand later we'll end up with more operations since we lost the 601 // original type. 602 // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to 603 // TargetLowering. 604 if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) && 605 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTPOP, NVT)) { 606 if (SDValue Result = TLI.expandCTPOP(N, DAG)) { 607 Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result); 608 return Result; 609 } 610 } 611 612 // Zero extend to the promoted type and do the count or parity there. 613 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 614 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op); 615 } 616 617 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 618 SDValue Op = GetPromotedInteger(N->getOperand(0)); 619 EVT OVT = N->getValueType(0); 620 EVT NVT = Op.getValueType(); 621 SDLoc dl(N); 622 623 // If the larger CTTZ isn't supported by the target, try to expand now. 624 // If we expand later we'll end up with more operations since we lost the 625 // original type. Don't expand if we can use CTPOP or CTLZ expansion on the 626 // larger type. 627 if (!OVT.isVector() && TLI.isTypeLegal(NVT) && 628 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ, NVT) && 629 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ_ZERO_UNDEF, NVT) && 630 !TLI.isOperationLegal(ISD::CTPOP, NVT) && 631 !TLI.isOperationLegal(ISD::CTLZ, NVT)) { 632 if (SDValue Result = TLI.expandCTTZ(N, DAG)) { 633 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); 634 return Result; 635 } 636 } 637 638 if (N->getOpcode() == ISD::CTTZ) { 639 // The count is the same in the promoted type except if the original 640 // value was zero. This can be handled by setting the bit just off 641 // the top of the original type. 642 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), 643 OVT.getScalarSizeInBits()); 644 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); 645 } 646 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 647 } 648 649 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 650 SDLoc dl(N); 651 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 652 653 SDValue Op0 = N->getOperand(0); 654 SDValue Op1 = N->getOperand(1); 655 656 // If the input also needs to be promoted, do that first so we can get a 657 // get a good idea for the output type. 658 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType()) 659 == TargetLowering::TypePromoteInteger) { 660 SDValue In = GetPromotedInteger(Op0); 661 662 // If the new type is larger than NVT, use it. We probably won't need to 663 // promote it again. 664 EVT SVT = In.getValueType().getScalarType(); 665 if (SVT.bitsGE(NVT)) { 666 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1); 667 return DAG.getAnyExtOrTrunc(Ext, dl, NVT); 668 } 669 } 670 671 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1); 672 } 673 674 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 675 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 676 unsigned NewOpc = N->getOpcode(); 677 SDLoc dl(N); 678 679 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 680 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 681 // and SINT conversions are Custom, there is no way to tell which is 682 // preferable. We choose SINT because that's the right thing on PPC.) 683 if (N->getOpcode() == ISD::FP_TO_UINT && 684 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 685 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 686 NewOpc = ISD::FP_TO_SINT; 687 688 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && 689 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) && 690 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) 691 NewOpc = ISD::STRICT_FP_TO_SINT; 692 693 if (N->getOpcode() == ISD::VP_FP_TO_UINT && 694 !TLI.isOperationLegal(ISD::VP_FP_TO_UINT, NVT) && 695 TLI.isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, NVT)) 696 NewOpc = ISD::VP_FP_TO_SINT; 697 698 SDValue Res; 699 if (N->isStrictFPOpcode()) { 700 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, 701 {N->getOperand(0), N->getOperand(1)}); 702 // Legalize the chain result - switch anything that used the old chain to 703 // use the new one. 704 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 705 } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) { 706 Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1), 707 N->getOperand(2)}); 708 } else { 709 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 710 } 711 712 // Assert that the converted value fits in the original type. If it doesn't 713 // (eg: because the value being converted is too big), then the result of the 714 // original operation was undefined anyway, so the assert is still correct. 715 // 716 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: 717 // before legalization: fp-to-uint16, 65534. -> 0xfffe 718 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe 719 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT || 720 N->getOpcode() == ISD::STRICT_FP_TO_UINT || 721 N->getOpcode() == ISD::VP_FP_TO_UINT) 722 ? ISD::AssertZext 723 : ISD::AssertSext, 724 dl, NVT, Res, 725 DAG.getValueType(N->getValueType(0).getScalarType())); 726 } 727 728 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { 729 // Promote the result type, while keeping the original width in Op1. 730 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 731 SDLoc dl(N); 732 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), 733 N->getOperand(1)); 734 } 735 736 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) { 737 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 738 SDLoc dl(N); 739 740 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 741 } 742 743 SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) { 744 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 745 SDLoc dl(N); 746 747 SDValue Res = 748 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); 749 750 // Legalize the chain result - switch anything that used the old chain to 751 // use the new one. 752 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 753 return Res; 754 } 755 756 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 757 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 758 SDLoc dl(N); 759 760 if (getTypeAction(N->getOperand(0).getValueType()) 761 == TargetLowering::TypePromoteInteger) { 762 SDValue Res = GetPromotedInteger(N->getOperand(0)); 763 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 764 765 // If the result and operand types are the same after promotion, simplify 766 // to an in-register extension. Unless this is a VP_*_EXTEND. 767 if (NVT == Res.getValueType() && N->getNumOperands() == 1) { 768 // The high bits are not guaranteed to be anything. Insert an extend. 769 if (N->getOpcode() == ISD::SIGN_EXTEND) 770 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 771 DAG.getValueType(N->getOperand(0).getValueType())); 772 if (N->getOpcode() == ISD::ZERO_EXTEND) 773 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); 774 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 775 return Res; 776 } 777 } 778 779 // Otherwise, just extend the original operand all the way to the larger type. 780 if (N->getNumOperands() != 1) { 781 assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); 782 assert(N->isVPOpcode() && "Expected VP opcode"); 783 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), 784 N->getOperand(1), N->getOperand(2)); 785 } 786 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 787 } 788 789 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 790 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 791 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 792 ISD::LoadExtType ExtType = 793 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 794 SDLoc dl(N); 795 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 796 N->getMemoryVT(), N->getMemOperand()); 797 798 // Legalize the chain result - switch anything that used the old chain to 799 // use the new one. 800 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 801 return Res; 802 } 803 804 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { 805 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 806 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 807 808 ISD::LoadExtType ExtType = N->getExtensionType(); 809 if (ExtType == ISD::NON_EXTLOAD) 810 ExtType = ISD::EXTLOAD; 811 812 SDLoc dl(N); 813 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), 814 N->getOffset(), N->getMask(), ExtPassThru, 815 N->getMemoryVT(), N->getMemOperand(), 816 N->getAddressingMode(), ExtType, 817 N->isExpandingLoad()); 818 // Legalize the chain result - switch anything that used the old chain to 819 // use the new one. 820 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 821 return Res; 822 } 823 824 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { 825 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 826 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 827 assert(NVT == ExtPassThru.getValueType() && 828 "Gather result type and the passThru argument type should be the same"); 829 830 ISD::LoadExtType ExtType = N->getExtensionType(); 831 if (ExtType == ISD::NON_EXTLOAD) 832 ExtType = ISD::EXTLOAD; 833 834 SDLoc dl(N); 835 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), 836 N->getIndex(), N->getScale() }; 837 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), 838 N->getMemoryVT(), dl, Ops, 839 N->getMemOperand(), N->getIndexType(), 840 ExtType); 841 // Legalize the chain result - switch anything that used the old chain to 842 // use the new one. 843 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 844 return Res; 845 } 846 847 /// Promote the overflow flag of an overflowing arithmetic node. 848 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 849 // Change the return type of the boolean result while obeying 850 // getSetCCResultType. 851 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 852 EVT VT = N->getValueType(0); 853 EVT SVT = getSetCCResultType(VT); 854 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; 855 unsigned NumOps = N->getNumOperands(); 856 assert(NumOps <= 3 && "Too many operands"); 857 if (NumOps == 3) 858 Ops[2] = N->getOperand(2); 859 860 SDLoc dl(N); 861 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), 862 ArrayRef(Ops, NumOps)); 863 864 // Modified the sum result - switch anything that used the old sum to use 865 // the new one. 866 ReplaceValueWith(SDValue(N, 0), Res); 867 868 // Convert to the expected type. 869 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); 870 } 871 872 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { 873 // If the promoted type is legal, we can convert this to: 874 // 1. ANY_EXTEND iN to iM 875 // 2. SHL by M-N 876 // 3. [US][ADD|SUB|SHL]SAT 877 // 4. L/ASHR by M-N 878 // Else it is more efficient to convert this to a min and a max 879 // operation in the higher precision arithmetic. 880 SDLoc dl(N); 881 SDValue Op1 = N->getOperand(0); 882 SDValue Op2 = N->getOperand(1); 883 unsigned OldBits = Op1.getScalarValueSizeInBits(); 884 885 unsigned Opcode = N->getOpcode(); 886 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; 887 888 SDValue Op1Promoted, Op2Promoted; 889 if (IsShift) { 890 Op1Promoted = GetPromotedInteger(Op1); 891 Op2Promoted = ZExtPromotedInteger(Op2); 892 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { 893 Op1Promoted = ZExtPromotedInteger(Op1); 894 Op2Promoted = ZExtPromotedInteger(Op2); 895 } else { 896 Op1Promoted = SExtPromotedInteger(Op1); 897 Op2Promoted = SExtPromotedInteger(Op2); 898 } 899 EVT PromotedType = Op1Promoted.getValueType(); 900 unsigned NewBits = PromotedType.getScalarSizeInBits(); 901 902 if (Opcode == ISD::UADDSAT) { 903 APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits); 904 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 905 SDValue Add = 906 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); 907 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); 908 } 909 910 // USUBSAT can always be promoted as long as we have zero-extended the args. 911 if (Opcode == ISD::USUBSAT) 912 return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted, 913 Op2Promoted); 914 915 // Shift cannot use a min/max expansion, we can't detect overflow if all of 916 // the bits have been shifted out. 917 if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) { 918 unsigned ShiftOp; 919 switch (Opcode) { 920 case ISD::SADDSAT: 921 case ISD::SSUBSAT: 922 case ISD::SSHLSAT: 923 ShiftOp = ISD::SRA; 924 break; 925 case ISD::USHLSAT: 926 ShiftOp = ISD::SRL; 927 break; 928 default: 929 llvm_unreachable("Expected opcode to be signed or unsigned saturation " 930 "addition, subtraction or left shift"); 931 } 932 933 unsigned SHLAmount = NewBits - OldBits; 934 SDValue ShiftAmount = 935 DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl); 936 Op1Promoted = 937 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); 938 if (!IsShift) 939 Op2Promoted = 940 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); 941 942 SDValue Result = 943 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); 944 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); 945 } 946 947 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; 948 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits); 949 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits); 950 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType); 951 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 952 SDValue Result = 953 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); 954 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); 955 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); 956 return Result; 957 } 958 959 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { 960 // Can just promote the operands then continue with operation. 961 SDLoc dl(N); 962 SDValue Op1Promoted, Op2Promoted; 963 bool Signed = 964 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; 965 bool Saturating = 966 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; 967 if (Signed) { 968 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 969 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 970 } else { 971 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 972 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 973 } 974 EVT OldType = N->getOperand(0).getValueType(); 975 EVT PromotedType = Op1Promoted.getValueType(); 976 unsigned DiffSize = 977 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); 978 979 if (Saturating) { 980 // Promoting the operand and result values changes the saturation width, 981 // which is extends the values that we clamp to on saturation. This could be 982 // resolved by shifting one of the operands the same amount, which would 983 // also shift the result we compare against, then shifting back. 984 Op1Promoted = 985 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 986 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); 987 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 988 Op2Promoted, N->getOperand(2)); 989 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; 990 return DAG.getNode(ShiftOp, dl, PromotedType, Result, 991 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); 992 } 993 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, 994 N->getOperand(2)); 995 } 996 997 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, 998 unsigned SatW, bool Signed, 999 const TargetLowering &TLI, 1000 SelectionDAG &DAG) { 1001 EVT VT = V.getValueType(); 1002 unsigned VTW = VT.getScalarSizeInBits(); 1003 1004 if (!Signed) { 1005 // Saturate to the unsigned maximum by getting the minimum of V and the 1006 // maximum. 1007 return DAG.getNode(ISD::UMIN, dl, VT, V, 1008 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW), 1009 dl, VT)); 1010 } 1011 1012 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the 1013 // signed minimum of it and V. 1014 V = DAG.getNode(ISD::SMIN, dl, VT, V, 1015 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1), 1016 dl, VT)); 1017 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the 1018 // signed maximum of it and V. 1019 V = DAG.getNode(ISD::SMAX, dl, VT, V, 1020 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1), 1021 dl, VT)); 1022 return V; 1023 } 1024 1025 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, 1026 unsigned Scale, const TargetLowering &TLI, 1027 SelectionDAG &DAG, unsigned SatW = 0) { 1028 EVT VT = LHS.getValueType(); 1029 unsigned VTSize = VT.getScalarSizeInBits(); 1030 bool Signed = N->getOpcode() == ISD::SDIVFIX || 1031 N->getOpcode() == ISD::SDIVFIXSAT; 1032 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 1033 N->getOpcode() == ISD::UDIVFIXSAT; 1034 1035 SDLoc dl(N); 1036 // Widen the types by a factor of two. This is guaranteed to expand, since it 1037 // will always have enough high bits in the LHS to shift into. 1038 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2); 1039 if (VT.isVector()) 1040 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, 1041 VT.getVectorElementCount()); 1042 LHS = DAG.getExtOrTrunc(Signed, LHS, dl, WideVT); 1043 RHS = DAG.getExtOrTrunc(Signed, RHS, dl, WideVT); 1044 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, 1045 DAG); 1046 assert(Res && "Expanding DIVFIX with wide type failed?"); 1047 if (Saturating) { 1048 // If the caller has told us to saturate at something less, use that width 1049 // instead of the type before doubling. However, it cannot be more than 1050 // what we just widened! 1051 assert(SatW <= VTSize && 1052 "Tried to saturate to more than the original type?"); 1053 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed, 1054 TLI, DAG); 1055 } 1056 return DAG.getZExtOrTrunc(Res, dl, VT); 1057 } 1058 1059 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { 1060 SDLoc dl(N); 1061 SDValue Op1Promoted, Op2Promoted; 1062 bool Signed = N->getOpcode() == ISD::SDIVFIX || 1063 N->getOpcode() == ISD::SDIVFIXSAT; 1064 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 1065 N->getOpcode() == ISD::UDIVFIXSAT; 1066 if (Signed) { 1067 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 1068 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 1069 } else { 1070 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 1071 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 1072 } 1073 EVT PromotedType = Op1Promoted.getValueType(); 1074 unsigned Scale = N->getConstantOperandVal(2); 1075 1076 // If the type is already legal and the operation is legal in that type, we 1077 // should not early expand. 1078 if (TLI.isTypeLegal(PromotedType)) { 1079 TargetLowering::LegalizeAction Action = 1080 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale); 1081 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { 1082 unsigned Diff = PromotedType.getScalarSizeInBits() - 1083 N->getValueType(0).getScalarSizeInBits(); 1084 if (Saturating) 1085 Op1Promoted = 1086 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 1087 DAG.getShiftAmountConstant(Diff, PromotedType, dl)); 1088 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 1089 Op2Promoted, N->getOperand(2)); 1090 if (Saturating) 1091 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res, 1092 DAG.getShiftAmountConstant(Diff, PromotedType, dl)); 1093 return Res; 1094 } 1095 } 1096 1097 // See if we can perform the division in this type without expanding. 1098 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted, 1099 Op2Promoted, Scale, DAG)) { 1100 if (Saturating) 1101 Res = SaturateWidenedDIVFIX(Res, dl, 1102 N->getValueType(0).getScalarSizeInBits(), 1103 Signed, TLI, DAG); 1104 return Res; 1105 } 1106 // If we cannot, expand it to twice the type width. If we are saturating, give 1107 // it the original width as a saturating width so we don't need to emit 1108 // two saturations. 1109 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG, 1110 N->getValueType(0).getScalarSizeInBits()); 1111 } 1112 1113 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 1114 if (ResNo == 1) 1115 return PromoteIntRes_Overflow(N); 1116 1117 // The operation overflowed iff the result in the larger type is not the 1118 // sign extension of its truncation to the original type. 1119 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1120 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1121 EVT OVT = N->getOperand(0).getValueType(); 1122 EVT NVT = LHS.getValueType(); 1123 SDLoc dl(N); 1124 1125 // Do the arithmetic in the larger type. 1126 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 1127 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1128 1129 // Calculate the overflow flag: sign extend the arithmetic result from 1130 // the original type. 1131 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 1132 DAG.getValueType(OVT)); 1133 // Overflowed if and only if this is not equal to Res. 1134 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1135 1136 // Use the calculated overflow everywhere. 1137 ReplaceValueWith(SDValue(N, 1), Ofl); 1138 1139 return Res; 1140 } 1141 1142 SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) { 1143 SDValue Mask = N->getOperand(0); 1144 1145 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 1146 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 1147 1148 unsigned Opcode = N->getOpcode(); 1149 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) 1150 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS, 1151 N->getOperand(3)); 1152 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS); 1153 } 1154 1155 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 1156 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 1157 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 1158 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 1159 LHS.getValueType(), N->getOperand(0), 1160 N->getOperand(1), LHS, RHS, N->getOperand(4)); 1161 } 1162 1163 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 1164 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 1165 EVT InVT = N->getOperand(OpNo).getValueType(); 1166 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1167 1168 EVT SVT = getSetCCResultType(InVT); 1169 1170 // If we got back a type that needs to be promoted, this likely means the 1171 // the input type also needs to be promoted. So get the promoted type for 1172 // the input and try the query again. 1173 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { 1174 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 1175 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 1176 SVT = getSetCCResultType(InVT); 1177 } else { 1178 // Input type isn't promoted, just use the default promoted type. 1179 SVT = NVT; 1180 } 1181 } 1182 1183 SDLoc dl(N); 1184 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && 1185 "Vector compare must return a vector result!"); 1186 1187 // Get the SETCC result using the canonical SETCC type. 1188 SDValue SetCC; 1189 if (N->isStrictFPOpcode()) { 1190 SDVTList VTs = DAG.getVTList({SVT, MVT::Other}); 1191 SDValue Opers[] = {N->getOperand(0), N->getOperand(1), 1192 N->getOperand(2), N->getOperand(3)}; 1193 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags()); 1194 // Legalize the chain result - switch anything that used the old chain to 1195 // use the new one. 1196 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1)); 1197 } else 1198 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), 1199 N->getOperand(1), N->getOperand(2), N->getFlags()); 1200 1201 // Convert to the expected type. 1202 return DAG.getSExtOrTrunc(SetCC, dl, NVT); 1203 } 1204 1205 SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) { 1206 SDLoc DL(N); 1207 SDValue Arg = N->getOperand(0); 1208 SDValue Test = N->getOperand(1); 1209 EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1210 return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test); 1211 } 1212 1213 SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) { 1214 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 1215 EVT VT = N->getValueType(0); 1216 1217 SDLoc dl(N); 1218 SDValue Res = 1219 DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, NVT), N->getOperand(0)); 1220 1221 ReplaceValueWith(SDValue(N, 0), Res); 1222 return Res.getValue(1); 1223 } 1224 1225 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 1226 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1227 SDValue RHS = N->getOperand(1); 1228 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1229 RHS = ZExtPromotedInteger(RHS); 1230 if (N->getOpcode() != ISD::VP_SHL) 1231 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1232 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1233 N->getOperand(2), N->getOperand(3)); 1234 } 1235 1236 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 1237 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1238 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 1239 Op.getValueType(), Op, N->getOperand(1)); 1240 } 1241 1242 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 1243 // The input may have strange things in the top bits of the registers, but 1244 // these operations don't care. They may have weird bits going out, but 1245 // that too is okay if they are integer operations. 1246 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1247 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 1248 if (N->getNumOperands() == 2) 1249 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1250 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1251 assert(N->isVPOpcode() && "Expected VP opcode"); 1252 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1253 N->getOperand(2), N->getOperand(3)); 1254 } 1255 1256 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { 1257 // Sign extend the input. 1258 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1259 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1260 if (N->getNumOperands() == 2) 1261 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1262 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1263 assert(N->isVPOpcode() && "Expected VP opcode"); 1264 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1265 N->getOperand(2), N->getOperand(3)); 1266 } 1267 1268 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { 1269 // Zero extend the input. 1270 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1271 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1272 if (N->getNumOperands() == 2) 1273 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1274 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1275 assert(N->isVPOpcode() && "Expected VP opcode"); 1276 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1277 N->getOperand(2), N->getOperand(3)); 1278 } 1279 1280 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { 1281 // It doesn't matter if we sign extend or zero extend in the inputs. So do 1282 // whatever is best for the target. 1283 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0)); 1284 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1)); 1285 return DAG.getNode(N->getOpcode(), SDLoc(N), 1286 LHS.getValueType(), LHS, RHS); 1287 } 1288 1289 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 1290 // The input value must be properly sign extended. 1291 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1292 SDValue RHS = N->getOperand(1); 1293 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1294 RHS = ZExtPromotedInteger(RHS); 1295 if (N->getOpcode() != ISD::VP_ASHR) 1296 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1297 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1298 N->getOperand(2), N->getOperand(3)); 1299 } 1300 1301 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 1302 // The input value must be properly zero extended. 1303 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1304 SDValue RHS = N->getOperand(1); 1305 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1306 RHS = ZExtPromotedInteger(RHS); 1307 if (N->getOpcode() != ISD::VP_LSHR) 1308 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1309 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1310 N->getOperand(2), N->getOperand(3)); 1311 } 1312 1313 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { 1314 // Lower the rotate to shifts and ORs which can be promoted. 1315 SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG); 1316 ReplaceValueWith(SDValue(N, 0), Res); 1317 return SDValue(); 1318 } 1319 1320 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { 1321 SDValue Hi = GetPromotedInteger(N->getOperand(0)); 1322 SDValue Lo = GetPromotedInteger(N->getOperand(1)); 1323 SDValue Amt = N->getOperand(2); 1324 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger) 1325 Amt = ZExtPromotedInteger(Amt); 1326 EVT AmtVT = Amt.getValueType(); 1327 1328 SDLoc DL(N); 1329 EVT OldVT = N->getOperand(0).getValueType(); 1330 EVT VT = Lo.getValueType(); 1331 unsigned Opcode = N->getOpcode(); 1332 bool IsFSHR = Opcode == ISD::FSHR; 1333 unsigned OldBits = OldVT.getScalarSizeInBits(); 1334 unsigned NewBits = VT.getScalarSizeInBits(); 1335 1336 // Amount has to be interpreted modulo the old bit width. 1337 Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt, 1338 DAG.getConstant(OldBits, DL, AmtVT)); 1339 1340 // If the promoted type is twice the size (or more), then we use the 1341 // traditional funnel 'double' shift codegen. This isn't necessary if the 1342 // shift amount is constant. 1343 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. 1344 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). 1345 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) && 1346 !TLI.isOperationLegalOrCustom(Opcode, VT)) { 1347 SDValue HiShift = DAG.getConstant(OldBits, DL, VT); 1348 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift); 1349 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT); 1350 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo); 1351 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt); 1352 if (!IsFSHR) 1353 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift); 1354 return Res; 1355 } 1356 1357 // Shift Lo up to occupy the upper bits of the promoted type. 1358 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT); 1359 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset); 1360 1361 // Increase Amount to shift the result into the lower bits of the promoted 1362 // type. 1363 if (IsFSHR) 1364 Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset); 1365 1366 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt); 1367 } 1368 1369 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 1370 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1371 SDValue Res; 1372 SDValue InOp = N->getOperand(0); 1373 SDLoc dl(N); 1374 1375 switch (getTypeAction(InOp.getValueType())) { 1376 default: llvm_unreachable("Unknown type action!"); 1377 case TargetLowering::TypeLegal: 1378 case TargetLowering::TypeExpandInteger: 1379 Res = InOp; 1380 break; 1381 case TargetLowering::TypePromoteInteger: 1382 Res = GetPromotedInteger(InOp); 1383 break; 1384 case TargetLowering::TypeSplitVector: { 1385 EVT InVT = InOp.getValueType(); 1386 assert(InVT.isVector() && "Cannot split scalar types"); 1387 ElementCount NumElts = InVT.getVectorElementCount(); 1388 assert(NumElts == NVT.getVectorElementCount() && 1389 "Dst and Src must have the same number of elements"); 1390 assert(isPowerOf2_32(NumElts.getKnownMinValue()) && 1391 "Promoted vector type must be a power of two"); 1392 1393 SDValue EOp1, EOp2; 1394 GetSplitVector(InOp, EOp1, EOp2); 1395 1396 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 1397 NumElts.divideCoefficientBy(2)); 1398 if (N->getOpcode() == ISD::TRUNCATE) { 1399 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 1400 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 1401 } else { 1402 assert(N->getOpcode() == ISD::VP_TRUNCATE && 1403 "Expected VP_TRUNCATE opcode"); 1404 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 1405 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 1406 std::tie(EVLLo, EVLHi) = 1407 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); 1408 EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo); 1409 EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi); 1410 } 1411 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 1412 } 1413 // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some 1414 // targets. 1415 case TargetLowering::TypeWidenVector: { 1416 SDValue WideInOp = GetWidenedVector(InOp); 1417 1418 // Truncate widened InOp. 1419 unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); 1420 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), 1421 N->getValueType(0).getScalarType(), NumElem); 1422 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); 1423 1424 // Zero extend so that the elements are of same type as those of NVT 1425 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), 1426 NumElem); 1427 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); 1428 1429 // Extract the low NVT subvector. 1430 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl); 1431 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); 1432 } 1433 } 1434 1435 // Truncate to NVT instead of VT 1436 if (N->getOpcode() == ISD::VP_TRUNCATE) 1437 return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1), 1438 N->getOperand(2)); 1439 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 1440 } 1441 1442 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 1443 if (ResNo == 1) 1444 return PromoteIntRes_Overflow(N); 1445 1446 // The operation overflowed iff the result in the larger type is not the 1447 // zero extension of its truncation to the original type. 1448 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1449 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1450 EVT OVT = N->getOperand(0).getValueType(); 1451 EVT NVT = LHS.getValueType(); 1452 SDLoc dl(N); 1453 1454 // Do the arithmetic in the larger type. 1455 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 1456 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1457 1458 // Calculate the overflow flag: zero extend the arithmetic result from 1459 // the original type. 1460 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 1461 // Overflowed if and only if this is not equal to Res. 1462 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1463 1464 // Use the calculated overflow everywhere. 1465 ReplaceValueWith(SDValue(N, 1), Ofl); 1466 1467 return Res; 1468 } 1469 1470 // Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that 1471 // the third operand of ADDE/SUBE nodes is carry flag, which differs from 1472 // the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean. 1473 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N, 1474 unsigned ResNo) { 1475 if (ResNo == 1) 1476 return PromoteIntRes_Overflow(N); 1477 1478 // We need to sign-extend the operands so the carry value computed by the 1479 // wide operation will be equivalent to the carry value computed by the 1480 // narrow operation. 1481 // An UADDO_CARRY can generate carry only if any of the operands has its 1482 // most significant bit set. Sign extension propagates the most significant 1483 // bit into the higher bits which means the extra bit that the narrow 1484 // addition would need (i.e. the carry) will be propagated through the higher 1485 // bits of the wide addition. 1486 // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will 1487 // be preserved by sign extension. 1488 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1489 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1490 1491 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; 1492 1493 // Do the arithmetic in the wide type. 1494 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), 1495 LHS, RHS, N->getOperand(2)); 1496 1497 // Update the users of the original carry/borrow value. 1498 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 1499 1500 return SDValue(Res.getNode(), 0); 1501 } 1502 1503 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, 1504 unsigned ResNo) { 1505 assert(ResNo == 1 && "Don't know how to promote other results yet."); 1506 return PromoteIntRes_Overflow(N); 1507 } 1508 1509 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { 1510 EVT OVT = N->getValueType(0); 1511 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 1512 1513 // If a larger ABS or SMAX isn't supported by the target, try to expand now. 1514 // If we expand later we'll end up sign extending more than just the sra input 1515 // in sra+xor+sub expansion. 1516 if (!OVT.isVector() && 1517 !TLI.isOperationLegalOrCustomOrPromote(ISD::ABS, NVT) && 1518 !TLI.isOperationLegal(ISD::SMAX, NVT)) { 1519 if (SDValue Res = TLI.expandABS(N, DAG)) 1520 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res); 1521 } 1522 1523 SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); 1524 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); 1525 } 1526 1527 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 1528 // Promote the overflow bit trivially. 1529 if (ResNo == 1) 1530 return PromoteIntRes_Overflow(N); 1531 1532 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 1533 SDLoc DL(N); 1534 EVT SmallVT = LHS.getValueType(); 1535 1536 // To determine if the result overflowed in a larger type, we extend the 1537 // input to the larger type, do the multiply (checking if it overflows), 1538 // then also check the high bits of the result to see if overflow happened 1539 // there. 1540 if (N->getOpcode() == ISD::SMULO) { 1541 LHS = SExtPromotedInteger(LHS); 1542 RHS = SExtPromotedInteger(RHS); 1543 } else { 1544 LHS = ZExtPromotedInteger(LHS); 1545 RHS = ZExtPromotedInteger(RHS); 1546 } 1547 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 1548 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 1549 1550 // Overflow occurred if it occurred in the larger type, or if the high part 1551 // of the result does not zero/sign-extend the low part. Check this second 1552 // possibility first. 1553 SDValue Overflow; 1554 if (N->getOpcode() == ISD::UMULO) { 1555 // Unsigned overflow occurred if the high part is non-zero. 1556 unsigned Shift = SmallVT.getScalarSizeInBits(); 1557 SDValue Hi = 1558 DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 1559 DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL)); 1560 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 1561 DAG.getConstant(0, DL, Hi.getValueType()), 1562 ISD::SETNE); 1563 } else { 1564 // Signed overflow occurred if the high part does not sign extend the low. 1565 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 1566 Mul, DAG.getValueType(SmallVT)); 1567 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 1568 } 1569 1570 // The only other way for overflow to occur is if the multiplication in the 1571 // larger type itself overflowed. 1572 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 1573 SDValue(Mul.getNode(), 1)); 1574 1575 // Use the calculated overflow everywhere. 1576 ReplaceValueWith(SDValue(N, 1), Overflow); 1577 return Mul; 1578 } 1579 1580 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 1581 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 1582 N->getValueType(0))); 1583 } 1584 1585 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { 1586 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1587 1588 const APInt &MulImm = N->getConstantOperandAPInt(0); 1589 return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits())); 1590 } 1591 1592 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 1593 SDValue Chain = N->getOperand(0); // Get the chain. 1594 SDValue Ptr = N->getOperand(1); // Get the pointer. 1595 EVT VT = N->getValueType(0); 1596 SDLoc dl(N); 1597 1598 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 1599 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 1600 // The argument is passed as NumRegs registers of type RegVT. 1601 1602 SmallVector<SDValue, 8> Parts(NumRegs); 1603 for (unsigned i = 0; i < NumRegs; ++i) { 1604 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 1605 N->getConstantOperandVal(3)); 1606 Chain = Parts[i].getValue(1); 1607 } 1608 1609 // Handle endianness of the load. 1610 if (DAG.getDataLayout().isBigEndian()) 1611 std::reverse(Parts.begin(), Parts.end()); 1612 1613 // Assemble the parts in the promoted type. 1614 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1615 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 1616 for (unsigned i = 1; i < NumRegs; ++i) { 1617 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 1618 // Shift it to the right position and "or" it in. 1619 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 1620 DAG.getConstant(i * RegVT.getSizeInBits(), dl, 1621 TLI.getPointerTy(DAG.getDataLayout()))); 1622 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 1623 } 1624 1625 // Modified the chain result - switch anything that used the old chain to 1626 // use the new one. 1627 ReplaceValueWith(SDValue(N, 1), Chain); 1628 1629 return Res; 1630 } 1631 1632 //===----------------------------------------------------------------------===// 1633 // Integer Operand Promotion 1634 //===----------------------------------------------------------------------===// 1635 1636 /// PromoteIntegerOperand - This method is called when the specified operand of 1637 /// the specified node is found to need promotion. At this point, all of the 1638 /// result types of the node are known to be legal, but other operands of the 1639 /// node may need promotion or expansion as well as the specified one. 1640 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 1641 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); 1642 dbgs() << "\n"); 1643 SDValue Res = SDValue(); 1644 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 1645 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 1646 return false; 1647 } 1648 1649 switch (N->getOpcode()) { 1650 default: 1651 #ifndef NDEBUG 1652 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 1653 N->dump(&DAG); dbgs() << "\n"; 1654 #endif 1655 report_fatal_error("Do not know how to promote this operator's operand!"); 1656 1657 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 1658 case ISD::ATOMIC_STORE: 1659 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 1660 break; 1661 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 1662 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 1663 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 1664 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 1665 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 1666 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 1667 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 1668 case ISD::INSERT_VECTOR_ELT: 1669 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo); 1670 break; 1671 case ISD::SPLAT_VECTOR: 1672 case ISD::SCALAR_TO_VECTOR: 1673 Res = PromoteIntOp_ScalarOp(N); 1674 break; 1675 case ISD::VSELECT: 1676 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 1677 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 1678 case ISD::VP_SETCC: 1679 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 1680 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 1681 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break; 1682 case ISD::VP_SINT_TO_FP: 1683 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 1684 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; 1685 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 1686 OpNo); break; 1687 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), 1688 OpNo); break; 1689 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), 1690 OpNo); break; 1691 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), 1692 OpNo); break; 1693 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), 1694 OpNo); break; 1695 case ISD::VP_TRUNCATE: 1696 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 1697 case ISD::BF16_TO_FP: 1698 case ISD::FP16_TO_FP: 1699 case ISD::VP_UINT_TO_FP: 1700 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 1701 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; 1702 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 1703 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break; 1704 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; 1705 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break; 1706 1707 case ISD::SHL: 1708 case ISD::SRA: 1709 case ISD::SRL: 1710 case ISD::ROTL: 1711 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 1712 1713 case ISD::FSHL: 1714 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break; 1715 1716 case ISD::SADDO_CARRY: 1717 case ISD::SSUBO_CARRY: 1718 case ISD::UADDO_CARRY: 1719 case ISD::USUBO_CARRY: Res = PromoteIntOp_ADDSUBO_CARRY(N, OpNo); break; 1720 1721 case ISD::FRAMEADDR: 1722 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; 1723 1724 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break; 1725 1726 case ISD::SMULFIX: 1727 case ISD::SMULFIXSAT: 1728 case ISD::UMULFIX: 1729 case ISD::UMULFIXSAT: 1730 case ISD::SDIVFIX: 1731 case ISD::SDIVFIXSAT: 1732 case ISD::UDIVFIX: 1733 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; 1734 case ISD::FPOWI: 1735 case ISD::STRICT_FPOWI: 1736 case ISD::FLDEXP: 1737 case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break; 1738 case ISD::VECREDUCE_ADD: 1739 case ISD::VECREDUCE_MUL: 1740 case ISD::VECREDUCE_AND: 1741 case ISD::VECREDUCE_OR: 1742 case ISD::VECREDUCE_XOR: 1743 case ISD::VECREDUCE_SMAX: 1744 case ISD::VECREDUCE_SMIN: 1745 case ISD::VECREDUCE_UMAX: 1746 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; 1747 case ISD::VP_REDUCE_ADD: 1748 case ISD::VP_REDUCE_MUL: 1749 case ISD::VP_REDUCE_AND: 1750 case ISD::VP_REDUCE_OR: 1751 case ISD::VP_REDUCE_XOR: 1752 case ISD::VP_REDUCE_SMAX: 1753 case ISD::VP_REDUCE_SMIN: 1754 case ISD::VP_REDUCE_UMAX: 1755 case ISD::VP_REDUCE_UMIN: 1756 Res = PromoteIntOp_VP_REDUCE(N, OpNo); 1757 break; 1758 1759 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; 1760 case ISD::STACKMAP: 1761 Res = PromoteIntOp_STACKMAP(N, OpNo); 1762 break; 1763 case ISD::PATCHPOINT: 1764 Res = PromoteIntOp_PATCHPOINT(N, OpNo); 1765 break; 1766 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 1767 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 1768 Res = PromoteIntOp_VP_STRIDED(N, OpNo); 1769 break; 1770 } 1771 1772 // If the result is null, the sub-method took care of registering results etc. 1773 if (!Res.getNode()) return false; 1774 1775 // If the result is N, the sub-method updated N in place. Tell the legalizer 1776 // core about this. 1777 if (Res.getNode() == N) 1778 return true; 1779 1780 const bool IsStrictFp = N->isStrictFPOpcode(); 1781 assert(Res.getValueType() == N->getValueType(0) && 1782 N->getNumValues() == (IsStrictFp ? 2 : 1) && 1783 "Invalid operand expansion"); 1784 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: "; 1785 Res.dump()); 1786 1787 ReplaceValueWith(SDValue(N, 0), Res); 1788 if (IsStrictFp) 1789 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 1790 1791 return false; 1792 } 1793 1794 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is 1795 /// shared among BR_CC, SELECT_CC, and SETCC handlers. 1796 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS, 1797 ISD::CondCode CCCode) { 1798 // We have to insert explicit sign or zero extends. Note that we could 1799 // insert sign extends for ALL conditions. For those operations where either 1800 // zero or sign extension would be valid, we ask the target which extension 1801 // it would prefer. 1802 1803 // Signed comparisons always require sign extension. 1804 if (ISD::isSignedIntSetCC(CCCode)) { 1805 LHS = SExtPromotedInteger(LHS); 1806 RHS = SExtPromotedInteger(RHS); 1807 return; 1808 } 1809 1810 assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) && 1811 "Unknown integer comparison!"); 1812 1813 SDValue OpL = GetPromotedInteger(LHS); 1814 SDValue OpR = GetPromotedInteger(RHS); 1815 1816 if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) { 1817 // The target would prefer to promote the comparison operand with sign 1818 // extension. Honor that unless the promoted values are already zero 1819 // extended. 1820 unsigned OpLEffectiveBits = 1821 DAG.computeKnownBits(OpL).countMaxActiveBits(); 1822 unsigned OpREffectiveBits = 1823 DAG.computeKnownBits(OpR).countMaxActiveBits(); 1824 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && 1825 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { 1826 LHS = OpL; 1827 RHS = OpR; 1828 return; 1829 } 1830 1831 // The promoted values aren't zero extended, use a sext_inreg. 1832 LHS = SExtPromotedInteger(LHS); 1833 RHS = SExtPromotedInteger(RHS); 1834 return; 1835 } 1836 1837 // Prefer to promote the comparison operand with zero extension. 1838 1839 // If the width of OpL/OpR excluding the duplicated sign bits is no greater 1840 // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation 1841 // that we might not be able to remove. 1842 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL); 1843 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR); 1844 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && 1845 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { 1846 LHS = OpL; 1847 RHS = OpR; 1848 return; 1849 } 1850 1851 // Otherwise, use zext_inreg. 1852 LHS = ZExtPromotedInteger(LHS); 1853 RHS = ZExtPromotedInteger(RHS); 1854 } 1855 1856 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 1857 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1858 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 1859 } 1860 1861 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 1862 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 1863 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 1864 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); 1865 } 1866 1867 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 1868 // This should only occur in unusual situations like bitcasting to an 1869 // x86_fp80, so just turn it into a store+load 1870 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 1871 } 1872 1873 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 1874 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1875 1876 SDValue LHS = N->getOperand(2); 1877 SDValue RHS = N->getOperand(3); 1878 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 1879 1880 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 1881 // legal types. 1882 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1883 N->getOperand(1), LHS, RHS, N->getOperand(4)), 1884 0); 1885 } 1886 1887 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 1888 assert(OpNo == 1 && "only know how to promote condition"); 1889 1890 // Promote all the way up to the canonical SetCC type. 1891 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 1892 1893 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 1894 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 1895 N->getOperand(2)), 0); 1896 } 1897 1898 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 1899 // Since the result type is legal, the operands must promote to it. 1900 EVT OVT = N->getOperand(0).getValueType(); 1901 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 1902 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 1903 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 1904 SDLoc dl(N); 1905 1906 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 1907 DAG.getConstant(OVT.getSizeInBits(), dl, 1908 TLI.getPointerTy(DAG.getDataLayout()))); 1909 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 1910 } 1911 1912 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 1913 // The vector type is legal but the element type is not. This implies 1914 // that the vector is a power-of-two in length and that the element 1915 // type does not have a strange size (eg: it is not i1). 1916 EVT VecVT = N->getValueType(0); 1917 unsigned NumElts = VecVT.getVectorNumElements(); 1918 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 1919 "Legal vector of one illegal element?"); 1920 1921 // Promote the inserted value. The type does not need to match the 1922 // vector element type. Check that any extra bits introduced will be 1923 // truncated away. 1924 assert(N->getOperand(0).getValueSizeInBits() >= 1925 N->getValueType(0).getScalarSizeInBits() && 1926 "Type of inserted value narrower than vector element type!"); 1927 1928 SmallVector<SDValue, 16> NewOps; 1929 for (unsigned i = 0; i < NumElts; ++i) 1930 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 1931 1932 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1933 } 1934 1935 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 1936 unsigned OpNo) { 1937 if (OpNo == 1) { 1938 // Promote the inserted value. This is valid because the type does not 1939 // have to match the vector element type. 1940 1941 // Check that any extra bits introduced will be truncated away. 1942 assert(N->getOperand(1).getValueSizeInBits() >= 1943 N->getValueType(0).getScalarSizeInBits() && 1944 "Type of inserted value narrower than vector element type!"); 1945 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1946 GetPromotedInteger(N->getOperand(1)), 1947 N->getOperand(2)), 1948 0); 1949 } 1950 1951 assert(OpNo == 2 && "Different operand and result vector types?"); 1952 1953 // Promote the index. 1954 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 1955 TLI.getVectorIdxTy(DAG.getDataLayout())); 1956 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1957 N->getOperand(1), Idx), 0); 1958 } 1959 1960 SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) { 1961 // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated, 1962 // so just promote the operand in place. 1963 return SDValue(DAG.UpdateNodeOperands(N, 1964 GetPromotedInteger(N->getOperand(0))), 0); 1965 } 1966 1967 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 1968 assert(OpNo == 0 && "Only know how to promote the condition!"); 1969 SDValue Cond = N->getOperand(0); 1970 EVT OpTy = N->getOperand(1).getValueType(); 1971 1972 if (N->getOpcode() == ISD::VSELECT) 1973 if (SDValue Res = WidenVSELECTMask(N)) 1974 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 1975 Res, N->getOperand(1), N->getOperand(2)); 1976 1977 // Promote all the way up to the canonical SetCC type. 1978 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 1979 Cond = PromoteTargetBoolean(Cond, OpVT); 1980 1981 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 1982 N->getOperand(2)), 0); 1983 } 1984 1985 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 1986 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1987 1988 SDValue LHS = N->getOperand(0); 1989 SDValue RHS = N->getOperand(1); 1990 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1991 1992 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1993 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1994 N->getOperand(3), N->getOperand(4)), 0); 1995 } 1996 1997 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1998 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1999 2000 SDValue LHS = N->getOperand(0); 2001 SDValue RHS = N->getOperand(1); 2002 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 2003 2004 // The CC (#2) is always legal. 2005 if (N->getOpcode() == ISD::SETCC) 2006 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 2007 2008 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode"); 2009 2010 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 2011 N->getOperand(3), N->getOperand(4)), 2012 0); 2013 } 2014 2015 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 2016 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2017 ZExtPromotedInteger(N->getOperand(1))), 0); 2018 } 2019 2020 SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) { 2021 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), 2022 ZExtPromotedInteger(N->getOperand(2))), 0); 2023 } 2024 2025 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 2026 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2027 SDLoc dl(N); 2028 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 2029 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 2030 Op, DAG.getValueType(N->getOperand(0).getValueType())); 2031 } 2032 2033 SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) { 2034 SDLoc dl(N); 2035 EVT VT = N->getValueType(0); 2036 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2037 // FIXME: There is no VP_ANY_EXTEND yet. 2038 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1), 2039 N->getOperand(2)); 2040 unsigned Diff = 2041 VT.getScalarSizeInBits() - N->getOperand(0).getScalarValueSizeInBits(); 2042 SDValue ShAmt = DAG.getShiftAmountConstant(Diff, VT, dl); 2043 // FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts. 2044 SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShAmt, N->getOperand(1), 2045 N->getOperand(2)); 2046 return DAG.getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt, N->getOperand(1), 2047 N->getOperand(2)); 2048 } 2049 2050 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 2051 if (N->getOpcode() == ISD::VP_SINT_TO_FP) 2052 return SDValue(DAG.UpdateNodeOperands(N, 2053 SExtPromotedInteger(N->getOperand(0)), 2054 N->getOperand(1), N->getOperand(2)), 2055 0); 2056 return SDValue(DAG.UpdateNodeOperands(N, 2057 SExtPromotedInteger(N->getOperand(0))), 0); 2058 } 2059 2060 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { 2061 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2062 SExtPromotedInteger(N->getOperand(1))), 0); 2063 } 2064 2065 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 2066 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2067 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 2068 SDLoc dl(N); 2069 2070 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 2071 2072 // Truncate the value and store the result. 2073 return DAG.getTruncStore(Ch, dl, Val, Ptr, 2074 N->getMemoryVT(), N->getMemOperand()); 2075 } 2076 2077 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, 2078 unsigned OpNo) { 2079 SDValue DataOp = N->getValue(); 2080 SDValue Mask = N->getMask(); 2081 2082 if (OpNo == 4) { 2083 // The Mask. Update in place. 2084 EVT DataVT = DataOp.getValueType(); 2085 Mask = PromoteTargetBoolean(Mask, DataVT); 2086 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2087 NewOps[4] = Mask; 2088 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2089 } 2090 2091 assert(OpNo == 1 && "Unexpected operand for promotion"); 2092 DataOp = GetPromotedInteger(DataOp); 2093 2094 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(), 2095 N->getOffset(), Mask, N->getMemoryVT(), 2096 N->getMemOperand(), N->getAddressingMode(), 2097 /*IsTruncating*/ true, N->isCompressingStore()); 2098 } 2099 2100 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, 2101 unsigned OpNo) { 2102 assert(OpNo == 3 && "Only know how to promote the mask!"); 2103 EVT DataVT = N->getValueType(0); 2104 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2105 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2106 NewOps[OpNo] = Mask; 2107 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 2108 if (Res == N) 2109 return SDValue(Res, 0); 2110 2111 // Update triggered CSE, do our own replacement since caller can't. 2112 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 2113 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 2114 return SDValue(); 2115 } 2116 2117 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, 2118 unsigned OpNo) { 2119 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 2120 2121 if (OpNo == 2) { 2122 // The Mask 2123 EVT DataVT = N->getValueType(0); 2124 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2125 } else if (OpNo == 4) { 2126 // The Index 2127 if (N->isIndexSigned()) 2128 // Need to sign extend the index since the bits will likely be used. 2129 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2130 else 2131 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 2132 } else 2133 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 2134 2135 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 2136 if (Res == N) 2137 return SDValue(Res, 0); 2138 2139 // Update triggered CSE, do our own replacement since caller can't. 2140 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 2141 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 2142 return SDValue(); 2143 } 2144 2145 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, 2146 unsigned OpNo) { 2147 bool TruncateStore = N->isTruncatingStore(); 2148 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 2149 2150 if (OpNo == 2) { 2151 // The Mask 2152 EVT DataVT = N->getValue().getValueType(); 2153 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2154 } else if (OpNo == 4) { 2155 // The Index 2156 if (N->isIndexSigned()) 2157 // Need to sign extend the index since the bits will likely be used. 2158 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2159 else 2160 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 2161 } else { 2162 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 2163 TruncateStore = true; 2164 } 2165 2166 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), 2167 SDLoc(N), NewOps, N->getMemOperand(), 2168 N->getIndexType(), TruncateStore); 2169 } 2170 2171 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 2172 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2173 if (N->getOpcode() == ISD::VP_TRUNCATE) 2174 return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op, 2175 N->getOperand(1), N->getOperand(2)); 2176 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 2177 } 2178 2179 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 2180 if (N->getOpcode() == ISD::VP_UINT_TO_FP) 2181 return SDValue(DAG.UpdateNodeOperands(N, 2182 ZExtPromotedInteger(N->getOperand(0)), 2183 N->getOperand(1), N->getOperand(2)), 2184 0); 2185 return SDValue(DAG.UpdateNodeOperands(N, 2186 ZExtPromotedInteger(N->getOperand(0))), 0); 2187 } 2188 2189 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { 2190 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2191 ZExtPromotedInteger(N->getOperand(1))), 0); 2192 } 2193 2194 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 2195 SDLoc dl(N); 2196 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2197 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 2198 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); 2199 } 2200 2201 SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) { 2202 SDLoc dl(N); 2203 EVT VT = N->getValueType(0); 2204 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2205 // FIXME: There is no VP_ANY_EXTEND yet. 2206 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1), 2207 N->getOperand(2)); 2208 APInt Imm = APInt::getLowBitsSet(VT.getScalarSizeInBits(), 2209 N->getOperand(0).getScalarValueSizeInBits()); 2210 return DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(Imm, dl, VT), 2211 N->getOperand(1), N->getOperand(2)); 2212 } 2213 2214 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBO_CARRY(SDNode *N, unsigned OpNo) { 2215 assert(OpNo == 2 && "Don't know how to promote this operand!"); 2216 2217 SDValue LHS = N->getOperand(0); 2218 SDValue RHS = N->getOperand(1); 2219 SDValue Carry = N->getOperand(2); 2220 SDLoc DL(N); 2221 2222 Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); 2223 2224 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); 2225 } 2226 2227 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { 2228 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 2229 return SDValue( 2230 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); 2231 } 2232 2233 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { 2234 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. 2235 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 2236 return SDValue(DAG.UpdateNodeOperands(N, Op), 0); 2237 } 2238 2239 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) { 2240 assert(OpNo > 1 && "Don't know how to promote this operand!"); 2241 // Promote the rw, locality, and cache type arguments to a supported integer 2242 // width. 2243 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 2244 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3)); 2245 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4)); 2246 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), 2247 Op2, Op3, Op4), 2248 0); 2249 } 2250 2251 SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) { 2252 bool IsStrict = N->isStrictFPOpcode(); 2253 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2254 2255 bool IsPowI = 2256 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; 2257 2258 // The integer operand is the last operand in FPOWI (or FLDEXP) (so the result 2259 // and floating point operand is already type legalized). 2260 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0)) 2261 : RTLIB::getLDEXP(N->getValueType(0)); 2262 2263 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 2264 SDValue Op = SExtPromotedInteger(N->getOperand(1)); 2265 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); 2266 } 2267 2268 // We can't just promote the exponent type in FPOWI, since we want to lower 2269 // the node to a libcall and we if we promote to a type larger than 2270 // sizeof(int) the libcall might not be according to the targets ABI. Instead 2271 // we rewrite to a libcall here directly, letting makeLibCall handle promotion 2272 // if the target accepts it according to shouldSignExtendTypeInLibCall. 2273 2274 unsigned OpOffset = IsStrict ? 1 : 0; 2275 // The exponent should fit in a sizeof(int) type for the libcall to be valid. 2276 assert(DAG.getLibInfo().getIntSize() == 2277 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && 2278 "POWI exponent should match with sizeof(int) when doing the libcall."); 2279 TargetLowering::MakeLibCallOptions CallOptions; 2280 CallOptions.setSExt(true); 2281 SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)}; 2282 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall( 2283 DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain); 2284 ReplaceValueWith(SDValue(N, 0), Tmp.first); 2285 if (IsStrict) 2286 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2287 return SDValue(); 2288 } 2289 2290 static unsigned getExtendForIntVecReduction(SDNode *N) { 2291 switch (N->getOpcode()) { 2292 default: 2293 llvm_unreachable("Expected integer vector reduction"); 2294 case ISD::VECREDUCE_ADD: 2295 case ISD::VECREDUCE_MUL: 2296 case ISD::VECREDUCE_AND: 2297 case ISD::VECREDUCE_OR: 2298 case ISD::VECREDUCE_XOR: 2299 case ISD::VP_REDUCE_ADD: 2300 case ISD::VP_REDUCE_MUL: 2301 case ISD::VP_REDUCE_AND: 2302 case ISD::VP_REDUCE_OR: 2303 case ISD::VP_REDUCE_XOR: 2304 return ISD::ANY_EXTEND; 2305 case ISD::VECREDUCE_SMAX: 2306 case ISD::VECREDUCE_SMIN: 2307 case ISD::VP_REDUCE_SMAX: 2308 case ISD::VP_REDUCE_SMIN: 2309 return ISD::SIGN_EXTEND; 2310 case ISD::VECREDUCE_UMAX: 2311 case ISD::VECREDUCE_UMIN: 2312 case ISD::VP_REDUCE_UMAX: 2313 case ISD::VP_REDUCE_UMIN: 2314 return ISD::ZERO_EXTEND; 2315 } 2316 } 2317 2318 SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) { 2319 switch (getExtendForIntVecReduction(N)) { 2320 default: 2321 llvm_unreachable("Impossible extension kind for integer reduction"); 2322 case ISD::ANY_EXTEND: 2323 return GetPromotedInteger(V); 2324 case ISD::SIGN_EXTEND: 2325 return SExtPromotedInteger(V); 2326 case ISD::ZERO_EXTEND: 2327 return ZExtPromotedInteger(V); 2328 } 2329 } 2330 2331 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { 2332 SDLoc dl(N); 2333 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0)); 2334 2335 EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType(); 2336 EVT InVT = Op.getValueType(); 2337 EVT EltVT = InVT.getVectorElementType(); 2338 EVT ResVT = N->getValueType(0); 2339 unsigned Opcode = N->getOpcode(); 2340 2341 // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if 2342 // vecreduce_xor is not legal 2343 if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 && 2344 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_XOR, InVT) && 2345 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_ADD, InVT)) 2346 Opcode = ISD::VECREDUCE_ADD; 2347 2348 // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if 2349 // vecreduce_or is not legal 2350 else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 && 2351 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_OR, InVT) && 2352 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMAX, InVT)) { 2353 Opcode = ISD::VECREDUCE_UMAX; 2354 // Can't use promoteTargetBoolean here because we still need 2355 // to either sign_ext or zero_ext in the undefined case. 2356 switch (TLI.getBooleanContents(InVT)) { 2357 case TargetLoweringBase::UndefinedBooleanContent: 2358 case TargetLoweringBase::ZeroOrOneBooleanContent: 2359 Op = ZExtPromotedInteger(N->getOperand(0)); 2360 break; 2361 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 2362 Op = SExtPromotedInteger(N->getOperand(0)); 2363 break; 2364 } 2365 } 2366 2367 // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if 2368 // vecreduce_and is not legal 2369 else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 && 2370 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_AND, InVT) && 2371 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMIN, InVT)) { 2372 Opcode = ISD::VECREDUCE_UMIN; 2373 // Can't use promoteTargetBoolean here because we still need 2374 // to either sign_ext or zero_ext in the undefined case. 2375 switch (TLI.getBooleanContents(InVT)) { 2376 case TargetLoweringBase::UndefinedBooleanContent: 2377 case TargetLoweringBase::ZeroOrOneBooleanContent: 2378 Op = ZExtPromotedInteger(N->getOperand(0)); 2379 break; 2380 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 2381 Op = SExtPromotedInteger(N->getOperand(0)); 2382 break; 2383 } 2384 } 2385 2386 if (ResVT.bitsGE(EltVT)) 2387 return DAG.getNode(Opcode, SDLoc(N), ResVT, Op); 2388 2389 // Result size must be >= element size. If this is not the case after 2390 // promotion, also promote the result type and then truncate. 2391 SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op); 2392 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce); 2393 } 2394 2395 SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) { 2396 SDLoc DL(N); 2397 SDValue Op = N->getOperand(OpNo); 2398 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2399 2400 if (OpNo == 2) { // Mask 2401 // Update in place. 2402 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType()); 2403 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2404 } 2405 2406 assert(OpNo == 1 && "Unexpected operand for promotion"); 2407 2408 Op = PromoteIntOpVectorReduction(N, Op); 2409 2410 NewOps[OpNo] = Op; 2411 2412 EVT VT = N->getValueType(0); 2413 EVT EltVT = Op.getValueType().getScalarType(); 2414 2415 if (VT.bitsGE(EltVT)) 2416 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps); 2417 2418 // Result size must be >= element/start-value size. If this is not the case 2419 // after promotion, also promote both the start value and result type and 2420 // then truncate. 2421 NewOps[0] = 2422 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0)); 2423 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps); 2424 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce); 2425 } 2426 2427 SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) { 2428 SDValue Op = ZExtPromotedInteger(N->getOperand(1)); 2429 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); 2430 } 2431 2432 SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) { 2433 assert(OpNo > 1); // Because the first two arguments are guaranteed legal. 2434 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 2435 SDValue Operand = N->getOperand(OpNo); 2436 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); 2437 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); 2438 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2439 } 2440 2441 SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { 2442 assert(OpNo >= 7); 2443 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 2444 SDValue Operand = N->getOperand(OpNo); 2445 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); 2446 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); 2447 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2448 } 2449 2450 SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { 2451 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || 2452 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); 2453 2454 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); 2455 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2456 2457 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2458 } 2459 2460 //===----------------------------------------------------------------------===// 2461 // Integer Result Expansion 2462 //===----------------------------------------------------------------------===// 2463 2464 /// ExpandIntegerResult - This method is called when the specified result of the 2465 /// specified node is found to need expansion. At this point, the node may also 2466 /// have invalid operands or may have other results that need promotion, we just 2467 /// know that (at least) one result needs expansion. 2468 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 2469 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); 2470 dbgs() << "\n"); 2471 SDValue Lo, Hi; 2472 Lo = Hi = SDValue(); 2473 2474 // See if the target wants to custom expand this node. 2475 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 2476 return; 2477 2478 switch (N->getOpcode()) { 2479 default: 2480 #ifndef NDEBUG 2481 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 2482 N->dump(&DAG); dbgs() << "\n"; 2483 #endif 2484 report_fatal_error("Do not know how to expand the result of this " 2485 "operator!"); 2486 2487 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break; 2488 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 2489 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; 2490 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 2491 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 2492 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; 2493 2494 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 2495 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 2496 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 2497 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 2498 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 2499 2500 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 2501 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 2502 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 2503 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; 2504 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 2505 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; 2506 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 2507 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; 2508 case ISD::CTLZ_ZERO_UNDEF: 2509 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 2510 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 2511 case ISD::CTTZ_ZERO_UNDEF: 2512 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 2513 case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break; 2514 case ISD::STRICT_FP_TO_SINT: 2515 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 2516 case ISD::STRICT_FP_TO_UINT: 2517 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 2518 case ISD::FP_TO_SINT_SAT: 2519 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; 2520 case ISD::STRICT_LROUND: 2521 case ISD::STRICT_LRINT: 2522 case ISD::LROUND: 2523 case ISD::LRINT: 2524 case ISD::STRICT_LLROUND: 2525 case ISD::STRICT_LLRINT: 2526 case ISD::LLROUND: 2527 case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break; 2528 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 2529 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 2530 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; 2531 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 2532 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 2533 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 2534 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 2535 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 2536 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 2537 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 2538 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 2539 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 2540 2541 case ISD::ATOMIC_LOAD_ADD: 2542 case ISD::ATOMIC_LOAD_SUB: 2543 case ISD::ATOMIC_LOAD_AND: 2544 case ISD::ATOMIC_LOAD_CLR: 2545 case ISD::ATOMIC_LOAD_OR: 2546 case ISD::ATOMIC_LOAD_XOR: 2547 case ISD::ATOMIC_LOAD_NAND: 2548 case ISD::ATOMIC_LOAD_MIN: 2549 case ISD::ATOMIC_LOAD_MAX: 2550 case ISD::ATOMIC_LOAD_UMIN: 2551 case ISD::ATOMIC_LOAD_UMAX: 2552 case ISD::ATOMIC_SWAP: 2553 case ISD::ATOMIC_CMP_SWAP: { 2554 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 2555 SplitInteger(Tmp.first, Lo, Hi); 2556 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2557 break; 2558 } 2559 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 2560 AtomicSDNode *AN = cast<AtomicSDNode>(N); 2561 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 2562 SDValue Tmp = DAG.getAtomicCmpSwap( 2563 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 2564 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 2565 AN->getMemOperand()); 2566 2567 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 2568 // success simply by comparing the loaded value against the ingoing 2569 // comparison. 2570 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 2571 N->getOperand(2), ISD::SETEQ); 2572 2573 SplitInteger(Tmp, Lo, Hi); 2574 ReplaceValueWith(SDValue(N, 1), Success); 2575 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 2576 break; 2577 } 2578 2579 case ISD::AND: 2580 case ISD::OR: 2581 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 2582 2583 case ISD::UMAX: 2584 case ISD::SMAX: 2585 case ISD::UMIN: 2586 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; 2587 2588 case ISD::ADD: 2589 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 2590 2591 case ISD::ADDC: 2592 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 2593 2594 case ISD::ADDE: 2595 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 2596 2597 case ISD::UADDO_CARRY: 2598 case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break; 2599 2600 case ISD::SADDO_CARRY: 2601 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; 2602 2603 case ISD::SHL: 2604 case ISD::SRA: 2605 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 2606 2607 case ISD::SADDO: 2608 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 2609 case ISD::UADDO: 2610 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 2611 case ISD::UMULO: 2612 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 2613 2614 case ISD::SADDSAT: 2615 case ISD::UADDSAT: 2616 case ISD::SSUBSAT: 2617 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; 2618 2619 case ISD::SSHLSAT: 2620 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; 2621 2622 case ISD::SMULFIX: 2623 case ISD::SMULFIXSAT: 2624 case ISD::UMULFIX: 2625 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; 2626 2627 case ISD::SDIVFIX: 2628 case ISD::SDIVFIXSAT: 2629 case ISD::UDIVFIX: 2630 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; 2631 2632 case ISD::VECREDUCE_ADD: 2633 case ISD::VECREDUCE_MUL: 2634 case ISD::VECREDUCE_AND: 2635 case ISD::VECREDUCE_OR: 2636 case ISD::VECREDUCE_XOR: 2637 case ISD::VECREDUCE_SMAX: 2638 case ISD::VECREDUCE_SMIN: 2639 case ISD::VECREDUCE_UMAX: 2640 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; 2641 2642 case ISD::ROTL: 2643 case ISD::ROTR: 2644 ExpandIntRes_Rotate(N, Lo, Hi); 2645 break; 2646 2647 case ISD::FSHL: 2648 case ISD::FSHR: 2649 ExpandIntRes_FunnelShift(N, Lo, Hi); 2650 break; 2651 2652 case ISD::VSCALE: 2653 ExpandIntRes_VSCALE(N, Lo, Hi); 2654 break; 2655 } 2656 2657 // If Lo/Hi is null, the sub-method took care of registering results etc. 2658 if (Lo.getNode()) 2659 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 2660 } 2661 2662 /// Lower an atomic node to the appropriate builtin call. 2663 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 2664 unsigned Opc = Node->getOpcode(); 2665 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2666 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering(); 2667 // Lower to outline atomic libcall if outline atomics enabled, 2668 // or to sync libcall otherwise 2669 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); 2670 EVT RetVT = Node->getValueType(0); 2671 TargetLowering::MakeLibCallOptions CallOptions; 2672 SmallVector<SDValue, 4> Ops; 2673 if (TLI.getLibcallName(LC)) { 2674 Ops.append(Node->op_begin() + 2, Node->op_end()); 2675 Ops.push_back(Node->getOperand(1)); 2676 } else { 2677 LC = RTLIB::getSYNC(Opc, VT); 2678 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2679 "Unexpected atomic op or value type!"); 2680 Ops.append(Node->op_begin() + 1, Node->op_end()); 2681 } 2682 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), 2683 Node->getOperand(0)); 2684 } 2685 2686 /// N is a shift by a value that needs to be expanded, 2687 /// and the shift amount is a constant 'Amt'. Expand the operation. 2688 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, 2689 SDValue &Lo, SDValue &Hi) { 2690 SDLoc DL(N); 2691 // Expand the incoming operand to be shifted, so that we have its parts 2692 SDValue InL, InH; 2693 GetExpandedInteger(N->getOperand(0), InL, InH); 2694 2695 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization 2696 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. 2697 if (!Amt) { 2698 Lo = InL; 2699 Hi = InH; 2700 return; 2701 } 2702 2703 EVT NVT = InL.getValueType(); 2704 unsigned VTBits = N->getValueType(0).getSizeInBits(); 2705 unsigned NVTBits = NVT.getSizeInBits(); 2706 EVT ShTy = N->getOperand(1).getValueType(); 2707 2708 if (N->getOpcode() == ISD::SHL) { 2709 if (Amt.uge(VTBits)) { 2710 Lo = Hi = DAG.getConstant(0, DL, NVT); 2711 } else if (Amt.ugt(NVTBits)) { 2712 Lo = DAG.getConstant(0, DL, NVT); 2713 Hi = DAG.getNode(ISD::SHL, DL, 2714 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2715 } else if (Amt == NVTBits) { 2716 Lo = DAG.getConstant(0, DL, NVT); 2717 Hi = InL; 2718 } else { 2719 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); 2720 Hi = DAG.getNode(ISD::OR, DL, NVT, 2721 DAG.getNode(ISD::SHL, DL, NVT, InH, 2722 DAG.getConstant(Amt, DL, ShTy)), 2723 DAG.getNode(ISD::SRL, DL, NVT, InL, 2724 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2725 } 2726 return; 2727 } 2728 2729 if (N->getOpcode() == ISD::SRL) { 2730 if (Amt.uge(VTBits)) { 2731 Lo = Hi = DAG.getConstant(0, DL, NVT); 2732 } else if (Amt.ugt(NVTBits)) { 2733 Lo = DAG.getNode(ISD::SRL, DL, 2734 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2735 Hi = DAG.getConstant(0, DL, NVT); 2736 } else if (Amt == NVTBits) { 2737 Lo = InH; 2738 Hi = DAG.getConstant(0, DL, NVT); 2739 } else { 2740 Lo = DAG.getNode(ISD::OR, DL, NVT, 2741 DAG.getNode(ISD::SRL, DL, NVT, InL, 2742 DAG.getConstant(Amt, DL, ShTy)), 2743 DAG.getNode(ISD::SHL, DL, NVT, InH, 2744 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2745 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2746 } 2747 return; 2748 } 2749 2750 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2751 if (Amt.uge(VTBits)) { 2752 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2753 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2754 } else if (Amt.ugt(NVTBits)) { 2755 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2756 DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2757 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2758 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2759 } else if (Amt == NVTBits) { 2760 Lo = InH; 2761 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2762 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2763 } else { 2764 Lo = DAG.getNode(ISD::OR, DL, NVT, 2765 DAG.getNode(ISD::SRL, DL, NVT, InL, 2766 DAG.getConstant(Amt, DL, ShTy)), 2767 DAG.getNode(ISD::SHL, DL, NVT, InH, 2768 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2769 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2770 } 2771 } 2772 2773 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 2774 /// this shift based on knowledge of the high bit of the shift amount. If we 2775 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual 2776 /// shift amount. 2777 bool DAGTypeLegalizer:: 2778 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2779 SDValue Amt = N->getOperand(1); 2780 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2781 EVT ShTy = Amt.getValueType(); 2782 unsigned ShBits = ShTy.getScalarSizeInBits(); 2783 unsigned NVTBits = NVT.getScalarSizeInBits(); 2784 assert(isPowerOf2_32(NVTBits) && 2785 "Expanded integer type size not a power of two!"); 2786 SDLoc dl(N); 2787 2788 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 2789 KnownBits Known = DAG.computeKnownBits(N->getOperand(1)); 2790 2791 // If we don't know anything about the high bits, exit. 2792 if (((Known.Zero|Known.One) & HighBitMask) == 0) 2793 return false; 2794 2795 // Get the incoming operand to be shifted. 2796 SDValue InL, InH; 2797 GetExpandedInteger(N->getOperand(0), InL, InH); 2798 2799 // If we know that any of the high bits of the shift amount are one, then we 2800 // can do this as a couple of simple shifts. 2801 if (Known.One.intersects(HighBitMask)) { 2802 // Mask out the high bit, which we know is set. 2803 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 2804 DAG.getConstant(~HighBitMask, dl, ShTy)); 2805 2806 switch (N->getOpcode()) { 2807 default: llvm_unreachable("Unknown shift"); 2808 case ISD::SHL: 2809 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. 2810 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 2811 return true; 2812 case ISD::SRL: 2813 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2814 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 2815 return true; 2816 case ISD::SRA: 2817 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 2818 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2819 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 2820 return true; 2821 } 2822 } 2823 2824 // If we know that all of the high bits of the shift amount are zero, then we 2825 // can do this as a couple of simple shifts. 2826 if (HighBitMask.isSubsetOf(Known.Zero)) { 2827 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 2828 // shift if x is zero. We can use XOR here because x is known to be smaller 2829 // than 32. 2830 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 2831 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2832 2833 unsigned Op1, Op2; 2834 switch (N->getOpcode()) { 2835 default: llvm_unreachable("Unknown shift"); 2836 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 2837 case ISD::SRL: 2838 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 2839 } 2840 2841 // When shifting right the arithmetic for Lo and Hi is swapped. 2842 if (N->getOpcode() != ISD::SHL) 2843 std::swap(InL, InH); 2844 2845 // Use a little trick to get the bits that move from Lo to Hi. First 2846 // shift by one bit. 2847 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); 2848 // Then compute the remaining shift with amount-1. 2849 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 2850 2851 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 2852 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 2853 2854 if (N->getOpcode() != ISD::SHL) 2855 std::swap(Hi, Lo); 2856 return true; 2857 } 2858 2859 return false; 2860 } 2861 2862 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 2863 /// of any size. 2864 bool DAGTypeLegalizer:: 2865 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2866 SDValue Amt = N->getOperand(1); 2867 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2868 EVT ShTy = Amt.getValueType(); 2869 unsigned NVTBits = NVT.getSizeInBits(); 2870 assert(isPowerOf2_32(NVTBits) && 2871 "Expanded integer type size not a power of two!"); 2872 SDLoc dl(N); 2873 2874 // Get the incoming operand to be shifted. 2875 SDValue InL, InH; 2876 GetExpandedInteger(N->getOperand(0), InL, InH); 2877 2878 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); 2879 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 2880 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 2881 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2882 Amt, NVBitsNode, ISD::SETULT); 2883 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2884 Amt, DAG.getConstant(0, dl, ShTy), 2885 ISD::SETEQ); 2886 2887 SDValue LoS, HiS, LoL, HiL; 2888 switch (N->getOpcode()) { 2889 default: llvm_unreachable("Unknown shift"); 2890 case ISD::SHL: 2891 // Short: ShAmt < NVTBits 2892 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 2893 HiS = DAG.getNode(ISD::OR, dl, NVT, 2894 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 2895 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 2896 2897 // Long: ShAmt >= NVTBits 2898 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. 2899 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 2900 2901 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 2902 Hi = DAG.getSelect(dl, NVT, isZero, InH, 2903 DAG.getSelect(dl, NVT, isShort, HiS, HiL)); 2904 return true; 2905 case ISD::SRL: 2906 // Short: ShAmt < NVTBits 2907 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 2908 LoS = DAG.getNode(ISD::OR, dl, NVT, 2909 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2910 // FIXME: If Amt is zero, the following shift generates an undefined result 2911 // on some architectures. 2912 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2913 2914 // Long: ShAmt >= NVTBits 2915 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2916 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2917 2918 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2919 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2920 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2921 return true; 2922 case ISD::SRA: 2923 // Short: ShAmt < NVTBits 2924 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 2925 LoS = DAG.getNode(ISD::OR, dl, NVT, 2926 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2927 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2928 2929 // Long: ShAmt >= NVTBits 2930 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 2931 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2932 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2933 2934 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2935 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2936 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2937 return true; 2938 } 2939 } 2940 2941 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { 2942 2943 switch (Op) { 2944 default: llvm_unreachable("invalid min/max opcode"); 2945 case ISD::SMAX: 2946 return std::make_pair(ISD::SETGT, ISD::UMAX); 2947 case ISD::UMAX: 2948 return std::make_pair(ISD::SETUGT, ISD::UMAX); 2949 case ISD::SMIN: 2950 return std::make_pair(ISD::SETLT, ISD::UMIN); 2951 case ISD::UMIN: 2952 return std::make_pair(ISD::SETULT, ISD::UMIN); 2953 } 2954 } 2955 2956 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, 2957 SDValue &Lo, SDValue &Hi) { 2958 SDLoc DL(N); 2959 2960 SDValue LHS = N->getOperand(0); 2961 SDValue RHS = N->getOperand(1); 2962 2963 // If the upper halves are all sign bits, then we can perform the MINMAX on 2964 // the lower half and sign-extend the result to the upper half. 2965 unsigned NumBits = N->getValueType(0).getScalarSizeInBits(); 2966 unsigned NumHalfBits = NumBits / 2; 2967 if (DAG.ComputeNumSignBits(LHS) > NumHalfBits && 2968 DAG.ComputeNumSignBits(RHS) > NumHalfBits) { 2969 SDValue LHSL, LHSH, RHSL, RHSH; 2970 GetExpandedInteger(LHS, LHSL, LHSH); 2971 GetExpandedInteger(RHS, RHSL, RHSH); 2972 EVT NVT = LHSL.getValueType(); 2973 2974 Lo = DAG.getNode(N->getOpcode(), DL, NVT, LHSL, RHSL); 2975 Hi = DAG.getNode(ISD::SRA, DL, NVT, Lo, 2976 DAG.getShiftAmountConstant(NumHalfBits - 1, NVT, DL)); 2977 return; 2978 } 2979 2980 // The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1. 2981 // The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL. 2982 if ((N->getOpcode() == ISD::SMAX && isNullConstant(RHS)) || 2983 (N->getOpcode() == ISD::SMIN && isAllOnesConstant(RHS))) { 2984 SDValue LHSL, LHSH, RHSL, RHSH; 2985 GetExpandedInteger(LHS, LHSL, LHSH); 2986 GetExpandedInteger(RHS, RHSL, RHSH); 2987 EVT NVT = LHSL.getValueType(); 2988 EVT CCT = getSetCCResultType(NVT); 2989 2990 SDValue HiNeg = 2991 DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT); 2992 if (N->getOpcode() == ISD::SMIN) { 2993 Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getConstant(-1, DL, NVT)); 2994 } else { 2995 Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL); 2996 } 2997 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 2998 return; 2999 } 3000 3001 const APInt *RHSVal = nullptr; 3002 if (auto *RHSConst = dyn_cast<ConstantSDNode>(RHS)) 3003 RHSVal = &RHSConst->getAPIntValue(); 3004 3005 // The high half of MIN/MAX is always just the the MIN/MAX of the 3006 // high halves of the operands. Expand this way if it appears profitable. 3007 if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) && 3008 (RHSVal->countLeadingOnes() >= NumHalfBits || 3009 RHSVal->countLeadingZeros() >= NumHalfBits)) { 3010 SDValue LHSL, LHSH, RHSL, RHSH; 3011 GetExpandedInteger(LHS, LHSL, LHSH); 3012 GetExpandedInteger(RHS, RHSL, RHSH); 3013 EVT NVT = LHSL.getValueType(); 3014 EVT CCT = getSetCCResultType(NVT); 3015 3016 ISD::NodeType LoOpc; 3017 ISD::CondCode CondC; 3018 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); 3019 3020 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 3021 // We need to know whether to select Lo part that corresponds to 'winning' 3022 // Hi part or if Hi parts are equal. 3023 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); 3024 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); 3025 3026 // Lo part corresponding to the 'winning' Hi part 3027 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); 3028 3029 // Recursed Lo part if Hi parts are equal, this uses unsigned version 3030 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); 3031 3032 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); 3033 return; 3034 } 3035 3036 // Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies 3037 // the compare. 3038 ISD::CondCode Pred; 3039 switch (N->getOpcode()) { 3040 default: llvm_unreachable("How did we get here?"); 3041 case ISD::SMAX: 3042 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) 3043 Pred = ISD::SETGE; 3044 else 3045 Pred = ISD::SETGT; 3046 break; 3047 case ISD::SMIN: 3048 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) 3049 Pred = ISD::SETLE; 3050 else 3051 Pred = ISD::SETLT; 3052 break; 3053 case ISD::UMAX: 3054 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) 3055 Pred = ISD::SETUGE; 3056 else 3057 Pred = ISD::SETUGT; 3058 break; 3059 case ISD::UMIN: 3060 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) 3061 Pred = ISD::SETULE; 3062 else 3063 Pred = ISD::SETULT; 3064 break; 3065 } 3066 EVT VT = N->getValueType(0); 3067 EVT CCT = getSetCCResultType(VT); 3068 SDValue Cond = DAG.getSetCC(DL, CCT, LHS, RHS, Pred); 3069 SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS); 3070 SplitInteger(Result, Lo, Hi); 3071 } 3072 3073 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 3074 SDValue &Lo, SDValue &Hi) { 3075 SDLoc dl(N); 3076 // Expand the subcomponents. 3077 SDValue LHSL, LHSH, RHSL, RHSH; 3078 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3079 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3080 3081 EVT NVT = LHSL.getValueType(); 3082 SDValue LoOps[2] = { LHSL, RHSL }; 3083 SDValue HiOps[3] = { LHSH, RHSH }; 3084 3085 bool HasOpCarry = TLI.isOperationLegalOrCustom( 3086 N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, 3087 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 3088 if (HasOpCarry) { 3089 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 3090 if (N->getOpcode() == ISD::ADD) { 3091 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 3092 HiOps[2] = Lo.getValue(1); 3093 Hi = DAG.computeKnownBits(HiOps[2]).isZero() 3094 ? DAG.getNode(ISD::UADDO, dl, VTList, ArrayRef(HiOps, 2)) 3095 : DAG.getNode(ISD::UADDO_CARRY, dl, VTList, HiOps); 3096 } else { 3097 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 3098 HiOps[2] = Lo.getValue(1); 3099 Hi = DAG.computeKnownBits(HiOps[2]).isZero() 3100 ? DAG.getNode(ISD::USUBO, dl, VTList, ArrayRef(HiOps, 2)) 3101 : DAG.getNode(ISD::USUBO_CARRY, dl, VTList, HiOps); 3102 } 3103 return; 3104 } 3105 3106 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 3107 // them. TODO: Teach operation legalization how to expand unsupported 3108 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 3109 // a carry of type MVT::Glue, but there doesn't seem to be any way to 3110 // generate a value of this type in the expanded code sequence. 3111 bool hasCarry = 3112 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 3113 ISD::ADDC : ISD::SUBC, 3114 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 3115 3116 if (hasCarry) { 3117 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 3118 if (N->getOpcode() == ISD::ADD) { 3119 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 3120 HiOps[2] = Lo.getValue(1); 3121 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 3122 } else { 3123 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 3124 HiOps[2] = Lo.getValue(1); 3125 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 3126 } 3127 return; 3128 } 3129 3130 bool hasOVF = 3131 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 3132 ISD::UADDO : ISD::USUBO, 3133 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 3134 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); 3135 3136 if (hasOVF) { 3137 EVT OvfVT = getSetCCResultType(NVT); 3138 SDVTList VTList = DAG.getVTList(NVT, OvfVT); 3139 int RevOpc; 3140 if (N->getOpcode() == ISD::ADD) { 3141 RevOpc = ISD::SUB; 3142 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 3143 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2)); 3144 } else { 3145 RevOpc = ISD::ADD; 3146 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 3147 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2)); 3148 } 3149 SDValue OVF = Lo.getValue(1); 3150 3151 switch (BoolType) { 3152 case TargetLoweringBase::UndefinedBooleanContent: 3153 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); 3154 [[fallthrough]]; 3155 case TargetLoweringBase::ZeroOrOneBooleanContent: 3156 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); 3157 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); 3158 break; 3159 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 3160 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); 3161 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); 3162 } 3163 return; 3164 } 3165 3166 if (N->getOpcode() == ISD::ADD) { 3167 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 3168 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2)); 3169 SDValue Cmp; 3170 // Special case: X+1 has a carry out if X+1==0. This may reduce the live 3171 // range of X. We assume comparing with 0 is cheap. 3172 if (isOneConstant(LoOps[1])) 3173 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 3174 DAG.getConstant(0, dl, NVT), ISD::SETEQ); 3175 else if (isAllOnesConstant(LoOps[1])) { 3176 if (isAllOnesConstant(HiOps[1])) 3177 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0], 3178 DAG.getConstant(0, dl, NVT), ISD::SETEQ); 3179 else 3180 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0], 3181 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3182 } else 3183 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 3184 ISD::SETULT); 3185 3186 SDValue Carry; 3187 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 3188 Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT); 3189 else 3190 Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 3191 DAG.getConstant(0, dl, NVT)); 3192 3193 if (isAllOnesConstant(LoOps[1]) && isAllOnesConstant(HiOps[1])) 3194 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps[0], Carry); 3195 else 3196 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); 3197 } else { 3198 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 3199 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2)); 3200 SDValue Cmp = 3201 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 3202 LoOps[0], LoOps[1], ISD::SETULT); 3203 3204 SDValue Borrow; 3205 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 3206 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); 3207 else 3208 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 3209 DAG.getConstant(0, dl, NVT)); 3210 3211 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 3212 } 3213 } 3214 3215 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 3216 SDValue &Lo, SDValue &Hi) { 3217 // Expand the subcomponents. 3218 SDValue LHSL, LHSH, RHSL, RHSH; 3219 SDLoc dl(N); 3220 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3221 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3222 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 3223 SDValue LoOps[2] = { LHSL, RHSL }; 3224 SDValue HiOps[3] = { LHSH, RHSH }; 3225 3226 if (N->getOpcode() == ISD::ADDC) { 3227 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 3228 HiOps[2] = Lo.getValue(1); 3229 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 3230 } else { 3231 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 3232 HiOps[2] = Lo.getValue(1); 3233 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 3234 } 3235 3236 // Legalized the flag result - switch anything that used the old flag to 3237 // use the new one. 3238 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3239 } 3240 3241 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 3242 SDValue &Lo, SDValue &Hi) { 3243 // Expand the subcomponents. 3244 SDValue LHSL, LHSH, RHSL, RHSH; 3245 SDLoc dl(N); 3246 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3247 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3248 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 3249 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 3250 SDValue HiOps[3] = { LHSH, RHSH }; 3251 3252 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3253 HiOps[2] = Lo.getValue(1); 3254 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 3255 3256 // Legalized the flag result - switch anything that used the old flag to 3257 // use the new one. 3258 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3259 } 3260 3261 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 3262 SDValue &Lo, SDValue &Hi) { 3263 SDValue LHS = N->getOperand(0); 3264 SDValue RHS = N->getOperand(1); 3265 SDLoc dl(N); 3266 3267 SDValue Ovf; 3268 3269 unsigned CarryOp, NoCarryOp; 3270 ISD::CondCode Cond; 3271 switch(N->getOpcode()) { 3272 case ISD::UADDO: 3273 CarryOp = ISD::UADDO_CARRY; 3274 NoCarryOp = ISD::ADD; 3275 Cond = ISD::SETULT; 3276 break; 3277 case ISD::USUBO: 3278 CarryOp = ISD::USUBO_CARRY; 3279 NoCarryOp = ISD::SUB; 3280 Cond = ISD::SETUGT; 3281 break; 3282 default: 3283 llvm_unreachable("Node has unexpected Opcode"); 3284 } 3285 3286 bool HasCarryOp = TLI.isOperationLegalOrCustom( 3287 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 3288 3289 if (HasCarryOp) { 3290 // Expand the subcomponents. 3291 SDValue LHSL, LHSH, RHSL, RHSH; 3292 GetExpandedInteger(LHS, LHSL, LHSH); 3293 GetExpandedInteger(RHS, RHSL, RHSH); 3294 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3295 SDValue LoOps[2] = { LHSL, RHSL }; 3296 SDValue HiOps[3] = { LHSH, RHSH }; 3297 3298 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3299 HiOps[2] = Lo.getValue(1); 3300 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps); 3301 3302 Ovf = Hi.getValue(1); 3303 } else { 3304 // Expand the result by simply replacing it with the equivalent 3305 // non-overflow-checking operation. 3306 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS); 3307 SplitInteger(Sum, Lo, Hi); 3308 3309 if (N->getOpcode() == ISD::UADDO && isOneConstant(RHS)) { 3310 // Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this 3311 // with (Lo | Hi) == 0. 3312 SDValue Or = DAG.getNode(ISD::OR, dl, Lo.getValueType(), Lo, Hi); 3313 Ovf = DAG.getSetCC(dl, N->getValueType(1), Or, 3314 DAG.getConstant(0, dl, Lo.getValueType()), ISD::SETEQ); 3315 } else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(RHS)) { 3316 // Special case: uaddo X, -1 overflows if X == 0. 3317 Ovf = 3318 DAG.getSetCC(dl, N->getValueType(1), LHS, 3319 DAG.getConstant(0, dl, LHS.getValueType()), ISD::SETNE); 3320 } else { 3321 // Calculate the overflow: addition overflows iff a + b < a, and 3322 // subtraction overflows iff a - b > a. 3323 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); 3324 } 3325 } 3326 3327 // Legalized the flag result - switch anything that used the old flag to 3328 // use the new one. 3329 ReplaceValueWith(SDValue(N, 1), Ovf); 3330 } 3331 3332 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo, 3333 SDValue &Hi) { 3334 // Expand the subcomponents. 3335 SDValue LHSL, LHSH, RHSL, RHSH; 3336 SDLoc dl(N); 3337 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3338 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3339 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3340 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 3341 SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; 3342 3343 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3344 HiOps[2] = Lo.getValue(1); 3345 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 3346 3347 // Legalized the flag result - switch anything that used the old flag to 3348 // use the new one. 3349 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3350 } 3351 3352 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, 3353 SDValue &Lo, SDValue &Hi) { 3354 // Expand the subcomponents. 3355 SDValue LHSL, LHSH, RHSL, RHSH; 3356 SDLoc dl(N); 3357 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3358 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3359 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3360 3361 // We need to use an unsigned carry op for the lo part. 3362 unsigned CarryOp = 3363 N->getOpcode() == ISD::SADDO_CARRY ? ISD::UADDO_CARRY : ISD::USUBO_CARRY; 3364 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) }); 3365 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 3366 3367 // Legalized the flag result - switch anything that used the old flag to 3368 // use the new one. 3369 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3370 } 3371 3372 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 3373 SDValue &Lo, SDValue &Hi) { 3374 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3375 SDLoc dl(N); 3376 SDValue Op = N->getOperand(0); 3377 if (Op.getValueType().bitsLE(NVT)) { 3378 // The low part is any extension of the input (which degenerates to a copy). 3379 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 3380 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 3381 } else { 3382 // For example, extension of an i48 to an i64. The operand type necessarily 3383 // promotes to the result type, so will end up being expanded too. 3384 assert(getTypeAction(Op.getValueType()) == 3385 TargetLowering::TypePromoteInteger && 3386 "Only know how to promote this result!"); 3387 SDValue Res = GetPromotedInteger(Op); 3388 assert(Res.getValueType() == N->getValueType(0) && 3389 "Operand over promoted?"); 3390 // Split the promoted operand. This will simplify when it is expanded. 3391 SplitInteger(Res, Lo, Hi); 3392 } 3393 } 3394 3395 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 3396 SDValue &Lo, SDValue &Hi) { 3397 SDLoc dl(N); 3398 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3399 EVT NVT = Lo.getValueType(); 3400 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3401 unsigned NVTBits = NVT.getSizeInBits(); 3402 unsigned EVTBits = EVT.getSizeInBits(); 3403 3404 if (NVTBits < EVTBits) { 3405 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 3406 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3407 EVTBits - NVTBits))); 3408 } else { 3409 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 3410 // The high part replicates the sign bit of Lo, make it explicit. 3411 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3412 DAG.getConstant(NVTBits - 1, dl, 3413 TLI.getPointerTy(DAG.getDataLayout()))); 3414 } 3415 } 3416 3417 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 3418 SDValue &Lo, SDValue &Hi) { 3419 SDLoc dl(N); 3420 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3421 EVT NVT = Lo.getValueType(); 3422 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3423 unsigned NVTBits = NVT.getSizeInBits(); 3424 unsigned EVTBits = EVT.getSizeInBits(); 3425 3426 if (NVTBits < EVTBits) { 3427 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 3428 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3429 EVTBits - NVTBits))); 3430 } else { 3431 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 3432 // The high part must be zero, make it explicit. 3433 Hi = DAG.getConstant(0, dl, NVT); 3434 } 3435 } 3436 3437 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, 3438 SDValue &Lo, SDValue &Hi) { 3439 SDLoc dl(N); 3440 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 3441 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); 3442 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); 3443 } 3444 3445 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 3446 SDValue &Lo, SDValue &Hi) { 3447 SDLoc dl(N); 3448 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 3449 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 3450 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 3451 } 3452 3453 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, 3454 SDValue &Hi) { 3455 SDLoc dl(N); 3456 // parity(HiLo) -> parity(Lo^Hi) 3457 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3458 EVT NVT = Lo.getValueType(); 3459 Lo = 3460 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi)); 3461 Hi = DAG.getConstant(0, dl, NVT); 3462 } 3463 3464 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 3465 SDValue &Lo, SDValue &Hi) { 3466 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3467 unsigned NBitWidth = NVT.getSizeInBits(); 3468 auto Constant = cast<ConstantSDNode>(N); 3469 const APInt &Cst = Constant->getAPIntValue(); 3470 bool IsTarget = Constant->isTargetOpcode(); 3471 bool IsOpaque = Constant->isOpaque(); 3472 SDLoc dl(N); 3473 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); 3474 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, 3475 IsOpaque); 3476 } 3477 3478 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { 3479 SDLoc dl(N); 3480 3481 SDValue N0 = N->getOperand(0); 3482 GetExpandedInteger(N0, Lo, Hi); 3483 EVT NVT = Lo.getValueType(); 3484 3485 // If the upper half is all sign bits, then we can perform the ABS on the 3486 // lower half and zero-extend. 3487 if (DAG.ComputeNumSignBits(N0) > NVT.getScalarSizeInBits()) { 3488 Lo = DAG.getNode(ISD::ABS, dl, NVT, Lo); 3489 Hi = DAG.getConstant(0, dl, NVT); 3490 return; 3491 } 3492 3493 // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence 3494 // we use in LegalizeDAG. The SUB part of the expansion is based on 3495 // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that 3496 // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further 3497 // expanded if needed. Shift expansion has a special case for filling with 3498 // sign bits so that we will only end up with one SRA. 3499 bool HasSubCarry = TLI.isOperationLegalOrCustom( 3500 ISD::USUBO_CARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 3501 if (HasSubCarry) { 3502 SDValue Sign = DAG.getNode( 3503 ISD::SRA, dl, NVT, Hi, 3504 DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl)); 3505 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 3506 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign); 3507 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign); 3508 Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign); 3509 Hi = DAG.getNode(ISD::USUBO_CARRY, dl, VTList, Hi, Sign, Lo.getValue(1)); 3510 return; 3511 } 3512 3513 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) 3514 EVT VT = N->getValueType(0); 3515 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, 3516 DAG.getConstant(0, dl, VT), N0); 3517 SDValue NegLo, NegHi; 3518 SplitInteger(Neg, NegLo, NegHi); 3519 3520 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 3521 DAG.getConstant(0, dl, NVT), ISD::SETLT); 3522 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); 3523 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); 3524 } 3525 3526 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 3527 SDValue &Lo, SDValue &Hi) { 3528 SDLoc dl(N); 3529 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 3530 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3531 EVT NVT = Lo.getValueType(); 3532 3533 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 3534 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3535 3536 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 3537 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 3538 3539 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 3540 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 3541 DAG.getConstant(NVT.getSizeInBits(), dl, 3542 NVT))); 3543 Hi = DAG.getConstant(0, dl, NVT); 3544 } 3545 3546 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 3547 SDValue &Lo, SDValue &Hi) { 3548 SDLoc dl(N); 3549 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 3550 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3551 EVT NVT = Lo.getValueType(); 3552 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 3553 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 3554 Hi = DAG.getConstant(0, dl, NVT); 3555 } 3556 3557 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 3558 SDValue &Lo, SDValue &Hi) { 3559 SDLoc dl(N); 3560 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 3561 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3562 EVT NVT = Lo.getValueType(); 3563 3564 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 3565 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3566 3567 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 3568 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 3569 3570 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 3571 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 3572 DAG.getConstant(NVT.getSizeInBits(), dl, 3573 NVT))); 3574 Hi = DAG.getConstant(0, dl, NVT); 3575 } 3576 3577 void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo, 3578 SDValue &Hi) { 3579 SDLoc dl(N); 3580 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3581 unsigned NBitWidth = NVT.getSizeInBits(); 3582 3583 Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0)); 3584 SDValue Chain = Lo.getValue(1); 3585 // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING 3586 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3587 DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl)); 3588 3589 // Legalize the chain result - switch anything that used the old chain to 3590 // use the new one. 3591 ReplaceValueWith(SDValue(N, 1), Chain); 3592 } 3593 3594 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 3595 SDValue &Hi) { 3596 SDLoc dl(N); 3597 EVT VT = N->getValueType(0); 3598 3599 bool IsStrict = N->isStrictFPOpcode(); 3600 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3601 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3602 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3603 Op = GetPromotedFloat(Op); 3604 3605 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3606 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3607 Op = GetSoftPromotedHalf(Op); 3608 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); 3609 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op); 3610 SplitInteger(Op, Lo, Hi); 3611 return; 3612 } 3613 3614 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 3615 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 3616 TargetLowering::MakeLibCallOptions CallOptions; 3617 CallOptions.setSExt(true); 3618 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3619 CallOptions, dl, Chain); 3620 SplitInteger(Tmp.first, Lo, Hi); 3621 3622 if (IsStrict) 3623 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3624 } 3625 3626 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 3627 SDValue &Hi) { 3628 SDLoc dl(N); 3629 EVT VT = N->getValueType(0); 3630 3631 bool IsStrict = N->isStrictFPOpcode(); 3632 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3633 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3634 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3635 Op = GetPromotedFloat(Op); 3636 3637 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3638 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3639 Op = GetSoftPromotedHalf(Op); 3640 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); 3641 Op = DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op); 3642 SplitInteger(Op, Lo, Hi); 3643 return; 3644 } 3645 3646 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 3647 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 3648 TargetLowering::MakeLibCallOptions CallOptions; 3649 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3650 CallOptions, dl, Chain); 3651 SplitInteger(Tmp.first, Lo, Hi); 3652 3653 if (IsStrict) 3654 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3655 } 3656 3657 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, 3658 SDValue &Hi) { 3659 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 3660 SplitInteger(Res, Lo, Hi); 3661 } 3662 3663 void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo, 3664 SDValue &Hi) { 3665 SDLoc dl(N); 3666 bool IsStrict = N->isStrictFPOpcode(); 3667 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3668 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3669 3670 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && 3671 "Input type needs to be promoted!"); 3672 3673 EVT VT = Op.getValueType(); 3674 3675 if (VT == MVT::f16) { 3676 VT = MVT::f32; 3677 // Extend to f32. 3678 if (IsStrict) { 3679 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { VT, MVT::Other }, {Chain, Op}); 3680 Chain = Op.getValue(1); 3681 } else { 3682 Op = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op); 3683 } 3684 } 3685 3686 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3687 if (N->getOpcode() == ISD::LROUND || 3688 N->getOpcode() == ISD::STRICT_LROUND) { 3689 if (VT == MVT::f32) 3690 LC = RTLIB::LROUND_F32; 3691 else if (VT == MVT::f64) 3692 LC = RTLIB::LROUND_F64; 3693 else if (VT == MVT::f80) 3694 LC = RTLIB::LROUND_F80; 3695 else if (VT == MVT::f128) 3696 LC = RTLIB::LROUND_F128; 3697 else if (VT == MVT::ppcf128) 3698 LC = RTLIB::LROUND_PPCF128; 3699 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!"); 3700 } else if (N->getOpcode() == ISD::LRINT || 3701 N->getOpcode() == ISD::STRICT_LRINT) { 3702 if (VT == MVT::f32) 3703 LC = RTLIB::LRINT_F32; 3704 else if (VT == MVT::f64) 3705 LC = RTLIB::LRINT_F64; 3706 else if (VT == MVT::f80) 3707 LC = RTLIB::LRINT_F80; 3708 else if (VT == MVT::f128) 3709 LC = RTLIB::LRINT_F128; 3710 else if (VT == MVT::ppcf128) 3711 LC = RTLIB::LRINT_PPCF128; 3712 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!"); 3713 } else if (N->getOpcode() == ISD::LLROUND || 3714 N->getOpcode() == ISD::STRICT_LLROUND) { 3715 if (VT == MVT::f32) 3716 LC = RTLIB::LLROUND_F32; 3717 else if (VT == MVT::f64) 3718 LC = RTLIB::LLROUND_F64; 3719 else if (VT == MVT::f80) 3720 LC = RTLIB::LLROUND_F80; 3721 else if (VT == MVT::f128) 3722 LC = RTLIB::LLROUND_F128; 3723 else if (VT == MVT::ppcf128) 3724 LC = RTLIB::LLROUND_PPCF128; 3725 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); 3726 } else if (N->getOpcode() == ISD::LLRINT || 3727 N->getOpcode() == ISD::STRICT_LLRINT) { 3728 if (VT == MVT::f32) 3729 LC = RTLIB::LLRINT_F32; 3730 else if (VT == MVT::f64) 3731 LC = RTLIB::LLRINT_F64; 3732 else if (VT == MVT::f80) 3733 LC = RTLIB::LLRINT_F80; 3734 else if (VT == MVT::f128) 3735 LC = RTLIB::LLRINT_F128; 3736 else if (VT == MVT::ppcf128) 3737 LC = RTLIB::LLRINT_PPCF128; 3738 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); 3739 } else 3740 llvm_unreachable("Unexpected opcode!"); 3741 3742 EVT RetVT = N->getValueType(0); 3743 3744 TargetLowering::MakeLibCallOptions CallOptions; 3745 CallOptions.setSExt(true); 3746 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, 3747 Op, CallOptions, dl, 3748 Chain); 3749 SplitInteger(Tmp.first, Lo, Hi); 3750 3751 if (N->isStrictFPOpcode()) 3752 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3753 } 3754 3755 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 3756 SDValue &Lo, SDValue &Hi) { 3757 if (N->isAtomic()) { 3758 // It's typical to have larger CAS than atomic load instructions. 3759 SDLoc dl(N); 3760 EVT VT = N->getMemoryVT(); 3761 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 3762 SDValue Zero = DAG.getConstant(0, dl, VT); 3763 SDValue Swap = DAG.getAtomicCmpSwap( 3764 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 3765 VT, VTs, N->getOperand(0), 3766 N->getOperand(1), Zero, Zero, N->getMemOperand()); 3767 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 3768 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 3769 return; 3770 } 3771 3772 if (ISD::isNormalLoad(N)) { 3773 ExpandRes_NormalLoad(N, Lo, Hi); 3774 return; 3775 } 3776 3777 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 3778 3779 EVT VT = N->getValueType(0); 3780 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3781 SDValue Ch = N->getChain(); 3782 SDValue Ptr = N->getBasePtr(); 3783 ISD::LoadExtType ExtType = N->getExtensionType(); 3784 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 3785 AAMDNodes AAInfo = N->getAAInfo(); 3786 SDLoc dl(N); 3787 3788 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 3789 3790 if (N->getMemoryVT().bitsLE(NVT)) { 3791 EVT MemVT = N->getMemoryVT(); 3792 3793 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, 3794 N->getOriginalAlign(), MMOFlags, AAInfo); 3795 3796 // Remember the chain. 3797 Ch = Lo.getValue(1); 3798 3799 if (ExtType == ISD::SEXTLOAD) { 3800 // The high part is obtained by SRA'ing all but one of the bits of the 3801 // lo part. 3802 unsigned LoSize = Lo.getValueSizeInBits(); 3803 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3804 DAG.getConstant(LoSize - 1, dl, 3805 TLI.getPointerTy(DAG.getDataLayout()))); 3806 } else if (ExtType == ISD::ZEXTLOAD) { 3807 // The high part is just a zero. 3808 Hi = DAG.getConstant(0, dl, NVT); 3809 } else { 3810 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 3811 // The high part is undefined. 3812 Hi = DAG.getUNDEF(NVT); 3813 } 3814 } else if (DAG.getDataLayout().isLittleEndian()) { 3815 // Little-endian - low bits are at low addresses. 3816 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 3817 N->getOriginalAlign(), MMOFlags, AAInfo); 3818 3819 unsigned ExcessBits = 3820 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 3821 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 3822 3823 // Increment the pointer to the other half. 3824 unsigned IncrementSize = NVT.getSizeInBits()/8; 3825 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 3826 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 3827 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 3828 N->getOriginalAlign(), MMOFlags, AAInfo); 3829 3830 // Build a factor node to remember that this load is independent of the 3831 // other one. 3832 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3833 Hi.getValue(1)); 3834 } else { 3835 // Big-endian - high bits are at low addresses. Favor aligned loads at 3836 // the cost of some bit-fiddling. 3837 EVT MemVT = N->getMemoryVT(); 3838 unsigned EBytes = MemVT.getStoreSize(); 3839 unsigned IncrementSize = NVT.getSizeInBits()/8; 3840 unsigned ExcessBits = (EBytes - IncrementSize)*8; 3841 3842 // Load both the high bits and maybe some of the low bits. 3843 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 3844 EVT::getIntegerVT(*DAG.getContext(), 3845 MemVT.getSizeInBits() - ExcessBits), 3846 N->getOriginalAlign(), MMOFlags, AAInfo); 3847 3848 // Increment the pointer to the other half. 3849 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 3850 // Load the rest of the low bits. 3851 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 3852 N->getPointerInfo().getWithOffset(IncrementSize), 3853 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 3854 N->getOriginalAlign(), MMOFlags, AAInfo); 3855 3856 // Build a factor node to remember that this load is independent of the 3857 // other one. 3858 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3859 Hi.getValue(1)); 3860 3861 if (ExcessBits < NVT.getSizeInBits()) { 3862 // Transfer low bits from the bottom of Hi to the top of Lo. 3863 Lo = DAG.getNode( 3864 ISD::OR, dl, NVT, Lo, 3865 DAG.getNode(ISD::SHL, dl, NVT, Hi, 3866 DAG.getConstant(ExcessBits, dl, 3867 TLI.getPointerTy(DAG.getDataLayout())))); 3868 // Move high bits to the right position in Hi. 3869 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, 3870 Hi, 3871 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 3872 TLI.getPointerTy(DAG.getDataLayout()))); 3873 } 3874 } 3875 3876 // Legalize the chain result - switch anything that used the old chain to 3877 // use the new one. 3878 ReplaceValueWith(SDValue(N, 1), Ch); 3879 } 3880 3881 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 3882 SDValue &Lo, SDValue &Hi) { 3883 SDLoc dl(N); 3884 SDValue LL, LH, RL, RH; 3885 GetExpandedInteger(N->getOperand(0), LL, LH); 3886 GetExpandedInteger(N->getOperand(1), RL, RH); 3887 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 3888 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 3889 } 3890 3891 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 3892 SDValue &Lo, SDValue &Hi) { 3893 EVT VT = N->getValueType(0); 3894 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3895 SDLoc dl(N); 3896 3897 SDValue LL, LH, RL, RH; 3898 GetExpandedInteger(N->getOperand(0), LL, LH); 3899 GetExpandedInteger(N->getOperand(1), RL, RH); 3900 3901 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, 3902 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3903 LL, LH, RL, RH)) 3904 return; 3905 3906 // If nothing else, we can make a libcall. 3907 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3908 if (VT == MVT::i16) 3909 LC = RTLIB::MUL_I16; 3910 else if (VT == MVT::i32) 3911 LC = RTLIB::MUL_I32; 3912 else if (VT == MVT::i64) 3913 LC = RTLIB::MUL_I64; 3914 else if (VT == MVT::i128) 3915 LC = RTLIB::MUL_I128; 3916 3917 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 3918 // We'll expand the multiplication by brute force because we have no other 3919 // options. This is a trivially-generalized version of the code from 3920 // Hacker's Delight (itself derived from Knuth's Algorithm M from section 3921 // 4.3.1). 3922 unsigned Bits = NVT.getSizeInBits(); 3923 unsigned HalfBits = Bits >> 1; 3924 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, 3925 NVT); 3926 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); 3927 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); 3928 3929 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); 3930 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); 3931 3932 SDValue Shift = DAG.getShiftAmountConstant(HalfBits, NVT, dl); 3933 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); 3934 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); 3935 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); 3936 3937 SDValue U = DAG.getNode(ISD::ADD, dl, NVT, 3938 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); 3939 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); 3940 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); 3941 3942 SDValue V = DAG.getNode(ISD::ADD, dl, NVT, 3943 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); 3944 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); 3945 3946 SDValue W = DAG.getNode(ISD::ADD, dl, NVT, 3947 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), 3948 DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); 3949 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, 3950 DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); 3951 3952 Hi = DAG.getNode(ISD::ADD, dl, NVT, W, 3953 DAG.getNode(ISD::ADD, dl, NVT, 3954 DAG.getNode(ISD::MUL, dl, NVT, RH, LL), 3955 DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); 3956 return; 3957 } 3958 3959 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3960 TargetLowering::MakeLibCallOptions CallOptions; 3961 CallOptions.setSExt(true); 3962 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, 3963 Lo, Hi); 3964 } 3965 3966 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, 3967 SDValue &Hi) { 3968 SDLoc DL(N); 3969 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3970 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); 3971 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); 3972 Lo = R.getValue(0); 3973 Hi = R.getValue(1); 3974 ReplaceValueWith(SDValue(N, 1), R.getValue(2)); 3975 } 3976 3977 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, 3978 SDValue &Hi) { 3979 SDValue Result = TLI.expandAddSubSat(N, DAG); 3980 SplitInteger(Result, Lo, Hi); 3981 } 3982 3983 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, 3984 SDValue &Hi) { 3985 SDValue Result = TLI.expandShlSat(N, DAG); 3986 SplitInteger(Result, Lo, Hi); 3987 } 3988 3989 /// This performs an expansion of the integer result for a fixed point 3990 /// multiplication. The default expansion performs rounding down towards 3991 /// negative infinity, though targets that do care about rounding should specify 3992 /// a target hook for rounding and provide their own expansion or lowering of 3993 /// fixed point multiplication to be consistent with rounding. 3994 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, 3995 SDValue &Hi) { 3996 SDLoc dl(N); 3997 EVT VT = N->getValueType(0); 3998 unsigned VTSize = VT.getScalarSizeInBits(); 3999 SDValue LHS = N->getOperand(0); 4000 SDValue RHS = N->getOperand(1); 4001 uint64_t Scale = N->getConstantOperandVal(2); 4002 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || 4003 N->getOpcode() == ISD::UMULFIXSAT); 4004 bool Signed = (N->getOpcode() == ISD::SMULFIX || 4005 N->getOpcode() == ISD::SMULFIXSAT); 4006 4007 // Handle special case when scale is equal to zero. 4008 if (!Scale) { 4009 SDValue Result; 4010 if (!Saturating) { 4011 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 4012 } else { 4013 EVT BoolVT = getSetCCResultType(VT); 4014 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; 4015 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); 4016 SDValue Product = Result.getValue(0); 4017 SDValue Overflow = Result.getValue(1); 4018 if (Signed) { 4019 APInt MinVal = APInt::getSignedMinValue(VTSize); 4020 APInt MaxVal = APInt::getSignedMaxValue(VTSize); 4021 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); 4022 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 4023 SDValue Zero = DAG.getConstant(0, dl, VT); 4024 // Xor the inputs, if resulting sign bit is 0 the product will be 4025 // positive, else negative. 4026 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); 4027 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); 4028 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); 4029 Result = DAG.getSelect(dl, VT, Overflow, Result, Product); 4030 } else { 4031 // For unsigned multiplication, we only need to check the max since we 4032 // can't really overflow towards zero. 4033 APInt MaxVal = APInt::getMaxValue(VTSize); 4034 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 4035 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product); 4036 } 4037 } 4038 SplitInteger(Result, Lo, Hi); 4039 return; 4040 } 4041 4042 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will 4043 // cover for unhandled cases below, while still being valid for UMULFIX[SAT]. 4044 assert(Scale <= VTSize && "Scale can't be larger than the value type size."); 4045 4046 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4047 SDValue LL, LH, RL, RH; 4048 GetExpandedInteger(LHS, LL, LH); 4049 GetExpandedInteger(RHS, RL, RH); 4050 SmallVector<SDValue, 4> Result; 4051 4052 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; 4053 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, 4054 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 4055 LL, LH, RL, RH)) { 4056 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); 4057 return; 4058 } 4059 4060 unsigned NVTSize = NVT.getScalarSizeInBits(); 4061 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " 4062 "the size of the current value type"); 4063 4064 // After getting the multiplication result in 4 parts, we need to perform a 4065 // shift right by the amount of the scale to get the result in that scale. 4066 // 4067 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in 4068 // 128 bits that are cut into 4 32-bit parts: 4069 // 4070 // HH HL LH LL 4071 // |---32---|---32---|---32---|---32---| 4072 // 128 96 64 32 0 4073 // 4074 // |------VTSize-----| 4075 // 4076 // |NVTSize-| 4077 // 4078 // The resulting Lo and Hi would normally be in LL and LH after the shift. But 4079 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift 4080 // amount and get Lo and Hi using two funnel shifts. Or for the special case 4081 // when Scale is a multiple of NVTSize we can just pick the result without 4082 // shifting. 4083 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. 4084 if (Scale % NVTSize) { 4085 SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl); 4086 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0], 4087 ShiftAmount); 4088 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1], 4089 ShiftAmount); 4090 } else { 4091 Lo = Result[Part0]; 4092 Hi = Result[Part0 + 1]; 4093 } 4094 4095 // Unless saturation is requested we are done. The result is in <Hi,Lo>. 4096 if (!Saturating) 4097 return; 4098 4099 // Can not overflow when there is no integer part. 4100 if (Scale == VTSize) 4101 return; 4102 4103 // To handle saturation we must check for overflow in the multiplication. 4104 // 4105 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) 4106 // aren't all zeroes. 4107 // 4108 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) 4109 // aren't all ones or all zeroes. 4110 // 4111 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the 4112 // highest bit of HH determines saturation direction in the event of signed 4113 // saturation. 4114 4115 SDValue ResultHL = Result[2]; 4116 SDValue ResultHH = Result[3]; 4117 4118 SDValue SatMax, SatMin; 4119 SDValue NVTZero = DAG.getConstant(0, dl, NVT); 4120 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); 4121 EVT BoolNVT = getSetCCResultType(NVT); 4122 4123 if (!Signed) { 4124 if (Scale < NVTSize) { 4125 // Overflow happened if ((HH | (HL >> Scale)) != 0). 4126 SDValue HLAdjusted = 4127 DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 4128 DAG.getShiftAmountConstant(Scale, NVT, dl)); 4129 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH); 4130 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE); 4131 } else if (Scale == NVTSize) { 4132 // Overflow happened if (HH != 0). 4133 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE); 4134 } else if (Scale < VTSize) { 4135 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0). 4136 SDValue HLAdjusted = 4137 DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 4138 DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl)); 4139 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE); 4140 } else 4141 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" 4142 "(and saturation can't happen with Scale==VTSize)."); 4143 4144 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi); 4145 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo); 4146 return; 4147 } 4148 4149 if (Scale < NVTSize) { 4150 // The number of overflow bits we can check are VTSize - Scale + 1 (we 4151 // include the sign bit). If these top bits are > 0, then we overflowed past 4152 // the max value. If these top bits are < -1, then we overflowed past the 4153 // min value. Otherwise, we did not overflow. 4154 unsigned OverflowBits = VTSize - Scale + 1; 4155 assert(OverflowBits <= VTSize && OverflowBits > NVTSize && 4156 "Extent of overflow bits must start within HL"); 4157 SDValue HLHiMask = DAG.getConstant( 4158 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); 4159 SDValue HLLoMask = DAG.getConstant( 4160 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); 4161 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). 4162 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 4163 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 4164 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); 4165 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 4166 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT)); 4167 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). 4168 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 4169 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 4170 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); 4171 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 4172 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT)); 4173 } else if (Scale == NVTSize) { 4174 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). 4175 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 4176 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 4177 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); 4178 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 4179 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg)); 4180 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). 4181 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 4182 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 4183 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); 4184 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 4185 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos)); 4186 } else if (Scale < VTSize) { 4187 // This is similar to the case when we saturate if Scale < NVTSize, but we 4188 // only need to check HH. 4189 unsigned OverflowBits = VTSize - Scale + 1; 4190 SDValue HHHiMask = DAG.getConstant( 4191 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); 4192 SDValue HHLoMask = DAG.getConstant( 4193 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); 4194 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); 4195 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); 4196 } else 4197 llvm_unreachable("Illegal scale for signed fixed point mul."); 4198 4199 // Saturate to signed maximum. 4200 APInt MaxHi = APInt::getSignedMaxValue(NVTSize); 4201 APInt MaxLo = APInt::getAllOnes(NVTSize); 4202 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi); 4203 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo); 4204 // Saturate to signed minimum. 4205 APInt MinHi = APInt::getSignedMinValue(NVTSize); 4206 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi); 4207 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); 4208 } 4209 4210 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, 4211 SDValue &Hi) { 4212 SDLoc dl(N); 4213 // Try expanding in the existing type first. 4214 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0), 4215 N->getOperand(1), 4216 N->getConstantOperandVal(2), DAG); 4217 4218 if (!Res) 4219 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1), 4220 N->getConstantOperandVal(2), TLI, DAG); 4221 SplitInteger(Res, Lo, Hi); 4222 } 4223 4224 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 4225 SDValue &Lo, SDValue &Hi) { 4226 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && 4227 "Node has unexpected Opcode"); 4228 SDValue LHS = Node->getOperand(0); 4229 SDValue RHS = Node->getOperand(1); 4230 SDLoc dl(Node); 4231 4232 SDValue Ovf; 4233 4234 bool IsAdd = Node->getOpcode() == ISD::SADDO; 4235 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY; 4236 4237 bool HasCarryOp = TLI.isOperationLegalOrCustom( 4238 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 4239 4240 if (HasCarryOp) { 4241 // Expand the subcomponents. 4242 SDValue LHSL, LHSH, RHSL, RHSH; 4243 GetExpandedInteger(LHS, LHSL, LHSH); 4244 GetExpandedInteger(RHS, RHSL, RHSH); 4245 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1)); 4246 4247 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL}); 4248 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 4249 4250 Ovf = Hi.getValue(1); 4251 } else { 4252 // Expand the result by simply replacing it with the equivalent 4253 // non-overflow-checking operation. 4254 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 4255 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 4256 LHS, RHS); 4257 SplitInteger(Sum, Lo, Hi); 4258 4259 // Compute the overflow. 4260 // 4261 // LHSSign -> LHS < 0 4262 // RHSSign -> RHS < 0 4263 // SumSign -> Sum < 0 4264 // 4265 // Add: 4266 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 4267 // Sub: 4268 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 4269 // 4270 // To get better codegen we can rewrite this by doing bitwise math on 4271 // the integers and extract the final sign bit at the end. So the 4272 // above becomes: 4273 // 4274 // Add: 4275 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0 4276 // Sub: 4277 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0 4278 // 4279 // NOTE: This is different than the expansion we do in expandSADDSUBO 4280 // because it is more costly to determine the RHS is > 0 for SSUBO with the 4281 // integers split. 4282 EVT VT = LHS.getValueType(); 4283 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); 4284 if (IsAdd) 4285 SignsMatch = DAG.getNOT(dl, SignsMatch, VT); 4286 4287 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum); 4288 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE); 4289 EVT OType = Node->getValueType(1); 4290 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT); 4291 } 4292 4293 // Use the calculated overflow everywhere. 4294 ReplaceValueWith(SDValue(Node, 1), Ovf); 4295 } 4296 4297 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 4298 SDValue &Lo, SDValue &Hi) { 4299 EVT VT = N->getValueType(0); 4300 SDLoc dl(N); 4301 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4302 4303 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 4304 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4305 SplitInteger(Res.getValue(0), Lo, Hi); 4306 return; 4307 } 4308 4309 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4310 if (VT == MVT::i16) 4311 LC = RTLIB::SDIV_I16; 4312 else if (VT == MVT::i32) 4313 LC = RTLIB::SDIV_I32; 4314 else if (VT == MVT::i64) 4315 LC = RTLIB::SDIV_I64; 4316 else if (VT == MVT::i128) 4317 LC = RTLIB::SDIV_I128; 4318 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 4319 4320 TargetLowering::MakeLibCallOptions CallOptions; 4321 CallOptions.setSExt(true); 4322 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4323 } 4324 4325 void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo, 4326 SDValue &Hi) { 4327 SDLoc dl(N); 4328 SDValue Shiftee = N->getOperand(0); 4329 EVT VT = Shiftee.getValueType(); 4330 SDValue ShAmt = N->getOperand(1); 4331 EVT ShAmtVT = ShAmt.getValueType(); 4332 4333 // This legalization is optimal when the shift is by a multiple of byte width, 4334 // %x * 8 <-> %x << 3 so 3 low bits should be be known zero. 4335 bool ShiftByByteMultiple = 4336 DAG.computeKnownBits(ShAmt).countMinTrailingZeros() >= 3; 4337 4338 // If we can't do it as one step, we'll have two uses of shift amount, 4339 // and thus must freeze it. 4340 if (!ShiftByByteMultiple) 4341 ShAmt = DAG.getFreeze(ShAmt); 4342 4343 unsigned VTBitWidth = VT.getScalarSizeInBits(); 4344 assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?"); 4345 unsigned VTByteWidth = VTBitWidth / 8; 4346 assert(isPowerOf2_32(VTByteWidth) && 4347 "Shiftee type size is not a power of two!"); 4348 unsigned StackSlotByteWidth = 2 * VTByteWidth; 4349 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth; 4350 EVT StackSlotVT = EVT::getIntegerVT(*DAG.getContext(), StackSlotBitWidth); 4351 4352 // Get a temporary stack slot 2x the width of our VT. 4353 // FIXME: reuse stack slots? 4354 // FIXME: should we be more picky about alignment? 4355 Align StackSlotAlignment(1); 4356 SDValue StackPtr = DAG.CreateStackTemporary( 4357 TypeSize::getFixed(StackSlotByteWidth), StackSlotAlignment); 4358 EVT PtrTy = StackPtr.getValueType(); 4359 SDValue Ch = DAG.getEntryNode(); 4360 4361 MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack( 4362 DAG.getMachineFunction(), 4363 cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex()); 4364 4365 // Extend the value, that is being shifted, to the entire stack slot's width. 4366 SDValue Init; 4367 if (N->getOpcode() != ISD::SHL) { 4368 unsigned WideningOpc = 4369 N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4370 Init = DAG.getNode(WideningOpc, dl, StackSlotVT, Shiftee); 4371 } else { 4372 // For left-shifts, pad the Shiftee's LSB with zeros to twice it's width. 4373 SDValue AllZeros = DAG.getConstant(0, dl, VT); 4374 Init = DAG.getNode(ISD::BUILD_PAIR, dl, StackSlotVT, AllZeros, Shiftee); 4375 } 4376 // And spill it into the stack slot. 4377 Ch = DAG.getStore(Ch, dl, Init, StackPtr, StackPtrInfo, StackSlotAlignment); 4378 4379 // Now, compute the full-byte offset into stack slot from where we can load. 4380 // We have shift amount, which is in bits, but in multiples of byte. 4381 // So just divide by CHAR_BIT. 4382 SDNodeFlags Flags; 4383 if (ShiftByByteMultiple) 4384 Flags.setExact(true); 4385 SDValue ByteOffset = DAG.getNode(ISD::SRL, dl, ShAmtVT, ShAmt, 4386 DAG.getConstant(3, dl, ShAmtVT), Flags); 4387 // And clamp it, because OOB load is an immediate UB, 4388 // while shift overflow would have *just* been poison. 4389 ByteOffset = DAG.getNode(ISD::AND, dl, ShAmtVT, ByteOffset, 4390 DAG.getConstant(VTByteWidth - 1, dl, ShAmtVT)); 4391 // We have exactly two strategies on indexing into stack slot here: 4392 // 1. upwards starting from the beginning of the slot 4393 // 2. downwards starting from the middle of the slot 4394 // On little-endian machine, we pick 1. for right shifts and 2. for left-shift 4395 // and vice versa on big-endian machine. 4396 bool WillIndexUpwards = N->getOpcode() != ISD::SHL; 4397 if (DAG.getDataLayout().isBigEndian()) 4398 WillIndexUpwards = !WillIndexUpwards; 4399 4400 SDValue AdjStackPtr; 4401 if (WillIndexUpwards) { 4402 AdjStackPtr = StackPtr; 4403 } else { 4404 AdjStackPtr = DAG.getMemBasePlusOffset( 4405 StackPtr, DAG.getConstant(VTByteWidth, dl, PtrTy), dl); 4406 ByteOffset = DAG.getNegative(ByteOffset, dl, ShAmtVT); 4407 } 4408 4409 // Get the pointer somewhere into the stack slot from which we need to load. 4410 ByteOffset = DAG.getSExtOrTrunc(ByteOffset, dl, PtrTy); 4411 AdjStackPtr = DAG.getMemBasePlusOffset(AdjStackPtr, ByteOffset, dl); 4412 4413 // And load it! While the load is not legal, legalizing it is obvious. 4414 SDValue Res = DAG.getLoad( 4415 VT, dl, Ch, AdjStackPtr, 4416 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), Align(1)); 4417 // We've performed the shift by a CHAR_BIT * [_ShAmt / CHAR_BIT_] 4418 4419 // If we may still have a less-than-CHAR_BIT to shift by, do so now. 4420 if (!ShiftByByteMultiple) { 4421 SDValue ShAmtRem = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, 4422 DAG.getConstant(7, dl, ShAmtVT)); 4423 Res = DAG.getNode(N->getOpcode(), dl, VT, Res, ShAmtRem); 4424 } 4425 4426 // Finally, split the computed value. 4427 SplitInteger(Res, Lo, Hi); 4428 } 4429 4430 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 4431 SDValue &Lo, SDValue &Hi) { 4432 EVT VT = N->getValueType(0); 4433 SDLoc dl(N); 4434 4435 // If we can emit an efficient shift operation, do so now. Check to see if 4436 // the RHS is a constant. 4437 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 4438 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); 4439 4440 // If we can determine that the high bit of the shift is zero or one, even if 4441 // the low bits are variable, emit this shift in an optimized form. 4442 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 4443 return; 4444 4445 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 4446 unsigned PartsOpc; 4447 if (N->getOpcode() == ISD::SHL) { 4448 PartsOpc = ISD::SHL_PARTS; 4449 } else if (N->getOpcode() == ISD::SRL) { 4450 PartsOpc = ISD::SRL_PARTS; 4451 } else { 4452 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 4453 PartsOpc = ISD::SRA_PARTS; 4454 } 4455 4456 // Next check to see if the target supports this SHL_PARTS operation or if it 4457 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for 4458 // size, but create a libcall instead. 4459 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4460 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 4461 const bool LegalOrCustom = 4462 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 4463 Action == TargetLowering::Custom; 4464 4465 unsigned ExpansionFactor = 1; 4466 // That VT->NVT expansion is one step. But will we re-expand NVT? 4467 for (EVT TmpVT = NVT;;) { 4468 EVT NewTMPVT = TLI.getTypeToTransformTo(*DAG.getContext(), TmpVT); 4469 if (NewTMPVT == TmpVT) 4470 break; 4471 TmpVT = NewTMPVT; 4472 ++ExpansionFactor; 4473 } 4474 4475 TargetLowering::ShiftLegalizationStrategy S = 4476 TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor); 4477 4478 if (S == TargetLowering::ShiftLegalizationStrategy::ExpandThroughStack) 4479 return ExpandIntRes_ShiftThroughStack(N, Lo, Hi); 4480 4481 if (LegalOrCustom && 4482 S != TargetLowering::ShiftLegalizationStrategy::LowerToLibcall) { 4483 // Expand the subcomponents. 4484 SDValue LHSL, LHSH; 4485 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 4486 EVT VT = LHSL.getValueType(); 4487 4488 // If the shift amount operand is coming from a vector legalization it may 4489 // have an illegal type. Fix that first by casting the operand, otherwise 4490 // the new SHL_PARTS operation would need further legalization. 4491 SDValue ShiftOp = N->getOperand(1); 4492 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 4493 if (ShiftOp.getValueType() != ShiftTy) 4494 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 4495 4496 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 4497 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 4498 Hi = Lo.getValue(1); 4499 return; 4500 } 4501 4502 // Otherwise, emit a libcall. 4503 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4504 bool isSigned; 4505 if (N->getOpcode() == ISD::SHL) { 4506 isSigned = false; /*sign irrelevant*/ 4507 if (VT == MVT::i16) 4508 LC = RTLIB::SHL_I16; 4509 else if (VT == MVT::i32) 4510 LC = RTLIB::SHL_I32; 4511 else if (VT == MVT::i64) 4512 LC = RTLIB::SHL_I64; 4513 else if (VT == MVT::i128) 4514 LC = RTLIB::SHL_I128; 4515 } else if (N->getOpcode() == ISD::SRL) { 4516 isSigned = false; 4517 if (VT == MVT::i16) 4518 LC = RTLIB::SRL_I16; 4519 else if (VT == MVT::i32) 4520 LC = RTLIB::SRL_I32; 4521 else if (VT == MVT::i64) 4522 LC = RTLIB::SRL_I64; 4523 else if (VT == MVT::i128) 4524 LC = RTLIB::SRL_I128; 4525 } else { 4526 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 4527 isSigned = true; 4528 if (VT == MVT::i16) 4529 LC = RTLIB::SRA_I16; 4530 else if (VT == MVT::i32) 4531 LC = RTLIB::SRA_I32; 4532 else if (VT == MVT::i64) 4533 LC = RTLIB::SRA_I64; 4534 else if (VT == MVT::i128) 4535 LC = RTLIB::SRA_I128; 4536 } 4537 4538 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 4539 EVT ShAmtTy = 4540 EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize()); 4541 SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy); 4542 SDValue Ops[2] = {N->getOperand(0), ShAmt}; 4543 TargetLowering::MakeLibCallOptions CallOptions; 4544 CallOptions.setSExt(isSigned); 4545 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4546 return; 4547 } 4548 4549 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 4550 llvm_unreachable("Unsupported shift!"); 4551 } 4552 4553 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 4554 SDValue &Lo, SDValue &Hi) { 4555 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4556 SDLoc dl(N); 4557 SDValue Op = N->getOperand(0); 4558 if (Op.getValueType().bitsLE(NVT)) { 4559 // The low part is sign extension of the input (degenerates to a copy). 4560 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 4561 // The high part is obtained by SRA'ing all but one of the bits of low part. 4562 unsigned LoSize = NVT.getSizeInBits(); 4563 Hi = DAG.getNode( 4564 ISD::SRA, dl, NVT, Lo, 4565 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); 4566 } else { 4567 // For example, extension of an i48 to an i64. The operand type necessarily 4568 // promotes to the result type, so will end up being expanded too. 4569 assert(getTypeAction(Op.getValueType()) == 4570 TargetLowering::TypePromoteInteger && 4571 "Only know how to promote this result!"); 4572 SDValue Res = GetPromotedInteger(Op); 4573 assert(Res.getValueType() == N->getValueType(0) && 4574 "Operand over promoted?"); 4575 // Split the promoted operand. This will simplify when it is expanded. 4576 SplitInteger(Res, Lo, Hi); 4577 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 4578 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 4579 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 4580 ExcessBits))); 4581 } 4582 } 4583 4584 void DAGTypeLegalizer:: 4585 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 4586 SDLoc dl(N); 4587 GetExpandedInteger(N->getOperand(0), Lo, Hi); 4588 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 4589 4590 if (EVT.bitsLE(Lo.getValueType())) { 4591 // sext_inreg the low part if needed. 4592 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 4593 N->getOperand(1)); 4594 4595 // The high part gets the sign extension from the lo-part. This handles 4596 // things like sextinreg V:i64 from i8. 4597 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 4598 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, 4599 TLI.getPointerTy(DAG.getDataLayout()))); 4600 } else { 4601 // For example, extension of an i48 to an i64. Leave the low part alone, 4602 // sext_inreg the high part. 4603 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); 4604 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 4605 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 4606 ExcessBits))); 4607 } 4608 } 4609 4610 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 4611 SDValue &Lo, SDValue &Hi) { 4612 EVT VT = N->getValueType(0); 4613 SDLoc dl(N); 4614 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4615 4616 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 4617 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4618 SplitInteger(Res.getValue(1), Lo, Hi); 4619 return; 4620 } 4621 4622 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4623 if (VT == MVT::i16) 4624 LC = RTLIB::SREM_I16; 4625 else if (VT == MVT::i32) 4626 LC = RTLIB::SREM_I32; 4627 else if (VT == MVT::i64) 4628 LC = RTLIB::SREM_I64; 4629 else if (VT == MVT::i128) 4630 LC = RTLIB::SREM_I128; 4631 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 4632 4633 TargetLowering::MakeLibCallOptions CallOptions; 4634 CallOptions.setSExt(true); 4635 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4636 } 4637 4638 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 4639 SDValue &Lo, SDValue &Hi) { 4640 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4641 SDLoc dl(N); 4642 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 4643 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), 4644 N->getOperand(0), 4645 DAG.getConstant(NVT.getSizeInBits(), dl, 4646 TLI.getPointerTy(DAG.getDataLayout()))); 4647 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 4648 } 4649 4650 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 4651 SDValue &Lo, SDValue &Hi) { 4652 EVT VT = N->getValueType(0); 4653 SDLoc dl(N); 4654 4655 if (N->getOpcode() == ISD::UMULO) { 4656 // This section expands the operation into the following sequence of 4657 // instructions. `iNh` here refers to a type which has half the bit width of 4658 // the type the original operation operated on. 4659 // 4660 // %0 = %LHS.HI != 0 && %RHS.HI != 0 4661 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) 4662 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) 4663 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) 4664 // %4 = add iNh %1.0, %2.0 as iN 4665 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH) 4666 // 4667 // %lo = %3.LO 4668 // %hi = %5.0 4669 // %ovf = %0 || %1.1 || %2.1 || %5.1 4670 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 4671 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; 4672 GetExpandedInteger(LHS, LHSLow, LHSHigh); 4673 GetExpandedInteger(RHS, RHSLow, RHSHigh); 4674 EVT HalfVT = LHSLow.getValueType(); 4675 EVT BitVT = N->getValueType(1); 4676 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT); 4677 4678 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT); 4679 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT, 4680 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE), 4681 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE)); 4682 4683 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow); 4684 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1)); 4685 4686 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow); 4687 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1)); 4688 4689 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two); 4690 4691 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not 4692 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this 4693 // operation recursively legalized?). 4694 // 4695 // Many backends understand this pattern and will convert into LOHI 4696 // themselves, if applicable. 4697 SDValue Three = DAG.getNode(ISD::MUL, dl, VT, 4698 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow), 4699 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow)); 4700 SplitInteger(Three, Lo, Hi); 4701 4702 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum); 4703 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1)); 4704 ReplaceValueWith(SDValue(N, 1), Overflow); 4705 return; 4706 } 4707 4708 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 4709 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 4710 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 4711 4712 // Replace this with a libcall that will check overflow. 4713 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4714 if (VT == MVT::i32) 4715 LC = RTLIB::MULO_I32; 4716 else if (VT == MVT::i64) 4717 LC = RTLIB::MULO_I64; 4718 else if (VT == MVT::i128) 4719 LC = RTLIB::MULO_I128; 4720 4721 // If we don't have the libcall or if the function we are compiling is the 4722 // implementation of the expected libcall (avoid inf-loop), expand inline. 4723 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC) || 4724 TLI.getLibcallName(LC) == DAG.getMachineFunction().getName()) { 4725 // FIXME: This is not an optimal expansion, but better than crashing. 4726 EVT WideVT = 4727 EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2); 4728 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0)); 4729 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1)); 4730 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); 4731 SDValue MulLo, MulHi; 4732 SplitInteger(Mul, MulLo, MulHi); 4733 SDValue SRA = 4734 DAG.getNode(ISD::SRA, dl, VT, MulLo, 4735 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT)); 4736 SDValue Overflow = 4737 DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE); 4738 SplitInteger(MulLo, Lo, Hi); 4739 ReplaceValueWith(SDValue(N, 1), Overflow); 4740 return; 4741 } 4742 4743 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 4744 // Temporary for the overflow value, default it to zero. 4745 SDValue Chain = 4746 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, 4747 MachinePointerInfo()); 4748 4749 TargetLowering::ArgListTy Args; 4750 TargetLowering::ArgListEntry Entry; 4751 for (const SDValue &Op : N->op_values()) { 4752 EVT ArgVT = Op.getValueType(); 4753 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 4754 Entry.Node = Op; 4755 Entry.Ty = ArgTy; 4756 Entry.IsSExt = true; 4757 Entry.IsZExt = false; 4758 Args.push_back(Entry); 4759 } 4760 4761 // Also pass the address of the overflow check. 4762 Entry.Node = Temp; 4763 Entry.Ty = PtrTy->getPointerTo(); 4764 Entry.IsSExt = true; 4765 Entry.IsZExt = false; 4766 Args.push_back(Entry); 4767 4768 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 4769 4770 TargetLowering::CallLoweringInfo CLI(DAG); 4771 CLI.setDebugLoc(dl) 4772 .setChain(Chain) 4773 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) 4774 .setSExtResult(); 4775 4776 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 4777 4778 SplitInteger(CallInfo.first, Lo, Hi); 4779 SDValue Temp2 = 4780 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); 4781 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 4782 DAG.getConstant(0, dl, PtrVT), 4783 ISD::SETNE); 4784 // Use the overflow from the libcall everywhere. 4785 ReplaceValueWith(SDValue(N, 1), Ofl); 4786 } 4787 4788 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 4789 SDValue &Lo, SDValue &Hi) { 4790 EVT VT = N->getValueType(0); 4791 SDLoc dl(N); 4792 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4793 4794 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4795 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4796 SplitInteger(Res.getValue(0), Lo, Hi); 4797 return; 4798 } 4799 4800 // Try to expand UDIV by constant. 4801 if (isa<ConstantSDNode>(N->getOperand(1))) { 4802 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4803 // Only if the new type is legal. 4804 if (isTypeLegal(NVT)) { 4805 SDValue InL, InH; 4806 GetExpandedInteger(N->getOperand(0), InL, InH); 4807 SmallVector<SDValue> Result; 4808 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) { 4809 Lo = Result[0]; 4810 Hi = Result[1]; 4811 return; 4812 } 4813 } 4814 } 4815 4816 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4817 if (VT == MVT::i16) 4818 LC = RTLIB::UDIV_I16; 4819 else if (VT == MVT::i32) 4820 LC = RTLIB::UDIV_I32; 4821 else if (VT == MVT::i64) 4822 LC = RTLIB::UDIV_I64; 4823 else if (VT == MVT::i128) 4824 LC = RTLIB::UDIV_I128; 4825 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 4826 4827 TargetLowering::MakeLibCallOptions CallOptions; 4828 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4829 } 4830 4831 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 4832 SDValue &Lo, SDValue &Hi) { 4833 EVT VT = N->getValueType(0); 4834 SDLoc dl(N); 4835 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4836 4837 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4838 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4839 SplitInteger(Res.getValue(1), Lo, Hi); 4840 return; 4841 } 4842 4843 // Try to expand UREM by constant. 4844 if (isa<ConstantSDNode>(N->getOperand(1))) { 4845 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4846 // Only if the new type is legal. 4847 if (isTypeLegal(NVT)) { 4848 SDValue InL, InH; 4849 GetExpandedInteger(N->getOperand(0), InL, InH); 4850 SmallVector<SDValue> Result; 4851 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) { 4852 Lo = Result[0]; 4853 Hi = Result[1]; 4854 return; 4855 } 4856 } 4857 } 4858 4859 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4860 if (VT == MVT::i16) 4861 LC = RTLIB::UREM_I16; 4862 else if (VT == MVT::i32) 4863 LC = RTLIB::UREM_I32; 4864 else if (VT == MVT::i64) 4865 LC = RTLIB::UREM_I64; 4866 else if (VT == MVT::i128) 4867 LC = RTLIB::UREM_I128; 4868 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 4869 4870 TargetLowering::MakeLibCallOptions CallOptions; 4871 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4872 } 4873 4874 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 4875 SDValue &Lo, SDValue &Hi) { 4876 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4877 SDLoc dl(N); 4878 SDValue Op = N->getOperand(0); 4879 if (Op.getValueType().bitsLE(NVT)) { 4880 // The low part is zero extension of the input (degenerates to a copy). 4881 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 4882 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. 4883 } else { 4884 // For example, extension of an i48 to an i64. The operand type necessarily 4885 // promotes to the result type, so will end up being expanded too. 4886 assert(getTypeAction(Op.getValueType()) == 4887 TargetLowering::TypePromoteInteger && 4888 "Only know how to promote this result!"); 4889 SDValue Res = GetPromotedInteger(Op); 4890 assert(Res.getValueType() == N->getValueType(0) && 4891 "Operand over promoted?"); 4892 // Split the promoted operand. This will simplify when it is expanded. 4893 SplitInteger(Res, Lo, Hi); 4894 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 4895 Hi = DAG.getZeroExtendInReg(Hi, dl, 4896 EVT::getIntegerVT(*DAG.getContext(), 4897 ExcessBits)); 4898 } 4899 } 4900 4901 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 4902 SDValue &Lo, SDValue &Hi) { 4903 SDLoc dl(N); 4904 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 4905 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 4906 SDValue Zero = DAG.getConstant(0, dl, VT); 4907 SDValue Swap = DAG.getAtomicCmpSwap( 4908 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 4909 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 4910 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); 4911 4912 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 4913 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 4914 } 4915 4916 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, 4917 SDValue &Lo, SDValue &Hi) { 4918 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate 4919 // both halves independently. 4920 SDValue Res = TLI.expandVecReduce(N, DAG); 4921 SplitInteger(Res, Lo, Hi); 4922 } 4923 4924 void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, 4925 SDValue &Lo, SDValue &Hi) { 4926 // Delegate to funnel-shift expansion. 4927 SDLoc DL(N); 4928 unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR; 4929 SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0), 4930 N->getOperand(0), N->getOperand(1)); 4931 SplitInteger(Res, Lo, Hi); 4932 } 4933 4934 void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo, 4935 SDValue &Hi) { 4936 // Values numbered from least significant to most significant. 4937 SDValue In1, In2, In3, In4; 4938 GetExpandedInteger(N->getOperand(0), In3, In4); 4939 GetExpandedInteger(N->getOperand(1), In1, In2); 4940 EVT HalfVT = In1.getValueType(); 4941 4942 SDLoc DL(N); 4943 unsigned Opc = N->getOpcode(); 4944 SDValue ShAmt = N->getOperand(2); 4945 EVT ShAmtVT = ShAmt.getValueType(); 4946 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT); 4947 4948 // If the shift amount is at least half the bitwidth, swap the inputs. 4949 unsigned HalfVTBits = HalfVT.getScalarSizeInBits(); 4950 SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt, 4951 DAG.getConstant(HalfVTBits, DL, ShAmtVT)); 4952 SDValue Cond = 4953 DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT), 4954 Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ); 4955 4956 // Expand to a pair of funnel shifts. 4957 EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout()); 4958 SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT); 4959 4960 SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2); 4961 SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3); 4962 SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4); 4963 Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt); 4964 Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt); 4965 } 4966 4967 void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, 4968 SDValue &Hi) { 4969 EVT VT = N->getValueType(0); 4970 EVT HalfVT = 4971 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2); 4972 SDLoc dl(N); 4973 4974 // We assume VSCALE(1) fits into a legal integer. 4975 APInt One(HalfVT.getSizeInBits(), 1); 4976 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One); 4977 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase); 4978 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0)); 4979 SplitInteger(Res, Lo, Hi); 4980 } 4981 4982 //===----------------------------------------------------------------------===// 4983 // Integer Operand Expansion 4984 //===----------------------------------------------------------------------===// 4985 4986 /// ExpandIntegerOperand - This method is called when the specified operand of 4987 /// the specified node is found to need expansion. At this point, all of the 4988 /// result types of the node are known to be legal, but other operands of the 4989 /// node may need promotion or expansion as well as the specified one. 4990 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 4991 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); 4992 dbgs() << "\n"); 4993 SDValue Res = SDValue(); 4994 4995 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 4996 return false; 4997 4998 switch (N->getOpcode()) { 4999 default: 5000 #ifndef NDEBUG 5001 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 5002 N->dump(&DAG); dbgs() << "\n"; 5003 #endif 5004 report_fatal_error("Do not know how to expand this operator's operand!"); 5005 5006 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 5007 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 5008 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 5009 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 5010 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 5011 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 5012 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break; 5013 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 5014 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 5015 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; 5016 case ISD::STRICT_SINT_TO_FP: 5017 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 5018 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 5019 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 5020 case ISD::STRICT_UINT_TO_FP: 5021 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 5022 5023 case ISD::SHL: 5024 case ISD::SRA: 5025 case ISD::SRL: 5026 case ISD::ROTL: 5027 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 5028 case ISD::RETURNADDR: 5029 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 5030 5031 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 5032 case ISD::STACKMAP: 5033 Res = ExpandIntOp_STACKMAP(N, OpNo); 5034 break; 5035 case ISD::PATCHPOINT: 5036 Res = ExpandIntOp_PATCHPOINT(N, OpNo); 5037 break; 5038 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 5039 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 5040 Res = ExpandIntOp_VP_STRIDED(N, OpNo); 5041 break; 5042 } 5043 5044 // If the result is null, the sub-method took care of registering results etc. 5045 if (!Res.getNode()) return false; 5046 5047 // If the result is N, the sub-method updated N in place. Tell the legalizer 5048 // core about this. 5049 if (Res.getNode() == N) 5050 return true; 5051 5052 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 5053 "Invalid operand expansion"); 5054 5055 ReplaceValueWith(SDValue(N, 0), Res); 5056 return false; 5057 } 5058 5059 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 5060 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 5061 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 5062 SDValue &NewRHS, 5063 ISD::CondCode &CCCode, 5064 const SDLoc &dl) { 5065 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 5066 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 5067 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 5068 5069 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 5070 if (RHSLo == RHSHi) { 5071 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 5072 if (RHSCST->isAllOnes()) { 5073 // Equality comparison to -1. 5074 NewLHS = DAG.getNode(ISD::AND, dl, 5075 LHSLo.getValueType(), LHSLo, LHSHi); 5076 NewRHS = RHSLo; 5077 return; 5078 } 5079 } 5080 } 5081 5082 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 5083 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 5084 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 5085 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); 5086 return; 5087 } 5088 5089 // If this is a comparison of the sign bit, just look at the top part. 5090 // X > -1, x < 0 5091 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 5092 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0 5093 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1 5094 NewLHS = LHSHi; 5095 NewRHS = RHSHi; 5096 return; 5097 } 5098 5099 // FIXME: This generated code sucks. 5100 ISD::CondCode LowCC; 5101 switch (CCCode) { 5102 default: llvm_unreachable("Unknown integer setcc!"); 5103 case ISD::SETLT: 5104 case ISD::SETULT: LowCC = ISD::SETULT; break; 5105 case ISD::SETGT: 5106 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 5107 case ISD::SETLE: 5108 case ISD::SETULE: LowCC = ISD::SETULE; break; 5109 case ISD::SETGE: 5110 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 5111 } 5112 5113 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison 5114 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands 5115 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; 5116 5117 // NOTE: on targets without efficient SELECT of bools, we can always use 5118 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 5119 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 5120 nullptr); 5121 SDValue LoCmp, HiCmp; 5122 if (TLI.isTypeLegal(LHSLo.getValueType()) && 5123 TLI.isTypeLegal(RHSLo.getValueType())) 5124 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, 5125 RHSLo, LowCC, false, DagCombineInfo, dl); 5126 if (!LoCmp.getNode()) 5127 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 5128 RHSLo, LowCC); 5129 if (TLI.isTypeLegal(LHSHi.getValueType()) && 5130 TLI.isTypeLegal(RHSHi.getValueType())) 5131 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, 5132 RHSHi, CCCode, false, DagCombineInfo, dl); 5133 if (!HiCmp.getNode()) 5134 HiCmp = 5135 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), 5136 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 5137 5138 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); 5139 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); 5140 5141 bool EqAllowed = ISD::isTrueWhenEqual(CCCode); 5142 5143 // FIXME: Is the HiCmpC->isOne() here correct for 5144 // ZeroOrNegativeOneBooleanContent. 5145 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) || 5146 (!EqAllowed && 5147 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) { 5148 // For LE / GE, if high part is known false, ignore the low part. 5149 // For LT / GT: if low part is known false, return the high part. 5150 // if high part is known true, ignore the low part. 5151 NewLHS = HiCmp; 5152 NewRHS = SDValue(); 5153 return; 5154 } 5155 5156 if (LHSHi == RHSHi) { 5157 // Comparing the low bits is enough. 5158 NewLHS = LoCmp; 5159 NewRHS = SDValue(); 5160 return; 5161 } 5162 5163 // Lower with SETCCCARRY if the target supports it. 5164 EVT HiVT = LHSHi.getValueType(); 5165 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); 5166 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); 5167 5168 // FIXME: Make all targets support this, then remove the other lowering. 5169 if (HasSETCCCARRY) { 5170 // SETCCCARRY can detect < and >= directly. For > and <=, flip 5171 // operands and condition code. 5172 bool FlipOperands = false; 5173 switch (CCCode) { 5174 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; 5175 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; 5176 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; 5177 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; 5178 default: break; 5179 } 5180 if (FlipOperands) { 5181 std::swap(LHSLo, RHSLo); 5182 std::swap(LHSHi, RHSHi); 5183 } 5184 // Perform a wide subtraction, feeding the carry from the low part into 5185 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high 5186 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is 5187 // zero or positive iff LHS >= RHS. 5188 EVT LoVT = LHSLo.getValueType(); 5189 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT)); 5190 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo); 5191 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT), 5192 LHSHi, RHSHi, LowCmp.getValue(1), 5193 DAG.getCondCode(CCCode)); 5194 NewLHS = Res; 5195 NewRHS = SDValue(); 5196 return; 5197 } 5198 5199 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, 5200 false, DagCombineInfo, dl); 5201 if (!NewLHS.getNode()) 5202 NewLHS = 5203 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); 5204 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); 5205 NewRHS = SDValue(); 5206 } 5207 5208 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 5209 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 5210 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 5211 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 5212 5213 // If ExpandSetCCOperands returned a scalar, we need to compare the result 5214 // against zero to select between true and false values. 5215 if (!NewRHS.getNode()) { 5216 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 5217 CCCode = ISD::SETNE; 5218 } 5219 5220 // Update N to have the operands specified. 5221 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 5222 DAG.getCondCode(CCCode), NewLHS, NewRHS, 5223 N->getOperand(4)), 0); 5224 } 5225 5226 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 5227 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 5228 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 5229 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 5230 5231 // If ExpandSetCCOperands returned a scalar, we need to compare the result 5232 // against zero to select between true and false values. 5233 if (!NewRHS.getNode()) { 5234 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 5235 CCCode = ISD::SETNE; 5236 } 5237 5238 // Update N to have the operands specified. 5239 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 5240 N->getOperand(2), N->getOperand(3), 5241 DAG.getCondCode(CCCode)), 0); 5242 } 5243 5244 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 5245 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 5246 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 5247 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 5248 5249 // If ExpandSetCCOperands returned a scalar, use it. 5250 if (!NewRHS.getNode()) { 5251 assert(NewLHS.getValueType() == N->getValueType(0) && 5252 "Unexpected setcc expansion!"); 5253 return NewLHS; 5254 } 5255 5256 // Otherwise, update N to have the operands specified. 5257 return SDValue( 5258 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); 5259 } 5260 5261 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { 5262 SDValue LHS = N->getOperand(0); 5263 SDValue RHS = N->getOperand(1); 5264 SDValue Carry = N->getOperand(2); 5265 SDValue Cond = N->getOperand(3); 5266 SDLoc dl = SDLoc(N); 5267 5268 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 5269 GetExpandedInteger(LHS, LHSLo, LHSHi); 5270 GetExpandedInteger(RHS, RHSLo, RHSHi); 5271 5272 // Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high. 5273 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); 5274 SDValue LowCmp = 5275 DAG.getNode(ISD::USUBO_CARRY, dl, VTList, LHSLo, RHSLo, Carry); 5276 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, 5277 LowCmp.getValue(1), Cond); 5278 } 5279 5280 SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) { 5281 // Split the operand and replace with SPLAT_VECTOR_PARTS. 5282 SDValue Lo, Hi; 5283 GetExpandedInteger(N->getOperand(0), Lo, Hi); 5284 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo, 5285 Hi); 5286 } 5287 5288 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 5289 // The value being shifted is legal, but the shift amount is too big. 5290 // It follows that either the result of the shift is undefined, or the 5291 // upper half of the shift amount is zero. Just use the lower half. 5292 SDValue Lo, Hi; 5293 GetExpandedInteger(N->getOperand(1), Lo, Hi); 5294 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 5295 } 5296 5297 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 5298 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 5299 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 5300 // constant to valid type. 5301 SDValue Lo, Hi; 5302 GetExpandedInteger(N->getOperand(0), Lo, Hi); 5303 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 5304 } 5305 5306 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 5307 bool IsStrict = N->isStrictFPOpcode(); 5308 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 5309 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 5310 EVT DstVT = N->getValueType(0); 5311 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 5312 assert(LC != RTLIB::UNKNOWN_LIBCALL && 5313 "Don't know how to expand this SINT_TO_FP!"); 5314 TargetLowering::MakeLibCallOptions CallOptions; 5315 CallOptions.setSExt(true); 5316 std::pair<SDValue, SDValue> Tmp = 5317 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 5318 5319 if (!IsStrict) 5320 return Tmp.first; 5321 5322 ReplaceValueWith(SDValue(N, 1), Tmp.second); 5323 ReplaceValueWith(SDValue(N, 0), Tmp.first); 5324 return SDValue(); 5325 } 5326 5327 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 5328 if (N->isAtomic()) { 5329 // It's typical to have larger CAS than atomic store instructions. 5330 SDLoc dl(N); 5331 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 5332 N->getMemoryVT(), 5333 N->getOperand(0), N->getOperand(2), 5334 N->getOperand(1), 5335 N->getMemOperand()); 5336 return Swap.getValue(1); 5337 } 5338 if (ISD::isNormalStore(N)) 5339 return ExpandOp_NormalStore(N, OpNo); 5340 5341 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 5342 assert(OpNo == 1 && "Can only expand the stored value so far"); 5343 5344 EVT VT = N->getOperand(1).getValueType(); 5345 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5346 SDValue Ch = N->getChain(); 5347 SDValue Ptr = N->getBasePtr(); 5348 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 5349 AAMDNodes AAInfo = N->getAAInfo(); 5350 SDLoc dl(N); 5351 SDValue Lo, Hi; 5352 5353 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 5354 5355 if (N->getMemoryVT().bitsLE(NVT)) { 5356 GetExpandedInteger(N->getValue(), Lo, Hi); 5357 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 5358 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags, 5359 AAInfo); 5360 } 5361 5362 if (DAG.getDataLayout().isLittleEndian()) { 5363 // Little-endian - low bits are at low addresses. 5364 GetExpandedInteger(N->getValue(), Lo, Hi); 5365 5366 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 5367 N->getOriginalAlign(), MMOFlags, AAInfo); 5368 5369 unsigned ExcessBits = 5370 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 5371 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 5372 5373 // Increment the pointer to the other half. 5374 unsigned IncrementSize = NVT.getSizeInBits()/8; 5375 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 5376 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 5377 N->getPointerInfo().getWithOffset(IncrementSize), 5378 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo); 5379 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 5380 } 5381 5382 // Big-endian - high bits are at low addresses. Favor aligned stores at 5383 // the cost of some bit-fiddling. 5384 GetExpandedInteger(N->getValue(), Lo, Hi); 5385 5386 EVT ExtVT = N->getMemoryVT(); 5387 unsigned EBytes = ExtVT.getStoreSize(); 5388 unsigned IncrementSize = NVT.getSizeInBits()/8; 5389 unsigned ExcessBits = (EBytes - IncrementSize)*8; 5390 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 5391 ExtVT.getSizeInBits() - ExcessBits); 5392 5393 if (ExcessBits < NVT.getSizeInBits()) { 5394 // Transfer high bits from the top of Lo to the bottom of Hi. 5395 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 5396 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 5397 TLI.getPointerTy(DAG.getDataLayout()))); 5398 Hi = DAG.getNode( 5399 ISD::OR, dl, NVT, Hi, 5400 DAG.getNode(ISD::SRL, dl, NVT, Lo, 5401 DAG.getConstant(ExcessBits, dl, 5402 TLI.getPointerTy(DAG.getDataLayout())))); 5403 } 5404 5405 // Store both the high bits and maybe some of the low bits. 5406 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, 5407 N->getOriginalAlign(), MMOFlags, AAInfo); 5408 5409 // Increment the pointer to the other half. 5410 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 5411 // Store the lowest ExcessBits bits in the second half. 5412 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 5413 N->getPointerInfo().getWithOffset(IncrementSize), 5414 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 5415 N->getOriginalAlign(), MMOFlags, AAInfo); 5416 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 5417 } 5418 5419 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 5420 SDValue InL, InH; 5421 GetExpandedInteger(N->getOperand(0), InL, InH); 5422 // Just truncate the low part of the source. 5423 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 5424 } 5425 5426 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 5427 bool IsStrict = N->isStrictFPOpcode(); 5428 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 5429 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 5430 EVT DstVT = N->getValueType(0); 5431 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT); 5432 assert(LC != RTLIB::UNKNOWN_LIBCALL && 5433 "Don't know how to expand this UINT_TO_FP!"); 5434 TargetLowering::MakeLibCallOptions CallOptions; 5435 CallOptions.setSExt(true); 5436 std::pair<SDValue, SDValue> Tmp = 5437 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 5438 5439 if (!IsStrict) 5440 return Tmp.first; 5441 5442 ReplaceValueWith(SDValue(N, 1), Tmp.second); 5443 ReplaceValueWith(SDValue(N, 0), Tmp.first); 5444 return SDValue(); 5445 } 5446 5447 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 5448 SDLoc dl(N); 5449 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 5450 cast<AtomicSDNode>(N)->getMemoryVT(), 5451 N->getOperand(0), 5452 N->getOperand(1), N->getOperand(2), 5453 cast<AtomicSDNode>(N)->getMemOperand()); 5454 return Swap.getValue(1); 5455 } 5456 5457 SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { 5458 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || 5459 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); 5460 5461 SDValue Hi; // The upper half is dropped out. 5462 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); 5463 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo], Hi); 5464 5465 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 5466 } 5467 5468 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { 5469 SDLoc dl(N); 5470 5471 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5472 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5473 EVT OutVT = V0.getValueType(); 5474 5475 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2)); 5476 } 5477 5478 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) { 5479 SDLoc dl(N); 5480 5481 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5482 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5483 EVT ResVT = V0.getValueType(); 5484 SDValue Res = DAG.getNode(N->getOpcode(), dl, 5485 DAG.getVTList(ResVT, ResVT), V0, V1); 5486 SetPromotedInteger(SDValue(N, 0), Res.getValue(0)); 5487 SetPromotedInteger(SDValue(N, 1), Res.getValue(1)); 5488 return SDValue(); 5489 } 5490 5491 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 5492 5493 EVT OutVT = N->getValueType(0); 5494 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5495 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5496 EVT NOutVTElem = NOutVT.getVectorElementType(); 5497 5498 SDLoc dl(N); 5499 SDValue BaseIdx = N->getOperand(1); 5500 5501 // TODO: We may be able to use this for types other than scalable 5502 // vectors and fix those tests that expect BUILD_VECTOR to be used 5503 if (OutVT.isScalableVector()) { 5504 SDValue InOp0 = N->getOperand(0); 5505 EVT InVT = InOp0.getValueType(); 5506 5507 // Try and extract from a smaller type so that it eventually falls 5508 // into the promotion code below. 5509 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector || 5510 getTypeAction(InVT) == TargetLowering::TypeLegal) { 5511 EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext()); 5512 unsigned NElts = NInVT.getVectorMinNumElements(); 5513 uint64_t IdxVal = cast<ConstantSDNode>(BaseIdx)->getZExtValue(); 5514 5515 SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0, 5516 DAG.getConstant(alignDown(IdxVal, NElts), dl, 5517 BaseIdx.getValueType())); 5518 SDValue Step2 = DAG.getNode( 5519 ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1, 5520 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType())); 5521 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2); 5522 } 5523 5524 // Try and extract from a widened type. 5525 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 5526 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx}; 5527 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops); 5528 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); 5529 } 5530 5531 // Promote operands and see if this is handled by target lowering, 5532 // Otherwise, use the BUILD_VECTOR approach below 5533 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 5534 // Collect the (promoted) operands 5535 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx }; 5536 5537 EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); 5538 assert(PromEltVT.bitsLE(NOutVTElem) && 5539 "Promoted operand has an element type greater than result"); 5540 5541 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT); 5542 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops); 5543 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); 5544 } 5545 } 5546 5547 if (OutVT.isScalableVector()) 5548 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR"); 5549 5550 SDValue InOp0 = N->getOperand(0); 5551 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) 5552 InOp0 = GetPromotedInteger(N->getOperand(0)); 5553 5554 EVT InVT = InOp0.getValueType(); 5555 5556 unsigned OutNumElems = OutVT.getVectorNumElements(); 5557 SmallVector<SDValue, 8> Ops; 5558 Ops.reserve(OutNumElems); 5559 for (unsigned i = 0; i != OutNumElems; ++i) { 5560 5561 // Extract the element from the original vector. 5562 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 5563 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); 5564 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 5565 InVT.getVectorElementType(), N->getOperand(0), Index); 5566 5567 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); 5568 // Insert the converted element to the new vector. 5569 Ops.push_back(Op); 5570 } 5571 5572 return DAG.getBuildVector(NOutVT, dl, Ops); 5573 } 5574 5575 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) { 5576 EVT OutVT = N->getValueType(0); 5577 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5578 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5579 5580 SDLoc dl(N); 5581 SDValue Vec = N->getOperand(0); 5582 SDValue SubVec = N->getOperand(1); 5583 SDValue Idx = N->getOperand(2); 5584 5585 EVT SubVecVT = SubVec.getValueType(); 5586 EVT NSubVT = 5587 EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(), 5588 SubVecVT.getVectorElementCount()); 5589 5590 Vec = GetPromotedInteger(Vec); 5591 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec); 5592 5593 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx); 5594 } 5595 5596 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) { 5597 SDLoc dl(N); 5598 5599 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5600 EVT OutVT = V0.getValueType(); 5601 5602 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0); 5603 } 5604 5605 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 5606 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 5607 EVT VT = N->getValueType(0); 5608 SDLoc dl(N); 5609 5610 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); 5611 5612 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5613 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5614 EVT OutVT = V0.getValueType(); 5615 5616 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); 5617 } 5618 5619 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 5620 EVT OutVT = N->getValueType(0); 5621 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5622 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5623 unsigned NumElems = N->getNumOperands(); 5624 EVT NOutVTElem = NOutVT.getVectorElementType(); 5625 TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(NOutVT); 5626 unsigned NOutExtOpc = TargetLowering::getExtendForContent(NOutBoolType); 5627 SDLoc dl(N); 5628 5629 SmallVector<SDValue, 8> Ops; 5630 Ops.reserve(NumElems); 5631 for (unsigned i = 0; i != NumElems; ++i) { 5632 SDValue Op = N->getOperand(i); 5633 EVT OpVT = Op.getValueType(); 5634 // BUILD_VECTOR integer operand types are allowed to be larger than the 5635 // result's element type. This may still be true after the promotion. For 5636 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 5637 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 5638 if (OpVT.bitsLT(NOutVTElem)) { 5639 unsigned ExtOpc = ISD::ANY_EXTEND; 5640 // Attempt to extend constant bool vectors to match target's BooleanContent. 5641 // While not necessary, this improves chances of the constant correctly 5642 // folding with compare results (e.g. for NOT patterns). 5643 if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant) 5644 ExtOpc = NOutExtOpc; 5645 Op = DAG.getNode(ExtOpc, dl, NOutVTElem, Op); 5646 } 5647 Ops.push_back(Op); 5648 } 5649 5650 return DAG.getBuildVector(NOutVT, dl, Ops); 5651 } 5652 5653 SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) { 5654 5655 SDLoc dl(N); 5656 5657 assert(!N->getOperand(0).getValueType().isVector() && 5658 "Input must be a scalar"); 5659 5660 EVT OutVT = N->getValueType(0); 5661 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5662 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5663 EVT NOutElemVT = NOutVT.getVectorElementType(); 5664 5665 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0)); 5666 5667 return DAG.getNode(N->getOpcode(), dl, NOutVT, Op); 5668 } 5669 5670 SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) { 5671 SDLoc dl(N); 5672 EVT OutVT = N->getValueType(0); 5673 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5674 assert(NOutVT.isScalableVector() && 5675 "Type must be promoted to a scalable vector type"); 5676 const APInt &StepVal = N->getConstantOperandAPInt(0); 5677 return DAG.getStepVector(dl, NOutVT, 5678 StepVal.sext(NOutVT.getScalarSizeInBits())); 5679 } 5680 5681 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 5682 SDLoc dl(N); 5683 5684 EVT OutVT = N->getValueType(0); 5685 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5686 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5687 5688 unsigned NumOperands = N->getNumOperands(); 5689 unsigned NumOutElem = NOutVT.getVectorMinNumElements(); 5690 EVT OutElemTy = NOutVT.getVectorElementType(); 5691 if (OutVT.isScalableVector()) { 5692 // Find the largest promoted element type for each of the operands. 5693 SDUse *MaxSizedValue = std::max_element( 5694 N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) { 5695 EVT AVT = A.getValueType().getVectorElementType(); 5696 EVT BVT = B.getValueType().getVectorElementType(); 5697 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits(); 5698 }); 5699 EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType(); 5700 5701 // Then promote all vectors to the largest element type. 5702 SmallVector<SDValue, 8> Ops; 5703 for (unsigned I = 0; I < NumOperands; ++I) { 5704 SDValue Op = N->getOperand(I); 5705 EVT OpVT = Op.getValueType(); 5706 if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger) 5707 Op = GetPromotedInteger(Op); 5708 else 5709 assert(getTypeAction(OpVT) == TargetLowering::TypeLegal && 5710 "Unhandled legalization type"); 5711 5712 if (OpVT.getVectorElementType().getScalarSizeInBits() < 5713 MaxElementVT.getScalarSizeInBits()) 5714 Op = DAG.getAnyExtOrTrunc(Op, dl, 5715 OpVT.changeVectorElementType(MaxElementVT)); 5716 Ops.push_back(Op); 5717 } 5718 5719 // Do the CONCAT on the promoted type and finally truncate to (the promoted) 5720 // NOutVT. 5721 return DAG.getAnyExtOrTrunc( 5722 DAG.getNode(ISD::CONCAT_VECTORS, dl, 5723 OutVT.changeVectorElementType(MaxElementVT), Ops), 5724 dl, NOutVT); 5725 } 5726 5727 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 5728 assert(NumElem * NumOperands == NumOutElem && 5729 "Unexpected number of elements"); 5730 5731 // Take the elements from the first vector. 5732 SmallVector<SDValue, 8> Ops(NumOutElem); 5733 for (unsigned i = 0; i < NumOperands; ++i) { 5734 SDValue Op = N->getOperand(i); 5735 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) 5736 Op = GetPromotedInteger(Op); 5737 EVT SclrTy = Op.getValueType().getVectorElementType(); 5738 assert(NumElem == Op.getValueType().getVectorNumElements() && 5739 "Unexpected number of elements"); 5740 5741 for (unsigned j = 0; j < NumElem; ++j) { 5742 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, 5743 DAG.getVectorIdxConstant(j, dl)); 5744 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); 5745 } 5746 } 5747 5748 return DAG.getBuildVector(NOutVT, dl, Ops); 5749 } 5750 5751 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { 5752 EVT VT = N->getValueType(0); 5753 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5754 assert(NVT.isVector() && "This type must be promoted to a vector type"); 5755 5756 SDLoc dl(N); 5757 5758 // For operands whose TypeAction is to promote, extend the promoted node 5759 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion 5760 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to 5761 // type.. 5762 if (getTypeAction(N->getOperand(0).getValueType()) 5763 == TargetLowering::TypePromoteInteger) { 5764 SDValue Promoted; 5765 5766 switch(N->getOpcode()) { 5767 case ISD::SIGN_EXTEND_VECTOR_INREG: 5768 Promoted = SExtPromotedInteger(N->getOperand(0)); 5769 break; 5770 case ISD::ZERO_EXTEND_VECTOR_INREG: 5771 Promoted = ZExtPromotedInteger(N->getOperand(0)); 5772 break; 5773 case ISD::ANY_EXTEND_VECTOR_INREG: 5774 Promoted = GetPromotedInteger(N->getOperand(0)); 5775 break; 5776 default: 5777 llvm_unreachable("Node has unexpected Opcode"); 5778 } 5779 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); 5780 } 5781 5782 // Directly extend to the appropriate transform-to type. 5783 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 5784 } 5785 5786 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 5787 EVT OutVT = N->getValueType(0); 5788 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5789 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5790 5791 EVT NOutVTElem = NOutVT.getVectorElementType(); 5792 5793 SDLoc dl(N); 5794 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5795 5796 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 5797 NOutVTElem, N->getOperand(1)); 5798 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 5799 V0, ConvElem, N->getOperand(2)); 5800 } 5801 5802 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { 5803 // The VECREDUCE result size may be larger than the element size, so 5804 // we can simply change the result type. 5805 SDLoc dl(N); 5806 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5807 return DAG.getNode(N->getOpcode(), dl, NVT, N->ops()); 5808 } 5809 5810 SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) { 5811 // The VP_REDUCE result size may be larger than the element size, so we can 5812 // simply change the result type. However the start value and result must be 5813 // the same. 5814 SDLoc DL(N); 5815 SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0)); 5816 return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start, 5817 N->getOperand(1), N->getOperand(2), N->getOperand(3)); 5818 } 5819 5820 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 5821 SDLoc dl(N); 5822 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5823 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, 5824 TLI.getVectorIdxTy(DAG.getDataLayout())); 5825 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 5826 V0->getValueType(0).getScalarType(), V0, V1); 5827 5828 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 5829 // element types. If this is the case then we need to expand the outgoing 5830 // value and not truncate it. 5831 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 5832 } 5833 5834 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) { 5835 SDLoc dl(N); 5836 // The result type is equal to the first input operand's type, so the 5837 // type that needs promoting must be the second source vector. 5838 SDValue V0 = N->getOperand(0); 5839 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5840 SDValue Idx = N->getOperand(2); 5841 EVT PromVT = EVT::getVectorVT(*DAG.getContext(), 5842 V1.getValueType().getVectorElementType(), 5843 V0.getValueType().getVectorElementCount()); 5844 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT); 5845 SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx); 5846 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 5847 } 5848 5849 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { 5850 SDLoc dl(N); 5851 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5852 MVT InVT = V0.getValueType().getSimpleVT(); 5853 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), 5854 N->getValueType(0).getVectorNumElements()); 5855 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); 5856 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 5857 } 5858 5859 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 5860 SDLoc dl(N); 5861 5862 EVT ResVT = N->getValueType(0); 5863 unsigned NumElems = N->getNumOperands(); 5864 5865 if (ResVT.isScalableVector()) { 5866 SDValue ResVec = DAG.getUNDEF(ResVT); 5867 5868 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { 5869 SDValue Op = N->getOperand(OpIdx); 5870 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); 5871 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op, 5872 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl)); 5873 } 5874 5875 return ResVec; 5876 } 5877 5878 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 5879 5880 SmallVector<SDValue, 8> NewOps; 5881 NewOps.reserve(NumElems); 5882 5883 // For each incoming vector 5884 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 5885 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 5886 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 5887 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 5888 5889 for (unsigned i=0; i<NumElem; ++i) { 5890 // Extract element from incoming vector 5891 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, 5892 DAG.getVectorIdxConstant(i, dl)); 5893 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 5894 NewOps.push_back(Tr); 5895 } 5896 } 5897 5898 return DAG.getBuildVector(N->getValueType(0), dl, NewOps); 5899 } 5900 5901 SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) { 5902 assert(OpNo > 1); 5903 SDValue Op = N->getOperand(OpNo); 5904 5905 // FIXME: Non-constant operands are not yet handled: 5906 // - https://github.com/llvm/llvm-project/issues/26431 5907 // - https://github.com/llvm/llvm-project/issues/55957 5908 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op); 5909 if (!CN) 5910 return SDValue(); 5911 5912 // Copy operands before the one being expanded. 5913 SmallVector<SDValue> NewOps; 5914 for (unsigned I = 0; I < OpNo; I++) 5915 NewOps.push_back(N->getOperand(I)); 5916 5917 EVT Ty = Op.getValueType(); 5918 SDLoc DL = SDLoc(N); 5919 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { 5920 NewOps.push_back( 5921 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); 5922 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); 5923 } else { 5924 // FIXME: https://github.com/llvm/llvm-project/issues/55609 5925 return SDValue(); 5926 } 5927 5928 // Copy remaining operands. 5929 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) 5930 NewOps.push_back(N->getOperand(I)); 5931 5932 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); 5933 5934 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 5935 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 5936 5937 return SDValue(); // Signal that we have replaced the node already. 5938 } 5939 5940 SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { 5941 assert(OpNo >= 7); 5942 SDValue Op = N->getOperand(OpNo); 5943 5944 // FIXME: Non-constant operands are not yet handled: 5945 // - https://github.com/llvm/llvm-project/issues/26431 5946 // - https://github.com/llvm/llvm-project/issues/55957 5947 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op); 5948 if (!CN) 5949 return SDValue(); 5950 5951 // Copy operands before the one being expanded. 5952 SmallVector<SDValue> NewOps; 5953 for (unsigned I = 0; I < OpNo; I++) 5954 NewOps.push_back(N->getOperand(I)); 5955 5956 EVT Ty = Op.getValueType(); 5957 SDLoc DL = SDLoc(N); 5958 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { 5959 NewOps.push_back( 5960 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); 5961 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); 5962 } else { 5963 // FIXME: https://github.com/llvm/llvm-project/issues/55609 5964 return SDValue(); 5965 } 5966 5967 // Copy remaining operands. 5968 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) 5969 NewOps.push_back(N->getOperand(I)); 5970 5971 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); 5972 5973 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 5974 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 5975 5976 return SDValue(); // Signal that we have replaced the node already. 5977 } 5978