1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements integer type expansion and promotion for LegalizeTypes. 10 // Promotion is the act of changing a computation in an illegal type into a 11 // computation in a larger type. For example, implementing i8 arithmetic in an 12 // i32 register (often needed on powerpc). 13 // Expansion is the act of changing a computation in an illegal type into a 14 // computation in two identical registers of a smaller type. For example, 15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit 16 // targets). 17 // 18 //===----------------------------------------------------------------------===// 19 20 #include "LegalizeTypes.h" 21 #include "llvm/Analysis/TargetLibraryInfo.h" 22 #include "llvm/CodeGen/StackMaps.h" 23 #include "llvm/CodeGen/TargetLowering.h" 24 #include "llvm/IR/DerivedTypes.h" 25 #include "llvm/Support/ErrorHandling.h" 26 #include "llvm/Support/KnownBits.h" 27 #include "llvm/Support/raw_ostream.h" 28 #include <algorithm> 29 using namespace llvm; 30 31 #define DEBUG_TYPE "legalize-types" 32 33 //===----------------------------------------------------------------------===// 34 // Integer Result Promotion 35 //===----------------------------------------------------------------------===// 36 37 /// PromoteIntegerResult - This method is called when a result of a node is 38 /// found to be in need of promotion to a larger type. At this point, the node 39 /// may also have invalid operands or may have other results that need 40 /// expansion, we just know that (at least) one result needs promotion. 41 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 42 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG)); 43 SDValue Res = SDValue(); 44 45 // See if the target wants to custom expand this node. 46 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 47 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 48 return; 49 } 50 51 switch (N->getOpcode()) { 52 default: 53 #ifndef NDEBUG 54 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 55 N->dump(&DAG); dbgs() << "\n"; 56 #endif 57 report_fatal_error("Do not know how to promote this operator!"); 58 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 59 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 60 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 61 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 62 case ISD::VP_BITREVERSE: 63 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; 64 case ISD::VP_BSWAP: 65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 66 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 67 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 68 case ISD::VP_CTLZ_ZERO_UNDEF: 69 case ISD::VP_CTLZ: 70 case ISD::CTLZ_ZERO_UNDEF: 71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 72 case ISD::PARITY: 73 case ISD::VP_CTPOP: 74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; 75 case ISD::VP_CTTZ_ZERO_UNDEF: 76 case ISD::VP_CTTZ: 77 case ISD::CTTZ_ZERO_UNDEF: 78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 79 case ISD::EXTRACT_VECTOR_ELT: 80 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 81 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 82 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); 83 break; 84 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); 85 break; 86 case ISD::SELECT: 87 case ISD::VSELECT: 88 case ISD::VP_SELECT: 89 case ISD::VP_MERGE: 90 Res = PromoteIntRes_Select(N); 91 break; 92 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 93 case ISD::STRICT_FSETCC: 94 case ISD::STRICT_FSETCCS: 95 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 96 case ISD::SMIN: 97 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; 98 case ISD::UMIN: 99 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; 100 101 case ISD::SHL: 102 case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break; 103 case ISD::SIGN_EXTEND_INREG: 104 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 105 case ISD::SRA: 106 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break; 107 case ISD::SRL: 108 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break; 109 case ISD::VP_TRUNCATE: 110 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 111 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 112 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 113 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; 114 115 case ISD::EXTRACT_SUBVECTOR: 116 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 117 case ISD::INSERT_SUBVECTOR: 118 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break; 119 case ISD::VECTOR_REVERSE: 120 Res = PromoteIntRes_VECTOR_REVERSE(N); break; 121 case ISD::VECTOR_SHUFFLE: 122 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 123 case ISD::VECTOR_SPLICE: 124 Res = PromoteIntRes_VECTOR_SPLICE(N); break; 125 case ISD::VECTOR_INTERLEAVE: 126 case ISD::VECTOR_DEINTERLEAVE: 127 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N); 128 return; 129 case ISD::INSERT_VECTOR_ELT: 130 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 131 case ISD::BUILD_VECTOR: 132 Res = PromoteIntRes_BUILD_VECTOR(N); 133 break; 134 case ISD::SPLAT_VECTOR: 135 case ISD::SCALAR_TO_VECTOR: 136 Res = PromoteIntRes_ScalarOp(N); 137 break; 138 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break; 139 case ISD::CONCAT_VECTORS: 140 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 141 142 case ISD::ANY_EXTEND_VECTOR_INREG: 143 case ISD::SIGN_EXTEND_VECTOR_INREG: 144 case ISD::ZERO_EXTEND_VECTOR_INREG: 145 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; 146 147 case ISD::SIGN_EXTEND: 148 case ISD::VP_SIGN_EXTEND: 149 case ISD::ZERO_EXTEND: 150 case ISD::VP_ZERO_EXTEND: 151 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 152 153 case ISD::VP_FP_TO_SINT: 154 case ISD::VP_FP_TO_UINT: 155 case ISD::STRICT_FP_TO_SINT: 156 case ISD::STRICT_FP_TO_UINT: 157 case ISD::FP_TO_SINT: 158 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 159 160 case ISD::FP_TO_SINT_SAT: 161 case ISD::FP_TO_UINT_SAT: 162 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; 163 164 case ISD::FP_TO_BF16: 165 case ISD::FP_TO_FP16: 166 Res = PromoteIntRes_FP_TO_FP16_BF16(N); 167 break; 168 case ISD::STRICT_FP_TO_FP16: 169 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N); 170 break; 171 case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break; 172 173 case ISD::AND: 174 case ISD::OR: 175 case ISD::XOR: 176 case ISD::ADD: 177 case ISD::SUB: 178 case ISD::MUL: 179 case ISD::VP_AND: 180 case ISD::VP_OR: 181 case ISD::VP_XOR: 182 case ISD::VP_ADD: 183 case ISD::VP_SUB: 184 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 185 186 case ISD::VP_SMIN: 187 case ISD::VP_SMAX: 188 case ISD::SDIV: 189 case ISD::SREM: 190 case ISD::VP_SDIV: 191 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; 192 193 case ISD::VP_UMIN: 194 case ISD::VP_UMAX: 195 case ISD::UDIV: 196 case ISD::UREM: 197 case ISD::VP_UDIV: 198 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; 199 200 case ISD::SADDO: 201 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 202 case ISD::UADDO: 203 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 204 case ISD::SMULO: 205 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 206 207 case ISD::ADDE: 208 case ISD::SUBE: 209 case ISD::UADDO_CARRY: 210 case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break; 211 212 case ISD::SADDO_CARRY: 213 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; 214 215 case ISD::SADDSAT: 216 case ISD::UADDSAT: 217 case ISD::SSUBSAT: 218 case ISD::USUBSAT: 219 case ISD::SSHLSAT: 220 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break; 221 222 case ISD::SMULFIX: 223 case ISD::SMULFIXSAT: 224 case ISD::UMULFIX: 225 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; 226 227 case ISD::SDIVFIX: 228 case ISD::SDIVFIXSAT: 229 case ISD::UDIVFIX: 230 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; 231 232 case ISD::ABS: Res = PromoteIntRes_ABS(N); break; 233 234 case ISD::ATOMIC_LOAD: 235 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 236 237 case ISD::ATOMIC_LOAD_ADD: 238 case ISD::ATOMIC_LOAD_SUB: 239 case ISD::ATOMIC_LOAD_AND: 240 case ISD::ATOMIC_LOAD_CLR: 241 case ISD::ATOMIC_LOAD_OR: 242 case ISD::ATOMIC_LOAD_XOR: 243 case ISD::ATOMIC_LOAD_NAND: 244 case ISD::ATOMIC_LOAD_MIN: 245 case ISD::ATOMIC_LOAD_MAX: 246 case ISD::ATOMIC_LOAD_UMIN: 247 case ISD::ATOMIC_LOAD_UMAX: 248 case ISD::ATOMIC_SWAP: 249 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 250 251 case ISD::ATOMIC_CMP_SWAP: 252 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 253 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 254 break; 255 256 case ISD::VECREDUCE_ADD: 257 case ISD::VECREDUCE_MUL: 258 case ISD::VECREDUCE_AND: 259 case ISD::VECREDUCE_OR: 260 case ISD::VECREDUCE_XOR: 261 case ISD::VECREDUCE_SMAX: 262 case ISD::VECREDUCE_SMIN: 263 case ISD::VECREDUCE_UMAX: 264 case ISD::VECREDUCE_UMIN: 265 Res = PromoteIntRes_VECREDUCE(N); 266 break; 267 268 case ISD::VP_REDUCE_ADD: 269 case ISD::VP_REDUCE_MUL: 270 case ISD::VP_REDUCE_AND: 271 case ISD::VP_REDUCE_OR: 272 case ISD::VP_REDUCE_XOR: 273 case ISD::VP_REDUCE_SMAX: 274 case ISD::VP_REDUCE_SMIN: 275 case ISD::VP_REDUCE_UMAX: 276 case ISD::VP_REDUCE_UMIN: 277 Res = PromoteIntRes_VP_REDUCE(N); 278 break; 279 280 case ISD::FREEZE: 281 Res = PromoteIntRes_FREEZE(N); 282 break; 283 284 case ISD::ROTL: 285 case ISD::ROTR: 286 Res = PromoteIntRes_Rotate(N); 287 break; 288 289 case ISD::FSHL: 290 case ISD::FSHR: 291 Res = PromoteIntRes_FunnelShift(N); 292 break; 293 294 case ISD::VP_FSHL: 295 case ISD::VP_FSHR: 296 Res = PromoteIntRes_VPFunnelShift(N); 297 break; 298 299 case ISD::IS_FPCLASS: 300 Res = PromoteIntRes_IS_FPCLASS(N); 301 break; 302 case ISD::FFREXP: 303 Res = PromoteIntRes_FFREXP(N); 304 break; 305 306 case ISD::LRINT: 307 case ISD::LLRINT: 308 Res = PromoteIntRes_XRINT(N); 309 break; 310 } 311 312 // If the result is null then the sub-method took care of registering it. 313 if (Res.getNode()) 314 SetPromotedInteger(SDValue(N, ResNo), Res); 315 } 316 317 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 318 unsigned ResNo) { 319 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 320 return GetPromotedInteger(Op); 321 } 322 323 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 324 // Sign-extend the new bits, and continue the assertion. 325 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 326 return DAG.getNode(ISD::AssertSext, SDLoc(N), 327 Op.getValueType(), Op, N->getOperand(1)); 328 } 329 330 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 331 // Zero the new bits, and continue the assertion. 332 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 333 return DAG.getNode(ISD::AssertZext, SDLoc(N), 334 Op.getValueType(), Op, N->getOperand(1)); 335 } 336 337 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 338 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 339 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 340 N->getMemoryVT(), ResVT, 341 N->getChain(), N->getBasePtr(), 342 N->getMemOperand()); 343 // Legalize the chain result - switch anything that used the old chain to 344 // use the new one. 345 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 346 return Res; 347 } 348 349 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 350 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 351 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 352 N->getMemoryVT(), 353 N->getChain(), N->getBasePtr(), 354 Op2, N->getMemOperand()); 355 // Legalize the chain result - switch anything that used the old chain to 356 // use the new one. 357 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 358 return Res; 359 } 360 361 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 362 unsigned ResNo) { 363 if (ResNo == 1) { 364 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 365 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 366 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 367 368 // Only use the result of getSetCCResultType if it is legal, 369 // otherwise just use the promoted result type (NVT). 370 if (!TLI.isTypeLegal(SVT)) 371 SVT = NVT; 372 373 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 374 SDValue Res = DAG.getAtomicCmpSwap( 375 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 376 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 377 N->getMemOperand()); 378 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 379 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 380 return DAG.getSExtOrTrunc(Res.getValue(1), SDLoc(N), NVT); 381 } 382 383 // Op2 is used for the comparison and thus must be extended according to the 384 // target's atomic operations. Op3 is merely stored and so can be left alone. 385 SDValue Op2 = N->getOperand(2); 386 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 387 switch (TLI.getExtendForAtomicCmpSwapArg()) { 388 case ISD::SIGN_EXTEND: 389 Op2 = SExtPromotedInteger(Op2); 390 break; 391 case ISD::ZERO_EXTEND: 392 Op2 = ZExtPromotedInteger(Op2); 393 break; 394 case ISD::ANY_EXTEND: 395 Op2 = GetPromotedInteger(Op2); 396 break; 397 default: 398 llvm_unreachable("Invalid atomic op extension"); 399 } 400 401 SDVTList VTs = 402 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 403 SDValue Res = DAG.getAtomicCmpSwap( 404 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 405 N->getBasePtr(), Op2, Op3, N->getMemOperand()); 406 // Update the use to N with the newly created Res. 407 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) 408 ReplaceValueWith(SDValue(N, i), Res.getValue(i)); 409 return Res; 410 } 411 412 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 413 SDValue InOp = N->getOperand(0); 414 EVT InVT = InOp.getValueType(); 415 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 416 EVT OutVT = N->getValueType(0); 417 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 418 SDLoc dl(N); 419 420 switch (getTypeAction(InVT)) { 421 case TargetLowering::TypeLegal: 422 break; 423 case TargetLowering::TypePromoteInteger: 424 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 425 // The input promotes to the same size. Convert the promoted value. 426 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 427 break; 428 case TargetLowering::TypeSoftenFloat: 429 // Promote the integer operand by hand. 430 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 431 case TargetLowering::TypeSoftPromoteHalf: 432 // Promote the integer operand by hand. 433 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp)); 434 case TargetLowering::TypePromoteFloat: { 435 // Convert the promoted float by hand. 436 if (!NOutVT.isVector()) 437 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); 438 break; 439 } 440 case TargetLowering::TypeExpandInteger: 441 case TargetLowering::TypeExpandFloat: 442 break; 443 case TargetLowering::TypeScalarizeVector: 444 // Convert the element to an integer and promote it by hand. 445 if (!NOutVT.isVector()) 446 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 447 BitConvertToInteger(GetScalarizedVector(InOp))); 448 break; 449 case TargetLowering::TypeScalarizeScalableVector: 450 report_fatal_error("Scalarization of scalable vectors is not supported."); 451 case TargetLowering::TypeSplitVector: { 452 if (!NOutVT.isVector()) { 453 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 454 // pieces of the input into integers and reassemble in the final type. 455 SDValue Lo, Hi; 456 GetSplitVector(N->getOperand(0), Lo, Hi); 457 Lo = BitConvertToInteger(Lo); 458 Hi = BitConvertToInteger(Hi); 459 460 if (DAG.getDataLayout().isBigEndian()) 461 std::swap(Lo, Hi); 462 463 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 464 EVT::getIntegerVT(*DAG.getContext(), 465 NOutVT.getSizeInBits()), 466 JoinIntegers(Lo, Hi)); 467 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 468 } 469 break; 470 } 471 case TargetLowering::TypeWidenVector: 472 // The input is widened to the same size. Convert to the widened value. 473 // Make sure that the outgoing value is not a vector, because this would 474 // make us bitcast between two vectors which are legalized in different ways. 475 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) { 476 SDValue Res = 477 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 478 479 // For big endian targets we need to shift the casted value or the 480 // interesting bits will end up at the wrong place. 481 if (DAG.getDataLayout().isBigEndian()) { 482 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 483 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!"); 484 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res, 485 DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl)); 486 } 487 return Res; 488 } 489 // If the output type is also a vector and widening it to the same size 490 // as the widened input type would be a legal type, we can widen the bitcast 491 // and handle the promotion after. 492 if (NOutVT.isVector()) { 493 TypeSize WidenInSize = NInVT.getSizeInBits(); 494 TypeSize OutSize = OutVT.getSizeInBits(); 495 if (WidenInSize.hasKnownScalarFactor(OutSize)) { 496 unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize); 497 EVT WideOutVT = 498 EVT::getVectorVT(*DAG.getContext(), OutVT.getVectorElementType(), 499 OutVT.getVectorElementCount() * Scale); 500 if (isTypeLegal(WideOutVT)) { 501 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); 502 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, 503 DAG.getVectorIdxConstant(0, dl)); 504 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp); 505 } 506 } 507 } 508 } 509 510 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 511 CreateStackStoreLoad(InOp, OutVT)); 512 } 513 514 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { 515 SDValue V = GetPromotedInteger(N->getOperand(0)); 516 return DAG.getNode(ISD::FREEZE, SDLoc(N), 517 V.getValueType(), V); 518 } 519 520 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 521 SDValue Op = GetPromotedInteger(N->getOperand(0)); 522 EVT OVT = N->getValueType(0); 523 EVT NVT = Op.getValueType(); 524 SDLoc dl(N); 525 526 // If the larger BSWAP isn't supported by the target, try to expand now. 527 // If we expand later we'll end up with more operations since we lost the 528 // original type. We only do this for scalars since we have a shuffle 529 // based lowering for vectors in LegalizeVectorOps. 530 if (!OVT.isVector() && 531 !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) { 532 if (SDValue Res = TLI.expandBSWAP(N, DAG)) 533 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); 534 } 535 536 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 537 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl); 538 if (N->getOpcode() == ISD::BSWAP) 539 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 540 ShAmt); 541 SDValue Mask = N->getOperand(1); 542 SDValue EVL = N->getOperand(2); 543 return DAG.getNode(ISD::VP_LSHR, dl, NVT, 544 DAG.getNode(ISD::VP_BSWAP, dl, NVT, Op, Mask, EVL), ShAmt, 545 Mask, EVL); 546 } 547 548 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { 549 SDValue Op = GetPromotedInteger(N->getOperand(0)); 550 EVT OVT = N->getValueType(0); 551 EVT NVT = Op.getValueType(); 552 SDLoc dl(N); 553 554 // If the larger BITREVERSE isn't supported by the target, try to expand now. 555 // If we expand later we'll end up with more operations since we lost the 556 // original type. We only do this for scalars since we have a shuffle 557 // based lowering for vectors in LegalizeVectorOps. 558 if (!OVT.isVector() && OVT.isSimple() && 559 !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) { 560 if (SDValue Res = TLI.expandBITREVERSE(N, DAG)) 561 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); 562 } 563 564 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 565 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl); 566 if (N->getOpcode() == ISD::BITREVERSE) 567 return DAG.getNode(ISD::SRL, dl, NVT, 568 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), ShAmt); 569 SDValue Mask = N->getOperand(1); 570 SDValue EVL = N->getOperand(2); 571 return DAG.getNode(ISD::VP_LSHR, dl, NVT, 572 DAG.getNode(ISD::VP_BITREVERSE, dl, NVT, Op, Mask, EVL), 573 ShAmt, Mask, EVL); 574 } 575 576 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 577 // The pair element type may be legal, or may not promote to the same type as 578 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 579 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 580 TLI.getTypeToTransformTo(*DAG.getContext(), 581 N->getValueType(0)), JoinIntegers(N->getOperand(0), 582 N->getOperand(1))); 583 } 584 585 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 586 EVT VT = N->getValueType(0); 587 // FIXME there is no actual debug info here 588 SDLoc dl(N); 589 // Zero extend things like i1, sign extend everything else. It shouldn't 590 // matter in theory which one we pick, but this tends to give better code? 591 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 592 SDValue Result = DAG.getNode(Opc, dl, 593 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 594 SDValue(N, 0)); 595 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 596 return Result; 597 } 598 599 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 600 EVT OVT = N->getValueType(0); 601 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 602 SDLoc dl(N); 603 604 // If the larger CTLZ isn't supported by the target, try to expand now. 605 // If we expand later we'll end up with more operations since we lost the 606 // original type. 607 if (!OVT.isVector() && TLI.isTypeLegal(NVT) && 608 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ, NVT) && 609 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ_ZERO_UNDEF, NVT)) { 610 if (SDValue Result = TLI.expandCTLZ(N, DAG)) { 611 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); 612 return Result; 613 } 614 } 615 616 // Zero extend to the promoted type and do the count there. 617 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 618 619 // Subtract off the extra leading bits in the bigger type. 620 SDValue ExtractLeadingBits = DAG.getConstant( 621 NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, NVT); 622 if (!N->isVPOpcode()) 623 return DAG.getNode(ISD::SUB, dl, NVT, 624 DAG.getNode(N->getOpcode(), dl, NVT, Op), 625 ExtractLeadingBits); 626 SDValue Mask = N->getOperand(1); 627 SDValue EVL = N->getOperand(2); 628 return DAG.getNode(ISD::VP_SUB, dl, NVT, 629 DAG.getNode(N->getOpcode(), dl, NVT, Op, Mask, EVL), 630 ExtractLeadingBits, Mask, EVL); 631 } 632 633 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { 634 EVT OVT = N->getValueType(0); 635 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 636 637 // If the larger CTPOP isn't supported by the target, try to expand now. 638 // If we expand later we'll end up with more operations since we lost the 639 // original type. 640 // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to 641 // TargetLowering. 642 if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) && 643 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTPOP, NVT)) { 644 if (SDValue Result = TLI.expandCTPOP(N, DAG)) { 645 Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result); 646 return Result; 647 } 648 } 649 650 // Zero extend to the promoted type and do the count or parity there. 651 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 652 if (!N->isVPOpcode()) 653 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op); 654 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op, 655 N->getOperand(1), N->getOperand(2)); 656 } 657 658 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 659 SDValue Op = GetPromotedInteger(N->getOperand(0)); 660 EVT OVT = N->getValueType(0); 661 EVT NVT = Op.getValueType(); 662 SDLoc dl(N); 663 664 // If the larger CTTZ isn't supported by the target, try to expand now. 665 // If we expand later we'll end up with more operations since we lost the 666 // original type. Don't expand if we can use CTPOP or CTLZ expansion on the 667 // larger type. 668 if (!OVT.isVector() && TLI.isTypeLegal(NVT) && 669 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ, NVT) && 670 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ_ZERO_UNDEF, NVT) && 671 !TLI.isOperationLegal(ISD::CTPOP, NVT) && 672 !TLI.isOperationLegal(ISD::CTLZ, NVT)) { 673 if (SDValue Result = TLI.expandCTTZ(N, DAG)) { 674 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); 675 return Result; 676 } 677 } 678 679 if (N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::VP_CTTZ) { 680 // The count is the same in the promoted type except if the original 681 // value was zero. This can be handled by setting the bit just off 682 // the top of the original type. 683 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), 684 OVT.getScalarSizeInBits()); 685 if (N->getOpcode() == ISD::CTTZ) 686 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); 687 else 688 Op = 689 DAG.getNode(ISD::VP_OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT), 690 N->getOperand(1), N->getOperand(2)); 691 } 692 if (!N->isVPOpcode()) 693 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 694 return DAG.getNode(N->getOpcode(), dl, NVT, Op, N->getOperand(1), 695 N->getOperand(2)); 696 } 697 698 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 699 SDLoc dl(N); 700 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 701 702 SDValue Op0 = N->getOperand(0); 703 SDValue Op1 = N->getOperand(1); 704 705 // If the input also needs to be promoted, do that first so we can get a 706 // get a good idea for the output type. 707 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType()) 708 == TargetLowering::TypePromoteInteger) { 709 SDValue In = GetPromotedInteger(Op0); 710 711 // If the new type is larger than NVT, use it. We probably won't need to 712 // promote it again. 713 EVT SVT = In.getValueType().getScalarType(); 714 if (SVT.bitsGE(NVT)) { 715 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1); 716 return DAG.getAnyExtOrTrunc(Ext, dl, NVT); 717 } 718 } 719 720 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1); 721 } 722 723 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 725 unsigned NewOpc = N->getOpcode(); 726 SDLoc dl(N); 727 728 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 729 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 730 // and SINT conversions are Custom, there is no way to tell which is 731 // preferable. We choose SINT because that's the right thing on PPC.) 732 if (N->getOpcode() == ISD::FP_TO_UINT && 733 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 734 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 735 NewOpc = ISD::FP_TO_SINT; 736 737 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && 738 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) && 739 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) 740 NewOpc = ISD::STRICT_FP_TO_SINT; 741 742 if (N->getOpcode() == ISD::VP_FP_TO_UINT && 743 !TLI.isOperationLegal(ISD::VP_FP_TO_UINT, NVT) && 744 TLI.isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, NVT)) 745 NewOpc = ISD::VP_FP_TO_SINT; 746 747 SDValue Res; 748 if (N->isStrictFPOpcode()) { 749 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, 750 {N->getOperand(0), N->getOperand(1)}); 751 // Legalize the chain result - switch anything that used the old chain to 752 // use the new one. 753 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 754 } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) { 755 Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1), 756 N->getOperand(2)}); 757 } else { 758 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 759 } 760 761 // Assert that the converted value fits in the original type. If it doesn't 762 // (eg: because the value being converted is too big), then the result of the 763 // original operation was undefined anyway, so the assert is still correct. 764 // 765 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: 766 // before legalization: fp-to-uint16, 65534. -> 0xfffe 767 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe 768 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT || 769 N->getOpcode() == ISD::STRICT_FP_TO_UINT || 770 N->getOpcode() == ISD::VP_FP_TO_UINT) 771 ? ISD::AssertZext 772 : ISD::AssertSext, 773 dl, NVT, Res, 774 DAG.getValueType(N->getValueType(0).getScalarType())); 775 } 776 777 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { 778 // Promote the result type, while keeping the original width in Op1. 779 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 780 SDLoc dl(N); 781 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), 782 N->getOperand(1)); 783 } 784 785 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) { 786 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 787 SDLoc dl(N); 788 789 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 790 } 791 792 SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) { 793 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 794 SDLoc dl(N); 795 796 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), 797 N->getOperand(0), N->getOperand(1)); 798 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 799 return Res; 800 } 801 802 SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) { 803 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 804 SDLoc dl(N); 805 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 806 } 807 808 SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) { 809 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 810 SDLoc dl(N); 811 812 SDValue Res = 813 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); 814 815 // Legalize the chain result - switch anything that used the old chain to 816 // use the new one. 817 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 818 return Res; 819 } 820 821 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 822 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 823 SDLoc dl(N); 824 825 if (getTypeAction(N->getOperand(0).getValueType()) 826 == TargetLowering::TypePromoteInteger) { 827 SDValue Res = GetPromotedInteger(N->getOperand(0)); 828 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 829 830 // If the result and operand types are the same after promotion, simplify 831 // to an in-register extension. Unless this is a VP_*_EXTEND. 832 if (NVT == Res.getValueType() && N->getNumOperands() == 1) { 833 // The high bits are not guaranteed to be anything. Insert an extend. 834 if (N->getOpcode() == ISD::SIGN_EXTEND) 835 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 836 DAG.getValueType(N->getOperand(0).getValueType())); 837 if (N->getOpcode() == ISD::ZERO_EXTEND) 838 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); 839 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 840 return Res; 841 } 842 } 843 844 // Otherwise, just extend the original operand all the way to the larger type. 845 if (N->getNumOperands() != 1) { 846 assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); 847 assert(N->isVPOpcode() && "Expected VP opcode"); 848 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), 849 N->getOperand(1), N->getOperand(2)); 850 } 851 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 852 } 853 854 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 855 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 856 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 857 ISD::LoadExtType ExtType = 858 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 859 SDLoc dl(N); 860 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 861 N->getMemoryVT(), N->getMemOperand()); 862 863 // Legalize the chain result - switch anything that used the old chain to 864 // use the new one. 865 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 866 return Res; 867 } 868 869 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { 870 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 871 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 872 873 ISD::LoadExtType ExtType = N->getExtensionType(); 874 if (ExtType == ISD::NON_EXTLOAD) 875 ExtType = ISD::EXTLOAD; 876 877 SDLoc dl(N); 878 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), 879 N->getOffset(), N->getMask(), ExtPassThru, 880 N->getMemoryVT(), N->getMemOperand(), 881 N->getAddressingMode(), ExtType, 882 N->isExpandingLoad()); 883 // Legalize the chain result - switch anything that used the old chain to 884 // use the new one. 885 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 886 return Res; 887 } 888 889 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { 890 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 891 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 892 assert(NVT == ExtPassThru.getValueType() && 893 "Gather result type and the passThru argument type should be the same"); 894 895 ISD::LoadExtType ExtType = N->getExtensionType(); 896 if (ExtType == ISD::NON_EXTLOAD) 897 ExtType = ISD::EXTLOAD; 898 899 SDLoc dl(N); 900 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), 901 N->getIndex(), N->getScale() }; 902 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), 903 N->getMemoryVT(), dl, Ops, 904 N->getMemOperand(), N->getIndexType(), 905 ExtType); 906 // Legalize the chain result - switch anything that used the old chain to 907 // use the new one. 908 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 909 return Res; 910 } 911 912 /// Promote the overflow flag of an overflowing arithmetic node. 913 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 914 // Change the return type of the boolean result while obeying 915 // getSetCCResultType. 916 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 917 EVT VT = N->getValueType(0); 918 EVT SVT = getSetCCResultType(VT); 919 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; 920 unsigned NumOps = N->getNumOperands(); 921 assert(NumOps <= 3 && "Too many operands"); 922 if (NumOps == 3) 923 Ops[2] = N->getOperand(2); 924 925 SDLoc dl(N); 926 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), 927 ArrayRef(Ops, NumOps)); 928 929 // Modified the sum result - switch anything that used the old sum to use 930 // the new one. 931 ReplaceValueWith(SDValue(N, 0), Res); 932 933 // Convert to the expected type. 934 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); 935 } 936 937 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { 938 // If the promoted type is legal, we can convert this to: 939 // 1. ANY_EXTEND iN to iM 940 // 2. SHL by M-N 941 // 3. [US][ADD|SUB|SHL]SAT 942 // 4. L/ASHR by M-N 943 // Else it is more efficient to convert this to a min and a max 944 // operation in the higher precision arithmetic. 945 SDLoc dl(N); 946 SDValue Op1 = N->getOperand(0); 947 SDValue Op2 = N->getOperand(1); 948 unsigned OldBits = Op1.getScalarValueSizeInBits(); 949 950 unsigned Opcode = N->getOpcode(); 951 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; 952 953 SDValue Op1Promoted, Op2Promoted; 954 if (IsShift) { 955 Op1Promoted = GetPromotedInteger(Op1); 956 Op2Promoted = ZExtPromotedInteger(Op2); 957 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { 958 Op1Promoted = ZExtPromotedInteger(Op1); 959 Op2Promoted = ZExtPromotedInteger(Op2); 960 } else { 961 Op1Promoted = SExtPromotedInteger(Op1); 962 Op2Promoted = SExtPromotedInteger(Op2); 963 } 964 EVT PromotedType = Op1Promoted.getValueType(); 965 unsigned NewBits = PromotedType.getScalarSizeInBits(); 966 967 if (Opcode == ISD::UADDSAT) { 968 APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits); 969 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 970 SDValue Add = 971 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); 972 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); 973 } 974 975 // USUBSAT can always be promoted as long as we have zero-extended the args. 976 if (Opcode == ISD::USUBSAT) 977 return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted, 978 Op2Promoted); 979 980 // Shift cannot use a min/max expansion, we can't detect overflow if all of 981 // the bits have been shifted out. 982 if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) { 983 unsigned ShiftOp; 984 switch (Opcode) { 985 case ISD::SADDSAT: 986 case ISD::SSUBSAT: 987 case ISD::SSHLSAT: 988 ShiftOp = ISD::SRA; 989 break; 990 case ISD::USHLSAT: 991 ShiftOp = ISD::SRL; 992 break; 993 default: 994 llvm_unreachable("Expected opcode to be signed or unsigned saturation " 995 "addition, subtraction or left shift"); 996 } 997 998 unsigned SHLAmount = NewBits - OldBits; 999 SDValue ShiftAmount = 1000 DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl); 1001 Op1Promoted = 1002 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); 1003 if (!IsShift) 1004 Op2Promoted = 1005 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); 1006 1007 SDValue Result = 1008 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); 1009 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); 1010 } 1011 1012 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; 1013 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits); 1014 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits); 1015 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType); 1016 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 1017 SDValue Result = 1018 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); 1019 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); 1020 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); 1021 return Result; 1022 } 1023 1024 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { 1025 // Can just promote the operands then continue with operation. 1026 SDLoc dl(N); 1027 SDValue Op1Promoted, Op2Promoted; 1028 bool Signed = 1029 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; 1030 bool Saturating = 1031 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; 1032 if (Signed) { 1033 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 1034 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 1035 } else { 1036 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 1037 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 1038 } 1039 EVT OldType = N->getOperand(0).getValueType(); 1040 EVT PromotedType = Op1Promoted.getValueType(); 1041 unsigned DiffSize = 1042 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); 1043 1044 if (Saturating) { 1045 // Promoting the operand and result values changes the saturation width, 1046 // which is extends the values that we clamp to on saturation. This could be 1047 // resolved by shifting one of the operands the same amount, which would 1048 // also shift the result we compare against, then shifting back. 1049 Op1Promoted = 1050 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 1051 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); 1052 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 1053 Op2Promoted, N->getOperand(2)); 1054 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; 1055 return DAG.getNode(ShiftOp, dl, PromotedType, Result, 1056 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); 1057 } 1058 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, 1059 N->getOperand(2)); 1060 } 1061 1062 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, 1063 unsigned SatW, bool Signed, 1064 const TargetLowering &TLI, 1065 SelectionDAG &DAG) { 1066 EVT VT = V.getValueType(); 1067 unsigned VTW = VT.getScalarSizeInBits(); 1068 1069 if (!Signed) { 1070 // Saturate to the unsigned maximum by getting the minimum of V and the 1071 // maximum. 1072 return DAG.getNode(ISD::UMIN, dl, VT, V, 1073 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW), 1074 dl, VT)); 1075 } 1076 1077 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the 1078 // signed minimum of it and V. 1079 V = DAG.getNode(ISD::SMIN, dl, VT, V, 1080 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1), 1081 dl, VT)); 1082 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the 1083 // signed maximum of it and V. 1084 V = DAG.getNode(ISD::SMAX, dl, VT, V, 1085 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1), 1086 dl, VT)); 1087 return V; 1088 } 1089 1090 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, 1091 unsigned Scale, const TargetLowering &TLI, 1092 SelectionDAG &DAG, unsigned SatW = 0) { 1093 EVT VT = LHS.getValueType(); 1094 unsigned VTSize = VT.getScalarSizeInBits(); 1095 bool Signed = N->getOpcode() == ISD::SDIVFIX || 1096 N->getOpcode() == ISD::SDIVFIXSAT; 1097 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 1098 N->getOpcode() == ISD::UDIVFIXSAT; 1099 1100 SDLoc dl(N); 1101 // Widen the types by a factor of two. This is guaranteed to expand, since it 1102 // will always have enough high bits in the LHS to shift into. 1103 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2); 1104 if (VT.isVector()) 1105 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, 1106 VT.getVectorElementCount()); 1107 LHS = DAG.getExtOrTrunc(Signed, LHS, dl, WideVT); 1108 RHS = DAG.getExtOrTrunc(Signed, RHS, dl, WideVT); 1109 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, 1110 DAG); 1111 assert(Res && "Expanding DIVFIX with wide type failed?"); 1112 if (Saturating) { 1113 // If the caller has told us to saturate at something less, use that width 1114 // instead of the type before doubling. However, it cannot be more than 1115 // what we just widened! 1116 assert(SatW <= VTSize && 1117 "Tried to saturate to more than the original type?"); 1118 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed, 1119 TLI, DAG); 1120 } 1121 return DAG.getZExtOrTrunc(Res, dl, VT); 1122 } 1123 1124 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { 1125 SDLoc dl(N); 1126 SDValue Op1Promoted, Op2Promoted; 1127 bool Signed = N->getOpcode() == ISD::SDIVFIX || 1128 N->getOpcode() == ISD::SDIVFIXSAT; 1129 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 1130 N->getOpcode() == ISD::UDIVFIXSAT; 1131 if (Signed) { 1132 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 1133 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 1134 } else { 1135 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 1136 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 1137 } 1138 EVT PromotedType = Op1Promoted.getValueType(); 1139 unsigned Scale = N->getConstantOperandVal(2); 1140 1141 // If the type is already legal and the operation is legal in that type, we 1142 // should not early expand. 1143 if (TLI.isTypeLegal(PromotedType)) { 1144 TargetLowering::LegalizeAction Action = 1145 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale); 1146 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { 1147 unsigned Diff = PromotedType.getScalarSizeInBits() - 1148 N->getValueType(0).getScalarSizeInBits(); 1149 if (Saturating) 1150 Op1Promoted = 1151 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 1152 DAG.getShiftAmountConstant(Diff, PromotedType, dl)); 1153 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 1154 Op2Promoted, N->getOperand(2)); 1155 if (Saturating) 1156 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res, 1157 DAG.getShiftAmountConstant(Diff, PromotedType, dl)); 1158 return Res; 1159 } 1160 } 1161 1162 // See if we can perform the division in this type without expanding. 1163 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted, 1164 Op2Promoted, Scale, DAG)) { 1165 if (Saturating) 1166 Res = SaturateWidenedDIVFIX(Res, dl, 1167 N->getValueType(0).getScalarSizeInBits(), 1168 Signed, TLI, DAG); 1169 return Res; 1170 } 1171 // If we cannot, expand it to twice the type width. If we are saturating, give 1172 // it the original width as a saturating width so we don't need to emit 1173 // two saturations. 1174 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG, 1175 N->getValueType(0).getScalarSizeInBits()); 1176 } 1177 1178 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 1179 if (ResNo == 1) 1180 return PromoteIntRes_Overflow(N); 1181 1182 // The operation overflowed iff the result in the larger type is not the 1183 // sign extension of its truncation to the original type. 1184 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1185 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1186 EVT OVT = N->getOperand(0).getValueType(); 1187 EVT NVT = LHS.getValueType(); 1188 SDLoc dl(N); 1189 1190 // Do the arithmetic in the larger type. 1191 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 1192 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1193 1194 // Calculate the overflow flag: sign extend the arithmetic result from 1195 // the original type. 1196 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 1197 DAG.getValueType(OVT)); 1198 // Overflowed if and only if this is not equal to Res. 1199 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1200 1201 // Use the calculated overflow everywhere. 1202 ReplaceValueWith(SDValue(N, 1), Ofl); 1203 1204 return Res; 1205 } 1206 1207 SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) { 1208 SDValue Mask = N->getOperand(0); 1209 1210 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 1211 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 1212 1213 unsigned Opcode = N->getOpcode(); 1214 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) 1215 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS, 1216 N->getOperand(3)); 1217 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS); 1218 } 1219 1220 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 1221 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 1222 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 1223 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 1224 LHS.getValueType(), N->getOperand(0), 1225 N->getOperand(1), LHS, RHS, N->getOperand(4)); 1226 } 1227 1228 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 1229 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 1230 EVT InVT = N->getOperand(OpNo).getValueType(); 1231 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1232 1233 EVT SVT = getSetCCResultType(InVT); 1234 1235 // If we got back a type that needs to be promoted, this likely means the 1236 // the input type also needs to be promoted. So get the promoted type for 1237 // the input and try the query again. 1238 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { 1239 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 1240 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 1241 SVT = getSetCCResultType(InVT); 1242 } else { 1243 // Input type isn't promoted, just use the default promoted type. 1244 SVT = NVT; 1245 } 1246 } 1247 1248 SDLoc dl(N); 1249 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && 1250 "Vector compare must return a vector result!"); 1251 1252 // Get the SETCC result using the canonical SETCC type. 1253 SDValue SetCC; 1254 if (N->isStrictFPOpcode()) { 1255 SDVTList VTs = DAG.getVTList({SVT, MVT::Other}); 1256 SDValue Opers[] = {N->getOperand(0), N->getOperand(1), 1257 N->getOperand(2), N->getOperand(3)}; 1258 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags()); 1259 // Legalize the chain result - switch anything that used the old chain to 1260 // use the new one. 1261 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1)); 1262 } else 1263 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), 1264 N->getOperand(1), N->getOperand(2), N->getFlags()); 1265 1266 // Convert to the expected type. 1267 return DAG.getSExtOrTrunc(SetCC, dl, NVT); 1268 } 1269 1270 SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) { 1271 SDLoc DL(N); 1272 SDValue Arg = N->getOperand(0); 1273 SDValue Test = N->getOperand(1); 1274 EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1275 return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test); 1276 } 1277 1278 SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) { 1279 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 1280 EVT VT = N->getValueType(0); 1281 1282 SDLoc dl(N); 1283 SDValue Res = 1284 DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, NVT), N->getOperand(0)); 1285 1286 ReplaceValueWith(SDValue(N, 0), Res); 1287 return Res.getValue(1); 1288 } 1289 1290 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 1291 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1292 SDValue RHS = N->getOperand(1); 1293 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1294 RHS = ZExtPromotedInteger(RHS); 1295 if (N->getOpcode() != ISD::VP_SHL) 1296 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1297 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1298 N->getOperand(2), N->getOperand(3)); 1299 } 1300 1301 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 1302 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1303 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 1304 Op.getValueType(), Op, N->getOperand(1)); 1305 } 1306 1307 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 1308 // The input may have strange things in the top bits of the registers, but 1309 // these operations don't care. They may have weird bits going out, but 1310 // that too is okay if they are integer operations. 1311 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1312 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 1313 if (N->getNumOperands() == 2) 1314 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1315 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1316 assert(N->isVPOpcode() && "Expected VP opcode"); 1317 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1318 N->getOperand(2), N->getOperand(3)); 1319 } 1320 1321 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { 1322 // Sign extend the input. 1323 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1324 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1325 if (N->getNumOperands() == 2) 1326 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1327 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1328 assert(N->isVPOpcode() && "Expected VP opcode"); 1329 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1330 N->getOperand(2), N->getOperand(3)); 1331 } 1332 1333 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { 1334 // Zero extend the input. 1335 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1336 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1337 if (N->getNumOperands() == 2) 1338 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1339 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1340 assert(N->isVPOpcode() && "Expected VP opcode"); 1341 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1342 N->getOperand(2), N->getOperand(3)); 1343 } 1344 1345 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { 1346 // It doesn't matter if we sign extend or zero extend in the inputs. So do 1347 // whatever is best for the target. 1348 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0)); 1349 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1)); 1350 return DAG.getNode(N->getOpcode(), SDLoc(N), 1351 LHS.getValueType(), LHS, RHS); 1352 } 1353 1354 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 1355 // The input value must be properly sign extended. 1356 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1357 SDValue RHS = N->getOperand(1); 1358 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1359 RHS = ZExtPromotedInteger(RHS); 1360 if (N->getOpcode() != ISD::VP_ASHR) 1361 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1362 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1363 N->getOperand(2), N->getOperand(3)); 1364 } 1365 1366 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 1367 // The input value must be properly zero extended. 1368 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1369 SDValue RHS = N->getOperand(1); 1370 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1371 RHS = ZExtPromotedInteger(RHS); 1372 if (N->getOpcode() != ISD::VP_LSHR) 1373 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1374 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1375 N->getOperand(2), N->getOperand(3)); 1376 } 1377 1378 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { 1379 // Lower the rotate to shifts and ORs which can be promoted. 1380 SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG); 1381 ReplaceValueWith(SDValue(N, 0), Res); 1382 return SDValue(); 1383 } 1384 1385 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { 1386 SDValue Hi = GetPromotedInteger(N->getOperand(0)); 1387 SDValue Lo = GetPromotedInteger(N->getOperand(1)); 1388 SDValue Amt = N->getOperand(2); 1389 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger) 1390 Amt = ZExtPromotedInteger(Amt); 1391 EVT AmtVT = Amt.getValueType(); 1392 1393 SDLoc DL(N); 1394 EVT OldVT = N->getOperand(0).getValueType(); 1395 EVT VT = Lo.getValueType(); 1396 unsigned Opcode = N->getOpcode(); 1397 bool IsFSHR = Opcode == ISD::FSHR; 1398 unsigned OldBits = OldVT.getScalarSizeInBits(); 1399 unsigned NewBits = VT.getScalarSizeInBits(); 1400 1401 // Amount has to be interpreted modulo the old bit width. 1402 Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt, 1403 DAG.getConstant(OldBits, DL, AmtVT)); 1404 1405 // If the promoted type is twice the size (or more), then we use the 1406 // traditional funnel 'double' shift codegen. This isn't necessary if the 1407 // shift amount is constant. 1408 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. 1409 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). 1410 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) && 1411 !TLI.isOperationLegalOrCustom(Opcode, VT)) { 1412 SDValue HiShift = DAG.getConstant(OldBits, DL, VT); 1413 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift); 1414 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT); 1415 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo); 1416 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt); 1417 if (!IsFSHR) 1418 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift); 1419 return Res; 1420 } 1421 1422 // Shift Lo up to occupy the upper bits of the promoted type. 1423 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT); 1424 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset); 1425 1426 // Increase Amount to shift the result into the lower bits of the promoted 1427 // type. 1428 if (IsFSHR) 1429 Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset); 1430 1431 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt); 1432 } 1433 1434 // A vp version of PromoteIntRes_FunnelShift. 1435 SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(SDNode *N) { 1436 SDValue Hi = GetPromotedInteger(N->getOperand(0)); 1437 SDValue Lo = GetPromotedInteger(N->getOperand(1)); 1438 SDValue Amt = N->getOperand(2); 1439 SDValue Mask = N->getOperand(3); 1440 SDValue EVL = N->getOperand(4); 1441 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger) 1442 Amt = ZExtPromotedInteger(Amt); 1443 EVT AmtVT = Amt.getValueType(); 1444 1445 SDLoc DL(N); 1446 EVT OldVT = N->getOperand(0).getValueType(); 1447 EVT VT = Lo.getValueType(); 1448 unsigned Opcode = N->getOpcode(); 1449 bool IsFSHR = Opcode == ISD::VP_FSHR; 1450 unsigned OldBits = OldVT.getScalarSizeInBits(); 1451 unsigned NewBits = VT.getScalarSizeInBits(); 1452 1453 // Amount has to be interpreted modulo the old bit width. 1454 Amt = DAG.getNode(ISD::VP_UREM, DL, AmtVT, Amt, 1455 DAG.getConstant(OldBits, DL, AmtVT), Mask, EVL); 1456 1457 // If the promoted type is twice the size (or more), then we use the 1458 // traditional funnel 'double' shift codegen. This isn't necessary if the 1459 // shift amount is constant. 1460 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. 1461 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). 1462 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) && 1463 !TLI.isOperationLegalOrCustom(Opcode, VT)) { 1464 SDValue HiShift = DAG.getConstant(OldBits, DL, VT); 1465 Hi = DAG.getNode(ISD::VP_SHL, DL, VT, Hi, HiShift, Mask, EVL); 1466 // FIXME: Replace it by vp operations. 1467 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT); 1468 SDValue Res = DAG.getNode(ISD::VP_OR, DL, VT, Hi, Lo, Mask, EVL); 1469 Res = DAG.getNode(IsFSHR ? ISD::VP_LSHR : ISD::VP_SHL, DL, VT, Res, Amt, 1470 Mask, EVL); 1471 if (!IsFSHR) 1472 Res = DAG.getNode(ISD::VP_LSHR, DL, VT, Res, HiShift, Mask, EVL); 1473 return Res; 1474 } 1475 1476 // Shift Lo up to occupy the upper bits of the promoted type. 1477 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT); 1478 Lo = DAG.getNode(ISD::VP_SHL, DL, VT, Lo, ShiftOffset, Mask, EVL); 1479 1480 // Increase Amount to shift the result into the lower bits of the promoted 1481 // type. 1482 if (IsFSHR) 1483 Amt = DAG.getNode(ISD::VP_ADD, DL, AmtVT, Amt, ShiftOffset, Mask, EVL); 1484 1485 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt, Mask, EVL); 1486 } 1487 1488 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 1489 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1490 SDValue Res; 1491 SDValue InOp = N->getOperand(0); 1492 SDLoc dl(N); 1493 1494 switch (getTypeAction(InOp.getValueType())) { 1495 default: llvm_unreachable("Unknown type action!"); 1496 case TargetLowering::TypeLegal: 1497 case TargetLowering::TypeExpandInteger: 1498 Res = InOp; 1499 break; 1500 case TargetLowering::TypePromoteInteger: 1501 Res = GetPromotedInteger(InOp); 1502 break; 1503 case TargetLowering::TypeSplitVector: { 1504 EVT InVT = InOp.getValueType(); 1505 assert(InVT.isVector() && "Cannot split scalar types"); 1506 ElementCount NumElts = InVT.getVectorElementCount(); 1507 assert(NumElts == NVT.getVectorElementCount() && 1508 "Dst and Src must have the same number of elements"); 1509 assert(isPowerOf2_32(NumElts.getKnownMinValue()) && 1510 "Promoted vector type must be a power of two"); 1511 1512 SDValue EOp1, EOp2; 1513 GetSplitVector(InOp, EOp1, EOp2); 1514 1515 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 1516 NumElts.divideCoefficientBy(2)); 1517 if (N->getOpcode() == ISD::TRUNCATE) { 1518 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 1519 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 1520 } else { 1521 assert(N->getOpcode() == ISD::VP_TRUNCATE && 1522 "Expected VP_TRUNCATE opcode"); 1523 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 1524 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 1525 std::tie(EVLLo, EVLHi) = 1526 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); 1527 EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo); 1528 EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi); 1529 } 1530 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 1531 } 1532 // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some 1533 // targets. 1534 case TargetLowering::TypeWidenVector: { 1535 SDValue WideInOp = GetWidenedVector(InOp); 1536 1537 // Truncate widened InOp. 1538 unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); 1539 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), 1540 N->getValueType(0).getScalarType(), NumElem); 1541 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); 1542 1543 // Zero extend so that the elements are of same type as those of NVT 1544 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), 1545 NumElem); 1546 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); 1547 1548 // Extract the low NVT subvector. 1549 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl); 1550 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); 1551 } 1552 } 1553 1554 // Truncate to NVT instead of VT 1555 if (N->getOpcode() == ISD::VP_TRUNCATE) 1556 return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1), 1557 N->getOperand(2)); 1558 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 1559 } 1560 1561 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 1562 if (ResNo == 1) 1563 return PromoteIntRes_Overflow(N); 1564 1565 // The operation overflowed iff the result in the larger type is not the 1566 // zero extension of its truncation to the original type. 1567 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1568 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1569 EVT OVT = N->getOperand(0).getValueType(); 1570 EVT NVT = LHS.getValueType(); 1571 SDLoc dl(N); 1572 1573 // Do the arithmetic in the larger type. 1574 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 1575 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1576 1577 // Calculate the overflow flag: zero extend the arithmetic result from 1578 // the original type. 1579 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 1580 // Overflowed if and only if this is not equal to Res. 1581 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1582 1583 // Use the calculated overflow everywhere. 1584 ReplaceValueWith(SDValue(N, 1), Ofl); 1585 1586 return Res; 1587 } 1588 1589 // Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that 1590 // the third operand of ADDE/SUBE nodes is carry flag, which differs from 1591 // the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean. 1592 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N, 1593 unsigned ResNo) { 1594 if (ResNo == 1) 1595 return PromoteIntRes_Overflow(N); 1596 1597 // We need to sign-extend the operands so the carry value computed by the 1598 // wide operation will be equivalent to the carry value computed by the 1599 // narrow operation. 1600 // An UADDO_CARRY can generate carry only if any of the operands has its 1601 // most significant bit set. Sign extension propagates the most significant 1602 // bit into the higher bits which means the extra bit that the narrow 1603 // addition would need (i.e. the carry) will be propagated through the higher 1604 // bits of the wide addition. 1605 // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will 1606 // be preserved by sign extension. 1607 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1608 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1609 1610 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; 1611 1612 // Do the arithmetic in the wide type. 1613 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), 1614 LHS, RHS, N->getOperand(2)); 1615 1616 // Update the users of the original carry/borrow value. 1617 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 1618 1619 return SDValue(Res.getNode(), 0); 1620 } 1621 1622 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, 1623 unsigned ResNo) { 1624 assert(ResNo == 1 && "Don't know how to promote other results yet."); 1625 return PromoteIntRes_Overflow(N); 1626 } 1627 1628 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { 1629 EVT OVT = N->getValueType(0); 1630 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 1631 1632 // If a larger ABS or SMAX isn't supported by the target, try to expand now. 1633 // If we expand later we'll end up sign extending more than just the sra input 1634 // in sra+xor+sub expansion. 1635 if (!OVT.isVector() && 1636 !TLI.isOperationLegalOrCustomOrPromote(ISD::ABS, NVT) && 1637 !TLI.isOperationLegal(ISD::SMAX, NVT)) { 1638 if (SDValue Res = TLI.expandABS(N, DAG)) 1639 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res); 1640 } 1641 1642 SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); 1643 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); 1644 } 1645 1646 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 1647 // Promote the overflow bit trivially. 1648 if (ResNo == 1) 1649 return PromoteIntRes_Overflow(N); 1650 1651 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 1652 SDLoc DL(N); 1653 EVT SmallVT = LHS.getValueType(); 1654 1655 // To determine if the result overflowed in a larger type, we extend the 1656 // input to the larger type, do the multiply (checking if it overflows), 1657 // then also check the high bits of the result to see if overflow happened 1658 // there. 1659 if (N->getOpcode() == ISD::SMULO) { 1660 LHS = SExtPromotedInteger(LHS); 1661 RHS = SExtPromotedInteger(RHS); 1662 } else { 1663 LHS = ZExtPromotedInteger(LHS); 1664 RHS = ZExtPromotedInteger(RHS); 1665 } 1666 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 1667 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 1668 1669 // Overflow occurred if it occurred in the larger type, or if the high part 1670 // of the result does not zero/sign-extend the low part. Check this second 1671 // possibility first. 1672 SDValue Overflow; 1673 if (N->getOpcode() == ISD::UMULO) { 1674 // Unsigned overflow occurred if the high part is non-zero. 1675 unsigned Shift = SmallVT.getScalarSizeInBits(); 1676 SDValue Hi = 1677 DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 1678 DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL)); 1679 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 1680 DAG.getConstant(0, DL, Hi.getValueType()), 1681 ISD::SETNE); 1682 } else { 1683 // Signed overflow occurred if the high part does not sign extend the low. 1684 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 1685 Mul, DAG.getValueType(SmallVT)); 1686 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 1687 } 1688 1689 // The only other way for overflow to occur is if the multiplication in the 1690 // larger type itself overflowed. 1691 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 1692 SDValue(Mul.getNode(), 1)); 1693 1694 // Use the calculated overflow everywhere. 1695 ReplaceValueWith(SDValue(N, 1), Overflow); 1696 return Mul; 1697 } 1698 1699 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 1700 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 1701 N->getValueType(0))); 1702 } 1703 1704 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { 1705 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1706 1707 const APInt &MulImm = N->getConstantOperandAPInt(0); 1708 return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits())); 1709 } 1710 1711 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 1712 SDValue Chain = N->getOperand(0); // Get the chain. 1713 SDValue Ptr = N->getOperand(1); // Get the pointer. 1714 EVT VT = N->getValueType(0); 1715 SDLoc dl(N); 1716 1717 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 1718 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 1719 // The argument is passed as NumRegs registers of type RegVT. 1720 1721 SmallVector<SDValue, 8> Parts(NumRegs); 1722 for (unsigned i = 0; i < NumRegs; ++i) { 1723 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 1724 N->getConstantOperandVal(3)); 1725 Chain = Parts[i].getValue(1); 1726 } 1727 1728 // Handle endianness of the load. 1729 if (DAG.getDataLayout().isBigEndian()) 1730 std::reverse(Parts.begin(), Parts.end()); 1731 1732 // Assemble the parts in the promoted type. 1733 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1734 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 1735 for (unsigned i = 1; i < NumRegs; ++i) { 1736 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 1737 // Shift it to the right position and "or" it in. 1738 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 1739 DAG.getConstant(i * RegVT.getSizeInBits(), dl, 1740 TLI.getPointerTy(DAG.getDataLayout()))); 1741 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 1742 } 1743 1744 // Modified the chain result - switch anything that used the old chain to 1745 // use the new one. 1746 ReplaceValueWith(SDValue(N, 1), Chain); 1747 1748 return Res; 1749 } 1750 1751 //===----------------------------------------------------------------------===// 1752 // Integer Operand Promotion 1753 //===----------------------------------------------------------------------===// 1754 1755 /// PromoteIntegerOperand - This method is called when the specified operand of 1756 /// the specified node is found to need promotion. At this point, all of the 1757 /// result types of the node are known to be legal, but other operands of the 1758 /// node may need promotion or expansion as well as the specified one. 1759 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 1760 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG)); 1761 SDValue Res = SDValue(); 1762 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 1763 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 1764 return false; 1765 } 1766 1767 switch (N->getOpcode()) { 1768 default: 1769 #ifndef NDEBUG 1770 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 1771 N->dump(&DAG); dbgs() << "\n"; 1772 #endif 1773 report_fatal_error("Do not know how to promote this operator's operand!"); 1774 1775 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 1776 case ISD::ATOMIC_STORE: 1777 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 1778 break; 1779 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 1780 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 1781 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 1782 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 1783 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 1784 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 1785 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 1786 case ISD::INSERT_VECTOR_ELT: 1787 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo); 1788 break; 1789 case ISD::SPLAT_VECTOR: 1790 case ISD::SCALAR_TO_VECTOR: 1791 Res = PromoteIntOp_ScalarOp(N); 1792 break; 1793 case ISD::VSELECT: 1794 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 1795 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 1796 case ISD::VP_SETCC: 1797 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 1798 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 1799 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break; 1800 case ISD::VP_SINT_TO_FP: 1801 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 1802 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; 1803 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 1804 OpNo); break; 1805 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), 1806 OpNo); break; 1807 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), 1808 OpNo); break; 1809 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), 1810 OpNo); break; 1811 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), 1812 OpNo); break; 1813 case ISD::VP_TRUNCATE: 1814 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 1815 case ISD::BF16_TO_FP: 1816 case ISD::FP16_TO_FP: 1817 case ISD::VP_UINT_TO_FP: 1818 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 1819 case ISD::STRICT_FP16_TO_FP: 1820 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; 1821 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 1822 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break; 1823 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; 1824 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break; 1825 1826 case ISD::SHL: 1827 case ISD::SRA: 1828 case ISD::SRL: 1829 case ISD::ROTL: 1830 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 1831 1832 case ISD::FSHL: 1833 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break; 1834 1835 case ISD::SADDO_CARRY: 1836 case ISD::SSUBO_CARRY: 1837 case ISD::UADDO_CARRY: 1838 case ISD::USUBO_CARRY: Res = PromoteIntOp_ADDSUBO_CARRY(N, OpNo); break; 1839 1840 case ISD::FRAMEADDR: 1841 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; 1842 1843 case ISD::SMULFIX: 1844 case ISD::SMULFIXSAT: 1845 case ISD::UMULFIX: 1846 case ISD::UMULFIXSAT: 1847 case ISD::SDIVFIX: 1848 case ISD::SDIVFIXSAT: 1849 case ISD::UDIVFIX: 1850 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; 1851 case ISD::FPOWI: 1852 case ISD::STRICT_FPOWI: 1853 case ISD::FLDEXP: 1854 case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break; 1855 case ISD::VECREDUCE_ADD: 1856 case ISD::VECREDUCE_MUL: 1857 case ISD::VECREDUCE_AND: 1858 case ISD::VECREDUCE_OR: 1859 case ISD::VECREDUCE_XOR: 1860 case ISD::VECREDUCE_SMAX: 1861 case ISD::VECREDUCE_SMIN: 1862 case ISD::VECREDUCE_UMAX: 1863 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; 1864 case ISD::VP_REDUCE_ADD: 1865 case ISD::VP_REDUCE_MUL: 1866 case ISD::VP_REDUCE_AND: 1867 case ISD::VP_REDUCE_OR: 1868 case ISD::VP_REDUCE_XOR: 1869 case ISD::VP_REDUCE_SMAX: 1870 case ISD::VP_REDUCE_SMIN: 1871 case ISD::VP_REDUCE_UMAX: 1872 case ISD::VP_REDUCE_UMIN: 1873 Res = PromoteIntOp_VP_REDUCE(N, OpNo); 1874 break; 1875 1876 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; 1877 case ISD::STACKMAP: 1878 Res = PromoteIntOp_STACKMAP(N, OpNo); 1879 break; 1880 case ISD::PATCHPOINT: 1881 Res = PromoteIntOp_PATCHPOINT(N, OpNo); 1882 break; 1883 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 1884 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 1885 Res = PromoteIntOp_VP_STRIDED(N, OpNo); 1886 break; 1887 case ISD::EXPERIMENTAL_VP_SPLICE: 1888 Res = PromoteIntOp_VP_SPLICE(N, OpNo); 1889 break; 1890 } 1891 1892 // If the result is null, the sub-method took care of registering results etc. 1893 if (!Res.getNode()) return false; 1894 1895 // If the result is N, the sub-method updated N in place. Tell the legalizer 1896 // core about this. 1897 if (Res.getNode() == N) 1898 return true; 1899 1900 const bool IsStrictFp = N->isStrictFPOpcode(); 1901 assert(Res.getValueType() == N->getValueType(0) && 1902 N->getNumValues() == (IsStrictFp ? 2 : 1) && 1903 "Invalid operand expansion"); 1904 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: "; 1905 Res.dump()); 1906 1907 ReplaceValueWith(SDValue(N, 0), Res); 1908 if (IsStrictFp) 1909 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 1910 1911 return false; 1912 } 1913 1914 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is 1915 /// shared among BR_CC, SELECT_CC, and SETCC handlers. 1916 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS, 1917 ISD::CondCode CCCode) { 1918 // We have to insert explicit sign or zero extends. Note that we could 1919 // insert sign extends for ALL conditions. For those operations where either 1920 // zero or sign extension would be valid, we ask the target which extension 1921 // it would prefer. 1922 1923 // Signed comparisons always require sign extension. 1924 if (ISD::isSignedIntSetCC(CCCode)) { 1925 LHS = SExtPromotedInteger(LHS); 1926 RHS = SExtPromotedInteger(RHS); 1927 return; 1928 } 1929 1930 assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) && 1931 "Unknown integer comparison!"); 1932 1933 SDValue OpL = GetPromotedInteger(LHS); 1934 SDValue OpR = GetPromotedInteger(RHS); 1935 1936 if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) { 1937 // The target would prefer to promote the comparison operand with sign 1938 // extension. Honor that unless the promoted values are already zero 1939 // extended. 1940 unsigned OpLEffectiveBits = 1941 DAG.computeKnownBits(OpL).countMaxActiveBits(); 1942 unsigned OpREffectiveBits = 1943 DAG.computeKnownBits(OpR).countMaxActiveBits(); 1944 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && 1945 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { 1946 LHS = OpL; 1947 RHS = OpR; 1948 return; 1949 } 1950 1951 // The promoted values aren't zero extended, use a sext_inreg. 1952 LHS = SExtPromotedInteger(LHS); 1953 RHS = SExtPromotedInteger(RHS); 1954 return; 1955 } 1956 1957 // Prefer to promote the comparison operand with zero extension. 1958 1959 // If the width of OpL/OpR excluding the duplicated sign bits is no greater 1960 // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation 1961 // that we might not be able to remove. 1962 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL); 1963 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR); 1964 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && 1965 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { 1966 LHS = OpL; 1967 RHS = OpR; 1968 return; 1969 } 1970 1971 // Otherwise, use zext_inreg. 1972 LHS = ZExtPromotedInteger(LHS); 1973 RHS = ZExtPromotedInteger(RHS); 1974 } 1975 1976 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 1977 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1978 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 1979 } 1980 1981 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 1982 SDValue Op1 = GetPromotedInteger(N->getOperand(1)); 1983 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 1984 N->getChain(), Op1, N->getBasePtr(), N->getMemOperand()); 1985 } 1986 1987 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 1988 // This should only occur in unusual situations like bitcasting to an 1989 // x86_fp80, so just turn it into a store+load 1990 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 1991 } 1992 1993 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 1994 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1995 1996 SDValue LHS = N->getOperand(2); 1997 SDValue RHS = N->getOperand(3); 1998 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 1999 2000 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 2001 // legal types. 2002 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2003 N->getOperand(1), LHS, RHS, N->getOperand(4)), 2004 0); 2005 } 2006 2007 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 2008 assert(OpNo == 1 && "only know how to promote condition"); 2009 2010 // Promote all the way up to the canonical SetCC type. 2011 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 2012 2013 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 2014 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 2015 N->getOperand(2)), 0); 2016 } 2017 2018 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 2019 // Since the result type is legal, the operands must promote to it. 2020 EVT OVT = N->getOperand(0).getValueType(); 2021 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 2022 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 2023 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 2024 SDLoc dl(N); 2025 2026 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 2027 DAG.getConstant(OVT.getSizeInBits(), dl, 2028 TLI.getPointerTy(DAG.getDataLayout()))); 2029 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 2030 } 2031 2032 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 2033 // The vector type is legal but the element type is not. This implies 2034 // that the vector is a power-of-two in length and that the element 2035 // type does not have a strange size (eg: it is not i1). 2036 EVT VecVT = N->getValueType(0); 2037 unsigned NumElts = VecVT.getVectorNumElements(); 2038 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 2039 "Legal vector of one illegal element?"); 2040 2041 // Promote the inserted value. The type does not need to match the 2042 // vector element type. Check that any extra bits introduced will be 2043 // truncated away. 2044 assert(N->getOperand(0).getValueSizeInBits() >= 2045 N->getValueType(0).getScalarSizeInBits() && 2046 "Type of inserted value narrower than vector element type!"); 2047 2048 SmallVector<SDValue, 16> NewOps; 2049 for (unsigned i = 0; i < NumElts; ++i) 2050 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 2051 2052 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2053 } 2054 2055 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 2056 unsigned OpNo) { 2057 if (OpNo == 1) { 2058 // Promote the inserted value. This is valid because the type does not 2059 // have to match the vector element type. 2060 2061 // Check that any extra bits introduced will be truncated away. 2062 assert(N->getOperand(1).getValueSizeInBits() >= 2063 N->getValueType(0).getScalarSizeInBits() && 2064 "Type of inserted value narrower than vector element type!"); 2065 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2066 GetPromotedInteger(N->getOperand(1)), 2067 N->getOperand(2)), 2068 0); 2069 } 2070 2071 assert(OpNo == 2 && "Different operand and result vector types?"); 2072 2073 // Promote the index. 2074 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 2075 TLI.getVectorIdxTy(DAG.getDataLayout())); 2076 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2077 N->getOperand(1), Idx), 0); 2078 } 2079 2080 SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) { 2081 // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated, 2082 // so just promote the operand in place. 2083 return SDValue(DAG.UpdateNodeOperands(N, 2084 GetPromotedInteger(N->getOperand(0))), 0); 2085 } 2086 2087 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 2088 assert(OpNo == 0 && "Only know how to promote the condition!"); 2089 SDValue Cond = N->getOperand(0); 2090 EVT OpTy = N->getOperand(1).getValueType(); 2091 2092 if (N->getOpcode() == ISD::VSELECT) 2093 if (SDValue Res = WidenVSELECTMask(N)) 2094 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 2095 Res, N->getOperand(1), N->getOperand(2)); 2096 2097 // Promote all the way up to the canonical SetCC type. 2098 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 2099 Cond = PromoteTargetBoolean(Cond, OpVT); 2100 2101 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 2102 N->getOperand(2)), 0); 2103 } 2104 2105 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 2106 assert(OpNo == 0 && "Don't know how to promote this operand!"); 2107 2108 SDValue LHS = N->getOperand(0); 2109 SDValue RHS = N->getOperand(1); 2110 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 2111 2112 // The CC (#4) and the possible return values (#2 and #3) have legal types. 2113 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 2114 N->getOperand(3), N->getOperand(4)), 0); 2115 } 2116 2117 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 2118 assert(OpNo == 0 && "Don't know how to promote this operand!"); 2119 2120 SDValue LHS = N->getOperand(0); 2121 SDValue RHS = N->getOperand(1); 2122 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 2123 2124 // The CC (#2) is always legal. 2125 if (N->getOpcode() == ISD::SETCC) 2126 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 2127 2128 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode"); 2129 2130 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 2131 N->getOperand(3), N->getOperand(4)), 2132 0); 2133 } 2134 2135 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 2136 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2137 ZExtPromotedInteger(N->getOperand(1))), 0); 2138 } 2139 2140 SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) { 2141 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), 2142 ZExtPromotedInteger(N->getOperand(2))), 0); 2143 } 2144 2145 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 2146 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2147 SDLoc dl(N); 2148 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 2149 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 2150 Op, DAG.getValueType(N->getOperand(0).getValueType())); 2151 } 2152 2153 SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) { 2154 SDLoc dl(N); 2155 EVT VT = N->getValueType(0); 2156 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2157 // FIXME: There is no VP_ANY_EXTEND yet. 2158 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1), 2159 N->getOperand(2)); 2160 unsigned Diff = 2161 VT.getScalarSizeInBits() - N->getOperand(0).getScalarValueSizeInBits(); 2162 SDValue ShAmt = DAG.getShiftAmountConstant(Diff, VT, dl); 2163 // FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts. 2164 SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShAmt, N->getOperand(1), 2165 N->getOperand(2)); 2166 return DAG.getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt, N->getOperand(1), 2167 N->getOperand(2)); 2168 } 2169 2170 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 2171 if (N->getOpcode() == ISD::VP_SINT_TO_FP) 2172 return SDValue(DAG.UpdateNodeOperands(N, 2173 SExtPromotedInteger(N->getOperand(0)), 2174 N->getOperand(1), N->getOperand(2)), 2175 0); 2176 return SDValue(DAG.UpdateNodeOperands(N, 2177 SExtPromotedInteger(N->getOperand(0))), 0); 2178 } 2179 2180 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { 2181 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2182 SExtPromotedInteger(N->getOperand(1))), 0); 2183 } 2184 2185 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 2186 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2187 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 2188 SDLoc dl(N); 2189 2190 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 2191 2192 // Truncate the value and store the result. 2193 return DAG.getTruncStore(Ch, dl, Val, Ptr, 2194 N->getMemoryVT(), N->getMemOperand()); 2195 } 2196 2197 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, 2198 unsigned OpNo) { 2199 SDValue DataOp = N->getValue(); 2200 SDValue Mask = N->getMask(); 2201 2202 if (OpNo == 4) { 2203 // The Mask. Update in place. 2204 EVT DataVT = DataOp.getValueType(); 2205 Mask = PromoteTargetBoolean(Mask, DataVT); 2206 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2207 NewOps[4] = Mask; 2208 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2209 } 2210 2211 assert(OpNo == 1 && "Unexpected operand for promotion"); 2212 DataOp = GetPromotedInteger(DataOp); 2213 2214 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(), 2215 N->getOffset(), Mask, N->getMemoryVT(), 2216 N->getMemOperand(), N->getAddressingMode(), 2217 /*IsTruncating*/ true, N->isCompressingStore()); 2218 } 2219 2220 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, 2221 unsigned OpNo) { 2222 assert(OpNo == 3 && "Only know how to promote the mask!"); 2223 EVT DataVT = N->getValueType(0); 2224 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2225 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2226 NewOps[OpNo] = Mask; 2227 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 2228 if (Res == N) 2229 return SDValue(Res, 0); 2230 2231 // Update triggered CSE, do our own replacement since caller can't. 2232 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 2233 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 2234 return SDValue(); 2235 } 2236 2237 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, 2238 unsigned OpNo) { 2239 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 2240 2241 if (OpNo == 2) { 2242 // The Mask 2243 EVT DataVT = N->getValueType(0); 2244 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2245 } else if (OpNo == 4) { 2246 // The Index 2247 if (N->isIndexSigned()) 2248 // Need to sign extend the index since the bits will likely be used. 2249 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2250 else 2251 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 2252 } else 2253 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 2254 2255 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 2256 if (Res == N) 2257 return SDValue(Res, 0); 2258 2259 // Update triggered CSE, do our own replacement since caller can't. 2260 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 2261 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 2262 return SDValue(); 2263 } 2264 2265 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, 2266 unsigned OpNo) { 2267 bool TruncateStore = N->isTruncatingStore(); 2268 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 2269 2270 if (OpNo == 2) { 2271 // The Mask 2272 EVT DataVT = N->getValue().getValueType(); 2273 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2274 } else if (OpNo == 4) { 2275 // The Index 2276 if (N->isIndexSigned()) 2277 // Need to sign extend the index since the bits will likely be used. 2278 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2279 else 2280 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 2281 } else { 2282 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 2283 TruncateStore = true; 2284 } 2285 2286 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), 2287 SDLoc(N), NewOps, N->getMemOperand(), 2288 N->getIndexType(), TruncateStore); 2289 } 2290 2291 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 2292 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2293 if (N->getOpcode() == ISD::VP_TRUNCATE) 2294 return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op, 2295 N->getOperand(1), N->getOperand(2)); 2296 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 2297 } 2298 2299 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 2300 if (N->getOpcode() == ISD::VP_UINT_TO_FP) 2301 return SDValue(DAG.UpdateNodeOperands(N, 2302 ZExtPromotedInteger(N->getOperand(0)), 2303 N->getOperand(1), N->getOperand(2)), 2304 0); 2305 return SDValue(DAG.UpdateNodeOperands(N, 2306 ZExtPromotedInteger(N->getOperand(0))), 0); 2307 } 2308 2309 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { 2310 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2311 ZExtPromotedInteger(N->getOperand(1))), 0); 2312 } 2313 2314 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 2315 SDLoc dl(N); 2316 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2317 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 2318 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); 2319 } 2320 2321 SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) { 2322 SDLoc dl(N); 2323 EVT VT = N->getValueType(0); 2324 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2325 // FIXME: There is no VP_ANY_EXTEND yet. 2326 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1), 2327 N->getOperand(2)); 2328 APInt Imm = APInt::getLowBitsSet(VT.getScalarSizeInBits(), 2329 N->getOperand(0).getScalarValueSizeInBits()); 2330 return DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(Imm, dl, VT), 2331 N->getOperand(1), N->getOperand(2)); 2332 } 2333 2334 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBO_CARRY(SDNode *N, unsigned OpNo) { 2335 assert(OpNo == 2 && "Don't know how to promote this operand!"); 2336 2337 SDValue LHS = N->getOperand(0); 2338 SDValue RHS = N->getOperand(1); 2339 SDValue Carry = N->getOperand(2); 2340 SDLoc DL(N); 2341 2342 Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); 2343 2344 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); 2345 } 2346 2347 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { 2348 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 2349 return SDValue( 2350 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); 2351 } 2352 2353 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { 2354 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. 2355 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 2356 return SDValue(DAG.UpdateNodeOperands(N, Op), 0); 2357 } 2358 2359 SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) { 2360 bool IsStrict = N->isStrictFPOpcode(); 2361 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2362 2363 bool IsPowI = 2364 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; 2365 2366 // The integer operand is the last operand in FPOWI (or FLDEXP) (so the result 2367 // and floating point operand is already type legalized). 2368 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0)) 2369 : RTLIB::getLDEXP(N->getValueType(0)); 2370 2371 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 2372 SDValue Op = SExtPromotedInteger(N->getOperand(1)); 2373 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); 2374 } 2375 2376 // We can't just promote the exponent type in FPOWI, since we want to lower 2377 // the node to a libcall and we if we promote to a type larger than 2378 // sizeof(int) the libcall might not be according to the targets ABI. Instead 2379 // we rewrite to a libcall here directly, letting makeLibCall handle promotion 2380 // if the target accepts it according to shouldSignExtendTypeInLibCall. 2381 2382 unsigned OpOffset = IsStrict ? 1 : 0; 2383 // The exponent should fit in a sizeof(int) type for the libcall to be valid. 2384 assert(DAG.getLibInfo().getIntSize() == 2385 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && 2386 "POWI exponent should match with sizeof(int) when doing the libcall."); 2387 TargetLowering::MakeLibCallOptions CallOptions; 2388 CallOptions.setSExt(true); 2389 SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)}; 2390 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall( 2391 DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain); 2392 ReplaceValueWith(SDValue(N, 0), Tmp.first); 2393 if (IsStrict) 2394 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2395 return SDValue(); 2396 } 2397 2398 static unsigned getExtendForIntVecReduction(SDNode *N) { 2399 switch (N->getOpcode()) { 2400 default: 2401 llvm_unreachable("Expected integer vector reduction"); 2402 case ISD::VECREDUCE_ADD: 2403 case ISD::VECREDUCE_MUL: 2404 case ISD::VECREDUCE_AND: 2405 case ISD::VECREDUCE_OR: 2406 case ISD::VECREDUCE_XOR: 2407 case ISD::VP_REDUCE_ADD: 2408 case ISD::VP_REDUCE_MUL: 2409 case ISD::VP_REDUCE_AND: 2410 case ISD::VP_REDUCE_OR: 2411 case ISD::VP_REDUCE_XOR: 2412 return ISD::ANY_EXTEND; 2413 case ISD::VECREDUCE_SMAX: 2414 case ISD::VECREDUCE_SMIN: 2415 case ISD::VP_REDUCE_SMAX: 2416 case ISD::VP_REDUCE_SMIN: 2417 return ISD::SIGN_EXTEND; 2418 case ISD::VECREDUCE_UMAX: 2419 case ISD::VECREDUCE_UMIN: 2420 case ISD::VP_REDUCE_UMAX: 2421 case ISD::VP_REDUCE_UMIN: 2422 return ISD::ZERO_EXTEND; 2423 } 2424 } 2425 2426 SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) { 2427 switch (getExtendForIntVecReduction(N)) { 2428 default: 2429 llvm_unreachable("Impossible extension kind for integer reduction"); 2430 case ISD::ANY_EXTEND: 2431 return GetPromotedInteger(V); 2432 case ISD::SIGN_EXTEND: 2433 return SExtPromotedInteger(V); 2434 case ISD::ZERO_EXTEND: 2435 return ZExtPromotedInteger(V); 2436 } 2437 } 2438 2439 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { 2440 SDLoc dl(N); 2441 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0)); 2442 2443 EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType(); 2444 EVT InVT = Op.getValueType(); 2445 EVT EltVT = InVT.getVectorElementType(); 2446 EVT ResVT = N->getValueType(0); 2447 unsigned Opcode = N->getOpcode(); 2448 2449 // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if 2450 // vecreduce_xor is not legal 2451 if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 && 2452 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_XOR, InVT) && 2453 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_ADD, InVT)) 2454 Opcode = ISD::VECREDUCE_ADD; 2455 2456 // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if 2457 // vecreduce_or is not legal 2458 else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 && 2459 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_OR, InVT) && 2460 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMAX, InVT)) { 2461 Opcode = ISD::VECREDUCE_UMAX; 2462 // Can't use promoteTargetBoolean here because we still need 2463 // to either sign_ext or zero_ext in the undefined case. 2464 switch (TLI.getBooleanContents(InVT)) { 2465 case TargetLoweringBase::UndefinedBooleanContent: 2466 case TargetLoweringBase::ZeroOrOneBooleanContent: 2467 Op = ZExtPromotedInteger(N->getOperand(0)); 2468 break; 2469 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 2470 Op = SExtPromotedInteger(N->getOperand(0)); 2471 break; 2472 } 2473 } 2474 2475 // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if 2476 // vecreduce_and is not legal 2477 else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 && 2478 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_AND, InVT) && 2479 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMIN, InVT)) { 2480 Opcode = ISD::VECREDUCE_UMIN; 2481 // Can't use promoteTargetBoolean here because we still need 2482 // to either sign_ext or zero_ext in the undefined case. 2483 switch (TLI.getBooleanContents(InVT)) { 2484 case TargetLoweringBase::UndefinedBooleanContent: 2485 case TargetLoweringBase::ZeroOrOneBooleanContent: 2486 Op = ZExtPromotedInteger(N->getOperand(0)); 2487 break; 2488 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 2489 Op = SExtPromotedInteger(N->getOperand(0)); 2490 break; 2491 } 2492 } 2493 2494 if (ResVT.bitsGE(EltVT)) 2495 return DAG.getNode(Opcode, SDLoc(N), ResVT, Op); 2496 2497 // Result size must be >= element size. If this is not the case after 2498 // promotion, also promote the result type and then truncate. 2499 SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op); 2500 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce); 2501 } 2502 2503 SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) { 2504 SDLoc DL(N); 2505 SDValue Op = N->getOperand(OpNo); 2506 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2507 2508 if (OpNo == 2) { // Mask 2509 // Update in place. 2510 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType()); 2511 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2512 } 2513 2514 assert(OpNo == 1 && "Unexpected operand for promotion"); 2515 2516 Op = PromoteIntOpVectorReduction(N, Op); 2517 2518 NewOps[OpNo] = Op; 2519 2520 EVT VT = N->getValueType(0); 2521 EVT EltVT = Op.getValueType().getScalarType(); 2522 2523 if (VT.bitsGE(EltVT)) 2524 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps); 2525 2526 // Result size must be >= element/start-value size. If this is not the case 2527 // after promotion, also promote both the start value and result type and 2528 // then truncate. 2529 NewOps[0] = 2530 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0)); 2531 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps); 2532 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce); 2533 } 2534 2535 SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) { 2536 SDValue Op = ZExtPromotedInteger(N->getOperand(1)); 2537 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); 2538 } 2539 2540 SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) { 2541 assert(OpNo > 1); // Because the first two arguments are guaranteed legal. 2542 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 2543 SDValue Operand = N->getOperand(OpNo); 2544 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); 2545 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); 2546 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2547 } 2548 2549 SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { 2550 assert(OpNo >= 7); 2551 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 2552 SDValue Operand = N->getOperand(OpNo); 2553 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); 2554 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); 2555 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2556 } 2557 2558 SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { 2559 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || 2560 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); 2561 2562 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); 2563 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2564 2565 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2566 } 2567 2568 SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo) { 2569 SmallVector<SDValue, 6> NewOps(N->op_begin(), N->op_end()); 2570 2571 if (OpNo == 2) { // Offset operand 2572 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2573 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2574 } 2575 2576 assert((OpNo == 4 || OpNo == 5) && "Unexpected operand for promotion"); 2577 2578 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 2579 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2580 } 2581 2582 //===----------------------------------------------------------------------===// 2583 // Integer Result Expansion 2584 //===----------------------------------------------------------------------===// 2585 2586 /// ExpandIntegerResult - This method is called when the specified result of the 2587 /// specified node is found to need expansion. At this point, the node may also 2588 /// have invalid operands or may have other results that need promotion, we just 2589 /// know that (at least) one result needs expansion. 2590 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 2591 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG)); 2592 SDValue Lo, Hi; 2593 Lo = Hi = SDValue(); 2594 2595 // See if the target wants to custom expand this node. 2596 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 2597 return; 2598 2599 switch (N->getOpcode()) { 2600 default: 2601 #ifndef NDEBUG 2602 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 2603 N->dump(&DAG); dbgs() << "\n"; 2604 #endif 2605 report_fatal_error("Do not know how to expand the result of this " 2606 "operator!"); 2607 2608 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break; 2609 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 2610 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; 2611 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 2612 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 2613 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; 2614 2615 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 2616 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 2617 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 2618 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 2619 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 2620 2621 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 2622 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 2623 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 2624 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; 2625 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 2626 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; 2627 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 2628 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; 2629 case ISD::CTLZ_ZERO_UNDEF: 2630 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 2631 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 2632 case ISD::CTTZ_ZERO_UNDEF: 2633 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 2634 case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break; 2635 case ISD::STRICT_FP_TO_SINT: 2636 case ISD::FP_TO_SINT: 2637 case ISD::STRICT_FP_TO_UINT: 2638 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_XINT(N, Lo, Hi); break; 2639 case ISD::FP_TO_SINT_SAT: 2640 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; 2641 case ISD::STRICT_LROUND: 2642 case ISD::STRICT_LRINT: 2643 case ISD::LROUND: 2644 case ISD::LRINT: 2645 case ISD::STRICT_LLROUND: 2646 case ISD::STRICT_LLRINT: 2647 case ISD::LLROUND: 2648 case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break; 2649 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 2650 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 2651 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; 2652 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 2653 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 2654 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 2655 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 2656 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 2657 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 2658 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 2659 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 2660 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 2661 2662 case ISD::ATOMIC_LOAD_ADD: 2663 case ISD::ATOMIC_LOAD_SUB: 2664 case ISD::ATOMIC_LOAD_AND: 2665 case ISD::ATOMIC_LOAD_CLR: 2666 case ISD::ATOMIC_LOAD_OR: 2667 case ISD::ATOMIC_LOAD_XOR: 2668 case ISD::ATOMIC_LOAD_NAND: 2669 case ISD::ATOMIC_LOAD_MIN: 2670 case ISD::ATOMIC_LOAD_MAX: 2671 case ISD::ATOMIC_LOAD_UMIN: 2672 case ISD::ATOMIC_LOAD_UMAX: 2673 case ISD::ATOMIC_SWAP: 2674 case ISD::ATOMIC_CMP_SWAP: { 2675 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 2676 SplitInteger(Tmp.first, Lo, Hi); 2677 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2678 break; 2679 } 2680 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 2681 AtomicSDNode *AN = cast<AtomicSDNode>(N); 2682 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 2683 SDValue Tmp = DAG.getAtomicCmpSwap( 2684 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 2685 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 2686 AN->getMemOperand()); 2687 2688 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 2689 // success simply by comparing the loaded value against the ingoing 2690 // comparison. 2691 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 2692 N->getOperand(2), ISD::SETEQ); 2693 2694 SplitInteger(Tmp, Lo, Hi); 2695 ReplaceValueWith(SDValue(N, 1), Success); 2696 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 2697 break; 2698 } 2699 2700 case ISD::AND: 2701 case ISD::OR: 2702 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 2703 2704 case ISD::UMAX: 2705 case ISD::SMAX: 2706 case ISD::UMIN: 2707 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; 2708 2709 case ISD::ADD: 2710 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 2711 2712 case ISD::ADDC: 2713 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 2714 2715 case ISD::ADDE: 2716 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 2717 2718 case ISD::UADDO_CARRY: 2719 case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break; 2720 2721 case ISD::SADDO_CARRY: 2722 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; 2723 2724 case ISD::SHL: 2725 case ISD::SRA: 2726 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 2727 2728 case ISD::SADDO: 2729 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 2730 case ISD::UADDO: 2731 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 2732 case ISD::UMULO: 2733 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 2734 2735 case ISD::SADDSAT: 2736 case ISD::UADDSAT: 2737 case ISD::SSUBSAT: 2738 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; 2739 2740 case ISD::SSHLSAT: 2741 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; 2742 2743 case ISD::SMULFIX: 2744 case ISD::SMULFIXSAT: 2745 case ISD::UMULFIX: 2746 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; 2747 2748 case ISD::SDIVFIX: 2749 case ISD::SDIVFIXSAT: 2750 case ISD::UDIVFIX: 2751 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; 2752 2753 case ISD::VECREDUCE_ADD: 2754 case ISD::VECREDUCE_MUL: 2755 case ISD::VECREDUCE_AND: 2756 case ISD::VECREDUCE_OR: 2757 case ISD::VECREDUCE_XOR: 2758 case ISD::VECREDUCE_SMAX: 2759 case ISD::VECREDUCE_SMIN: 2760 case ISD::VECREDUCE_UMAX: 2761 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; 2762 2763 case ISD::ROTL: 2764 case ISD::ROTR: 2765 ExpandIntRes_Rotate(N, Lo, Hi); 2766 break; 2767 2768 case ISD::FSHL: 2769 case ISD::FSHR: 2770 ExpandIntRes_FunnelShift(N, Lo, Hi); 2771 break; 2772 2773 case ISD::VSCALE: 2774 ExpandIntRes_VSCALE(N, Lo, Hi); 2775 break; 2776 } 2777 2778 // If Lo/Hi is null, the sub-method took care of registering results etc. 2779 if (Lo.getNode()) 2780 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 2781 } 2782 2783 /// Lower an atomic node to the appropriate builtin call. 2784 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 2785 unsigned Opc = Node->getOpcode(); 2786 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2787 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering(); 2788 // Lower to outline atomic libcall if outline atomics enabled, 2789 // or to sync libcall otherwise 2790 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); 2791 EVT RetVT = Node->getValueType(0); 2792 TargetLowering::MakeLibCallOptions CallOptions; 2793 SmallVector<SDValue, 4> Ops; 2794 if (TLI.getLibcallName(LC)) { 2795 Ops.append(Node->op_begin() + 2, Node->op_end()); 2796 Ops.push_back(Node->getOperand(1)); 2797 } else { 2798 LC = RTLIB::getSYNC(Opc, VT); 2799 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2800 "Unexpected atomic op or value type!"); 2801 Ops.append(Node->op_begin() + 1, Node->op_end()); 2802 } 2803 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), 2804 Node->getOperand(0)); 2805 } 2806 2807 /// N is a shift by a value that needs to be expanded, 2808 /// and the shift amount is a constant 'Amt'. Expand the operation. 2809 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, 2810 SDValue &Lo, SDValue &Hi) { 2811 SDLoc DL(N); 2812 // Expand the incoming operand to be shifted, so that we have its parts 2813 SDValue InL, InH; 2814 GetExpandedInteger(N->getOperand(0), InL, InH); 2815 2816 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization 2817 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. 2818 if (!Amt) { 2819 Lo = InL; 2820 Hi = InH; 2821 return; 2822 } 2823 2824 EVT NVT = InL.getValueType(); 2825 unsigned VTBits = N->getValueType(0).getSizeInBits(); 2826 unsigned NVTBits = NVT.getSizeInBits(); 2827 EVT ShTy = N->getOperand(1).getValueType(); 2828 2829 if (N->getOpcode() == ISD::SHL) { 2830 if (Amt.uge(VTBits)) { 2831 Lo = Hi = DAG.getConstant(0, DL, NVT); 2832 } else if (Amt.ugt(NVTBits)) { 2833 Lo = DAG.getConstant(0, DL, NVT); 2834 Hi = DAG.getNode(ISD::SHL, DL, 2835 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2836 } else if (Amt == NVTBits) { 2837 Lo = DAG.getConstant(0, DL, NVT); 2838 Hi = InL; 2839 } else { 2840 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); 2841 Hi = DAG.getNode(ISD::OR, DL, NVT, 2842 DAG.getNode(ISD::SHL, DL, NVT, InH, 2843 DAG.getConstant(Amt, DL, ShTy)), 2844 DAG.getNode(ISD::SRL, DL, NVT, InL, 2845 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2846 } 2847 return; 2848 } 2849 2850 if (N->getOpcode() == ISD::SRL) { 2851 if (Amt.uge(VTBits)) { 2852 Lo = Hi = DAG.getConstant(0, DL, NVT); 2853 } else if (Amt.ugt(NVTBits)) { 2854 Lo = DAG.getNode(ISD::SRL, DL, 2855 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2856 Hi = DAG.getConstant(0, DL, NVT); 2857 } else if (Amt == NVTBits) { 2858 Lo = InH; 2859 Hi = DAG.getConstant(0, DL, NVT); 2860 } else { 2861 Lo = DAG.getNode(ISD::OR, DL, NVT, 2862 DAG.getNode(ISD::SRL, DL, NVT, InL, 2863 DAG.getConstant(Amt, DL, ShTy)), 2864 DAG.getNode(ISD::SHL, DL, NVT, InH, 2865 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2866 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2867 } 2868 return; 2869 } 2870 2871 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2872 if (Amt.uge(VTBits)) { 2873 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2874 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2875 } else if (Amt.ugt(NVTBits)) { 2876 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2877 DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2878 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2879 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2880 } else if (Amt == NVTBits) { 2881 Lo = InH; 2882 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2883 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2884 } else { 2885 Lo = DAG.getNode(ISD::OR, DL, NVT, 2886 DAG.getNode(ISD::SRL, DL, NVT, InL, 2887 DAG.getConstant(Amt, DL, ShTy)), 2888 DAG.getNode(ISD::SHL, DL, NVT, InH, 2889 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2890 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2891 } 2892 } 2893 2894 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 2895 /// this shift based on knowledge of the high bit of the shift amount. If we 2896 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual 2897 /// shift amount. 2898 bool DAGTypeLegalizer:: 2899 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2900 SDValue Amt = N->getOperand(1); 2901 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2902 EVT ShTy = Amt.getValueType(); 2903 unsigned ShBits = ShTy.getScalarSizeInBits(); 2904 unsigned NVTBits = NVT.getScalarSizeInBits(); 2905 assert(isPowerOf2_32(NVTBits) && 2906 "Expanded integer type size not a power of two!"); 2907 SDLoc dl(N); 2908 2909 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 2910 KnownBits Known = DAG.computeKnownBits(N->getOperand(1)); 2911 2912 // If we don't know anything about the high bits, exit. 2913 if (((Known.Zero|Known.One) & HighBitMask) == 0) 2914 return false; 2915 2916 // Get the incoming operand to be shifted. 2917 SDValue InL, InH; 2918 GetExpandedInteger(N->getOperand(0), InL, InH); 2919 2920 // If we know that any of the high bits of the shift amount are one, then we 2921 // can do this as a couple of simple shifts. 2922 if (Known.One.intersects(HighBitMask)) { 2923 // Mask out the high bit, which we know is set. 2924 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 2925 DAG.getConstant(~HighBitMask, dl, ShTy)); 2926 2927 switch (N->getOpcode()) { 2928 default: llvm_unreachable("Unknown shift"); 2929 case ISD::SHL: 2930 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. 2931 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 2932 return true; 2933 case ISD::SRL: 2934 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2935 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 2936 return true; 2937 case ISD::SRA: 2938 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 2939 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2940 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 2941 return true; 2942 } 2943 } 2944 2945 // If we know that all of the high bits of the shift amount are zero, then we 2946 // can do this as a couple of simple shifts. 2947 if (HighBitMask.isSubsetOf(Known.Zero)) { 2948 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 2949 // shift if x is zero. We can use XOR here because x is known to be smaller 2950 // than 32. 2951 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 2952 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2953 2954 unsigned Op1, Op2; 2955 switch (N->getOpcode()) { 2956 default: llvm_unreachable("Unknown shift"); 2957 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 2958 case ISD::SRL: 2959 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 2960 } 2961 2962 // When shifting right the arithmetic for Lo and Hi is swapped. 2963 if (N->getOpcode() != ISD::SHL) 2964 std::swap(InL, InH); 2965 2966 // Use a little trick to get the bits that move from Lo to Hi. First 2967 // shift by one bit. 2968 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); 2969 // Then compute the remaining shift with amount-1. 2970 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 2971 2972 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 2973 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 2974 2975 if (N->getOpcode() != ISD::SHL) 2976 std::swap(Hi, Lo); 2977 return true; 2978 } 2979 2980 return false; 2981 } 2982 2983 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 2984 /// of any size. 2985 bool DAGTypeLegalizer:: 2986 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2987 SDValue Amt = N->getOperand(1); 2988 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2989 EVT ShTy = Amt.getValueType(); 2990 unsigned NVTBits = NVT.getSizeInBits(); 2991 assert(isPowerOf2_32(NVTBits) && 2992 "Expanded integer type size not a power of two!"); 2993 SDLoc dl(N); 2994 2995 // Get the incoming operand to be shifted. 2996 SDValue InL, InH; 2997 GetExpandedInteger(N->getOperand(0), InL, InH); 2998 2999 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); 3000 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 3001 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 3002 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 3003 Amt, NVBitsNode, ISD::SETULT); 3004 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), 3005 Amt, DAG.getConstant(0, dl, ShTy), 3006 ISD::SETEQ); 3007 3008 SDValue LoS, HiS, LoL, HiL; 3009 switch (N->getOpcode()) { 3010 default: llvm_unreachable("Unknown shift"); 3011 case ISD::SHL: 3012 // Short: ShAmt < NVTBits 3013 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 3014 HiS = DAG.getNode(ISD::OR, dl, NVT, 3015 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 3016 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 3017 3018 // Long: ShAmt >= NVTBits 3019 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. 3020 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 3021 3022 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 3023 Hi = DAG.getSelect(dl, NVT, isZero, InH, 3024 DAG.getSelect(dl, NVT, isShort, HiS, HiL)); 3025 return true; 3026 case ISD::SRL: 3027 // Short: ShAmt < NVTBits 3028 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 3029 LoS = DAG.getNode(ISD::OR, dl, NVT, 3030 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 3031 // FIXME: If Amt is zero, the following shift generates an undefined result 3032 // on some architectures. 3033 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 3034 3035 // Long: ShAmt >= NVTBits 3036 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. 3037 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 3038 3039 Lo = DAG.getSelect(dl, NVT, isZero, InL, 3040 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 3041 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 3042 return true; 3043 case ISD::SRA: 3044 // Short: ShAmt < NVTBits 3045 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 3046 LoS = DAG.getNode(ISD::OR, dl, NVT, 3047 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 3048 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 3049 3050 // Long: ShAmt >= NVTBits 3051 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 3052 DAG.getConstant(NVTBits - 1, dl, ShTy)); 3053 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 3054 3055 Lo = DAG.getSelect(dl, NVT, isZero, InL, 3056 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 3057 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 3058 return true; 3059 } 3060 } 3061 3062 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { 3063 3064 switch (Op) { 3065 default: llvm_unreachable("invalid min/max opcode"); 3066 case ISD::SMAX: 3067 return std::make_pair(ISD::SETGT, ISD::UMAX); 3068 case ISD::UMAX: 3069 return std::make_pair(ISD::SETUGT, ISD::UMAX); 3070 case ISD::SMIN: 3071 return std::make_pair(ISD::SETLT, ISD::UMIN); 3072 case ISD::UMIN: 3073 return std::make_pair(ISD::SETULT, ISD::UMIN); 3074 } 3075 } 3076 3077 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, 3078 SDValue &Lo, SDValue &Hi) { 3079 SDLoc DL(N); 3080 3081 SDValue LHS = N->getOperand(0); 3082 SDValue RHS = N->getOperand(1); 3083 3084 // If the upper halves are all sign bits, then we can perform the MINMAX on 3085 // the lower half and sign-extend the result to the upper half. 3086 unsigned NumBits = N->getValueType(0).getScalarSizeInBits(); 3087 unsigned NumHalfBits = NumBits / 2; 3088 if (DAG.ComputeNumSignBits(LHS) > NumHalfBits && 3089 DAG.ComputeNumSignBits(RHS) > NumHalfBits) { 3090 SDValue LHSL, LHSH, RHSL, RHSH; 3091 GetExpandedInteger(LHS, LHSL, LHSH); 3092 GetExpandedInteger(RHS, RHSL, RHSH); 3093 EVT NVT = LHSL.getValueType(); 3094 3095 Lo = DAG.getNode(N->getOpcode(), DL, NVT, LHSL, RHSL); 3096 Hi = DAG.getNode(ISD::SRA, DL, NVT, Lo, 3097 DAG.getShiftAmountConstant(NumHalfBits - 1, NVT, DL)); 3098 return; 3099 } 3100 3101 // The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1. 3102 // The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL. 3103 if ((N->getOpcode() == ISD::SMAX && isNullConstant(RHS)) || 3104 (N->getOpcode() == ISD::SMIN && isAllOnesConstant(RHS))) { 3105 SDValue LHSL, LHSH, RHSL, RHSH; 3106 GetExpandedInteger(LHS, LHSL, LHSH); 3107 GetExpandedInteger(RHS, RHSL, RHSH); 3108 EVT NVT = LHSL.getValueType(); 3109 EVT CCT = getSetCCResultType(NVT); 3110 3111 SDValue HiNeg = 3112 DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT); 3113 if (N->getOpcode() == ISD::SMIN) { 3114 Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getConstant(-1, DL, NVT)); 3115 } else { 3116 Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL); 3117 } 3118 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 3119 return; 3120 } 3121 3122 const APInt *RHSVal = nullptr; 3123 if (auto *RHSConst = dyn_cast<ConstantSDNode>(RHS)) 3124 RHSVal = &RHSConst->getAPIntValue(); 3125 3126 // The high half of MIN/MAX is always just the the MIN/MAX of the 3127 // high halves of the operands. Expand this way if it appears profitable. 3128 if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) && 3129 (RHSVal->countLeadingOnes() >= NumHalfBits || 3130 RHSVal->countLeadingZeros() >= NumHalfBits)) { 3131 SDValue LHSL, LHSH, RHSL, RHSH; 3132 GetExpandedInteger(LHS, LHSL, LHSH); 3133 GetExpandedInteger(RHS, RHSL, RHSH); 3134 EVT NVT = LHSL.getValueType(); 3135 EVT CCT = getSetCCResultType(NVT); 3136 3137 ISD::NodeType LoOpc; 3138 ISD::CondCode CondC; 3139 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); 3140 3141 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 3142 // We need to know whether to select Lo part that corresponds to 'winning' 3143 // Hi part or if Hi parts are equal. 3144 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); 3145 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); 3146 3147 // Lo part corresponding to the 'winning' Hi part 3148 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); 3149 3150 // Recursed Lo part if Hi parts are equal, this uses unsigned version 3151 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); 3152 3153 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); 3154 return; 3155 } 3156 3157 // Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies 3158 // the compare. 3159 ISD::CondCode Pred; 3160 switch (N->getOpcode()) { 3161 default: llvm_unreachable("How did we get here?"); 3162 case ISD::SMAX: 3163 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) 3164 Pred = ISD::SETGE; 3165 else 3166 Pred = ISD::SETGT; 3167 break; 3168 case ISD::SMIN: 3169 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) 3170 Pred = ISD::SETLE; 3171 else 3172 Pred = ISD::SETLT; 3173 break; 3174 case ISD::UMAX: 3175 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) 3176 Pred = ISD::SETUGE; 3177 else 3178 Pred = ISD::SETUGT; 3179 break; 3180 case ISD::UMIN: 3181 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) 3182 Pred = ISD::SETULE; 3183 else 3184 Pred = ISD::SETULT; 3185 break; 3186 } 3187 EVT VT = N->getValueType(0); 3188 EVT CCT = getSetCCResultType(VT); 3189 SDValue Cond = DAG.getSetCC(DL, CCT, LHS, RHS, Pred); 3190 SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS); 3191 SplitInteger(Result, Lo, Hi); 3192 } 3193 3194 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 3195 SDValue &Lo, SDValue &Hi) { 3196 SDLoc dl(N); 3197 // Expand the subcomponents. 3198 SDValue LHSL, LHSH, RHSL, RHSH; 3199 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3200 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3201 3202 EVT NVT = LHSL.getValueType(); 3203 SDValue LoOps[2] = { LHSL, RHSL }; 3204 SDValue HiOps[3] = { LHSH, RHSH }; 3205 3206 bool HasOpCarry = TLI.isOperationLegalOrCustom( 3207 N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, 3208 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 3209 if (HasOpCarry) { 3210 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 3211 if (N->getOpcode() == ISD::ADD) { 3212 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 3213 HiOps[2] = Lo.getValue(1); 3214 Hi = DAG.computeKnownBits(HiOps[2]).isZero() 3215 ? DAG.getNode(ISD::UADDO, dl, VTList, ArrayRef(HiOps, 2)) 3216 : DAG.getNode(ISD::UADDO_CARRY, dl, VTList, HiOps); 3217 } else { 3218 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 3219 HiOps[2] = Lo.getValue(1); 3220 Hi = DAG.computeKnownBits(HiOps[2]).isZero() 3221 ? DAG.getNode(ISD::USUBO, dl, VTList, ArrayRef(HiOps, 2)) 3222 : DAG.getNode(ISD::USUBO_CARRY, dl, VTList, HiOps); 3223 } 3224 return; 3225 } 3226 3227 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 3228 // them. TODO: Teach operation legalization how to expand unsupported 3229 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 3230 // a carry of type MVT::Glue, but there doesn't seem to be any way to 3231 // generate a value of this type in the expanded code sequence. 3232 bool hasCarry = 3233 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 3234 ISD::ADDC : ISD::SUBC, 3235 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 3236 3237 if (hasCarry) { 3238 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 3239 if (N->getOpcode() == ISD::ADD) { 3240 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 3241 HiOps[2] = Lo.getValue(1); 3242 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 3243 } else { 3244 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 3245 HiOps[2] = Lo.getValue(1); 3246 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 3247 } 3248 return; 3249 } 3250 3251 bool hasOVF = 3252 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 3253 ISD::UADDO : ISD::USUBO, 3254 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 3255 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); 3256 3257 if (hasOVF) { 3258 EVT OvfVT = getSetCCResultType(NVT); 3259 SDVTList VTList = DAG.getVTList(NVT, OvfVT); 3260 int RevOpc; 3261 if (N->getOpcode() == ISD::ADD) { 3262 RevOpc = ISD::SUB; 3263 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 3264 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2)); 3265 } else { 3266 RevOpc = ISD::ADD; 3267 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 3268 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2)); 3269 } 3270 SDValue OVF = Lo.getValue(1); 3271 3272 switch (BoolType) { 3273 case TargetLoweringBase::UndefinedBooleanContent: 3274 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); 3275 [[fallthrough]]; 3276 case TargetLoweringBase::ZeroOrOneBooleanContent: 3277 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); 3278 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); 3279 break; 3280 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 3281 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); 3282 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); 3283 } 3284 return; 3285 } 3286 3287 if (N->getOpcode() == ISD::ADD) { 3288 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 3289 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2)); 3290 SDValue Cmp; 3291 // Special case: X+1 has a carry out if X+1==0. This may reduce the live 3292 // range of X. We assume comparing with 0 is cheap. 3293 if (isOneConstant(LoOps[1])) 3294 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 3295 DAG.getConstant(0, dl, NVT), ISD::SETEQ); 3296 else if (isAllOnesConstant(LoOps[1])) { 3297 if (isAllOnesConstant(HiOps[1])) 3298 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0], 3299 DAG.getConstant(0, dl, NVT), ISD::SETEQ); 3300 else 3301 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0], 3302 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3303 } else 3304 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 3305 ISD::SETULT); 3306 3307 SDValue Carry; 3308 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 3309 Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT); 3310 else 3311 Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 3312 DAG.getConstant(0, dl, NVT)); 3313 3314 if (isAllOnesConstant(LoOps[1]) && isAllOnesConstant(HiOps[1])) 3315 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps[0], Carry); 3316 else 3317 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); 3318 } else { 3319 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 3320 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2)); 3321 SDValue Cmp = 3322 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 3323 LoOps[0], LoOps[1], ISD::SETULT); 3324 3325 SDValue Borrow; 3326 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 3327 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); 3328 else 3329 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 3330 DAG.getConstant(0, dl, NVT)); 3331 3332 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 3333 } 3334 } 3335 3336 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 3337 SDValue &Lo, SDValue &Hi) { 3338 // Expand the subcomponents. 3339 SDValue LHSL, LHSH, RHSL, RHSH; 3340 SDLoc dl(N); 3341 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3342 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3343 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 3344 SDValue LoOps[2] = { LHSL, RHSL }; 3345 SDValue HiOps[3] = { LHSH, RHSH }; 3346 3347 if (N->getOpcode() == ISD::ADDC) { 3348 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 3349 HiOps[2] = Lo.getValue(1); 3350 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 3351 } else { 3352 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 3353 HiOps[2] = Lo.getValue(1); 3354 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 3355 } 3356 3357 // Legalized the flag result - switch anything that used the old flag to 3358 // use the new one. 3359 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3360 } 3361 3362 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 3363 SDValue &Lo, SDValue &Hi) { 3364 // Expand the subcomponents. 3365 SDValue LHSL, LHSH, RHSL, RHSH; 3366 SDLoc dl(N); 3367 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3368 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3369 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 3370 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 3371 SDValue HiOps[3] = { LHSH, RHSH }; 3372 3373 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3374 HiOps[2] = Lo.getValue(1); 3375 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 3376 3377 // Legalized the flag result - switch anything that used the old flag to 3378 // use the new one. 3379 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3380 } 3381 3382 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 3383 SDValue &Lo, SDValue &Hi) { 3384 SDValue LHS = N->getOperand(0); 3385 SDValue RHS = N->getOperand(1); 3386 SDLoc dl(N); 3387 3388 SDValue Ovf; 3389 3390 unsigned CarryOp, NoCarryOp; 3391 ISD::CondCode Cond; 3392 switch(N->getOpcode()) { 3393 case ISD::UADDO: 3394 CarryOp = ISD::UADDO_CARRY; 3395 NoCarryOp = ISD::ADD; 3396 Cond = ISD::SETULT; 3397 break; 3398 case ISD::USUBO: 3399 CarryOp = ISD::USUBO_CARRY; 3400 NoCarryOp = ISD::SUB; 3401 Cond = ISD::SETUGT; 3402 break; 3403 default: 3404 llvm_unreachable("Node has unexpected Opcode"); 3405 } 3406 3407 bool HasCarryOp = TLI.isOperationLegalOrCustom( 3408 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 3409 3410 if (HasCarryOp) { 3411 // Expand the subcomponents. 3412 SDValue LHSL, LHSH, RHSL, RHSH; 3413 GetExpandedInteger(LHS, LHSL, LHSH); 3414 GetExpandedInteger(RHS, RHSL, RHSH); 3415 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3416 SDValue LoOps[2] = { LHSL, RHSL }; 3417 SDValue HiOps[3] = { LHSH, RHSH }; 3418 3419 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3420 HiOps[2] = Lo.getValue(1); 3421 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps); 3422 3423 Ovf = Hi.getValue(1); 3424 } else { 3425 // Expand the result by simply replacing it with the equivalent 3426 // non-overflow-checking operation. 3427 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS); 3428 SplitInteger(Sum, Lo, Hi); 3429 3430 if (N->getOpcode() == ISD::UADDO && isOneConstant(RHS)) { 3431 // Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this 3432 // with (Lo | Hi) == 0. 3433 SDValue Or = DAG.getNode(ISD::OR, dl, Lo.getValueType(), Lo, Hi); 3434 Ovf = DAG.getSetCC(dl, N->getValueType(1), Or, 3435 DAG.getConstant(0, dl, Lo.getValueType()), ISD::SETEQ); 3436 } else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(RHS)) { 3437 // Special case: uaddo X, -1 overflows if X == 0. 3438 Ovf = 3439 DAG.getSetCC(dl, N->getValueType(1), LHS, 3440 DAG.getConstant(0, dl, LHS.getValueType()), ISD::SETNE); 3441 } else { 3442 // Calculate the overflow: addition overflows iff a + b < a, and 3443 // subtraction overflows iff a - b > a. 3444 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); 3445 } 3446 } 3447 3448 // Legalized the flag result - switch anything that used the old flag to 3449 // use the new one. 3450 ReplaceValueWith(SDValue(N, 1), Ovf); 3451 } 3452 3453 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo, 3454 SDValue &Hi) { 3455 // Expand the subcomponents. 3456 SDValue LHSL, LHSH, RHSL, RHSH; 3457 SDLoc dl(N); 3458 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3459 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3460 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3461 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 3462 SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; 3463 3464 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3465 HiOps[2] = Lo.getValue(1); 3466 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 3467 3468 // Legalized the flag result - switch anything that used the old flag to 3469 // use the new one. 3470 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3471 } 3472 3473 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, 3474 SDValue &Lo, SDValue &Hi) { 3475 // Expand the subcomponents. 3476 SDValue LHSL, LHSH, RHSL, RHSH; 3477 SDLoc dl(N); 3478 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3479 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3480 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3481 3482 // We need to use an unsigned carry op for the lo part. 3483 unsigned CarryOp = 3484 N->getOpcode() == ISD::SADDO_CARRY ? ISD::UADDO_CARRY : ISD::USUBO_CARRY; 3485 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) }); 3486 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 3487 3488 // Legalized the flag result - switch anything that used the old flag to 3489 // use the new one. 3490 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3491 } 3492 3493 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 3494 SDValue &Lo, SDValue &Hi) { 3495 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3496 SDLoc dl(N); 3497 SDValue Op = N->getOperand(0); 3498 if (Op.getValueType().bitsLE(NVT)) { 3499 // The low part is any extension of the input (which degenerates to a copy). 3500 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 3501 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 3502 } else { 3503 // For example, extension of an i48 to an i64. The operand type necessarily 3504 // promotes to the result type, so will end up being expanded too. 3505 assert(getTypeAction(Op.getValueType()) == 3506 TargetLowering::TypePromoteInteger && 3507 "Only know how to promote this result!"); 3508 SDValue Res = GetPromotedInteger(Op); 3509 assert(Res.getValueType() == N->getValueType(0) && 3510 "Operand over promoted?"); 3511 // Split the promoted operand. This will simplify when it is expanded. 3512 SplitInteger(Res, Lo, Hi); 3513 } 3514 } 3515 3516 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 3517 SDValue &Lo, SDValue &Hi) { 3518 SDLoc dl(N); 3519 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3520 EVT NVT = Lo.getValueType(); 3521 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3522 unsigned NVTBits = NVT.getSizeInBits(); 3523 unsigned EVTBits = EVT.getSizeInBits(); 3524 3525 if (NVTBits < EVTBits) { 3526 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 3527 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3528 EVTBits - NVTBits))); 3529 } else { 3530 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 3531 // The high part replicates the sign bit of Lo, make it explicit. 3532 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3533 DAG.getConstant(NVTBits - 1, dl, 3534 TLI.getPointerTy(DAG.getDataLayout()))); 3535 } 3536 } 3537 3538 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 3539 SDValue &Lo, SDValue &Hi) { 3540 SDLoc dl(N); 3541 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3542 EVT NVT = Lo.getValueType(); 3543 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3544 unsigned NVTBits = NVT.getSizeInBits(); 3545 unsigned EVTBits = EVT.getSizeInBits(); 3546 3547 if (NVTBits < EVTBits) { 3548 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 3549 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3550 EVTBits - NVTBits))); 3551 } else { 3552 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 3553 // The high part must be zero, make it explicit. 3554 Hi = DAG.getConstant(0, dl, NVT); 3555 } 3556 } 3557 3558 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, 3559 SDValue &Lo, SDValue &Hi) { 3560 SDLoc dl(N); 3561 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 3562 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); 3563 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); 3564 } 3565 3566 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 3567 SDValue &Lo, SDValue &Hi) { 3568 SDLoc dl(N); 3569 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 3570 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 3571 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 3572 } 3573 3574 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, 3575 SDValue &Hi) { 3576 SDLoc dl(N); 3577 // parity(HiLo) -> parity(Lo^Hi) 3578 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3579 EVT NVT = Lo.getValueType(); 3580 Lo = 3581 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi)); 3582 Hi = DAG.getConstant(0, dl, NVT); 3583 } 3584 3585 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 3586 SDValue &Lo, SDValue &Hi) { 3587 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3588 unsigned NBitWidth = NVT.getSizeInBits(); 3589 auto Constant = cast<ConstantSDNode>(N); 3590 const APInt &Cst = Constant->getAPIntValue(); 3591 bool IsTarget = Constant->isTargetOpcode(); 3592 bool IsOpaque = Constant->isOpaque(); 3593 SDLoc dl(N); 3594 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); 3595 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, 3596 IsOpaque); 3597 } 3598 3599 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { 3600 SDLoc dl(N); 3601 3602 SDValue N0 = N->getOperand(0); 3603 GetExpandedInteger(N0, Lo, Hi); 3604 EVT NVT = Lo.getValueType(); 3605 3606 // If the upper half is all sign bits, then we can perform the ABS on the 3607 // lower half and zero-extend. 3608 if (DAG.ComputeNumSignBits(N0) > NVT.getScalarSizeInBits()) { 3609 Lo = DAG.getNode(ISD::ABS, dl, NVT, Lo); 3610 Hi = DAG.getConstant(0, dl, NVT); 3611 return; 3612 } 3613 3614 // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence 3615 // we use in LegalizeDAG. The SUB part of the expansion is based on 3616 // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that 3617 // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further 3618 // expanded if needed. Shift expansion has a special case for filling with 3619 // sign bits so that we will only end up with one SRA. 3620 bool HasSubCarry = TLI.isOperationLegalOrCustom( 3621 ISD::USUBO_CARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 3622 if (HasSubCarry) { 3623 SDValue Sign = DAG.getNode( 3624 ISD::SRA, dl, NVT, Hi, 3625 DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl)); 3626 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 3627 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign); 3628 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign); 3629 Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign); 3630 Hi = DAG.getNode(ISD::USUBO_CARRY, dl, VTList, Hi, Sign, Lo.getValue(1)); 3631 return; 3632 } 3633 3634 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) 3635 EVT VT = N->getValueType(0); 3636 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, 3637 DAG.getConstant(0, dl, VT), N0); 3638 SDValue NegLo, NegHi; 3639 SplitInteger(Neg, NegLo, NegHi); 3640 3641 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 3642 DAG.getConstant(0, dl, NVT), ISD::SETLT); 3643 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); 3644 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); 3645 } 3646 3647 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 3648 SDValue &Lo, SDValue &Hi) { 3649 SDLoc dl(N); 3650 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 3651 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3652 EVT NVT = Lo.getValueType(); 3653 3654 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 3655 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3656 3657 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 3658 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 3659 3660 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 3661 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 3662 DAG.getConstant(NVT.getSizeInBits(), dl, 3663 NVT))); 3664 Hi = DAG.getConstant(0, dl, NVT); 3665 } 3666 3667 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 3668 SDValue &Lo, SDValue &Hi) { 3669 SDLoc dl(N); 3670 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 3671 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3672 EVT NVT = Lo.getValueType(); 3673 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 3674 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 3675 Hi = DAG.getConstant(0, dl, NVT); 3676 } 3677 3678 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 3679 SDValue &Lo, SDValue &Hi) { 3680 SDLoc dl(N); 3681 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 3682 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3683 EVT NVT = Lo.getValueType(); 3684 3685 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 3686 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3687 3688 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 3689 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 3690 3691 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 3692 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 3693 DAG.getConstant(NVT.getSizeInBits(), dl, 3694 NVT))); 3695 Hi = DAG.getConstant(0, dl, NVT); 3696 } 3697 3698 void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo, 3699 SDValue &Hi) { 3700 SDLoc dl(N); 3701 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3702 unsigned NBitWidth = NVT.getSizeInBits(); 3703 3704 Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0)); 3705 SDValue Chain = Lo.getValue(1); 3706 // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING 3707 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3708 DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl)); 3709 3710 // Legalize the chain result - switch anything that used the old chain to 3711 // use the new one. 3712 ReplaceValueWith(SDValue(N, 1), Chain); 3713 } 3714 3715 // Helper for producing an FP_EXTEND/STRICT_FP_EXTEND of Op. 3716 static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, 3717 SDLoc DL, SelectionDAG &DAG) { 3718 if (IsStrict) { 3719 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}); 3720 Chain = Op.getValue(1); 3721 return Op; 3722 } 3723 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Op); 3724 } 3725 3726 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo, 3727 SDValue &Hi) { 3728 SDLoc dl(N); 3729 EVT VT = N->getValueType(0); 3730 3731 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || 3732 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 3733 bool IsStrict = N->isStrictFPOpcode(); 3734 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3735 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3736 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3737 Op = GetPromotedFloat(Op); 3738 3739 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3740 EVT OFPVT = Op.getValueType(); 3741 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), OFPVT); 3742 Op = GetSoftPromotedHalf(Op); 3743 Op = DAG.getNode(OFPVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP, dl, 3744 NFPVT, Op); 3745 Op = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, VT, Op); 3746 SplitInteger(Op, Lo, Hi); 3747 return; 3748 } 3749 3750 if (Op.getValueType() == MVT::bf16) { 3751 // Extend to f32 as there is no bf16 libcall. 3752 Op = fpExtendHelper(Op, Chain, IsStrict, MVT::f32, dl, DAG); 3753 } 3754 3755 RTLIB::Libcall LC = IsSigned ? RTLIB::getFPTOSINT(Op.getValueType(), VT) 3756 : RTLIB::getFPTOUINT(Op.getValueType(), VT); 3757 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-xint conversion!"); 3758 TargetLowering::MakeLibCallOptions CallOptions; 3759 CallOptions.setSExt(true); 3760 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3761 CallOptions, dl, Chain); 3762 SplitInteger(Tmp.first, Lo, Hi); 3763 3764 if (IsStrict) 3765 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3766 } 3767 3768 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, 3769 SDValue &Hi) { 3770 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 3771 SplitInteger(Res, Lo, Hi); 3772 } 3773 3774 void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo, 3775 SDValue &Hi) { 3776 SDLoc dl(N); 3777 bool IsStrict = N->isStrictFPOpcode(); 3778 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3779 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3780 3781 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && 3782 "Input type needs to be promoted!"); 3783 3784 EVT VT = Op.getValueType(); 3785 3786 if (VT == MVT::f16) { 3787 // Extend to f32. 3788 VT = MVT::f32; 3789 Op = fpExtendHelper(Op, Chain, IsStrict, VT, dl, DAG); 3790 } 3791 3792 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3793 if (N->getOpcode() == ISD::LROUND || 3794 N->getOpcode() == ISD::STRICT_LROUND) { 3795 if (VT == MVT::f32) 3796 LC = RTLIB::LROUND_F32; 3797 else if (VT == MVT::f64) 3798 LC = RTLIB::LROUND_F64; 3799 else if (VT == MVT::f80) 3800 LC = RTLIB::LROUND_F80; 3801 else if (VT == MVT::f128) 3802 LC = RTLIB::LROUND_F128; 3803 else if (VT == MVT::ppcf128) 3804 LC = RTLIB::LROUND_PPCF128; 3805 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!"); 3806 } else if (N->getOpcode() == ISD::LRINT || 3807 N->getOpcode() == ISD::STRICT_LRINT) { 3808 if (VT == MVT::f32) 3809 LC = RTLIB::LRINT_F32; 3810 else if (VT == MVT::f64) 3811 LC = RTLIB::LRINT_F64; 3812 else if (VT == MVT::f80) 3813 LC = RTLIB::LRINT_F80; 3814 else if (VT == MVT::f128) 3815 LC = RTLIB::LRINT_F128; 3816 else if (VT == MVT::ppcf128) 3817 LC = RTLIB::LRINT_PPCF128; 3818 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!"); 3819 } else if (N->getOpcode() == ISD::LLROUND || 3820 N->getOpcode() == ISD::STRICT_LLROUND) { 3821 if (VT == MVT::f32) 3822 LC = RTLIB::LLROUND_F32; 3823 else if (VT == MVT::f64) 3824 LC = RTLIB::LLROUND_F64; 3825 else if (VT == MVT::f80) 3826 LC = RTLIB::LLROUND_F80; 3827 else if (VT == MVT::f128) 3828 LC = RTLIB::LLROUND_F128; 3829 else if (VT == MVT::ppcf128) 3830 LC = RTLIB::LLROUND_PPCF128; 3831 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); 3832 } else if (N->getOpcode() == ISD::LLRINT || 3833 N->getOpcode() == ISD::STRICT_LLRINT) { 3834 if (VT == MVT::f32) 3835 LC = RTLIB::LLRINT_F32; 3836 else if (VT == MVT::f64) 3837 LC = RTLIB::LLRINT_F64; 3838 else if (VT == MVT::f80) 3839 LC = RTLIB::LLRINT_F80; 3840 else if (VT == MVT::f128) 3841 LC = RTLIB::LLRINT_F128; 3842 else if (VT == MVT::ppcf128) 3843 LC = RTLIB::LLRINT_PPCF128; 3844 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); 3845 } else 3846 llvm_unreachable("Unexpected opcode!"); 3847 3848 EVT RetVT = N->getValueType(0); 3849 3850 TargetLowering::MakeLibCallOptions CallOptions; 3851 CallOptions.setSExt(true); 3852 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, 3853 Op, CallOptions, dl, 3854 Chain); 3855 SplitInteger(Tmp.first, Lo, Hi); 3856 3857 if (N->isStrictFPOpcode()) 3858 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3859 } 3860 3861 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 3862 SDValue &Lo, SDValue &Hi) { 3863 assert(!N->isAtomic() && "Should have been a ATOMIC_LOAD?"); 3864 3865 if (ISD::isNormalLoad(N)) { 3866 ExpandRes_NormalLoad(N, Lo, Hi); 3867 return; 3868 } 3869 3870 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 3871 3872 EVT VT = N->getValueType(0); 3873 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3874 SDValue Ch = N->getChain(); 3875 SDValue Ptr = N->getBasePtr(); 3876 ISD::LoadExtType ExtType = N->getExtensionType(); 3877 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 3878 AAMDNodes AAInfo = N->getAAInfo(); 3879 SDLoc dl(N); 3880 3881 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 3882 3883 if (N->getMemoryVT().bitsLE(NVT)) { 3884 EVT MemVT = N->getMemoryVT(); 3885 3886 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, 3887 N->getOriginalAlign(), MMOFlags, AAInfo); 3888 3889 // Remember the chain. 3890 Ch = Lo.getValue(1); 3891 3892 if (ExtType == ISD::SEXTLOAD) { 3893 // The high part is obtained by SRA'ing all but one of the bits of the 3894 // lo part. 3895 unsigned LoSize = Lo.getValueSizeInBits(); 3896 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3897 DAG.getConstant(LoSize - 1, dl, 3898 TLI.getPointerTy(DAG.getDataLayout()))); 3899 } else if (ExtType == ISD::ZEXTLOAD) { 3900 // The high part is just a zero. 3901 Hi = DAG.getConstant(0, dl, NVT); 3902 } else { 3903 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 3904 // The high part is undefined. 3905 Hi = DAG.getUNDEF(NVT); 3906 } 3907 } else if (DAG.getDataLayout().isLittleEndian()) { 3908 // Little-endian - low bits are at low addresses. 3909 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 3910 N->getOriginalAlign(), MMOFlags, AAInfo); 3911 3912 unsigned ExcessBits = 3913 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 3914 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 3915 3916 // Increment the pointer to the other half. 3917 unsigned IncrementSize = NVT.getSizeInBits()/8; 3918 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl); 3919 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 3920 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 3921 N->getOriginalAlign(), MMOFlags, AAInfo); 3922 3923 // Build a factor node to remember that this load is independent of the 3924 // other one. 3925 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3926 Hi.getValue(1)); 3927 } else { 3928 // Big-endian - high bits are at low addresses. Favor aligned loads at 3929 // the cost of some bit-fiddling. 3930 EVT MemVT = N->getMemoryVT(); 3931 unsigned EBytes = MemVT.getStoreSize(); 3932 unsigned IncrementSize = NVT.getSizeInBits()/8; 3933 unsigned ExcessBits = (EBytes - IncrementSize)*8; 3934 3935 // Load both the high bits and maybe some of the low bits. 3936 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 3937 EVT::getIntegerVT(*DAG.getContext(), 3938 MemVT.getSizeInBits() - ExcessBits), 3939 N->getOriginalAlign(), MMOFlags, AAInfo); 3940 3941 // Increment the pointer to the other half. 3942 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl); 3943 // Load the rest of the low bits. 3944 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 3945 N->getPointerInfo().getWithOffset(IncrementSize), 3946 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 3947 N->getOriginalAlign(), MMOFlags, AAInfo); 3948 3949 // Build a factor node to remember that this load is independent of the 3950 // other one. 3951 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3952 Hi.getValue(1)); 3953 3954 if (ExcessBits < NVT.getSizeInBits()) { 3955 // Transfer low bits from the bottom of Hi to the top of Lo. 3956 Lo = DAG.getNode( 3957 ISD::OR, dl, NVT, Lo, 3958 DAG.getNode(ISD::SHL, dl, NVT, Hi, 3959 DAG.getConstant(ExcessBits, dl, 3960 TLI.getPointerTy(DAG.getDataLayout())))); 3961 // Move high bits to the right position in Hi. 3962 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, 3963 Hi, 3964 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 3965 TLI.getPointerTy(DAG.getDataLayout()))); 3966 } 3967 } 3968 3969 // Legalize the chain result - switch anything that used the old chain to 3970 // use the new one. 3971 ReplaceValueWith(SDValue(N, 1), Ch); 3972 } 3973 3974 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 3975 SDValue &Lo, SDValue &Hi) { 3976 SDLoc dl(N); 3977 SDValue LL, LH, RL, RH; 3978 GetExpandedInteger(N->getOperand(0), LL, LH); 3979 GetExpandedInteger(N->getOperand(1), RL, RH); 3980 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 3981 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 3982 } 3983 3984 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 3985 SDValue &Lo, SDValue &Hi) { 3986 EVT VT = N->getValueType(0); 3987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3988 SDLoc dl(N); 3989 3990 SDValue LL, LH, RL, RH; 3991 GetExpandedInteger(N->getOperand(0), LL, LH); 3992 GetExpandedInteger(N->getOperand(1), RL, RH); 3993 3994 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, 3995 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3996 LL, LH, RL, RH)) 3997 return; 3998 3999 // If nothing else, we can make a libcall. 4000 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4001 if (VT == MVT::i16) 4002 LC = RTLIB::MUL_I16; 4003 else if (VT == MVT::i32) 4004 LC = RTLIB::MUL_I32; 4005 else if (VT == MVT::i64) 4006 LC = RTLIB::MUL_I64; 4007 else if (VT == MVT::i128) 4008 LC = RTLIB::MUL_I128; 4009 4010 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 4011 // We'll expand the multiplication by brute force because we have no other 4012 // options. This is a trivially-generalized version of the code from 4013 // Hacker's Delight (itself derived from Knuth's Algorithm M from section 4014 // 4.3.1). 4015 unsigned Bits = NVT.getSizeInBits(); 4016 unsigned HalfBits = Bits >> 1; 4017 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, 4018 NVT); 4019 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); 4020 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); 4021 4022 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); 4023 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); 4024 4025 SDValue Shift = DAG.getShiftAmountConstant(HalfBits, NVT, dl); 4026 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); 4027 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); 4028 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); 4029 4030 SDValue U = DAG.getNode(ISD::ADD, dl, NVT, 4031 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); 4032 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); 4033 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); 4034 4035 SDValue V = DAG.getNode(ISD::ADD, dl, NVT, 4036 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); 4037 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); 4038 4039 SDValue W = DAG.getNode(ISD::ADD, dl, NVT, 4040 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), 4041 DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); 4042 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, 4043 DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); 4044 4045 Hi = DAG.getNode(ISD::ADD, dl, NVT, W, 4046 DAG.getNode(ISD::ADD, dl, NVT, 4047 DAG.getNode(ISD::MUL, dl, NVT, RH, LL), 4048 DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); 4049 return; 4050 } 4051 4052 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4053 TargetLowering::MakeLibCallOptions CallOptions; 4054 CallOptions.setSExt(true); 4055 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, 4056 Lo, Hi); 4057 } 4058 4059 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, 4060 SDValue &Hi) { 4061 SDLoc DL(N); 4062 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4063 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); 4064 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); 4065 Lo = R.getValue(0); 4066 Hi = R.getValue(1); 4067 ReplaceValueWith(SDValue(N, 1), R.getValue(2)); 4068 } 4069 4070 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, 4071 SDValue &Hi) { 4072 SDValue Result = TLI.expandAddSubSat(N, DAG); 4073 SplitInteger(Result, Lo, Hi); 4074 } 4075 4076 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, 4077 SDValue &Hi) { 4078 SDValue Result = TLI.expandShlSat(N, DAG); 4079 SplitInteger(Result, Lo, Hi); 4080 } 4081 4082 /// This performs an expansion of the integer result for a fixed point 4083 /// multiplication. The default expansion performs rounding down towards 4084 /// negative infinity, though targets that do care about rounding should specify 4085 /// a target hook for rounding and provide their own expansion or lowering of 4086 /// fixed point multiplication to be consistent with rounding. 4087 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, 4088 SDValue &Hi) { 4089 SDLoc dl(N); 4090 EVT VT = N->getValueType(0); 4091 unsigned VTSize = VT.getScalarSizeInBits(); 4092 SDValue LHS = N->getOperand(0); 4093 SDValue RHS = N->getOperand(1); 4094 uint64_t Scale = N->getConstantOperandVal(2); 4095 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || 4096 N->getOpcode() == ISD::UMULFIXSAT); 4097 bool Signed = (N->getOpcode() == ISD::SMULFIX || 4098 N->getOpcode() == ISD::SMULFIXSAT); 4099 4100 // Handle special case when scale is equal to zero. 4101 if (!Scale) { 4102 SDValue Result; 4103 if (!Saturating) { 4104 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 4105 } else { 4106 EVT BoolVT = getSetCCResultType(VT); 4107 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; 4108 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); 4109 SDValue Product = Result.getValue(0); 4110 SDValue Overflow = Result.getValue(1); 4111 if (Signed) { 4112 APInt MinVal = APInt::getSignedMinValue(VTSize); 4113 APInt MaxVal = APInt::getSignedMaxValue(VTSize); 4114 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); 4115 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 4116 SDValue Zero = DAG.getConstant(0, dl, VT); 4117 // Xor the inputs, if resulting sign bit is 0 the product will be 4118 // positive, else negative. 4119 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); 4120 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); 4121 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); 4122 Result = DAG.getSelect(dl, VT, Overflow, Result, Product); 4123 } else { 4124 // For unsigned multiplication, we only need to check the max since we 4125 // can't really overflow towards zero. 4126 APInt MaxVal = APInt::getMaxValue(VTSize); 4127 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 4128 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product); 4129 } 4130 } 4131 SplitInteger(Result, Lo, Hi); 4132 return; 4133 } 4134 4135 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will 4136 // cover for unhandled cases below, while still being valid for UMULFIX[SAT]. 4137 assert(Scale <= VTSize && "Scale can't be larger than the value type size."); 4138 4139 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4140 SDValue LL, LH, RL, RH; 4141 GetExpandedInteger(LHS, LL, LH); 4142 GetExpandedInteger(RHS, RL, RH); 4143 SmallVector<SDValue, 4> Result; 4144 4145 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; 4146 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, 4147 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 4148 LL, LH, RL, RH)) { 4149 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); 4150 return; 4151 } 4152 4153 unsigned NVTSize = NVT.getScalarSizeInBits(); 4154 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " 4155 "the size of the current value type"); 4156 4157 // After getting the multiplication result in 4 parts, we need to perform a 4158 // shift right by the amount of the scale to get the result in that scale. 4159 // 4160 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in 4161 // 128 bits that are cut into 4 32-bit parts: 4162 // 4163 // HH HL LH LL 4164 // |---32---|---32---|---32---|---32---| 4165 // 128 96 64 32 0 4166 // 4167 // |------VTSize-----| 4168 // 4169 // |NVTSize-| 4170 // 4171 // The resulting Lo and Hi would normally be in LL and LH after the shift. But 4172 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift 4173 // amount and get Lo and Hi using two funnel shifts. Or for the special case 4174 // when Scale is a multiple of NVTSize we can just pick the result without 4175 // shifting. 4176 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. 4177 if (Scale % NVTSize) { 4178 SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl); 4179 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0], 4180 ShiftAmount); 4181 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1], 4182 ShiftAmount); 4183 } else { 4184 Lo = Result[Part0]; 4185 Hi = Result[Part0 + 1]; 4186 } 4187 4188 // Unless saturation is requested we are done. The result is in <Hi,Lo>. 4189 if (!Saturating) 4190 return; 4191 4192 // Can not overflow when there is no integer part. 4193 if (Scale == VTSize) 4194 return; 4195 4196 // To handle saturation we must check for overflow in the multiplication. 4197 // 4198 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) 4199 // aren't all zeroes. 4200 // 4201 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) 4202 // aren't all ones or all zeroes. 4203 // 4204 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the 4205 // highest bit of HH determines saturation direction in the event of signed 4206 // saturation. 4207 4208 SDValue ResultHL = Result[2]; 4209 SDValue ResultHH = Result[3]; 4210 4211 SDValue SatMax, SatMin; 4212 SDValue NVTZero = DAG.getConstant(0, dl, NVT); 4213 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); 4214 EVT BoolNVT = getSetCCResultType(NVT); 4215 4216 if (!Signed) { 4217 if (Scale < NVTSize) { 4218 // Overflow happened if ((HH | (HL >> Scale)) != 0). 4219 SDValue HLAdjusted = 4220 DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 4221 DAG.getShiftAmountConstant(Scale, NVT, dl)); 4222 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH); 4223 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE); 4224 } else if (Scale == NVTSize) { 4225 // Overflow happened if (HH != 0). 4226 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE); 4227 } else if (Scale < VTSize) { 4228 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0). 4229 SDValue HLAdjusted = 4230 DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 4231 DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl)); 4232 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE); 4233 } else 4234 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" 4235 "(and saturation can't happen with Scale==VTSize)."); 4236 4237 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi); 4238 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo); 4239 return; 4240 } 4241 4242 if (Scale < NVTSize) { 4243 // The number of overflow bits we can check are VTSize - Scale + 1 (we 4244 // include the sign bit). If these top bits are > 0, then we overflowed past 4245 // the max value. If these top bits are < -1, then we overflowed past the 4246 // min value. Otherwise, we did not overflow. 4247 unsigned OverflowBits = VTSize - Scale + 1; 4248 assert(OverflowBits <= VTSize && OverflowBits > NVTSize && 4249 "Extent of overflow bits must start within HL"); 4250 SDValue HLHiMask = DAG.getConstant( 4251 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); 4252 SDValue HLLoMask = DAG.getConstant( 4253 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); 4254 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). 4255 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 4256 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 4257 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); 4258 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 4259 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT)); 4260 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). 4261 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 4262 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 4263 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); 4264 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 4265 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT)); 4266 } else if (Scale == NVTSize) { 4267 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). 4268 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 4269 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 4270 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); 4271 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 4272 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg)); 4273 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). 4274 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 4275 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 4276 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); 4277 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 4278 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos)); 4279 } else if (Scale < VTSize) { 4280 // This is similar to the case when we saturate if Scale < NVTSize, but we 4281 // only need to check HH. 4282 unsigned OverflowBits = VTSize - Scale + 1; 4283 SDValue HHHiMask = DAG.getConstant( 4284 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); 4285 SDValue HHLoMask = DAG.getConstant( 4286 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); 4287 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); 4288 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); 4289 } else 4290 llvm_unreachable("Illegal scale for signed fixed point mul."); 4291 4292 // Saturate to signed maximum. 4293 APInt MaxHi = APInt::getSignedMaxValue(NVTSize); 4294 APInt MaxLo = APInt::getAllOnes(NVTSize); 4295 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi); 4296 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo); 4297 // Saturate to signed minimum. 4298 APInt MinHi = APInt::getSignedMinValue(NVTSize); 4299 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi); 4300 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); 4301 } 4302 4303 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, 4304 SDValue &Hi) { 4305 SDLoc dl(N); 4306 // Try expanding in the existing type first. 4307 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0), 4308 N->getOperand(1), 4309 N->getConstantOperandVal(2), DAG); 4310 4311 if (!Res) 4312 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1), 4313 N->getConstantOperandVal(2), TLI, DAG); 4314 SplitInteger(Res, Lo, Hi); 4315 } 4316 4317 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 4318 SDValue &Lo, SDValue &Hi) { 4319 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && 4320 "Node has unexpected Opcode"); 4321 SDValue LHS = Node->getOperand(0); 4322 SDValue RHS = Node->getOperand(1); 4323 SDLoc dl(Node); 4324 4325 SDValue Ovf; 4326 4327 bool IsAdd = Node->getOpcode() == ISD::SADDO; 4328 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY; 4329 4330 bool HasCarryOp = TLI.isOperationLegalOrCustom( 4331 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 4332 4333 if (HasCarryOp) { 4334 // Expand the subcomponents. 4335 SDValue LHSL, LHSH, RHSL, RHSH; 4336 GetExpandedInteger(LHS, LHSL, LHSH); 4337 GetExpandedInteger(RHS, RHSL, RHSH); 4338 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1)); 4339 4340 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL}); 4341 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 4342 4343 Ovf = Hi.getValue(1); 4344 } else { 4345 // Expand the result by simply replacing it with the equivalent 4346 // non-overflow-checking operation. 4347 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 4348 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 4349 LHS, RHS); 4350 SplitInteger(Sum, Lo, Hi); 4351 4352 // Compute the overflow. 4353 // 4354 // LHSSign -> LHS < 0 4355 // RHSSign -> RHS < 0 4356 // SumSign -> Sum < 0 4357 // 4358 // Add: 4359 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 4360 // Sub: 4361 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 4362 // 4363 // To get better codegen we can rewrite this by doing bitwise math on 4364 // the integers and extract the final sign bit at the end. So the 4365 // above becomes: 4366 // 4367 // Add: 4368 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0 4369 // Sub: 4370 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0 4371 // 4372 // NOTE: This is different than the expansion we do in expandSADDSUBO 4373 // because it is more costly to determine the RHS is > 0 for SSUBO with the 4374 // integers split. 4375 EVT VT = LHS.getValueType(); 4376 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); 4377 if (IsAdd) 4378 SignsMatch = DAG.getNOT(dl, SignsMatch, VT); 4379 4380 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum); 4381 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE); 4382 EVT OType = Node->getValueType(1); 4383 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT); 4384 } 4385 4386 // Use the calculated overflow everywhere. 4387 ReplaceValueWith(SDValue(Node, 1), Ovf); 4388 } 4389 4390 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 4391 SDValue &Lo, SDValue &Hi) { 4392 EVT VT = N->getValueType(0); 4393 SDLoc dl(N); 4394 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4395 4396 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 4397 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4398 SplitInteger(Res.getValue(0), Lo, Hi); 4399 return; 4400 } 4401 4402 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4403 if (VT == MVT::i16) 4404 LC = RTLIB::SDIV_I16; 4405 else if (VT == MVT::i32) 4406 LC = RTLIB::SDIV_I32; 4407 else if (VT == MVT::i64) 4408 LC = RTLIB::SDIV_I64; 4409 else if (VT == MVT::i128) 4410 LC = RTLIB::SDIV_I128; 4411 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 4412 4413 TargetLowering::MakeLibCallOptions CallOptions; 4414 CallOptions.setSExt(true); 4415 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4416 } 4417 4418 void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo, 4419 SDValue &Hi) { 4420 SDLoc dl(N); 4421 SDValue Shiftee = N->getOperand(0); 4422 EVT VT = Shiftee.getValueType(); 4423 SDValue ShAmt = N->getOperand(1); 4424 EVT ShAmtVT = ShAmt.getValueType(); 4425 4426 // This legalization is optimal when the shift is by a multiple of byte width, 4427 // %x * 8 <-> %x << 3 so 3 low bits should be be known zero. 4428 bool ShiftByByteMultiple = 4429 DAG.computeKnownBits(ShAmt).countMinTrailingZeros() >= 3; 4430 4431 // If we can't do it as one step, we'll have two uses of shift amount, 4432 // and thus must freeze it. 4433 if (!ShiftByByteMultiple) 4434 ShAmt = DAG.getFreeze(ShAmt); 4435 4436 unsigned VTBitWidth = VT.getScalarSizeInBits(); 4437 assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?"); 4438 unsigned VTByteWidth = VTBitWidth / 8; 4439 assert(isPowerOf2_32(VTByteWidth) && 4440 "Shiftee type size is not a power of two!"); 4441 unsigned StackSlotByteWidth = 2 * VTByteWidth; 4442 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth; 4443 EVT StackSlotVT = EVT::getIntegerVT(*DAG.getContext(), StackSlotBitWidth); 4444 4445 // Get a temporary stack slot 2x the width of our VT. 4446 // FIXME: reuse stack slots? 4447 // FIXME: should we be more picky about alignment? 4448 Align StackSlotAlignment(1); 4449 SDValue StackPtr = DAG.CreateStackTemporary( 4450 TypeSize::getFixed(StackSlotByteWidth), StackSlotAlignment); 4451 EVT PtrTy = StackPtr.getValueType(); 4452 SDValue Ch = DAG.getEntryNode(); 4453 4454 MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack( 4455 DAG.getMachineFunction(), 4456 cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex()); 4457 4458 // Extend the value, that is being shifted, to the entire stack slot's width. 4459 SDValue Init; 4460 if (N->getOpcode() != ISD::SHL) { 4461 unsigned WideningOpc = 4462 N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4463 Init = DAG.getNode(WideningOpc, dl, StackSlotVT, Shiftee); 4464 } else { 4465 // For left-shifts, pad the Shiftee's LSB with zeros to twice it's width. 4466 SDValue AllZeros = DAG.getConstant(0, dl, VT); 4467 Init = DAG.getNode(ISD::BUILD_PAIR, dl, StackSlotVT, AllZeros, Shiftee); 4468 } 4469 // And spill it into the stack slot. 4470 Ch = DAG.getStore(Ch, dl, Init, StackPtr, StackPtrInfo, StackSlotAlignment); 4471 4472 // Now, compute the full-byte offset into stack slot from where we can load. 4473 // We have shift amount, which is in bits, but in multiples of byte. 4474 // So just divide by CHAR_BIT. 4475 SDNodeFlags Flags; 4476 if (ShiftByByteMultiple) 4477 Flags.setExact(true); 4478 SDValue ByteOffset = DAG.getNode(ISD::SRL, dl, ShAmtVT, ShAmt, 4479 DAG.getConstant(3, dl, ShAmtVT), Flags); 4480 // And clamp it, because OOB load is an immediate UB, 4481 // while shift overflow would have *just* been poison. 4482 ByteOffset = DAG.getNode(ISD::AND, dl, ShAmtVT, ByteOffset, 4483 DAG.getConstant(VTByteWidth - 1, dl, ShAmtVT)); 4484 // We have exactly two strategies on indexing into stack slot here: 4485 // 1. upwards starting from the beginning of the slot 4486 // 2. downwards starting from the middle of the slot 4487 // On little-endian machine, we pick 1. for right shifts and 2. for left-shift 4488 // and vice versa on big-endian machine. 4489 bool WillIndexUpwards = N->getOpcode() != ISD::SHL; 4490 if (DAG.getDataLayout().isBigEndian()) 4491 WillIndexUpwards = !WillIndexUpwards; 4492 4493 SDValue AdjStackPtr; 4494 if (WillIndexUpwards) { 4495 AdjStackPtr = StackPtr; 4496 } else { 4497 AdjStackPtr = DAG.getMemBasePlusOffset( 4498 StackPtr, DAG.getConstant(VTByteWidth, dl, PtrTy), dl); 4499 ByteOffset = DAG.getNegative(ByteOffset, dl, ShAmtVT); 4500 } 4501 4502 // Get the pointer somewhere into the stack slot from which we need to load. 4503 ByteOffset = DAG.getSExtOrTrunc(ByteOffset, dl, PtrTy); 4504 AdjStackPtr = DAG.getMemBasePlusOffset(AdjStackPtr, ByteOffset, dl); 4505 4506 // And load it! While the load is not legal, legalizing it is obvious. 4507 SDValue Res = DAG.getLoad( 4508 VT, dl, Ch, AdjStackPtr, 4509 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), Align(1)); 4510 // We've performed the shift by a CHAR_BIT * [_ShAmt / CHAR_BIT_] 4511 4512 // If we may still have a less-than-CHAR_BIT to shift by, do so now. 4513 if (!ShiftByByteMultiple) { 4514 SDValue ShAmtRem = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, 4515 DAG.getConstant(7, dl, ShAmtVT)); 4516 Res = DAG.getNode(N->getOpcode(), dl, VT, Res, ShAmtRem); 4517 } 4518 4519 // Finally, split the computed value. 4520 SplitInteger(Res, Lo, Hi); 4521 } 4522 4523 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 4524 SDValue &Lo, SDValue &Hi) { 4525 EVT VT = N->getValueType(0); 4526 SDLoc dl(N); 4527 4528 // If we can emit an efficient shift operation, do so now. Check to see if 4529 // the RHS is a constant. 4530 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 4531 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); 4532 4533 // If we can determine that the high bit of the shift is zero or one, even if 4534 // the low bits are variable, emit this shift in an optimized form. 4535 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 4536 return; 4537 4538 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 4539 unsigned PartsOpc; 4540 if (N->getOpcode() == ISD::SHL) { 4541 PartsOpc = ISD::SHL_PARTS; 4542 } else if (N->getOpcode() == ISD::SRL) { 4543 PartsOpc = ISD::SRL_PARTS; 4544 } else { 4545 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 4546 PartsOpc = ISD::SRA_PARTS; 4547 } 4548 4549 // Next check to see if the target supports this SHL_PARTS operation or if it 4550 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for 4551 // size, but create a libcall instead. 4552 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4553 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 4554 const bool LegalOrCustom = 4555 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 4556 Action == TargetLowering::Custom; 4557 4558 unsigned ExpansionFactor = 1; 4559 // That VT->NVT expansion is one step. But will we re-expand NVT? 4560 for (EVT TmpVT = NVT;;) { 4561 EVT NewTMPVT = TLI.getTypeToTransformTo(*DAG.getContext(), TmpVT); 4562 if (NewTMPVT == TmpVT) 4563 break; 4564 TmpVT = NewTMPVT; 4565 ++ExpansionFactor; 4566 } 4567 4568 TargetLowering::ShiftLegalizationStrategy S = 4569 TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor); 4570 4571 if (S == TargetLowering::ShiftLegalizationStrategy::ExpandThroughStack) 4572 return ExpandIntRes_ShiftThroughStack(N, Lo, Hi); 4573 4574 if (LegalOrCustom && 4575 S != TargetLowering::ShiftLegalizationStrategy::LowerToLibcall) { 4576 // Expand the subcomponents. 4577 SDValue LHSL, LHSH; 4578 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 4579 EVT VT = LHSL.getValueType(); 4580 4581 // If the shift amount operand is coming from a vector legalization it may 4582 // have an illegal type. Fix that first by casting the operand, otherwise 4583 // the new SHL_PARTS operation would need further legalization. 4584 SDValue ShiftOp = N->getOperand(1); 4585 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 4586 if (ShiftOp.getValueType() != ShiftTy) 4587 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 4588 4589 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 4590 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 4591 Hi = Lo.getValue(1); 4592 return; 4593 } 4594 4595 // Otherwise, emit a libcall. 4596 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4597 bool isSigned; 4598 if (N->getOpcode() == ISD::SHL) { 4599 isSigned = false; /*sign irrelevant*/ 4600 if (VT == MVT::i16) 4601 LC = RTLIB::SHL_I16; 4602 else if (VT == MVT::i32) 4603 LC = RTLIB::SHL_I32; 4604 else if (VT == MVT::i64) 4605 LC = RTLIB::SHL_I64; 4606 else if (VT == MVT::i128) 4607 LC = RTLIB::SHL_I128; 4608 } else if (N->getOpcode() == ISD::SRL) { 4609 isSigned = false; 4610 if (VT == MVT::i16) 4611 LC = RTLIB::SRL_I16; 4612 else if (VT == MVT::i32) 4613 LC = RTLIB::SRL_I32; 4614 else if (VT == MVT::i64) 4615 LC = RTLIB::SRL_I64; 4616 else if (VT == MVT::i128) 4617 LC = RTLIB::SRL_I128; 4618 } else { 4619 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 4620 isSigned = true; 4621 if (VT == MVT::i16) 4622 LC = RTLIB::SRA_I16; 4623 else if (VT == MVT::i32) 4624 LC = RTLIB::SRA_I32; 4625 else if (VT == MVT::i64) 4626 LC = RTLIB::SRA_I64; 4627 else if (VT == MVT::i128) 4628 LC = RTLIB::SRA_I128; 4629 } 4630 4631 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 4632 EVT ShAmtTy = 4633 EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize()); 4634 SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy); 4635 SDValue Ops[2] = {N->getOperand(0), ShAmt}; 4636 TargetLowering::MakeLibCallOptions CallOptions; 4637 CallOptions.setSExt(isSigned); 4638 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4639 return; 4640 } 4641 4642 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 4643 llvm_unreachable("Unsupported shift!"); 4644 } 4645 4646 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 4647 SDValue &Lo, SDValue &Hi) { 4648 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4649 SDLoc dl(N); 4650 SDValue Op = N->getOperand(0); 4651 if (Op.getValueType().bitsLE(NVT)) { 4652 // The low part is sign extension of the input (degenerates to a copy). 4653 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 4654 // The high part is obtained by SRA'ing all but one of the bits of low part. 4655 unsigned LoSize = NVT.getSizeInBits(); 4656 Hi = DAG.getNode( 4657 ISD::SRA, dl, NVT, Lo, 4658 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); 4659 } else { 4660 // For example, extension of an i48 to an i64. The operand type necessarily 4661 // promotes to the result type, so will end up being expanded too. 4662 assert(getTypeAction(Op.getValueType()) == 4663 TargetLowering::TypePromoteInteger && 4664 "Only know how to promote this result!"); 4665 SDValue Res = GetPromotedInteger(Op); 4666 assert(Res.getValueType() == N->getValueType(0) && 4667 "Operand over promoted?"); 4668 // Split the promoted operand. This will simplify when it is expanded. 4669 SplitInteger(Res, Lo, Hi); 4670 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 4671 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 4672 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 4673 ExcessBits))); 4674 } 4675 } 4676 4677 void DAGTypeLegalizer:: 4678 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 4679 SDLoc dl(N); 4680 GetExpandedInteger(N->getOperand(0), Lo, Hi); 4681 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 4682 4683 if (EVT.bitsLE(Lo.getValueType())) { 4684 // sext_inreg the low part if needed. 4685 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 4686 N->getOperand(1)); 4687 4688 // The high part gets the sign extension from the lo-part. This handles 4689 // things like sextinreg V:i64 from i8. 4690 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 4691 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, 4692 TLI.getPointerTy(DAG.getDataLayout()))); 4693 } else { 4694 // For example, extension of an i48 to an i64. Leave the low part alone, 4695 // sext_inreg the high part. 4696 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); 4697 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 4698 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 4699 ExcessBits))); 4700 } 4701 } 4702 4703 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 4704 SDValue &Lo, SDValue &Hi) { 4705 EVT VT = N->getValueType(0); 4706 SDLoc dl(N); 4707 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4708 4709 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 4710 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4711 SplitInteger(Res.getValue(1), Lo, Hi); 4712 return; 4713 } 4714 4715 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4716 if (VT == MVT::i16) 4717 LC = RTLIB::SREM_I16; 4718 else if (VT == MVT::i32) 4719 LC = RTLIB::SREM_I32; 4720 else if (VT == MVT::i64) 4721 LC = RTLIB::SREM_I64; 4722 else if (VT == MVT::i128) 4723 LC = RTLIB::SREM_I128; 4724 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 4725 4726 TargetLowering::MakeLibCallOptions CallOptions; 4727 CallOptions.setSExt(true); 4728 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4729 } 4730 4731 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 4732 SDValue &Lo, SDValue &Hi) { 4733 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4734 SDLoc dl(N); 4735 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 4736 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), 4737 N->getOperand(0), 4738 DAG.getConstant(NVT.getSizeInBits(), dl, 4739 TLI.getPointerTy(DAG.getDataLayout()))); 4740 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 4741 } 4742 4743 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 4744 SDValue &Lo, SDValue &Hi) { 4745 EVT VT = N->getValueType(0); 4746 SDLoc dl(N); 4747 4748 if (N->getOpcode() == ISD::UMULO) { 4749 // This section expands the operation into the following sequence of 4750 // instructions. `iNh` here refers to a type which has half the bit width of 4751 // the type the original operation operated on. 4752 // 4753 // %0 = %LHS.HI != 0 && %RHS.HI != 0 4754 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) 4755 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) 4756 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) 4757 // %4 = add iNh %1.0, %2.0 as iN 4758 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH) 4759 // 4760 // %lo = %3.LO 4761 // %hi = %5.0 4762 // %ovf = %0 || %1.1 || %2.1 || %5.1 4763 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 4764 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; 4765 GetExpandedInteger(LHS, LHSLow, LHSHigh); 4766 GetExpandedInteger(RHS, RHSLow, RHSHigh); 4767 EVT HalfVT = LHSLow.getValueType(); 4768 EVT BitVT = N->getValueType(1); 4769 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT); 4770 4771 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT); 4772 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT, 4773 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE), 4774 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE)); 4775 4776 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow); 4777 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1)); 4778 4779 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow); 4780 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1)); 4781 4782 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two); 4783 4784 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not 4785 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this 4786 // operation recursively legalized?). 4787 // 4788 // Many backends understand this pattern and will convert into LOHI 4789 // themselves, if applicable. 4790 SDValue Three = DAG.getNode(ISD::MUL, dl, VT, 4791 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow), 4792 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow)); 4793 SplitInteger(Three, Lo, Hi); 4794 4795 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum); 4796 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1)); 4797 ReplaceValueWith(SDValue(N, 1), Overflow); 4798 return; 4799 } 4800 4801 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 4802 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 4803 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 4804 4805 // Replace this with a libcall that will check overflow. 4806 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4807 if (VT == MVT::i32) 4808 LC = RTLIB::MULO_I32; 4809 else if (VT == MVT::i64) 4810 LC = RTLIB::MULO_I64; 4811 else if (VT == MVT::i128) 4812 LC = RTLIB::MULO_I128; 4813 4814 // If we don't have the libcall or if the function we are compiling is the 4815 // implementation of the expected libcall (avoid inf-loop), expand inline. 4816 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC) || 4817 TLI.getLibcallName(LC) == DAG.getMachineFunction().getName()) { 4818 // FIXME: This is not an optimal expansion, but better than crashing. 4819 EVT WideVT = 4820 EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2); 4821 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0)); 4822 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1)); 4823 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); 4824 SDValue MulLo, MulHi; 4825 SplitInteger(Mul, MulLo, MulHi); 4826 SDValue SRA = 4827 DAG.getNode(ISD::SRA, dl, VT, MulLo, 4828 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT)); 4829 SDValue Overflow = 4830 DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE); 4831 SplitInteger(MulLo, Lo, Hi); 4832 ReplaceValueWith(SDValue(N, 1), Overflow); 4833 return; 4834 } 4835 4836 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 4837 // Temporary for the overflow value, default it to zero. 4838 SDValue Chain = 4839 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, 4840 MachinePointerInfo()); 4841 4842 TargetLowering::ArgListTy Args; 4843 TargetLowering::ArgListEntry Entry; 4844 for (const SDValue &Op : N->op_values()) { 4845 EVT ArgVT = Op.getValueType(); 4846 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 4847 Entry.Node = Op; 4848 Entry.Ty = ArgTy; 4849 Entry.IsSExt = true; 4850 Entry.IsZExt = false; 4851 Args.push_back(Entry); 4852 } 4853 4854 // Also pass the address of the overflow check. 4855 Entry.Node = Temp; 4856 Entry.Ty = PointerType::getUnqual(PtrTy->getContext()); 4857 Entry.IsSExt = true; 4858 Entry.IsZExt = false; 4859 Args.push_back(Entry); 4860 4861 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 4862 4863 TargetLowering::CallLoweringInfo CLI(DAG); 4864 CLI.setDebugLoc(dl) 4865 .setChain(Chain) 4866 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) 4867 .setSExtResult(); 4868 4869 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 4870 4871 SplitInteger(CallInfo.first, Lo, Hi); 4872 SDValue Temp2 = 4873 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); 4874 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 4875 DAG.getConstant(0, dl, PtrVT), 4876 ISD::SETNE); 4877 // Use the overflow from the libcall everywhere. 4878 ReplaceValueWith(SDValue(N, 1), Ofl); 4879 } 4880 4881 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 4882 SDValue &Lo, SDValue &Hi) { 4883 EVT VT = N->getValueType(0); 4884 SDLoc dl(N); 4885 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4886 4887 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4888 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4889 SplitInteger(Res.getValue(0), Lo, Hi); 4890 return; 4891 } 4892 4893 // Try to expand UDIV by constant. 4894 if (isa<ConstantSDNode>(N->getOperand(1))) { 4895 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4896 // Only if the new type is legal. 4897 if (isTypeLegal(NVT)) { 4898 SDValue InL, InH; 4899 GetExpandedInteger(N->getOperand(0), InL, InH); 4900 SmallVector<SDValue> Result; 4901 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) { 4902 Lo = Result[0]; 4903 Hi = Result[1]; 4904 return; 4905 } 4906 } 4907 } 4908 4909 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4910 if (VT == MVT::i16) 4911 LC = RTLIB::UDIV_I16; 4912 else if (VT == MVT::i32) 4913 LC = RTLIB::UDIV_I32; 4914 else if (VT == MVT::i64) 4915 LC = RTLIB::UDIV_I64; 4916 else if (VT == MVT::i128) 4917 LC = RTLIB::UDIV_I128; 4918 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 4919 4920 TargetLowering::MakeLibCallOptions CallOptions; 4921 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4922 } 4923 4924 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 4925 SDValue &Lo, SDValue &Hi) { 4926 EVT VT = N->getValueType(0); 4927 SDLoc dl(N); 4928 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4929 4930 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4931 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4932 SplitInteger(Res.getValue(1), Lo, Hi); 4933 return; 4934 } 4935 4936 // Try to expand UREM by constant. 4937 if (isa<ConstantSDNode>(N->getOperand(1))) { 4938 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4939 // Only if the new type is legal. 4940 if (isTypeLegal(NVT)) { 4941 SDValue InL, InH; 4942 GetExpandedInteger(N->getOperand(0), InL, InH); 4943 SmallVector<SDValue> Result; 4944 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) { 4945 Lo = Result[0]; 4946 Hi = Result[1]; 4947 return; 4948 } 4949 } 4950 } 4951 4952 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4953 if (VT == MVT::i16) 4954 LC = RTLIB::UREM_I16; 4955 else if (VT == MVT::i32) 4956 LC = RTLIB::UREM_I32; 4957 else if (VT == MVT::i64) 4958 LC = RTLIB::UREM_I64; 4959 else if (VT == MVT::i128) 4960 LC = RTLIB::UREM_I128; 4961 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 4962 4963 TargetLowering::MakeLibCallOptions CallOptions; 4964 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4965 } 4966 4967 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 4968 SDValue &Lo, SDValue &Hi) { 4969 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4970 SDLoc dl(N); 4971 SDValue Op = N->getOperand(0); 4972 if (Op.getValueType().bitsLE(NVT)) { 4973 // The low part is zero extension of the input (degenerates to a copy). 4974 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 4975 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. 4976 } else { 4977 // For example, extension of an i48 to an i64. The operand type necessarily 4978 // promotes to the result type, so will end up being expanded too. 4979 assert(getTypeAction(Op.getValueType()) == 4980 TargetLowering::TypePromoteInteger && 4981 "Only know how to promote this result!"); 4982 SDValue Res = GetPromotedInteger(Op); 4983 assert(Res.getValueType() == N->getValueType(0) && 4984 "Operand over promoted?"); 4985 // Split the promoted operand. This will simplify when it is expanded. 4986 SplitInteger(Res, Lo, Hi); 4987 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 4988 Hi = DAG.getZeroExtendInReg(Hi, dl, 4989 EVT::getIntegerVT(*DAG.getContext(), 4990 ExcessBits)); 4991 } 4992 } 4993 4994 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 4995 SDValue &Lo, SDValue &Hi) { 4996 SDLoc dl(N); 4997 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 4998 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 4999 SDValue Zero = DAG.getConstant(0, dl, VT); 5000 SDValue Swap = DAG.getAtomicCmpSwap( 5001 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 5002 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 5003 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); 5004 5005 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 5006 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 5007 } 5008 5009 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, 5010 SDValue &Lo, SDValue &Hi) { 5011 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate 5012 // both halves independently. 5013 SDValue Res = TLI.expandVecReduce(N, DAG); 5014 SplitInteger(Res, Lo, Hi); 5015 } 5016 5017 void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, 5018 SDValue &Lo, SDValue &Hi) { 5019 // Delegate to funnel-shift expansion. 5020 SDLoc DL(N); 5021 unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR; 5022 SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0), 5023 N->getOperand(0), N->getOperand(1)); 5024 SplitInteger(Res, Lo, Hi); 5025 } 5026 5027 void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo, 5028 SDValue &Hi) { 5029 // Values numbered from least significant to most significant. 5030 SDValue In1, In2, In3, In4; 5031 GetExpandedInteger(N->getOperand(0), In3, In4); 5032 GetExpandedInteger(N->getOperand(1), In1, In2); 5033 EVT HalfVT = In1.getValueType(); 5034 5035 SDLoc DL(N); 5036 unsigned Opc = N->getOpcode(); 5037 SDValue ShAmt = N->getOperand(2); 5038 EVT ShAmtVT = ShAmt.getValueType(); 5039 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT); 5040 5041 // If the shift amount is at least half the bitwidth, swap the inputs. 5042 unsigned HalfVTBits = HalfVT.getScalarSizeInBits(); 5043 SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt, 5044 DAG.getConstant(HalfVTBits, DL, ShAmtVT)); 5045 SDValue Cond = 5046 DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT), 5047 Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ); 5048 5049 // Expand to a pair of funnel shifts. 5050 EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout()); 5051 SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT); 5052 5053 SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2); 5054 SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3); 5055 SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4); 5056 Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt); 5057 Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt); 5058 } 5059 5060 void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, 5061 SDValue &Hi) { 5062 EVT VT = N->getValueType(0); 5063 EVT HalfVT = 5064 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2); 5065 SDLoc dl(N); 5066 5067 // We assume VSCALE(1) fits into a legal integer. 5068 APInt One(HalfVT.getSizeInBits(), 1); 5069 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One); 5070 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase); 5071 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0)); 5072 SplitInteger(Res, Lo, Hi); 5073 } 5074 5075 //===----------------------------------------------------------------------===// 5076 // Integer Operand Expansion 5077 //===----------------------------------------------------------------------===// 5078 5079 /// ExpandIntegerOperand - This method is called when the specified operand of 5080 /// the specified node is found to need expansion. At this point, all of the 5081 /// result types of the node are known to be legal, but other operands of the 5082 /// node may need promotion or expansion as well as the specified one. 5083 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 5084 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG)); 5085 SDValue Res = SDValue(); 5086 5087 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 5088 return false; 5089 5090 switch (N->getOpcode()) { 5091 default: 5092 #ifndef NDEBUG 5093 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 5094 N->dump(&DAG); dbgs() << "\n"; 5095 #endif 5096 report_fatal_error("Do not know how to expand this operator's operand!"); 5097 5098 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 5099 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 5100 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 5101 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 5102 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 5103 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 5104 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break; 5105 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 5106 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 5107 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; 5108 case ISD::STRICT_SINT_TO_FP: 5109 case ISD::SINT_TO_FP: 5110 case ISD::STRICT_UINT_TO_FP: 5111 case ISD::UINT_TO_FP: Res = ExpandIntOp_XINT_TO_FP(N); break; 5112 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 5113 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 5114 5115 case ISD::SHL: 5116 case ISD::SRA: 5117 case ISD::SRL: 5118 case ISD::ROTL: 5119 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 5120 case ISD::RETURNADDR: 5121 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 5122 5123 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 5124 case ISD::STACKMAP: 5125 Res = ExpandIntOp_STACKMAP(N, OpNo); 5126 break; 5127 case ISD::PATCHPOINT: 5128 Res = ExpandIntOp_PATCHPOINT(N, OpNo); 5129 break; 5130 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 5131 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 5132 Res = ExpandIntOp_VP_STRIDED(N, OpNo); 5133 break; 5134 } 5135 5136 // If the result is null, the sub-method took care of registering results etc. 5137 if (!Res.getNode()) return false; 5138 5139 // If the result is N, the sub-method updated N in place. Tell the legalizer 5140 // core about this. 5141 if (Res.getNode() == N) 5142 return true; 5143 5144 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 5145 "Invalid operand expansion"); 5146 5147 ReplaceValueWith(SDValue(N, 0), Res); 5148 return false; 5149 } 5150 5151 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 5152 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 5153 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 5154 SDValue &NewRHS, 5155 ISD::CondCode &CCCode, 5156 const SDLoc &dl) { 5157 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 5158 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 5159 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 5160 5161 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 5162 if (RHSLo == RHSHi && isAllOnesConstant(RHSLo)) { 5163 // Equality comparison to -1. 5164 NewLHS = DAG.getNode(ISD::AND, dl, LHSLo.getValueType(), LHSLo, LHSHi); 5165 NewRHS = RHSLo; 5166 return; 5167 } 5168 5169 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 5170 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 5171 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 5172 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); 5173 return; 5174 } 5175 5176 // If this is a comparison of the sign bit, just look at the top part. 5177 // X > -1, x < 0 5178 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 5179 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0 5180 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1 5181 NewLHS = LHSHi; 5182 NewRHS = RHSHi; 5183 return; 5184 } 5185 5186 // FIXME: This generated code sucks. 5187 ISD::CondCode LowCC; 5188 switch (CCCode) { 5189 default: llvm_unreachable("Unknown integer setcc!"); 5190 case ISD::SETLT: 5191 case ISD::SETULT: LowCC = ISD::SETULT; break; 5192 case ISD::SETGT: 5193 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 5194 case ISD::SETLE: 5195 case ISD::SETULE: LowCC = ISD::SETULE; break; 5196 case ISD::SETGE: 5197 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 5198 } 5199 5200 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison 5201 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands 5202 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; 5203 5204 // NOTE: on targets without efficient SELECT of bools, we can always use 5205 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 5206 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 5207 nullptr); 5208 SDValue LoCmp, HiCmp; 5209 if (TLI.isTypeLegal(LHSLo.getValueType()) && 5210 TLI.isTypeLegal(RHSLo.getValueType())) 5211 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, 5212 RHSLo, LowCC, false, DagCombineInfo, dl); 5213 if (!LoCmp.getNode()) 5214 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 5215 RHSLo, LowCC); 5216 if (TLI.isTypeLegal(LHSHi.getValueType()) && 5217 TLI.isTypeLegal(RHSHi.getValueType())) 5218 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, 5219 RHSHi, CCCode, false, DagCombineInfo, dl); 5220 if (!HiCmp.getNode()) 5221 HiCmp = 5222 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), 5223 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 5224 5225 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); 5226 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); 5227 5228 bool EqAllowed = ISD::isTrueWhenEqual(CCCode); 5229 5230 // FIXME: Is the HiCmpC->isOne() here correct for 5231 // ZeroOrNegativeOneBooleanContent. 5232 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) || 5233 (!EqAllowed && 5234 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) { 5235 // For LE / GE, if high part is known false, ignore the low part. 5236 // For LT / GT: if low part is known false, return the high part. 5237 // if high part is known true, ignore the low part. 5238 NewLHS = HiCmp; 5239 NewRHS = SDValue(); 5240 return; 5241 } 5242 5243 if (LHSHi == RHSHi) { 5244 // Comparing the low bits is enough. 5245 NewLHS = LoCmp; 5246 NewRHS = SDValue(); 5247 return; 5248 } 5249 5250 // Lower with SETCCCARRY if the target supports it. 5251 EVT HiVT = LHSHi.getValueType(); 5252 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); 5253 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); 5254 5255 // FIXME: Make all targets support this, then remove the other lowering. 5256 if (HasSETCCCARRY) { 5257 // SETCCCARRY can detect < and >= directly. For > and <=, flip 5258 // operands and condition code. 5259 bool FlipOperands = false; 5260 switch (CCCode) { 5261 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; 5262 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; 5263 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; 5264 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; 5265 default: break; 5266 } 5267 if (FlipOperands) { 5268 std::swap(LHSLo, RHSLo); 5269 std::swap(LHSHi, RHSHi); 5270 } 5271 // Perform a wide subtraction, feeding the carry from the low part into 5272 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high 5273 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is 5274 // zero or positive iff LHS >= RHS. 5275 EVT LoVT = LHSLo.getValueType(); 5276 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT)); 5277 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo); 5278 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT), 5279 LHSHi, RHSHi, LowCmp.getValue(1), 5280 DAG.getCondCode(CCCode)); 5281 NewLHS = Res; 5282 NewRHS = SDValue(); 5283 return; 5284 } 5285 5286 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, 5287 false, DagCombineInfo, dl); 5288 if (!NewLHS.getNode()) 5289 NewLHS = 5290 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); 5291 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); 5292 NewRHS = SDValue(); 5293 } 5294 5295 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 5296 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 5297 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 5298 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 5299 5300 // If ExpandSetCCOperands returned a scalar, we need to compare the result 5301 // against zero to select between true and false values. 5302 if (!NewRHS.getNode()) { 5303 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 5304 CCCode = ISD::SETNE; 5305 } 5306 5307 // Update N to have the operands specified. 5308 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 5309 DAG.getCondCode(CCCode), NewLHS, NewRHS, 5310 N->getOperand(4)), 0); 5311 } 5312 5313 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 5314 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 5315 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 5316 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 5317 5318 // If ExpandSetCCOperands returned a scalar, we need to compare the result 5319 // against zero to select between true and false values. 5320 if (!NewRHS.getNode()) { 5321 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 5322 CCCode = ISD::SETNE; 5323 } 5324 5325 // Update N to have the operands specified. 5326 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 5327 N->getOperand(2), N->getOperand(3), 5328 DAG.getCondCode(CCCode)), 0); 5329 } 5330 5331 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 5332 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 5333 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 5334 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 5335 5336 // If ExpandSetCCOperands returned a scalar, use it. 5337 if (!NewRHS.getNode()) { 5338 assert(NewLHS.getValueType() == N->getValueType(0) && 5339 "Unexpected setcc expansion!"); 5340 return NewLHS; 5341 } 5342 5343 // Otherwise, update N to have the operands specified. 5344 return SDValue( 5345 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); 5346 } 5347 5348 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { 5349 SDValue LHS = N->getOperand(0); 5350 SDValue RHS = N->getOperand(1); 5351 SDValue Carry = N->getOperand(2); 5352 SDValue Cond = N->getOperand(3); 5353 SDLoc dl = SDLoc(N); 5354 5355 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 5356 GetExpandedInteger(LHS, LHSLo, LHSHi); 5357 GetExpandedInteger(RHS, RHSLo, RHSHi); 5358 5359 // Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high. 5360 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); 5361 SDValue LowCmp = 5362 DAG.getNode(ISD::USUBO_CARRY, dl, VTList, LHSLo, RHSLo, Carry); 5363 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, 5364 LowCmp.getValue(1), Cond); 5365 } 5366 5367 SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) { 5368 // Split the operand and replace with SPLAT_VECTOR_PARTS. 5369 SDValue Lo, Hi; 5370 GetExpandedInteger(N->getOperand(0), Lo, Hi); 5371 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo, 5372 Hi); 5373 } 5374 5375 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 5376 // The value being shifted is legal, but the shift amount is too big. 5377 // It follows that either the result of the shift is undefined, or the 5378 // upper half of the shift amount is zero. Just use the lower half. 5379 SDValue Lo, Hi; 5380 GetExpandedInteger(N->getOperand(1), Lo, Hi); 5381 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 5382 } 5383 5384 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 5385 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 5386 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 5387 // constant to valid type. 5388 SDValue Lo, Hi; 5389 GetExpandedInteger(N->getOperand(0), Lo, Hi); 5390 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 5391 } 5392 5393 SDValue DAGTypeLegalizer::ExpandIntOp_XINT_TO_FP(SDNode *N) { 5394 bool IsStrict = N->isStrictFPOpcode(); 5395 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || 5396 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 5397 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 5398 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 5399 EVT DstVT = N->getValueType(0); 5400 RTLIB::Libcall LC = IsSigned ? RTLIB::getSINTTOFP(Op.getValueType(), DstVT) 5401 : RTLIB::getUINTTOFP(Op.getValueType(), DstVT); 5402 assert(LC != RTLIB::UNKNOWN_LIBCALL && 5403 "Don't know how to expand this XINT_TO_FP!"); 5404 TargetLowering::MakeLibCallOptions CallOptions; 5405 CallOptions.setSExt(true); 5406 std::pair<SDValue, SDValue> Tmp = 5407 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 5408 5409 if (!IsStrict) 5410 return Tmp.first; 5411 5412 ReplaceValueWith(SDValue(N, 1), Tmp.second); 5413 ReplaceValueWith(SDValue(N, 0), Tmp.first); 5414 return SDValue(); 5415 } 5416 5417 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 5418 assert(!N->isAtomic() && "Should have been a ATOMIC_STORE?"); 5419 5420 if (ISD::isNormalStore(N)) 5421 return ExpandOp_NormalStore(N, OpNo); 5422 5423 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 5424 assert(OpNo == 1 && "Can only expand the stored value so far"); 5425 5426 EVT VT = N->getOperand(1).getValueType(); 5427 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5428 SDValue Ch = N->getChain(); 5429 SDValue Ptr = N->getBasePtr(); 5430 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 5431 AAMDNodes AAInfo = N->getAAInfo(); 5432 SDLoc dl(N); 5433 SDValue Lo, Hi; 5434 5435 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 5436 5437 if (N->getMemoryVT().bitsLE(NVT)) { 5438 GetExpandedInteger(N->getValue(), Lo, Hi); 5439 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 5440 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags, 5441 AAInfo); 5442 } 5443 5444 if (DAG.getDataLayout().isLittleEndian()) { 5445 // Little-endian - low bits are at low addresses. 5446 GetExpandedInteger(N->getValue(), Lo, Hi); 5447 5448 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 5449 N->getOriginalAlign(), MMOFlags, AAInfo); 5450 5451 unsigned ExcessBits = 5452 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 5453 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 5454 5455 // Increment the pointer to the other half. 5456 unsigned IncrementSize = NVT.getSizeInBits()/8; 5457 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize)); 5458 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 5459 N->getPointerInfo().getWithOffset(IncrementSize), 5460 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo); 5461 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 5462 } 5463 5464 // Big-endian - high bits are at low addresses. Favor aligned stores at 5465 // the cost of some bit-fiddling. 5466 GetExpandedInteger(N->getValue(), Lo, Hi); 5467 5468 EVT ExtVT = N->getMemoryVT(); 5469 unsigned EBytes = ExtVT.getStoreSize(); 5470 unsigned IncrementSize = NVT.getSizeInBits()/8; 5471 unsigned ExcessBits = (EBytes - IncrementSize)*8; 5472 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 5473 ExtVT.getSizeInBits() - ExcessBits); 5474 5475 if (ExcessBits < NVT.getSizeInBits()) { 5476 // Transfer high bits from the top of Lo to the bottom of Hi. 5477 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 5478 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 5479 TLI.getPointerTy(DAG.getDataLayout()))); 5480 Hi = DAG.getNode( 5481 ISD::OR, dl, NVT, Hi, 5482 DAG.getNode(ISD::SRL, dl, NVT, Lo, 5483 DAG.getConstant(ExcessBits, dl, 5484 TLI.getPointerTy(DAG.getDataLayout())))); 5485 } 5486 5487 // Store both the high bits and maybe some of the low bits. 5488 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, 5489 N->getOriginalAlign(), MMOFlags, AAInfo); 5490 5491 // Increment the pointer to the other half. 5492 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize)); 5493 // Store the lowest ExcessBits bits in the second half. 5494 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 5495 N->getPointerInfo().getWithOffset(IncrementSize), 5496 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 5497 N->getOriginalAlign(), MMOFlags, AAInfo); 5498 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 5499 } 5500 5501 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 5502 SDValue InL, InH; 5503 GetExpandedInteger(N->getOperand(0), InL, InH); 5504 // Just truncate the low part of the source. 5505 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 5506 } 5507 5508 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 5509 SDLoc dl(N); 5510 SDValue Swap = 5511 DAG.getAtomic(ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(N)->getMemoryVT(), 5512 N->getOperand(0), N->getOperand(2), N->getOperand(1), 5513 cast<AtomicSDNode>(N)->getMemOperand()); 5514 return Swap.getValue(1); 5515 } 5516 5517 SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { 5518 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || 5519 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); 5520 5521 SDValue Hi; // The upper half is dropped out. 5522 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); 5523 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo], Hi); 5524 5525 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 5526 } 5527 5528 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { 5529 SDLoc dl(N); 5530 5531 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5532 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5533 EVT OutVT = V0.getValueType(); 5534 5535 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2)); 5536 } 5537 5538 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) { 5539 SDLoc dl(N); 5540 5541 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5542 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5543 EVT ResVT = V0.getValueType(); 5544 SDValue Res = DAG.getNode(N->getOpcode(), dl, 5545 DAG.getVTList(ResVT, ResVT), V0, V1); 5546 SetPromotedInteger(SDValue(N, 0), Res.getValue(0)); 5547 SetPromotedInteger(SDValue(N, 1), Res.getValue(1)); 5548 return SDValue(); 5549 } 5550 5551 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 5552 5553 EVT OutVT = N->getValueType(0); 5554 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5555 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5556 EVT NOutVTElem = NOutVT.getVectorElementType(); 5557 5558 SDLoc dl(N); 5559 SDValue BaseIdx = N->getOperand(1); 5560 5561 // TODO: We may be able to use this for types other than scalable 5562 // vectors and fix those tests that expect BUILD_VECTOR to be used 5563 if (OutVT.isScalableVector()) { 5564 SDValue InOp0 = N->getOperand(0); 5565 EVT InVT = InOp0.getValueType(); 5566 5567 // Try and extract from a smaller type so that it eventually falls 5568 // into the promotion code below. 5569 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector || 5570 getTypeAction(InVT) == TargetLowering::TypeLegal) { 5571 EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext()); 5572 unsigned NElts = NInVT.getVectorMinNumElements(); 5573 uint64_t IdxVal = BaseIdx->getAsZExtVal(); 5574 5575 SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0, 5576 DAG.getConstant(alignDown(IdxVal, NElts), dl, 5577 BaseIdx.getValueType())); 5578 SDValue Step2 = DAG.getNode( 5579 ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1, 5580 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType())); 5581 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2); 5582 } 5583 5584 // Try and extract from a widened type. 5585 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 5586 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx}; 5587 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops); 5588 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); 5589 } 5590 5591 // Promote operands and see if this is handled by target lowering, 5592 // Otherwise, use the BUILD_VECTOR approach below 5593 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 5594 // Collect the (promoted) operands 5595 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx }; 5596 5597 EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); 5598 assert(PromEltVT.bitsLE(NOutVTElem) && 5599 "Promoted operand has an element type greater than result"); 5600 5601 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT); 5602 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops); 5603 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); 5604 } 5605 } 5606 5607 if (OutVT.isScalableVector()) 5608 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR"); 5609 5610 SDValue InOp0 = N->getOperand(0); 5611 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) 5612 InOp0 = GetPromotedInteger(N->getOperand(0)); 5613 5614 EVT InVT = InOp0.getValueType(); 5615 5616 unsigned OutNumElems = OutVT.getVectorNumElements(); 5617 SmallVector<SDValue, 8> Ops; 5618 Ops.reserve(OutNumElems); 5619 for (unsigned i = 0; i != OutNumElems; ++i) { 5620 5621 // Extract the element from the original vector. 5622 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 5623 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); 5624 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 5625 InVT.getVectorElementType(), N->getOperand(0), Index); 5626 5627 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); 5628 // Insert the converted element to the new vector. 5629 Ops.push_back(Op); 5630 } 5631 5632 return DAG.getBuildVector(NOutVT, dl, Ops); 5633 } 5634 5635 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) { 5636 EVT OutVT = N->getValueType(0); 5637 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5638 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5639 5640 SDLoc dl(N); 5641 SDValue Vec = N->getOperand(0); 5642 SDValue SubVec = N->getOperand(1); 5643 SDValue Idx = N->getOperand(2); 5644 5645 EVT SubVecVT = SubVec.getValueType(); 5646 EVT NSubVT = 5647 EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(), 5648 SubVecVT.getVectorElementCount()); 5649 5650 Vec = GetPromotedInteger(Vec); 5651 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec); 5652 5653 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx); 5654 } 5655 5656 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) { 5657 SDLoc dl(N); 5658 5659 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5660 EVT OutVT = V0.getValueType(); 5661 5662 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0); 5663 } 5664 5665 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 5666 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 5667 EVT VT = N->getValueType(0); 5668 SDLoc dl(N); 5669 5670 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); 5671 5672 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5673 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5674 EVT OutVT = V0.getValueType(); 5675 5676 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); 5677 } 5678 5679 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 5680 EVT OutVT = N->getValueType(0); 5681 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5682 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5683 unsigned NumElems = N->getNumOperands(); 5684 EVT NOutVTElem = NOutVT.getVectorElementType(); 5685 TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(NOutVT); 5686 unsigned NOutExtOpc = TargetLowering::getExtendForContent(NOutBoolType); 5687 SDLoc dl(N); 5688 5689 SmallVector<SDValue, 8> Ops; 5690 Ops.reserve(NumElems); 5691 for (unsigned i = 0; i != NumElems; ++i) { 5692 SDValue Op = N->getOperand(i); 5693 EVT OpVT = Op.getValueType(); 5694 // BUILD_VECTOR integer operand types are allowed to be larger than the 5695 // result's element type. This may still be true after the promotion. For 5696 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 5697 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 5698 if (OpVT.bitsLT(NOutVTElem)) { 5699 unsigned ExtOpc = ISD::ANY_EXTEND; 5700 // Attempt to extend constant bool vectors to match target's BooleanContent. 5701 // While not necessary, this improves chances of the constant correctly 5702 // folding with compare results (e.g. for NOT patterns). 5703 if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant) 5704 ExtOpc = NOutExtOpc; 5705 Op = DAG.getNode(ExtOpc, dl, NOutVTElem, Op); 5706 } 5707 Ops.push_back(Op); 5708 } 5709 5710 return DAG.getBuildVector(NOutVT, dl, Ops); 5711 } 5712 5713 SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) { 5714 5715 SDLoc dl(N); 5716 5717 assert(!N->getOperand(0).getValueType().isVector() && 5718 "Input must be a scalar"); 5719 5720 EVT OutVT = N->getValueType(0); 5721 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5722 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5723 EVT NOutElemVT = NOutVT.getVectorElementType(); 5724 5725 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0)); 5726 5727 return DAG.getNode(N->getOpcode(), dl, NOutVT, Op); 5728 } 5729 5730 SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) { 5731 SDLoc dl(N); 5732 EVT OutVT = N->getValueType(0); 5733 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5734 assert(NOutVT.isScalableVector() && 5735 "Type must be promoted to a scalable vector type"); 5736 const APInt &StepVal = N->getConstantOperandAPInt(0); 5737 return DAG.getStepVector(dl, NOutVT, 5738 StepVal.sext(NOutVT.getScalarSizeInBits())); 5739 } 5740 5741 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 5742 SDLoc dl(N); 5743 5744 EVT OutVT = N->getValueType(0); 5745 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5746 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5747 5748 unsigned NumOperands = N->getNumOperands(); 5749 unsigned NumOutElem = NOutVT.getVectorMinNumElements(); 5750 EVT OutElemTy = NOutVT.getVectorElementType(); 5751 if (OutVT.isScalableVector()) { 5752 // Find the largest promoted element type for each of the operands. 5753 SDUse *MaxSizedValue = std::max_element( 5754 N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) { 5755 EVT AVT = A.getValueType().getVectorElementType(); 5756 EVT BVT = B.getValueType().getVectorElementType(); 5757 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits(); 5758 }); 5759 EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType(); 5760 5761 // Then promote all vectors to the largest element type. 5762 SmallVector<SDValue, 8> Ops; 5763 for (unsigned I = 0; I < NumOperands; ++I) { 5764 SDValue Op = N->getOperand(I); 5765 EVT OpVT = Op.getValueType(); 5766 if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger) 5767 Op = GetPromotedInteger(Op); 5768 else 5769 assert(getTypeAction(OpVT) == TargetLowering::TypeLegal && 5770 "Unhandled legalization type"); 5771 5772 if (OpVT.getVectorElementType().getScalarSizeInBits() < 5773 MaxElementVT.getScalarSizeInBits()) 5774 Op = DAG.getAnyExtOrTrunc(Op, dl, 5775 OpVT.changeVectorElementType(MaxElementVT)); 5776 Ops.push_back(Op); 5777 } 5778 5779 // Do the CONCAT on the promoted type and finally truncate to (the promoted) 5780 // NOutVT. 5781 return DAG.getAnyExtOrTrunc( 5782 DAG.getNode(ISD::CONCAT_VECTORS, dl, 5783 OutVT.changeVectorElementType(MaxElementVT), Ops), 5784 dl, NOutVT); 5785 } 5786 5787 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 5788 assert(NumElem * NumOperands == NumOutElem && 5789 "Unexpected number of elements"); 5790 5791 // Take the elements from the first vector. 5792 SmallVector<SDValue, 8> Ops(NumOutElem); 5793 for (unsigned i = 0; i < NumOperands; ++i) { 5794 SDValue Op = N->getOperand(i); 5795 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) 5796 Op = GetPromotedInteger(Op); 5797 EVT SclrTy = Op.getValueType().getVectorElementType(); 5798 assert(NumElem == Op.getValueType().getVectorNumElements() && 5799 "Unexpected number of elements"); 5800 5801 for (unsigned j = 0; j < NumElem; ++j) { 5802 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, 5803 DAG.getVectorIdxConstant(j, dl)); 5804 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); 5805 } 5806 } 5807 5808 return DAG.getBuildVector(NOutVT, dl, Ops); 5809 } 5810 5811 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { 5812 EVT VT = N->getValueType(0); 5813 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5814 assert(NVT.isVector() && "This type must be promoted to a vector type"); 5815 5816 SDLoc dl(N); 5817 5818 // For operands whose TypeAction is to promote, extend the promoted node 5819 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion 5820 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to 5821 // type.. 5822 if (getTypeAction(N->getOperand(0).getValueType()) 5823 == TargetLowering::TypePromoteInteger) { 5824 SDValue Promoted; 5825 5826 switch(N->getOpcode()) { 5827 case ISD::SIGN_EXTEND_VECTOR_INREG: 5828 Promoted = SExtPromotedInteger(N->getOperand(0)); 5829 break; 5830 case ISD::ZERO_EXTEND_VECTOR_INREG: 5831 Promoted = ZExtPromotedInteger(N->getOperand(0)); 5832 break; 5833 case ISD::ANY_EXTEND_VECTOR_INREG: 5834 Promoted = GetPromotedInteger(N->getOperand(0)); 5835 break; 5836 default: 5837 llvm_unreachable("Node has unexpected Opcode"); 5838 } 5839 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); 5840 } 5841 5842 // Directly extend to the appropriate transform-to type. 5843 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 5844 } 5845 5846 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 5847 EVT OutVT = N->getValueType(0); 5848 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5849 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5850 5851 EVT NOutVTElem = NOutVT.getVectorElementType(); 5852 5853 SDLoc dl(N); 5854 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5855 5856 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 5857 NOutVTElem, N->getOperand(1)); 5858 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 5859 V0, ConvElem, N->getOperand(2)); 5860 } 5861 5862 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { 5863 // The VECREDUCE result size may be larger than the element size, so 5864 // we can simply change the result type. 5865 SDLoc dl(N); 5866 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5867 return DAG.getNode(N->getOpcode(), dl, NVT, N->ops()); 5868 } 5869 5870 SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) { 5871 // The VP_REDUCE result size may be larger than the element size, so we can 5872 // simply change the result type. However the start value and result must be 5873 // the same. 5874 SDLoc DL(N); 5875 SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0)); 5876 return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start, 5877 N->getOperand(1), N->getOperand(2), N->getOperand(3)); 5878 } 5879 5880 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 5881 SDLoc dl(N); 5882 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5883 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, 5884 TLI.getVectorIdxTy(DAG.getDataLayout())); 5885 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 5886 V0->getValueType(0).getScalarType(), V0, V1); 5887 5888 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 5889 // element types. If this is the case then we need to expand the outgoing 5890 // value and not truncate it. 5891 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 5892 } 5893 5894 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) { 5895 SDLoc dl(N); 5896 // The result type is equal to the first input operand's type, so the 5897 // type that needs promoting must be the second source vector. 5898 SDValue V0 = N->getOperand(0); 5899 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5900 SDValue Idx = N->getOperand(2); 5901 EVT PromVT = EVT::getVectorVT(*DAG.getContext(), 5902 V1.getValueType().getVectorElementType(), 5903 V0.getValueType().getVectorElementCount()); 5904 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT); 5905 SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx); 5906 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 5907 } 5908 5909 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { 5910 SDLoc dl(N); 5911 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5912 MVT InVT = V0.getValueType().getSimpleVT(); 5913 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), 5914 N->getValueType(0).getVectorNumElements()); 5915 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); 5916 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 5917 } 5918 5919 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 5920 SDLoc dl(N); 5921 5922 EVT ResVT = N->getValueType(0); 5923 unsigned NumElems = N->getNumOperands(); 5924 5925 if (ResVT.isScalableVector()) { 5926 SDValue ResVec = DAG.getUNDEF(ResVT); 5927 5928 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { 5929 SDValue Op = N->getOperand(OpIdx); 5930 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); 5931 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op, 5932 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl)); 5933 } 5934 5935 return ResVec; 5936 } 5937 5938 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 5939 5940 SmallVector<SDValue, 8> NewOps; 5941 NewOps.reserve(NumElems); 5942 5943 // For each incoming vector 5944 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 5945 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 5946 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 5947 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 5948 5949 for (unsigned i=0; i<NumElem; ++i) { 5950 // Extract element from incoming vector 5951 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, 5952 DAG.getVectorIdxConstant(i, dl)); 5953 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 5954 NewOps.push_back(Tr); 5955 } 5956 } 5957 5958 return DAG.getBuildVector(N->getValueType(0), dl, NewOps); 5959 } 5960 5961 SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) { 5962 assert(OpNo > 1); 5963 SDValue Op = N->getOperand(OpNo); 5964 5965 // FIXME: Non-constant operands are not yet handled: 5966 // - https://github.com/llvm/llvm-project/issues/26431 5967 // - https://github.com/llvm/llvm-project/issues/55957 5968 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op); 5969 if (!CN) 5970 return SDValue(); 5971 5972 // Copy operands before the one being expanded. 5973 SmallVector<SDValue> NewOps; 5974 for (unsigned I = 0; I < OpNo; I++) 5975 NewOps.push_back(N->getOperand(I)); 5976 5977 EVT Ty = Op.getValueType(); 5978 SDLoc DL = SDLoc(N); 5979 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { 5980 NewOps.push_back( 5981 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); 5982 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); 5983 } else { 5984 // FIXME: https://github.com/llvm/llvm-project/issues/55609 5985 return SDValue(); 5986 } 5987 5988 // Copy remaining operands. 5989 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) 5990 NewOps.push_back(N->getOperand(I)); 5991 5992 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); 5993 5994 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 5995 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 5996 5997 return SDValue(); // Signal that we have replaced the node already. 5998 } 5999 6000 SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { 6001 assert(OpNo >= 7); 6002 SDValue Op = N->getOperand(OpNo); 6003 6004 // FIXME: Non-constant operands are not yet handled: 6005 // - https://github.com/llvm/llvm-project/issues/26431 6006 // - https://github.com/llvm/llvm-project/issues/55957 6007 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op); 6008 if (!CN) 6009 return SDValue(); 6010 6011 // Copy operands before the one being expanded. 6012 SmallVector<SDValue> NewOps; 6013 for (unsigned I = 0; I < OpNo; I++) 6014 NewOps.push_back(N->getOperand(I)); 6015 6016 EVT Ty = Op.getValueType(); 6017 SDLoc DL = SDLoc(N); 6018 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { 6019 NewOps.push_back( 6020 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); 6021 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); 6022 } else { 6023 // FIXME: https://github.com/llvm/llvm-project/issues/55609 6024 return SDValue(); 6025 } 6026 6027 // Copy remaining operands. 6028 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) 6029 NewOps.push_back(N->getOperand(I)); 6030 6031 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); 6032 6033 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 6034 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 6035 6036 return SDValue(); // Signal that we have replaced the node already. 6037 } 6038