1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements integer type expansion and promotion for LegalizeTypes. 10 // Promotion is the act of changing a computation in an illegal type into a 11 // computation in a larger type. For example, implementing i8 arithmetic in an 12 // i32 register (often needed on powerpc). 13 // Expansion is the act of changing a computation in an illegal type into a 14 // computation in two identical registers of a smaller type. For example, 15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit 16 // targets). 17 // 18 //===----------------------------------------------------------------------===// 19 20 #include "LegalizeTypes.h" 21 #include "llvm/Analysis/TargetLibraryInfo.h" 22 #include "llvm/CodeGen/StackMaps.h" 23 #include "llvm/IR/DerivedTypes.h" 24 #include "llvm/Support/ErrorHandling.h" 25 #include "llvm/Support/KnownBits.h" 26 #include "llvm/Support/raw_ostream.h" 27 #include <algorithm> 28 using namespace llvm; 29 30 #define DEBUG_TYPE "legalize-types" 31 32 //===----------------------------------------------------------------------===// 33 // Integer Result Promotion 34 //===----------------------------------------------------------------------===// 35 36 /// PromoteIntegerResult - This method is called when a result of a node is 37 /// found to be in need of promotion to a larger type. At this point, the node 38 /// may also have invalid operands or may have other results that need 39 /// expansion, we just know that (at least) one result needs promotion. 40 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 41 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); 42 dbgs() << "\n"); 43 SDValue Res = SDValue(); 44 45 // See if the target wants to custom expand this node. 46 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 47 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 48 return; 49 } 50 51 switch (N->getOpcode()) { 52 default: 53 #ifndef NDEBUG 54 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 55 N->dump(&DAG); dbgs() << "\n"; 56 #endif 57 llvm_unreachable("Do not know how to promote this operator!"); 58 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 59 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 60 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 61 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 62 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; 63 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 64 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 65 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 66 case ISD::CTLZ_ZERO_UNDEF: 67 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 68 case ISD::PARITY: 69 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; 70 case ISD::CTTZ_ZERO_UNDEF: 71 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 72 case ISD::EXTRACT_VECTOR_ELT: 73 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 74 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 75 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); 76 break; 77 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); 78 break; 79 case ISD::SELECT: 80 case ISD::VSELECT: 81 case ISD::VP_SELECT: 82 case ISD::VP_MERGE: 83 Res = PromoteIntRes_Select(N); 84 break; 85 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 86 case ISD::STRICT_FSETCC: 87 case ISD::STRICT_FSETCCS: 88 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 89 case ISD::SMIN: 90 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; 91 case ISD::UMIN: 92 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; 93 94 case ISD::SHL: 95 case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break; 96 case ISD::SIGN_EXTEND_INREG: 97 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 98 case ISD::SRA: 99 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break; 100 case ISD::SRL: 101 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break; 102 case ISD::VP_TRUNCATE: 103 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 104 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 105 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 106 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; 107 108 case ISD::EXTRACT_SUBVECTOR: 109 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 110 case ISD::INSERT_SUBVECTOR: 111 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break; 112 case ISD::VECTOR_REVERSE: 113 Res = PromoteIntRes_VECTOR_REVERSE(N); break; 114 case ISD::VECTOR_SHUFFLE: 115 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 116 case ISD::VECTOR_SPLICE: 117 Res = PromoteIntRes_VECTOR_SPLICE(N); break; 118 case ISD::INSERT_VECTOR_ELT: 119 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 120 case ISD::BUILD_VECTOR: 121 Res = PromoteIntRes_BUILD_VECTOR(N); 122 break; 123 case ISD::SPLAT_VECTOR: 124 case ISD::SCALAR_TO_VECTOR: 125 Res = PromoteIntRes_ScalarOp(N); 126 break; 127 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break; 128 case ISD::CONCAT_VECTORS: 129 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 130 131 case ISD::ANY_EXTEND_VECTOR_INREG: 132 case ISD::SIGN_EXTEND_VECTOR_INREG: 133 case ISD::ZERO_EXTEND_VECTOR_INREG: 134 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; 135 136 case ISD::SIGN_EXTEND: 137 case ISD::ZERO_EXTEND: 138 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 139 140 case ISD::VP_FP_TO_SINT: 141 case ISD::VP_FP_TO_UINT: 142 case ISD::STRICT_FP_TO_SINT: 143 case ISD::STRICT_FP_TO_UINT: 144 case ISD::FP_TO_SINT: 145 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 146 147 case ISD::FP_TO_SINT_SAT: 148 case ISD::FP_TO_UINT_SAT: 149 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; 150 151 case ISD::FP_TO_BF16: 152 case ISD::FP_TO_FP16: 153 Res = PromoteIntRes_FP_TO_FP16_BF16(N); 154 break; 155 156 case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break; 157 158 case ISD::AND: 159 case ISD::OR: 160 case ISD::XOR: 161 case ISD::ADD: 162 case ISD::SUB: 163 case ISD::MUL: 164 case ISD::VP_AND: 165 case ISD::VP_OR: 166 case ISD::VP_XOR: 167 case ISD::VP_ADD: 168 case ISD::VP_SUB: 169 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 170 171 case ISD::VP_SMIN: 172 case ISD::VP_SMAX: 173 case ISD::SDIV: 174 case ISD::SREM: 175 case ISD::VP_SDIV: 176 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; 177 178 case ISD::VP_UMIN: 179 case ISD::VP_UMAX: 180 case ISD::UDIV: 181 case ISD::UREM: 182 case ISD::VP_UDIV: 183 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; 184 185 case ISD::SADDO: 186 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 187 case ISD::UADDO: 188 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 189 case ISD::SMULO: 190 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 191 192 case ISD::ADDE: 193 case ISD::SUBE: 194 case ISD::ADDCARRY: 195 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break; 196 197 case ISD::SADDO_CARRY: 198 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; 199 200 case ISD::SADDSAT: 201 case ISD::UADDSAT: 202 case ISD::SSUBSAT: 203 case ISD::USUBSAT: 204 case ISD::SSHLSAT: 205 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break; 206 207 case ISD::SMULFIX: 208 case ISD::SMULFIXSAT: 209 case ISD::UMULFIX: 210 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; 211 212 case ISD::SDIVFIX: 213 case ISD::SDIVFIXSAT: 214 case ISD::UDIVFIX: 215 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; 216 217 case ISD::ABS: Res = PromoteIntRes_ABS(N); break; 218 219 case ISD::ATOMIC_LOAD: 220 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 221 222 case ISD::ATOMIC_LOAD_ADD: 223 case ISD::ATOMIC_LOAD_SUB: 224 case ISD::ATOMIC_LOAD_AND: 225 case ISD::ATOMIC_LOAD_CLR: 226 case ISD::ATOMIC_LOAD_OR: 227 case ISD::ATOMIC_LOAD_XOR: 228 case ISD::ATOMIC_LOAD_NAND: 229 case ISD::ATOMIC_LOAD_MIN: 230 case ISD::ATOMIC_LOAD_MAX: 231 case ISD::ATOMIC_LOAD_UMIN: 232 case ISD::ATOMIC_LOAD_UMAX: 233 case ISD::ATOMIC_SWAP: 234 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 235 236 case ISD::ATOMIC_CMP_SWAP: 237 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 238 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 239 break; 240 241 case ISD::VECREDUCE_ADD: 242 case ISD::VECREDUCE_MUL: 243 case ISD::VECREDUCE_AND: 244 case ISD::VECREDUCE_OR: 245 case ISD::VECREDUCE_XOR: 246 case ISD::VECREDUCE_SMAX: 247 case ISD::VECREDUCE_SMIN: 248 case ISD::VECREDUCE_UMAX: 249 case ISD::VECREDUCE_UMIN: 250 Res = PromoteIntRes_VECREDUCE(N); 251 break; 252 253 case ISD::VP_REDUCE_ADD: 254 case ISD::VP_REDUCE_MUL: 255 case ISD::VP_REDUCE_AND: 256 case ISD::VP_REDUCE_OR: 257 case ISD::VP_REDUCE_XOR: 258 case ISD::VP_REDUCE_SMAX: 259 case ISD::VP_REDUCE_SMIN: 260 case ISD::VP_REDUCE_UMAX: 261 case ISD::VP_REDUCE_UMIN: 262 Res = PromoteIntRes_VP_REDUCE(N); 263 break; 264 265 case ISD::FREEZE: 266 Res = PromoteIntRes_FREEZE(N); 267 break; 268 269 case ISD::ROTL: 270 case ISD::ROTR: 271 Res = PromoteIntRes_Rotate(N); 272 break; 273 274 case ISD::FSHL: 275 case ISD::FSHR: 276 Res = PromoteIntRes_FunnelShift(N); 277 break; 278 279 case ISD::IS_FPCLASS: 280 Res = PromoteIntRes_IS_FPCLASS(N); 281 break; 282 } 283 284 // If the result is null then the sub-method took care of registering it. 285 if (Res.getNode()) 286 SetPromotedInteger(SDValue(N, ResNo), Res); 287 } 288 289 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 290 unsigned ResNo) { 291 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 292 return GetPromotedInteger(Op); 293 } 294 295 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 296 // Sign-extend the new bits, and continue the assertion. 297 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 298 return DAG.getNode(ISD::AssertSext, SDLoc(N), 299 Op.getValueType(), Op, N->getOperand(1)); 300 } 301 302 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 303 // Zero the new bits, and continue the assertion. 304 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 305 return DAG.getNode(ISD::AssertZext, SDLoc(N), 306 Op.getValueType(), Op, N->getOperand(1)); 307 } 308 309 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 310 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 311 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 312 N->getMemoryVT(), ResVT, 313 N->getChain(), N->getBasePtr(), 314 N->getMemOperand()); 315 // Legalize the chain result - switch anything that used the old chain to 316 // use the new one. 317 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 318 return Res; 319 } 320 321 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 322 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 323 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 324 N->getMemoryVT(), 325 N->getChain(), N->getBasePtr(), 326 Op2, N->getMemOperand()); 327 // Legalize the chain result - switch anything that used the old chain to 328 // use the new one. 329 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 330 return Res; 331 } 332 333 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 334 unsigned ResNo) { 335 if (ResNo == 1) { 336 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 337 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 338 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 339 340 // Only use the result of getSetCCResultType if it is legal, 341 // otherwise just use the promoted result type (NVT). 342 if (!TLI.isTypeLegal(SVT)) 343 SVT = NVT; 344 345 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 346 SDValue Res = DAG.getAtomicCmpSwap( 347 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 348 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 349 N->getMemOperand()); 350 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 351 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 352 return Res.getValue(1); 353 } 354 355 // Op2 is used for the comparison and thus must be extended according to the 356 // target's atomic operations. Op3 is merely stored and so can be left alone. 357 SDValue Op2 = N->getOperand(2); 358 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 359 switch (TLI.getExtendForAtomicCmpSwapArg()) { 360 case ISD::SIGN_EXTEND: 361 Op2 = SExtPromotedInteger(Op2); 362 break; 363 case ISD::ZERO_EXTEND: 364 Op2 = ZExtPromotedInteger(Op2); 365 break; 366 case ISD::ANY_EXTEND: 367 Op2 = GetPromotedInteger(Op2); 368 break; 369 default: 370 llvm_unreachable("Invalid atomic op extension"); 371 } 372 373 SDVTList VTs = 374 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 375 SDValue Res = DAG.getAtomicCmpSwap( 376 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 377 N->getBasePtr(), Op2, Op3, N->getMemOperand()); 378 // Update the use to N with the newly created Res. 379 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) 380 ReplaceValueWith(SDValue(N, i), Res.getValue(i)); 381 return Res; 382 } 383 384 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 385 SDValue InOp = N->getOperand(0); 386 EVT InVT = InOp.getValueType(); 387 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 388 EVT OutVT = N->getValueType(0); 389 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 390 SDLoc dl(N); 391 392 switch (getTypeAction(InVT)) { 393 case TargetLowering::TypeLegal: 394 break; 395 case TargetLowering::TypePromoteInteger: 396 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 397 // The input promotes to the same size. Convert the promoted value. 398 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 399 break; 400 case TargetLowering::TypeSoftenFloat: 401 // Promote the integer operand by hand. 402 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 403 case TargetLowering::TypeSoftPromoteHalf: 404 // Promote the integer operand by hand. 405 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp)); 406 case TargetLowering::TypePromoteFloat: { 407 // Convert the promoted float by hand. 408 if (!NOutVT.isVector()) 409 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); 410 break; 411 } 412 case TargetLowering::TypeExpandInteger: 413 case TargetLowering::TypeExpandFloat: 414 break; 415 case TargetLowering::TypeScalarizeVector: 416 // Convert the element to an integer and promote it by hand. 417 if (!NOutVT.isVector()) 418 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 419 BitConvertToInteger(GetScalarizedVector(InOp))); 420 break; 421 case TargetLowering::TypeScalarizeScalableVector: 422 report_fatal_error("Scalarization of scalable vectors is not supported."); 423 case TargetLowering::TypeSplitVector: { 424 if (!NOutVT.isVector()) { 425 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 426 // pieces of the input into integers and reassemble in the final type. 427 SDValue Lo, Hi; 428 GetSplitVector(N->getOperand(0), Lo, Hi); 429 Lo = BitConvertToInteger(Lo); 430 Hi = BitConvertToInteger(Hi); 431 432 if (DAG.getDataLayout().isBigEndian()) 433 std::swap(Lo, Hi); 434 435 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 436 EVT::getIntegerVT(*DAG.getContext(), 437 NOutVT.getSizeInBits()), 438 JoinIntegers(Lo, Hi)); 439 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 440 } 441 break; 442 } 443 case TargetLowering::TypeWidenVector: 444 // The input is widened to the same size. Convert to the widened value. 445 // Make sure that the outgoing value is not a vector, because this would 446 // make us bitcast between two vectors which are legalized in different ways. 447 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) { 448 SDValue Res = 449 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 450 451 // For big endian targets we need to shift the casted value or the 452 // interesting bits will end up at the wrong place. 453 if (DAG.getDataLayout().isBigEndian()) { 454 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 455 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!"); 456 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res, 457 DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl)); 458 } 459 return Res; 460 } 461 // If the output type is also a vector and widening it to the same size 462 // as the widened input type would be a legal type, we can widen the bitcast 463 // and handle the promotion after. 464 if (NOutVT.isVector()) { 465 TypeSize WidenInSize = NInVT.getSizeInBits(); 466 TypeSize OutSize = OutVT.getSizeInBits(); 467 if (WidenInSize.hasKnownScalarFactor(OutSize)) { 468 unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize); 469 EVT WideOutVT = 470 EVT::getVectorVT(*DAG.getContext(), OutVT.getVectorElementType(), 471 OutVT.getVectorElementCount() * Scale); 472 if (isTypeLegal(WideOutVT)) { 473 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); 474 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, 475 DAG.getVectorIdxConstant(0, dl)); 476 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp); 477 } 478 } 479 } 480 } 481 482 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 483 CreateStackStoreLoad(InOp, OutVT)); 484 } 485 486 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { 487 SDValue V = GetPromotedInteger(N->getOperand(0)); 488 return DAG.getNode(ISD::FREEZE, SDLoc(N), 489 V.getValueType(), V); 490 } 491 492 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 493 SDValue Op = GetPromotedInteger(N->getOperand(0)); 494 EVT OVT = N->getValueType(0); 495 EVT NVT = Op.getValueType(); 496 SDLoc dl(N); 497 498 // If the larger BSWAP isn't supported by the target, try to expand now. 499 // If we expand later we'll end up with more operations since we lost the 500 // original type. We only do this for scalars since we have a shuffle 501 // based lowering for vectors in LegalizeVectorOps. 502 if (!OVT.isVector() && 503 !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) { 504 if (SDValue Res = TLI.expandBSWAP(N, DAG)) 505 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); 506 } 507 508 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 509 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 510 DAG.getShiftAmountConstant(DiffBits, NVT, dl)); 511 } 512 513 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { 514 SDValue Op = GetPromotedInteger(N->getOperand(0)); 515 EVT OVT = N->getValueType(0); 516 EVT NVT = Op.getValueType(); 517 SDLoc dl(N); 518 519 // If the larger BITREVERSE isn't supported by the target, try to expand now. 520 // If we expand later we'll end up with more operations since we lost the 521 // original type. We only do this for scalars since we have a shuffle 522 // based lowering for vectors in LegalizeVectorOps. 523 if (!OVT.isVector() && OVT.isSimple() && 524 !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) { 525 if (SDValue Res = TLI.expandBITREVERSE(N, DAG)) 526 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); 527 } 528 529 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 530 return DAG.getNode(ISD::SRL, dl, NVT, 531 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), 532 DAG.getShiftAmountConstant(DiffBits, NVT, dl)); 533 } 534 535 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 536 // The pair element type may be legal, or may not promote to the same type as 537 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 538 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 539 TLI.getTypeToTransformTo(*DAG.getContext(), 540 N->getValueType(0)), JoinIntegers(N->getOperand(0), 541 N->getOperand(1))); 542 } 543 544 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 545 EVT VT = N->getValueType(0); 546 // FIXME there is no actual debug info here 547 SDLoc dl(N); 548 // Zero extend things like i1, sign extend everything else. It shouldn't 549 // matter in theory which one we pick, but this tends to give better code? 550 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 551 SDValue Result = DAG.getNode(Opc, dl, 552 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 553 SDValue(N, 0)); 554 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 555 return Result; 556 } 557 558 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 559 EVT OVT = N->getValueType(0); 560 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 561 SDLoc dl(N); 562 563 // If the larger CTLZ isn't supported by the target, try to expand now. 564 // If we expand later we'll end up with more operations since we lost the 565 // original type. 566 if (!OVT.isVector() && TLI.isTypeLegal(NVT) && 567 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ, NVT) && 568 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ_ZERO_UNDEF, NVT)) { 569 if (SDValue Result = TLI.expandCTLZ(N, DAG)) { 570 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); 571 return Result; 572 } 573 } 574 575 // Zero extend to the promoted type and do the count there. 576 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 577 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 578 // Subtract off the extra leading bits in the bigger type. 579 return DAG.getNode( 580 ISD::SUB, dl, NVT, Op, 581 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, 582 NVT)); 583 } 584 585 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { 586 EVT OVT = N->getValueType(0); 587 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 588 589 // If the larger CTPOP isn't supported by the target, try to expand now. 590 // If we expand later we'll end up with more operations since we lost the 591 // original type. 592 // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to 593 // TargetLowering. 594 if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) && 595 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTPOP, NVT)) { 596 if (SDValue Result = TLI.expandCTPOP(N, DAG)) { 597 Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result); 598 return Result; 599 } 600 } 601 602 // Zero extend to the promoted type and do the count or parity there. 603 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 604 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op); 605 } 606 607 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 608 SDValue Op = GetPromotedInteger(N->getOperand(0)); 609 EVT OVT = N->getValueType(0); 610 EVT NVT = Op.getValueType(); 611 SDLoc dl(N); 612 613 // If the larger CTTZ isn't supported by the target, try to expand now. 614 // If we expand later we'll end up with more operations since we lost the 615 // original type. Don't expand if we can use CTPOP or CTLZ expansion on the 616 // larger type. 617 if (!OVT.isVector() && TLI.isTypeLegal(NVT) && 618 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ, NVT) && 619 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ_ZERO_UNDEF, NVT) && 620 !TLI.isOperationLegal(ISD::CTPOP, NVT) && 621 !TLI.isOperationLegal(ISD::CTLZ, NVT)) { 622 if (SDValue Result = TLI.expandCTTZ(N, DAG)) { 623 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); 624 return Result; 625 } 626 } 627 628 if (N->getOpcode() == ISD::CTTZ) { 629 // The count is the same in the promoted type except if the original 630 // value was zero. This can be handled by setting the bit just off 631 // the top of the original type. 632 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), 633 OVT.getScalarSizeInBits()); 634 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); 635 } 636 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 637 } 638 639 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 640 SDLoc dl(N); 641 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 642 643 SDValue Op0 = N->getOperand(0); 644 SDValue Op1 = N->getOperand(1); 645 646 // If the input also needs to be promoted, do that first so we can get a 647 // get a good idea for the output type. 648 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType()) 649 == TargetLowering::TypePromoteInteger) { 650 SDValue In = GetPromotedInteger(Op0); 651 652 // If the new type is larger than NVT, use it. We probably won't need to 653 // promote it again. 654 EVT SVT = In.getValueType().getScalarType(); 655 if (SVT.bitsGE(NVT)) { 656 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1); 657 return DAG.getAnyExtOrTrunc(Ext, dl, NVT); 658 } 659 } 660 661 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1); 662 } 663 664 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 665 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 666 unsigned NewOpc = N->getOpcode(); 667 SDLoc dl(N); 668 669 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 670 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 671 // and SINT conversions are Custom, there is no way to tell which is 672 // preferable. We choose SINT because that's the right thing on PPC.) 673 if (N->getOpcode() == ISD::FP_TO_UINT && 674 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 675 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 676 NewOpc = ISD::FP_TO_SINT; 677 678 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && 679 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) && 680 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) 681 NewOpc = ISD::STRICT_FP_TO_SINT; 682 683 if (N->getOpcode() == ISD::VP_FP_TO_UINT && 684 !TLI.isOperationLegal(ISD::VP_FP_TO_UINT, NVT) && 685 TLI.isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, NVT)) 686 NewOpc = ISD::VP_FP_TO_SINT; 687 688 SDValue Res; 689 if (N->isStrictFPOpcode()) { 690 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, 691 {N->getOperand(0), N->getOperand(1)}); 692 // Legalize the chain result - switch anything that used the old chain to 693 // use the new one. 694 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 695 } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) { 696 Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1), 697 N->getOperand(2)}); 698 } else { 699 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 700 } 701 702 // Assert that the converted value fits in the original type. If it doesn't 703 // (eg: because the value being converted is too big), then the result of the 704 // original operation was undefined anyway, so the assert is still correct. 705 // 706 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: 707 // before legalization: fp-to-uint16, 65534. -> 0xfffe 708 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe 709 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT || 710 N->getOpcode() == ISD::STRICT_FP_TO_UINT || 711 N->getOpcode() == ISD::VP_FP_TO_UINT) 712 ? ISD::AssertZext 713 : ISD::AssertSext, 714 dl, NVT, Res, 715 DAG.getValueType(N->getValueType(0).getScalarType())); 716 } 717 718 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { 719 // Promote the result type, while keeping the original width in Op1. 720 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 721 SDLoc dl(N); 722 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), 723 N->getOperand(1)); 724 } 725 726 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) { 727 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 728 SDLoc dl(N); 729 730 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 731 } 732 733 SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) { 734 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 735 SDLoc dl(N); 736 737 SDValue Res = 738 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); 739 740 // Legalize the chain result - switch anything that used the old chain to 741 // use the new one. 742 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 743 return Res; 744 } 745 746 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 747 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 748 SDLoc dl(N); 749 750 if (getTypeAction(N->getOperand(0).getValueType()) 751 == TargetLowering::TypePromoteInteger) { 752 SDValue Res = GetPromotedInteger(N->getOperand(0)); 753 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 754 755 // If the result and operand types are the same after promotion, simplify 756 // to an in-register extension. 757 if (NVT == Res.getValueType()) { 758 // The high bits are not guaranteed to be anything. Insert an extend. 759 if (N->getOpcode() == ISD::SIGN_EXTEND) 760 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 761 DAG.getValueType(N->getOperand(0).getValueType())); 762 if (N->getOpcode() == ISD::ZERO_EXTEND) 763 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); 764 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 765 return Res; 766 } 767 } 768 769 // Otherwise, just extend the original operand all the way to the larger type. 770 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 771 } 772 773 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 774 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 775 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 776 ISD::LoadExtType ExtType = 777 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 778 SDLoc dl(N); 779 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 780 N->getMemoryVT(), N->getMemOperand()); 781 782 // Legalize the chain result - switch anything that used the old chain to 783 // use the new one. 784 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 785 return Res; 786 } 787 788 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { 789 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 790 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 791 792 ISD::LoadExtType ExtType = N->getExtensionType(); 793 if (ExtType == ISD::NON_EXTLOAD) 794 ExtType = ISD::EXTLOAD; 795 796 SDLoc dl(N); 797 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), 798 N->getOffset(), N->getMask(), ExtPassThru, 799 N->getMemoryVT(), N->getMemOperand(), 800 N->getAddressingMode(), ExtType, 801 N->isExpandingLoad()); 802 // Legalize the chain result - switch anything that used the old chain to 803 // use the new one. 804 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 805 return Res; 806 } 807 808 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { 809 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 810 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); 811 assert(NVT == ExtPassThru.getValueType() && 812 "Gather result type and the passThru argument type should be the same"); 813 814 ISD::LoadExtType ExtType = N->getExtensionType(); 815 if (ExtType == ISD::NON_EXTLOAD) 816 ExtType = ISD::EXTLOAD; 817 818 SDLoc dl(N); 819 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), 820 N->getIndex(), N->getScale() }; 821 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), 822 N->getMemoryVT(), dl, Ops, 823 N->getMemOperand(), N->getIndexType(), 824 ExtType); 825 // Legalize the chain result - switch anything that used the old chain to 826 // use the new one. 827 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 828 return Res; 829 } 830 831 /// Promote the overflow flag of an overflowing arithmetic node. 832 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 833 // Change the return type of the boolean result while obeying 834 // getSetCCResultType. 835 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 836 EVT VT = N->getValueType(0); 837 EVT SVT = getSetCCResultType(VT); 838 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; 839 unsigned NumOps = N->getNumOperands(); 840 assert(NumOps <= 3 && "Too many operands"); 841 if (NumOps == 3) 842 Ops[2] = N->getOperand(2); 843 844 SDLoc dl(N); 845 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), 846 ArrayRef(Ops, NumOps)); 847 848 // Modified the sum result - switch anything that used the old sum to use 849 // the new one. 850 ReplaceValueWith(SDValue(N, 0), Res); 851 852 // Convert to the expected type. 853 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); 854 } 855 856 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { 857 // If the promoted type is legal, we can convert this to: 858 // 1. ANY_EXTEND iN to iM 859 // 2. SHL by M-N 860 // 3. [US][ADD|SUB|SHL]SAT 861 // 4. L/ASHR by M-N 862 // Else it is more efficient to convert this to a min and a max 863 // operation in the higher precision arithmetic. 864 SDLoc dl(N); 865 SDValue Op1 = N->getOperand(0); 866 SDValue Op2 = N->getOperand(1); 867 unsigned OldBits = Op1.getScalarValueSizeInBits(); 868 869 unsigned Opcode = N->getOpcode(); 870 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; 871 872 SDValue Op1Promoted, Op2Promoted; 873 if (IsShift) { 874 Op1Promoted = GetPromotedInteger(Op1); 875 Op2Promoted = ZExtPromotedInteger(Op2); 876 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { 877 Op1Promoted = ZExtPromotedInteger(Op1); 878 Op2Promoted = ZExtPromotedInteger(Op2); 879 } else { 880 Op1Promoted = SExtPromotedInteger(Op1); 881 Op2Promoted = SExtPromotedInteger(Op2); 882 } 883 EVT PromotedType = Op1Promoted.getValueType(); 884 unsigned NewBits = PromotedType.getScalarSizeInBits(); 885 886 if (Opcode == ISD::UADDSAT) { 887 APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits); 888 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 889 SDValue Add = 890 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); 891 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); 892 } 893 894 // USUBSAT can always be promoted as long as we have zero-extended the args. 895 if (Opcode == ISD::USUBSAT) 896 return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted, 897 Op2Promoted); 898 899 // Shift cannot use a min/max expansion, we can't detect overflow if all of 900 // the bits have been shifted out. 901 if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) { 902 unsigned ShiftOp; 903 switch (Opcode) { 904 case ISD::SADDSAT: 905 case ISD::SSUBSAT: 906 case ISD::SSHLSAT: 907 ShiftOp = ISD::SRA; 908 break; 909 case ISD::USHLSAT: 910 ShiftOp = ISD::SRL; 911 break; 912 default: 913 llvm_unreachable("Expected opcode to be signed or unsigned saturation " 914 "addition, subtraction or left shift"); 915 } 916 917 unsigned SHLAmount = NewBits - OldBits; 918 SDValue ShiftAmount = 919 DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl); 920 Op1Promoted = 921 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); 922 if (!IsShift) 923 Op2Promoted = 924 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); 925 926 SDValue Result = 927 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); 928 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); 929 } 930 931 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; 932 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits); 933 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits); 934 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType); 935 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); 936 SDValue Result = 937 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); 938 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); 939 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); 940 return Result; 941 } 942 943 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { 944 // Can just promote the operands then continue with operation. 945 SDLoc dl(N); 946 SDValue Op1Promoted, Op2Promoted; 947 bool Signed = 948 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; 949 bool Saturating = 950 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; 951 if (Signed) { 952 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 953 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 954 } else { 955 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 956 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 957 } 958 EVT OldType = N->getOperand(0).getValueType(); 959 EVT PromotedType = Op1Promoted.getValueType(); 960 unsigned DiffSize = 961 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); 962 963 if (Saturating) { 964 // Promoting the operand and result values changes the saturation width, 965 // which is extends the values that we clamp to on saturation. This could be 966 // resolved by shifting one of the operands the same amount, which would 967 // also shift the result we compare against, then shifting back. 968 Op1Promoted = 969 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 970 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); 971 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 972 Op2Promoted, N->getOperand(2)); 973 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; 974 return DAG.getNode(ShiftOp, dl, PromotedType, Result, 975 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); 976 } 977 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, 978 N->getOperand(2)); 979 } 980 981 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, 982 unsigned SatW, bool Signed, 983 const TargetLowering &TLI, 984 SelectionDAG &DAG) { 985 EVT VT = V.getValueType(); 986 unsigned VTW = VT.getScalarSizeInBits(); 987 988 if (!Signed) { 989 // Saturate to the unsigned maximum by getting the minimum of V and the 990 // maximum. 991 return DAG.getNode(ISD::UMIN, dl, VT, V, 992 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW), 993 dl, VT)); 994 } 995 996 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the 997 // signed minimum of it and V. 998 V = DAG.getNode(ISD::SMIN, dl, VT, V, 999 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1), 1000 dl, VT)); 1001 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the 1002 // signed maximum of it and V. 1003 V = DAG.getNode(ISD::SMAX, dl, VT, V, 1004 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1), 1005 dl, VT)); 1006 return V; 1007 } 1008 1009 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, 1010 unsigned Scale, const TargetLowering &TLI, 1011 SelectionDAG &DAG, unsigned SatW = 0) { 1012 EVT VT = LHS.getValueType(); 1013 unsigned VTSize = VT.getScalarSizeInBits(); 1014 bool Signed = N->getOpcode() == ISD::SDIVFIX || 1015 N->getOpcode() == ISD::SDIVFIXSAT; 1016 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 1017 N->getOpcode() == ISD::UDIVFIXSAT; 1018 1019 SDLoc dl(N); 1020 // Widen the types by a factor of two. This is guaranteed to expand, since it 1021 // will always have enough high bits in the LHS to shift into. 1022 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2); 1023 if (VT.isVector()) 1024 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, 1025 VT.getVectorElementCount()); 1026 if (Signed) { 1027 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT); 1028 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT); 1029 } else { 1030 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT); 1031 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT); 1032 } 1033 1034 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, 1035 DAG); 1036 assert(Res && "Expanding DIVFIX with wide type failed?"); 1037 if (Saturating) { 1038 // If the caller has told us to saturate at something less, use that width 1039 // instead of the type before doubling. However, it cannot be more than 1040 // what we just widened! 1041 assert(SatW <= VTSize && 1042 "Tried to saturate to more than the original type?"); 1043 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed, 1044 TLI, DAG); 1045 } 1046 return DAG.getZExtOrTrunc(Res, dl, VT); 1047 } 1048 1049 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { 1050 SDLoc dl(N); 1051 SDValue Op1Promoted, Op2Promoted; 1052 bool Signed = N->getOpcode() == ISD::SDIVFIX || 1053 N->getOpcode() == ISD::SDIVFIXSAT; 1054 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || 1055 N->getOpcode() == ISD::UDIVFIXSAT; 1056 if (Signed) { 1057 Op1Promoted = SExtPromotedInteger(N->getOperand(0)); 1058 Op2Promoted = SExtPromotedInteger(N->getOperand(1)); 1059 } else { 1060 Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); 1061 Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); 1062 } 1063 EVT PromotedType = Op1Promoted.getValueType(); 1064 unsigned Scale = N->getConstantOperandVal(2); 1065 1066 // If the type is already legal and the operation is legal in that type, we 1067 // should not early expand. 1068 if (TLI.isTypeLegal(PromotedType)) { 1069 TargetLowering::LegalizeAction Action = 1070 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale); 1071 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { 1072 unsigned Diff = PromotedType.getScalarSizeInBits() - 1073 N->getValueType(0).getScalarSizeInBits(); 1074 if (Saturating) 1075 Op1Promoted = 1076 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, 1077 DAG.getShiftAmountConstant(Diff, PromotedType, dl)); 1078 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, 1079 Op2Promoted, N->getOperand(2)); 1080 if (Saturating) 1081 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res, 1082 DAG.getShiftAmountConstant(Diff, PromotedType, dl)); 1083 return Res; 1084 } 1085 } 1086 1087 // See if we can perform the division in this type without expanding. 1088 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted, 1089 Op2Promoted, Scale, DAG)) { 1090 if (Saturating) 1091 Res = SaturateWidenedDIVFIX(Res, dl, 1092 N->getValueType(0).getScalarSizeInBits(), 1093 Signed, TLI, DAG); 1094 return Res; 1095 } 1096 // If we cannot, expand it to twice the type width. If we are saturating, give 1097 // it the original width as a saturating width so we don't need to emit 1098 // two saturations. 1099 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG, 1100 N->getValueType(0).getScalarSizeInBits()); 1101 } 1102 1103 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 1104 if (ResNo == 1) 1105 return PromoteIntRes_Overflow(N); 1106 1107 // The operation overflowed iff the result in the larger type is not the 1108 // sign extension of its truncation to the original type. 1109 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1110 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1111 EVT OVT = N->getOperand(0).getValueType(); 1112 EVT NVT = LHS.getValueType(); 1113 SDLoc dl(N); 1114 1115 // Do the arithmetic in the larger type. 1116 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 1117 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1118 1119 // Calculate the overflow flag: sign extend the arithmetic result from 1120 // the original type. 1121 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 1122 DAG.getValueType(OVT)); 1123 // Overflowed if and only if this is not equal to Res. 1124 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1125 1126 // Use the calculated overflow everywhere. 1127 ReplaceValueWith(SDValue(N, 1), Ofl); 1128 1129 return Res; 1130 } 1131 1132 SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) { 1133 SDValue Mask = N->getOperand(0); 1134 1135 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 1136 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 1137 1138 unsigned Opcode = N->getOpcode(); 1139 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) 1140 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS, 1141 N->getOperand(3)); 1142 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS); 1143 } 1144 1145 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 1146 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 1147 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 1148 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 1149 LHS.getValueType(), N->getOperand(0), 1150 N->getOperand(1), LHS, RHS, N->getOperand(4)); 1151 } 1152 1153 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 1154 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 1155 EVT InVT = N->getOperand(OpNo).getValueType(); 1156 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1157 1158 EVT SVT = getSetCCResultType(InVT); 1159 1160 // If we got back a type that needs to be promoted, this likely means the 1161 // the input type also needs to be promoted. So get the promoted type for 1162 // the input and try the query again. 1163 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { 1164 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 1165 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 1166 SVT = getSetCCResultType(InVT); 1167 } else { 1168 // Input type isn't promoted, just use the default promoted type. 1169 SVT = NVT; 1170 } 1171 } 1172 1173 SDLoc dl(N); 1174 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && 1175 "Vector compare must return a vector result!"); 1176 1177 // Get the SETCC result using the canonical SETCC type. 1178 SDValue SetCC; 1179 if (N->isStrictFPOpcode()) { 1180 EVT VTs[] = {SVT, MVT::Other}; 1181 SDValue Opers[] = {N->getOperand(0), N->getOperand(1), 1182 N->getOperand(2), N->getOperand(3)}; 1183 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers); 1184 // Legalize the chain result - switch anything that used the old chain to 1185 // use the new one. 1186 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1)); 1187 } else 1188 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), 1189 N->getOperand(1), N->getOperand(2)); 1190 1191 // Convert to the expected type. 1192 return DAG.getSExtOrTrunc(SetCC, dl, NVT); 1193 } 1194 1195 SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) { 1196 SDLoc DL(N); 1197 SDValue Arg = N->getOperand(0); 1198 SDValue Test = N->getOperand(1); 1199 EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1200 return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test); 1201 } 1202 1203 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 1204 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1205 SDValue RHS = N->getOperand(1); 1206 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1207 RHS = ZExtPromotedInteger(RHS); 1208 if (N->getOpcode() != ISD::VP_SHL) 1209 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1210 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1211 N->getOperand(2), N->getOperand(3)); 1212 } 1213 1214 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 1215 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1216 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 1217 Op.getValueType(), Op, N->getOperand(1)); 1218 } 1219 1220 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 1221 // The input may have strange things in the top bits of the registers, but 1222 // these operations don't care. They may have weird bits going out, but 1223 // that too is okay if they are integer operations. 1224 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 1225 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 1226 if (N->getNumOperands() == 2) 1227 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1228 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1229 assert(N->isVPOpcode() && "Expected VP opcode"); 1230 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1231 N->getOperand(2), N->getOperand(3)); 1232 } 1233 1234 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { 1235 // Sign extend the input. 1236 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1237 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1238 if (N->getNumOperands() == 2) 1239 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1240 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1241 assert(N->isVPOpcode() && "Expected VP opcode"); 1242 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1243 N->getOperand(2), N->getOperand(3)); 1244 } 1245 1246 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { 1247 // Zero extend the input. 1248 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1249 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1250 if (N->getNumOperands() == 2) 1251 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1252 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1253 assert(N->isVPOpcode() && "Expected VP opcode"); 1254 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1255 N->getOperand(2), N->getOperand(3)); 1256 } 1257 1258 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { 1259 // It doesn't matter if we sign extend or zero extend in the inputs. So do 1260 // whatever is best for the target. 1261 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0)); 1262 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1)); 1263 return DAG.getNode(N->getOpcode(), SDLoc(N), 1264 LHS.getValueType(), LHS, RHS); 1265 } 1266 1267 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 1268 // The input value must be properly sign extended. 1269 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1270 SDValue RHS = N->getOperand(1); 1271 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1272 RHS = ZExtPromotedInteger(RHS); 1273 if (N->getOpcode() != ISD::VP_ASHR) 1274 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1275 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1276 N->getOperand(2), N->getOperand(3)); 1277 } 1278 1279 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 1280 // The input value must be properly zero extended. 1281 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1282 SDValue RHS = N->getOperand(1); 1283 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 1284 RHS = ZExtPromotedInteger(RHS); 1285 if (N->getOpcode() != ISD::VP_LSHR) 1286 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); 1287 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, 1288 N->getOperand(2), N->getOperand(3)); 1289 } 1290 1291 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { 1292 // Lower the rotate to shifts and ORs which can be promoted. 1293 SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG); 1294 ReplaceValueWith(SDValue(N, 0), Res); 1295 return SDValue(); 1296 } 1297 1298 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { 1299 SDValue Hi = GetPromotedInteger(N->getOperand(0)); 1300 SDValue Lo = GetPromotedInteger(N->getOperand(1)); 1301 SDValue Amt = N->getOperand(2); 1302 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger) 1303 Amt = ZExtPromotedInteger(Amt); 1304 EVT AmtVT = Amt.getValueType(); 1305 1306 SDLoc DL(N); 1307 EVT OldVT = N->getOperand(0).getValueType(); 1308 EVT VT = Lo.getValueType(); 1309 unsigned Opcode = N->getOpcode(); 1310 bool IsFSHR = Opcode == ISD::FSHR; 1311 unsigned OldBits = OldVT.getScalarSizeInBits(); 1312 unsigned NewBits = VT.getScalarSizeInBits(); 1313 1314 // Amount has to be interpreted modulo the old bit width. 1315 Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt, 1316 DAG.getConstant(OldBits, DL, AmtVT)); 1317 1318 // If the promoted type is twice the size (or more), then we use the 1319 // traditional funnel 'double' shift codegen. This isn't necessary if the 1320 // shift amount is constant. 1321 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. 1322 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). 1323 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) && 1324 !TLI.isOperationLegalOrCustom(Opcode, VT)) { 1325 SDValue HiShift = DAG.getConstant(OldBits, DL, VT); 1326 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift); 1327 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT); 1328 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo); 1329 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt); 1330 if (!IsFSHR) 1331 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift); 1332 return Res; 1333 } 1334 1335 // Shift Lo up to occupy the upper bits of the promoted type. 1336 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT); 1337 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset); 1338 1339 // Increase Amount to shift the result into the lower bits of the promoted 1340 // type. 1341 if (IsFSHR) 1342 Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset); 1343 1344 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt); 1345 } 1346 1347 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 1348 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1349 SDValue Res; 1350 SDValue InOp = N->getOperand(0); 1351 SDLoc dl(N); 1352 1353 switch (getTypeAction(InOp.getValueType())) { 1354 default: llvm_unreachable("Unknown type action!"); 1355 case TargetLowering::TypeLegal: 1356 case TargetLowering::TypeExpandInteger: 1357 Res = InOp; 1358 break; 1359 case TargetLowering::TypePromoteInteger: 1360 Res = GetPromotedInteger(InOp); 1361 break; 1362 case TargetLowering::TypeSplitVector: { 1363 EVT InVT = InOp.getValueType(); 1364 assert(InVT.isVector() && "Cannot split scalar types"); 1365 ElementCount NumElts = InVT.getVectorElementCount(); 1366 assert(NumElts == NVT.getVectorElementCount() && 1367 "Dst and Src must have the same number of elements"); 1368 assert(isPowerOf2_32(NumElts.getKnownMinValue()) && 1369 "Promoted vector type must be a power of two"); 1370 1371 SDValue EOp1, EOp2; 1372 GetSplitVector(InOp, EOp1, EOp2); 1373 1374 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 1375 NumElts.divideCoefficientBy(2)); 1376 if (N->getOpcode() == ISD::TRUNCATE) { 1377 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 1378 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 1379 } else { 1380 assert(N->getOpcode() == ISD::VP_TRUNCATE && 1381 "Expected VP_TRUNCATE opcode"); 1382 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 1383 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 1384 std::tie(EVLLo, EVLHi) = 1385 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); 1386 EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo); 1387 EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi); 1388 } 1389 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 1390 } 1391 // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some 1392 // targets. 1393 case TargetLowering::TypeWidenVector: { 1394 SDValue WideInOp = GetWidenedVector(InOp); 1395 1396 // Truncate widened InOp. 1397 unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); 1398 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), 1399 N->getValueType(0).getScalarType(), NumElem); 1400 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); 1401 1402 // Zero extend so that the elements are of same type as those of NVT 1403 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), 1404 NumElem); 1405 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); 1406 1407 // Extract the low NVT subvector. 1408 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl); 1409 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); 1410 } 1411 } 1412 1413 // Truncate to NVT instead of VT 1414 if (N->getOpcode() == ISD::VP_TRUNCATE) 1415 return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1), 1416 N->getOperand(2)); 1417 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 1418 } 1419 1420 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 1421 if (ResNo == 1) 1422 return PromoteIntRes_Overflow(N); 1423 1424 // The operation overflowed iff the result in the larger type is not the 1425 // zero extension of its truncation to the original type. 1426 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 1427 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 1428 EVT OVT = N->getOperand(0).getValueType(); 1429 EVT NVT = LHS.getValueType(); 1430 SDLoc dl(N); 1431 1432 // Do the arithmetic in the larger type. 1433 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 1434 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 1435 1436 // Calculate the overflow flag: zero extend the arithmetic result from 1437 // the original type. 1438 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 1439 // Overflowed if and only if this is not equal to Res. 1440 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 1441 1442 // Use the calculated overflow everywhere. 1443 ReplaceValueWith(SDValue(N, 1), Ofl); 1444 1445 return Res; 1446 } 1447 1448 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that 1449 // the third operand of ADDE/SUBE nodes is carry flag, which differs from 1450 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean. 1451 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) { 1452 if (ResNo == 1) 1453 return PromoteIntRes_Overflow(N); 1454 1455 // We need to sign-extend the operands so the carry value computed by the 1456 // wide operation will be equivalent to the carry value computed by the 1457 // narrow operation. 1458 // An ADDCARRY can generate carry only if any of the operands has its 1459 // most significant bit set. Sign extension propagates the most significant 1460 // bit into the higher bits which means the extra bit that the narrow 1461 // addition would need (i.e. the carry) will be propagated through the higher 1462 // bits of the wide addition. 1463 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be 1464 // preserved by sign extension. 1465 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 1466 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 1467 1468 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; 1469 1470 // Do the arithmetic in the wide type. 1471 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), 1472 LHS, RHS, N->getOperand(2)); 1473 1474 // Update the users of the original carry/borrow value. 1475 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 1476 1477 return SDValue(Res.getNode(), 0); 1478 } 1479 1480 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, 1481 unsigned ResNo) { 1482 assert(ResNo == 1 && "Don't know how to promote other results yet."); 1483 return PromoteIntRes_Overflow(N); 1484 } 1485 1486 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { 1487 EVT OVT = N->getValueType(0); 1488 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 1489 1490 // If a larger ABS or SMAX isn't supported by the target, try to expand now. 1491 // If we expand later we'll end up sign extending more than just the sra input 1492 // in sra+xor+sub expansion. 1493 if (!OVT.isVector() && 1494 !TLI.isOperationLegalOrCustomOrPromote(ISD::ABS, NVT) && 1495 !TLI.isOperationLegal(ISD::SMAX, NVT)) { 1496 if (SDValue Res = TLI.expandABS(N, DAG)) 1497 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res); 1498 } 1499 1500 SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); 1501 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); 1502 } 1503 1504 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 1505 // Promote the overflow bit trivially. 1506 if (ResNo == 1) 1507 return PromoteIntRes_Overflow(N); 1508 1509 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 1510 SDLoc DL(N); 1511 EVT SmallVT = LHS.getValueType(); 1512 1513 // To determine if the result overflowed in a larger type, we extend the 1514 // input to the larger type, do the multiply (checking if it overflows), 1515 // then also check the high bits of the result to see if overflow happened 1516 // there. 1517 if (N->getOpcode() == ISD::SMULO) { 1518 LHS = SExtPromotedInteger(LHS); 1519 RHS = SExtPromotedInteger(RHS); 1520 } else { 1521 LHS = ZExtPromotedInteger(LHS); 1522 RHS = ZExtPromotedInteger(RHS); 1523 } 1524 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 1525 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 1526 1527 // Overflow occurred if it occurred in the larger type, or if the high part 1528 // of the result does not zero/sign-extend the low part. Check this second 1529 // possibility first. 1530 SDValue Overflow; 1531 if (N->getOpcode() == ISD::UMULO) { 1532 // Unsigned overflow occurred if the high part is non-zero. 1533 unsigned Shift = SmallVT.getScalarSizeInBits(); 1534 SDValue Hi = 1535 DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 1536 DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL)); 1537 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 1538 DAG.getConstant(0, DL, Hi.getValueType()), 1539 ISD::SETNE); 1540 } else { 1541 // Signed overflow occurred if the high part does not sign extend the low. 1542 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 1543 Mul, DAG.getValueType(SmallVT)); 1544 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 1545 } 1546 1547 // The only other way for overflow to occur is if the multiplication in the 1548 // larger type itself overflowed. 1549 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 1550 SDValue(Mul.getNode(), 1)); 1551 1552 // Use the calculated overflow everywhere. 1553 ReplaceValueWith(SDValue(N, 1), Overflow); 1554 return Mul; 1555 } 1556 1557 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 1558 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 1559 N->getValueType(0))); 1560 } 1561 1562 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { 1563 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1564 1565 const APInt &MulImm = N->getConstantOperandAPInt(0); 1566 return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits())); 1567 } 1568 1569 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 1570 SDValue Chain = N->getOperand(0); // Get the chain. 1571 SDValue Ptr = N->getOperand(1); // Get the pointer. 1572 EVT VT = N->getValueType(0); 1573 SDLoc dl(N); 1574 1575 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 1576 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 1577 // The argument is passed as NumRegs registers of type RegVT. 1578 1579 SmallVector<SDValue, 8> Parts(NumRegs); 1580 for (unsigned i = 0; i < NumRegs; ++i) { 1581 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 1582 N->getConstantOperandVal(3)); 1583 Chain = Parts[i].getValue(1); 1584 } 1585 1586 // Handle endianness of the load. 1587 if (DAG.getDataLayout().isBigEndian()) 1588 std::reverse(Parts.begin(), Parts.end()); 1589 1590 // Assemble the parts in the promoted type. 1591 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1592 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 1593 for (unsigned i = 1; i < NumRegs; ++i) { 1594 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 1595 // Shift it to the right position and "or" it in. 1596 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 1597 DAG.getConstant(i * RegVT.getSizeInBits(), dl, 1598 TLI.getPointerTy(DAG.getDataLayout()))); 1599 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 1600 } 1601 1602 // Modified the chain result - switch anything that used the old chain to 1603 // use the new one. 1604 ReplaceValueWith(SDValue(N, 1), Chain); 1605 1606 return Res; 1607 } 1608 1609 //===----------------------------------------------------------------------===// 1610 // Integer Operand Promotion 1611 //===----------------------------------------------------------------------===// 1612 1613 /// PromoteIntegerOperand - This method is called when the specified operand of 1614 /// the specified node is found to need promotion. At this point, all of the 1615 /// result types of the node are known to be legal, but other operands of the 1616 /// node may need promotion or expansion as well as the specified one. 1617 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 1618 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); 1619 dbgs() << "\n"); 1620 SDValue Res = SDValue(); 1621 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 1622 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 1623 return false; 1624 } 1625 1626 switch (N->getOpcode()) { 1627 default: 1628 #ifndef NDEBUG 1629 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 1630 N->dump(&DAG); dbgs() << "\n"; 1631 #endif 1632 llvm_unreachable("Do not know how to promote this operator's operand!"); 1633 1634 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 1635 case ISD::ATOMIC_STORE: 1636 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 1637 break; 1638 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 1639 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 1640 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 1641 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 1642 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 1643 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 1644 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 1645 case ISD::INSERT_VECTOR_ELT: 1646 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo); 1647 break; 1648 case ISD::SPLAT_VECTOR: 1649 case ISD::SCALAR_TO_VECTOR: 1650 Res = PromoteIntOp_ScalarOp(N); 1651 break; 1652 case ISD::VSELECT: 1653 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 1654 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 1655 case ISD::VP_SETCC: 1656 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 1657 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 1658 case ISD::VP_SINT_TO_FP: 1659 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 1660 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; 1661 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 1662 OpNo); break; 1663 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), 1664 OpNo); break; 1665 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), 1666 OpNo); break; 1667 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), 1668 OpNo); break; 1669 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), 1670 OpNo); break; 1671 case ISD::VP_TRUNCATE: 1672 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 1673 case ISD::BF16_TO_FP: 1674 case ISD::FP16_TO_FP: 1675 case ISD::VP_UINT_TO_FP: 1676 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 1677 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; 1678 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 1679 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; 1680 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break; 1681 1682 case ISD::SHL: 1683 case ISD::SRA: 1684 case ISD::SRL: 1685 case ISD::ROTL: 1686 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 1687 1688 case ISD::FSHL: 1689 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break; 1690 1691 case ISD::SADDO_CARRY: 1692 case ISD::SSUBO_CARRY: 1693 case ISD::ADDCARRY: 1694 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break; 1695 1696 case ISD::FRAMEADDR: 1697 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; 1698 1699 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break; 1700 1701 case ISD::SMULFIX: 1702 case ISD::SMULFIXSAT: 1703 case ISD::UMULFIX: 1704 case ISD::UMULFIXSAT: 1705 case ISD::SDIVFIX: 1706 case ISD::SDIVFIXSAT: 1707 case ISD::UDIVFIX: 1708 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; 1709 1710 case ISD::FPOWI: 1711 case ISD::STRICT_FPOWI: Res = PromoteIntOp_FPOWI(N); break; 1712 1713 case ISD::VECREDUCE_ADD: 1714 case ISD::VECREDUCE_MUL: 1715 case ISD::VECREDUCE_AND: 1716 case ISD::VECREDUCE_OR: 1717 case ISD::VECREDUCE_XOR: 1718 case ISD::VECREDUCE_SMAX: 1719 case ISD::VECREDUCE_SMIN: 1720 case ISD::VECREDUCE_UMAX: 1721 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; 1722 case ISD::VP_REDUCE_ADD: 1723 case ISD::VP_REDUCE_MUL: 1724 case ISD::VP_REDUCE_AND: 1725 case ISD::VP_REDUCE_OR: 1726 case ISD::VP_REDUCE_XOR: 1727 case ISD::VP_REDUCE_SMAX: 1728 case ISD::VP_REDUCE_SMIN: 1729 case ISD::VP_REDUCE_UMAX: 1730 case ISD::VP_REDUCE_UMIN: 1731 Res = PromoteIntOp_VP_REDUCE(N, OpNo); 1732 break; 1733 1734 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; 1735 case ISD::STACKMAP: 1736 Res = PromoteIntOp_STACKMAP(N, OpNo); 1737 break; 1738 case ISD::PATCHPOINT: 1739 Res = PromoteIntOp_PATCHPOINT(N, OpNo); 1740 break; 1741 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 1742 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 1743 Res = PromoteIntOp_VP_STRIDED(N, OpNo); 1744 break; 1745 } 1746 1747 // If the result is null, the sub-method took care of registering results etc. 1748 if (!Res.getNode()) return false; 1749 1750 // If the result is N, the sub-method updated N in place. Tell the legalizer 1751 // core about this. 1752 if (Res.getNode() == N) 1753 return true; 1754 1755 const bool IsStrictFp = N->isStrictFPOpcode(); 1756 assert(Res.getValueType() == N->getValueType(0) && 1757 N->getNumValues() == (IsStrictFp ? 2 : 1) && 1758 "Invalid operand expansion"); 1759 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: "; 1760 Res.dump()); 1761 1762 ReplaceValueWith(SDValue(N, 0), Res); 1763 if (IsStrictFp) 1764 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 1765 1766 return false; 1767 } 1768 1769 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is 1770 /// shared among BR_CC, SELECT_CC, and SETCC handlers. 1771 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS, 1772 ISD::CondCode CCCode) { 1773 // We have to insert explicit sign or zero extends. Note that we could 1774 // insert sign extends for ALL conditions. For those operations where either 1775 // zero or sign extension would be valid, we ask the target which extension 1776 // it would prefer. 1777 1778 // Signed comparisons always require sign extension. 1779 if (ISD::isSignedIntSetCC(CCCode)) { 1780 LHS = SExtPromotedInteger(LHS); 1781 RHS = SExtPromotedInteger(RHS); 1782 return; 1783 } 1784 1785 assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) && 1786 "Unknown integer comparison!"); 1787 1788 SDValue OpL = GetPromotedInteger(LHS); 1789 SDValue OpR = GetPromotedInteger(RHS); 1790 1791 if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) { 1792 // The target would prefer to promote the comparison operand with sign 1793 // extension. Honor that unless the promoted values are already zero 1794 // extended. 1795 unsigned OpLEffectiveBits = 1796 DAG.computeKnownBits(OpL).countMaxActiveBits(); 1797 unsigned OpREffectiveBits = 1798 DAG.computeKnownBits(OpR).countMaxActiveBits(); 1799 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && 1800 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { 1801 LHS = OpL; 1802 RHS = OpR; 1803 return; 1804 } 1805 1806 // The promoted values aren't zero extended, use a sext_inreg. 1807 LHS = SExtPromotedInteger(LHS); 1808 RHS = SExtPromotedInteger(RHS); 1809 return; 1810 } 1811 1812 // Prefer to promote the comparison operand with zero extension. 1813 1814 // If the width of OpL/OpR excluding the duplicated sign bits is no greater 1815 // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation 1816 // that we might not be able to remove. 1817 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL); 1818 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR); 1819 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && 1820 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { 1821 LHS = OpL; 1822 RHS = OpR; 1823 return; 1824 } 1825 1826 // Otherwise, use zext_inreg. 1827 LHS = ZExtPromotedInteger(LHS); 1828 RHS = ZExtPromotedInteger(RHS); 1829 } 1830 1831 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 1832 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1833 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 1834 } 1835 1836 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 1837 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 1838 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 1839 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); 1840 } 1841 1842 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 1843 // This should only occur in unusual situations like bitcasting to an 1844 // x86_fp80, so just turn it into a store+load 1845 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 1846 } 1847 1848 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 1849 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1850 1851 SDValue LHS = N->getOperand(2); 1852 SDValue RHS = N->getOperand(3); 1853 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 1854 1855 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 1856 // legal types. 1857 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1858 N->getOperand(1), LHS, RHS, N->getOperand(4)), 1859 0); 1860 } 1861 1862 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 1863 assert(OpNo == 1 && "only know how to promote condition"); 1864 1865 // Promote all the way up to the canonical SetCC type. 1866 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 1867 1868 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 1869 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 1870 N->getOperand(2)), 0); 1871 } 1872 1873 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 1874 // Since the result type is legal, the operands must promote to it. 1875 EVT OVT = N->getOperand(0).getValueType(); 1876 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 1877 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 1878 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 1879 SDLoc dl(N); 1880 1881 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 1882 DAG.getConstant(OVT.getSizeInBits(), dl, 1883 TLI.getPointerTy(DAG.getDataLayout()))); 1884 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 1885 } 1886 1887 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 1888 // The vector type is legal but the element type is not. This implies 1889 // that the vector is a power-of-two in length and that the element 1890 // type does not have a strange size (eg: it is not i1). 1891 EVT VecVT = N->getValueType(0); 1892 unsigned NumElts = VecVT.getVectorNumElements(); 1893 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 1894 "Legal vector of one illegal element?"); 1895 1896 // Promote the inserted value. The type does not need to match the 1897 // vector element type. Check that any extra bits introduced will be 1898 // truncated away. 1899 assert(N->getOperand(0).getValueSizeInBits() >= 1900 N->getValueType(0).getScalarSizeInBits() && 1901 "Type of inserted value narrower than vector element type!"); 1902 1903 SmallVector<SDValue, 16> NewOps; 1904 for (unsigned i = 0; i < NumElts; ++i) 1905 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 1906 1907 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1908 } 1909 1910 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 1911 unsigned OpNo) { 1912 if (OpNo == 1) { 1913 // Promote the inserted value. This is valid because the type does not 1914 // have to match the vector element type. 1915 1916 // Check that any extra bits introduced will be truncated away. 1917 assert(N->getOperand(1).getValueSizeInBits() >= 1918 N->getValueType(0).getScalarSizeInBits() && 1919 "Type of inserted value narrower than vector element type!"); 1920 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1921 GetPromotedInteger(N->getOperand(1)), 1922 N->getOperand(2)), 1923 0); 1924 } 1925 1926 assert(OpNo == 2 && "Different operand and result vector types?"); 1927 1928 // Promote the index. 1929 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 1930 TLI.getVectorIdxTy(DAG.getDataLayout())); 1931 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1932 N->getOperand(1), Idx), 0); 1933 } 1934 1935 SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) { 1936 // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated, 1937 // so just promote the operand in place. 1938 return SDValue(DAG.UpdateNodeOperands(N, 1939 GetPromotedInteger(N->getOperand(0))), 0); 1940 } 1941 1942 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 1943 assert(OpNo == 0 && "Only know how to promote the condition!"); 1944 SDValue Cond = N->getOperand(0); 1945 EVT OpTy = N->getOperand(1).getValueType(); 1946 1947 if (N->getOpcode() == ISD::VSELECT) 1948 if (SDValue Res = WidenVSELECTMask(N)) 1949 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 1950 Res, N->getOperand(1), N->getOperand(2)); 1951 1952 // Promote all the way up to the canonical SetCC type. 1953 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 1954 Cond = PromoteTargetBoolean(Cond, OpVT); 1955 1956 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 1957 N->getOperand(2)), 0); 1958 } 1959 1960 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 1961 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1962 1963 SDValue LHS = N->getOperand(0); 1964 SDValue RHS = N->getOperand(1); 1965 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1966 1967 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1968 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1969 N->getOperand(3), N->getOperand(4)), 0); 1970 } 1971 1972 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1973 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1974 1975 SDValue LHS = N->getOperand(0); 1976 SDValue RHS = N->getOperand(1); 1977 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 1978 1979 // The CC (#2) is always legal. 1980 if (N->getOpcode() == ISD::SETCC) 1981 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 1982 1983 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode"); 1984 1985 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1986 N->getOperand(3), N->getOperand(4)), 1987 0); 1988 } 1989 1990 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 1991 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1992 ZExtPromotedInteger(N->getOperand(1))), 0); 1993 } 1994 1995 SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) { 1996 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), 1997 ZExtPromotedInteger(N->getOperand(2))), 0); 1998 } 1999 2000 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 2001 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2002 SDLoc dl(N); 2003 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 2004 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 2005 Op, DAG.getValueType(N->getOperand(0).getValueType())); 2006 } 2007 2008 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 2009 if (N->getOpcode() == ISD::VP_SINT_TO_FP) 2010 return SDValue(DAG.UpdateNodeOperands(N, 2011 SExtPromotedInteger(N->getOperand(0)), 2012 N->getOperand(1), N->getOperand(2)), 2013 0); 2014 return SDValue(DAG.UpdateNodeOperands(N, 2015 SExtPromotedInteger(N->getOperand(0))), 0); 2016 } 2017 2018 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { 2019 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2020 SExtPromotedInteger(N->getOperand(1))), 0); 2021 } 2022 2023 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 2024 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2025 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 2026 SDLoc dl(N); 2027 2028 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 2029 2030 // Truncate the value and store the result. 2031 return DAG.getTruncStore(Ch, dl, Val, Ptr, 2032 N->getMemoryVT(), N->getMemOperand()); 2033 } 2034 2035 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, 2036 unsigned OpNo) { 2037 SDValue DataOp = N->getValue(); 2038 SDValue Mask = N->getMask(); 2039 2040 if (OpNo == 4) { 2041 // The Mask. Update in place. 2042 EVT DataVT = DataOp.getValueType(); 2043 Mask = PromoteTargetBoolean(Mask, DataVT); 2044 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2045 NewOps[4] = Mask; 2046 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2047 } 2048 2049 assert(OpNo == 1 && "Unexpected operand for promotion"); 2050 DataOp = GetPromotedInteger(DataOp); 2051 2052 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(), 2053 N->getOffset(), Mask, N->getMemoryVT(), 2054 N->getMemOperand(), N->getAddressingMode(), 2055 /*IsTruncating*/ true, N->isCompressingStore()); 2056 } 2057 2058 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, 2059 unsigned OpNo) { 2060 assert(OpNo == 3 && "Only know how to promote the mask!"); 2061 EVT DataVT = N->getValueType(0); 2062 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2063 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2064 NewOps[OpNo] = Mask; 2065 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 2066 if (Res == N) 2067 return SDValue(Res, 0); 2068 2069 // Update triggered CSE, do our own replacement since caller can't. 2070 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 2071 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 2072 return SDValue(); 2073 } 2074 2075 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, 2076 unsigned OpNo) { 2077 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 2078 2079 if (OpNo == 2) { 2080 // The Mask 2081 EVT DataVT = N->getValueType(0); 2082 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2083 } else if (OpNo == 4) { 2084 // The Index 2085 if (N->isIndexSigned()) 2086 // Need to sign extend the index since the bits will likely be used. 2087 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2088 else 2089 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 2090 } else 2091 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 2092 2093 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); 2094 if (Res == N) 2095 return SDValue(Res, 0); 2096 2097 // Update triggered CSE, do our own replacement since caller can't. 2098 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); 2099 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); 2100 return SDValue(); 2101 } 2102 2103 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, 2104 unsigned OpNo) { 2105 bool TruncateStore = N->isTruncatingStore(); 2106 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 2107 2108 if (OpNo == 2) { 2109 // The Mask 2110 EVT DataVT = N->getValue().getValueType(); 2111 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 2112 } else if (OpNo == 4) { 2113 // The Index 2114 if (N->isIndexSigned()) 2115 // Need to sign extend the index since the bits will likely be used. 2116 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2117 else 2118 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); 2119 } else { 2120 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 2121 TruncateStore = true; 2122 } 2123 2124 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), 2125 SDLoc(N), NewOps, N->getMemOperand(), 2126 N->getIndexType(), TruncateStore); 2127 } 2128 2129 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 2130 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2131 if (N->getOpcode() == ISD::VP_TRUNCATE) 2132 return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op, 2133 N->getOperand(1), N->getOperand(2)); 2134 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 2135 } 2136 2137 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 2138 if (N->getOpcode() == ISD::VP_UINT_TO_FP) 2139 return SDValue(DAG.UpdateNodeOperands(N, 2140 ZExtPromotedInteger(N->getOperand(0)), 2141 N->getOperand(1), N->getOperand(2)), 2142 0); 2143 return SDValue(DAG.UpdateNodeOperands(N, 2144 ZExtPromotedInteger(N->getOperand(0))), 0); 2145 } 2146 2147 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { 2148 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2149 ZExtPromotedInteger(N->getOperand(1))), 0); 2150 } 2151 2152 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 2153 SDLoc dl(N); 2154 SDValue Op = GetPromotedInteger(N->getOperand(0)); 2155 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 2156 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); 2157 } 2158 2159 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { 2160 assert(OpNo == 2 && "Don't know how to promote this operand!"); 2161 2162 SDValue LHS = N->getOperand(0); 2163 SDValue RHS = N->getOperand(1); 2164 SDValue Carry = N->getOperand(2); 2165 SDLoc DL(N); 2166 2167 Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); 2168 2169 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); 2170 } 2171 2172 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { 2173 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 2174 return SDValue( 2175 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); 2176 } 2177 2178 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { 2179 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. 2180 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 2181 return SDValue(DAG.UpdateNodeOperands(N, Op), 0); 2182 } 2183 2184 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) { 2185 assert(OpNo > 1 && "Don't know how to promote this operand!"); 2186 // Promote the rw, locality, and cache type arguments to a supported integer 2187 // width. 2188 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); 2189 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3)); 2190 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4)); 2191 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), 2192 Op2, Op3, Op4), 2193 0); 2194 } 2195 2196 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) { 2197 bool IsStrict = N->isStrictFPOpcode(); 2198 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2199 2200 // The integer operand is the last operand in FPOWI (so the result and 2201 // floating point operand is already type legalized). 2202 2203 // We can't just promote the exponent type in FPOWI, since we want to lower 2204 // the node to a libcall and we if we promote to a type larger than 2205 // sizeof(int) the libcall might not be according to the targets ABI. Instead 2206 // we rewrite to a libcall here directly, letting makeLibCall handle promotion 2207 // if the target accepts it according to shouldSignExtendTypeInLibCall. 2208 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0)); 2209 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); 2210 if (!TLI.getLibcallName(LC)) { 2211 // Some targets don't have a powi libcall; use pow instead. 2212 // FIXME: Implement this if some target needs it. 2213 DAG.getContext()->emitError("Don't know how to promote fpowi to fpow"); 2214 return DAG.getUNDEF(N->getValueType(0)); 2215 } 2216 unsigned OpOffset = IsStrict ? 1 : 0; 2217 // The exponent should fit in a sizeof(int) type for the libcall to be valid. 2218 assert(DAG.getLibInfo().getIntSize() == 2219 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && 2220 "POWI exponent should match with sizeof(int) when doing the libcall."); 2221 TargetLowering::MakeLibCallOptions CallOptions; 2222 CallOptions.setSExt(true); 2223 SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)}; 2224 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall( 2225 DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain); 2226 ReplaceValueWith(SDValue(N, 0), Tmp.first); 2227 if (IsStrict) 2228 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2229 return SDValue(); 2230 } 2231 2232 static unsigned getExtendForIntVecReduction(SDNode *N) { 2233 switch (N->getOpcode()) { 2234 default: 2235 llvm_unreachable("Expected integer vector reduction"); 2236 case ISD::VECREDUCE_ADD: 2237 case ISD::VECREDUCE_MUL: 2238 case ISD::VECREDUCE_AND: 2239 case ISD::VECREDUCE_OR: 2240 case ISD::VECREDUCE_XOR: 2241 case ISD::VP_REDUCE_ADD: 2242 case ISD::VP_REDUCE_MUL: 2243 case ISD::VP_REDUCE_AND: 2244 case ISD::VP_REDUCE_OR: 2245 case ISD::VP_REDUCE_XOR: 2246 return ISD::ANY_EXTEND; 2247 case ISD::VECREDUCE_SMAX: 2248 case ISD::VECREDUCE_SMIN: 2249 case ISD::VP_REDUCE_SMAX: 2250 case ISD::VP_REDUCE_SMIN: 2251 return ISD::SIGN_EXTEND; 2252 case ISD::VECREDUCE_UMAX: 2253 case ISD::VECREDUCE_UMIN: 2254 case ISD::VP_REDUCE_UMAX: 2255 case ISD::VP_REDUCE_UMIN: 2256 return ISD::ZERO_EXTEND; 2257 } 2258 } 2259 2260 SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) { 2261 switch (getExtendForIntVecReduction(N)) { 2262 default: 2263 llvm_unreachable("Impossible extension kind for integer reduction"); 2264 case ISD::ANY_EXTEND: 2265 return GetPromotedInteger(V); 2266 case ISD::SIGN_EXTEND: 2267 return SExtPromotedInteger(V); 2268 case ISD::ZERO_EXTEND: 2269 return ZExtPromotedInteger(V); 2270 } 2271 } 2272 2273 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { 2274 SDLoc dl(N); 2275 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0)); 2276 2277 EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType(); 2278 EVT InVT = Op.getValueType(); 2279 EVT EltVT = InVT.getVectorElementType(); 2280 EVT ResVT = N->getValueType(0); 2281 unsigned Opcode = N->getOpcode(); 2282 2283 // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if 2284 // vecreduce_xor is not legal 2285 if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 && 2286 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_XOR, InVT) && 2287 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_ADD, InVT)) 2288 Opcode = ISD::VECREDUCE_ADD; 2289 2290 // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if 2291 // vecreduce_or is not legal 2292 else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 && 2293 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_OR, InVT) && 2294 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMAX, InVT)) 2295 Opcode = ISD::VECREDUCE_UMAX; 2296 2297 // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if 2298 // vecreduce_and is not legal 2299 else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 && 2300 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_AND, InVT) && 2301 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMIN, InVT)) 2302 Opcode = ISD::VECREDUCE_UMIN; 2303 2304 if (ResVT.bitsGE(EltVT)) 2305 return DAG.getNode(Opcode, SDLoc(N), ResVT, Op); 2306 2307 // Result size must be >= element size. If this is not the case after 2308 // promotion, also promote the result type and then truncate. 2309 SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op); 2310 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce); 2311 } 2312 2313 SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) { 2314 SDLoc DL(N); 2315 SDValue Op = N->getOperand(OpNo); 2316 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 2317 2318 if (OpNo == 2) { // Mask 2319 // Update in place. 2320 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType()); 2321 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2322 } 2323 2324 assert(OpNo == 1 && "Unexpected operand for promotion"); 2325 2326 Op = PromoteIntOpVectorReduction(N, Op); 2327 2328 NewOps[OpNo] = Op; 2329 2330 EVT VT = N->getValueType(0); 2331 EVT EltVT = Op.getValueType().getScalarType(); 2332 2333 if (VT.bitsGE(EltVT)) 2334 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps); 2335 2336 // Result size must be >= element/start-value size. If this is not the case 2337 // after promotion, also promote both the start value and result type and 2338 // then truncate. 2339 NewOps[0] = 2340 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0)); 2341 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps); 2342 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce); 2343 } 2344 2345 SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) { 2346 SDValue Op = ZExtPromotedInteger(N->getOperand(1)); 2347 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); 2348 } 2349 2350 SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) { 2351 assert(OpNo > 1); // Because the first two arguments are guaranteed legal. 2352 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 2353 SDValue Operand = N->getOperand(OpNo); 2354 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); 2355 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); 2356 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2357 } 2358 2359 SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { 2360 assert(OpNo >= 7); 2361 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 2362 SDValue Operand = N->getOperand(OpNo); 2363 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); 2364 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); 2365 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2366 } 2367 2368 SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { 2369 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || 2370 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); 2371 2372 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); 2373 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 2374 2375 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 2376 } 2377 2378 //===----------------------------------------------------------------------===// 2379 // Integer Result Expansion 2380 //===----------------------------------------------------------------------===// 2381 2382 /// ExpandIntegerResult - This method is called when the specified result of the 2383 /// specified node is found to need expansion. At this point, the node may also 2384 /// have invalid operands or may have other results that need promotion, we just 2385 /// know that (at least) one result needs expansion. 2386 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 2387 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); 2388 dbgs() << "\n"); 2389 SDValue Lo, Hi; 2390 Lo = Hi = SDValue(); 2391 2392 // See if the target wants to custom expand this node. 2393 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 2394 return; 2395 2396 switch (N->getOpcode()) { 2397 default: 2398 #ifndef NDEBUG 2399 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 2400 N->dump(&DAG); dbgs() << "\n"; 2401 #endif 2402 report_fatal_error("Do not know how to expand the result of this " 2403 "operator!"); 2404 2405 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break; 2406 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 2407 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; 2408 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 2409 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 2410 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; 2411 2412 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 2413 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 2414 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 2415 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 2416 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 2417 2418 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 2419 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 2420 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 2421 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; 2422 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 2423 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; 2424 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 2425 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; 2426 case ISD::CTLZ_ZERO_UNDEF: 2427 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 2428 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 2429 case ISD::CTTZ_ZERO_UNDEF: 2430 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 2431 case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break; 2432 case ISD::STRICT_FP_TO_SINT: 2433 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 2434 case ISD::STRICT_FP_TO_UINT: 2435 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 2436 case ISD::FP_TO_SINT_SAT: 2437 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; 2438 case ISD::STRICT_LROUND: 2439 case ISD::STRICT_LRINT: 2440 case ISD::LROUND: 2441 case ISD::LRINT: 2442 case ISD::STRICT_LLROUND: 2443 case ISD::STRICT_LLRINT: 2444 case ISD::LLROUND: 2445 case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break; 2446 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 2447 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 2448 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; 2449 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 2450 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 2451 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 2452 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 2453 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 2454 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 2455 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 2456 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 2457 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 2458 2459 case ISD::ATOMIC_LOAD_ADD: 2460 case ISD::ATOMIC_LOAD_SUB: 2461 case ISD::ATOMIC_LOAD_AND: 2462 case ISD::ATOMIC_LOAD_CLR: 2463 case ISD::ATOMIC_LOAD_OR: 2464 case ISD::ATOMIC_LOAD_XOR: 2465 case ISD::ATOMIC_LOAD_NAND: 2466 case ISD::ATOMIC_LOAD_MIN: 2467 case ISD::ATOMIC_LOAD_MAX: 2468 case ISD::ATOMIC_LOAD_UMIN: 2469 case ISD::ATOMIC_LOAD_UMAX: 2470 case ISD::ATOMIC_SWAP: 2471 case ISD::ATOMIC_CMP_SWAP: { 2472 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 2473 SplitInteger(Tmp.first, Lo, Hi); 2474 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2475 break; 2476 } 2477 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 2478 AtomicSDNode *AN = cast<AtomicSDNode>(N); 2479 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 2480 SDValue Tmp = DAG.getAtomicCmpSwap( 2481 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 2482 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 2483 AN->getMemOperand()); 2484 2485 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 2486 // success simply by comparing the loaded value against the ingoing 2487 // comparison. 2488 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 2489 N->getOperand(2), ISD::SETEQ); 2490 2491 SplitInteger(Tmp, Lo, Hi); 2492 ReplaceValueWith(SDValue(N, 1), Success); 2493 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 2494 break; 2495 } 2496 2497 case ISD::AND: 2498 case ISD::OR: 2499 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 2500 2501 case ISD::UMAX: 2502 case ISD::SMAX: 2503 case ISD::UMIN: 2504 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; 2505 2506 case ISD::ADD: 2507 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 2508 2509 case ISD::ADDC: 2510 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 2511 2512 case ISD::ADDE: 2513 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 2514 2515 case ISD::ADDCARRY: 2516 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break; 2517 2518 case ISD::SADDO_CARRY: 2519 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; 2520 2521 case ISD::SHL: 2522 case ISD::SRA: 2523 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 2524 2525 case ISD::SADDO: 2526 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 2527 case ISD::UADDO: 2528 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 2529 case ISD::UMULO: 2530 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 2531 2532 case ISD::SADDSAT: 2533 case ISD::UADDSAT: 2534 case ISD::SSUBSAT: 2535 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; 2536 2537 case ISD::SSHLSAT: 2538 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; 2539 2540 case ISD::SMULFIX: 2541 case ISD::SMULFIXSAT: 2542 case ISD::UMULFIX: 2543 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; 2544 2545 case ISD::SDIVFIX: 2546 case ISD::SDIVFIXSAT: 2547 case ISD::UDIVFIX: 2548 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; 2549 2550 case ISD::VECREDUCE_ADD: 2551 case ISD::VECREDUCE_MUL: 2552 case ISD::VECREDUCE_AND: 2553 case ISD::VECREDUCE_OR: 2554 case ISD::VECREDUCE_XOR: 2555 case ISD::VECREDUCE_SMAX: 2556 case ISD::VECREDUCE_SMIN: 2557 case ISD::VECREDUCE_UMAX: 2558 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; 2559 2560 case ISD::ROTL: 2561 case ISD::ROTR: 2562 ExpandIntRes_Rotate(N, Lo, Hi); 2563 break; 2564 2565 case ISD::FSHL: 2566 case ISD::FSHR: 2567 ExpandIntRes_FunnelShift(N, Lo, Hi); 2568 break; 2569 2570 case ISD::VSCALE: 2571 ExpandIntRes_VSCALE(N, Lo, Hi); 2572 break; 2573 } 2574 2575 // If Lo/Hi is null, the sub-method took care of registering results etc. 2576 if (Lo.getNode()) 2577 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 2578 } 2579 2580 /// Lower an atomic node to the appropriate builtin call. 2581 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 2582 unsigned Opc = Node->getOpcode(); 2583 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2584 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering(); 2585 // Lower to outline atomic libcall if outline atomics enabled, 2586 // or to sync libcall otherwise 2587 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); 2588 EVT RetVT = Node->getValueType(0); 2589 TargetLowering::MakeLibCallOptions CallOptions; 2590 SmallVector<SDValue, 4> Ops; 2591 if (TLI.getLibcallName(LC)) { 2592 Ops.append(Node->op_begin() + 2, Node->op_end()); 2593 Ops.push_back(Node->getOperand(1)); 2594 } else { 2595 LC = RTLIB::getSYNC(Opc, VT); 2596 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2597 "Unexpected atomic op or value type!"); 2598 Ops.append(Node->op_begin() + 1, Node->op_end()); 2599 } 2600 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), 2601 Node->getOperand(0)); 2602 } 2603 2604 /// N is a shift by a value that needs to be expanded, 2605 /// and the shift amount is a constant 'Amt'. Expand the operation. 2606 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, 2607 SDValue &Lo, SDValue &Hi) { 2608 SDLoc DL(N); 2609 // Expand the incoming operand to be shifted, so that we have its parts 2610 SDValue InL, InH; 2611 GetExpandedInteger(N->getOperand(0), InL, InH); 2612 2613 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization 2614 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. 2615 if (!Amt) { 2616 Lo = InL; 2617 Hi = InH; 2618 return; 2619 } 2620 2621 EVT NVT = InL.getValueType(); 2622 unsigned VTBits = N->getValueType(0).getSizeInBits(); 2623 unsigned NVTBits = NVT.getSizeInBits(); 2624 EVT ShTy = N->getOperand(1).getValueType(); 2625 2626 if (N->getOpcode() == ISD::SHL) { 2627 if (Amt.uge(VTBits)) { 2628 Lo = Hi = DAG.getConstant(0, DL, NVT); 2629 } else if (Amt.ugt(NVTBits)) { 2630 Lo = DAG.getConstant(0, DL, NVT); 2631 Hi = DAG.getNode(ISD::SHL, DL, 2632 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2633 } else if (Amt == NVTBits) { 2634 Lo = DAG.getConstant(0, DL, NVT); 2635 Hi = InL; 2636 } else { 2637 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); 2638 Hi = DAG.getNode(ISD::OR, DL, NVT, 2639 DAG.getNode(ISD::SHL, DL, NVT, InH, 2640 DAG.getConstant(Amt, DL, ShTy)), 2641 DAG.getNode(ISD::SRL, DL, NVT, InL, 2642 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2643 } 2644 return; 2645 } 2646 2647 if (N->getOpcode() == ISD::SRL) { 2648 if (Amt.uge(VTBits)) { 2649 Lo = Hi = DAG.getConstant(0, DL, NVT); 2650 } else if (Amt.ugt(NVTBits)) { 2651 Lo = DAG.getNode(ISD::SRL, DL, 2652 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2653 Hi = DAG.getConstant(0, DL, NVT); 2654 } else if (Amt == NVTBits) { 2655 Lo = InH; 2656 Hi = DAG.getConstant(0, DL, NVT); 2657 } else { 2658 Lo = DAG.getNode(ISD::OR, DL, NVT, 2659 DAG.getNode(ISD::SRL, DL, NVT, InL, 2660 DAG.getConstant(Amt, DL, ShTy)), 2661 DAG.getNode(ISD::SHL, DL, NVT, InH, 2662 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2663 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2664 } 2665 return; 2666 } 2667 2668 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2669 if (Amt.uge(VTBits)) { 2670 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2671 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2672 } else if (Amt.ugt(NVTBits)) { 2673 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 2674 DAG.getConstant(Amt - NVTBits, DL, ShTy)); 2675 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2676 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2677 } else if (Amt == NVTBits) { 2678 Lo = InH; 2679 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 2680 DAG.getConstant(NVTBits - 1, DL, ShTy)); 2681 } else { 2682 Lo = DAG.getNode(ISD::OR, DL, NVT, 2683 DAG.getNode(ISD::SRL, DL, NVT, InL, 2684 DAG.getConstant(Amt, DL, ShTy)), 2685 DAG.getNode(ISD::SHL, DL, NVT, InH, 2686 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 2687 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 2688 } 2689 } 2690 2691 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 2692 /// this shift based on knowledge of the high bit of the shift amount. If we 2693 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual 2694 /// shift amount. 2695 bool DAGTypeLegalizer:: 2696 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2697 SDValue Amt = N->getOperand(1); 2698 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2699 EVT ShTy = Amt.getValueType(); 2700 unsigned ShBits = ShTy.getScalarSizeInBits(); 2701 unsigned NVTBits = NVT.getScalarSizeInBits(); 2702 assert(isPowerOf2_32(NVTBits) && 2703 "Expanded integer type size not a power of two!"); 2704 SDLoc dl(N); 2705 2706 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 2707 KnownBits Known = DAG.computeKnownBits(N->getOperand(1)); 2708 2709 // If we don't know anything about the high bits, exit. 2710 if (((Known.Zero|Known.One) & HighBitMask) == 0) 2711 return false; 2712 2713 // Get the incoming operand to be shifted. 2714 SDValue InL, InH; 2715 GetExpandedInteger(N->getOperand(0), InL, InH); 2716 2717 // If we know that any of the high bits of the shift amount are one, then we 2718 // can do this as a couple of simple shifts. 2719 if (Known.One.intersects(HighBitMask)) { 2720 // Mask out the high bit, which we know is set. 2721 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 2722 DAG.getConstant(~HighBitMask, dl, ShTy)); 2723 2724 switch (N->getOpcode()) { 2725 default: llvm_unreachable("Unknown shift"); 2726 case ISD::SHL: 2727 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. 2728 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 2729 return true; 2730 case ISD::SRL: 2731 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2732 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 2733 return true; 2734 case ISD::SRA: 2735 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 2736 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2737 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 2738 return true; 2739 } 2740 } 2741 2742 // If we know that all of the high bits of the shift amount are zero, then we 2743 // can do this as a couple of simple shifts. 2744 if (HighBitMask.isSubsetOf(Known.Zero)) { 2745 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 2746 // shift if x is zero. We can use XOR here because x is known to be smaller 2747 // than 32. 2748 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 2749 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2750 2751 unsigned Op1, Op2; 2752 switch (N->getOpcode()) { 2753 default: llvm_unreachable("Unknown shift"); 2754 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 2755 case ISD::SRL: 2756 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 2757 } 2758 2759 // When shifting right the arithmetic for Lo and Hi is swapped. 2760 if (N->getOpcode() != ISD::SHL) 2761 std::swap(InL, InH); 2762 2763 // Use a little trick to get the bits that move from Lo to Hi. First 2764 // shift by one bit. 2765 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); 2766 // Then compute the remaining shift with amount-1. 2767 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 2768 2769 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 2770 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 2771 2772 if (N->getOpcode() != ISD::SHL) 2773 std::swap(Hi, Lo); 2774 return true; 2775 } 2776 2777 return false; 2778 } 2779 2780 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 2781 /// of any size. 2782 bool DAGTypeLegalizer:: 2783 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 2784 SDValue Amt = N->getOperand(1); 2785 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2786 EVT ShTy = Amt.getValueType(); 2787 unsigned NVTBits = NVT.getSizeInBits(); 2788 assert(isPowerOf2_32(NVTBits) && 2789 "Expanded integer type size not a power of two!"); 2790 SDLoc dl(N); 2791 2792 // Get the incoming operand to be shifted. 2793 SDValue InL, InH; 2794 GetExpandedInteger(N->getOperand(0), InL, InH); 2795 2796 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); 2797 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 2798 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 2799 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2800 Amt, NVBitsNode, ISD::SETULT); 2801 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), 2802 Amt, DAG.getConstant(0, dl, ShTy), 2803 ISD::SETEQ); 2804 2805 SDValue LoS, HiS, LoL, HiL; 2806 switch (N->getOpcode()) { 2807 default: llvm_unreachable("Unknown shift"); 2808 case ISD::SHL: 2809 // Short: ShAmt < NVTBits 2810 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 2811 HiS = DAG.getNode(ISD::OR, dl, NVT, 2812 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 2813 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 2814 2815 // Long: ShAmt >= NVTBits 2816 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. 2817 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 2818 2819 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 2820 Hi = DAG.getSelect(dl, NVT, isZero, InH, 2821 DAG.getSelect(dl, NVT, isShort, HiS, HiL)); 2822 return true; 2823 case ISD::SRL: 2824 // Short: ShAmt < NVTBits 2825 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 2826 LoS = DAG.getNode(ISD::OR, dl, NVT, 2827 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2828 // FIXME: If Amt is zero, the following shift generates an undefined result 2829 // on some architectures. 2830 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2831 2832 // Long: ShAmt >= NVTBits 2833 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. 2834 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2835 2836 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2837 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2838 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2839 return true; 2840 case ISD::SRA: 2841 // Short: ShAmt < NVTBits 2842 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 2843 LoS = DAG.getNode(ISD::OR, dl, NVT, 2844 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 2845 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 2846 2847 // Long: ShAmt >= NVTBits 2848 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 2849 DAG.getConstant(NVTBits - 1, dl, ShTy)); 2850 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 2851 2852 Lo = DAG.getSelect(dl, NVT, isZero, InL, 2853 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 2854 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 2855 return true; 2856 } 2857 } 2858 2859 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { 2860 2861 switch (Op) { 2862 default: llvm_unreachable("invalid min/max opcode"); 2863 case ISD::SMAX: 2864 return std::make_pair(ISD::SETGT, ISD::UMAX); 2865 case ISD::UMAX: 2866 return std::make_pair(ISD::SETUGT, ISD::UMAX); 2867 case ISD::SMIN: 2868 return std::make_pair(ISD::SETLT, ISD::UMIN); 2869 case ISD::UMIN: 2870 return std::make_pair(ISD::SETULT, ISD::UMIN); 2871 } 2872 } 2873 2874 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, 2875 SDValue &Lo, SDValue &Hi) { 2876 SDLoc DL(N); 2877 ISD::NodeType LoOpc; 2878 ISD::CondCode CondC; 2879 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); 2880 2881 SDValue LHS = N->getOperand(0); 2882 SDValue RHS = N->getOperand(1); 2883 2884 // Expand the subcomponents. 2885 SDValue LHSL, LHSH, RHSL, RHSH; 2886 GetExpandedInteger(LHS, LHSL, LHSH); 2887 GetExpandedInteger(RHS, RHSL, RHSH); 2888 2889 // Value types 2890 EVT NVT = LHSL.getValueType(); 2891 EVT CCT = getSetCCResultType(NVT); 2892 2893 // If the upper halves are all sign bits, then we can perform the MINMAX on 2894 // the lower half and sign-extend the result to the upper half. 2895 unsigned NumHalfBits = NVT.getScalarSizeInBits(); 2896 if (DAG.ComputeNumSignBits(LHS) > NumHalfBits && 2897 DAG.ComputeNumSignBits(RHS) > NumHalfBits) { 2898 Lo = DAG.getNode(N->getOpcode(), DL, NVT, LHSL, RHSL); 2899 Hi = DAG.getNode(ISD::SRA, DL, NVT, Lo, 2900 DAG.getShiftAmountConstant(NumHalfBits - 1, NVT, DL)); 2901 return; 2902 } 2903 2904 // Hi part is always the same op 2905 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 2906 2907 // We need to know whether to select Lo part that corresponds to 'winning' 2908 // Hi part or if Hi parts are equal. 2909 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); 2910 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); 2911 2912 // Lo part corresponding to the 'winning' Hi part 2913 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); 2914 2915 // Recursed Lo part if Hi parts are equal, this uses unsigned version 2916 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); 2917 2918 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); 2919 } 2920 2921 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 2922 SDValue &Lo, SDValue &Hi) { 2923 SDLoc dl(N); 2924 // Expand the subcomponents. 2925 SDValue LHSL, LHSH, RHSL, RHSH; 2926 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2927 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2928 2929 EVT NVT = LHSL.getValueType(); 2930 SDValue LoOps[2] = { LHSL, RHSL }; 2931 SDValue HiOps[3] = { LHSH, RHSH }; 2932 2933 bool HasOpCarry = TLI.isOperationLegalOrCustom( 2934 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY, 2935 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2936 if (HasOpCarry) { 2937 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 2938 if (N->getOpcode() == ISD::ADD) { 2939 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 2940 HiOps[2] = Lo.getValue(1); 2941 Hi = DAG.computeKnownBits(HiOps[2]).isZero() 2942 ? DAG.getNode(ISD::UADDO, dl, VTList, ArrayRef(HiOps, 2)) 2943 : DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps); 2944 } else { 2945 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 2946 HiOps[2] = Lo.getValue(1); 2947 Hi = DAG.computeKnownBits(HiOps[2]).isZero() 2948 ? DAG.getNode(ISD::USUBO, dl, VTList, ArrayRef(HiOps, 2)) 2949 : DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps); 2950 } 2951 return; 2952 } 2953 2954 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 2955 // them. TODO: Teach operation legalization how to expand unsupported 2956 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 2957 // a carry of type MVT::Glue, but there doesn't seem to be any way to 2958 // generate a value of this type in the expanded code sequence. 2959 bool hasCarry = 2960 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 2961 ISD::ADDC : ISD::SUBC, 2962 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2963 2964 if (hasCarry) { 2965 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 2966 if (N->getOpcode() == ISD::ADD) { 2967 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 2968 HiOps[2] = Lo.getValue(1); 2969 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 2970 } else { 2971 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 2972 HiOps[2] = Lo.getValue(1); 2973 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 2974 } 2975 return; 2976 } 2977 2978 bool hasOVF = 2979 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 2980 ISD::UADDO : ISD::USUBO, 2981 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 2982 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); 2983 2984 if (hasOVF) { 2985 EVT OvfVT = getSetCCResultType(NVT); 2986 SDVTList VTList = DAG.getVTList(NVT, OvfVT); 2987 int RevOpc; 2988 if (N->getOpcode() == ISD::ADD) { 2989 RevOpc = ISD::SUB; 2990 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 2991 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2)); 2992 } else { 2993 RevOpc = ISD::ADD; 2994 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 2995 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2)); 2996 } 2997 SDValue OVF = Lo.getValue(1); 2998 2999 switch (BoolType) { 3000 case TargetLoweringBase::UndefinedBooleanContent: 3001 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); 3002 [[fallthrough]]; 3003 case TargetLoweringBase::ZeroOrOneBooleanContent: 3004 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); 3005 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); 3006 break; 3007 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 3008 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); 3009 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); 3010 } 3011 return; 3012 } 3013 3014 if (N->getOpcode() == ISD::ADD) { 3015 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 3016 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2)); 3017 SDValue Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 3018 ISD::SETULT); 3019 3020 SDValue Carry; 3021 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 3022 Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT); 3023 else 3024 Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 3025 DAG.getConstant(0, dl, NVT)); 3026 3027 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); 3028 } else { 3029 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 3030 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2)); 3031 SDValue Cmp = 3032 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 3033 LoOps[0], LoOps[1], ISD::SETULT); 3034 3035 SDValue Borrow; 3036 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 3037 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); 3038 else 3039 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 3040 DAG.getConstant(0, dl, NVT)); 3041 3042 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 3043 } 3044 } 3045 3046 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 3047 SDValue &Lo, SDValue &Hi) { 3048 // Expand the subcomponents. 3049 SDValue LHSL, LHSH, RHSL, RHSH; 3050 SDLoc dl(N); 3051 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3052 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3053 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 3054 SDValue LoOps[2] = { LHSL, RHSL }; 3055 SDValue HiOps[3] = { LHSH, RHSH }; 3056 3057 if (N->getOpcode() == ISD::ADDC) { 3058 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 3059 HiOps[2] = Lo.getValue(1); 3060 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 3061 } else { 3062 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 3063 HiOps[2] = Lo.getValue(1); 3064 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 3065 } 3066 3067 // Legalized the flag result - switch anything that used the old flag to 3068 // use the new one. 3069 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3070 } 3071 3072 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 3073 SDValue &Lo, SDValue &Hi) { 3074 // Expand the subcomponents. 3075 SDValue LHSL, LHSH, RHSL, RHSH; 3076 SDLoc dl(N); 3077 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3078 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3079 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 3080 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 3081 SDValue HiOps[3] = { LHSH, RHSH }; 3082 3083 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3084 HiOps[2] = Lo.getValue(1); 3085 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 3086 3087 // Legalized the flag result - switch anything that used the old flag to 3088 // use the new one. 3089 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3090 } 3091 3092 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 3093 SDValue &Lo, SDValue &Hi) { 3094 SDValue LHS = N->getOperand(0); 3095 SDValue RHS = N->getOperand(1); 3096 SDLoc dl(N); 3097 3098 SDValue Ovf; 3099 3100 unsigned CarryOp, NoCarryOp; 3101 ISD::CondCode Cond; 3102 switch(N->getOpcode()) { 3103 case ISD::UADDO: 3104 CarryOp = ISD::ADDCARRY; 3105 NoCarryOp = ISD::ADD; 3106 Cond = ISD::SETULT; 3107 break; 3108 case ISD::USUBO: 3109 CarryOp = ISD::SUBCARRY; 3110 NoCarryOp = ISD::SUB; 3111 Cond = ISD::SETUGT; 3112 break; 3113 default: 3114 llvm_unreachable("Node has unexpected Opcode"); 3115 } 3116 3117 bool HasCarryOp = TLI.isOperationLegalOrCustom( 3118 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 3119 3120 if (HasCarryOp) { 3121 // Expand the subcomponents. 3122 SDValue LHSL, LHSH, RHSL, RHSH; 3123 GetExpandedInteger(LHS, LHSL, LHSH); 3124 GetExpandedInteger(RHS, RHSL, RHSH); 3125 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3126 SDValue LoOps[2] = { LHSL, RHSL }; 3127 SDValue HiOps[3] = { LHSH, RHSH }; 3128 3129 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3130 HiOps[2] = Lo.getValue(1); 3131 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps); 3132 3133 Ovf = Hi.getValue(1); 3134 } else { 3135 // Expand the result by simply replacing it with the equivalent 3136 // non-overflow-checking operation. 3137 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS); 3138 SplitInteger(Sum, Lo, Hi); 3139 3140 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 3141 // overflows iff a - b > a. 3142 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); 3143 } 3144 3145 // Legalized the flag result - switch anything that used the old flag to 3146 // use the new one. 3147 ReplaceValueWith(SDValue(N, 1), Ovf); 3148 } 3149 3150 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N, 3151 SDValue &Lo, SDValue &Hi) { 3152 // Expand the subcomponents. 3153 SDValue LHSL, LHSH, RHSL, RHSH; 3154 SDLoc dl(N); 3155 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3156 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3157 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3158 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 3159 SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; 3160 3161 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 3162 HiOps[2] = Lo.getValue(1); 3163 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 3164 3165 // Legalized the flag result - switch anything that used the old flag to 3166 // use the new one. 3167 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3168 } 3169 3170 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, 3171 SDValue &Lo, SDValue &Hi) { 3172 // Expand the subcomponents. 3173 SDValue LHSL, LHSH, RHSL, RHSH; 3174 SDLoc dl(N); 3175 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 3176 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 3177 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 3178 3179 // We need to use an unsigned carry op for the lo part. 3180 unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY 3181 : ISD::SUBCARRY; 3182 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) }); 3183 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 3184 3185 // Legalized the flag result - switch anything that used the old flag to 3186 // use the new one. 3187 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 3188 } 3189 3190 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 3191 SDValue &Lo, SDValue &Hi) { 3192 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3193 SDLoc dl(N); 3194 SDValue Op = N->getOperand(0); 3195 if (Op.getValueType().bitsLE(NVT)) { 3196 // The low part is any extension of the input (which degenerates to a copy). 3197 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 3198 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 3199 } else { 3200 // For example, extension of an i48 to an i64. The operand type necessarily 3201 // promotes to the result type, so will end up being expanded too. 3202 assert(getTypeAction(Op.getValueType()) == 3203 TargetLowering::TypePromoteInteger && 3204 "Only know how to promote this result!"); 3205 SDValue Res = GetPromotedInteger(Op); 3206 assert(Res.getValueType() == N->getValueType(0) && 3207 "Operand over promoted?"); 3208 // Split the promoted operand. This will simplify when it is expanded. 3209 SplitInteger(Res, Lo, Hi); 3210 } 3211 } 3212 3213 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 3214 SDValue &Lo, SDValue &Hi) { 3215 SDLoc dl(N); 3216 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3217 EVT NVT = Lo.getValueType(); 3218 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3219 unsigned NVTBits = NVT.getSizeInBits(); 3220 unsigned EVTBits = EVT.getSizeInBits(); 3221 3222 if (NVTBits < EVTBits) { 3223 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 3224 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3225 EVTBits - NVTBits))); 3226 } else { 3227 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 3228 // The high part replicates the sign bit of Lo, make it explicit. 3229 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3230 DAG.getConstant(NVTBits - 1, dl, 3231 TLI.getPointerTy(DAG.getDataLayout()))); 3232 } 3233 } 3234 3235 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 3236 SDValue &Lo, SDValue &Hi) { 3237 SDLoc dl(N); 3238 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3239 EVT NVT = Lo.getValueType(); 3240 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 3241 unsigned NVTBits = NVT.getSizeInBits(); 3242 unsigned EVTBits = EVT.getSizeInBits(); 3243 3244 if (NVTBits < EVTBits) { 3245 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 3246 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 3247 EVTBits - NVTBits))); 3248 } else { 3249 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 3250 // The high part must be zero, make it explicit. 3251 Hi = DAG.getConstant(0, dl, NVT); 3252 } 3253 } 3254 3255 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, 3256 SDValue &Lo, SDValue &Hi) { 3257 SDLoc dl(N); 3258 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 3259 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); 3260 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); 3261 } 3262 3263 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 3264 SDValue &Lo, SDValue &Hi) { 3265 SDLoc dl(N); 3266 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 3267 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 3268 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 3269 } 3270 3271 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, 3272 SDValue &Hi) { 3273 SDLoc dl(N); 3274 // parity(HiLo) -> parity(Lo^Hi) 3275 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3276 EVT NVT = Lo.getValueType(); 3277 Lo = 3278 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi)); 3279 Hi = DAG.getConstant(0, dl, NVT); 3280 } 3281 3282 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 3283 SDValue &Lo, SDValue &Hi) { 3284 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3285 unsigned NBitWidth = NVT.getSizeInBits(); 3286 auto Constant = cast<ConstantSDNode>(N); 3287 const APInt &Cst = Constant->getAPIntValue(); 3288 bool IsTarget = Constant->isTargetOpcode(); 3289 bool IsOpaque = Constant->isOpaque(); 3290 SDLoc dl(N); 3291 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); 3292 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, 3293 IsOpaque); 3294 } 3295 3296 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { 3297 SDLoc dl(N); 3298 3299 SDValue N0 = N->getOperand(0); 3300 GetExpandedInteger(N0, Lo, Hi); 3301 EVT NVT = Lo.getValueType(); 3302 3303 // If the upper half is all sign bits, then we can perform the ABS on the 3304 // lower half and zero-extend. 3305 if (DAG.ComputeNumSignBits(N0) > NVT.getScalarSizeInBits()) { 3306 Lo = DAG.getNode(ISD::ABS, dl, NVT, Lo); 3307 Hi = DAG.getConstant(0, dl, NVT); 3308 return; 3309 } 3310 3311 // If we have SUBCARRY, use the expanded form of the sra+xor+sub sequence we 3312 // use in LegalizeDAG. The SUB part of the expansion is based on 3313 // ExpandIntRes_ADDSUB which also uses SUBCARRY/USUBO after checking that 3314 // SUBCARRY is LegalOrCustom. Each of the pieces here can be further expanded 3315 // if needed. Shift expansion has a special case for filling with sign bits 3316 // so that we will only end up with one SRA. 3317 bool HasSubCarry = TLI.isOperationLegalOrCustom( 3318 ISD::SUBCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 3319 if (HasSubCarry) { 3320 SDValue Sign = DAG.getNode( 3321 ISD::SRA, dl, NVT, Hi, 3322 DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl)); 3323 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 3324 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign); 3325 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign); 3326 Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign); 3327 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, Hi, Sign, Lo.getValue(1)); 3328 return; 3329 } 3330 3331 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) 3332 EVT VT = N->getValueType(0); 3333 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, 3334 DAG.getConstant(0, dl, VT), N0); 3335 SDValue NegLo, NegHi; 3336 SplitInteger(Neg, NegLo, NegHi); 3337 3338 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 3339 DAG.getConstant(0, dl, NVT), ISD::SETLT); 3340 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); 3341 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); 3342 } 3343 3344 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 3345 SDValue &Lo, SDValue &Hi) { 3346 SDLoc dl(N); 3347 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 3348 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3349 EVT NVT = Lo.getValueType(); 3350 3351 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 3352 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3353 3354 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 3355 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 3356 3357 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 3358 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 3359 DAG.getConstant(NVT.getSizeInBits(), dl, 3360 NVT))); 3361 Hi = DAG.getConstant(0, dl, NVT); 3362 } 3363 3364 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 3365 SDValue &Lo, SDValue &Hi) { 3366 SDLoc dl(N); 3367 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 3368 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3369 EVT NVT = Lo.getValueType(); 3370 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 3371 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 3372 Hi = DAG.getConstant(0, dl, NVT); 3373 } 3374 3375 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 3376 SDValue &Lo, SDValue &Hi) { 3377 SDLoc dl(N); 3378 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 3379 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3380 EVT NVT = Lo.getValueType(); 3381 3382 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 3383 DAG.getConstant(0, dl, NVT), ISD::SETNE); 3384 3385 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 3386 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 3387 3388 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 3389 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 3390 DAG.getConstant(NVT.getSizeInBits(), dl, 3391 NVT))); 3392 Hi = DAG.getConstant(0, dl, NVT); 3393 } 3394 3395 void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo, 3396 SDValue &Hi) { 3397 SDLoc dl(N); 3398 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3399 unsigned NBitWidth = NVT.getSizeInBits(); 3400 3401 Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0)); 3402 SDValue Chain = Lo.getValue(1); 3403 // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING 3404 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3405 DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl)); 3406 3407 // Legalize the chain result - switch anything that used the old chain to 3408 // use the new one. 3409 ReplaceValueWith(SDValue(N, 1), Chain); 3410 } 3411 3412 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 3413 SDValue &Hi) { 3414 SDLoc dl(N); 3415 EVT VT = N->getValueType(0); 3416 3417 bool IsStrict = N->isStrictFPOpcode(); 3418 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3419 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3420 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3421 Op = GetPromotedFloat(Op); 3422 3423 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3424 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3425 Op = GetSoftPromotedHalf(Op); 3426 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); 3427 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op); 3428 SplitInteger(Op, Lo, Hi); 3429 return; 3430 } 3431 3432 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 3433 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 3434 TargetLowering::MakeLibCallOptions CallOptions; 3435 CallOptions.setSExt(true); 3436 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3437 CallOptions, dl, Chain); 3438 SplitInteger(Tmp.first, Lo, Hi); 3439 3440 if (IsStrict) 3441 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3442 } 3443 3444 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 3445 SDValue &Hi) { 3446 SDLoc dl(N); 3447 EVT VT = N->getValueType(0); 3448 3449 bool IsStrict = N->isStrictFPOpcode(); 3450 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3451 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3452 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 3453 Op = GetPromotedFloat(Op); 3454 3455 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { 3456 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3457 Op = GetSoftPromotedHalf(Op); 3458 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); 3459 Op = DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op); 3460 SplitInteger(Op, Lo, Hi); 3461 return; 3462 } 3463 3464 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 3465 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 3466 TargetLowering::MakeLibCallOptions CallOptions; 3467 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, 3468 CallOptions, dl, Chain); 3469 SplitInteger(Tmp.first, Lo, Hi); 3470 3471 if (IsStrict) 3472 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3473 } 3474 3475 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, 3476 SDValue &Hi) { 3477 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 3478 SplitInteger(Res, Lo, Hi); 3479 } 3480 3481 void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo, 3482 SDValue &Hi) { 3483 SDLoc dl(N); 3484 bool IsStrict = N->isStrictFPOpcode(); 3485 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3486 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3487 3488 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && 3489 "Input type needs to be promoted!"); 3490 3491 EVT VT = Op.getValueType(); 3492 3493 if (VT == MVT::f16) { 3494 VT = MVT::f32; 3495 // Extend to f32. 3496 if (IsStrict) { 3497 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { VT, MVT::Other }, {Chain, Op}); 3498 Chain = Op.getValue(1); 3499 } else { 3500 Op = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op); 3501 } 3502 } 3503 3504 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3505 if (N->getOpcode() == ISD::LROUND || 3506 N->getOpcode() == ISD::STRICT_LROUND) { 3507 if (VT == MVT::f32) 3508 LC = RTLIB::LROUND_F32; 3509 else if (VT == MVT::f64) 3510 LC = RTLIB::LROUND_F64; 3511 else if (VT == MVT::f80) 3512 LC = RTLIB::LROUND_F80; 3513 else if (VT == MVT::f128) 3514 LC = RTLIB::LROUND_F128; 3515 else if (VT == MVT::ppcf128) 3516 LC = RTLIB::LROUND_PPCF128; 3517 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!"); 3518 } else if (N->getOpcode() == ISD::LRINT || 3519 N->getOpcode() == ISD::STRICT_LRINT) { 3520 if (VT == MVT::f32) 3521 LC = RTLIB::LRINT_F32; 3522 else if (VT == MVT::f64) 3523 LC = RTLIB::LRINT_F64; 3524 else if (VT == MVT::f80) 3525 LC = RTLIB::LRINT_F80; 3526 else if (VT == MVT::f128) 3527 LC = RTLIB::LRINT_F128; 3528 else if (VT == MVT::ppcf128) 3529 LC = RTLIB::LRINT_PPCF128; 3530 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!"); 3531 } else if (N->getOpcode() == ISD::LLROUND || 3532 N->getOpcode() == ISD::STRICT_LLROUND) { 3533 if (VT == MVT::f32) 3534 LC = RTLIB::LLROUND_F32; 3535 else if (VT == MVT::f64) 3536 LC = RTLIB::LLROUND_F64; 3537 else if (VT == MVT::f80) 3538 LC = RTLIB::LLROUND_F80; 3539 else if (VT == MVT::f128) 3540 LC = RTLIB::LLROUND_F128; 3541 else if (VT == MVT::ppcf128) 3542 LC = RTLIB::LLROUND_PPCF128; 3543 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); 3544 } else if (N->getOpcode() == ISD::LLRINT || 3545 N->getOpcode() == ISD::STRICT_LLRINT) { 3546 if (VT == MVT::f32) 3547 LC = RTLIB::LLRINT_F32; 3548 else if (VT == MVT::f64) 3549 LC = RTLIB::LLRINT_F64; 3550 else if (VT == MVT::f80) 3551 LC = RTLIB::LLRINT_F80; 3552 else if (VT == MVT::f128) 3553 LC = RTLIB::LLRINT_F128; 3554 else if (VT == MVT::ppcf128) 3555 LC = RTLIB::LLRINT_PPCF128; 3556 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); 3557 } else 3558 llvm_unreachable("Unexpected opcode!"); 3559 3560 EVT RetVT = N->getValueType(0); 3561 3562 TargetLowering::MakeLibCallOptions CallOptions; 3563 CallOptions.setSExt(true); 3564 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, 3565 Op, CallOptions, dl, 3566 Chain); 3567 SplitInteger(Tmp.first, Lo, Hi); 3568 3569 if (N->isStrictFPOpcode()) 3570 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3571 } 3572 3573 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 3574 SDValue &Lo, SDValue &Hi) { 3575 if (N->isAtomic()) { 3576 // It's typical to have larger CAS than atomic load instructions. 3577 SDLoc dl(N); 3578 EVT VT = N->getMemoryVT(); 3579 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 3580 SDValue Zero = DAG.getConstant(0, dl, VT); 3581 SDValue Swap = DAG.getAtomicCmpSwap( 3582 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 3583 VT, VTs, N->getOperand(0), 3584 N->getOperand(1), Zero, Zero, N->getMemOperand()); 3585 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 3586 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 3587 return; 3588 } 3589 3590 if (ISD::isNormalLoad(N)) { 3591 ExpandRes_NormalLoad(N, Lo, Hi); 3592 return; 3593 } 3594 3595 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 3596 3597 EVT VT = N->getValueType(0); 3598 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3599 SDValue Ch = N->getChain(); 3600 SDValue Ptr = N->getBasePtr(); 3601 ISD::LoadExtType ExtType = N->getExtensionType(); 3602 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 3603 AAMDNodes AAInfo = N->getAAInfo(); 3604 SDLoc dl(N); 3605 3606 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 3607 3608 if (N->getMemoryVT().bitsLE(NVT)) { 3609 EVT MemVT = N->getMemoryVT(); 3610 3611 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, 3612 N->getOriginalAlign(), MMOFlags, AAInfo); 3613 3614 // Remember the chain. 3615 Ch = Lo.getValue(1); 3616 3617 if (ExtType == ISD::SEXTLOAD) { 3618 // The high part is obtained by SRA'ing all but one of the bits of the 3619 // lo part. 3620 unsigned LoSize = Lo.getValueSizeInBits(); 3621 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 3622 DAG.getConstant(LoSize - 1, dl, 3623 TLI.getPointerTy(DAG.getDataLayout()))); 3624 } else if (ExtType == ISD::ZEXTLOAD) { 3625 // The high part is just a zero. 3626 Hi = DAG.getConstant(0, dl, NVT); 3627 } else { 3628 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 3629 // The high part is undefined. 3630 Hi = DAG.getUNDEF(NVT); 3631 } 3632 } else if (DAG.getDataLayout().isLittleEndian()) { 3633 // Little-endian - low bits are at low addresses. 3634 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 3635 N->getOriginalAlign(), MMOFlags, AAInfo); 3636 3637 unsigned ExcessBits = 3638 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 3639 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 3640 3641 // Increment the pointer to the other half. 3642 unsigned IncrementSize = NVT.getSizeInBits()/8; 3643 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 3644 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 3645 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 3646 N->getOriginalAlign(), MMOFlags, AAInfo); 3647 3648 // Build a factor node to remember that this load is independent of the 3649 // other one. 3650 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3651 Hi.getValue(1)); 3652 } else { 3653 // Big-endian - high bits are at low addresses. Favor aligned loads at 3654 // the cost of some bit-fiddling. 3655 EVT MemVT = N->getMemoryVT(); 3656 unsigned EBytes = MemVT.getStoreSize(); 3657 unsigned IncrementSize = NVT.getSizeInBits()/8; 3658 unsigned ExcessBits = (EBytes - IncrementSize)*8; 3659 3660 // Load both the high bits and maybe some of the low bits. 3661 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 3662 EVT::getIntegerVT(*DAG.getContext(), 3663 MemVT.getSizeInBits() - ExcessBits), 3664 N->getOriginalAlign(), MMOFlags, AAInfo); 3665 3666 // Increment the pointer to the other half. 3667 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 3668 // Load the rest of the low bits. 3669 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 3670 N->getPointerInfo().getWithOffset(IncrementSize), 3671 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 3672 N->getOriginalAlign(), MMOFlags, AAInfo); 3673 3674 // Build a factor node to remember that this load is independent of the 3675 // other one. 3676 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3677 Hi.getValue(1)); 3678 3679 if (ExcessBits < NVT.getSizeInBits()) { 3680 // Transfer low bits from the bottom of Hi to the top of Lo. 3681 Lo = DAG.getNode( 3682 ISD::OR, dl, NVT, Lo, 3683 DAG.getNode(ISD::SHL, dl, NVT, Hi, 3684 DAG.getConstant(ExcessBits, dl, 3685 TLI.getPointerTy(DAG.getDataLayout())))); 3686 // Move high bits to the right position in Hi. 3687 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, 3688 Hi, 3689 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 3690 TLI.getPointerTy(DAG.getDataLayout()))); 3691 } 3692 } 3693 3694 // Legalize the chain result - switch anything that used the old chain to 3695 // use the new one. 3696 ReplaceValueWith(SDValue(N, 1), Ch); 3697 } 3698 3699 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 3700 SDValue &Lo, SDValue &Hi) { 3701 SDLoc dl(N); 3702 SDValue LL, LH, RL, RH; 3703 GetExpandedInteger(N->getOperand(0), LL, LH); 3704 GetExpandedInteger(N->getOperand(1), RL, RH); 3705 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 3706 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 3707 } 3708 3709 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 3710 SDValue &Lo, SDValue &Hi) { 3711 EVT VT = N->getValueType(0); 3712 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3713 SDLoc dl(N); 3714 3715 SDValue LL, LH, RL, RH; 3716 GetExpandedInteger(N->getOperand(0), LL, LH); 3717 GetExpandedInteger(N->getOperand(1), RL, RH); 3718 3719 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, 3720 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3721 LL, LH, RL, RH)) 3722 return; 3723 3724 // If nothing else, we can make a libcall. 3725 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3726 if (VT == MVT::i16) 3727 LC = RTLIB::MUL_I16; 3728 else if (VT == MVT::i32) 3729 LC = RTLIB::MUL_I32; 3730 else if (VT == MVT::i64) 3731 LC = RTLIB::MUL_I64; 3732 else if (VT == MVT::i128) 3733 LC = RTLIB::MUL_I128; 3734 3735 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 3736 // We'll expand the multiplication by brute force because we have no other 3737 // options. This is a trivially-generalized version of the code from 3738 // Hacker's Delight (itself derived from Knuth's Algorithm M from section 3739 // 4.3.1). 3740 unsigned Bits = NVT.getSizeInBits(); 3741 unsigned HalfBits = Bits >> 1; 3742 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, 3743 NVT); 3744 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); 3745 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); 3746 3747 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); 3748 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); 3749 3750 SDValue Shift = DAG.getShiftAmountConstant(HalfBits, NVT, dl); 3751 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); 3752 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); 3753 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); 3754 3755 SDValue U = DAG.getNode(ISD::ADD, dl, NVT, 3756 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); 3757 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); 3758 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); 3759 3760 SDValue V = DAG.getNode(ISD::ADD, dl, NVT, 3761 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); 3762 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); 3763 3764 SDValue W = DAG.getNode(ISD::ADD, dl, NVT, 3765 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), 3766 DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); 3767 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, 3768 DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); 3769 3770 Hi = DAG.getNode(ISD::ADD, dl, NVT, W, 3771 DAG.getNode(ISD::ADD, dl, NVT, 3772 DAG.getNode(ISD::MUL, dl, NVT, RH, LL), 3773 DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); 3774 return; 3775 } 3776 3777 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 3778 TargetLowering::MakeLibCallOptions CallOptions; 3779 CallOptions.setSExt(true); 3780 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, 3781 Lo, Hi); 3782 } 3783 3784 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, 3785 SDValue &Hi) { 3786 SDLoc DL(N); 3787 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3788 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); 3789 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); 3790 Lo = R.getValue(0); 3791 Hi = R.getValue(1); 3792 ReplaceValueWith(SDValue(N, 1), R.getValue(2)); 3793 } 3794 3795 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, 3796 SDValue &Hi) { 3797 SDValue Result = TLI.expandAddSubSat(N, DAG); 3798 SplitInteger(Result, Lo, Hi); 3799 } 3800 3801 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, 3802 SDValue &Hi) { 3803 SDValue Result = TLI.expandShlSat(N, DAG); 3804 SplitInteger(Result, Lo, Hi); 3805 } 3806 3807 /// This performs an expansion of the integer result for a fixed point 3808 /// multiplication. The default expansion performs rounding down towards 3809 /// negative infinity, though targets that do care about rounding should specify 3810 /// a target hook for rounding and provide their own expansion or lowering of 3811 /// fixed point multiplication to be consistent with rounding. 3812 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, 3813 SDValue &Hi) { 3814 SDLoc dl(N); 3815 EVT VT = N->getValueType(0); 3816 unsigned VTSize = VT.getScalarSizeInBits(); 3817 SDValue LHS = N->getOperand(0); 3818 SDValue RHS = N->getOperand(1); 3819 uint64_t Scale = N->getConstantOperandVal(2); 3820 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || 3821 N->getOpcode() == ISD::UMULFIXSAT); 3822 bool Signed = (N->getOpcode() == ISD::SMULFIX || 3823 N->getOpcode() == ISD::SMULFIXSAT); 3824 3825 // Handle special case when scale is equal to zero. 3826 if (!Scale) { 3827 SDValue Result; 3828 if (!Saturating) { 3829 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 3830 } else { 3831 EVT BoolVT = getSetCCResultType(VT); 3832 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; 3833 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); 3834 SDValue Product = Result.getValue(0); 3835 SDValue Overflow = Result.getValue(1); 3836 if (Signed) { 3837 APInt MinVal = APInt::getSignedMinValue(VTSize); 3838 APInt MaxVal = APInt::getSignedMaxValue(VTSize); 3839 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); 3840 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 3841 SDValue Zero = DAG.getConstant(0, dl, VT); 3842 // Xor the inputs, if resulting sign bit is 0 the product will be 3843 // positive, else negative. 3844 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); 3845 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); 3846 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); 3847 Result = DAG.getSelect(dl, VT, Overflow, Result, Product); 3848 } else { 3849 // For unsigned multiplication, we only need to check the max since we 3850 // can't really overflow towards zero. 3851 APInt MaxVal = APInt::getMaxValue(VTSize); 3852 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); 3853 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product); 3854 } 3855 } 3856 SplitInteger(Result, Lo, Hi); 3857 return; 3858 } 3859 3860 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will 3861 // cover for unhandled cases below, while still being valid for UMULFIX[SAT]. 3862 assert(Scale <= VTSize && "Scale can't be larger than the value type size."); 3863 3864 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3865 SDValue LL, LH, RL, RH; 3866 GetExpandedInteger(LHS, LL, LH); 3867 GetExpandedInteger(RHS, RL, RH); 3868 SmallVector<SDValue, 4> Result; 3869 3870 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; 3871 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, 3872 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 3873 LL, LH, RL, RH)) { 3874 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); 3875 return; 3876 } 3877 3878 unsigned NVTSize = NVT.getScalarSizeInBits(); 3879 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " 3880 "the size of the current value type"); 3881 3882 // After getting the multiplication result in 4 parts, we need to perform a 3883 // shift right by the amount of the scale to get the result in that scale. 3884 // 3885 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in 3886 // 128 bits that are cut into 4 32-bit parts: 3887 // 3888 // HH HL LH LL 3889 // |---32---|---32---|---32---|---32---| 3890 // 128 96 64 32 0 3891 // 3892 // |------VTSize-----| 3893 // 3894 // |NVTSize-| 3895 // 3896 // The resulting Lo and Hi would normally be in LL and LH after the shift. But 3897 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift 3898 // amount and get Lo and Hi using two funnel shifts. Or for the special case 3899 // when Scale is a multiple of NVTSize we can just pick the result without 3900 // shifting. 3901 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. 3902 if (Scale % NVTSize) { 3903 SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl); 3904 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0], 3905 ShiftAmount); 3906 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1], 3907 ShiftAmount); 3908 } else { 3909 Lo = Result[Part0]; 3910 Hi = Result[Part0 + 1]; 3911 } 3912 3913 // Unless saturation is requested we are done. The result is in <Hi,Lo>. 3914 if (!Saturating) 3915 return; 3916 3917 // Can not overflow when there is no integer part. 3918 if (Scale == VTSize) 3919 return; 3920 3921 // To handle saturation we must check for overflow in the multiplication. 3922 // 3923 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) 3924 // aren't all zeroes. 3925 // 3926 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) 3927 // aren't all ones or all zeroes. 3928 // 3929 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the 3930 // highest bit of HH determines saturation direction in the event of signed 3931 // saturation. 3932 3933 SDValue ResultHL = Result[2]; 3934 SDValue ResultHH = Result[3]; 3935 3936 SDValue SatMax, SatMin; 3937 SDValue NVTZero = DAG.getConstant(0, dl, NVT); 3938 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); 3939 EVT BoolNVT = getSetCCResultType(NVT); 3940 3941 if (!Signed) { 3942 if (Scale < NVTSize) { 3943 // Overflow happened if ((HH | (HL >> Scale)) != 0). 3944 SDValue HLAdjusted = 3945 DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 3946 DAG.getShiftAmountConstant(Scale, NVT, dl)); 3947 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH); 3948 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE); 3949 } else if (Scale == NVTSize) { 3950 // Overflow happened if (HH != 0). 3951 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE); 3952 } else if (Scale < VTSize) { 3953 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0). 3954 SDValue HLAdjusted = 3955 DAG.getNode(ISD::SRL, dl, NVT, ResultHL, 3956 DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl)); 3957 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE); 3958 } else 3959 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" 3960 "(and saturation can't happen with Scale==VTSize)."); 3961 3962 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi); 3963 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo); 3964 return; 3965 } 3966 3967 if (Scale < NVTSize) { 3968 // The number of overflow bits we can check are VTSize - Scale + 1 (we 3969 // include the sign bit). If these top bits are > 0, then we overflowed past 3970 // the max value. If these top bits are < -1, then we overflowed past the 3971 // min value. Otherwise, we did not overflow. 3972 unsigned OverflowBits = VTSize - Scale + 1; 3973 assert(OverflowBits <= VTSize && OverflowBits > NVTSize && 3974 "Extent of overflow bits must start within HL"); 3975 SDValue HLHiMask = DAG.getConstant( 3976 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); 3977 SDValue HLLoMask = DAG.getConstant( 3978 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); 3979 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). 3980 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 3981 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 3982 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); 3983 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 3984 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT)); 3985 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). 3986 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 3987 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 3988 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); 3989 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 3990 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT)); 3991 } else if (Scale == NVTSize) { 3992 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). 3993 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); 3994 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); 3995 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); 3996 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, 3997 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg)); 3998 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). 3999 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); 4000 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); 4001 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); 4002 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, 4003 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos)); 4004 } else if (Scale < VTSize) { 4005 // This is similar to the case when we saturate if Scale < NVTSize, but we 4006 // only need to check HH. 4007 unsigned OverflowBits = VTSize - Scale + 1; 4008 SDValue HHHiMask = DAG.getConstant( 4009 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); 4010 SDValue HHLoMask = DAG.getConstant( 4011 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); 4012 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); 4013 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); 4014 } else 4015 llvm_unreachable("Illegal scale for signed fixed point mul."); 4016 4017 // Saturate to signed maximum. 4018 APInt MaxHi = APInt::getSignedMaxValue(NVTSize); 4019 APInt MaxLo = APInt::getAllOnes(NVTSize); 4020 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi); 4021 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo); 4022 // Saturate to signed minimum. 4023 APInt MinHi = APInt::getSignedMinValue(NVTSize); 4024 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi); 4025 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); 4026 } 4027 4028 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, 4029 SDValue &Hi) { 4030 SDLoc dl(N); 4031 // Try expanding in the existing type first. 4032 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0), 4033 N->getOperand(1), 4034 N->getConstantOperandVal(2), DAG); 4035 4036 if (!Res) 4037 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1), 4038 N->getConstantOperandVal(2), TLI, DAG); 4039 SplitInteger(Res, Lo, Hi); 4040 } 4041 4042 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 4043 SDValue &Lo, SDValue &Hi) { 4044 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && 4045 "Node has unexpected Opcode"); 4046 SDValue LHS = Node->getOperand(0); 4047 SDValue RHS = Node->getOperand(1); 4048 SDLoc dl(Node); 4049 4050 SDValue Ovf; 4051 4052 bool IsAdd = Node->getOpcode() == ISD::SADDO; 4053 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY; 4054 4055 bool HasCarryOp = TLI.isOperationLegalOrCustom( 4056 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 4057 4058 if (HasCarryOp) { 4059 // Expand the subcomponents. 4060 SDValue LHSL, LHSH, RHSL, RHSH; 4061 GetExpandedInteger(LHS, LHSL, LHSH); 4062 GetExpandedInteger(RHS, RHSL, RHSH); 4063 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1)); 4064 4065 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL}); 4066 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); 4067 4068 Ovf = Hi.getValue(1); 4069 } else { 4070 // Expand the result by simply replacing it with the equivalent 4071 // non-overflow-checking operation. 4072 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 4073 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 4074 LHS, RHS); 4075 SplitInteger(Sum, Lo, Hi); 4076 4077 // Compute the overflow. 4078 // 4079 // LHSSign -> LHS < 0 4080 // RHSSign -> RHS < 0 4081 // SumSign -> Sum < 0 4082 // 4083 // Add: 4084 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 4085 // Sub: 4086 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 4087 // 4088 // To get better codegen we can rewrite this by doing bitwise math on 4089 // the integers and extract the final sign bit at the end. So the 4090 // above becomes: 4091 // 4092 // Add: 4093 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0 4094 // Sub: 4095 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0 4096 // 4097 // NOTE: This is different than the expansion we do in expandSADDSUBO 4098 // because it is more costly to determine the RHS is > 0 for SSUBO with the 4099 // integers split. 4100 EVT VT = LHS.getValueType(); 4101 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); 4102 if (IsAdd) 4103 SignsMatch = DAG.getNOT(dl, SignsMatch, VT); 4104 4105 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum); 4106 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE); 4107 EVT OType = Node->getValueType(1); 4108 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT); 4109 } 4110 4111 // Use the calculated overflow everywhere. 4112 ReplaceValueWith(SDValue(Node, 1), Ovf); 4113 } 4114 4115 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 4116 SDValue &Lo, SDValue &Hi) { 4117 EVT VT = N->getValueType(0); 4118 SDLoc dl(N); 4119 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4120 4121 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 4122 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4123 SplitInteger(Res.getValue(0), Lo, Hi); 4124 return; 4125 } 4126 4127 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4128 if (VT == MVT::i16) 4129 LC = RTLIB::SDIV_I16; 4130 else if (VT == MVT::i32) 4131 LC = RTLIB::SDIV_I32; 4132 else if (VT == MVT::i64) 4133 LC = RTLIB::SDIV_I64; 4134 else if (VT == MVT::i128) 4135 LC = RTLIB::SDIV_I128; 4136 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 4137 4138 TargetLowering::MakeLibCallOptions CallOptions; 4139 CallOptions.setSExt(true); 4140 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4141 } 4142 4143 void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo, 4144 SDValue &Hi) { 4145 SDLoc dl(N); 4146 SDValue Shiftee = N->getOperand(0); 4147 EVT VT = Shiftee.getValueType(); 4148 SDValue ShAmt = N->getOperand(1); 4149 EVT ShAmtVT = ShAmt.getValueType(); 4150 4151 // This legalization is optimal when the shift is by a multiple of byte width, 4152 // %x * 8 <-> %x << 3 so 3 low bits should be be known zero. 4153 bool ShiftByByteMultiple = 4154 DAG.computeKnownBits(ShAmt).countMinTrailingZeros() >= 3; 4155 4156 // If we can't do it as one step, we'll have two uses of shift amount, 4157 // and thus must freeze it. 4158 if (!ShiftByByteMultiple) 4159 ShAmt = DAG.getFreeze(ShAmt); 4160 4161 unsigned VTBitWidth = VT.getScalarSizeInBits(); 4162 assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?"); 4163 unsigned VTByteWidth = VTBitWidth / 8; 4164 assert(isPowerOf2_32(VTByteWidth) && 4165 "Shiftee type size is not a power of two!"); 4166 unsigned StackSlotByteWidth = 2 * VTByteWidth; 4167 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth; 4168 EVT StackSlotVT = EVT::getIntegerVT(*DAG.getContext(), StackSlotBitWidth); 4169 4170 // Get a temporary stack slot 2x the width of our VT. 4171 // FIXME: reuse stack slots? 4172 // FIXME: should we be more picky about alignment? 4173 Align StackSlotAlignment(1); 4174 SDValue StackPtr = DAG.CreateStackTemporary( 4175 TypeSize::getFixed(StackSlotByteWidth), StackSlotAlignment); 4176 EVT PtrTy = StackPtr.getValueType(); 4177 SDValue Ch = DAG.getEntryNode(); 4178 4179 MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack( 4180 DAG.getMachineFunction(), 4181 cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex()); 4182 4183 // Extend the value, that is being shifted, to the entire stack slot's width. 4184 SDValue Init; 4185 if (N->getOpcode() != ISD::SHL) { 4186 unsigned WideningOpc = 4187 N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4188 Init = DAG.getNode(WideningOpc, dl, StackSlotVT, Shiftee); 4189 } else { 4190 // For left-shifts, pad the Shiftee's LSB with zeros to twice it's width. 4191 SDValue AllZeros = DAG.getConstant(0, dl, VT); 4192 Init = DAG.getNode(ISD::BUILD_PAIR, dl, StackSlotVT, AllZeros, Shiftee); 4193 } 4194 // And spill it into the stack slot. 4195 Ch = DAG.getStore(Ch, dl, Init, StackPtr, StackPtrInfo, StackSlotAlignment); 4196 4197 // Now, compute the full-byte offset into stack slot from where we can load. 4198 // We have shift amount, which is in bits, but in multiples of byte. 4199 // So just divide by CHAR_BIT. 4200 SDNodeFlags Flags; 4201 if (ShiftByByteMultiple) 4202 Flags.setExact(true); 4203 SDValue ByteOffset = DAG.getNode(ISD::SRL, dl, ShAmtVT, ShAmt, 4204 DAG.getConstant(3, dl, ShAmtVT), Flags); 4205 // And clamp it, because OOB load is an immediate UB, 4206 // while shift overflow would have *just* been poison. 4207 ByteOffset = DAG.getNode(ISD::AND, dl, ShAmtVT, ByteOffset, 4208 DAG.getConstant(VTByteWidth - 1, dl, ShAmtVT)); 4209 // We have exactly two strategies on indexing into stack slot here: 4210 // 1. upwards starting from the beginning of the slot 4211 // 2. downwards starting from the middle of the slot 4212 // On little-endian machine, we pick 1. for right shifts and 2. for left-shift 4213 // and vice versa on big-endian machine. 4214 bool WillIndexUpwards = N->getOpcode() != ISD::SHL; 4215 if (DAG.getDataLayout().isBigEndian()) 4216 WillIndexUpwards = !WillIndexUpwards; 4217 4218 SDValue AdjStackPtr; 4219 if (WillIndexUpwards) { 4220 AdjStackPtr = StackPtr; 4221 } else { 4222 AdjStackPtr = DAG.getMemBasePlusOffset( 4223 StackPtr, DAG.getConstant(VTByteWidth, dl, PtrTy), dl); 4224 ByteOffset = DAG.getNegative(ByteOffset, dl, ShAmtVT); 4225 } 4226 4227 // Get the pointer somewhere into the stack slot from which we need to load. 4228 ByteOffset = DAG.getSExtOrTrunc(ByteOffset, dl, PtrTy); 4229 AdjStackPtr = DAG.getMemBasePlusOffset(AdjStackPtr, ByteOffset, dl); 4230 4231 // And load it! While the load is not legal, legalizing it is obvious. 4232 SDValue Res = DAG.getLoad( 4233 VT, dl, Ch, AdjStackPtr, 4234 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), Align(1)); 4235 // We've performed the shift by a CHAR_BIT * [_ShAmt / CHAR_BIT_] 4236 4237 // If we may still have a less-than-CHAR_BIT to shift by, do so now. 4238 if (!ShiftByByteMultiple) { 4239 SDValue ShAmtRem = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, 4240 DAG.getConstant(7, dl, ShAmtVT)); 4241 Res = DAG.getNode(N->getOpcode(), dl, VT, Res, ShAmtRem); 4242 } 4243 4244 // Finally, split the computed value. 4245 SplitInteger(Res, Lo, Hi); 4246 } 4247 4248 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 4249 SDValue &Lo, SDValue &Hi) { 4250 EVT VT = N->getValueType(0); 4251 SDLoc dl(N); 4252 4253 // If we can emit an efficient shift operation, do so now. Check to see if 4254 // the RHS is a constant. 4255 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 4256 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); 4257 4258 // If we can determine that the high bit of the shift is zero or one, even if 4259 // the low bits are variable, emit this shift in an optimized form. 4260 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 4261 return; 4262 4263 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 4264 unsigned PartsOpc; 4265 if (N->getOpcode() == ISD::SHL) { 4266 PartsOpc = ISD::SHL_PARTS; 4267 } else if (N->getOpcode() == ISD::SRL) { 4268 PartsOpc = ISD::SRL_PARTS; 4269 } else { 4270 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 4271 PartsOpc = ISD::SRA_PARTS; 4272 } 4273 4274 // Next check to see if the target supports this SHL_PARTS operation or if it 4275 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for 4276 // size, but create a libcall instead. 4277 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4278 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 4279 const bool LegalOrCustom = 4280 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 4281 Action == TargetLowering::Custom; 4282 4283 unsigned ExpansionFactor = 1; 4284 // That VT->NVT expansion is one step. But will we re-expand NVT? 4285 for (EVT TmpVT = NVT;;) { 4286 EVT NewTMPVT = TLI.getTypeToTransformTo(*DAG.getContext(), TmpVT); 4287 if (NewTMPVT == TmpVT) 4288 break; 4289 TmpVT = NewTMPVT; 4290 ++ExpansionFactor; 4291 } 4292 4293 TargetLowering::ShiftLegalizationStrategy S = 4294 TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor); 4295 4296 if (S == TargetLowering::ShiftLegalizationStrategy::ExpandThroughStack) 4297 return ExpandIntRes_ShiftThroughStack(N, Lo, Hi); 4298 4299 if (LegalOrCustom && 4300 S != TargetLowering::ShiftLegalizationStrategy::LowerToLibcall) { 4301 // Expand the subcomponents. 4302 SDValue LHSL, LHSH; 4303 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 4304 EVT VT = LHSL.getValueType(); 4305 4306 // If the shift amount operand is coming from a vector legalization it may 4307 // have an illegal type. Fix that first by casting the operand, otherwise 4308 // the new SHL_PARTS operation would need further legalization. 4309 SDValue ShiftOp = N->getOperand(1); 4310 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 4311 if (ShiftOp.getValueType() != ShiftTy) 4312 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 4313 4314 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 4315 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 4316 Hi = Lo.getValue(1); 4317 return; 4318 } 4319 4320 // Otherwise, emit a libcall. 4321 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4322 bool isSigned; 4323 if (N->getOpcode() == ISD::SHL) { 4324 isSigned = false; /*sign irrelevant*/ 4325 if (VT == MVT::i16) 4326 LC = RTLIB::SHL_I16; 4327 else if (VT == MVT::i32) 4328 LC = RTLIB::SHL_I32; 4329 else if (VT == MVT::i64) 4330 LC = RTLIB::SHL_I64; 4331 else if (VT == MVT::i128) 4332 LC = RTLIB::SHL_I128; 4333 } else if (N->getOpcode() == ISD::SRL) { 4334 isSigned = false; 4335 if (VT == MVT::i16) 4336 LC = RTLIB::SRL_I16; 4337 else if (VT == MVT::i32) 4338 LC = RTLIB::SRL_I32; 4339 else if (VT == MVT::i64) 4340 LC = RTLIB::SRL_I64; 4341 else if (VT == MVT::i128) 4342 LC = RTLIB::SRL_I128; 4343 } else { 4344 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 4345 isSigned = true; 4346 if (VT == MVT::i16) 4347 LC = RTLIB::SRA_I16; 4348 else if (VT == MVT::i32) 4349 LC = RTLIB::SRA_I32; 4350 else if (VT == MVT::i64) 4351 LC = RTLIB::SRA_I64; 4352 else if (VT == MVT::i128) 4353 LC = RTLIB::SRA_I128; 4354 } 4355 4356 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 4357 EVT ShAmtTy = 4358 EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize()); 4359 SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy); 4360 SDValue Ops[2] = {N->getOperand(0), ShAmt}; 4361 TargetLowering::MakeLibCallOptions CallOptions; 4362 CallOptions.setSExt(isSigned); 4363 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4364 return; 4365 } 4366 4367 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 4368 llvm_unreachable("Unsupported shift!"); 4369 } 4370 4371 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 4372 SDValue &Lo, SDValue &Hi) { 4373 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4374 SDLoc dl(N); 4375 SDValue Op = N->getOperand(0); 4376 if (Op.getValueType().bitsLE(NVT)) { 4377 // The low part is sign extension of the input (degenerates to a copy). 4378 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 4379 // The high part is obtained by SRA'ing all but one of the bits of low part. 4380 unsigned LoSize = NVT.getSizeInBits(); 4381 Hi = DAG.getNode( 4382 ISD::SRA, dl, NVT, Lo, 4383 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); 4384 } else { 4385 // For example, extension of an i48 to an i64. The operand type necessarily 4386 // promotes to the result type, so will end up being expanded too. 4387 assert(getTypeAction(Op.getValueType()) == 4388 TargetLowering::TypePromoteInteger && 4389 "Only know how to promote this result!"); 4390 SDValue Res = GetPromotedInteger(Op); 4391 assert(Res.getValueType() == N->getValueType(0) && 4392 "Operand over promoted?"); 4393 // Split the promoted operand. This will simplify when it is expanded. 4394 SplitInteger(Res, Lo, Hi); 4395 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 4396 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 4397 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 4398 ExcessBits))); 4399 } 4400 } 4401 4402 void DAGTypeLegalizer:: 4403 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 4404 SDLoc dl(N); 4405 GetExpandedInteger(N->getOperand(0), Lo, Hi); 4406 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 4407 4408 if (EVT.bitsLE(Lo.getValueType())) { 4409 // sext_inreg the low part if needed. 4410 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 4411 N->getOperand(1)); 4412 4413 // The high part gets the sign extension from the lo-part. This handles 4414 // things like sextinreg V:i64 from i8. 4415 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 4416 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, 4417 TLI.getPointerTy(DAG.getDataLayout()))); 4418 } else { 4419 // For example, extension of an i48 to an i64. Leave the low part alone, 4420 // sext_inreg the high part. 4421 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); 4422 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 4423 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 4424 ExcessBits))); 4425 } 4426 } 4427 4428 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 4429 SDValue &Lo, SDValue &Hi) { 4430 EVT VT = N->getValueType(0); 4431 SDLoc dl(N); 4432 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4433 4434 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 4435 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4436 SplitInteger(Res.getValue(1), Lo, Hi); 4437 return; 4438 } 4439 4440 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4441 if (VT == MVT::i16) 4442 LC = RTLIB::SREM_I16; 4443 else if (VT == MVT::i32) 4444 LC = RTLIB::SREM_I32; 4445 else if (VT == MVT::i64) 4446 LC = RTLIB::SREM_I64; 4447 else if (VT == MVT::i128) 4448 LC = RTLIB::SREM_I128; 4449 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 4450 4451 TargetLowering::MakeLibCallOptions CallOptions; 4452 CallOptions.setSExt(true); 4453 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4454 } 4455 4456 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 4457 SDValue &Lo, SDValue &Hi) { 4458 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4459 SDLoc dl(N); 4460 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 4461 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), 4462 N->getOperand(0), 4463 DAG.getConstant(NVT.getSizeInBits(), dl, 4464 TLI.getPointerTy(DAG.getDataLayout()))); 4465 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 4466 } 4467 4468 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 4469 SDValue &Lo, SDValue &Hi) { 4470 EVT VT = N->getValueType(0); 4471 SDLoc dl(N); 4472 4473 if (N->getOpcode() == ISD::UMULO) { 4474 // This section expands the operation into the following sequence of 4475 // instructions. `iNh` here refers to a type which has half the bit width of 4476 // the type the original operation operated on. 4477 // 4478 // %0 = %LHS.HI != 0 && %RHS.HI != 0 4479 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) 4480 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) 4481 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) 4482 // %4 = add iNh %1.0, %2.0 as iN 4483 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH) 4484 // 4485 // %lo = %3.LO 4486 // %hi = %5.0 4487 // %ovf = %0 || %1.1 || %2.1 || %5.1 4488 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 4489 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; 4490 GetExpandedInteger(LHS, LHSLow, LHSHigh); 4491 GetExpandedInteger(RHS, RHSLow, RHSHigh); 4492 EVT HalfVT = LHSLow.getValueType(); 4493 EVT BitVT = N->getValueType(1); 4494 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT); 4495 4496 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT); 4497 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT, 4498 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE), 4499 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE)); 4500 4501 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow); 4502 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1)); 4503 4504 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow); 4505 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1)); 4506 4507 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two); 4508 4509 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not 4510 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this 4511 // operation recursively legalized?). 4512 // 4513 // Many backends understand this pattern and will convert into LOHI 4514 // themselves, if applicable. 4515 SDValue Three = DAG.getNode(ISD::MUL, dl, VT, 4516 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow), 4517 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow)); 4518 SplitInteger(Three, Lo, Hi); 4519 4520 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum); 4521 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1)); 4522 ReplaceValueWith(SDValue(N, 1), Overflow); 4523 return; 4524 } 4525 4526 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 4527 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 4528 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 4529 4530 // Replace this with a libcall that will check overflow. 4531 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4532 if (VT == MVT::i32) 4533 LC = RTLIB::MULO_I32; 4534 else if (VT == MVT::i64) 4535 LC = RTLIB::MULO_I64; 4536 else if (VT == MVT::i128) 4537 LC = RTLIB::MULO_I128; 4538 4539 // If we don't have the libcall or if the function we are compiling is the 4540 // implementation of the expected libcall (avoid inf-loop), expand inline. 4541 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC) || 4542 TLI.getLibcallName(LC) == DAG.getMachineFunction().getName()) { 4543 // FIXME: This is not an optimal expansion, but better than crashing. 4544 EVT WideVT = 4545 EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2); 4546 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0)); 4547 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1)); 4548 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); 4549 SDValue MulLo, MulHi; 4550 SplitInteger(Mul, MulLo, MulHi); 4551 SDValue SRA = 4552 DAG.getNode(ISD::SRA, dl, VT, MulLo, 4553 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT)); 4554 SDValue Overflow = 4555 DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE); 4556 SplitInteger(MulLo, Lo, Hi); 4557 ReplaceValueWith(SDValue(N, 1), Overflow); 4558 return; 4559 } 4560 4561 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 4562 // Temporary for the overflow value, default it to zero. 4563 SDValue Chain = 4564 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, 4565 MachinePointerInfo()); 4566 4567 TargetLowering::ArgListTy Args; 4568 TargetLowering::ArgListEntry Entry; 4569 for (const SDValue &Op : N->op_values()) { 4570 EVT ArgVT = Op.getValueType(); 4571 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 4572 Entry.Node = Op; 4573 Entry.Ty = ArgTy; 4574 Entry.IsSExt = true; 4575 Entry.IsZExt = false; 4576 Args.push_back(Entry); 4577 } 4578 4579 // Also pass the address of the overflow check. 4580 Entry.Node = Temp; 4581 Entry.Ty = PtrTy->getPointerTo(); 4582 Entry.IsSExt = true; 4583 Entry.IsZExt = false; 4584 Args.push_back(Entry); 4585 4586 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 4587 4588 TargetLowering::CallLoweringInfo CLI(DAG); 4589 CLI.setDebugLoc(dl) 4590 .setChain(Chain) 4591 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) 4592 .setSExtResult(); 4593 4594 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 4595 4596 SplitInteger(CallInfo.first, Lo, Hi); 4597 SDValue Temp2 = 4598 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); 4599 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 4600 DAG.getConstant(0, dl, PtrVT), 4601 ISD::SETNE); 4602 // Use the overflow from the libcall everywhere. 4603 ReplaceValueWith(SDValue(N, 1), Ofl); 4604 } 4605 4606 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 4607 SDValue &Lo, SDValue &Hi) { 4608 EVT VT = N->getValueType(0); 4609 SDLoc dl(N); 4610 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4611 4612 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4613 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4614 SplitInteger(Res.getValue(0), Lo, Hi); 4615 return; 4616 } 4617 4618 // Try to expand UDIV by constant. 4619 if (isa<ConstantSDNode>(N->getOperand(1))) { 4620 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4621 // Only if the new type is legal. 4622 if (isTypeLegal(NVT)) { 4623 SDValue InL, InH; 4624 GetExpandedInteger(N->getOperand(0), InL, InH); 4625 SmallVector<SDValue> Result; 4626 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) { 4627 Lo = Result[0]; 4628 Hi = Result[1]; 4629 return; 4630 } 4631 } 4632 } 4633 4634 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4635 if (VT == MVT::i16) 4636 LC = RTLIB::UDIV_I16; 4637 else if (VT == MVT::i32) 4638 LC = RTLIB::UDIV_I32; 4639 else if (VT == MVT::i64) 4640 LC = RTLIB::UDIV_I64; 4641 else if (VT == MVT::i128) 4642 LC = RTLIB::UDIV_I128; 4643 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 4644 4645 TargetLowering::MakeLibCallOptions CallOptions; 4646 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4647 } 4648 4649 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 4650 SDValue &Lo, SDValue &Hi) { 4651 EVT VT = N->getValueType(0); 4652 SDLoc dl(N); 4653 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 4654 4655 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 4656 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 4657 SplitInteger(Res.getValue(1), Lo, Hi); 4658 return; 4659 } 4660 4661 // Try to expand UREM by constant. 4662 if (isa<ConstantSDNode>(N->getOperand(1))) { 4663 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4664 // Only if the new type is legal. 4665 if (isTypeLegal(NVT)) { 4666 SDValue InL, InH; 4667 GetExpandedInteger(N->getOperand(0), InL, InH); 4668 SmallVector<SDValue> Result; 4669 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) { 4670 Lo = Result[0]; 4671 Hi = Result[1]; 4672 return; 4673 } 4674 } 4675 } 4676 4677 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 4678 if (VT == MVT::i16) 4679 LC = RTLIB::UREM_I16; 4680 else if (VT == MVT::i32) 4681 LC = RTLIB::UREM_I32; 4682 else if (VT == MVT::i64) 4683 LC = RTLIB::UREM_I64; 4684 else if (VT == MVT::i128) 4685 LC = RTLIB::UREM_I128; 4686 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 4687 4688 TargetLowering::MakeLibCallOptions CallOptions; 4689 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); 4690 } 4691 4692 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 4693 SDValue &Lo, SDValue &Hi) { 4694 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4695 SDLoc dl(N); 4696 SDValue Op = N->getOperand(0); 4697 if (Op.getValueType().bitsLE(NVT)) { 4698 // The low part is zero extension of the input (degenerates to a copy). 4699 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 4700 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. 4701 } else { 4702 // For example, extension of an i48 to an i64. The operand type necessarily 4703 // promotes to the result type, so will end up being expanded too. 4704 assert(getTypeAction(Op.getValueType()) == 4705 TargetLowering::TypePromoteInteger && 4706 "Only know how to promote this result!"); 4707 SDValue Res = GetPromotedInteger(Op); 4708 assert(Res.getValueType() == N->getValueType(0) && 4709 "Operand over promoted?"); 4710 // Split the promoted operand. This will simplify when it is expanded. 4711 SplitInteger(Res, Lo, Hi); 4712 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 4713 Hi = DAG.getZeroExtendInReg(Hi, dl, 4714 EVT::getIntegerVT(*DAG.getContext(), 4715 ExcessBits)); 4716 } 4717 } 4718 4719 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 4720 SDValue &Lo, SDValue &Hi) { 4721 SDLoc dl(N); 4722 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 4723 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 4724 SDValue Zero = DAG.getConstant(0, dl, VT); 4725 SDValue Swap = DAG.getAtomicCmpSwap( 4726 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 4727 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 4728 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); 4729 4730 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 4731 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 4732 } 4733 4734 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, 4735 SDValue &Lo, SDValue &Hi) { 4736 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate 4737 // both halves independently. 4738 SDValue Res = TLI.expandVecReduce(N, DAG); 4739 SplitInteger(Res, Lo, Hi); 4740 } 4741 4742 void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, 4743 SDValue &Lo, SDValue &Hi) { 4744 // Delegate to funnel-shift expansion. 4745 SDLoc DL(N); 4746 unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR; 4747 SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0), 4748 N->getOperand(0), N->getOperand(1)); 4749 SplitInteger(Res, Lo, Hi); 4750 } 4751 4752 void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo, 4753 SDValue &Hi) { 4754 // Values numbered from least significant to most significant. 4755 SDValue In1, In2, In3, In4; 4756 GetExpandedInteger(N->getOperand(0), In3, In4); 4757 GetExpandedInteger(N->getOperand(1), In1, In2); 4758 EVT HalfVT = In1.getValueType(); 4759 4760 SDLoc DL(N); 4761 unsigned Opc = N->getOpcode(); 4762 SDValue ShAmt = N->getOperand(2); 4763 EVT ShAmtVT = ShAmt.getValueType(); 4764 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT); 4765 4766 // If the shift amount is at least half the bitwidth, swap the inputs. 4767 unsigned HalfVTBits = HalfVT.getScalarSizeInBits(); 4768 SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt, 4769 DAG.getConstant(HalfVTBits, DL, ShAmtVT)); 4770 SDValue Cond = 4771 DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT), 4772 Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ); 4773 4774 // Expand to a pair of funnel shifts. 4775 EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout()); 4776 SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT); 4777 4778 SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2); 4779 SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3); 4780 SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4); 4781 Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt); 4782 Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt); 4783 } 4784 4785 void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, 4786 SDValue &Hi) { 4787 EVT VT = N->getValueType(0); 4788 EVT HalfVT = 4789 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2); 4790 SDLoc dl(N); 4791 4792 // We assume VSCALE(1) fits into a legal integer. 4793 APInt One(HalfVT.getSizeInBits(), 1); 4794 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One); 4795 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase); 4796 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0)); 4797 SplitInteger(Res, Lo, Hi); 4798 } 4799 4800 //===----------------------------------------------------------------------===// 4801 // Integer Operand Expansion 4802 //===----------------------------------------------------------------------===// 4803 4804 /// ExpandIntegerOperand - This method is called when the specified operand of 4805 /// the specified node is found to need expansion. At this point, all of the 4806 /// result types of the node are known to be legal, but other operands of the 4807 /// node may need promotion or expansion as well as the specified one. 4808 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 4809 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); 4810 dbgs() << "\n"); 4811 SDValue Res = SDValue(); 4812 4813 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 4814 return false; 4815 4816 switch (N->getOpcode()) { 4817 default: 4818 #ifndef NDEBUG 4819 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 4820 N->dump(&DAG); dbgs() << "\n"; 4821 #endif 4822 report_fatal_error("Do not know how to expand this operator's operand!"); 4823 4824 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 4825 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 4826 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 4827 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 4828 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 4829 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 4830 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break; 4831 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 4832 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 4833 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; 4834 case ISD::STRICT_SINT_TO_FP: 4835 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 4836 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 4837 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 4838 case ISD::STRICT_UINT_TO_FP: 4839 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 4840 4841 case ISD::SHL: 4842 case ISD::SRA: 4843 case ISD::SRL: 4844 case ISD::ROTL: 4845 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 4846 case ISD::RETURNADDR: 4847 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 4848 4849 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 4850 case ISD::STACKMAP: 4851 Res = ExpandIntOp_STACKMAP(N, OpNo); 4852 break; 4853 case ISD::PATCHPOINT: 4854 Res = ExpandIntOp_PATCHPOINT(N, OpNo); 4855 break; 4856 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 4857 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 4858 Res = ExpandIntOp_VP_STRIDED(N, OpNo); 4859 break; 4860 } 4861 4862 // If the result is null, the sub-method took care of registering results etc. 4863 if (!Res.getNode()) return false; 4864 4865 // If the result is N, the sub-method updated N in place. Tell the legalizer 4866 // core about this. 4867 if (Res.getNode() == N) 4868 return true; 4869 4870 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 4871 "Invalid operand expansion"); 4872 4873 ReplaceValueWith(SDValue(N, 0), Res); 4874 return false; 4875 } 4876 4877 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 4878 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 4879 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 4880 SDValue &NewRHS, 4881 ISD::CondCode &CCCode, 4882 const SDLoc &dl) { 4883 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 4884 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 4885 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 4886 4887 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 4888 if (RHSLo == RHSHi) { 4889 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 4890 if (RHSCST->isAllOnes()) { 4891 // Equality comparison to -1. 4892 NewLHS = DAG.getNode(ISD::AND, dl, 4893 LHSLo.getValueType(), LHSLo, LHSHi); 4894 NewRHS = RHSLo; 4895 return; 4896 } 4897 } 4898 } 4899 4900 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 4901 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 4902 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 4903 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); 4904 return; 4905 } 4906 4907 // If this is a comparison of the sign bit, just look at the top part. 4908 // X > -1, x < 0 4909 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 4910 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0 4911 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1 4912 NewLHS = LHSHi; 4913 NewRHS = RHSHi; 4914 return; 4915 } 4916 4917 // FIXME: This generated code sucks. 4918 ISD::CondCode LowCC; 4919 switch (CCCode) { 4920 default: llvm_unreachable("Unknown integer setcc!"); 4921 case ISD::SETLT: 4922 case ISD::SETULT: LowCC = ISD::SETULT; break; 4923 case ISD::SETGT: 4924 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 4925 case ISD::SETLE: 4926 case ISD::SETULE: LowCC = ISD::SETULE; break; 4927 case ISD::SETGE: 4928 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 4929 } 4930 4931 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison 4932 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands 4933 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; 4934 4935 // NOTE: on targets without efficient SELECT of bools, we can always use 4936 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 4937 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 4938 nullptr); 4939 SDValue LoCmp, HiCmp; 4940 if (TLI.isTypeLegal(LHSLo.getValueType()) && 4941 TLI.isTypeLegal(RHSLo.getValueType())) 4942 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, 4943 RHSLo, LowCC, false, DagCombineInfo, dl); 4944 if (!LoCmp.getNode()) 4945 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 4946 RHSLo, LowCC); 4947 if (TLI.isTypeLegal(LHSHi.getValueType()) && 4948 TLI.isTypeLegal(RHSHi.getValueType())) 4949 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, 4950 RHSHi, CCCode, false, DagCombineInfo, dl); 4951 if (!HiCmp.getNode()) 4952 HiCmp = 4953 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), 4954 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 4955 4956 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); 4957 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); 4958 4959 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 4960 CCCode == ISD::SETUGE || CCCode == ISD::SETULE); 4961 4962 // FIXME: Is the HiCmpC->isOne() here correct for 4963 // ZeroOrNegativeOneBooleanContent. 4964 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) || 4965 (!EqAllowed && 4966 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) { 4967 // For LE / GE, if high part is known false, ignore the low part. 4968 // For LT / GT: if low part is known false, return the high part. 4969 // if high part is known true, ignore the low part. 4970 NewLHS = HiCmp; 4971 NewRHS = SDValue(); 4972 return; 4973 } 4974 4975 if (LHSHi == RHSHi) { 4976 // Comparing the low bits is enough. 4977 NewLHS = LoCmp; 4978 NewRHS = SDValue(); 4979 return; 4980 } 4981 4982 // Lower with SETCCCARRY if the target supports it. 4983 EVT HiVT = LHSHi.getValueType(); 4984 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); 4985 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); 4986 4987 // FIXME: Make all targets support this, then remove the other lowering. 4988 if (HasSETCCCARRY) { 4989 // SETCCCARRY can detect < and >= directly. For > and <=, flip 4990 // operands and condition code. 4991 bool FlipOperands = false; 4992 switch (CCCode) { 4993 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; 4994 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; 4995 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; 4996 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; 4997 default: break; 4998 } 4999 if (FlipOperands) { 5000 std::swap(LHSLo, RHSLo); 5001 std::swap(LHSHi, RHSHi); 5002 } 5003 // Perform a wide subtraction, feeding the carry from the low part into 5004 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high 5005 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is 5006 // zero or positive iff LHS >= RHS. 5007 EVT LoVT = LHSLo.getValueType(); 5008 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT)); 5009 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo); 5010 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT), 5011 LHSHi, RHSHi, LowCmp.getValue(1), 5012 DAG.getCondCode(CCCode)); 5013 NewLHS = Res; 5014 NewRHS = SDValue(); 5015 return; 5016 } 5017 5018 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, 5019 false, DagCombineInfo, dl); 5020 if (!NewLHS.getNode()) 5021 NewLHS = 5022 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); 5023 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); 5024 NewRHS = SDValue(); 5025 } 5026 5027 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 5028 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 5029 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 5030 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 5031 5032 // If ExpandSetCCOperands returned a scalar, we need to compare the result 5033 // against zero to select between true and false values. 5034 if (!NewRHS.getNode()) { 5035 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 5036 CCCode = ISD::SETNE; 5037 } 5038 5039 // Update N to have the operands specified. 5040 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 5041 DAG.getCondCode(CCCode), NewLHS, NewRHS, 5042 N->getOperand(4)), 0); 5043 } 5044 5045 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 5046 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 5047 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 5048 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 5049 5050 // If ExpandSetCCOperands returned a scalar, we need to compare the result 5051 // against zero to select between true and false values. 5052 if (!NewRHS.getNode()) { 5053 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 5054 CCCode = ISD::SETNE; 5055 } 5056 5057 // Update N to have the operands specified. 5058 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 5059 N->getOperand(2), N->getOperand(3), 5060 DAG.getCondCode(CCCode)), 0); 5061 } 5062 5063 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 5064 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 5065 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 5066 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 5067 5068 // If ExpandSetCCOperands returned a scalar, use it. 5069 if (!NewRHS.getNode()) { 5070 assert(NewLHS.getValueType() == N->getValueType(0) && 5071 "Unexpected setcc expansion!"); 5072 return NewLHS; 5073 } 5074 5075 // Otherwise, update N to have the operands specified. 5076 return SDValue( 5077 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); 5078 } 5079 5080 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { 5081 SDValue LHS = N->getOperand(0); 5082 SDValue RHS = N->getOperand(1); 5083 SDValue Carry = N->getOperand(2); 5084 SDValue Cond = N->getOperand(3); 5085 SDLoc dl = SDLoc(N); 5086 5087 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 5088 GetExpandedInteger(LHS, LHSLo, LHSHi); 5089 GetExpandedInteger(RHS, RHSLo, RHSHi); 5090 5091 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high. 5092 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); 5093 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry); 5094 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, 5095 LowCmp.getValue(1), Cond); 5096 } 5097 5098 SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) { 5099 // Split the operand and replace with SPLAT_VECTOR_PARTS. 5100 SDValue Lo, Hi; 5101 GetExpandedInteger(N->getOperand(0), Lo, Hi); 5102 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo, 5103 Hi); 5104 } 5105 5106 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 5107 // The value being shifted is legal, but the shift amount is too big. 5108 // It follows that either the result of the shift is undefined, or the 5109 // upper half of the shift amount is zero. Just use the lower half. 5110 SDValue Lo, Hi; 5111 GetExpandedInteger(N->getOperand(1), Lo, Hi); 5112 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 5113 } 5114 5115 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 5116 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 5117 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 5118 // constant to valid type. 5119 SDValue Lo, Hi; 5120 GetExpandedInteger(N->getOperand(0), Lo, Hi); 5121 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 5122 } 5123 5124 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 5125 bool IsStrict = N->isStrictFPOpcode(); 5126 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 5127 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 5128 EVT DstVT = N->getValueType(0); 5129 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 5130 assert(LC != RTLIB::UNKNOWN_LIBCALL && 5131 "Don't know how to expand this SINT_TO_FP!"); 5132 TargetLowering::MakeLibCallOptions CallOptions; 5133 CallOptions.setSExt(true); 5134 std::pair<SDValue, SDValue> Tmp = 5135 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 5136 5137 if (!IsStrict) 5138 return Tmp.first; 5139 5140 ReplaceValueWith(SDValue(N, 1), Tmp.second); 5141 ReplaceValueWith(SDValue(N, 0), Tmp.first); 5142 return SDValue(); 5143 } 5144 5145 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 5146 if (N->isAtomic()) { 5147 // It's typical to have larger CAS than atomic store instructions. 5148 SDLoc dl(N); 5149 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 5150 N->getMemoryVT(), 5151 N->getOperand(0), N->getOperand(2), 5152 N->getOperand(1), 5153 N->getMemOperand()); 5154 return Swap.getValue(1); 5155 } 5156 if (ISD::isNormalStore(N)) 5157 return ExpandOp_NormalStore(N, OpNo); 5158 5159 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 5160 assert(OpNo == 1 && "Can only expand the stored value so far"); 5161 5162 EVT VT = N->getOperand(1).getValueType(); 5163 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5164 SDValue Ch = N->getChain(); 5165 SDValue Ptr = N->getBasePtr(); 5166 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 5167 AAMDNodes AAInfo = N->getAAInfo(); 5168 SDLoc dl(N); 5169 SDValue Lo, Hi; 5170 5171 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 5172 5173 if (N->getMemoryVT().bitsLE(NVT)) { 5174 GetExpandedInteger(N->getValue(), Lo, Hi); 5175 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 5176 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags, 5177 AAInfo); 5178 } 5179 5180 if (DAG.getDataLayout().isLittleEndian()) { 5181 // Little-endian - low bits are at low addresses. 5182 GetExpandedInteger(N->getValue(), Lo, Hi); 5183 5184 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 5185 N->getOriginalAlign(), MMOFlags, AAInfo); 5186 5187 unsigned ExcessBits = 5188 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 5189 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 5190 5191 // Increment the pointer to the other half. 5192 unsigned IncrementSize = NVT.getSizeInBits()/8; 5193 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 5194 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 5195 N->getPointerInfo().getWithOffset(IncrementSize), 5196 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo); 5197 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 5198 } 5199 5200 // Big-endian - high bits are at low addresses. Favor aligned stores at 5201 // the cost of some bit-fiddling. 5202 GetExpandedInteger(N->getValue(), Lo, Hi); 5203 5204 EVT ExtVT = N->getMemoryVT(); 5205 unsigned EBytes = ExtVT.getStoreSize(); 5206 unsigned IncrementSize = NVT.getSizeInBits()/8; 5207 unsigned ExcessBits = (EBytes - IncrementSize)*8; 5208 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 5209 ExtVT.getSizeInBits() - ExcessBits); 5210 5211 if (ExcessBits < NVT.getSizeInBits()) { 5212 // Transfer high bits from the top of Lo to the bottom of Hi. 5213 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 5214 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 5215 TLI.getPointerTy(DAG.getDataLayout()))); 5216 Hi = DAG.getNode( 5217 ISD::OR, dl, NVT, Hi, 5218 DAG.getNode(ISD::SRL, dl, NVT, Lo, 5219 DAG.getConstant(ExcessBits, dl, 5220 TLI.getPointerTy(DAG.getDataLayout())))); 5221 } 5222 5223 // Store both the high bits and maybe some of the low bits. 5224 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, 5225 N->getOriginalAlign(), MMOFlags, AAInfo); 5226 5227 // Increment the pointer to the other half. 5228 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 5229 // Store the lowest ExcessBits bits in the second half. 5230 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 5231 N->getPointerInfo().getWithOffset(IncrementSize), 5232 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 5233 N->getOriginalAlign(), MMOFlags, AAInfo); 5234 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 5235 } 5236 5237 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 5238 SDValue InL, InH; 5239 GetExpandedInteger(N->getOperand(0), InL, InH); 5240 // Just truncate the low part of the source. 5241 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 5242 } 5243 5244 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 5245 bool IsStrict = N->isStrictFPOpcode(); 5246 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 5247 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 5248 EVT DstVT = N->getValueType(0); 5249 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT); 5250 assert(LC != RTLIB::UNKNOWN_LIBCALL && 5251 "Don't know how to expand this UINT_TO_FP!"); 5252 TargetLowering::MakeLibCallOptions CallOptions; 5253 CallOptions.setSExt(true); 5254 std::pair<SDValue, SDValue> Tmp = 5255 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); 5256 5257 if (!IsStrict) 5258 return Tmp.first; 5259 5260 ReplaceValueWith(SDValue(N, 1), Tmp.second); 5261 ReplaceValueWith(SDValue(N, 0), Tmp.first); 5262 return SDValue(); 5263 } 5264 5265 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 5266 SDLoc dl(N); 5267 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 5268 cast<AtomicSDNode>(N)->getMemoryVT(), 5269 N->getOperand(0), 5270 N->getOperand(1), N->getOperand(2), 5271 cast<AtomicSDNode>(N)->getMemOperand()); 5272 return Swap.getValue(1); 5273 } 5274 5275 SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { 5276 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || 5277 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); 5278 5279 SDValue Hi; // The upper half is dropped out. 5280 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); 5281 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo], Hi); 5282 5283 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 5284 } 5285 5286 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { 5287 SDLoc dl(N); 5288 5289 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5290 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5291 EVT OutVT = V0.getValueType(); 5292 5293 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2)); 5294 } 5295 5296 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 5297 5298 EVT OutVT = N->getValueType(0); 5299 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5300 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5301 EVT NOutVTElem = NOutVT.getVectorElementType(); 5302 5303 SDLoc dl(N); 5304 SDValue BaseIdx = N->getOperand(1); 5305 5306 // TODO: We may be able to use this for types other than scalable 5307 // vectors and fix those tests that expect BUILD_VECTOR to be used 5308 if (OutVT.isScalableVector()) { 5309 SDValue InOp0 = N->getOperand(0); 5310 EVT InVT = InOp0.getValueType(); 5311 5312 // Try and extract from a smaller type so that it eventually falls 5313 // into the promotion code below. 5314 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector || 5315 getTypeAction(InVT) == TargetLowering::TypeLegal) { 5316 EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext()); 5317 unsigned NElts = NInVT.getVectorMinNumElements(); 5318 uint64_t IdxVal = cast<ConstantSDNode>(BaseIdx)->getZExtValue(); 5319 5320 SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0, 5321 DAG.getConstant(alignDown(IdxVal, NElts), dl, 5322 BaseIdx.getValueType())); 5323 SDValue Step2 = DAG.getNode( 5324 ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1, 5325 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType())); 5326 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2); 5327 } 5328 5329 // Try and extract from a widened type. 5330 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 5331 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx}; 5332 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops); 5333 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); 5334 } 5335 5336 // Promote operands and see if this is handled by target lowering, 5337 // Otherwise, use the BUILD_VECTOR approach below 5338 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { 5339 // Collect the (promoted) operands 5340 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx }; 5341 5342 EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); 5343 assert(PromEltVT.bitsLE(NOutVTElem) && 5344 "Promoted operand has an element type greater than result"); 5345 5346 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT); 5347 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops); 5348 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); 5349 } 5350 } 5351 5352 if (OutVT.isScalableVector()) 5353 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR"); 5354 5355 SDValue InOp0 = N->getOperand(0); 5356 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) 5357 InOp0 = GetPromotedInteger(N->getOperand(0)); 5358 5359 EVT InVT = InOp0.getValueType(); 5360 5361 unsigned OutNumElems = OutVT.getVectorNumElements(); 5362 SmallVector<SDValue, 8> Ops; 5363 Ops.reserve(OutNumElems); 5364 for (unsigned i = 0; i != OutNumElems; ++i) { 5365 5366 // Extract the element from the original vector. 5367 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 5368 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); 5369 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 5370 InVT.getVectorElementType(), N->getOperand(0), Index); 5371 5372 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); 5373 // Insert the converted element to the new vector. 5374 Ops.push_back(Op); 5375 } 5376 5377 return DAG.getBuildVector(NOutVT, dl, Ops); 5378 } 5379 5380 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) { 5381 EVT OutVT = N->getValueType(0); 5382 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5383 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5384 5385 SDLoc dl(N); 5386 SDValue Vec = N->getOperand(0); 5387 SDValue SubVec = N->getOperand(1); 5388 SDValue Idx = N->getOperand(2); 5389 5390 EVT SubVecVT = SubVec.getValueType(); 5391 EVT NSubVT = 5392 EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(), 5393 SubVecVT.getVectorElementCount()); 5394 5395 Vec = GetPromotedInteger(Vec); 5396 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec); 5397 5398 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx); 5399 } 5400 5401 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) { 5402 SDLoc dl(N); 5403 5404 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5405 EVT OutVT = V0.getValueType(); 5406 5407 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0); 5408 } 5409 5410 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 5411 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 5412 EVT VT = N->getValueType(0); 5413 SDLoc dl(N); 5414 5415 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); 5416 5417 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5418 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5419 EVT OutVT = V0.getValueType(); 5420 5421 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); 5422 } 5423 5424 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 5425 EVT OutVT = N->getValueType(0); 5426 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5427 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5428 unsigned NumElems = N->getNumOperands(); 5429 EVT NOutVTElem = NOutVT.getVectorElementType(); 5430 TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(NOutVT); 5431 unsigned NOutExtOpc = TargetLowering::getExtendForContent(NOutBoolType); 5432 SDLoc dl(N); 5433 5434 SmallVector<SDValue, 8> Ops; 5435 Ops.reserve(NumElems); 5436 for (unsigned i = 0; i != NumElems; ++i) { 5437 SDValue Op = N->getOperand(i); 5438 EVT OpVT = Op.getValueType(); 5439 // BUILD_VECTOR integer operand types are allowed to be larger than the 5440 // result's element type. This may still be true after the promotion. For 5441 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 5442 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 5443 if (OpVT.bitsLT(NOutVTElem)) { 5444 unsigned ExtOpc = ISD::ANY_EXTEND; 5445 // Attempt to extend constant bool vectors to match target's BooleanContent. 5446 // While not necessary, this improves chances of the constant correctly 5447 // folding with compare results (e.g. for NOT patterns). 5448 if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant) 5449 ExtOpc = NOutExtOpc; 5450 Op = DAG.getNode(ExtOpc, dl, NOutVTElem, Op); 5451 } 5452 Ops.push_back(Op); 5453 } 5454 5455 return DAG.getBuildVector(NOutVT, dl, Ops); 5456 } 5457 5458 SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) { 5459 5460 SDLoc dl(N); 5461 5462 assert(!N->getOperand(0).getValueType().isVector() && 5463 "Input must be a scalar"); 5464 5465 EVT OutVT = N->getValueType(0); 5466 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5467 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5468 EVT NOutElemVT = NOutVT.getVectorElementType(); 5469 5470 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0)); 5471 5472 return DAG.getNode(N->getOpcode(), dl, NOutVT, Op); 5473 } 5474 5475 SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) { 5476 SDLoc dl(N); 5477 EVT OutVT = N->getValueType(0); 5478 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5479 assert(NOutVT.isScalableVector() && 5480 "Type must be promoted to a scalable vector type"); 5481 const APInt &StepVal = N->getConstantOperandAPInt(0); 5482 return DAG.getStepVector(dl, NOutVT, 5483 StepVal.sext(NOutVT.getScalarSizeInBits())); 5484 } 5485 5486 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 5487 SDLoc dl(N); 5488 5489 EVT OutVT = N->getValueType(0); 5490 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5491 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5492 5493 unsigned NumOperands = N->getNumOperands(); 5494 unsigned NumOutElem = NOutVT.getVectorMinNumElements(); 5495 EVT OutElemTy = NOutVT.getVectorElementType(); 5496 if (OutVT.isScalableVector()) { 5497 // Find the largest promoted element type for each of the operands. 5498 SDUse *MaxSizedValue = std::max_element( 5499 N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) { 5500 EVT AVT = A.getValueType().getVectorElementType(); 5501 EVT BVT = B.getValueType().getVectorElementType(); 5502 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits(); 5503 }); 5504 EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType(); 5505 5506 // Then promote all vectors to the largest element type. 5507 SmallVector<SDValue, 8> Ops; 5508 for (unsigned I = 0; I < NumOperands; ++I) { 5509 SDValue Op = N->getOperand(I); 5510 EVT OpVT = Op.getValueType(); 5511 if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger) 5512 Op = GetPromotedInteger(Op); 5513 else 5514 assert(getTypeAction(OpVT) == TargetLowering::TypeLegal && 5515 "Unhandled legalization type"); 5516 5517 if (OpVT.getVectorElementType().getScalarSizeInBits() < 5518 MaxElementVT.getScalarSizeInBits()) 5519 Op = DAG.getAnyExtOrTrunc(Op, dl, 5520 OpVT.changeVectorElementType(MaxElementVT)); 5521 Ops.push_back(Op); 5522 } 5523 5524 // Do the CONCAT on the promoted type and finally truncate to (the promoted) 5525 // NOutVT. 5526 return DAG.getAnyExtOrTrunc( 5527 DAG.getNode(ISD::CONCAT_VECTORS, dl, 5528 OutVT.changeVectorElementType(MaxElementVT), Ops), 5529 dl, NOutVT); 5530 } 5531 5532 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 5533 assert(NumElem * NumOperands == NumOutElem && 5534 "Unexpected number of elements"); 5535 5536 // Take the elements from the first vector. 5537 SmallVector<SDValue, 8> Ops(NumOutElem); 5538 for (unsigned i = 0; i < NumOperands; ++i) { 5539 SDValue Op = N->getOperand(i); 5540 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) 5541 Op = GetPromotedInteger(Op); 5542 EVT SclrTy = Op.getValueType().getVectorElementType(); 5543 assert(NumElem == Op.getValueType().getVectorNumElements() && 5544 "Unexpected number of elements"); 5545 5546 for (unsigned j = 0; j < NumElem; ++j) { 5547 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, 5548 DAG.getVectorIdxConstant(j, dl)); 5549 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); 5550 } 5551 } 5552 5553 return DAG.getBuildVector(NOutVT, dl, Ops); 5554 } 5555 5556 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { 5557 EVT VT = N->getValueType(0); 5558 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5559 assert(NVT.isVector() && "This type must be promoted to a vector type"); 5560 5561 SDLoc dl(N); 5562 5563 // For operands whose TypeAction is to promote, extend the promoted node 5564 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion 5565 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to 5566 // type.. 5567 if (getTypeAction(N->getOperand(0).getValueType()) 5568 == TargetLowering::TypePromoteInteger) { 5569 SDValue Promoted; 5570 5571 switch(N->getOpcode()) { 5572 case ISD::SIGN_EXTEND_VECTOR_INREG: 5573 Promoted = SExtPromotedInteger(N->getOperand(0)); 5574 break; 5575 case ISD::ZERO_EXTEND_VECTOR_INREG: 5576 Promoted = ZExtPromotedInteger(N->getOperand(0)); 5577 break; 5578 case ISD::ANY_EXTEND_VECTOR_INREG: 5579 Promoted = GetPromotedInteger(N->getOperand(0)); 5580 break; 5581 default: 5582 llvm_unreachable("Node has unexpected Opcode"); 5583 } 5584 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); 5585 } 5586 5587 // Directly extend to the appropriate transform-to type. 5588 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 5589 } 5590 5591 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 5592 EVT OutVT = N->getValueType(0); 5593 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 5594 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 5595 5596 EVT NOutVTElem = NOutVT.getVectorElementType(); 5597 5598 SDLoc dl(N); 5599 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5600 5601 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 5602 NOutVTElem, N->getOperand(1)); 5603 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 5604 V0, ConvElem, N->getOperand(2)); 5605 } 5606 5607 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { 5608 // The VECREDUCE result size may be larger than the element size, so 5609 // we can simply change the result type. 5610 SDLoc dl(N); 5611 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5612 return DAG.getNode(N->getOpcode(), dl, NVT, N->ops()); 5613 } 5614 5615 SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) { 5616 // The VP_REDUCE result size may be larger than the element size, so we can 5617 // simply change the result type. However the start value and result must be 5618 // the same. 5619 SDLoc DL(N); 5620 SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0)); 5621 return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start, 5622 N->getOperand(1), N->getOperand(2), N->getOperand(3)); 5623 } 5624 5625 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 5626 SDLoc dl(N); 5627 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5628 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, 5629 TLI.getVectorIdxTy(DAG.getDataLayout())); 5630 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 5631 V0->getValueType(0).getScalarType(), V0, V1); 5632 5633 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 5634 // element types. If this is the case then we need to expand the outgoing 5635 // value and not truncate it. 5636 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 5637 } 5638 5639 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) { 5640 SDLoc dl(N); 5641 // The result type is equal to the first input operand's type, so the 5642 // type that needs promoting must be the second source vector. 5643 SDValue V0 = N->getOperand(0); 5644 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 5645 SDValue Idx = N->getOperand(2); 5646 EVT PromVT = EVT::getVectorVT(*DAG.getContext(), 5647 V1.getValueType().getVectorElementType(), 5648 V0.getValueType().getVectorElementCount()); 5649 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT); 5650 SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx); 5651 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 5652 } 5653 5654 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { 5655 SDLoc dl(N); 5656 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 5657 MVT InVT = V0.getValueType().getSimpleVT(); 5658 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), 5659 N->getValueType(0).getVectorNumElements()); 5660 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); 5661 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 5662 } 5663 5664 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 5665 SDLoc dl(N); 5666 5667 EVT ResVT = N->getValueType(0); 5668 unsigned NumElems = N->getNumOperands(); 5669 5670 if (ResVT.isScalableVector()) { 5671 SDValue ResVec = DAG.getUNDEF(ResVT); 5672 5673 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { 5674 SDValue Op = N->getOperand(OpIdx); 5675 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); 5676 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op, 5677 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl)); 5678 } 5679 5680 return ResVec; 5681 } 5682 5683 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 5684 5685 SmallVector<SDValue, 8> NewOps; 5686 NewOps.reserve(NumElems); 5687 5688 // For each incoming vector 5689 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 5690 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 5691 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 5692 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 5693 5694 for (unsigned i=0; i<NumElem; ++i) { 5695 // Extract element from incoming vector 5696 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, 5697 DAG.getVectorIdxConstant(i, dl)); 5698 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 5699 NewOps.push_back(Tr); 5700 } 5701 } 5702 5703 return DAG.getBuildVector(N->getValueType(0), dl, NewOps); 5704 } 5705 5706 SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) { 5707 assert(OpNo > 1); 5708 SDValue Op = N->getOperand(OpNo); 5709 5710 // FIXME: Non-constant operands are not yet handled: 5711 // - https://github.com/llvm/llvm-project/issues/26431 5712 // - https://github.com/llvm/llvm-project/issues/55957 5713 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op); 5714 if (!CN) 5715 return SDValue(); 5716 5717 // Copy operands before the one being expanded. 5718 SmallVector<SDValue> NewOps; 5719 for (unsigned I = 0; I < OpNo; I++) 5720 NewOps.push_back(N->getOperand(I)); 5721 5722 EVT Ty = Op.getValueType(); 5723 SDLoc DL = SDLoc(N); 5724 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { 5725 NewOps.push_back( 5726 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); 5727 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); 5728 } else { 5729 // FIXME: https://github.com/llvm/llvm-project/issues/55609 5730 return SDValue(); 5731 } 5732 5733 // Copy remaining operands. 5734 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) 5735 NewOps.push_back(N->getOperand(I)); 5736 5737 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); 5738 5739 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 5740 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 5741 5742 return SDValue(); // Signal that we have replaced the node already. 5743 } 5744 5745 SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { 5746 assert(OpNo >= 7); 5747 SDValue Op = N->getOperand(OpNo); 5748 5749 // FIXME: Non-constant operands are not yet handled: 5750 // - https://github.com/llvm/llvm-project/issues/26431 5751 // - https://github.com/llvm/llvm-project/issues/55957 5752 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op); 5753 if (!CN) 5754 return SDValue(); 5755 5756 // Copy operands before the one being expanded. 5757 SmallVector<SDValue> NewOps; 5758 for (unsigned I = 0; I < OpNo; I++) 5759 NewOps.push_back(N->getOperand(I)); 5760 5761 EVT Ty = Op.getValueType(); 5762 SDLoc DL = SDLoc(N); 5763 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { 5764 NewOps.push_back( 5765 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); 5766 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); 5767 } else { 5768 // FIXME: https://github.com/llvm/llvm-project/issues/55609 5769 return SDValue(); 5770 } 5771 5772 // Copy remaining operands. 5773 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) 5774 NewOps.push_back(N->getOperand(I)); 5775 5776 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); 5777 5778 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 5779 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 5780 5781 return SDValue(); // Signal that we have replaced the node already. 5782 } 5783