1 //===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements generic type expansion and splitting for LegalizeTypes. 10 // The routines here perform legalization when the details of the type (such as 11 // whether it is an integer or a float) do not matter. 12 // Expansion is the act of changing a computation in an illegal type to be a 13 // computation in two identical registers of a smaller type. The Lo/Hi part 14 // is required to be stored first in memory on little/big-endian machines. 15 // Splitting is the act of changing a computation in an illegal type to be a 16 // computation in two not necessarily identical registers of a smaller type. 17 // There are no requirements on how the type is represented in memory. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/IR/DataLayout.h" 23 using namespace llvm; 24 25 #define DEBUG_TYPE "legalize-types" 26 27 //===----------------------------------------------------------------------===// 28 // Generic Result Expansion. 29 //===----------------------------------------------------------------------===// 30 31 // These routines assume that the Lo/Hi part is stored first in memory on 32 // little/big-endian machines, followed by the Hi/Lo part. This means that 33 // they cannot be used as is on vectors, for which Lo is always stored first. 34 void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 35 SDValue &Lo, SDValue &Hi) { 36 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 37 GetExpandedOp(Op, Lo, Hi); 38 } 39 40 void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) { 41 EVT OutVT = N->getValueType(0); 42 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 43 SDValue InOp = N->getOperand(0); 44 EVT InVT = InOp.getValueType(); 45 SDLoc dl(N); 46 47 // Handle some special cases efficiently. 48 switch (getTypeAction(InVT)) { 49 case TargetLowering::TypeLegal: 50 case TargetLowering::TypePromoteInteger: 51 break; 52 case TargetLowering::TypePromoteFloat: 53 case TargetLowering::TypeSoftPromoteHalf: 54 llvm_unreachable("Bitcast of a promotion-needing float should never need" 55 "expansion"); 56 case TargetLowering::TypeSoftenFloat: 57 SplitInteger(GetSoftenedFloat(InOp), Lo, Hi); 58 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 59 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 60 return; 61 case TargetLowering::TypeExpandInteger: 62 case TargetLowering::TypeExpandFloat: { 63 auto &DL = DAG.getDataLayout(); 64 // Convert the expanded pieces of the input. 65 GetExpandedOp(InOp, Lo, Hi); 66 if (TLI.hasBigEndianPartOrdering(InVT, DL) != 67 TLI.hasBigEndianPartOrdering(OutVT, DL)) 68 std::swap(Lo, Hi); 69 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 70 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 71 return; 72 } 73 case TargetLowering::TypeSplitVector: 74 GetSplitVector(InOp, Lo, Hi); 75 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 76 std::swap(Lo, Hi); 77 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 78 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 79 return; 80 case TargetLowering::TypeScalarizeVector: 81 // Convert the element instead. 82 SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi); 83 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 84 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 85 return; 86 case TargetLowering::TypeScalarizeScalableVector: 87 report_fatal_error("Scalarization of scalable vectors is not supported."); 88 case TargetLowering::TypeWidenVector: { 89 assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST"); 90 InOp = GetWidenedVector(InOp); 91 EVT LoVT, HiVT; 92 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT); 93 std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT); 94 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 95 std::swap(Lo, Hi); 96 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 97 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 98 return; 99 } 100 } 101 102 if (InVT.isVector() && OutVT.isInteger()) { 103 // Handle cases like i64 = BITCAST v1i64 on x86, where the operand 104 // is legal but the result is not. 105 unsigned NumElems = 2; 106 EVT ElemVT = NOutVT; 107 EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems); 108 109 // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>. 110 while (!isTypeLegal(NVT)) { 111 unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2; 112 // If the element size is smaller than byte, bail. 113 if (NewSizeInBits < 8) 114 break; 115 NumElems *= 2; 116 ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits); 117 NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems); 118 } 119 120 if (isTypeLegal(NVT)) { 121 SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp); 122 123 SmallVector<SDValue, 8> Vals; 124 for (unsigned i = 0; i < NumElems; ++i) 125 Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT, 126 CastInOp, DAG.getVectorIdxConstant(i, dl))); 127 128 // Build Lo, Hi pair by pairing extracted elements if needed. 129 unsigned Slot = 0; 130 for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) { 131 // Each iteration will BUILD_PAIR two nodes and append the result until 132 // there are only two nodes left, i.e. Lo and Hi. 133 SDValue LHS = Vals[Slot]; 134 SDValue RHS = Vals[Slot + 1]; 135 136 if (DAG.getDataLayout().isBigEndian()) 137 std::swap(LHS, RHS); 138 139 Vals.push_back(DAG.getNode( 140 ISD::BUILD_PAIR, dl, 141 EVT::getIntegerVT(*DAG.getContext(), LHS.getValueSizeInBits() << 1), 142 LHS, RHS)); 143 } 144 Lo = Vals[Slot++]; 145 Hi = Vals[Slot++]; 146 147 if (DAG.getDataLayout().isBigEndian()) 148 std::swap(Lo, Hi); 149 150 return; 151 } 152 } 153 154 // Lower the bit-convert to a store/load from the stack. 155 assert(NOutVT.isByteSized() && "Expanded type not byte sized!"); 156 157 // Create the stack frame object. Make sure it is aligned for both 158 // the source and expanded destination types. 159 160 // In cases where the vector is illegal it will be broken down into parts 161 // and stored in parts - we should use the alignment for the smallest part. 162 Align InAlign = DAG.getReducedAlign(InVT, /*UseABI=*/false); 163 Align NOutAlign = DAG.getReducedAlign(NOutVT, /*UseABI=*/false); 164 Align Align = std::max(InAlign, NOutAlign); 165 SDValue StackPtr = DAG.CreateStackTemporary(InVT.getStoreSize(), Align); 166 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 167 MachinePointerInfo PtrInfo = 168 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); 169 170 // Emit a store to the stack slot. 171 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo); 172 173 // Load the first half from the stack slot. 174 Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, NOutAlign); 175 176 // Increment the pointer to the other half. 177 unsigned IncrementSize = NOutVT.getSizeInBits() / 8; 178 StackPtr = DAG.getMemBasePlusOffset(StackPtr, IncrementSize, dl); 179 180 // Load the second half from the stack slot. 181 Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr, 182 PtrInfo.getWithOffset(IncrementSize), NOutAlign); 183 184 // Handle endianness of the load. 185 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 186 std::swap(Lo, Hi); 187 } 188 189 void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo, 190 SDValue &Hi) { 191 // Return the operands. 192 Lo = N->getOperand(0); 193 Hi = N->getOperand(1); 194 } 195 196 void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo, 197 SDValue &Hi) { 198 GetExpandedOp(N->getOperand(0), Lo, Hi); 199 SDValue Part = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? 200 Hi : Lo; 201 202 assert(Part.getValueType() == N->getValueType(0) && 203 "Type twice as big as expanded type not itself expanded!"); 204 205 GetPairElements(Part, Lo, Hi); 206 } 207 208 void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, 209 SDValue &Hi) { 210 SDValue OldVec = N->getOperand(0); 211 unsigned OldElts = OldVec.getValueType().getVectorNumElements(); 212 EVT OldEltVT = OldVec.getValueType().getVectorElementType(); 213 SDLoc dl(N); 214 215 // Convert to a vector of the expanded element type, for example 216 // <3 x i64> -> <6 x i32>. 217 EVT OldVT = N->getValueType(0); 218 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); 219 220 if (OldVT != OldEltVT) { 221 // The result of EXTRACT_VECTOR_ELT may be larger than the element type of 222 // the input vector. If so, extend the elements of the input vector to the 223 // same bitwidth as the result before expanding. 224 assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!"); 225 EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldElts); 226 OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0)); 227 } 228 229 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, 230 EVT::getVectorVT(*DAG.getContext(), 231 NewVT, 2*OldElts), 232 OldVec); 233 234 // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector. 235 SDValue Idx = N->getOperand(1); 236 237 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); 238 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); 239 240 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 241 DAG.getConstant(1, dl, Idx.getValueType())); 242 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); 243 244 if (DAG.getDataLayout().isBigEndian()) 245 std::swap(Lo, Hi); 246 } 247 248 void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo, 249 SDValue &Hi) { 250 assert(ISD::isNormalLoad(N) && "This routine only for normal loads!"); 251 SDLoc dl(N); 252 253 LoadSDNode *LD = cast<LoadSDNode>(N); 254 assert(!LD->isAtomic() && "Atomics can not be split"); 255 EVT ValueVT = LD->getValueType(0); 256 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT); 257 SDValue Chain = LD->getChain(); 258 SDValue Ptr = LD->getBasePtr(); 259 AAMDNodes AAInfo = LD->getAAInfo(); 260 261 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 262 263 Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), 264 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), 265 AAInfo); 266 267 // Increment the pointer to the other half. 268 unsigned IncrementSize = NVT.getSizeInBits() / 8; 269 Ptr = DAG.getMemBasePlusOffset(Ptr, IncrementSize, dl); 270 Hi = DAG.getLoad( 271 NVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(IncrementSize), 272 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), AAInfo); 273 274 // Build a factor node to remember that this load is independent of the 275 // other one. 276 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 277 Hi.getValue(1)); 278 279 // Handle endianness of the load. 280 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) 281 std::swap(Lo, Hi); 282 283 // Modified the chain - switch anything that used the old chain to use 284 // the new one. 285 ReplaceValueWith(SDValue(N, 1), Chain); 286 } 287 288 void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { 289 EVT OVT = N->getValueType(0); 290 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 291 SDValue Chain = N->getOperand(0); 292 SDValue Ptr = N->getOperand(1); 293 SDLoc dl(N); 294 const unsigned Align = N->getConstantOperandVal(3); 295 296 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align); 297 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0); 298 Chain = Hi.getValue(1); 299 300 // Handle endianness of the load. 301 if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout())) 302 std::swap(Lo, Hi); 303 304 // Modified the chain - switch anything that used the old chain to use 305 // the new one. 306 ReplaceValueWith(SDValue(N, 1), Chain); 307 } 308 309 310 //===--------------------------------------------------------------------===// 311 // Generic Operand Expansion. 312 //===--------------------------------------------------------------------===// 313 314 void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements, 315 SmallVectorImpl<SDValue> &Ops, 316 EVT EltVT) { 317 assert(Op.getValueType().isInteger()); 318 SDLoc DL(Op); 319 SDValue Parts[2]; 320 321 if (NumElements > 1) { 322 NumElements >>= 1; 323 SplitInteger(Op, Parts[0], Parts[1]); 324 if (DAG.getDataLayout().isBigEndian()) 325 std::swap(Parts[0], Parts[1]); 326 IntegerToVector(Parts[0], NumElements, Ops, EltVT); 327 IntegerToVector(Parts[1], NumElements, Ops, EltVT); 328 } else { 329 Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op)); 330 } 331 } 332 333 SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) { 334 SDLoc dl(N); 335 if (N->getValueType(0).isVector() && 336 N->getOperand(0).getValueType().isInteger()) { 337 // An illegal expanding type is being converted to a legal vector type. 338 // Make a two element vector out of the expanded parts and convert that 339 // instead, but only if the new vector type is legal (otherwise there 340 // is no point, and it might create expansion loops). For example, on 341 // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32. 342 // 343 // FIXME: I'm not sure why we are first trying to split the input into 344 // a 2 element vector, so I'm leaving it here to maintain the current 345 // behavior. 346 unsigned NumElts = 2; 347 EVT OVT = N->getOperand(0).getValueType(); 348 EVT NVT = EVT::getVectorVT(*DAG.getContext(), 349 TLI.getTypeToTransformTo(*DAG.getContext(), OVT), 350 NumElts); 351 if (!isTypeLegal(NVT)) { 352 // If we can't find a legal type by splitting the integer in half, 353 // then we can use the node's value type. 354 NumElts = N->getValueType(0).getVectorNumElements(); 355 NVT = N->getValueType(0); 356 } 357 358 SmallVector<SDValue, 8> Ops; 359 IntegerToVector(N->getOperand(0), NumElts, Ops, NVT.getVectorElementType()); 360 361 SDValue Vec = 362 DAG.getBuildVector(NVT, dl, makeArrayRef(Ops.data(), NumElts)); 363 return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec); 364 } 365 366 // Otherwise, store to a temporary and load out again as the new type. 367 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 368 } 369 370 SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) { 371 // The vector type is legal but the element type needs expansion. 372 EVT VecVT = N->getValueType(0); 373 unsigned NumElts = VecVT.getVectorNumElements(); 374 EVT OldVT = N->getOperand(0).getValueType(); 375 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); 376 SDLoc dl(N); 377 378 assert(OldVT == VecVT.getVectorElementType() && 379 "BUILD_VECTOR operand type doesn't match vector element type!"); 380 381 // Build a vector of twice the length out of the expanded elements. 382 // For example <3 x i64> -> <6 x i32>. 383 SmallVector<SDValue, 16> NewElts; 384 NewElts.reserve(NumElts*2); 385 386 for (unsigned i = 0; i < NumElts; ++i) { 387 SDValue Lo, Hi; 388 GetExpandedOp(N->getOperand(i), Lo, Hi); 389 if (DAG.getDataLayout().isBigEndian()) 390 std::swap(Lo, Hi); 391 NewElts.push_back(Lo); 392 NewElts.push_back(Hi); 393 } 394 395 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size()); 396 SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts); 397 398 // Convert the new vector to the old vector type. 399 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); 400 } 401 402 SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) { 403 SDValue Lo, Hi; 404 GetExpandedOp(N->getOperand(0), Lo, Hi); 405 return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? Hi : Lo; 406 } 407 408 SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) { 409 // The vector type is legal but the element type needs expansion. 410 EVT VecVT = N->getValueType(0); 411 unsigned NumElts = VecVT.getVectorNumElements(); 412 SDLoc dl(N); 413 414 SDValue Val = N->getOperand(1); 415 EVT OldEVT = Val.getValueType(); 416 EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT); 417 418 assert(OldEVT == VecVT.getVectorElementType() && 419 "Inserted element type doesn't match vector element type!"); 420 421 // Bitconvert to a vector of twice the length with elements of the expanded 422 // type, insert the expanded vector elements, and then convert back. 423 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2); 424 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, 425 NewVecVT, N->getOperand(0)); 426 427 SDValue Lo, Hi; 428 GetExpandedOp(Val, Lo, Hi); 429 if (DAG.getDataLayout().isBigEndian()) 430 std::swap(Lo, Hi); 431 432 SDValue Idx = N->getOperand(2); 433 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); 434 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx); 435 Idx = DAG.getNode(ISD::ADD, dl, 436 Idx.getValueType(), Idx, 437 DAG.getConstant(1, dl, Idx.getValueType())); 438 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx); 439 440 // Convert the new vector to the old vector type. 441 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); 442 } 443 444 SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) { 445 SDLoc dl(N); 446 EVT VT = N->getValueType(0); 447 assert(VT.getVectorElementType() == N->getOperand(0).getValueType() && 448 "SCALAR_TO_VECTOR operand type doesn't match vector element type!"); 449 unsigned NumElts = VT.getVectorNumElements(); 450 SmallVector<SDValue, 16> Ops(NumElts); 451 Ops[0] = N->getOperand(0); 452 SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType()); 453 for (unsigned i = 1; i < NumElts; ++i) 454 Ops[i] = UndefVal; 455 return DAG.getBuildVector(VT, dl, Ops); 456 } 457 458 SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) { 459 assert(ISD::isNormalStore(N) && "This routine only for normal stores!"); 460 assert(OpNo == 1 && "Can only expand the stored value so far"); 461 SDLoc dl(N); 462 463 StoreSDNode *St = cast<StoreSDNode>(N); 464 assert(!St->isAtomic() && "Atomics can not be split"); 465 EVT ValueVT = St->getValue().getValueType(); 466 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT); 467 SDValue Chain = St->getChain(); 468 SDValue Ptr = St->getBasePtr(); 469 AAMDNodes AAInfo = St->getAAInfo(); 470 471 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 472 unsigned IncrementSize = NVT.getSizeInBits() / 8; 473 474 SDValue Lo, Hi; 475 GetExpandedOp(St->getValue(), Lo, Hi); 476 477 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) 478 std::swap(Lo, Hi); 479 480 Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(), 481 St->getOriginalAlign(), St->getMemOperand()->getFlags(), 482 AAInfo); 483 484 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize); 485 Hi = DAG.getStore( 486 Chain, dl, Hi, Ptr, St->getPointerInfo().getWithOffset(IncrementSize), 487 St->getOriginalAlign(), St->getMemOperand()->getFlags(), AAInfo); 488 489 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 490 } 491 492 493 //===--------------------------------------------------------------------===// 494 // Generic Result Splitting. 495 //===--------------------------------------------------------------------===// 496 497 // Be careful to make no assumptions about which of Lo/Hi is stored first in 498 // memory (for vectors it is always Lo first followed by Hi in the following 499 // bytes; for integers and floats it is Lo first if and only if the machine is 500 // little-endian). 501 502 void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 503 SDValue &Lo, SDValue &Hi) { 504 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 505 GetSplitOp(Op, Lo, Hi); 506 } 507 508 void DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) { 509 SDValue LL, LH, RL, RH, CL, CH; 510 SDLoc dl(N); 511 GetSplitOp(N->getOperand(1), LL, LH); 512 GetSplitOp(N->getOperand(2), RL, RH); 513 514 SDValue Cond = N->getOperand(0); 515 CL = CH = Cond; 516 if (Cond.getValueType().isVector()) { 517 if (SDValue Res = WidenVSELECTAndMask(N)) 518 std::tie(CL, CH) = DAG.SplitVector(Res->getOperand(0), dl); 519 // Check if there are already splitted versions of the vector available and 520 // use those instead of splitting the mask operand again. 521 else if (getTypeAction(Cond.getValueType()) == 522 TargetLowering::TypeSplitVector) 523 GetSplitVector(Cond, CL, CH); 524 // It seems to improve code to generate two narrow SETCCs as opposed to 525 // splitting a wide result vector. 526 else if (Cond.getOpcode() == ISD::SETCC) { 527 // If the condition is a vXi1 vector, and the LHS of the setcc is a legal 528 // type and the setcc result type is the same vXi1, then leave the setcc 529 // alone. 530 EVT CondLHSVT = Cond.getOperand(0).getValueType(); 531 if (Cond.getValueType().getVectorElementType() == MVT::i1 && 532 isTypeLegal(CondLHSVT) && 533 getSetCCResultType(CondLHSVT) == Cond.getValueType()) 534 std::tie(CL, CH) = DAG.SplitVector(Cond, dl); 535 else 536 SplitVecRes_SETCC(Cond.getNode(), CL, CH); 537 } else 538 std::tie(CL, CH) = DAG.SplitVector(Cond, dl); 539 } 540 541 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL); 542 Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH); 543 } 544 545 void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo, 546 SDValue &Hi) { 547 SDValue LL, LH, RL, RH; 548 SDLoc dl(N); 549 GetSplitOp(N->getOperand(2), LL, LH); 550 GetSplitOp(N->getOperand(3), RL, RH); 551 552 Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0), 553 N->getOperand(1), LL, RL, N->getOperand(4)); 554 Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0), 555 N->getOperand(1), LH, RH, N->getOperand(4)); 556 } 557 558 void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) { 559 EVT LoVT, HiVT; 560 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 561 Lo = DAG.getUNDEF(LoVT); 562 Hi = DAG.getUNDEF(HiVT); 563 } 564 565 void DAGTypeLegalizer::SplitRes_FREEZE(SDNode *N, SDValue &Lo, SDValue &Hi) { 566 SDValue L, H; 567 SDLoc dl(N); 568 GetSplitOp(N->getOperand(0), L, H); 569 570 Lo = DAG.getNode(ISD::FREEZE, dl, L.getValueType(), L); 571 Hi = DAG.getNode(ISD::FREEZE, dl, H.getValueType(), H); 572 } 573