1 //===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements generic type expansion and splitting for LegalizeTypes. 10 // The routines here perform legalization when the details of the type (such as 11 // whether it is an integer or a float) do not matter. 12 // Expansion is the act of changing a computation in an illegal type to be a 13 // computation in two identical registers of a smaller type. The Lo/Hi part 14 // is required to be stored first in memory on little/big-endian machines. 15 // Splitting is the act of changing a computation in an illegal type to be a 16 // computation in two not necessarily identical registers of a smaller type. 17 // There are no requirements on how the type is represented in memory. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/IR/DataLayout.h" 23 using namespace llvm; 24 25 #define DEBUG_TYPE "legalize-types" 26 27 //===----------------------------------------------------------------------===// 28 // Generic Result Expansion. 29 //===----------------------------------------------------------------------===// 30 31 // These routines assume that the Lo/Hi part is stored first in memory on 32 // little/big-endian machines, followed by the Hi/Lo part. This means that 33 // they cannot be used as is on vectors, for which Lo is always stored first. 34 void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 35 SDValue &Lo, SDValue &Hi) { 36 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 37 GetExpandedOp(Op, Lo, Hi); 38 } 39 40 void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) { 41 EVT OutVT = N->getValueType(0); 42 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 43 SDValue InOp = N->getOperand(0); 44 EVT InVT = InOp.getValueType(); 45 SDLoc dl(N); 46 47 // Handle some special cases efficiently. 48 switch (getTypeAction(InVT)) { 49 case TargetLowering::TypeLegal: 50 case TargetLowering::TypePromoteInteger: 51 break; 52 case TargetLowering::TypePromoteFloat: 53 case TargetLowering::TypeSoftPromoteHalf: 54 llvm_unreachable("Bitcast of a promotion-needing float should never need" 55 "expansion"); 56 case TargetLowering::TypeSoftenFloat: 57 SplitInteger(GetSoftenedFloat(InOp), Lo, Hi); 58 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 59 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 60 return; 61 case TargetLowering::TypeExpandInteger: 62 case TargetLowering::TypeExpandFloat: { 63 auto &DL = DAG.getDataLayout(); 64 // Convert the expanded pieces of the input. 65 GetExpandedOp(InOp, Lo, Hi); 66 if (TLI.hasBigEndianPartOrdering(InVT, DL) != 67 TLI.hasBigEndianPartOrdering(OutVT, DL)) 68 std::swap(Lo, Hi); 69 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 70 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 71 return; 72 } 73 case TargetLowering::TypeSplitVector: 74 GetSplitVector(InOp, Lo, Hi); 75 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 76 std::swap(Lo, Hi); 77 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 78 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 79 return; 80 case TargetLowering::TypeScalarizeVector: 81 // Convert the element instead. 82 SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi); 83 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 84 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 85 return; 86 case TargetLowering::TypeScalarizeScalableVector: 87 report_fatal_error("Scalarization of scalable vectors is not supported."); 88 case TargetLowering::TypeWidenVector: { 89 assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST"); 90 InOp = GetWidenedVector(InOp); 91 EVT LoVT, HiVT; 92 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT); 93 std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT); 94 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 95 std::swap(Lo, Hi); 96 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 97 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 98 return; 99 } 100 } 101 102 if (InVT.isVector() && OutVT.isInteger()) { 103 // Handle cases like i64 = BITCAST v1i64 on x86, where the operand 104 // is legal but the result is not. 105 unsigned NumElems = 2; 106 EVT ElemVT = NOutVT; 107 EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems); 108 109 // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>. 110 while (!isTypeLegal(NVT)) { 111 unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2; 112 // If the element size is smaller than byte, bail. 113 if (NewSizeInBits < 8) 114 break; 115 NumElems *= 2; 116 ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits); 117 NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems); 118 } 119 120 if (isTypeLegal(NVT)) { 121 SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp); 122 123 SmallVector<SDValue, 8> Vals; 124 for (unsigned i = 0; i < NumElems; ++i) 125 Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT, 126 CastInOp, DAG.getVectorIdxConstant(i, dl))); 127 128 // Build Lo, Hi pair by pairing extracted elements if needed. 129 unsigned Slot = 0; 130 for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) { 131 // Each iteration will BUILD_PAIR two nodes and append the result until 132 // there are only two nodes left, i.e. Lo and Hi. 133 SDValue LHS = Vals[Slot]; 134 SDValue RHS = Vals[Slot + 1]; 135 136 if (DAG.getDataLayout().isBigEndian()) 137 std::swap(LHS, RHS); 138 139 Vals.push_back(DAG.getNode( 140 ISD::BUILD_PAIR, dl, 141 EVT::getIntegerVT(*DAG.getContext(), LHS.getValueSizeInBits() << 1), 142 LHS, RHS)); 143 } 144 Lo = Vals[Slot++]; 145 Hi = Vals[Slot++]; 146 147 if (DAG.getDataLayout().isBigEndian()) 148 std::swap(Lo, Hi); 149 150 return; 151 } 152 } 153 154 // Lower the bit-convert to a store/load from the stack. 155 assert(NOutVT.isByteSized() && "Expanded type not byte sized!"); 156 157 // Create the stack frame object. Make sure it is aligned for both 158 // the source and expanded destination types. 159 160 // In cases where the vector is illegal it will be broken down into parts 161 // and stored in parts - we should use the alignment for the smallest part. 162 Align InAlign = DAG.getReducedAlign(InVT, /*UseABI=*/false); 163 Align NOutAlign = DAG.getReducedAlign(NOutVT, /*UseABI=*/false); 164 Align Align = std::max(InAlign, NOutAlign); 165 SDValue StackPtr = DAG.CreateStackTemporary(InVT.getStoreSize(), Align); 166 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 167 MachinePointerInfo PtrInfo = 168 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); 169 170 // Emit a store to the stack slot. 171 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo); 172 173 // Load the first half from the stack slot. 174 Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, NOutAlign); 175 176 // Increment the pointer to the other half. 177 unsigned IncrementSize = NOutVT.getSizeInBits() / 8; 178 StackPtr = 179 DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(IncrementSize), dl); 180 181 // Load the second half from the stack slot. 182 Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr, 183 PtrInfo.getWithOffset(IncrementSize), NOutAlign); 184 185 // Handle endianness of the load. 186 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 187 std::swap(Lo, Hi); 188 } 189 190 void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo, 191 SDValue &Hi) { 192 // Return the operands. 193 Lo = N->getOperand(0); 194 Hi = N->getOperand(1); 195 } 196 197 void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo, 198 SDValue &Hi) { 199 GetExpandedOp(N->getOperand(0), Lo, Hi); 200 SDValue Part = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? 201 Hi : Lo; 202 203 assert(Part.getValueType() == N->getValueType(0) && 204 "Type twice as big as expanded type not itself expanded!"); 205 206 GetPairElements(Part, Lo, Hi); 207 } 208 209 void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, 210 SDValue &Hi) { 211 SDValue OldVec = N->getOperand(0); 212 unsigned OldElts = OldVec.getValueType().getVectorNumElements(); 213 EVT OldEltVT = OldVec.getValueType().getVectorElementType(); 214 SDLoc dl(N); 215 216 // Convert to a vector of the expanded element type, for example 217 // <3 x i64> -> <6 x i32>. 218 EVT OldVT = N->getValueType(0); 219 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); 220 221 if (OldVT != OldEltVT) { 222 // The result of EXTRACT_VECTOR_ELT may be larger than the element type of 223 // the input vector. If so, extend the elements of the input vector to the 224 // same bitwidth as the result before expanding. 225 assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!"); 226 EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldElts); 227 OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0)); 228 } 229 230 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, 231 EVT::getVectorVT(*DAG.getContext(), 232 NewVT, 2*OldElts), 233 OldVec); 234 235 // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector. 236 SDValue Idx = N->getOperand(1); 237 238 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); 239 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); 240 241 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 242 DAG.getConstant(1, dl, Idx.getValueType())); 243 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); 244 245 if (DAG.getDataLayout().isBigEndian()) 246 std::swap(Lo, Hi); 247 } 248 249 void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo, 250 SDValue &Hi) { 251 assert(ISD::isNormalLoad(N) && "This routine only for normal loads!"); 252 SDLoc dl(N); 253 254 LoadSDNode *LD = cast<LoadSDNode>(N); 255 assert(!LD->isAtomic() && "Atomics can not be split"); 256 EVT ValueVT = LD->getValueType(0); 257 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT); 258 SDValue Chain = LD->getChain(); 259 SDValue Ptr = LD->getBasePtr(); 260 AAMDNodes AAInfo = LD->getAAInfo(); 261 262 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 263 264 Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), 265 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), 266 AAInfo); 267 268 // Increment the pointer to the other half. 269 unsigned IncrementSize = NVT.getSizeInBits() / 8; 270 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); 271 Hi = DAG.getLoad( 272 NVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(IncrementSize), 273 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), AAInfo); 274 275 // Build a factor node to remember that this load is independent of the 276 // other one. 277 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 278 Hi.getValue(1)); 279 280 // Handle endianness of the load. 281 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) 282 std::swap(Lo, Hi); 283 284 // Modified the chain - switch anything that used the old chain to use 285 // the new one. 286 ReplaceValueWith(SDValue(N, 1), Chain); 287 } 288 289 void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { 290 EVT OVT = N->getValueType(0); 291 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 292 SDValue Chain = N->getOperand(0); 293 SDValue Ptr = N->getOperand(1); 294 SDLoc dl(N); 295 const unsigned Align = N->getConstantOperandVal(3); 296 297 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align); 298 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0); 299 Chain = Hi.getValue(1); 300 301 // Handle endianness of the load. 302 if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout())) 303 std::swap(Lo, Hi); 304 305 // Modified the chain - switch anything that used the old chain to use 306 // the new one. 307 ReplaceValueWith(SDValue(N, 1), Chain); 308 } 309 310 311 //===--------------------------------------------------------------------===// 312 // Generic Operand Expansion. 313 //===--------------------------------------------------------------------===// 314 315 void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements, 316 SmallVectorImpl<SDValue> &Ops, 317 EVT EltVT) { 318 assert(Op.getValueType().isInteger()); 319 SDLoc DL(Op); 320 SDValue Parts[2]; 321 322 if (NumElements > 1) { 323 NumElements >>= 1; 324 SplitInteger(Op, Parts[0], Parts[1]); 325 if (DAG.getDataLayout().isBigEndian()) 326 std::swap(Parts[0], Parts[1]); 327 IntegerToVector(Parts[0], NumElements, Ops, EltVT); 328 IntegerToVector(Parts[1], NumElements, Ops, EltVT); 329 } else { 330 Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op)); 331 } 332 } 333 334 SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) { 335 SDLoc dl(N); 336 if (N->getValueType(0).isVector() && 337 N->getOperand(0).getValueType().isInteger()) { 338 // An illegal expanding type is being converted to a legal vector type. 339 // Make a two element vector out of the expanded parts and convert that 340 // instead, but only if the new vector type is legal (otherwise there 341 // is no point, and it might create expansion loops). For example, on 342 // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32. 343 // 344 // FIXME: I'm not sure why we are first trying to split the input into 345 // a 2 element vector, so I'm leaving it here to maintain the current 346 // behavior. 347 unsigned NumElts = 2; 348 EVT OVT = N->getOperand(0).getValueType(); 349 EVT NVT = EVT::getVectorVT(*DAG.getContext(), 350 TLI.getTypeToTransformTo(*DAG.getContext(), OVT), 351 NumElts); 352 if (!isTypeLegal(NVT)) { 353 // If we can't find a legal type by splitting the integer in half, 354 // then we can use the node's value type. 355 NumElts = N->getValueType(0).getVectorNumElements(); 356 NVT = N->getValueType(0); 357 } 358 359 SmallVector<SDValue, 8> Ops; 360 IntegerToVector(N->getOperand(0), NumElts, Ops, NVT.getVectorElementType()); 361 362 SDValue Vec = 363 DAG.getBuildVector(NVT, dl, makeArrayRef(Ops.data(), NumElts)); 364 return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec); 365 } 366 367 // Otherwise, store to a temporary and load out again as the new type. 368 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 369 } 370 371 SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) { 372 // The vector type is legal but the element type needs expansion. 373 EVT VecVT = N->getValueType(0); 374 unsigned NumElts = VecVT.getVectorNumElements(); 375 EVT OldVT = N->getOperand(0).getValueType(); 376 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); 377 SDLoc dl(N); 378 379 assert(OldVT == VecVT.getVectorElementType() && 380 "BUILD_VECTOR operand type doesn't match vector element type!"); 381 382 // Build a vector of twice the length out of the expanded elements. 383 // For example <3 x i64> -> <6 x i32>. 384 SmallVector<SDValue, 16> NewElts; 385 NewElts.reserve(NumElts*2); 386 387 for (unsigned i = 0; i < NumElts; ++i) { 388 SDValue Lo, Hi; 389 GetExpandedOp(N->getOperand(i), Lo, Hi); 390 if (DAG.getDataLayout().isBigEndian()) 391 std::swap(Lo, Hi); 392 NewElts.push_back(Lo); 393 NewElts.push_back(Hi); 394 } 395 396 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size()); 397 SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts); 398 399 // Convert the new vector to the old vector type. 400 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); 401 } 402 403 SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) { 404 SDValue Lo, Hi; 405 GetExpandedOp(N->getOperand(0), Lo, Hi); 406 return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? Hi : Lo; 407 } 408 409 SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) { 410 // The vector type is legal but the element type needs expansion. 411 EVT VecVT = N->getValueType(0); 412 unsigned NumElts = VecVT.getVectorNumElements(); 413 SDLoc dl(N); 414 415 SDValue Val = N->getOperand(1); 416 EVT OldEVT = Val.getValueType(); 417 EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT); 418 419 assert(OldEVT == VecVT.getVectorElementType() && 420 "Inserted element type doesn't match vector element type!"); 421 422 // Bitconvert to a vector of twice the length with elements of the expanded 423 // type, insert the expanded vector elements, and then convert back. 424 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2); 425 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, 426 NewVecVT, N->getOperand(0)); 427 428 SDValue Lo, Hi; 429 GetExpandedOp(Val, Lo, Hi); 430 if (DAG.getDataLayout().isBigEndian()) 431 std::swap(Lo, Hi); 432 433 SDValue Idx = N->getOperand(2); 434 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); 435 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx); 436 Idx = DAG.getNode(ISD::ADD, dl, 437 Idx.getValueType(), Idx, 438 DAG.getConstant(1, dl, Idx.getValueType())); 439 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx); 440 441 // Convert the new vector to the old vector type. 442 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); 443 } 444 445 SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) { 446 SDLoc dl(N); 447 EVT VT = N->getValueType(0); 448 assert(VT.getVectorElementType() == N->getOperand(0).getValueType() && 449 "SCALAR_TO_VECTOR operand type doesn't match vector element type!"); 450 unsigned NumElts = VT.getVectorNumElements(); 451 SmallVector<SDValue, 16> Ops(NumElts); 452 Ops[0] = N->getOperand(0); 453 SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType()); 454 for (unsigned i = 1; i < NumElts; ++i) 455 Ops[i] = UndefVal; 456 return DAG.getBuildVector(VT, dl, Ops); 457 } 458 459 SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) { 460 assert(ISD::isNormalStore(N) && "This routine only for normal stores!"); 461 assert(OpNo == 1 && "Can only expand the stored value so far"); 462 SDLoc dl(N); 463 464 StoreSDNode *St = cast<StoreSDNode>(N); 465 assert(!St->isAtomic() && "Atomics can not be split"); 466 EVT ValueVT = St->getValue().getValueType(); 467 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT); 468 SDValue Chain = St->getChain(); 469 SDValue Ptr = St->getBasePtr(); 470 AAMDNodes AAInfo = St->getAAInfo(); 471 472 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 473 unsigned IncrementSize = NVT.getSizeInBits() / 8; 474 475 SDValue Lo, Hi; 476 GetExpandedOp(St->getValue(), Lo, Hi); 477 478 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) 479 std::swap(Lo, Hi); 480 481 Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(), 482 St->getOriginalAlign(), St->getMemOperand()->getFlags(), 483 AAInfo); 484 485 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); 486 Hi = DAG.getStore( 487 Chain, dl, Hi, Ptr, St->getPointerInfo().getWithOffset(IncrementSize), 488 St->getOriginalAlign(), St->getMemOperand()->getFlags(), AAInfo); 489 490 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 491 } 492 493 494 //===--------------------------------------------------------------------===// 495 // Generic Result Splitting. 496 //===--------------------------------------------------------------------===// 497 498 // Be careful to make no assumptions about which of Lo/Hi is stored first in 499 // memory (for vectors it is always Lo first followed by Hi in the following 500 // bytes; for integers and floats it is Lo first if and only if the machine is 501 // little-endian). 502 503 void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 504 SDValue &Lo, SDValue &Hi) { 505 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 506 GetSplitOp(Op, Lo, Hi); 507 } 508 509 void DAGTypeLegalizer::SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi) { 510 SDValue LL, LH, RL, RH, CL, CH; 511 SDLoc dl(N); 512 unsigned Opcode = N->getOpcode(); 513 GetSplitOp(N->getOperand(1), LL, LH); 514 GetSplitOp(N->getOperand(2), RL, RH); 515 516 SDValue Cond = N->getOperand(0); 517 CL = CH = Cond; 518 if (Cond.getValueType().isVector()) { 519 if (SDValue Res = WidenVSELECTMask(N)) 520 std::tie(CL, CH) = DAG.SplitVector(Res, dl); 521 // Check if there are already splitted versions of the vector available and 522 // use those instead of splitting the mask operand again. 523 else if (getTypeAction(Cond.getValueType()) == 524 TargetLowering::TypeSplitVector) 525 GetSplitVector(Cond, CL, CH); 526 // It seems to improve code to generate two narrow SETCCs as opposed to 527 // splitting a wide result vector. 528 else if (Cond.getOpcode() == ISD::SETCC) { 529 // If the condition is a vXi1 vector, and the LHS of the setcc is a legal 530 // type and the setcc result type is the same vXi1, then leave the setcc 531 // alone. 532 EVT CondLHSVT = Cond.getOperand(0).getValueType(); 533 if (Cond.getValueType().getVectorElementType() == MVT::i1 && 534 isTypeLegal(CondLHSVT) && 535 getSetCCResultType(CondLHSVT) == Cond.getValueType()) 536 std::tie(CL, CH) = DAG.SplitVector(Cond, dl); 537 else 538 SplitVecRes_SETCC(Cond.getNode(), CL, CH); 539 } else 540 std::tie(CL, CH) = DAG.SplitVector(Cond, dl); 541 } 542 543 if (Opcode != ISD::VP_SELECT && Opcode != ISD::VP_MERGE) { 544 Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL); 545 Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH); 546 return; 547 } 548 549 SDValue EVLLo, EVLHi; 550 std::tie(EVLLo, EVLHi) = 551 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl); 552 553 Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL, EVLLo); 554 Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH, EVLHi); 555 } 556 557 void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo, 558 SDValue &Hi) { 559 SDValue LL, LH, RL, RH; 560 SDLoc dl(N); 561 GetSplitOp(N->getOperand(2), LL, LH); 562 GetSplitOp(N->getOperand(3), RL, RH); 563 564 Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0), 565 N->getOperand(1), LL, RL, N->getOperand(4)); 566 Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0), 567 N->getOperand(1), LH, RH, N->getOperand(4)); 568 } 569 570 void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) { 571 EVT LoVT, HiVT; 572 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 573 Lo = DAG.getUNDEF(LoVT); 574 Hi = DAG.getUNDEF(HiVT); 575 } 576 577 void DAGTypeLegalizer::SplitRes_FREEZE(SDNode *N, SDValue &Lo, SDValue &Hi) { 578 SDValue L, H; 579 SDLoc dl(N); 580 GetSplitOp(N->getOperand(0), L, H); 581 582 Lo = DAG.getNode(ISD::FREEZE, dl, L.getValueType(), L); 583 Hi = DAG.getNode(ISD::FREEZE, dl, H.getValueType(), H); 584 } 585 586 void DAGTypeLegalizer::SplitRes_ARITH_FENCE(SDNode *N, SDValue &Lo, 587 SDValue &Hi) { 588 SDValue L, H; 589 SDLoc DL(N); 590 GetSplitOp(N->getOperand(0), L, H); 591 592 Lo = DAG.getNode(ISD::ARITH_FENCE, DL, L.getValueType(), L); 593 Hi = DAG.getNode(ISD::ARITH_FENCE, DL, H.getValueType(), H); 594 } 595