1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file performs vector type splitting and scalarization for LegalizeTypes. 10 // Scalarization is the act of changing a computation in an illegal one-element 11 // vector type to be a computation in its scalar element type. For example, 12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which 14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 15 // types. 16 // Splitting is the act of changing a computation in an invalid vector type to 17 // be a computation in two vectors of half the size. For example, implementing 18 // <128 x f32> operations in terms of two <64 x f32> operations. 19 // 20 //===----------------------------------------------------------------------===// 21 22 #include "LegalizeTypes.h" 23 #include "llvm/ADT/SmallBitVector.h" 24 #include "llvm/Analysis/MemoryLocation.h" 25 #include "llvm/Analysis/VectorUtils.h" 26 #include "llvm/IR/DataLayout.h" 27 #include "llvm/Support/ErrorHandling.h" 28 #include "llvm/Support/TypeSize.h" 29 #include "llvm/Support/raw_ostream.h" 30 using namespace llvm; 31 32 #define DEBUG_TYPE "legalize-types" 33 34 //===----------------------------------------------------------------------===// 35 // Result Vector Scalarization: <1 x ty> -> ty. 36 //===----------------------------------------------------------------------===// 37 38 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 39 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 40 dbgs() << "\n"); 41 SDValue R = SDValue(); 42 43 switch (N->getOpcode()) { 44 default: 45 #ifndef NDEBUG 46 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 47 N->dump(&DAG); 48 dbgs() << "\n"; 49 #endif 50 report_fatal_error("Do not know how to scalarize the result of this " 51 "operator!\n"); 52 53 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 54 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 55 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 56 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 57 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 58 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 59 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 60 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 61 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 62 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 63 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 64 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 65 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 66 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 67 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 68 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 69 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break; 70 case ISD::ANY_EXTEND_VECTOR_INREG: 71 case ISD::SIGN_EXTEND_VECTOR_INREG: 72 case ISD::ZERO_EXTEND_VECTOR_INREG: 73 R = ScalarizeVecRes_VecInregOp(N); 74 break; 75 case ISD::ABS: 76 case ISD::ANY_EXTEND: 77 case ISD::BITREVERSE: 78 case ISD::BSWAP: 79 case ISD::CTLZ: 80 case ISD::CTLZ_ZERO_UNDEF: 81 case ISD::CTPOP: 82 case ISD::CTTZ: 83 case ISD::CTTZ_ZERO_UNDEF: 84 case ISD::FABS: 85 case ISD::FCEIL: 86 case ISD::FCOS: 87 case ISD::FEXP: 88 case ISD::FEXP2: 89 case ISD::FFLOOR: 90 case ISD::FLOG: 91 case ISD::FLOG10: 92 case ISD::FLOG2: 93 case ISD::FNEARBYINT: 94 case ISD::FNEG: 95 case ISD::FREEZE: 96 case ISD::ARITH_FENCE: 97 case ISD::FP_EXTEND: 98 case ISD::FP_TO_SINT: 99 case ISD::FP_TO_UINT: 100 case ISD::FRINT: 101 case ISD::FROUND: 102 case ISD::FROUNDEVEN: 103 case ISD::FSIN: 104 case ISD::FSQRT: 105 case ISD::FTRUNC: 106 case ISD::SIGN_EXTEND: 107 case ISD::SINT_TO_FP: 108 case ISD::TRUNCATE: 109 case ISD::UINT_TO_FP: 110 case ISD::ZERO_EXTEND: 111 case ISD::FCANONICALIZE: 112 R = ScalarizeVecRes_UnaryOp(N); 113 break; 114 115 case ISD::ADD: 116 case ISD::AND: 117 case ISD::FADD: 118 case ISD::FCOPYSIGN: 119 case ISD::FDIV: 120 case ISD::FMUL: 121 case ISD::FMINNUM: 122 case ISD::FMAXNUM: 123 case ISD::FMINNUM_IEEE: 124 case ISD::FMAXNUM_IEEE: 125 case ISD::FMINIMUM: 126 case ISD::FMAXIMUM: 127 case ISD::SMIN: 128 case ISD::SMAX: 129 case ISD::UMIN: 130 case ISD::UMAX: 131 132 case ISD::SADDSAT: 133 case ISD::UADDSAT: 134 case ISD::SSUBSAT: 135 case ISD::USUBSAT: 136 case ISD::SSHLSAT: 137 case ISD::USHLSAT: 138 139 case ISD::FPOW: 140 case ISD::FREM: 141 case ISD::FSUB: 142 case ISD::MUL: 143 case ISD::OR: 144 case ISD::SDIV: 145 case ISD::SREM: 146 case ISD::SUB: 147 case ISD::UDIV: 148 case ISD::UREM: 149 case ISD::XOR: 150 case ISD::SHL: 151 case ISD::SRA: 152 case ISD::SRL: 153 case ISD::ROTL: 154 case ISD::ROTR: 155 R = ScalarizeVecRes_BinOp(N); 156 break; 157 case ISD::FMA: 158 case ISD::FSHL: 159 case ISD::FSHR: 160 R = ScalarizeVecRes_TernaryOp(N); 161 break; 162 163 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 164 case ISD::STRICT_##DAGN: 165 #include "llvm/IR/ConstrainedOps.def" 166 R = ScalarizeVecRes_StrictFPOp(N); 167 break; 168 169 case ISD::FP_TO_UINT_SAT: 170 case ISD::FP_TO_SINT_SAT: 171 R = ScalarizeVecRes_FP_TO_XINT_SAT(N); 172 break; 173 174 case ISD::UADDO: 175 case ISD::SADDO: 176 case ISD::USUBO: 177 case ISD::SSUBO: 178 case ISD::UMULO: 179 case ISD::SMULO: 180 R = ScalarizeVecRes_OverflowOp(N, ResNo); 181 break; 182 case ISD::SMULFIX: 183 case ISD::SMULFIXSAT: 184 case ISD::UMULFIX: 185 case ISD::UMULFIXSAT: 186 case ISD::SDIVFIX: 187 case ISD::SDIVFIXSAT: 188 case ISD::UDIVFIX: 189 case ISD::UDIVFIXSAT: 190 R = ScalarizeVecRes_FIX(N); 191 break; 192 } 193 194 // If R is null, the sub-method took care of registering the result. 195 if (R.getNode()) 196 SetScalarizedVector(SDValue(N, ResNo), R); 197 } 198 199 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 200 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 201 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 202 return DAG.getNode(N->getOpcode(), SDLoc(N), 203 LHS.getValueType(), LHS, RHS, N->getFlags()); 204 } 205 206 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 207 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 208 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 209 SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 210 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1, 211 Op2, N->getFlags()); 212 } 213 214 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) { 215 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 216 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 217 SDValue Op2 = N->getOperand(2); 218 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1, 219 Op2, N->getFlags()); 220 } 221 222 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) { 223 EVT VT = N->getValueType(0).getVectorElementType(); 224 unsigned NumOpers = N->getNumOperands(); 225 SDValue Chain = N->getOperand(0); 226 EVT ValueVTs[] = {VT, MVT::Other}; 227 SDLoc dl(N); 228 229 SmallVector<SDValue, 4> Opers(NumOpers); 230 231 // The Chain is the first operand. 232 Opers[0] = Chain; 233 234 // Now process the remaining operands. 235 for (unsigned i = 1; i < NumOpers; ++i) { 236 SDValue Oper = N->getOperand(i); 237 EVT OperVT = Oper.getValueType(); 238 239 if (OperVT.isVector()) { 240 if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector) 241 Oper = GetScalarizedVector(Oper); 242 else 243 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 244 OperVT.getVectorElementType(), Oper, 245 DAG.getVectorIdxConstant(0, dl)); 246 } 247 248 Opers[i] = Oper; 249 } 250 251 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs), 252 Opers, N->getFlags()); 253 254 // Legalize the chain result - switch anything that used the old chain to 255 // use the new one. 256 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 257 return Result; 258 } 259 260 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N, 261 unsigned ResNo) { 262 SDLoc DL(N); 263 EVT ResVT = N->getValueType(0); 264 EVT OvVT = N->getValueType(1); 265 266 SDValue ScalarLHS, ScalarRHS; 267 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) { 268 ScalarLHS = GetScalarizedVector(N->getOperand(0)); 269 ScalarRHS = GetScalarizedVector(N->getOperand(1)); 270 } else { 271 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS; 272 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS); 273 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS); 274 ScalarLHS = ElemsLHS[0]; 275 ScalarRHS = ElemsRHS[0]; 276 } 277 278 SDVTList ScalarVTs = DAG.getVTList( 279 ResVT.getVectorElementType(), OvVT.getVectorElementType()); 280 SDNode *ScalarNode = DAG.getNode( 281 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode(); 282 ScalarNode->setFlags(N->getFlags()); 283 284 // Replace the other vector result not being explicitly scalarized here. 285 unsigned OtherNo = 1 - ResNo; 286 EVT OtherVT = N->getValueType(OtherNo); 287 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) { 288 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo)); 289 } else { 290 SDValue OtherVal = DAG.getNode( 291 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo)); 292 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 293 } 294 295 return SDValue(ScalarNode, ResNo); 296 } 297 298 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 299 unsigned ResNo) { 300 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 301 return GetScalarizedVector(Op); 302 } 303 304 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 305 SDValue Op = N->getOperand(0); 306 if (Op.getValueType().isVector() 307 && Op.getValueType().getVectorNumElements() == 1 308 && !isSimpleLegalType(Op.getValueType())) 309 Op = GetScalarizedVector(Op); 310 EVT NewVT = N->getValueType(0).getVectorElementType(); 311 return DAG.getNode(ISD::BITCAST, SDLoc(N), 312 NewVT, Op); 313 } 314 315 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 316 EVT EltVT = N->getValueType(0).getVectorElementType(); 317 SDValue InOp = N->getOperand(0); 318 // The BUILD_VECTOR operands may be of wider element types and 319 // we may need to truncate them back to the requested return type. 320 if (EltVT.isInteger()) 321 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 322 return InOp; 323 } 324 325 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 326 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 327 N->getValueType(0).getVectorElementType(), 328 N->getOperand(0), N->getOperand(1)); 329 } 330 331 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 332 SDLoc DL(N); 333 SDValue Op = N->getOperand(0); 334 EVT OpVT = Op.getValueType(); 335 // The result needs scalarizing, but it's not a given that the source does. 336 // See similar logic in ScalarizeVecRes_UnaryOp. 337 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 338 Op = GetScalarizedVector(Op); 339 } else { 340 EVT VT = OpVT.getVectorElementType(); 341 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, 342 DAG.getVectorIdxConstant(0, DL)); 343 } 344 return DAG.getNode(ISD::FP_ROUND, DL, 345 N->getValueType(0).getVectorElementType(), Op, 346 N->getOperand(1)); 347 } 348 349 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 350 SDValue Op = GetScalarizedVector(N->getOperand(0)); 351 return DAG.getNode(ISD::FPOWI, SDLoc(N), 352 Op.getValueType(), Op, N->getOperand(1)); 353 } 354 355 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 356 // The value to insert may have a wider type than the vector element type, 357 // so be sure to truncate it to the element type if necessary. 358 SDValue Op = N->getOperand(1); 359 EVT EltVT = N->getValueType(0).getVectorElementType(); 360 if (Op.getValueType() != EltVT) 361 // FIXME: Can this happen for floating point types? 362 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op); 363 return Op; 364 } 365 366 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 367 assert(N->isUnindexed() && "Indexed vector load?"); 368 369 SDValue Result = DAG.getLoad( 370 ISD::UNINDEXED, N->getExtensionType(), 371 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(), 372 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()), 373 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(), 374 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo()); 375 376 // Legalize the chain result - switch anything that used the old chain to 377 // use the new one. 378 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 379 return Result; 380 } 381 382 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 383 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 384 EVT DestVT = N->getValueType(0).getVectorElementType(); 385 SDValue Op = N->getOperand(0); 386 EVT OpVT = Op.getValueType(); 387 SDLoc DL(N); 388 // The result needs scalarizing, but it's not a given that the source does. 389 // This is a workaround for targets where it's impossible to scalarize the 390 // result of a conversion, because the source type is legal. 391 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32} 392 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is 393 // legal and was not scalarized. 394 // See the similar logic in ScalarizeVecRes_SETCC 395 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 396 Op = GetScalarizedVector(Op); 397 } else { 398 EVT VT = OpVT.getVectorElementType(); 399 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, 400 DAG.getVectorIdxConstant(0, DL)); 401 } 402 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags()); 403 } 404 405 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 406 EVT EltVT = N->getValueType(0).getVectorElementType(); 407 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 408 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 409 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT, 410 LHS, DAG.getValueType(ExtVT)); 411 } 412 413 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) { 414 SDLoc DL(N); 415 SDValue Op = N->getOperand(0); 416 417 EVT OpVT = Op.getValueType(); 418 EVT OpEltVT = OpVT.getVectorElementType(); 419 EVT EltVT = N->getValueType(0).getVectorElementType(); 420 421 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 422 Op = GetScalarizedVector(Op); 423 } else { 424 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op, 425 DAG.getVectorIdxConstant(0, DL)); 426 } 427 428 switch (N->getOpcode()) { 429 case ISD::ANY_EXTEND_VECTOR_INREG: 430 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op); 431 case ISD::SIGN_EXTEND_VECTOR_INREG: 432 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op); 433 case ISD::ZERO_EXTEND_VECTOR_INREG: 434 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op); 435 } 436 437 llvm_unreachable("Illegal extend_vector_inreg opcode"); 438 } 439 440 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 441 // If the operand is wider than the vector element type then it is implicitly 442 // truncated. Make that explicit here. 443 EVT EltVT = N->getValueType(0).getVectorElementType(); 444 SDValue InOp = N->getOperand(0); 445 if (InOp.getValueType() != EltVT) 446 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 447 return InOp; 448 } 449 450 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 451 SDValue Cond = N->getOperand(0); 452 EVT OpVT = Cond.getValueType(); 453 SDLoc DL(N); 454 // The vselect result and true/value operands needs scalarizing, but it's 455 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal. 456 // See the similar logic in ScalarizeVecRes_SETCC 457 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 458 Cond = GetScalarizedVector(Cond); 459 } else { 460 EVT VT = OpVT.getVectorElementType(); 461 Cond = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond, 462 DAG.getVectorIdxConstant(0, DL)); 463 } 464 465 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 466 TargetLowering::BooleanContent ScalarBool = 467 TLI.getBooleanContents(false, false); 468 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false); 469 470 // If integer and float booleans have different contents then we can't 471 // reliably optimize in all cases. There is a full explanation for this in 472 // DAGCombiner::visitSELECT() where the same issue affects folding 473 // (select C, 0, 1) to (xor C, 1). 474 if (TLI.getBooleanContents(false, false) != 475 TLI.getBooleanContents(false, true)) { 476 // At least try the common case where the boolean is generated by a 477 // comparison. 478 if (Cond->getOpcode() == ISD::SETCC) { 479 EVT OpVT = Cond->getOperand(0).getValueType(); 480 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType()); 481 VecBool = TLI.getBooleanContents(OpVT); 482 } else 483 ScalarBool = TargetLowering::UndefinedBooleanContent; 484 } 485 486 EVT CondVT = Cond.getValueType(); 487 if (ScalarBool != VecBool) { 488 switch (ScalarBool) { 489 case TargetLowering::UndefinedBooleanContent: 490 break; 491 case TargetLowering::ZeroOrOneBooleanContent: 492 assert(VecBool == TargetLowering::UndefinedBooleanContent || 493 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 494 // Vector read from all ones, scalar expects a single 1 so mask. 495 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT, 496 Cond, DAG.getConstant(1, SDLoc(N), CondVT)); 497 break; 498 case TargetLowering::ZeroOrNegativeOneBooleanContent: 499 assert(VecBool == TargetLowering::UndefinedBooleanContent || 500 VecBool == TargetLowering::ZeroOrOneBooleanContent); 501 // Vector reads from a one, scalar from all ones so sign extend. 502 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT, 503 Cond, DAG.getValueType(MVT::i1)); 504 break; 505 } 506 } 507 508 // Truncate the condition if needed 509 auto BoolVT = getSetCCResultType(CondVT); 510 if (BoolVT.bitsLT(CondVT)) 511 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond); 512 513 return DAG.getSelect(SDLoc(N), 514 LHS.getValueType(), Cond, LHS, 515 GetScalarizedVector(N->getOperand(2))); 516 } 517 518 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 519 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 520 return DAG.getSelect(SDLoc(N), 521 LHS.getValueType(), N->getOperand(0), LHS, 522 GetScalarizedVector(N->getOperand(2))); 523 } 524 525 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 526 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 527 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(), 528 N->getOperand(0), N->getOperand(1), 529 LHS, GetScalarizedVector(N->getOperand(3)), 530 N->getOperand(4)); 531 } 532 533 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 534 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 535 } 536 537 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 538 // Figure out if the scalar is the LHS or RHS and return it. 539 SDValue Arg = N->getOperand(2).getOperand(0); 540 if (Arg.isUndef()) 541 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 542 unsigned Op = !cast<ConstantSDNode>(Arg)->isZero(); 543 return GetScalarizedVector(N->getOperand(Op)); 544 } 545 546 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) { 547 SDValue Src = N->getOperand(0); 548 EVT SrcVT = Src.getValueType(); 549 SDLoc dl(N); 550 551 // Handle case where result is scalarized but operand is not 552 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector) 553 Src = GetScalarizedVector(Src); 554 else 555 Src = DAG.getNode( 556 ISD::EXTRACT_VECTOR_ELT, dl, SrcVT.getVectorElementType(), Src, 557 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 558 559 EVT DstVT = N->getValueType(0).getVectorElementType(); 560 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1)); 561 } 562 563 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 564 assert(N->getValueType(0).isVector() && 565 N->getOperand(0).getValueType().isVector() && 566 "Operand types must be vectors"); 567 SDValue LHS = N->getOperand(0); 568 SDValue RHS = N->getOperand(1); 569 EVT OpVT = LHS.getValueType(); 570 EVT NVT = N->getValueType(0).getVectorElementType(); 571 SDLoc DL(N); 572 573 // The result needs scalarizing, but it's not a given that the source does. 574 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 575 LHS = GetScalarizedVector(LHS); 576 RHS = GetScalarizedVector(RHS); 577 } else { 578 EVT VT = OpVT.getVectorElementType(); 579 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS, 580 DAG.getVectorIdxConstant(0, DL)); 581 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS, 582 DAG.getVectorIdxConstant(0, DL)); 583 } 584 585 // Turn it into a scalar SETCC. 586 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 587 N->getOperand(2)); 588 // Vectors may have a different boolean contents to scalars. Promote the 589 // value appropriately. 590 ISD::NodeType ExtendCode = 591 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 592 return DAG.getNode(ExtendCode, DL, NVT, Res); 593 } 594 595 SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) { 596 SDLoc DL(N); 597 SDValue Arg = N->getOperand(0); 598 SDValue Test = N->getOperand(1); 599 EVT ArgVT = Arg.getValueType(); 600 EVT ResultVT = N->getValueType(0).getVectorElementType(); 601 602 if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) { 603 Arg = GetScalarizedVector(Arg); 604 } else { 605 EVT VT = ArgVT.getVectorElementType(); 606 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Arg, 607 DAG.getVectorIdxConstant(0, DL)); 608 } 609 610 SDValue Res = 611 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags()); 612 // Vectors may have a different boolean contents to scalars. Promote the 613 // value appropriately. 614 ISD::NodeType ExtendCode = 615 TargetLowering::getExtendForContent(TLI.getBooleanContents(ArgVT)); 616 return DAG.getNode(ExtendCode, DL, ResultVT, Res); 617 } 618 619 //===----------------------------------------------------------------------===// 620 // Operand Vector Scalarization <1 x ty> -> ty. 621 //===----------------------------------------------------------------------===// 622 623 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 624 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 625 dbgs() << "\n"); 626 SDValue Res = SDValue(); 627 628 switch (N->getOpcode()) { 629 default: 630 #ifndef NDEBUG 631 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 632 N->dump(&DAG); 633 dbgs() << "\n"; 634 #endif 635 report_fatal_error("Do not know how to scalarize this operator's " 636 "operand!\n"); 637 case ISD::BITCAST: 638 Res = ScalarizeVecOp_BITCAST(N); 639 break; 640 case ISD::ANY_EXTEND: 641 case ISD::ZERO_EXTEND: 642 case ISD::SIGN_EXTEND: 643 case ISD::TRUNCATE: 644 case ISD::FP_TO_SINT: 645 case ISD::FP_TO_UINT: 646 case ISD::SINT_TO_FP: 647 case ISD::UINT_TO_FP: 648 Res = ScalarizeVecOp_UnaryOp(N); 649 break; 650 case ISD::STRICT_SINT_TO_FP: 651 case ISD::STRICT_UINT_TO_FP: 652 case ISD::STRICT_FP_TO_SINT: 653 case ISD::STRICT_FP_TO_UINT: 654 Res = ScalarizeVecOp_UnaryOp_StrictFP(N); 655 break; 656 case ISD::CONCAT_VECTORS: 657 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 658 break; 659 case ISD::EXTRACT_VECTOR_ELT: 660 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 661 break; 662 case ISD::VSELECT: 663 Res = ScalarizeVecOp_VSELECT(N); 664 break; 665 case ISD::SETCC: 666 Res = ScalarizeVecOp_VSETCC(N); 667 break; 668 case ISD::STORE: 669 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 670 break; 671 case ISD::STRICT_FP_ROUND: 672 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo); 673 break; 674 case ISD::FP_ROUND: 675 Res = ScalarizeVecOp_FP_ROUND(N, OpNo); 676 break; 677 case ISD::STRICT_FP_EXTEND: 678 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N); 679 break; 680 case ISD::FP_EXTEND: 681 Res = ScalarizeVecOp_FP_EXTEND(N); 682 break; 683 case ISD::VECREDUCE_FADD: 684 case ISD::VECREDUCE_FMUL: 685 case ISD::VECREDUCE_ADD: 686 case ISD::VECREDUCE_MUL: 687 case ISD::VECREDUCE_AND: 688 case ISD::VECREDUCE_OR: 689 case ISD::VECREDUCE_XOR: 690 case ISD::VECREDUCE_SMAX: 691 case ISD::VECREDUCE_SMIN: 692 case ISD::VECREDUCE_UMAX: 693 case ISD::VECREDUCE_UMIN: 694 case ISD::VECREDUCE_FMAX: 695 case ISD::VECREDUCE_FMIN: 696 Res = ScalarizeVecOp_VECREDUCE(N); 697 break; 698 case ISD::VECREDUCE_SEQ_FADD: 699 case ISD::VECREDUCE_SEQ_FMUL: 700 Res = ScalarizeVecOp_VECREDUCE_SEQ(N); 701 break; 702 } 703 704 // If the result is null, the sub-method took care of registering results etc. 705 if (!Res.getNode()) return false; 706 707 // If the result is N, the sub-method updated N in place. Tell the legalizer 708 // core about this. 709 if (Res.getNode() == N) 710 return true; 711 712 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 713 "Invalid operand expansion"); 714 715 ReplaceValueWith(SDValue(N, 0), Res); 716 return false; 717 } 718 719 /// If the value to convert is a vector that needs to be scalarized, it must be 720 /// <1 x ty>. Convert the element instead. 721 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 722 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 723 return DAG.getNode(ISD::BITCAST, SDLoc(N), 724 N->getValueType(0), Elt); 725 } 726 727 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>. 728 /// Do the operation on the element instead. 729 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) { 730 assert(N->getValueType(0).getVectorNumElements() == 1 && 731 "Unexpected vector type!"); 732 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 733 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N), 734 N->getValueType(0).getScalarType(), Elt); 735 // Revectorize the result so the types line up with what the uses of this 736 // expression expect. 737 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op); 738 } 739 740 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>. 741 /// Do the strict FP operation on the element instead. 742 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) { 743 assert(N->getValueType(0).getVectorNumElements() == 1 && 744 "Unexpected vector type!"); 745 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 746 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), 747 { N->getValueType(0).getScalarType(), MVT::Other }, 748 { N->getOperand(0), Elt }); 749 // Legalize the chain result - switch anything that used the old chain to 750 // use the new one. 751 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 752 // Revectorize the result so the types line up with what the uses of this 753 // expression expect. 754 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 755 756 // Do our own replacement and return SDValue() to tell the caller that we 757 // handled all replacements since caller can only handle a single result. 758 ReplaceValueWith(SDValue(N, 0), Res); 759 return SDValue(); 760 } 761 762 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead. 763 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 764 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 765 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 766 Ops[i] = GetScalarizedVector(N->getOperand(i)); 767 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops); 768 } 769 770 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>, 771 /// so just return the element, ignoring the index. 772 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 773 EVT VT = N->getValueType(0); 774 SDValue Res = GetScalarizedVector(N->getOperand(0)); 775 if (Res.getValueType() != VT) 776 Res = VT.isFloatingPoint() 777 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res) 778 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res); 779 return Res; 780 } 781 782 /// If the input condition is a vector that needs to be scalarized, it must be 783 /// <1 x i1>, so just convert to a normal ISD::SELECT 784 /// (still with vector output type since that was acceptable if we got here). 785 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) { 786 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0)); 787 EVT VT = N->getValueType(0); 788 789 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1), 790 N->getOperand(2)); 791 } 792 793 /// If the operand is a vector that needs to be scalarized then the 794 /// result must be v1i1, so just convert to a scalar SETCC and wrap 795 /// with a scalar_to_vector since the res type is legal if we got here 796 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) { 797 assert(N->getValueType(0).isVector() && 798 N->getOperand(0).getValueType().isVector() && 799 "Operand types must be vectors"); 800 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type"); 801 802 EVT VT = N->getValueType(0); 803 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 804 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 805 806 EVT OpVT = N->getOperand(0).getValueType(); 807 EVT NVT = VT.getVectorElementType(); 808 SDLoc DL(N); 809 // Turn it into a scalar SETCC. 810 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 811 N->getOperand(2)); 812 813 // Vectors may have a different boolean contents to scalars. Promote the 814 // value appropriately. 815 ISD::NodeType ExtendCode = 816 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 817 818 Res = DAG.getNode(ExtendCode, DL, NVT, Res); 819 820 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res); 821 } 822 823 /// If the value to store is a vector that needs to be scalarized, it must be 824 /// <1 x ty>. Just store the element. 825 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 826 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 827 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 828 SDLoc dl(N); 829 830 if (N->isTruncatingStore()) 831 return DAG.getTruncStore( 832 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 833 N->getBasePtr(), N->getPointerInfo(), 834 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(), 835 N->getMemOperand()->getFlags(), N->getAAInfo()); 836 837 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 838 N->getBasePtr(), N->getPointerInfo(), 839 N->getOriginalAlign(), N->getMemOperand()->getFlags(), 840 N->getAAInfo()); 841 } 842 843 /// If the value to round is a vector that needs to be scalarized, it must be 844 /// <1 x ty>. Convert the element instead. 845 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { 846 assert(OpNo == 0 && "Wrong operand for scalarization!"); 847 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 848 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N), 849 N->getValueType(0).getVectorElementType(), Elt, 850 N->getOperand(1)); 851 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 852 } 853 854 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, 855 unsigned OpNo) { 856 assert(OpNo == 1 && "Wrong operand for scalarization!"); 857 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 858 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 859 { N->getValueType(0).getVectorElementType(), 860 MVT::Other }, 861 { N->getOperand(0), Elt, N->getOperand(2) }); 862 // Legalize the chain result - switch anything that used the old chain to 863 // use the new one. 864 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 865 866 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 867 868 // Do our own replacement and return SDValue() to tell the caller that we 869 // handled all replacements since caller can only handle a single result. 870 ReplaceValueWith(SDValue(N, 0), Res); 871 return SDValue(); 872 } 873 874 /// If the value to extend is a vector that needs to be scalarized, it must be 875 /// <1 x ty>. Convert the element instead. 876 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) { 877 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 878 SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), 879 N->getValueType(0).getVectorElementType(), Elt); 880 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 881 } 882 883 /// If the value to extend is a vector that needs to be scalarized, it must be 884 /// <1 x ty>. Convert the element instead. 885 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) { 886 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 887 SDValue Res = 888 DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 889 {N->getValueType(0).getVectorElementType(), MVT::Other}, 890 {N->getOperand(0), Elt}); 891 // Legalize the chain result - switch anything that used the old chain to 892 // use the new one. 893 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 894 895 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 896 897 // Do our own replacement and return SDValue() to tell the caller that we 898 // handled all replacements since caller can only handle a single result. 899 ReplaceValueWith(SDValue(N, 0), Res); 900 return SDValue(); 901 } 902 903 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) { 904 SDValue Res = GetScalarizedVector(N->getOperand(0)); 905 // Result type may be wider than element type. 906 if (Res.getValueType() != N->getValueType(0)) 907 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res); 908 return Res; 909 } 910 911 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) { 912 SDValue AccOp = N->getOperand(0); 913 SDValue VecOp = N->getOperand(1); 914 915 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode()); 916 917 SDValue Op = GetScalarizedVector(VecOp); 918 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0), 919 AccOp, Op, N->getFlags()); 920 } 921 922 //===----------------------------------------------------------------------===// 923 // Result Vector Splitting 924 //===----------------------------------------------------------------------===// 925 926 /// This method is called when the specified result of the specified node is 927 /// found to need vector splitting. At this point, the node may also have 928 /// invalid operands or may have other results that need legalization, we just 929 /// know that (at least) one result needs vector splitting. 930 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 931 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n"); 932 SDValue Lo, Hi; 933 934 // See if the target wants to custom expand this node. 935 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 936 return; 937 938 switch (N->getOpcode()) { 939 default: 940 #ifndef NDEBUG 941 dbgs() << "SplitVectorResult #" << ResNo << ": "; 942 N->dump(&DAG); 943 dbgs() << "\n"; 944 #endif 945 report_fatal_error("Do not know how to split the result of this " 946 "operator!\n"); 947 948 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 949 case ISD::VSELECT: 950 case ISD::SELECT: 951 case ISD::VP_MERGE: 952 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break; 953 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 954 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 955 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 956 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 957 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 958 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 959 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break; 960 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 961 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break; 962 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break; 963 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 964 case ISD::SPLAT_VECTOR: 965 case ISD::SCALAR_TO_VECTOR: 966 SplitVecRes_ScalarOp(N, Lo, Hi); 967 break; 968 case ISD::STEP_VECTOR: 969 SplitVecRes_STEP_VECTOR(N, Lo, Hi); 970 break; 971 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 972 case ISD::LOAD: 973 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 974 break; 975 case ISD::VP_LOAD: 976 SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi); 977 break; 978 case ISD::MLOAD: 979 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi); 980 break; 981 case ISD::MGATHER: 982 case ISD::VP_GATHER: 983 SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true); 984 break; 985 case ISD::SETCC: 986 case ISD::VP_SETCC: 987 SplitVecRes_SETCC(N, Lo, Hi); 988 break; 989 case ISD::VECTOR_REVERSE: 990 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi); 991 break; 992 case ISD::VECTOR_SHUFFLE: 993 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 994 break; 995 case ISD::VECTOR_SPLICE: 996 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi); 997 break; 998 case ISD::VAARG: 999 SplitVecRes_VAARG(N, Lo, Hi); 1000 break; 1001 1002 case ISD::ANY_EXTEND_VECTOR_INREG: 1003 case ISD::SIGN_EXTEND_VECTOR_INREG: 1004 case ISD::ZERO_EXTEND_VECTOR_INREG: 1005 SplitVecRes_ExtVecInRegOp(N, Lo, Hi); 1006 break; 1007 1008 case ISD::ABS: 1009 case ISD::BITREVERSE: 1010 case ISD::BSWAP: 1011 case ISD::CTLZ: 1012 case ISD::CTTZ: 1013 case ISD::CTLZ_ZERO_UNDEF: 1014 case ISD::CTTZ_ZERO_UNDEF: 1015 case ISD::CTPOP: 1016 case ISD::FABS: 1017 case ISD::FCEIL: 1018 case ISD::FCOS: 1019 case ISD::FEXP: 1020 case ISD::FEXP2: 1021 case ISD::FFLOOR: 1022 case ISD::FLOG: 1023 case ISD::FLOG10: 1024 case ISD::FLOG2: 1025 case ISD::FNEARBYINT: 1026 case ISD::FNEG: case ISD::VP_FNEG: 1027 case ISD::FREEZE: 1028 case ISD::ARITH_FENCE: 1029 case ISD::FP_EXTEND: 1030 case ISD::VP_FP_EXTEND: 1031 case ISD::FP_ROUND: 1032 case ISD::VP_FP_ROUND: 1033 case ISD::FP_TO_SINT: 1034 case ISD::VP_FPTOSI: 1035 case ISD::FP_TO_UINT: 1036 case ISD::VP_FPTOUI: 1037 case ISD::FRINT: 1038 case ISD::FROUND: 1039 case ISD::FROUNDEVEN: 1040 case ISD::FSIN: 1041 case ISD::FSQRT: 1042 case ISD::FTRUNC: 1043 case ISD::SINT_TO_FP: 1044 case ISD::VP_SITOFP: 1045 case ISD::TRUNCATE: 1046 case ISD::VP_TRUNCATE: 1047 case ISD::UINT_TO_FP: 1048 case ISD::VP_UITOFP: 1049 case ISD::FCANONICALIZE: 1050 SplitVecRes_UnaryOp(N, Lo, Hi); 1051 break; 1052 1053 case ISD::ANY_EXTEND: 1054 case ISD::SIGN_EXTEND: 1055 case ISD::ZERO_EXTEND: 1056 case ISD::VP_SIGN_EXTEND: 1057 case ISD::VP_ZERO_EXTEND: 1058 SplitVecRes_ExtendOp(N, Lo, Hi); 1059 break; 1060 1061 case ISD::ADD: case ISD::VP_ADD: 1062 case ISD::SUB: case ISD::VP_SUB: 1063 case ISD::MUL: case ISD::VP_MUL: 1064 case ISD::MULHS: 1065 case ISD::MULHU: 1066 case ISD::FADD: case ISD::VP_FADD: 1067 case ISD::FSUB: case ISD::VP_FSUB: 1068 case ISD::FMUL: case ISD::VP_FMUL: 1069 case ISD::FMINNUM: 1070 case ISD::FMAXNUM: 1071 case ISD::FMINIMUM: 1072 case ISD::FMAXIMUM: 1073 case ISD::SDIV: case ISD::VP_SDIV: 1074 case ISD::UDIV: case ISD::VP_UDIV: 1075 case ISD::FDIV: case ISD::VP_FDIV: 1076 case ISD::FPOW: 1077 case ISD::AND: case ISD::VP_AND: 1078 case ISD::OR: case ISD::VP_OR: 1079 case ISD::XOR: case ISD::VP_XOR: 1080 case ISD::SHL: case ISD::VP_SHL: 1081 case ISD::SRA: case ISD::VP_ASHR: 1082 case ISD::SRL: case ISD::VP_LSHR: 1083 case ISD::UREM: case ISD::VP_UREM: 1084 case ISD::SREM: case ISD::VP_SREM: 1085 case ISD::FREM: case ISD::VP_FREM: 1086 case ISD::SMIN: 1087 case ISD::SMAX: 1088 case ISD::UMIN: 1089 case ISD::UMAX: 1090 case ISD::SADDSAT: 1091 case ISD::UADDSAT: 1092 case ISD::SSUBSAT: 1093 case ISD::USUBSAT: 1094 case ISD::SSHLSAT: 1095 case ISD::USHLSAT: 1096 case ISD::ROTL: 1097 case ISD::ROTR: 1098 SplitVecRes_BinOp(N, Lo, Hi); 1099 break; 1100 case ISD::FMA: case ISD::VP_FMA: 1101 case ISD::FSHL: 1102 case ISD::FSHR: 1103 SplitVecRes_TernaryOp(N, Lo, Hi); 1104 break; 1105 1106 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 1107 case ISD::STRICT_##DAGN: 1108 #include "llvm/IR/ConstrainedOps.def" 1109 SplitVecRes_StrictFPOp(N, Lo, Hi); 1110 break; 1111 1112 case ISD::FP_TO_UINT_SAT: 1113 case ISD::FP_TO_SINT_SAT: 1114 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi); 1115 break; 1116 1117 case ISD::UADDO: 1118 case ISD::SADDO: 1119 case ISD::USUBO: 1120 case ISD::SSUBO: 1121 case ISD::UMULO: 1122 case ISD::SMULO: 1123 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi); 1124 break; 1125 case ISD::SMULFIX: 1126 case ISD::SMULFIXSAT: 1127 case ISD::UMULFIX: 1128 case ISD::UMULFIXSAT: 1129 case ISD::SDIVFIX: 1130 case ISD::SDIVFIXSAT: 1131 case ISD::UDIVFIX: 1132 case ISD::UDIVFIXSAT: 1133 SplitVecRes_FIX(N, Lo, Hi); 1134 break; 1135 } 1136 1137 // If Lo/Hi is null, the sub-method took care of registering results etc. 1138 if (Lo.getNode()) 1139 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 1140 } 1141 1142 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT, 1143 MachinePointerInfo &MPI, SDValue &Ptr, 1144 uint64_t *ScaledOffset) { 1145 SDLoc DL(N); 1146 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8; 1147 1148 if (MemVT.isScalableVector()) { 1149 SDNodeFlags Flags; 1150 SDValue BytesIncrement = DAG.getVScale( 1151 DL, Ptr.getValueType(), 1152 APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize)); 1153 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); 1154 Flags.setNoUnsignedWrap(true); 1155 if (ScaledOffset) 1156 *ScaledOffset += IncrementSize; 1157 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement, 1158 Flags); 1159 } else { 1160 MPI = N->getPointerInfo().getWithOffset(IncrementSize); 1161 // Increment the pointer to the other half. 1162 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize)); 1163 } 1164 } 1165 1166 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) { 1167 return SplitMask(Mask, SDLoc(Mask)); 1168 } 1169 1170 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask, 1171 const SDLoc &DL) { 1172 SDValue MaskLo, MaskHi; 1173 EVT MaskVT = Mask.getValueType(); 1174 if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector) 1175 GetSplitVector(Mask, MaskLo, MaskHi); 1176 else 1177 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 1178 return std::make_pair(MaskLo, MaskHi); 1179 } 1180 1181 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) { 1182 SDValue LHSLo, LHSHi; 1183 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1184 SDValue RHSLo, RHSHi; 1185 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 1186 SDLoc dl(N); 1187 1188 const SDNodeFlags Flags = N->getFlags(); 1189 unsigned Opcode = N->getOpcode(); 1190 if (N->getNumOperands() == 2) { 1191 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags); 1192 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags); 1193 return; 1194 } 1195 1196 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1197 assert(N->isVPOpcode() && "Expected VP opcode"); 1198 1199 SDValue MaskLo, MaskHi; 1200 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2)); 1201 1202 SDValue EVLLo, EVLHi; 1203 std::tie(EVLLo, EVLHi) = 1204 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl); 1205 1206 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), 1207 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags); 1208 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), 1209 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags); 1210 } 1211 1212 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 1213 SDValue &Hi) { 1214 SDValue Op0Lo, Op0Hi; 1215 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 1216 SDValue Op1Lo, Op1Hi; 1217 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 1218 SDValue Op2Lo, Op2Hi; 1219 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 1220 SDLoc dl(N); 1221 1222 const SDNodeFlags Flags = N->getFlags(); 1223 unsigned Opcode = N->getOpcode(); 1224 if (N->getNumOperands() == 3) { 1225 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags); 1226 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags); 1227 return; 1228 } 1229 1230 assert(N->getNumOperands() == 5 && "Unexpected number of operands!"); 1231 assert(N->isVPOpcode() && "Expected VP opcode"); 1232 1233 SDValue MaskLo, MaskHi; 1234 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3)); 1235 1236 SDValue EVLLo, EVLHi; 1237 std::tie(EVLLo, EVLHi) = 1238 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), dl); 1239 1240 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), 1241 {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags); 1242 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), 1243 {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags); 1244 } 1245 1246 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) { 1247 SDValue LHSLo, LHSHi; 1248 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1249 SDValue RHSLo, RHSHi; 1250 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 1251 SDLoc dl(N); 1252 SDValue Op2 = N->getOperand(2); 1253 1254 unsigned Opcode = N->getOpcode(); 1255 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2, 1256 N->getFlags()); 1257 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2, 1258 N->getFlags()); 1259 } 1260 1261 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 1262 SDValue &Hi) { 1263 // We know the result is a vector. The input may be either a vector or a 1264 // scalar value. 1265 EVT LoVT, HiVT; 1266 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1267 SDLoc dl(N); 1268 1269 SDValue InOp = N->getOperand(0); 1270 EVT InVT = InOp.getValueType(); 1271 1272 // Handle some special cases efficiently. 1273 switch (getTypeAction(InVT)) { 1274 case TargetLowering::TypeLegal: 1275 case TargetLowering::TypePromoteInteger: 1276 case TargetLowering::TypePromoteFloat: 1277 case TargetLowering::TypeSoftPromoteHalf: 1278 case TargetLowering::TypeSoftenFloat: 1279 case TargetLowering::TypeScalarizeVector: 1280 case TargetLowering::TypeWidenVector: 1281 break; 1282 case TargetLowering::TypeExpandInteger: 1283 case TargetLowering::TypeExpandFloat: 1284 // A scalar to vector conversion, where the scalar needs expansion. 1285 // If the vector is being split in two then we can just convert the 1286 // expanded pieces. 1287 if (LoVT == HiVT) { 1288 GetExpandedOp(InOp, Lo, Hi); 1289 if (DAG.getDataLayout().isBigEndian()) 1290 std::swap(Lo, Hi); 1291 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1292 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1293 return; 1294 } 1295 break; 1296 case TargetLowering::TypeSplitVector: 1297 // If the input is a vector that needs to be split, convert each split 1298 // piece of the input now. 1299 GetSplitVector(InOp, Lo, Hi); 1300 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1301 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1302 return; 1303 case TargetLowering::TypeScalarizeScalableVector: 1304 report_fatal_error("Scalarization of scalable vectors is not supported."); 1305 } 1306 1307 // In the general case, convert the input to an integer and split it by hand. 1308 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 1309 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 1310 if (DAG.getDataLayout().isBigEndian()) 1311 std::swap(LoIntVT, HiIntVT); 1312 1313 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 1314 1315 if (DAG.getDataLayout().isBigEndian()) 1316 std::swap(Lo, Hi); 1317 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1318 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1319 } 1320 1321 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 1322 SDValue &Hi) { 1323 EVT LoVT, HiVT; 1324 SDLoc dl(N); 1325 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1326 unsigned LoNumElts = LoVT.getVectorNumElements(); 1327 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 1328 Lo = DAG.getBuildVector(LoVT, dl, LoOps); 1329 1330 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 1331 Hi = DAG.getBuildVector(HiVT, dl, HiOps); 1332 } 1333 1334 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 1335 SDValue &Hi) { 1336 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 1337 SDLoc dl(N); 1338 unsigned NumSubvectors = N->getNumOperands() / 2; 1339 if (NumSubvectors == 1) { 1340 Lo = N->getOperand(0); 1341 Hi = N->getOperand(1); 1342 return; 1343 } 1344 1345 EVT LoVT, HiVT; 1346 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1347 1348 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 1349 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps); 1350 1351 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 1352 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps); 1353 } 1354 1355 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 1356 SDValue &Hi) { 1357 SDValue Vec = N->getOperand(0); 1358 SDValue Idx = N->getOperand(1); 1359 SDLoc dl(N); 1360 1361 EVT LoVT, HiVT; 1362 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1363 1364 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 1365 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1366 Hi = DAG.getNode( 1367 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 1368 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl)); 1369 } 1370 1371 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, 1372 SDValue &Hi) { 1373 SDValue Vec = N->getOperand(0); 1374 SDValue SubVec = N->getOperand(1); 1375 SDValue Idx = N->getOperand(2); 1376 SDLoc dl(N); 1377 GetSplitVector(Vec, Lo, Hi); 1378 1379 EVT VecVT = Vec.getValueType(); 1380 EVT LoVT = Lo.getValueType(); 1381 EVT SubVecVT = SubVec.getValueType(); 1382 unsigned VecElems = VecVT.getVectorMinNumElements(); 1383 unsigned SubElems = SubVecVT.getVectorMinNumElements(); 1384 unsigned LoElems = LoVT.getVectorMinNumElements(); 1385 1386 // If we know the index is in the first half, and we know the subvector 1387 // doesn't cross the boundary between the halves, we can avoid spilling the 1388 // vector, and insert into the lower half of the split vector directly. 1389 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1390 if (IdxVal + SubElems <= LoElems) { 1391 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx); 1392 return; 1393 } 1394 // Similarly if the subvector is fully in the high half, but mind that we 1395 // can't tell whether a fixed-length subvector is fully within the high half 1396 // of a scalable vector. 1397 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() && 1398 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) { 1399 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec, 1400 DAG.getVectorIdxConstant(IdxVal - LoElems, dl)); 1401 return; 1402 } 1403 1404 // Spill the vector to the stack. 1405 // In cases where the vector is illegal it will be broken down into parts 1406 // and stored in parts - we should use the alignment for the smallest part. 1407 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false); 1408 SDValue StackPtr = 1409 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign); 1410 auto &MF = DAG.getMachineFunction(); 1411 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1412 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 1413 1414 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1415 SmallestAlign); 1416 1417 // Store the new subvector into the specified index. 1418 SDValue SubVecPtr = 1419 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx); 1420 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, 1421 MachinePointerInfo::getUnknownStack(MF)); 1422 1423 // Load the Lo part from the stack slot. 1424 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo, 1425 SmallestAlign); 1426 1427 // Increment the pointer to the other part. 1428 auto *Load = cast<LoadSDNode>(Lo); 1429 MachinePointerInfo MPI = Load->getPointerInfo(); 1430 IncrementPointer(Load, LoVT, MPI, StackPtr); 1431 1432 // Load the Hi part from the stack slot. 1433 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign); 1434 } 1435 1436 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 1437 SDValue &Hi) { 1438 SDLoc dl(N); 1439 GetSplitVector(N->getOperand(0), Lo, Hi); 1440 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 1441 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 1442 } 1443 1444 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, 1445 SDValue &Hi) { 1446 SDValue LHSLo, LHSHi; 1447 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1448 SDLoc DL(N); 1449 1450 SDValue RHSLo, RHSHi; 1451 SDValue RHS = N->getOperand(1); 1452 EVT RHSVT = RHS.getValueType(); 1453 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector) 1454 GetSplitVector(RHS, RHSLo, RHSHi); 1455 else 1456 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS)); 1457 1458 1459 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo); 1460 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi); 1461 } 1462 1463 void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo, 1464 SDValue &Hi) { 1465 SDLoc DL(N); 1466 SDValue ArgLo, ArgHi; 1467 SDValue Test = N->getOperand(1); 1468 GetSplitVector(N->getOperand(0), ArgLo, ArgHi); 1469 EVT LoVT, HiVT; 1470 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1471 1472 Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags()); 1473 Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags()); 1474 } 1475 1476 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 1477 SDValue &Hi) { 1478 SDValue LHSLo, LHSHi; 1479 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1480 SDLoc dl(N); 1481 1482 EVT LoVT, HiVT; 1483 std::tie(LoVT, HiVT) = 1484 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT()); 1485 1486 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 1487 DAG.getValueType(LoVT)); 1488 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 1489 DAG.getValueType(HiVT)); 1490 } 1491 1492 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, 1493 SDValue &Hi) { 1494 unsigned Opcode = N->getOpcode(); 1495 SDValue N0 = N->getOperand(0); 1496 1497 SDLoc dl(N); 1498 SDValue InLo, InHi; 1499 1500 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector) 1501 GetSplitVector(N0, InLo, InHi); 1502 else 1503 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0); 1504 1505 EVT InLoVT = InLo.getValueType(); 1506 unsigned InNumElements = InLoVT.getVectorNumElements(); 1507 1508 EVT OutLoVT, OutHiVT; 1509 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1510 unsigned OutNumElements = OutLoVT.getVectorNumElements(); 1511 assert((2 * OutNumElements) <= InNumElements && 1512 "Illegal extend vector in reg split"); 1513 1514 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the 1515 // input vector (i.e. we only use InLo): 1516 // OutLo will extend the first OutNumElements from InLo. 1517 // OutHi will extend the next OutNumElements from InLo. 1518 1519 // Shuffle the elements from InLo for OutHi into the bottom elements to 1520 // create a 'fake' InHi. 1521 SmallVector<int, 8> SplitHi(InNumElements, -1); 1522 for (unsigned i = 0; i != OutNumElements; ++i) 1523 SplitHi[i] = i + OutNumElements; 1524 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi); 1525 1526 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo); 1527 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi); 1528 } 1529 1530 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, 1531 SDValue &Hi) { 1532 unsigned NumOps = N->getNumOperands(); 1533 SDValue Chain = N->getOperand(0); 1534 EVT LoVT, HiVT; 1535 SDLoc dl(N); 1536 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1537 1538 SmallVector<SDValue, 4> OpsLo(NumOps); 1539 SmallVector<SDValue, 4> OpsHi(NumOps); 1540 1541 // The Chain is the first operand. 1542 OpsLo[0] = Chain; 1543 OpsHi[0] = Chain; 1544 1545 // Now process the remaining operands. 1546 for (unsigned i = 1; i < NumOps; ++i) { 1547 SDValue Op = N->getOperand(i); 1548 SDValue OpLo = Op; 1549 SDValue OpHi = Op; 1550 1551 EVT InVT = Op.getValueType(); 1552 if (InVT.isVector()) { 1553 // If the input also splits, handle it directly for a 1554 // compile time speedup. Otherwise split it by hand. 1555 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 1556 GetSplitVector(Op, OpLo, OpHi); 1557 else 1558 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i); 1559 } 1560 1561 OpsLo[i] = OpLo; 1562 OpsHi[i] = OpHi; 1563 } 1564 1565 EVT LoValueVTs[] = {LoVT, MVT::Other}; 1566 EVT HiValueVTs[] = {HiVT, MVT::Other}; 1567 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo, 1568 N->getFlags()); 1569 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi, 1570 N->getFlags()); 1571 1572 // Build a factor node to remember that this Op is independent of the 1573 // other one. 1574 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1575 Lo.getValue(1), Hi.getValue(1)); 1576 1577 // Legalize the chain result - switch anything that used the old chain to 1578 // use the new one. 1579 ReplaceValueWith(SDValue(N, 1), Chain); 1580 } 1581 1582 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { 1583 SDValue Chain = N->getOperand(0); 1584 EVT VT = N->getValueType(0); 1585 unsigned NE = VT.getVectorNumElements(); 1586 EVT EltVT = VT.getVectorElementType(); 1587 SDLoc dl(N); 1588 1589 SmallVector<SDValue, 8> Scalars; 1590 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 1591 1592 // If ResNE is 0, fully unroll the vector op. 1593 if (ResNE == 0) 1594 ResNE = NE; 1595 else if (NE > ResNE) 1596 NE = ResNE; 1597 1598 //The results of each unrolled operation, including the chain. 1599 EVT ChainVTs[] = {EltVT, MVT::Other}; 1600 SmallVector<SDValue, 8> Chains; 1601 1602 unsigned i; 1603 for (i = 0; i != NE; ++i) { 1604 Operands[0] = Chain; 1605 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) { 1606 SDValue Operand = N->getOperand(j); 1607 EVT OperandVT = Operand.getValueType(); 1608 if (OperandVT.isVector()) { 1609 EVT OperandEltVT = OperandVT.getVectorElementType(); 1610 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, 1611 Operand, DAG.getVectorIdxConstant(i, dl)); 1612 } else { 1613 Operands[j] = Operand; 1614 } 1615 } 1616 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands); 1617 Scalar.getNode()->setFlags(N->getFlags()); 1618 1619 //Add in the scalar as well as its chain value to the 1620 //result vectors. 1621 Scalars.push_back(Scalar); 1622 Chains.push_back(Scalar.getValue(1)); 1623 } 1624 1625 for (; i < ResNE; ++i) 1626 Scalars.push_back(DAG.getUNDEF(EltVT)); 1627 1628 // Build a new factor node to connect the chain back together. 1629 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 1630 ReplaceValueWith(SDValue(N, 1), Chain); 1631 1632 // Create a new BUILD_VECTOR node 1633 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE); 1634 return DAG.getBuildVector(VecVT, dl, Scalars); 1635 } 1636 1637 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, 1638 SDValue &Lo, SDValue &Hi) { 1639 SDLoc dl(N); 1640 EVT ResVT = N->getValueType(0); 1641 EVT OvVT = N->getValueType(1); 1642 EVT LoResVT, HiResVT, LoOvVT, HiOvVT; 1643 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT); 1644 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT); 1645 1646 SDValue LoLHS, HiLHS, LoRHS, HiRHS; 1647 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) { 1648 GetSplitVector(N->getOperand(0), LoLHS, HiLHS); 1649 GetSplitVector(N->getOperand(1), LoRHS, HiRHS); 1650 } else { 1651 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0); 1652 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1); 1653 } 1654 1655 unsigned Opcode = N->getOpcode(); 1656 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT); 1657 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT); 1658 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode(); 1659 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode(); 1660 LoNode->setFlags(N->getFlags()); 1661 HiNode->setFlags(N->getFlags()); 1662 1663 Lo = SDValue(LoNode, ResNo); 1664 Hi = SDValue(HiNode, ResNo); 1665 1666 // Replace the other vector result not being explicitly split here. 1667 unsigned OtherNo = 1 - ResNo; 1668 EVT OtherVT = N->getValueType(OtherNo); 1669 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) { 1670 SetSplitVector(SDValue(N, OtherNo), 1671 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo)); 1672 } else { 1673 SDValue OtherVal = DAG.getNode( 1674 ISD::CONCAT_VECTORS, dl, OtherVT, 1675 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo)); 1676 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 1677 } 1678 } 1679 1680 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 1681 SDValue &Hi) { 1682 SDValue Vec = N->getOperand(0); 1683 SDValue Elt = N->getOperand(1); 1684 SDValue Idx = N->getOperand(2); 1685 SDLoc dl(N); 1686 GetSplitVector(Vec, Lo, Hi); 1687 1688 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 1689 unsigned IdxVal = CIdx->getZExtValue(); 1690 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements(); 1691 if (IdxVal < LoNumElts) { 1692 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 1693 Lo.getValueType(), Lo, Elt, Idx); 1694 return; 1695 } else if (!Vec.getValueType().isScalableVector()) { 1696 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 1697 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl)); 1698 return; 1699 } 1700 } 1701 1702 // See if the target wants to custom expand this node. 1703 if (CustomLowerNode(N, N->getValueType(0), true)) 1704 return; 1705 1706 // Make the vector elements byte-addressable if they aren't already. 1707 EVT VecVT = Vec.getValueType(); 1708 EVT EltVT = VecVT.getVectorElementType(); 1709 if (VecVT.getScalarSizeInBits() < 8) { 1710 EltVT = MVT::i8; 1711 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 1712 VecVT.getVectorElementCount()); 1713 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec); 1714 // Extend the element type to match if needed. 1715 if (EltVT.bitsGT(Elt.getValueType())) 1716 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt); 1717 } 1718 1719 // Spill the vector to the stack. 1720 // In cases where the vector is illegal it will be broken down into parts 1721 // and stored in parts - we should use the alignment for the smallest part. 1722 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false); 1723 SDValue StackPtr = 1724 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign); 1725 auto &MF = DAG.getMachineFunction(); 1726 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1727 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 1728 1729 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1730 SmallestAlign); 1731 1732 // Store the new element. This may be larger than the vector element type, 1733 // so use a truncating store. 1734 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 1735 Store = DAG.getTruncStore( 1736 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT, 1737 commonAlignment(SmallestAlign, 1738 EltVT.getFixedSizeInBits() / 8)); 1739 1740 EVT LoVT, HiVT; 1741 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT); 1742 1743 // Load the Lo part from the stack slot. 1744 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign); 1745 1746 // Increment the pointer to the other part. 1747 auto Load = cast<LoadSDNode>(Lo); 1748 MachinePointerInfo MPI = Load->getPointerInfo(); 1749 IncrementPointer(Load, LoVT, MPI, StackPtr); 1750 1751 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign); 1752 1753 // If we adjusted the original type, we need to truncate the results. 1754 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1755 if (LoVT != Lo.getValueType()) 1756 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo); 1757 if (HiVT != Hi.getValueType()) 1758 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi); 1759 } 1760 1761 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, 1762 SDValue &Hi) { 1763 EVT LoVT, HiVT; 1764 SDLoc dl(N); 1765 assert(N->getValueType(0).isScalableVector() && 1766 "Only scalable vectors are supported for STEP_VECTOR"); 1767 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1768 SDValue Step = N->getOperand(0); 1769 1770 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step); 1771 1772 // Hi = Lo + (EltCnt * Step) 1773 EVT EltVT = Step.getValueType(); 1774 APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue(); 1775 SDValue StartOfHi = 1776 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements()); 1777 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType()); 1778 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi); 1779 1780 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step); 1781 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi); 1782 } 1783 1784 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, 1785 SDValue &Hi) { 1786 EVT LoVT, HiVT; 1787 SDLoc dl(N); 1788 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1789 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0)); 1790 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { 1791 Hi = DAG.getUNDEF(HiVT); 1792 } else { 1793 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode"); 1794 Hi = Lo; 1795 } 1796 } 1797 1798 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 1799 SDValue &Hi) { 1800 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 1801 EVT LoVT, HiVT; 1802 SDLoc dl(LD); 1803 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0)); 1804 1805 ISD::LoadExtType ExtType = LD->getExtensionType(); 1806 SDValue Ch = LD->getChain(); 1807 SDValue Ptr = LD->getBasePtr(); 1808 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 1809 EVT MemoryVT = LD->getMemoryVT(); 1810 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 1811 AAMDNodes AAInfo = LD->getAAInfo(); 1812 1813 EVT LoMemVT, HiMemVT; 1814 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1815 1816 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) { 1817 SDValue Value, NewChain; 1818 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG); 1819 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl); 1820 ReplaceValueWith(SDValue(LD, 1), NewChain); 1821 return; 1822 } 1823 1824 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 1825 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(), 1826 MMOFlags, AAInfo); 1827 1828 MachinePointerInfo MPI; 1829 IncrementPointer(LD, LoMemVT, MPI, Ptr); 1830 1831 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI, 1832 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo); 1833 1834 // Build a factor node to remember that this load is independent of the 1835 // other one. 1836 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1837 Hi.getValue(1)); 1838 1839 // Legalize the chain result - switch anything that used the old chain to 1840 // use the new one. 1841 ReplaceValueWith(SDValue(LD, 1), Ch); 1842 } 1843 1844 void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo, 1845 SDValue &Hi) { 1846 assert(LD->isUnindexed() && "Indexed VP load during type legalization!"); 1847 EVT LoVT, HiVT; 1848 SDLoc dl(LD); 1849 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0)); 1850 1851 ISD::LoadExtType ExtType = LD->getExtensionType(); 1852 SDValue Ch = LD->getChain(); 1853 SDValue Ptr = LD->getBasePtr(); 1854 SDValue Offset = LD->getOffset(); 1855 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset"); 1856 Align Alignment = LD->getOriginalAlign(); 1857 SDValue Mask = LD->getMask(); 1858 SDValue EVL = LD->getVectorLength(); 1859 EVT MemoryVT = LD->getMemoryVT(); 1860 1861 EVT LoMemVT, HiMemVT; 1862 bool HiIsEmpty = false; 1863 std::tie(LoMemVT, HiMemVT) = 1864 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty); 1865 1866 // Split Mask operand 1867 SDValue MaskLo, MaskHi; 1868 if (Mask.getOpcode() == ISD::SETCC) { 1869 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 1870 } else { 1871 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1872 GetSplitVector(Mask, MaskLo, MaskHi); 1873 else 1874 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1875 } 1876 1877 // Split EVL operand 1878 SDValue EVLLo, EVLHi; 1879 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl); 1880 1881 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 1882 LD->getPointerInfo(), MachineMemOperand::MOLoad, 1883 MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges()); 1884 1885 Lo = 1886 DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset, 1887 MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad()); 1888 1889 if (HiIsEmpty) { 1890 // The hi vp_load has zero storage size. We therefore simply set it to 1891 // the low vp_load and rely on subsequent removal from the chain. 1892 Hi = Lo; 1893 } else { 1894 // Generate hi vp_load. 1895 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG, 1896 LD->isExpandingLoad()); 1897 1898 MachinePointerInfo MPI; 1899 if (LoMemVT.isScalableVector()) 1900 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace()); 1901 else 1902 MPI = LD->getPointerInfo().getWithOffset( 1903 LoMemVT.getStoreSize().getFixedSize()); 1904 1905 MMO = DAG.getMachineFunction().getMachineMemOperand( 1906 MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment, 1907 LD->getAAInfo(), LD->getRanges()); 1908 1909 Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr, 1910 Offset, MaskHi, EVLHi, HiMemVT, MMO, 1911 LD->isExpandingLoad()); 1912 } 1913 1914 // Build a factor node to remember that this load is independent of the 1915 // other one. 1916 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1917 Hi.getValue(1)); 1918 1919 // Legalize the chain result - switch anything that used the old chain to 1920 // use the new one. 1921 ReplaceValueWith(SDValue(LD, 1), Ch); 1922 } 1923 1924 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, 1925 SDValue &Lo, SDValue &Hi) { 1926 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!"); 1927 EVT LoVT, HiVT; 1928 SDLoc dl(MLD); 1929 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0)); 1930 1931 SDValue Ch = MLD->getChain(); 1932 SDValue Ptr = MLD->getBasePtr(); 1933 SDValue Offset = MLD->getOffset(); 1934 assert(Offset.isUndef() && "Unexpected indexed masked load offset"); 1935 SDValue Mask = MLD->getMask(); 1936 SDValue PassThru = MLD->getPassThru(); 1937 Align Alignment = MLD->getOriginalAlign(); 1938 ISD::LoadExtType ExtType = MLD->getExtensionType(); 1939 1940 // Split Mask operand 1941 SDValue MaskLo, MaskHi; 1942 if (Mask.getOpcode() == ISD::SETCC) { 1943 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 1944 } else { 1945 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1946 GetSplitVector(Mask, MaskLo, MaskHi); 1947 else 1948 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1949 } 1950 1951 EVT MemoryVT = MLD->getMemoryVT(); 1952 EVT LoMemVT, HiMemVT; 1953 bool HiIsEmpty = false; 1954 std::tie(LoMemVT, HiMemVT) = 1955 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty); 1956 1957 SDValue PassThruLo, PassThruHi; 1958 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector) 1959 GetSplitVector(PassThru, PassThruLo, PassThruHi); 1960 else 1961 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); 1962 1963 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 1964 MLD->getPointerInfo(), MachineMemOperand::MOLoad, 1965 MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(), 1966 MLD->getRanges()); 1967 1968 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT, 1969 MMO, MLD->getAddressingMode(), ExtType, 1970 MLD->isExpandingLoad()); 1971 1972 if (HiIsEmpty) { 1973 // The hi masked load has zero storage size. We therefore simply set it to 1974 // the low masked load and rely on subsequent removal from the chain. 1975 Hi = Lo; 1976 } else { 1977 // Generate hi masked load. 1978 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG, 1979 MLD->isExpandingLoad()); 1980 1981 MachinePointerInfo MPI; 1982 if (LoMemVT.isScalableVector()) 1983 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace()); 1984 else 1985 MPI = MLD->getPointerInfo().getWithOffset( 1986 LoMemVT.getStoreSize().getFixedSize()); 1987 1988 MMO = DAG.getMachineFunction().getMachineMemOperand( 1989 MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment, 1990 MLD->getAAInfo(), MLD->getRanges()); 1991 1992 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi, 1993 HiMemVT, MMO, MLD->getAddressingMode(), ExtType, 1994 MLD->isExpandingLoad()); 1995 } 1996 1997 // Build a factor node to remember that this load is independent of the 1998 // other one. 1999 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2000 Hi.getValue(1)); 2001 2002 // Legalize the chain result - switch anything that used the old chain to 2003 // use the new one. 2004 ReplaceValueWith(SDValue(MLD, 1), Ch); 2005 2006 } 2007 2008 void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo, 2009 SDValue &Hi, bool SplitSETCC) { 2010 EVT LoVT, HiVT; 2011 SDLoc dl(N); 2012 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 2013 2014 SDValue Ch = N->getChain(); 2015 SDValue Ptr = N->getBasePtr(); 2016 struct Operands { 2017 SDValue Mask; 2018 SDValue Index; 2019 SDValue Scale; 2020 } Ops = [&]() -> Operands { 2021 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) { 2022 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()}; 2023 } 2024 auto *VPSC = cast<VPGatherSDNode>(N); 2025 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()}; 2026 }(); 2027 2028 EVT MemoryVT = N->getMemoryVT(); 2029 Align Alignment = N->getOriginalAlign(); 2030 2031 // Split Mask operand 2032 SDValue MaskLo, MaskHi; 2033 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) { 2034 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi); 2035 } else { 2036 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl); 2037 } 2038 2039 EVT LoMemVT, HiMemVT; 2040 // Split MemoryVT 2041 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 2042 2043 SDValue IndexHi, IndexLo; 2044 if (getTypeAction(Ops.Index.getValueType()) == 2045 TargetLowering::TypeSplitVector) 2046 GetSplitVector(Ops.Index, IndexLo, IndexHi); 2047 else 2048 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl); 2049 2050 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 2051 N->getPointerInfo(), MachineMemOperand::MOLoad, 2052 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges()); 2053 2054 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) { 2055 SDValue PassThru = MGT->getPassThru(); 2056 SDValue PassThruLo, PassThruHi; 2057 if (getTypeAction(PassThru.getValueType()) == 2058 TargetLowering::TypeSplitVector) 2059 GetSplitVector(PassThru, PassThruLo, PassThruHi); 2060 else 2061 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); 2062 2063 ISD::LoadExtType ExtType = MGT->getExtensionType(); 2064 ISD::MemIndexType IndexTy = MGT->getIndexType(); 2065 2066 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale}; 2067 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, 2068 OpsLo, MMO, IndexTy, ExtType); 2069 2070 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale}; 2071 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, 2072 OpsHi, MMO, IndexTy, ExtType); 2073 } else { 2074 auto *VPGT = cast<VPGatherSDNode>(N); 2075 SDValue EVLLo, EVLHi; 2076 std::tie(EVLLo, EVLHi) = 2077 DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl); 2078 2079 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo}; 2080 Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo, 2081 MMO, VPGT->getIndexType()); 2082 2083 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi}; 2084 Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi, 2085 MMO, VPGT->getIndexType()); 2086 } 2087 2088 // Build a factor node to remember that this load is independent of the 2089 // other one. 2090 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2091 Hi.getValue(1)); 2092 2093 // Legalize the chain result - switch anything that used the old chain to 2094 // use the new one. 2095 ReplaceValueWith(SDValue(N, 1), Ch); 2096 } 2097 2098 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 2099 assert(N->getValueType(0).isVector() && 2100 N->getOperand(0).getValueType().isVector() && 2101 "Operand types must be vectors"); 2102 2103 EVT LoVT, HiVT; 2104 SDLoc DL(N); 2105 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 2106 2107 // If the input also splits, handle it directly. Otherwise split it by hand. 2108 SDValue LL, LH, RL, RH; 2109 if (getTypeAction(N->getOperand(0).getValueType()) == 2110 TargetLowering::TypeSplitVector) 2111 GetSplitVector(N->getOperand(0), LL, LH); 2112 else 2113 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0); 2114 2115 if (getTypeAction(N->getOperand(1).getValueType()) == 2116 TargetLowering::TypeSplitVector) 2117 GetSplitVector(N->getOperand(1), RL, RH); 2118 else 2119 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1); 2120 2121 if (N->getOpcode() == ISD::SETCC) { 2122 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 2123 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 2124 } else { 2125 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode"); 2126 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 2127 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3)); 2128 std::tie(EVLLo, EVLHi) = 2129 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL); 2130 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2), MaskLo, 2131 EVLLo); 2132 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2), MaskHi, 2133 EVLHi); 2134 } 2135 } 2136 2137 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 2138 SDValue &Hi) { 2139 // Get the dest types - they may not match the input types, e.g. int_to_fp. 2140 EVT LoVT, HiVT; 2141 SDLoc dl(N); 2142 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 2143 2144 // If the input also splits, handle it directly for a compile time speedup. 2145 // Otherwise split it by hand. 2146 EVT InVT = N->getOperand(0).getValueType(); 2147 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 2148 GetSplitVector(N->getOperand(0), Lo, Hi); 2149 else 2150 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); 2151 2152 const SDNodeFlags Flags = N->getFlags(); 2153 unsigned Opcode = N->getOpcode(); 2154 if (N->getNumOperands() <= 2) { 2155 if (Opcode == ISD::FP_ROUND) { 2156 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags); 2157 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags); 2158 } else { 2159 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags); 2160 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags); 2161 } 2162 return; 2163 } 2164 2165 assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); 2166 assert(N->isVPOpcode() && "Expected VP opcode"); 2167 2168 SDValue MaskLo, MaskHi; 2169 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 2170 2171 SDValue EVLLo, EVLHi; 2172 std::tie(EVLLo, EVLHi) = 2173 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); 2174 2175 Lo = DAG.getNode(Opcode, dl, LoVT, {Lo, MaskLo, EVLLo}, Flags); 2176 Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags); 2177 } 2178 2179 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, 2180 SDValue &Hi) { 2181 SDLoc dl(N); 2182 EVT SrcVT = N->getOperand(0).getValueType(); 2183 EVT DestVT = N->getValueType(0); 2184 EVT LoVT, HiVT; 2185 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT); 2186 2187 // We can do better than a generic split operation if the extend is doing 2188 // more than just doubling the width of the elements and the following are 2189 // true: 2190 // - The number of vector elements is even, 2191 // - the source type is legal, 2192 // - the type of a split source is illegal, 2193 // - the type of an extended (by doubling element size) source is legal, and 2194 // - the type of that extended source when split is legal. 2195 // 2196 // This won't necessarily completely legalize the operation, but it will 2197 // more effectively move in the right direction and prevent falling down 2198 // to scalarization in many cases due to the input vector being split too 2199 // far. 2200 if (SrcVT.getVectorElementCount().isKnownEven() && 2201 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) { 2202 LLVMContext &Ctx = *DAG.getContext(); 2203 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx); 2204 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx); 2205 2206 EVT SplitLoVT, SplitHiVT; 2207 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT); 2208 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) && 2209 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) { 2210 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:"; 2211 N->dump(&DAG); dbgs() << "\n"); 2212 if (!N->isVPOpcode()) { 2213 // Extend the source vector by one step. 2214 SDValue NewSrc = 2215 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0)); 2216 // Get the low and high halves of the new, extended one step, vector. 2217 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl); 2218 // Extend those vector halves the rest of the way. 2219 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 2220 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 2221 return; 2222 } 2223 2224 // Extend the source vector by one step. 2225 SDValue NewSrc = 2226 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0), 2227 N->getOperand(1), N->getOperand(2)); 2228 // Get the low and high halves of the new, extended one step, vector. 2229 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl); 2230 2231 SDValue MaskLo, MaskHi; 2232 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 2233 2234 SDValue EVLLo, EVLHi; 2235 std::tie(EVLLo, EVLHi) = 2236 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); 2237 // Extend those vector halves the rest of the way. 2238 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo}); 2239 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi}); 2240 return; 2241 } 2242 } 2243 // Fall back to the generic unary operator splitting otherwise. 2244 SplitVecRes_UnaryOp(N, Lo, Hi); 2245 } 2246 2247 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 2248 SDValue &Lo, SDValue &Hi) { 2249 // The low and high parts of the original input give four input vectors. 2250 SDValue Inputs[4]; 2251 SDLoc DL(N); 2252 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 2253 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 2254 EVT NewVT = Inputs[0].getValueType(); 2255 unsigned NewElts = NewVT.getVectorNumElements(); 2256 2257 auto &&IsConstant = [](const SDValue &N) { 2258 APInt SplatValue; 2259 return N.getResNo() == 0 && 2260 (ISD::isConstantSplatVector(N.getNode(), SplatValue) || 2261 ISD::isBuildVectorOfConstantSDNodes(N.getNode())); 2262 }; 2263 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1, 2264 SDValue &Input2, 2265 ArrayRef<int> Mask) { 2266 assert(Input1->getOpcode() == ISD::BUILD_VECTOR && 2267 Input2->getOpcode() == ISD::BUILD_VECTOR && 2268 "Expected build vector node."); 2269 EVT EltVT = NewVT.getVectorElementType(); 2270 SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT)); 2271 for (unsigned I = 0; I < NewElts; ++I) { 2272 if (Mask[I] == UndefMaskElem) 2273 continue; 2274 unsigned Idx = Mask[I]; 2275 if (Idx >= NewElts) 2276 Ops[I] = Input2.getOperand(Idx - NewElts); 2277 else 2278 Ops[I] = Input1.getOperand(Idx); 2279 // Make the type of all elements the same as the element type. 2280 if (Ops[I].getValueType().bitsGT(EltVT)) 2281 Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]); 2282 } 2283 return DAG.getBuildVector(NewVT, DL, Ops); 2284 }; 2285 2286 // If Lo or Hi uses elements from at most two of the four input vectors, then 2287 // express it as a vector shuffle of those two inputs. Otherwise extract the 2288 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 2289 SmallVector<int> OrigMask(N->getMask().begin(), N->getMask().end()); 2290 // Try to pack incoming shuffles/inputs. 2291 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts, 2292 &DL](SmallVectorImpl<int> &Mask) { 2293 // Check if all inputs are shuffles of the same operands or non-shuffles. 2294 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs; 2295 for (unsigned Idx = 0; Idx < array_lengthof(Inputs); ++Idx) { 2296 SDValue Input = Inputs[Idx]; 2297 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode()); 2298 if (!Shuffle || 2299 Input.getOperand(0).getValueType() != Input.getValueType()) 2300 continue; 2301 ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))] 2302 .push_back(Idx); 2303 ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))] 2304 .push_back(Idx); 2305 } 2306 for (auto &P : ShufflesIdxs) { 2307 if (P.second.size() < 2) 2308 continue; 2309 // Use shuffles operands instead of shuffles themselves. 2310 // 1. Adjust mask. 2311 for (int &Idx : Mask) { 2312 if (Idx == UndefMaskElem) 2313 continue; 2314 unsigned SrcRegIdx = Idx / NewElts; 2315 if (Inputs[SrcRegIdx].isUndef()) { 2316 Idx = UndefMaskElem; 2317 continue; 2318 } 2319 auto *Shuffle = 2320 dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode()); 2321 if (!Shuffle || !is_contained(P.second, SrcRegIdx)) 2322 continue; 2323 int MaskElt = Shuffle->getMaskElt(Idx % NewElts); 2324 if (MaskElt == UndefMaskElem) { 2325 Idx = UndefMaskElem; 2326 continue; 2327 } 2328 Idx = MaskElt % NewElts + 2329 P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first 2330 ? 0 2331 : 1] * 2332 NewElts; 2333 } 2334 // 2. Update inputs. 2335 Inputs[P.second[0]] = P.first.first; 2336 Inputs[P.second[1]] = P.first.second; 2337 // Clear the pair data. 2338 P.second.clear(); 2339 ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear(); 2340 } 2341 // Check if any concat_vectors can be simplified. 2342 SmallBitVector UsedSubVector(2 * array_lengthof(Inputs)); 2343 for (int &Idx : Mask) { 2344 if (Idx == UndefMaskElem) 2345 continue; 2346 unsigned SrcRegIdx = Idx / NewElts; 2347 if (Inputs[SrcRegIdx].isUndef()) { 2348 Idx = UndefMaskElem; 2349 continue; 2350 } 2351 TargetLowering::LegalizeTypeAction TypeAction = 2352 getTypeAction(Inputs[SrcRegIdx].getValueType()); 2353 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS && 2354 Inputs[SrcRegIdx].getNumOperands() == 2 && 2355 !Inputs[SrcRegIdx].getOperand(1).isUndef() && 2356 (TypeAction == TargetLowering::TypeLegal || 2357 TypeAction == TargetLowering::TypeWidenVector)) 2358 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2)); 2359 } 2360 if (UsedSubVector.count() > 1) { 2361 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs; 2362 for (unsigned I = 0; I < array_lengthof(Inputs); ++I) { 2363 if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1)) 2364 continue; 2365 if (Pairs.empty() || Pairs.back().size() == 2) 2366 Pairs.emplace_back(); 2367 if (UsedSubVector.test(2 * I)) { 2368 Pairs.back().emplace_back(I, 0); 2369 } else { 2370 assert(UsedSubVector.test(2 * I + 1) && 2371 "Expected to be used one of the subvectors."); 2372 Pairs.back().emplace_back(I, 1); 2373 } 2374 } 2375 if (!Pairs.empty() && Pairs.front().size() > 1) { 2376 // Adjust mask. 2377 for (int &Idx : Mask) { 2378 if (Idx == UndefMaskElem) 2379 continue; 2380 unsigned SrcRegIdx = Idx / NewElts; 2381 auto *It = find_if( 2382 Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) { 2383 return Idxs.front().first == SrcRegIdx || 2384 Idxs.back().first == SrcRegIdx; 2385 }); 2386 if (It == Pairs.end()) 2387 continue; 2388 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) + 2389 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2)); 2390 } 2391 // Adjust inputs. 2392 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) { 2393 Inputs[Idxs.front().first] = DAG.getNode( 2394 ISD::CONCAT_VECTORS, DL, 2395 Inputs[Idxs.front().first].getValueType(), 2396 Inputs[Idxs.front().first].getOperand(Idxs.front().second), 2397 Inputs[Idxs.back().first].getOperand(Idxs.back().second)); 2398 } 2399 } 2400 } 2401 bool Changed; 2402 do { 2403 // Try to remove extra shuffles (except broadcasts) and shuffles with the 2404 // reused operands. 2405 Changed = false; 2406 for (unsigned I = 0; I < array_lengthof(Inputs); ++I) { 2407 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode()); 2408 if (!Shuffle) 2409 continue; 2410 if (Shuffle->getOperand(0).getValueType() != NewVT) 2411 continue; 2412 int Op = -1; 2413 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() && 2414 !Shuffle->isSplat()) { 2415 Op = 0; 2416 } else if (!Inputs[I].hasOneUse() && 2417 !Shuffle->getOperand(1).isUndef()) { 2418 // Find the only used operand, if possible. 2419 for (int &Idx : Mask) { 2420 if (Idx == UndefMaskElem) 2421 continue; 2422 unsigned SrcRegIdx = Idx / NewElts; 2423 if (SrcRegIdx != I) 2424 continue; 2425 int MaskElt = Shuffle->getMaskElt(Idx % NewElts); 2426 if (MaskElt == UndefMaskElem) { 2427 Idx = UndefMaskElem; 2428 continue; 2429 } 2430 int OpIdx = MaskElt / NewElts; 2431 if (Op == -1) { 2432 Op = OpIdx; 2433 continue; 2434 } 2435 if (Op != OpIdx) { 2436 Op = -1; 2437 break; 2438 } 2439 } 2440 } 2441 if (Op < 0) { 2442 // Try to check if one of the shuffle operands is used already. 2443 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) { 2444 if (Shuffle->getOperand(OpIdx).isUndef()) 2445 continue; 2446 auto *It = find(Inputs, Shuffle->getOperand(OpIdx)); 2447 if (It == std::end(Inputs)) 2448 continue; 2449 int FoundOp = std::distance(std::begin(Inputs), It); 2450 // Found that operand is used already. 2451 // 1. Fix the mask for the reused operand. 2452 for (int &Idx : Mask) { 2453 if (Idx == UndefMaskElem) 2454 continue; 2455 unsigned SrcRegIdx = Idx / NewElts; 2456 if (SrcRegIdx != I) 2457 continue; 2458 int MaskElt = Shuffle->getMaskElt(Idx % NewElts); 2459 if (MaskElt == UndefMaskElem) { 2460 Idx = UndefMaskElem; 2461 continue; 2462 } 2463 int MaskIdx = MaskElt / NewElts; 2464 if (OpIdx == MaskIdx) 2465 Idx = MaskElt % NewElts + FoundOp * NewElts; 2466 } 2467 // 2. Set Op to the unused OpIdx. 2468 Op = (OpIdx + 1) % 2; 2469 break; 2470 } 2471 } 2472 if (Op >= 0) { 2473 Changed = true; 2474 Inputs[I] = Shuffle->getOperand(Op); 2475 // Adjust mask. 2476 for (int &Idx : Mask) { 2477 if (Idx == UndefMaskElem) 2478 continue; 2479 unsigned SrcRegIdx = Idx / NewElts; 2480 if (SrcRegIdx != I) 2481 continue; 2482 int MaskElt = Shuffle->getMaskElt(Idx % NewElts); 2483 int OpIdx = MaskElt / NewElts; 2484 if (OpIdx != Op) 2485 continue; 2486 Idx = MaskElt % NewElts + SrcRegIdx * NewElts; 2487 } 2488 } 2489 } 2490 } while (Changed); 2491 }; 2492 TryPeekThroughShufflesInputs(OrigMask); 2493 // Proces unique inputs. 2494 auto &&MakeUniqueInputs = [&Inputs, &IsConstant, 2495 NewElts](SmallVectorImpl<int> &Mask) { 2496 SetVector<SDValue> UniqueInputs; 2497 SetVector<SDValue> UniqueConstantInputs; 2498 for (unsigned I = 0; I < array_lengthof(Inputs); ++I) { 2499 if (IsConstant(Inputs[I])) 2500 UniqueConstantInputs.insert(Inputs[I]); 2501 else if (!Inputs[I].isUndef()) 2502 UniqueInputs.insert(Inputs[I]); 2503 } 2504 // Adjust mask in case of reused inputs. Also, need to insert constant 2505 // inputs at first, otherwise it affects the final outcome. 2506 if (UniqueInputs.size() != array_lengthof(Inputs)) { 2507 auto &&UniqueVec = UniqueInputs.takeVector(); 2508 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector(); 2509 unsigned ConstNum = UniqueConstantVec.size(); 2510 for (int &Idx : Mask) { 2511 if (Idx == UndefMaskElem) 2512 continue; 2513 unsigned SrcRegIdx = Idx / NewElts; 2514 if (Inputs[SrcRegIdx].isUndef()) { 2515 Idx = UndefMaskElem; 2516 continue; 2517 } 2518 const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]); 2519 if (It != UniqueConstantVec.end()) { 2520 Idx = (Idx % NewElts) + 2521 NewElts * std::distance(UniqueConstantVec.begin(), It); 2522 assert(Idx >= 0 && "Expected defined mask idx."); 2523 continue; 2524 } 2525 const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]); 2526 assert(RegIt != UniqueVec.end() && "Cannot find non-const value."); 2527 Idx = (Idx % NewElts) + 2528 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum); 2529 assert(Idx >= 0 && "Expected defined mask idx."); 2530 } 2531 copy(UniqueConstantVec, std::begin(Inputs)); 2532 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum)); 2533 } 2534 }; 2535 MakeUniqueInputs(OrigMask); 2536 SDValue OrigInputs[4]; 2537 copy(Inputs, std::begin(OrigInputs)); 2538 for (unsigned High = 0; High < 2; ++High) { 2539 SDValue &Output = High ? Hi : Lo; 2540 2541 // Build a shuffle mask for the output, discovering on the fly which 2542 // input vectors to use as shuffle operands. 2543 unsigned FirstMaskIdx = High * NewElts; 2544 SmallVector<int> Mask(NewElts * array_lengthof(Inputs), UndefMaskElem); 2545 copy(makeArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin()); 2546 assert(!Output && "Expected default initialized initial value."); 2547 TryPeekThroughShufflesInputs(Mask); 2548 MakeUniqueInputs(Mask); 2549 SDValue TmpInputs[4]; 2550 copy(Inputs, std::begin(TmpInputs)); 2551 // Track changes in the output registers. 2552 int UsedIdx = -1; 2553 bool SecondIteration = false; 2554 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) { 2555 if (UsedIdx < 0) { 2556 UsedIdx = Idx; 2557 return false; 2558 } 2559 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx) 2560 SecondIteration = true; 2561 return SecondIteration; 2562 }; 2563 processShuffleMasks( 2564 Mask, array_lengthof(Inputs), array_lengthof(Inputs), 2565 /*NumOfUsedRegs=*/1, 2566 [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); }, 2567 [&Output, &DAG = DAG, NewVT, &DL, &Inputs, 2568 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) { 2569 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR) 2570 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask); 2571 else 2572 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx], 2573 DAG.getUNDEF(NewVT), Mask); 2574 Inputs[Idx] = Output; 2575 }, 2576 [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs, 2577 &TmpInputs, 2578 &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) { 2579 if (AccumulateResults(Idx1)) { 2580 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR && 2581 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR) 2582 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask); 2583 else 2584 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1], 2585 Inputs[Idx2], Mask); 2586 } else { 2587 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR && 2588 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR) 2589 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask); 2590 else 2591 Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1], 2592 TmpInputs[Idx2], Mask); 2593 } 2594 Inputs[Idx1] = Output; 2595 }); 2596 copy(OrigInputs, std::begin(Inputs)); 2597 } 2598 } 2599 2600 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { 2601 EVT OVT = N->getValueType(0); 2602 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext()); 2603 SDValue Chain = N->getOperand(0); 2604 SDValue Ptr = N->getOperand(1); 2605 SDValue SV = N->getOperand(2); 2606 SDLoc dl(N); 2607 2608 const Align Alignment = 2609 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext())); 2610 2611 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value()); 2612 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value()); 2613 Chain = Hi.getValue(1); 2614 2615 // Modified the chain - switch anything that used the old chain to use 2616 // the new one. 2617 ReplaceValueWith(SDValue(N, 1), Chain); 2618 } 2619 2620 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, 2621 SDValue &Hi) { 2622 EVT DstVTLo, DstVTHi; 2623 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0)); 2624 SDLoc dl(N); 2625 2626 SDValue SrcLo, SrcHi; 2627 EVT SrcVT = N->getOperand(0).getValueType(); 2628 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector) 2629 GetSplitVector(N->getOperand(0), SrcLo, SrcHi); 2630 else 2631 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0); 2632 2633 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1)); 2634 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1)); 2635 } 2636 2637 void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo, 2638 SDValue &Hi) { 2639 SDValue InLo, InHi; 2640 GetSplitVector(N->getOperand(0), InLo, InHi); 2641 SDLoc DL(N); 2642 2643 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi); 2644 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo); 2645 } 2646 2647 void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo, 2648 SDValue &Hi) { 2649 EVT VT = N->getValueType(0); 2650 SDLoc DL(N); 2651 2652 EVT LoVT, HiVT; 2653 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); 2654 2655 SDValue Expanded = TLI.expandVectorSplice(N, DAG); 2656 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded, 2657 DAG.getVectorIdxConstant(0, DL)); 2658 Hi = 2659 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded, 2660 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL)); 2661 } 2662 2663 //===----------------------------------------------------------------------===// 2664 // Operand Vector Splitting 2665 //===----------------------------------------------------------------------===// 2666 2667 /// This method is called when the specified operand of the specified node is 2668 /// found to need vector splitting. At this point, all of the result types of 2669 /// the node are known to be legal, but other operands of the node may need 2670 /// legalization as well as the specified one. 2671 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 2672 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n"); 2673 SDValue Res = SDValue(); 2674 2675 // See if the target wants to custom split this node. 2676 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2677 return false; 2678 2679 switch (N->getOpcode()) { 2680 default: 2681 #ifndef NDEBUG 2682 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 2683 N->dump(&DAG); 2684 dbgs() << "\n"; 2685 #endif 2686 report_fatal_error("Do not know how to split this operator's " 2687 "operand!\n"); 2688 2689 case ISD::VP_SETCC: 2690 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 2691 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 2692 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 2693 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break; 2694 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 2695 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 2696 case ISD::VP_TRUNCATE: 2697 case ISD::TRUNCATE: 2698 Res = SplitVecOp_TruncateHelper(N); 2699 break; 2700 case ISD::STRICT_FP_ROUND: 2701 case ISD::VP_FP_ROUND: 2702 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 2703 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break; 2704 case ISD::STORE: 2705 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 2706 break; 2707 case ISD::VP_STORE: 2708 Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo); 2709 break; 2710 case ISD::MSTORE: 2711 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo); 2712 break; 2713 case ISD::MSCATTER: 2714 case ISD::VP_SCATTER: 2715 Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo); 2716 break; 2717 case ISD::MGATHER: 2718 case ISD::VP_GATHER: 2719 Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo); 2720 break; 2721 case ISD::VSELECT: 2722 Res = SplitVecOp_VSELECT(N, OpNo); 2723 break; 2724 case ISD::STRICT_SINT_TO_FP: 2725 case ISD::STRICT_UINT_TO_FP: 2726 case ISD::SINT_TO_FP: 2727 case ISD::UINT_TO_FP: 2728 if (N->getValueType(0).bitsLT( 2729 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType())) 2730 Res = SplitVecOp_TruncateHelper(N); 2731 else 2732 Res = SplitVecOp_UnaryOp(N); 2733 break; 2734 case ISD::FP_TO_SINT_SAT: 2735 case ISD::FP_TO_UINT_SAT: 2736 Res = SplitVecOp_FP_TO_XINT_SAT(N); 2737 break; 2738 case ISD::FP_TO_SINT: 2739 case ISD::FP_TO_UINT: 2740 case ISD::STRICT_FP_TO_SINT: 2741 case ISD::STRICT_FP_TO_UINT: 2742 case ISD::STRICT_FP_EXTEND: 2743 case ISD::FP_EXTEND: 2744 case ISD::SIGN_EXTEND: 2745 case ISD::ZERO_EXTEND: 2746 case ISD::ANY_EXTEND: 2747 case ISD::FTRUNC: 2748 Res = SplitVecOp_UnaryOp(N); 2749 break; 2750 2751 case ISD::ANY_EXTEND_VECTOR_INREG: 2752 case ISD::SIGN_EXTEND_VECTOR_INREG: 2753 case ISD::ZERO_EXTEND_VECTOR_INREG: 2754 Res = SplitVecOp_ExtVecInRegOp(N); 2755 break; 2756 2757 case ISD::VECREDUCE_FADD: 2758 case ISD::VECREDUCE_FMUL: 2759 case ISD::VECREDUCE_ADD: 2760 case ISD::VECREDUCE_MUL: 2761 case ISD::VECREDUCE_AND: 2762 case ISD::VECREDUCE_OR: 2763 case ISD::VECREDUCE_XOR: 2764 case ISD::VECREDUCE_SMAX: 2765 case ISD::VECREDUCE_SMIN: 2766 case ISD::VECREDUCE_UMAX: 2767 case ISD::VECREDUCE_UMIN: 2768 case ISD::VECREDUCE_FMAX: 2769 case ISD::VECREDUCE_FMIN: 2770 Res = SplitVecOp_VECREDUCE(N, OpNo); 2771 break; 2772 case ISD::VECREDUCE_SEQ_FADD: 2773 case ISD::VECREDUCE_SEQ_FMUL: 2774 Res = SplitVecOp_VECREDUCE_SEQ(N); 2775 break; 2776 case ISD::VP_REDUCE_FADD: 2777 case ISD::VP_REDUCE_SEQ_FADD: 2778 case ISD::VP_REDUCE_FMUL: 2779 case ISD::VP_REDUCE_SEQ_FMUL: 2780 case ISD::VP_REDUCE_ADD: 2781 case ISD::VP_REDUCE_MUL: 2782 case ISD::VP_REDUCE_AND: 2783 case ISD::VP_REDUCE_OR: 2784 case ISD::VP_REDUCE_XOR: 2785 case ISD::VP_REDUCE_SMAX: 2786 case ISD::VP_REDUCE_SMIN: 2787 case ISD::VP_REDUCE_UMAX: 2788 case ISD::VP_REDUCE_UMIN: 2789 case ISD::VP_REDUCE_FMAX: 2790 case ISD::VP_REDUCE_FMIN: 2791 Res = SplitVecOp_VP_REDUCE(N, OpNo); 2792 break; 2793 } 2794 2795 // If the result is null, the sub-method took care of registering results etc. 2796 if (!Res.getNode()) return false; 2797 2798 // If the result is N, the sub-method updated N in place. Tell the legalizer 2799 // core about this. 2800 if (Res.getNode() == N) 2801 return true; 2802 2803 if (N->isStrictFPOpcode()) 2804 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && 2805 "Invalid operand expansion"); 2806 else 2807 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2808 "Invalid operand expansion"); 2809 2810 ReplaceValueWith(SDValue(N, 0), Res); 2811 return false; 2812 } 2813 2814 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 2815 // The only possibility for an illegal operand is the mask, since result type 2816 // legalization would have handled this node already otherwise. 2817 assert(OpNo == 0 && "Illegal operand must be mask"); 2818 2819 SDValue Mask = N->getOperand(0); 2820 SDValue Src0 = N->getOperand(1); 2821 SDValue Src1 = N->getOperand(2); 2822 EVT Src0VT = Src0.getValueType(); 2823 SDLoc DL(N); 2824 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?"); 2825 2826 SDValue Lo, Hi; 2827 GetSplitVector(N->getOperand(0), Lo, Hi); 2828 assert(Lo.getValueType() == Hi.getValueType() && 2829 "Lo and Hi have differing types"); 2830 2831 EVT LoOpVT, HiOpVT; 2832 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT); 2833 assert(LoOpVT == HiOpVT && "Asymmetric vector split?"); 2834 2835 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask; 2836 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL); 2837 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL); 2838 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL); 2839 2840 SDValue LoSelect = 2841 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 2842 SDValue HiSelect = 2843 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 2844 2845 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 2846 } 2847 2848 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) { 2849 EVT ResVT = N->getValueType(0); 2850 SDValue Lo, Hi; 2851 SDLoc dl(N); 2852 2853 SDValue VecOp = N->getOperand(OpNo); 2854 EVT VecVT = VecOp.getValueType(); 2855 assert(VecVT.isVector() && "Can only split reduce vector operand"); 2856 GetSplitVector(VecOp, Lo, Hi); 2857 EVT LoOpVT, HiOpVT; 2858 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT); 2859 2860 // Use the appropriate scalar instruction on the split subvectors before 2861 // reducing the now partially reduced smaller vector. 2862 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode()); 2863 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags()); 2864 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags()); 2865 } 2866 2867 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) { 2868 EVT ResVT = N->getValueType(0); 2869 SDValue Lo, Hi; 2870 SDLoc dl(N); 2871 2872 SDValue AccOp = N->getOperand(0); 2873 SDValue VecOp = N->getOperand(1); 2874 SDNodeFlags Flags = N->getFlags(); 2875 2876 EVT VecVT = VecOp.getValueType(); 2877 assert(VecVT.isVector() && "Can only split reduce vector operand"); 2878 GetSplitVector(VecOp, Lo, Hi); 2879 EVT LoOpVT, HiOpVT; 2880 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT); 2881 2882 // Reduce low half. 2883 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags); 2884 2885 // Reduce high half, using low half result as initial value. 2886 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags); 2887 } 2888 2889 SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) { 2890 assert(N->isVPOpcode() && "Expected VP opcode"); 2891 assert(OpNo == 1 && "Can only split reduce vector operand"); 2892 2893 unsigned Opc = N->getOpcode(); 2894 EVT ResVT = N->getValueType(0); 2895 SDValue Lo, Hi; 2896 SDLoc dl(N); 2897 2898 SDValue VecOp = N->getOperand(OpNo); 2899 EVT VecVT = VecOp.getValueType(); 2900 assert(VecVT.isVector() && "Can only split reduce vector operand"); 2901 GetSplitVector(VecOp, Lo, Hi); 2902 2903 SDValue MaskLo, MaskHi; 2904 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2)); 2905 2906 SDValue EVLLo, EVLHi; 2907 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl); 2908 2909 const SDNodeFlags Flags = N->getFlags(); 2910 2911 SDValue ResLo = 2912 DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags); 2913 return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags); 2914 } 2915 2916 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 2917 // The result has a legal vector type, but the input needs splitting. 2918 EVT ResVT = N->getValueType(0); 2919 SDValue Lo, Hi; 2920 SDLoc dl(N); 2921 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); 2922 EVT InVT = Lo.getValueType(); 2923 2924 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 2925 InVT.getVectorElementCount()); 2926 2927 if (N->isStrictFPOpcode()) { 2928 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 2929 { N->getOperand(0), Lo }); 2930 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 2931 { N->getOperand(0), Hi }); 2932 2933 // Build a factor node to remember that this operation is independent 2934 // of the other one. 2935 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2936 Hi.getValue(1)); 2937 2938 // Legalize the chain result - switch anything that used the old chain to 2939 // use the new one. 2940 ReplaceValueWith(SDValue(N, 1), Ch); 2941 } else if (N->getNumOperands() == 3) { 2942 assert(N->isVPOpcode() && "Expected VP opcode"); 2943 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 2944 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 2945 std::tie(EVLLo, EVLHi) = 2946 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); 2947 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo, MaskLo, EVLLo); 2948 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi, MaskHi, EVLHi); 2949 } else { 2950 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 2951 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 2952 } 2953 2954 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 2955 } 2956 2957 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 2958 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 2959 // end up being split all the way down to individual components. Convert the 2960 // split pieces into integers and reassemble. 2961 SDValue Lo, Hi; 2962 GetSplitVector(N->getOperand(0), Lo, Hi); 2963 Lo = BitConvertToInteger(Lo); 2964 Hi = BitConvertToInteger(Hi); 2965 2966 if (DAG.getDataLayout().isBigEndian()) 2967 std::swap(Lo, Hi); 2968 2969 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), 2970 JoinIntegers(Lo, Hi)); 2971 } 2972 2973 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N, 2974 unsigned OpNo) { 2975 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec."); 2976 // We know that the result type is legal. 2977 EVT ResVT = N->getValueType(0); 2978 2979 SDValue Vec = N->getOperand(0); 2980 SDValue SubVec = N->getOperand(1); 2981 SDValue Idx = N->getOperand(2); 2982 SDLoc dl(N); 2983 2984 SDValue Lo, Hi; 2985 GetSplitVector(SubVec, Lo, Hi); 2986 2987 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2988 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); 2989 2990 SDValue FirstInsertion = 2991 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx); 2992 SDValue SecondInsertion = 2993 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi, 2994 DAG.getVectorIdxConstant(IdxVal + LoElts, dl)); 2995 2996 return SecondInsertion; 2997 } 2998 2999 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 3000 // We know that the extracted result type is legal. 3001 EVT SubVT = N->getValueType(0); 3002 3003 SDValue Idx = N->getOperand(1); 3004 SDLoc dl(N); 3005 SDValue Lo, Hi; 3006 3007 if (SubVT.isScalableVector() != 3008 N->getOperand(0).getValueType().isScalableVector()) 3009 report_fatal_error("Extracting a fixed-length vector from an illegal " 3010 "scalable vector is not yet supported"); 3011 3012 GetSplitVector(N->getOperand(0), Lo, Hi); 3013 3014 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); 3015 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 3016 3017 if (IdxVal < LoElts) { 3018 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts && 3019 "Extracted subvector crosses vector split!"); 3020 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 3021 } else { 3022 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 3023 DAG.getVectorIdxConstant(IdxVal - LoElts, dl)); 3024 } 3025 } 3026 3027 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 3028 SDValue Vec = N->getOperand(0); 3029 SDValue Idx = N->getOperand(1); 3030 EVT VecVT = Vec.getValueType(); 3031 3032 if (isa<ConstantSDNode>(Idx)) { 3033 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 3034 3035 SDValue Lo, Hi; 3036 GetSplitVector(Vec, Lo, Hi); 3037 3038 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); 3039 3040 if (IdxVal < LoElts) 3041 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 3042 else if (!Vec.getValueType().isScalableVector()) 3043 return SDValue(DAG.UpdateNodeOperands(N, Hi, 3044 DAG.getConstant(IdxVal - LoElts, SDLoc(N), 3045 Idx.getValueType())), 0); 3046 } 3047 3048 // See if the target wants to custom expand this node. 3049 if (CustomLowerNode(N, N->getValueType(0), true)) 3050 return SDValue(); 3051 3052 // Make the vector elements byte-addressable if they aren't already. 3053 SDLoc dl(N); 3054 EVT EltVT = VecVT.getVectorElementType(); 3055 if (VecVT.getScalarSizeInBits() < 8) { 3056 EltVT = MVT::i8; 3057 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 3058 VecVT.getVectorElementCount()); 3059 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec); 3060 } 3061 3062 // Store the vector to the stack. 3063 // In cases where the vector is illegal it will be broken down into parts 3064 // and stored in parts - we should use the alignment for the smallest part. 3065 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false); 3066 SDValue StackPtr = 3067 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign); 3068 auto &MF = DAG.getMachineFunction(); 3069 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 3070 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 3071 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 3072 SmallestAlign); 3073 3074 // Load back the required element. 3075 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 3076 3077 // FIXME: This is to handle i1 vectors with elements promoted to i8. 3078 // i1 vector handling needs general improvement. 3079 if (N->getValueType(0).bitsLT(EltVT)) { 3080 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr, 3081 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction())); 3082 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0)); 3083 } 3084 3085 return DAG.getExtLoad( 3086 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 3087 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT, 3088 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8)); 3089 } 3090 3091 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) { 3092 SDValue Lo, Hi; 3093 3094 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so 3095 // splitting the result has the same effect as splitting the input operand. 3096 SplitVecRes_ExtVecInRegOp(N, Lo, Hi); 3097 3098 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi); 3099 } 3100 3101 SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) { 3102 (void)OpNo; 3103 SDValue Lo, Hi; 3104 SplitVecRes_Gather(N, Lo, Hi); 3105 3106 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi); 3107 ReplaceValueWith(SDValue(N, 0), Res); 3108 return SDValue(); 3109 } 3110 3111 SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) { 3112 assert(N->isUnindexed() && "Indexed vp_store of vector?"); 3113 SDValue Ch = N->getChain(); 3114 SDValue Ptr = N->getBasePtr(); 3115 SDValue Offset = N->getOffset(); 3116 assert(Offset.isUndef() && "Unexpected VP store offset"); 3117 SDValue Mask = N->getMask(); 3118 SDValue EVL = N->getVectorLength(); 3119 SDValue Data = N->getValue(); 3120 Align Alignment = N->getOriginalAlign(); 3121 SDLoc DL(N); 3122 3123 SDValue DataLo, DataHi; 3124 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 3125 // Split Data operand 3126 GetSplitVector(Data, DataLo, DataHi); 3127 else 3128 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL); 3129 3130 // Split Mask operand 3131 SDValue MaskLo, MaskHi; 3132 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) { 3133 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 3134 } else { 3135 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 3136 GetSplitVector(Mask, MaskLo, MaskHi); 3137 else 3138 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 3139 } 3140 3141 EVT MemoryVT = N->getMemoryVT(); 3142 EVT LoMemVT, HiMemVT; 3143 bool HiIsEmpty = false; 3144 std::tie(LoMemVT, HiMemVT) = 3145 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty); 3146 3147 // Split EVL 3148 SDValue EVLLo, EVLHi; 3149 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL); 3150 3151 SDValue Lo, Hi; 3152 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 3153 N->getPointerInfo(), MachineMemOperand::MOStore, 3154 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges()); 3155 3156 Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO, 3157 N->getAddressingMode(), N->isTruncatingStore(), 3158 N->isCompressingStore()); 3159 3160 // If the hi vp_store has zero storage size, only the lo vp_store is needed. 3161 if (HiIsEmpty) 3162 return Lo; 3163 3164 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG, 3165 N->isCompressingStore()); 3166 3167 MachinePointerInfo MPI; 3168 if (LoMemVT.isScalableVector()) { 3169 Alignment = commonAlignment(Alignment, 3170 LoMemVT.getSizeInBits().getKnownMinSize() / 8); 3171 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); 3172 } else 3173 MPI = N->getPointerInfo().getWithOffset( 3174 LoMemVT.getStoreSize().getFixedSize()); 3175 3176 MMO = DAG.getMachineFunction().getMachineMemOperand( 3177 MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment, 3178 N->getAAInfo(), N->getRanges()); 3179 3180 Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO, 3181 N->getAddressingMode(), N->isTruncatingStore(), 3182 N->isCompressingStore()); 3183 3184 // Build a factor node to remember that this store is independent of the 3185 // other one. 3186 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 3187 } 3188 3189 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N, 3190 unsigned OpNo) { 3191 assert(N->isUnindexed() && "Indexed masked store of vector?"); 3192 SDValue Ch = N->getChain(); 3193 SDValue Ptr = N->getBasePtr(); 3194 SDValue Offset = N->getOffset(); 3195 assert(Offset.isUndef() && "Unexpected indexed masked store offset"); 3196 SDValue Mask = N->getMask(); 3197 SDValue Data = N->getValue(); 3198 Align Alignment = N->getOriginalAlign(); 3199 SDLoc DL(N); 3200 3201 SDValue DataLo, DataHi; 3202 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 3203 // Split Data operand 3204 GetSplitVector(Data, DataLo, DataHi); 3205 else 3206 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL); 3207 3208 // Split Mask operand 3209 SDValue MaskLo, MaskHi; 3210 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) { 3211 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 3212 } else { 3213 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 3214 GetSplitVector(Mask, MaskLo, MaskHi); 3215 else 3216 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 3217 } 3218 3219 EVT MemoryVT = N->getMemoryVT(); 3220 EVT LoMemVT, HiMemVT; 3221 bool HiIsEmpty = false; 3222 std::tie(LoMemVT, HiMemVT) = 3223 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty); 3224 3225 SDValue Lo, Hi, Res; 3226 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 3227 N->getPointerInfo(), MachineMemOperand::MOStore, 3228 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges()); 3229 3230 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO, 3231 N->getAddressingMode(), N->isTruncatingStore(), 3232 N->isCompressingStore()); 3233 3234 if (HiIsEmpty) { 3235 // The hi masked store has zero storage size. 3236 // Only the lo masked store is needed. 3237 Res = Lo; 3238 } else { 3239 3240 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG, 3241 N->isCompressingStore()); 3242 3243 MachinePointerInfo MPI; 3244 if (LoMemVT.isScalableVector()) { 3245 Alignment = commonAlignment( 3246 Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8); 3247 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); 3248 } else 3249 MPI = N->getPointerInfo().getWithOffset( 3250 LoMemVT.getStoreSize().getFixedSize()); 3251 3252 MMO = DAG.getMachineFunction().getMachineMemOperand( 3253 MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment, 3254 N->getAAInfo(), N->getRanges()); 3255 3256 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO, 3257 N->getAddressingMode(), N->isTruncatingStore(), 3258 N->isCompressingStore()); 3259 3260 // Build a factor node to remember that this store is independent of the 3261 // other one. 3262 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 3263 } 3264 3265 return Res; 3266 } 3267 3268 SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) { 3269 SDValue Ch = N->getChain(); 3270 SDValue Ptr = N->getBasePtr(); 3271 EVT MemoryVT = N->getMemoryVT(); 3272 Align Alignment = N->getOriginalAlign(); 3273 SDLoc DL(N); 3274 struct Operands { 3275 SDValue Mask; 3276 SDValue Index; 3277 SDValue Scale; 3278 SDValue Data; 3279 } Ops = [&]() -> Operands { 3280 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) { 3281 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(), 3282 MSC->getValue()}; 3283 } 3284 auto *VPSC = cast<VPScatterSDNode>(N); 3285 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(), 3286 VPSC->getValue()}; 3287 }(); 3288 // Split all operands 3289 3290 EVT LoMemVT, HiMemVT; 3291 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 3292 3293 SDValue DataLo, DataHi; 3294 if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector) 3295 // Split Data operand 3296 GetSplitVector(Ops.Data, DataLo, DataHi); 3297 else 3298 std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL); 3299 3300 // Split Mask operand 3301 SDValue MaskLo, MaskHi; 3302 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) { 3303 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi); 3304 } else { 3305 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL); 3306 } 3307 3308 SDValue IndexHi, IndexLo; 3309 if (getTypeAction(Ops.Index.getValueType()) == 3310 TargetLowering::TypeSplitVector) 3311 GetSplitVector(Ops.Index, IndexLo, IndexHi); 3312 else 3313 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL); 3314 3315 SDValue Lo; 3316 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 3317 N->getPointerInfo(), MachineMemOperand::MOStore, 3318 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges()); 3319 3320 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) { 3321 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale}; 3322 Lo = 3323 DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO, 3324 MSC->getIndexType(), MSC->isTruncatingStore()); 3325 3326 // The order of the Scatter operation after split is well defined. The "Hi" 3327 // part comes after the "Lo". So these two operations should be chained one 3328 // after another. 3329 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale}; 3330 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, 3331 MMO, MSC->getIndexType(), 3332 MSC->isTruncatingStore()); 3333 } 3334 auto *VPSC = cast<VPScatterSDNode>(N); 3335 SDValue EVLLo, EVLHi; 3336 std::tie(EVLLo, EVLHi) = 3337 DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL); 3338 3339 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo}; 3340 Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO, 3341 VPSC->getIndexType()); 3342 3343 // The order of the Scatter operation after split is well defined. The "Hi" 3344 // part comes after the "Lo". So these two operations should be chained one 3345 // after another. 3346 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi}; 3347 return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO, 3348 VPSC->getIndexType()); 3349 } 3350 3351 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 3352 assert(N->isUnindexed() && "Indexed store of vector?"); 3353 assert(OpNo == 1 && "Can only split the stored value"); 3354 SDLoc DL(N); 3355 3356 bool isTruncating = N->isTruncatingStore(); 3357 SDValue Ch = N->getChain(); 3358 SDValue Ptr = N->getBasePtr(); 3359 EVT MemoryVT = N->getMemoryVT(); 3360 Align Alignment = N->getOriginalAlign(); 3361 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 3362 AAMDNodes AAInfo = N->getAAInfo(); 3363 SDValue Lo, Hi; 3364 GetSplitVector(N->getOperand(1), Lo, Hi); 3365 3366 EVT LoMemVT, HiMemVT; 3367 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 3368 3369 // Scalarize if the split halves are not byte-sized. 3370 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) 3371 return TLI.scalarizeVectorStore(N, DAG); 3372 3373 if (isTruncating) 3374 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT, 3375 Alignment, MMOFlags, AAInfo); 3376 else 3377 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags, 3378 AAInfo); 3379 3380 MachinePointerInfo MPI; 3381 IncrementPointer(N, LoMemVT, MPI, Ptr); 3382 3383 if (isTruncating) 3384 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI, 3385 HiMemVT, Alignment, MMOFlags, AAInfo); 3386 else 3387 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo); 3388 3389 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 3390 } 3391 3392 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 3393 SDLoc DL(N); 3394 3395 // The input operands all must have the same type, and we know the result 3396 // type is valid. Convert this to a buildvector which extracts all the 3397 // input elements. 3398 // TODO: If the input elements are power-two vectors, we could convert this to 3399 // a new CONCAT_VECTORS node with elements that are half-wide. 3400 SmallVector<SDValue, 32> Elts; 3401 EVT EltVT = N->getValueType(0).getVectorElementType(); 3402 for (const SDValue &Op : N->op_values()) { 3403 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 3404 i != e; ++i) { 3405 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op, 3406 DAG.getVectorIdxConstant(i, DL))); 3407 } 3408 } 3409 3410 return DAG.getBuildVector(N->getValueType(0), DL, Elts); 3411 } 3412 3413 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) { 3414 // The result type is legal, but the input type is illegal. If splitting 3415 // ends up with the result type of each half still being legal, just 3416 // do that. If, however, that would result in an illegal result type, 3417 // we can try to get more clever with power-two vectors. Specifically, 3418 // split the input type, but also widen the result element size, then 3419 // concatenate the halves and truncate again. For example, consider a target 3420 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit 3421 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: 3422 // %inlo = v4i32 extract_subvector %in, 0 3423 // %inhi = v4i32 extract_subvector %in, 4 3424 // %lo16 = v4i16 trunc v4i32 %inlo 3425 // %hi16 = v4i16 trunc v4i32 %inhi 3426 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 3427 // %res = v8i8 trunc v8i16 %in16 3428 // 3429 // Without this transform, the original truncate would end up being 3430 // scalarized, which is pretty much always a last resort. 3431 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 3432 SDValue InVec = N->getOperand(OpNo); 3433 EVT InVT = InVec->getValueType(0); 3434 EVT OutVT = N->getValueType(0); 3435 ElementCount NumElements = OutVT.getVectorElementCount(); 3436 bool IsFloat = OutVT.isFloatingPoint(); 3437 3438 unsigned InElementSize = InVT.getScalarSizeInBits(); 3439 unsigned OutElementSize = OutVT.getScalarSizeInBits(); 3440 3441 // Determine the split output VT. If its legal we can just split dirctly. 3442 EVT LoOutVT, HiOutVT; 3443 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT); 3444 assert(LoOutVT == HiOutVT && "Unequal split?"); 3445 3446 // If the input elements are only 1/2 the width of the result elements, 3447 // just use the normal splitting. Our trick only work if there's room 3448 // to split more than once. 3449 if (isTypeLegal(LoOutVT) || 3450 InElementSize <= OutElementSize * 2) 3451 return SplitVecOp_UnaryOp(N); 3452 SDLoc DL(N); 3453 3454 // Don't touch if this will be scalarized. 3455 EVT FinalVT = InVT; 3456 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector) 3457 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext()); 3458 3459 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector) 3460 return SplitVecOp_UnaryOp(N); 3461 3462 // Get the split input vector. 3463 SDValue InLoVec, InHiVec; 3464 GetSplitVector(InVec, InLoVec, InHiVec); 3465 3466 // Truncate them to 1/2 the element size. 3467 // 3468 // This assumes the number of elements is a power of two; any vector that 3469 // isn't should be widened, not split. 3470 EVT HalfElementVT = IsFloat ? 3471 EVT::getFloatingPointVT(InElementSize/2) : 3472 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); 3473 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, 3474 NumElements.divideCoefficientBy(2)); 3475 3476 SDValue HalfLo; 3477 SDValue HalfHi; 3478 SDValue Chain; 3479 if (N->isStrictFPOpcode()) { 3480 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other}, 3481 {N->getOperand(0), InLoVec}); 3482 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other}, 3483 {N->getOperand(0), InHiVec}); 3484 // Legalize the chain result - switch anything that used the old chain to 3485 // use the new one. 3486 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1), 3487 HalfHi.getValue(1)); 3488 } else { 3489 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec); 3490 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec); 3491 } 3492 3493 // Concatenate them to get the full intermediate truncation result. 3494 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); 3495 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, 3496 HalfHi); 3497 // Now finish up by truncating all the way down to the original result 3498 // type. This should normally be something that ends up being legal directly, 3499 // but in theory if a target has very wide vectors and an annoyingly 3500 // restricted set of legal types, this split can chain to build things up. 3501 3502 if (N->isStrictFPOpcode()) { 3503 SDValue Res = DAG.getNode( 3504 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other}, 3505 {Chain, InterVec, 3506 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))}); 3507 // Relink the chain 3508 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 3509 return Res; 3510 } 3511 3512 return IsFloat 3513 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec, 3514 DAG.getTargetConstant( 3515 0, DL, TLI.getPointerTy(DAG.getDataLayout()))) 3516 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); 3517 } 3518 3519 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 3520 assert(N->getValueType(0).isVector() && 3521 N->getOperand(0).getValueType().isVector() && 3522 "Operand types must be vectors"); 3523 // The result has a legal vector type, but the input needs splitting. 3524 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 3525 SDLoc DL(N); 3526 GetSplitVector(N->getOperand(0), Lo0, Hi0); 3527 GetSplitVector(N->getOperand(1), Lo1, Hi1); 3528 auto PartEltCnt = Lo0.getValueType().getVectorElementCount(); 3529 3530 LLVMContext &Context = *DAG.getContext(); 3531 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt); 3532 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2); 3533 3534 if (N->getOpcode() == ISD::SETCC) { 3535 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 3536 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 3537 } else { 3538 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode"); 3539 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 3540 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3)); 3541 std::tie(EVLLo, EVLHi) = 3542 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL); 3543 LoRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Lo0, Lo1, 3544 N->getOperand(2), MaskLo, EVLLo); 3545 HiRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Hi0, Hi1, 3546 N->getOperand(2), MaskHi, EVLHi); 3547 } 3548 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 3549 3550 EVT OpVT = N->getOperand(0).getValueType(); 3551 ISD::NodeType ExtendCode = 3552 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 3553 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con); 3554 } 3555 3556 3557 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 3558 // The result has a legal vector type, but the input needs splitting. 3559 EVT ResVT = N->getValueType(0); 3560 SDValue Lo, Hi; 3561 SDLoc DL(N); 3562 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); 3563 EVT InVT = Lo.getValueType(); 3564 3565 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 3566 InVT.getVectorElementCount()); 3567 3568 if (N->isStrictFPOpcode()) { 3569 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, 3570 { N->getOperand(0), Lo, N->getOperand(2) }); 3571 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, 3572 { N->getOperand(0), Hi, N->getOperand(2) }); 3573 // Legalize the chain result - switch anything that used the old chain to 3574 // use the new one. 3575 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 3576 Lo.getValue(1), Hi.getValue(1)); 3577 ReplaceValueWith(SDValue(N, 1), NewChain); 3578 } else if (N->getOpcode() == ISD::VP_FP_ROUND) { 3579 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 3580 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 3581 std::tie(EVLLo, EVLHi) = 3582 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), DL); 3583 Lo = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Lo, MaskLo, EVLLo); 3584 Hi = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Hi, MaskHi, EVLHi); 3585 } else { 3586 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 3587 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 3588 } 3589 3590 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 3591 } 3592 3593 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) { 3594 // The result (and the first input) has a legal vector type, but the second 3595 // input needs splitting. 3596 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements()); 3597 } 3598 3599 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) { 3600 EVT ResVT = N->getValueType(0); 3601 SDValue Lo, Hi; 3602 SDLoc dl(N); 3603 GetSplitVector(N->getOperand(0), Lo, Hi); 3604 EVT InVT = Lo.getValueType(); 3605 3606 EVT NewResVT = 3607 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 3608 InVT.getVectorElementCount()); 3609 3610 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1)); 3611 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1)); 3612 3613 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 3614 } 3615 3616 //===----------------------------------------------------------------------===// 3617 // Result Vector Widening 3618 //===----------------------------------------------------------------------===// 3619 3620 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 3621 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG); 3622 dbgs() << "\n"); 3623 3624 // See if the target wants to custom widen this node. 3625 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 3626 return; 3627 3628 SDValue Res = SDValue(); 3629 3630 auto unrollExpandedOp = [&]() { 3631 // We're going to widen this vector op to a legal type by padding with undef 3632 // elements. If the wide vector op is eventually going to be expanded to 3633 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary 3634 // libcalls on the undef elements. 3635 EVT VT = N->getValueType(0); 3636 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3637 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) && 3638 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) { 3639 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements()); 3640 return true; 3641 } 3642 return false; 3643 }; 3644 3645 switch (N->getOpcode()) { 3646 default: 3647 #ifndef NDEBUG 3648 dbgs() << "WidenVectorResult #" << ResNo << ": "; 3649 N->dump(&DAG); 3650 dbgs() << "\n"; 3651 #endif 3652 llvm_unreachable("Do not know how to widen the result of this operator!"); 3653 3654 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 3655 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 3656 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 3657 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 3658 case ISD::INSERT_SUBVECTOR: 3659 Res = WidenVecRes_INSERT_SUBVECTOR(N); 3660 break; 3661 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 3662 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 3663 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 3664 case ISD::STEP_VECTOR: 3665 case ISD::SPLAT_VECTOR: 3666 case ISD::SCALAR_TO_VECTOR: 3667 Res = WidenVecRes_ScalarOp(N); 3668 break; 3669 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 3670 case ISD::VSELECT: 3671 case ISD::SELECT: 3672 case ISD::VP_SELECT: 3673 case ISD::VP_MERGE: 3674 Res = WidenVecRes_Select(N); 3675 break; 3676 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 3677 case ISD::VP_SETCC: 3678 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 3679 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 3680 case ISD::VECTOR_SHUFFLE: 3681 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 3682 break; 3683 case ISD::VP_LOAD: 3684 Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(N)); 3685 break; 3686 case ISD::MLOAD: 3687 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N)); 3688 break; 3689 case ISD::MGATHER: 3690 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N)); 3691 break; 3692 case ISD::VP_GATHER: 3693 Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(N)); 3694 break; 3695 3696 case ISD::ADD: case ISD::VP_ADD: 3697 case ISD::AND: case ISD::VP_AND: 3698 case ISD::MUL: case ISD::VP_MUL: 3699 case ISD::MULHS: 3700 case ISD::MULHU: 3701 case ISD::OR: case ISD::VP_OR: 3702 case ISD::SUB: case ISD::VP_SUB: 3703 case ISD::XOR: case ISD::VP_XOR: 3704 case ISD::SHL: case ISD::VP_SHL: 3705 case ISD::SRA: case ISD::VP_ASHR: 3706 case ISD::SRL: case ISD::VP_LSHR: 3707 case ISD::FMINNUM: 3708 case ISD::FMAXNUM: 3709 case ISD::FMINIMUM: 3710 case ISD::FMAXIMUM: 3711 case ISD::SMIN: 3712 case ISD::SMAX: 3713 case ISD::UMIN: 3714 case ISD::UMAX: 3715 case ISD::UADDSAT: 3716 case ISD::SADDSAT: 3717 case ISD::USUBSAT: 3718 case ISD::SSUBSAT: 3719 case ISD::SSHLSAT: 3720 case ISD::USHLSAT: 3721 case ISD::ROTL: 3722 case ISD::ROTR: 3723 case ISD::AVGFLOORS: 3724 case ISD::AVGFLOORU: 3725 case ISD::AVGCEILS: 3726 case ISD::AVGCEILU: 3727 // Vector-predicated binary op widening. Note that -- unlike the 3728 // unpredicated versions -- we don't have to worry about trapping on 3729 // operations like UDIV, FADD, etc., as we pass on the original vector 3730 // length parameter. This means the widened elements containing garbage 3731 // aren't active. 3732 case ISD::VP_SDIV: 3733 case ISD::VP_UDIV: 3734 case ISD::VP_SREM: 3735 case ISD::VP_UREM: 3736 case ISD::VP_FADD: 3737 case ISD::VP_FSUB: 3738 case ISD::VP_FMUL: 3739 case ISD::VP_FDIV: 3740 case ISD::VP_FREM: 3741 Res = WidenVecRes_Binary(N); 3742 break; 3743 3744 case ISD::FPOW: 3745 case ISD::FREM: 3746 if (unrollExpandedOp()) 3747 break; 3748 // If the target has custom/legal support for the scalar FP intrinsic ops 3749 // (they are probably not destined to become libcalls), then widen those 3750 // like any other binary ops. 3751 LLVM_FALLTHROUGH; 3752 3753 case ISD::FADD: 3754 case ISD::FMUL: 3755 case ISD::FSUB: 3756 case ISD::FDIV: 3757 case ISD::SDIV: 3758 case ISD::UDIV: 3759 case ISD::SREM: 3760 case ISD::UREM: 3761 Res = WidenVecRes_BinaryCanTrap(N); 3762 break; 3763 3764 case ISD::SMULFIX: 3765 case ISD::SMULFIXSAT: 3766 case ISD::UMULFIX: 3767 case ISD::UMULFIXSAT: 3768 // These are binary operations, but with an extra operand that shouldn't 3769 // be widened (the scale). 3770 Res = WidenVecRes_BinaryWithExtraScalarOp(N); 3771 break; 3772 3773 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 3774 case ISD::STRICT_##DAGN: 3775 #include "llvm/IR/ConstrainedOps.def" 3776 Res = WidenVecRes_StrictFP(N); 3777 break; 3778 3779 case ISD::UADDO: 3780 case ISD::SADDO: 3781 case ISD::USUBO: 3782 case ISD::SSUBO: 3783 case ISD::UMULO: 3784 case ISD::SMULO: 3785 Res = WidenVecRes_OverflowOp(N, ResNo); 3786 break; 3787 3788 case ISD::FCOPYSIGN: 3789 Res = WidenVecRes_FCOPYSIGN(N); 3790 break; 3791 3792 case ISD::IS_FPCLASS: 3793 Res = WidenVecRes_IS_FPCLASS(N); 3794 break; 3795 3796 case ISD::FPOWI: 3797 Res = WidenVecRes_POWI(N); 3798 break; 3799 3800 case ISD::ANY_EXTEND_VECTOR_INREG: 3801 case ISD::SIGN_EXTEND_VECTOR_INREG: 3802 case ISD::ZERO_EXTEND_VECTOR_INREG: 3803 Res = WidenVecRes_EXTEND_VECTOR_INREG(N); 3804 break; 3805 3806 case ISD::ANY_EXTEND: 3807 case ISD::FP_EXTEND: 3808 case ISD::VP_FP_EXTEND: 3809 case ISD::FP_ROUND: 3810 case ISD::VP_FP_ROUND: 3811 case ISD::FP_TO_SINT: 3812 case ISD::VP_FPTOSI: 3813 case ISD::FP_TO_UINT: 3814 case ISD::VP_FPTOUI: 3815 case ISD::SIGN_EXTEND: 3816 case ISD::VP_SIGN_EXTEND: 3817 case ISD::SINT_TO_FP: 3818 case ISD::VP_SITOFP: 3819 case ISD::VP_TRUNCATE: 3820 case ISD::TRUNCATE: 3821 case ISD::UINT_TO_FP: 3822 case ISD::VP_UITOFP: 3823 case ISD::ZERO_EXTEND: 3824 case ISD::VP_ZERO_EXTEND: 3825 Res = WidenVecRes_Convert(N); 3826 break; 3827 3828 case ISD::FP_TO_SINT_SAT: 3829 case ISD::FP_TO_UINT_SAT: 3830 Res = WidenVecRes_FP_TO_XINT_SAT(N); 3831 break; 3832 3833 case ISD::FABS: 3834 case ISD::FCEIL: 3835 case ISD::FCOS: 3836 case ISD::FEXP: 3837 case ISD::FEXP2: 3838 case ISD::FFLOOR: 3839 case ISD::FLOG: 3840 case ISD::FLOG10: 3841 case ISD::FLOG2: 3842 case ISD::FNEARBYINT: 3843 case ISD::FRINT: 3844 case ISD::FROUND: 3845 case ISD::FROUNDEVEN: 3846 case ISD::FSIN: 3847 case ISD::FSQRT: 3848 case ISD::FTRUNC: 3849 if (unrollExpandedOp()) 3850 break; 3851 // If the target has custom/legal support for the scalar FP intrinsic ops 3852 // (they are probably not destined to become libcalls), then widen those 3853 // like any other unary ops. 3854 LLVM_FALLTHROUGH; 3855 3856 case ISD::ABS: 3857 case ISD::BITREVERSE: 3858 case ISD::BSWAP: 3859 case ISD::CTLZ: 3860 case ISD::CTLZ_ZERO_UNDEF: 3861 case ISD::CTPOP: 3862 case ISD::CTTZ: 3863 case ISD::CTTZ_ZERO_UNDEF: 3864 case ISD::FNEG: case ISD::VP_FNEG: 3865 case ISD::FREEZE: 3866 case ISD::ARITH_FENCE: 3867 case ISD::FCANONICALIZE: 3868 Res = WidenVecRes_Unary(N); 3869 break; 3870 case ISD::FMA: case ISD::VP_FMA: 3871 case ISD::FSHL: 3872 case ISD::FSHR: 3873 Res = WidenVecRes_Ternary(N); 3874 break; 3875 } 3876 3877 // If Res is null, the sub-method took care of registering the result. 3878 if (Res.getNode()) 3879 SetWidenedVector(SDValue(N, ResNo), Res); 3880 } 3881 3882 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 3883 // Ternary op widening. 3884 SDLoc dl(N); 3885 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3886 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 3887 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 3888 SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 3889 if (N->getNumOperands() == 3) 3890 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 3891 3892 assert(N->getNumOperands() == 5 && "Unexpected number of operands!"); 3893 assert(N->isVPOpcode() && "Expected VP opcode"); 3894 3895 SDValue Mask = 3896 GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount()); 3897 return DAG.getNode(N->getOpcode(), dl, WidenVT, 3898 {InOp1, InOp2, InOp3, Mask, N->getOperand(4)}); 3899 } 3900 3901 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 3902 // Binary op widening. 3903 SDLoc dl(N); 3904 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3905 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 3906 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 3907 if (N->getNumOperands() == 2) 3908 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, 3909 N->getFlags()); 3910 3911 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 3912 assert(N->isVPOpcode() && "Expected VP opcode"); 3913 3914 SDValue Mask = 3915 GetWidenedMask(N->getOperand(2), WidenVT.getVectorElementCount()); 3916 return DAG.getNode(N->getOpcode(), dl, WidenVT, 3917 {InOp1, InOp2, Mask, N->getOperand(3)}, N->getFlags()); 3918 } 3919 3920 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) { 3921 // Binary op widening, but with an extra operand that shouldn't be widened. 3922 SDLoc dl(N); 3923 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3924 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 3925 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 3926 SDValue InOp3 = N->getOperand(2); 3927 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3, 3928 N->getFlags()); 3929 } 3930 3931 // Given a vector of operations that have been broken up to widen, see 3932 // if we can collect them together into the next widest legal VT. This 3933 // implementation is trap-safe. 3934 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, 3935 SmallVectorImpl<SDValue> &ConcatOps, 3936 unsigned ConcatEnd, EVT VT, EVT MaxVT, 3937 EVT WidenVT) { 3938 // Check to see if we have a single operation with the widen type. 3939 if (ConcatEnd == 1) { 3940 VT = ConcatOps[0].getValueType(); 3941 if (VT == WidenVT) 3942 return ConcatOps[0]; 3943 } 3944 3945 SDLoc dl(ConcatOps[0]); 3946 EVT WidenEltVT = WidenVT.getVectorElementType(); 3947 3948 // while (Some element of ConcatOps is not of type MaxVT) { 3949 // From the end of ConcatOps, collect elements of the same type and put 3950 // them into an op of the next larger supported type 3951 // } 3952 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 3953 int Idx = ConcatEnd - 1; 3954 VT = ConcatOps[Idx--].getValueType(); 3955 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 3956 Idx--; 3957 3958 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 3959 EVT NextVT; 3960 do { 3961 NextSize *= 2; 3962 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 3963 } while (!TLI.isTypeLegal(NextVT)); 3964 3965 if (!VT.isVector()) { 3966 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 3967 SDValue VecOp = DAG.getUNDEF(NextVT); 3968 unsigned NumToInsert = ConcatEnd - Idx - 1; 3969 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 3970 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, 3971 ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl)); 3972 } 3973 ConcatOps[Idx+1] = VecOp; 3974 ConcatEnd = Idx + 2; 3975 } else { 3976 // Vector type, create a CONCAT_VECTORS of type NextVT 3977 SDValue undefVec = DAG.getUNDEF(VT); 3978 unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 3979 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 3980 unsigned RealVals = ConcatEnd - Idx - 1; 3981 unsigned SubConcatEnd = 0; 3982 unsigned SubConcatIdx = Idx + 1; 3983 while (SubConcatEnd < RealVals) 3984 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 3985 while (SubConcatEnd < OpsToConcat) 3986 SubConcatOps[SubConcatEnd++] = undefVec; 3987 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 3988 NextVT, SubConcatOps); 3989 ConcatEnd = SubConcatIdx + 1; 3990 } 3991 } 3992 3993 // Check to see if we have a single operation with the widen type. 3994 if (ConcatEnd == 1) { 3995 VT = ConcatOps[0].getValueType(); 3996 if (VT == WidenVT) 3997 return ConcatOps[0]; 3998 } 3999 4000 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 4001 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 4002 if (NumOps != ConcatEnd ) { 4003 SDValue UndefVal = DAG.getUNDEF(MaxVT); 4004 for (unsigned j = ConcatEnd; j < NumOps; ++j) 4005 ConcatOps[j] = UndefVal; 4006 } 4007 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 4008 makeArrayRef(ConcatOps.data(), NumOps)); 4009 } 4010 4011 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) { 4012 // Binary op widening for operations that can trap. 4013 unsigned Opcode = N->getOpcode(); 4014 SDLoc dl(N); 4015 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4016 EVT WidenEltVT = WidenVT.getVectorElementType(); 4017 EVT VT = WidenVT; 4018 unsigned NumElts = VT.getVectorMinNumElements(); 4019 const SDNodeFlags Flags = N->getFlags(); 4020 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 4021 NumElts = NumElts / 2; 4022 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 4023 } 4024 4025 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 4026 // Operation doesn't trap so just widen as normal. 4027 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 4028 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 4029 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags); 4030 } 4031 4032 // FIXME: Improve support for scalable vectors. 4033 assert(!VT.isScalableVector() && "Scalable vectors not handled yet."); 4034 4035 // No legal vector version so unroll the vector operation and then widen. 4036 if (NumElts == 1) 4037 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 4038 4039 // Since the operation can trap, apply operation on the original vector. 4040 EVT MaxVT = VT; 4041 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 4042 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 4043 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 4044 4045 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 4046 unsigned ConcatEnd = 0; // Current ConcatOps index. 4047 int Idx = 0; // Current Idx into input vectors. 4048 4049 // NumElts := greatest legal vector size (at most WidenVT) 4050 // while (orig. vector has unhandled elements) { 4051 // take munches of size NumElts from the beginning and add to ConcatOps 4052 // NumElts := next smaller supported vector size or 1 4053 // } 4054 while (CurNumElts != 0) { 4055 while (CurNumElts >= NumElts) { 4056 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 4057 DAG.getVectorIdxConstant(Idx, dl)); 4058 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 4059 DAG.getVectorIdxConstant(Idx, dl)); 4060 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags); 4061 Idx += NumElts; 4062 CurNumElts -= NumElts; 4063 } 4064 do { 4065 NumElts = NumElts / 2; 4066 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 4067 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 4068 4069 if (NumElts == 1) { 4070 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 4071 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 4072 InOp1, DAG.getVectorIdxConstant(Idx, dl)); 4073 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 4074 InOp2, DAG.getVectorIdxConstant(Idx, dl)); 4075 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 4076 EOp1, EOp2, Flags); 4077 } 4078 CurNumElts = 0; 4079 } 4080 } 4081 4082 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); 4083 } 4084 4085 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) { 4086 switch (N->getOpcode()) { 4087 case ISD::STRICT_FSETCC: 4088 case ISD::STRICT_FSETCCS: 4089 return WidenVecRes_STRICT_FSETCC(N); 4090 case ISD::STRICT_FP_EXTEND: 4091 case ISD::STRICT_FP_ROUND: 4092 case ISD::STRICT_FP_TO_SINT: 4093 case ISD::STRICT_FP_TO_UINT: 4094 case ISD::STRICT_SINT_TO_FP: 4095 case ISD::STRICT_UINT_TO_FP: 4096 return WidenVecRes_Convert_StrictFP(N); 4097 default: 4098 break; 4099 } 4100 4101 // StrictFP op widening for operations that can trap. 4102 unsigned NumOpers = N->getNumOperands(); 4103 unsigned Opcode = N->getOpcode(); 4104 SDLoc dl(N); 4105 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4106 EVT WidenEltVT = WidenVT.getVectorElementType(); 4107 EVT VT = WidenVT; 4108 unsigned NumElts = VT.getVectorNumElements(); 4109 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 4110 NumElts = NumElts / 2; 4111 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 4112 } 4113 4114 // No legal vector version so unroll the vector operation and then widen. 4115 if (NumElts == 1) 4116 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements()); 4117 4118 // Since the operation can trap, apply operation on the original vector. 4119 EVT MaxVT = VT; 4120 SmallVector<SDValue, 4> InOps; 4121 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 4122 4123 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 4124 SmallVector<SDValue, 16> Chains; 4125 unsigned ConcatEnd = 0; // Current ConcatOps index. 4126 int Idx = 0; // Current Idx into input vectors. 4127 4128 // The Chain is the first operand. 4129 InOps.push_back(N->getOperand(0)); 4130 4131 // Now process the remaining operands. 4132 for (unsigned i = 1; i < NumOpers; ++i) { 4133 SDValue Oper = N->getOperand(i); 4134 4135 if (Oper.getValueType().isVector()) { 4136 assert(Oper.getValueType() == N->getValueType(0) && 4137 "Invalid operand type to widen!"); 4138 Oper = GetWidenedVector(Oper); 4139 } 4140 4141 InOps.push_back(Oper); 4142 } 4143 4144 // NumElts := greatest legal vector size (at most WidenVT) 4145 // while (orig. vector has unhandled elements) { 4146 // take munches of size NumElts from the beginning and add to ConcatOps 4147 // NumElts := next smaller supported vector size or 1 4148 // } 4149 while (CurNumElts != 0) { 4150 while (CurNumElts >= NumElts) { 4151 SmallVector<SDValue, 4> EOps; 4152 4153 for (unsigned i = 0; i < NumOpers; ++i) { 4154 SDValue Op = InOps[i]; 4155 4156 if (Op.getValueType().isVector()) 4157 Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op, 4158 DAG.getVectorIdxConstant(Idx, dl)); 4159 4160 EOps.push_back(Op); 4161 } 4162 4163 EVT OperVT[] = {VT, MVT::Other}; 4164 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps); 4165 ConcatOps[ConcatEnd++] = Oper; 4166 Chains.push_back(Oper.getValue(1)); 4167 Idx += NumElts; 4168 CurNumElts -= NumElts; 4169 } 4170 do { 4171 NumElts = NumElts / 2; 4172 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 4173 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 4174 4175 if (NumElts == 1) { 4176 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 4177 SmallVector<SDValue, 4> EOps; 4178 4179 for (unsigned i = 0; i < NumOpers; ++i) { 4180 SDValue Op = InOps[i]; 4181 4182 if (Op.getValueType().isVector()) 4183 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op, 4184 DAG.getVectorIdxConstant(Idx, dl)); 4185 4186 EOps.push_back(Op); 4187 } 4188 4189 EVT WidenVT[] = {WidenEltVT, MVT::Other}; 4190 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps); 4191 ConcatOps[ConcatEnd++] = Oper; 4192 Chains.push_back(Oper.getValue(1)); 4193 } 4194 CurNumElts = 0; 4195 } 4196 } 4197 4198 // Build a factor node to remember all the Ops that have been created. 4199 SDValue NewChain; 4200 if (Chains.size() == 1) 4201 NewChain = Chains[0]; 4202 else 4203 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 4204 ReplaceValueWith(SDValue(N, 1), NewChain); 4205 4206 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); 4207 } 4208 4209 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) { 4210 SDLoc DL(N); 4211 EVT ResVT = N->getValueType(0); 4212 EVT OvVT = N->getValueType(1); 4213 EVT WideResVT, WideOvVT; 4214 SDValue WideLHS, WideRHS; 4215 4216 // TODO: This might result in a widen/split loop. 4217 if (ResNo == 0) { 4218 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT); 4219 WideOvVT = EVT::getVectorVT( 4220 *DAG.getContext(), OvVT.getVectorElementType(), 4221 WideResVT.getVectorNumElements()); 4222 4223 WideLHS = GetWidenedVector(N->getOperand(0)); 4224 WideRHS = GetWidenedVector(N->getOperand(1)); 4225 } else { 4226 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT); 4227 WideResVT = EVT::getVectorVT( 4228 *DAG.getContext(), ResVT.getVectorElementType(), 4229 WideOvVT.getVectorNumElements()); 4230 4231 SDValue Zero = DAG.getVectorIdxConstant(0, DL); 4232 WideLHS = DAG.getNode( 4233 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT), 4234 N->getOperand(0), Zero); 4235 WideRHS = DAG.getNode( 4236 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT), 4237 N->getOperand(1), Zero); 4238 } 4239 4240 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT); 4241 SDNode *WideNode = DAG.getNode( 4242 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode(); 4243 4244 // Replace the other vector result not being explicitly widened here. 4245 unsigned OtherNo = 1 - ResNo; 4246 EVT OtherVT = N->getValueType(OtherNo); 4247 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) { 4248 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo)); 4249 } else { 4250 SDValue Zero = DAG.getVectorIdxConstant(0, DL); 4251 SDValue OtherVal = DAG.getNode( 4252 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero); 4253 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 4254 } 4255 4256 return SDValue(WideNode, ResNo); 4257 } 4258 4259 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 4260 LLVMContext &Ctx = *DAG.getContext(); 4261 SDValue InOp = N->getOperand(0); 4262 SDLoc DL(N); 4263 4264 EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0)); 4265 ElementCount WidenEC = WidenVT.getVectorElementCount(); 4266 4267 EVT InVT = InOp.getValueType(); 4268 4269 unsigned Opcode = N->getOpcode(); 4270 const SDNodeFlags Flags = N->getFlags(); 4271 4272 // Handle the case of ZERO_EXTEND where the promoted InVT element size does 4273 // not equal that of WidenVT. 4274 if (N->getOpcode() == ISD::ZERO_EXTEND && 4275 getTypeAction(InVT) == TargetLowering::TypePromoteInteger && 4276 TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() != 4277 WidenVT.getScalarSizeInBits()) { 4278 InOp = ZExtPromotedInteger(InOp); 4279 InVT = InOp.getValueType(); 4280 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits()) 4281 Opcode = ISD::TRUNCATE; 4282 } 4283 4284 EVT InEltVT = InVT.getVectorElementType(); 4285 EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenEC); 4286 ElementCount InVTEC = InVT.getVectorElementCount(); 4287 4288 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 4289 InOp = GetWidenedVector(N->getOperand(0)); 4290 InVT = InOp.getValueType(); 4291 InVTEC = InVT.getVectorElementCount(); 4292 if (InVTEC == WidenEC) { 4293 if (N->getNumOperands() == 1) 4294 return DAG.getNode(Opcode, DL, WidenVT, InOp); 4295 if (N->getNumOperands() == 3) { 4296 assert(N->isVPOpcode() && "Expected VP opcode"); 4297 SDValue Mask = 4298 GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount()); 4299 return DAG.getNode(Opcode, DL, WidenVT, InOp, Mask, N->getOperand(2)); 4300 } 4301 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags); 4302 } 4303 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) { 4304 // If both input and result vector types are of same width, extend 4305 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which 4306 // accepts fewer elements in the result than in the input. 4307 if (Opcode == ISD::ANY_EXTEND) 4308 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 4309 if (Opcode == ISD::SIGN_EXTEND) 4310 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 4311 if (Opcode == ISD::ZERO_EXTEND) 4312 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 4313 } 4314 } 4315 4316 if (TLI.isTypeLegal(InWidenVT)) { 4317 // Because the result and the input are different vector types, widening 4318 // the result could create a legal type but widening the input might make 4319 // it an illegal type that might lead to repeatedly splitting the input 4320 // and then widening it. To avoid this, we widen the input only if 4321 // it results in a legal type. 4322 if (WidenEC.isKnownMultipleOf(InVTEC.getKnownMinValue())) { 4323 // Widen the input and call convert on the widened input vector. 4324 unsigned NumConcat = 4325 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue(); 4326 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT)); 4327 Ops[0] = InOp; 4328 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops); 4329 if (N->getNumOperands() == 1) 4330 return DAG.getNode(Opcode, DL, WidenVT, InVec); 4331 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags); 4332 } 4333 4334 if (InVTEC.isKnownMultipleOf(WidenEC.getKnownMinValue())) { 4335 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp, 4336 DAG.getVectorIdxConstant(0, DL)); 4337 // Extract the input and convert the shorten input vector. 4338 if (N->getNumOperands() == 1) 4339 return DAG.getNode(Opcode, DL, WidenVT, InVal); 4340 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags); 4341 } 4342 } 4343 4344 // Otherwise unroll into some nasty scalar code and rebuild the vector. 4345 EVT EltVT = WidenVT.getVectorElementType(); 4346 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(EltVT)); 4347 // Use the original element count so we don't do more scalar opts than 4348 // necessary. 4349 unsigned MinElts = N->getValueType(0).getVectorNumElements(); 4350 for (unsigned i=0; i < MinElts; ++i) { 4351 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 4352 DAG.getVectorIdxConstant(i, DL)); 4353 if (N->getNumOperands() == 1) 4354 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 4355 else 4356 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags); 4357 } 4358 4359 return DAG.getBuildVector(WidenVT, DL, Ops); 4360 } 4361 4362 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) { 4363 SDLoc dl(N); 4364 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4365 ElementCount WidenNumElts = WidenVT.getVectorElementCount(); 4366 4367 SDValue Src = N->getOperand(0); 4368 EVT SrcVT = Src.getValueType(); 4369 4370 // Also widen the input. 4371 if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) { 4372 Src = GetWidenedVector(Src); 4373 SrcVT = Src.getValueType(); 4374 } 4375 4376 // Input and output not widened to the same size, give up. 4377 if (WidenNumElts != SrcVT.getVectorElementCount()) 4378 return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue()); 4379 4380 return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1)); 4381 } 4382 4383 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) { 4384 SDValue InOp = N->getOperand(1); 4385 SDLoc DL(N); 4386 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 4387 4388 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4389 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4390 4391 EVT InVT = InOp.getValueType(); 4392 EVT InEltVT = InVT.getVectorElementType(); 4393 4394 unsigned Opcode = N->getOpcode(); 4395 4396 // FIXME: Optimizations need to be implemented here. 4397 4398 // Otherwise unroll into some nasty scalar code and rebuild the vector. 4399 EVT EltVT = WidenVT.getVectorElementType(); 4400 std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}}; 4401 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT)); 4402 SmallVector<SDValue, 32> OpChains; 4403 // Use the original element count so we don't do more scalar opts than 4404 // necessary. 4405 unsigned MinElts = N->getValueType(0).getVectorNumElements(); 4406 for (unsigned i=0; i < MinElts; ++i) { 4407 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 4408 DAG.getVectorIdxConstant(i, DL)); 4409 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps); 4410 OpChains.push_back(Ops[i].getValue(1)); 4411 } 4412 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains); 4413 ReplaceValueWith(SDValue(N, 1), NewChain); 4414 4415 return DAG.getBuildVector(WidenVT, DL, Ops); 4416 } 4417 4418 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) { 4419 unsigned Opcode = N->getOpcode(); 4420 SDValue InOp = N->getOperand(0); 4421 SDLoc DL(N); 4422 4423 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4424 EVT WidenSVT = WidenVT.getVectorElementType(); 4425 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4426 4427 EVT InVT = InOp.getValueType(); 4428 EVT InSVT = InVT.getVectorElementType(); 4429 unsigned InVTNumElts = InVT.getVectorNumElements(); 4430 4431 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 4432 InOp = GetWidenedVector(InOp); 4433 InVT = InOp.getValueType(); 4434 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) { 4435 switch (Opcode) { 4436 case ISD::ANY_EXTEND_VECTOR_INREG: 4437 case ISD::SIGN_EXTEND_VECTOR_INREG: 4438 case ISD::ZERO_EXTEND_VECTOR_INREG: 4439 return DAG.getNode(Opcode, DL, WidenVT, InOp); 4440 } 4441 } 4442 } 4443 4444 // Unroll, extend the scalars and rebuild the vector. 4445 SmallVector<SDValue, 16> Ops; 4446 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) { 4447 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp, 4448 DAG.getVectorIdxConstant(i, DL)); 4449 switch (Opcode) { 4450 case ISD::ANY_EXTEND_VECTOR_INREG: 4451 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val); 4452 break; 4453 case ISD::SIGN_EXTEND_VECTOR_INREG: 4454 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val); 4455 break; 4456 case ISD::ZERO_EXTEND_VECTOR_INREG: 4457 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val); 4458 break; 4459 default: 4460 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected"); 4461 } 4462 Ops.push_back(Val); 4463 } 4464 4465 while (Ops.size() != WidenNumElts) 4466 Ops.push_back(DAG.getUNDEF(WidenSVT)); 4467 4468 return DAG.getBuildVector(WidenVT, DL, Ops); 4469 } 4470 4471 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) { 4472 // If this is an FCOPYSIGN with same input types, we can treat it as a 4473 // normal (can trap) binary op. 4474 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType()) 4475 return WidenVecRes_BinaryCanTrap(N); 4476 4477 // If the types are different, fall back to unrolling. 4478 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4479 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 4480 } 4481 4482 SDValue DAGTypeLegalizer::WidenVecRes_IS_FPCLASS(SDNode *N) { 4483 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4484 SDValue Arg = GetWidenedVector(N->getOperand(0)); 4485 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, {Arg, N->getOperand(1)}, 4486 N->getFlags()); 4487 } 4488 4489 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 4490 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4491 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4492 SDValue ShOp = N->getOperand(1); 4493 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 4494 } 4495 4496 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 4497 // Unary op widening. 4498 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4499 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4500 if (N->getNumOperands() == 1) 4501 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp); 4502 4503 assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); 4504 assert(N->isVPOpcode() && "Expected VP opcode"); 4505 4506 SDValue Mask = 4507 GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount()); 4508 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, 4509 {InOp, Mask, N->getOperand(2)}); 4510 } 4511 4512 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 4513 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4514 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 4515 cast<VTSDNode>(N->getOperand(1))->getVT() 4516 .getVectorElementType(), 4517 WidenVT.getVectorNumElements()); 4518 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 4519 return DAG.getNode(N->getOpcode(), SDLoc(N), 4520 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 4521 } 4522 4523 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 4524 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 4525 return GetWidenedVector(WidenVec); 4526 } 4527 4528 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 4529 SDValue InOp = N->getOperand(0); 4530 EVT InVT = InOp.getValueType(); 4531 EVT VT = N->getValueType(0); 4532 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4533 SDLoc dl(N); 4534 4535 switch (getTypeAction(InVT)) { 4536 case TargetLowering::TypeLegal: 4537 break; 4538 case TargetLowering::TypeScalarizeScalableVector: 4539 report_fatal_error("Scalarization of scalable vectors is not supported."); 4540 case TargetLowering::TypePromoteInteger: { 4541 // If the incoming type is a vector that is being promoted, then 4542 // we know that the elements are arranged differently and that we 4543 // must perform the conversion using a stack slot. 4544 if (InVT.isVector()) 4545 break; 4546 4547 // If the InOp is promoted to the same size, convert it. Otherwise, 4548 // fall out of the switch and widen the promoted input. 4549 SDValue NInOp = GetPromotedInteger(InOp); 4550 EVT NInVT = NInOp.getValueType(); 4551 if (WidenVT.bitsEq(NInVT)) { 4552 // For big endian targets we need to shift the input integer or the 4553 // interesting bits will end up at the wrong place. 4554 if (DAG.getDataLayout().isBigEndian()) { 4555 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 4556 EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout()); 4557 assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!"); 4558 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp, 4559 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy)); 4560 } 4561 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp); 4562 } 4563 InOp = NInOp; 4564 InVT = NInVT; 4565 break; 4566 } 4567 case TargetLowering::TypeSoftenFloat: 4568 case TargetLowering::TypePromoteFloat: 4569 case TargetLowering::TypeSoftPromoteHalf: 4570 case TargetLowering::TypeExpandInteger: 4571 case TargetLowering::TypeExpandFloat: 4572 case TargetLowering::TypeScalarizeVector: 4573 case TargetLowering::TypeSplitVector: 4574 break; 4575 case TargetLowering::TypeWidenVector: 4576 // If the InOp is widened to the same size, convert it. Otherwise, fall 4577 // out of the switch and widen the widened input. 4578 InOp = GetWidenedVector(InOp); 4579 InVT = InOp.getValueType(); 4580 if (WidenVT.bitsEq(InVT)) 4581 // The input widens to the same size. Convert to the widen value. 4582 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 4583 break; 4584 } 4585 4586 unsigned WidenSize = WidenVT.getSizeInBits(); 4587 unsigned InSize = InVT.getSizeInBits(); 4588 // x86mmx is not an acceptable vector element type, so don't try. 4589 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 4590 // Determine new input vector type. The new input vector type will use 4591 // the same element type (if its a vector) or use the input type as a 4592 // vector. It is the same size as the type to widen to. 4593 EVT NewInVT; 4594 unsigned NewNumElts = WidenSize / InSize; 4595 if (InVT.isVector()) { 4596 EVT InEltVT = InVT.getVectorElementType(); 4597 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 4598 WidenSize / InEltVT.getSizeInBits()); 4599 } else { 4600 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 4601 } 4602 4603 if (TLI.isTypeLegal(NewInVT)) { 4604 SDValue NewVec; 4605 if (InVT.isVector()) { 4606 // Because the result and the input are different vector types, widening 4607 // the result could create a legal type but widening the input might make 4608 // it an illegal type that might lead to repeatedly splitting the input 4609 // and then widening it. To avoid this, we widen the input only if 4610 // it results in a legal type. 4611 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT)); 4612 Ops[0] = InOp; 4613 4614 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops); 4615 } else { 4616 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp); 4617 } 4618 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 4619 } 4620 } 4621 4622 return CreateStackStoreLoad(InOp, WidenVT); 4623 } 4624 4625 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 4626 SDLoc dl(N); 4627 // Build a vector with undefined for the new nodes. 4628 EVT VT = N->getValueType(0); 4629 4630 // Integer BUILD_VECTOR operands may be larger than the node's vector element 4631 // type. The UNDEFs need to have the same type as the existing operands. 4632 EVT EltVT = N->getOperand(0).getValueType(); 4633 unsigned NumElts = VT.getVectorNumElements(); 4634 4635 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4636 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4637 4638 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 4639 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!"); 4640 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT)); 4641 4642 return DAG.getBuildVector(WidenVT, dl, NewOps); 4643 } 4644 4645 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 4646 EVT InVT = N->getOperand(0).getValueType(); 4647 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4648 SDLoc dl(N); 4649 unsigned NumOperands = N->getNumOperands(); 4650 4651 bool InputWidened = false; // Indicates we need to widen the input. 4652 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 4653 unsigned WidenNumElts = WidenVT.getVectorMinNumElements(); 4654 unsigned NumInElts = InVT.getVectorMinNumElements(); 4655 if (WidenNumElts % NumInElts == 0) { 4656 // Add undef vectors to widen to correct length. 4657 unsigned NumConcat = WidenNumElts / NumInElts; 4658 SDValue UndefVal = DAG.getUNDEF(InVT); 4659 SmallVector<SDValue, 16> Ops(NumConcat); 4660 for (unsigned i=0; i < NumOperands; ++i) 4661 Ops[i] = N->getOperand(i); 4662 for (unsigned i = NumOperands; i != NumConcat; ++i) 4663 Ops[i] = UndefVal; 4664 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops); 4665 } 4666 } else { 4667 InputWidened = true; 4668 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 4669 // The inputs and the result are widen to the same value. 4670 unsigned i; 4671 for (i=1; i < NumOperands; ++i) 4672 if (!N->getOperand(i).isUndef()) 4673 break; 4674 4675 if (i == NumOperands) 4676 // Everything but the first operand is an UNDEF so just return the 4677 // widened first operand. 4678 return GetWidenedVector(N->getOperand(0)); 4679 4680 if (NumOperands == 2) { 4681 assert(!WidenVT.isScalableVector() && 4682 "Cannot use vector shuffles to widen CONCAT_VECTOR result"); 4683 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4684 unsigned NumInElts = InVT.getVectorNumElements(); 4685 4686 // Replace concat of two operands with a shuffle. 4687 SmallVector<int, 16> MaskOps(WidenNumElts, -1); 4688 for (unsigned i = 0; i < NumInElts; ++i) { 4689 MaskOps[i] = i; 4690 MaskOps[i + NumInElts] = i + WidenNumElts; 4691 } 4692 return DAG.getVectorShuffle(WidenVT, dl, 4693 GetWidenedVector(N->getOperand(0)), 4694 GetWidenedVector(N->getOperand(1)), 4695 MaskOps); 4696 } 4697 } 4698 } 4699 4700 assert(!WidenVT.isScalableVector() && 4701 "Cannot use build vectors to widen CONCAT_VECTOR result"); 4702 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4703 unsigned NumInElts = InVT.getVectorNumElements(); 4704 4705 // Fall back to use extracts and build vector. 4706 EVT EltVT = WidenVT.getVectorElementType(); 4707 SmallVector<SDValue, 16> Ops(WidenNumElts); 4708 unsigned Idx = 0; 4709 for (unsigned i=0; i < NumOperands; ++i) { 4710 SDValue InOp = N->getOperand(i); 4711 if (InputWidened) 4712 InOp = GetWidenedVector(InOp); 4713 for (unsigned j = 0; j < NumInElts; ++j) 4714 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 4715 DAG.getVectorIdxConstant(j, dl)); 4716 } 4717 SDValue UndefVal = DAG.getUNDEF(EltVT); 4718 for (; Idx < WidenNumElts; ++Idx) 4719 Ops[Idx] = UndefVal; 4720 return DAG.getBuildVector(WidenVT, dl, Ops); 4721 } 4722 4723 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) { 4724 EVT VT = N->getValueType(0); 4725 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4726 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 4727 SDValue InOp2 = N->getOperand(1); 4728 SDValue Idx = N->getOperand(2); 4729 SDLoc dl(N); 4730 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WidenVT, InOp1, InOp2, Idx); 4731 } 4732 4733 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 4734 EVT VT = N->getValueType(0); 4735 EVT EltVT = VT.getVectorElementType(); 4736 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4737 SDValue InOp = N->getOperand(0); 4738 SDValue Idx = N->getOperand(1); 4739 SDLoc dl(N); 4740 4741 auto InOpTypeAction = getTypeAction(InOp.getValueType()); 4742 if (InOpTypeAction == TargetLowering::TypeWidenVector) 4743 InOp = GetWidenedVector(InOp); 4744 4745 EVT InVT = InOp.getValueType(); 4746 4747 // Check if we can just return the input vector after widening. 4748 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 4749 if (IdxVal == 0 && InVT == WidenVT) 4750 return InOp; 4751 4752 // Check if we can extract from the vector. 4753 unsigned WidenNumElts = WidenVT.getVectorMinNumElements(); 4754 unsigned InNumElts = InVT.getVectorMinNumElements(); 4755 unsigned VTNumElts = VT.getVectorMinNumElements(); 4756 assert(IdxVal % VTNumElts == 0 && 4757 "Expected Idx to be a multiple of subvector minimum vector length"); 4758 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 4759 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 4760 4761 if (VT.isScalableVector()) { 4762 // Try to split the operation up into smaller extracts and concat the 4763 // results together, e.g. 4764 // nxv6i64 extract_subvector(nxv12i64, 6) 4765 // <-> 4766 // nxv8i64 concat( 4767 // nxv2i64 extract_subvector(nxv16i64, 6) 4768 // nxv2i64 extract_subvector(nxv16i64, 8) 4769 // nxv2i64 extract_subvector(nxv16i64, 10) 4770 // undef) 4771 unsigned GCD = greatestCommonDivisor(VTNumElts, WidenNumElts); 4772 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken " 4773 "down type's element count"); 4774 EVT PartVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 4775 ElementCount::getScalable(GCD)); 4776 // Avoid recursion around e.g. nxv1i8. 4777 if (getTypeAction(PartVT) != TargetLowering::TypeWidenVector) { 4778 SmallVector<SDValue> Parts; 4779 unsigned I = 0; 4780 for (; I < VTNumElts / GCD; ++I) 4781 Parts.push_back( 4782 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, PartVT, InOp, 4783 DAG.getVectorIdxConstant(IdxVal + I * GCD, dl))); 4784 for (; I < WidenNumElts / GCD; ++I) 4785 Parts.push_back(DAG.getUNDEF(PartVT)); 4786 4787 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Parts); 4788 } 4789 4790 report_fatal_error("Don't know how to widen the result of " 4791 "EXTRACT_SUBVECTOR for scalable vectors"); 4792 } 4793 4794 // We could try widening the input to the right length but for now, extract 4795 // the original elements, fill the rest with undefs and build a vector. 4796 SmallVector<SDValue, 16> Ops(WidenNumElts); 4797 unsigned i; 4798 for (i = 0; i < VTNumElts; ++i) 4799 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 4800 DAG.getVectorIdxConstant(IdxVal + i, dl)); 4801 4802 SDValue UndefVal = DAG.getUNDEF(EltVT); 4803 for (; i < WidenNumElts; ++i) 4804 Ops[i] = UndefVal; 4805 return DAG.getBuildVector(WidenVT, dl, Ops); 4806 } 4807 4808 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 4809 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4810 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), 4811 InOp.getValueType(), InOp, 4812 N->getOperand(1), N->getOperand(2)); 4813 } 4814 4815 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 4816 LoadSDNode *LD = cast<LoadSDNode>(N); 4817 ISD::LoadExtType ExtType = LD->getExtensionType(); 4818 4819 // A vector must always be stored in memory as-is, i.e. without any padding 4820 // between the elements, since various code depend on it, e.g. in the 4821 // handling of a bitcast of a vector type to int, which may be done with a 4822 // vector store followed by an integer load. A vector that does not have 4823 // elements that are byte-sized must therefore be stored as an integer 4824 // built out of the extracted vector elements. 4825 if (!LD->getMemoryVT().isByteSized()) { 4826 SDValue Value, NewChain; 4827 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG); 4828 ReplaceValueWith(SDValue(LD, 0), Value); 4829 ReplaceValueWith(SDValue(LD, 1), NewChain); 4830 return SDValue(); 4831 } 4832 4833 SDValue Result; 4834 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 4835 if (ExtType != ISD::NON_EXTLOAD) 4836 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 4837 else 4838 Result = GenWidenVectorLoads(LdChain, LD); 4839 4840 if (Result) { 4841 // If we generate a single load, we can use that for the chain. Otherwise, 4842 // build a factor node to remember the multiple loads are independent and 4843 // chain to that. 4844 SDValue NewChain; 4845 if (LdChain.size() == 1) 4846 NewChain = LdChain[0]; 4847 else 4848 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain); 4849 4850 // Modified the chain - switch anything that used the old chain to use 4851 // the new one. 4852 ReplaceValueWith(SDValue(N, 1), NewChain); 4853 4854 return Result; 4855 } 4856 4857 // Generate a vector-predicated load if it is custom/legal on the target. To 4858 // avoid possible recursion, only do this if the widened mask type is legal. 4859 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been 4860 // removed from the IR by the ExpandVectorPredication pass but we're 4861 // reintroducing them here. 4862 EVT LdVT = LD->getMemoryVT(); 4863 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), LdVT); 4864 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 4865 WideVT.getVectorElementCount()); 4866 if (ExtType == ISD::NON_EXTLOAD && WideVT.isScalableVector() && 4867 TLI.isOperationLegalOrCustom(ISD::VP_LOAD, WideVT) && 4868 TLI.isTypeLegal(WideMaskVT)) { 4869 SDLoc DL(N); 4870 SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT); 4871 MVT EVLVT = TLI.getVPExplicitVectorLengthTy(); 4872 unsigned NumVTElts = LdVT.getVectorMinNumElements(); 4873 SDValue EVL = 4874 DAG.getVScale(DL, EVLVT, APInt(EVLVT.getScalarSizeInBits(), NumVTElts)); 4875 const auto *MMO = LD->getMemOperand(); 4876 SDValue NewLoad = 4877 DAG.getLoadVP(WideVT, DL, LD->getChain(), LD->getBasePtr(), Mask, EVL, 4878 MMO->getPointerInfo(), MMO->getAlign(), MMO->getFlags(), 4879 MMO->getAAInfo()); 4880 4881 // Modified the chain - switch anything that used the old chain to use 4882 // the new one. 4883 ReplaceValueWith(SDValue(N, 1), NewLoad.getValue(1)); 4884 4885 return NewLoad; 4886 } 4887 4888 report_fatal_error("Unable to widen vector load"); 4889 } 4890 4891 SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) { 4892 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4893 SDValue Mask = N->getMask(); 4894 SDValue EVL = N->getVectorLength(); 4895 ISD::LoadExtType ExtType = N->getExtensionType(); 4896 SDLoc dl(N); 4897 4898 // The mask should be widened as well 4899 assert(getTypeAction(Mask.getValueType()) == 4900 TargetLowering::TypeWidenVector && 4901 "Unable to widen binary VP op"); 4902 Mask = GetWidenedVector(Mask); 4903 assert(Mask.getValueType().getVectorElementCount() == 4904 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType()) 4905 .getVectorElementCount() && 4906 "Unable to widen vector load"); 4907 4908 SDValue Res = 4909 DAG.getLoadVP(N->getAddressingMode(), ExtType, WidenVT, dl, N->getChain(), 4910 N->getBasePtr(), N->getOffset(), Mask, EVL, 4911 N->getMemoryVT(), N->getMemOperand(), N->isExpandingLoad()); 4912 // Legalize the chain result - switch anything that used the old chain to 4913 // use the new one. 4914 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 4915 return Res; 4916 } 4917 4918 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) { 4919 4920 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0)); 4921 SDValue Mask = N->getMask(); 4922 EVT MaskVT = Mask.getValueType(); 4923 SDValue PassThru = GetWidenedVector(N->getPassThru()); 4924 ISD::LoadExtType ExtType = N->getExtensionType(); 4925 SDLoc dl(N); 4926 4927 // The mask should be widened as well 4928 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 4929 MaskVT.getVectorElementType(), 4930 WidenVT.getVectorNumElements()); 4931 Mask = ModifyToType(Mask, WideMaskVT, true); 4932 4933 SDValue Res = DAG.getMaskedLoad( 4934 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, 4935 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(), 4936 ExtType, N->isExpandingLoad()); 4937 // Legalize the chain result - switch anything that used the old chain to 4938 // use the new one. 4939 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 4940 return Res; 4941 } 4942 4943 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) { 4944 4945 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4946 SDValue Mask = N->getMask(); 4947 EVT MaskVT = Mask.getValueType(); 4948 SDValue PassThru = GetWidenedVector(N->getPassThru()); 4949 SDValue Scale = N->getScale(); 4950 unsigned NumElts = WideVT.getVectorNumElements(); 4951 SDLoc dl(N); 4952 4953 // The mask should be widened as well 4954 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 4955 MaskVT.getVectorElementType(), 4956 WideVT.getVectorNumElements()); 4957 Mask = ModifyToType(Mask, WideMaskVT, true); 4958 4959 // Widen the Index operand 4960 SDValue Index = N->getIndex(); 4961 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(), 4962 Index.getValueType().getScalarType(), 4963 NumElts); 4964 Index = ModifyToType(Index, WideIndexVT); 4965 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index, 4966 Scale }; 4967 4968 // Widen the MemoryType 4969 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(), 4970 N->getMemoryVT().getScalarType(), NumElts); 4971 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other), 4972 WideMemVT, dl, Ops, N->getMemOperand(), 4973 N->getIndexType(), N->getExtensionType()); 4974 4975 // Legalize the chain result - switch anything that used the old chain to 4976 // use the new one. 4977 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 4978 return Res; 4979 } 4980 4981 SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) { 4982 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4983 SDValue Mask = N->getMask(); 4984 SDValue Scale = N->getScale(); 4985 ElementCount WideEC = WideVT.getVectorElementCount(); 4986 SDLoc dl(N); 4987 4988 SDValue Index = GetWidenedVector(N->getIndex()); 4989 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(), 4990 N->getMemoryVT().getScalarType(), WideEC); 4991 Mask = GetWidenedMask(Mask, WideEC); 4992 4993 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale, 4994 Mask, N->getVectorLength()}; 4995 SDValue Res = DAG.getGatherVP(DAG.getVTList(WideVT, MVT::Other), WideMemVT, 4996 dl, Ops, N->getMemOperand(), N->getIndexType()); 4997 4998 // Legalize the chain result - switch anything that used the old chain to 4999 // use the new one. 5000 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 5001 return Res; 5002 } 5003 5004 SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) { 5005 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5006 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0)); 5007 } 5008 5009 // Return true is this is a SETCC node or a strict version of it. 5010 static inline bool isSETCCOp(unsigned Opcode) { 5011 switch (Opcode) { 5012 case ISD::SETCC: 5013 case ISD::STRICT_FSETCC: 5014 case ISD::STRICT_FSETCCS: 5015 return true; 5016 } 5017 return false; 5018 } 5019 5020 // Return true if this is a node that could have two SETCCs as operands. 5021 static inline bool isLogicalMaskOp(unsigned Opcode) { 5022 switch (Opcode) { 5023 case ISD::AND: 5024 case ISD::OR: 5025 case ISD::XOR: 5026 return true; 5027 } 5028 return false; 5029 } 5030 5031 // If N is a SETCC or a strict variant of it, return the type 5032 // of the compare operands. 5033 static inline EVT getSETCCOperandType(SDValue N) { 5034 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 5035 return N->getOperand(OpNo).getValueType(); 5036 } 5037 5038 // This is used just for the assert in convertMask(). Check that this either 5039 // a SETCC or a previously handled SETCC by convertMask(). 5040 #ifndef NDEBUG 5041 static inline bool isSETCCorConvertedSETCC(SDValue N) { 5042 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) 5043 N = N.getOperand(0); 5044 else if (N.getOpcode() == ISD::CONCAT_VECTORS) { 5045 for (unsigned i = 1; i < N->getNumOperands(); ++i) 5046 if (!N->getOperand(i)->isUndef()) 5047 return false; 5048 N = N.getOperand(0); 5049 } 5050 5051 if (N.getOpcode() == ISD::TRUNCATE) 5052 N = N.getOperand(0); 5053 else if (N.getOpcode() == ISD::SIGN_EXTEND) 5054 N = N.getOperand(0); 5055 5056 if (isLogicalMaskOp(N.getOpcode())) 5057 return isSETCCorConvertedSETCC(N.getOperand(0)) && 5058 isSETCCorConvertedSETCC(N.getOperand(1)); 5059 5060 return (isSETCCOp(N.getOpcode()) || 5061 ISD::isBuildVectorOfConstantSDNodes(N.getNode())); 5062 } 5063 #endif 5064 5065 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT 5066 // to ToMaskVT if needed with vector extension or truncation. 5067 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT, 5068 EVT ToMaskVT) { 5069 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled. 5070 // FIXME: This code seems to be too restrictive, we might consider 5071 // generalizing it or dropping it. 5072 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument."); 5073 5074 // Make a new Mask node, with a legal result VT. 5075 SDValue Mask; 5076 SmallVector<SDValue, 4> Ops; 5077 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i) 5078 Ops.push_back(InMask->getOperand(i)); 5079 if (InMask->isStrictFPOpcode()) { 5080 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), 5081 { MaskVT, MVT::Other }, Ops); 5082 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1)); 5083 } 5084 else 5085 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops); 5086 5087 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign 5088 // extend or truncate is needed. 5089 LLVMContext &Ctx = *DAG.getContext(); 5090 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits(); 5091 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits(); 5092 if (MaskScalarBits < ToMaskScalBits) { 5093 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(), 5094 MaskVT.getVectorNumElements()); 5095 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask); 5096 } else if (MaskScalarBits > ToMaskScalBits) { 5097 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(), 5098 MaskVT.getVectorNumElements()); 5099 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask); 5100 } 5101 5102 assert(Mask->getValueType(0).getScalarSizeInBits() == 5103 ToMaskVT.getScalarSizeInBits() && 5104 "Mask should have the right element size by now."); 5105 5106 // Adjust Mask to the right number of elements. 5107 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements(); 5108 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) { 5109 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask)); 5110 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask, 5111 ZeroIdx); 5112 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) { 5113 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls); 5114 EVT SubVT = Mask->getValueType(0); 5115 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT)); 5116 SubOps[0] = Mask; 5117 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps); 5118 } 5119 5120 assert((Mask->getValueType(0) == ToMaskVT) && 5121 "A mask of ToMaskVT should have been produced by now."); 5122 5123 return Mask; 5124 } 5125 5126 // This method tries to handle some special cases for the vselect mask 5127 // and if needed adjusting the mask vector type to match that of the VSELECT. 5128 // Without it, many cases end up with scalarization of the SETCC, with many 5129 // unnecessary instructions. 5130 SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) { 5131 LLVMContext &Ctx = *DAG.getContext(); 5132 SDValue Cond = N->getOperand(0); 5133 5134 if (N->getOpcode() != ISD::VSELECT) 5135 return SDValue(); 5136 5137 if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode())) 5138 return SDValue(); 5139 5140 // If this is a splitted VSELECT that was previously already handled, do 5141 // nothing. 5142 EVT CondVT = Cond->getValueType(0); 5143 if (CondVT.getScalarSizeInBits() != 1) 5144 return SDValue(); 5145 5146 EVT VSelVT = N->getValueType(0); 5147 5148 // This method can't handle scalable vector types. 5149 // FIXME: This support could be added in the future. 5150 if (VSelVT.isScalableVector()) 5151 return SDValue(); 5152 5153 // Only handle vector types which are a power of 2. 5154 if (!isPowerOf2_64(VSelVT.getSizeInBits())) 5155 return SDValue(); 5156 5157 // Don't touch if this will be scalarized. 5158 EVT FinalVT = VSelVT; 5159 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector) 5160 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx); 5161 5162 if (FinalVT.getVectorNumElements() == 1) 5163 return SDValue(); 5164 5165 // If there is support for an i1 vector mask, don't touch. 5166 if (isSETCCOp(Cond.getOpcode())) { 5167 EVT SetCCOpVT = getSETCCOperandType(Cond); 5168 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal) 5169 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT); 5170 EVT SetCCResVT = getSetCCResultType(SetCCOpVT); 5171 if (SetCCResVT.getScalarSizeInBits() == 1) 5172 return SDValue(); 5173 } else if (CondVT.getScalarType() == MVT::i1) { 5174 // If there is support for an i1 vector mask (or only scalar i1 conditions), 5175 // don't touch. 5176 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal) 5177 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT); 5178 5179 if (CondVT.getScalarType() == MVT::i1) 5180 return SDValue(); 5181 } 5182 5183 // Widen the vselect result type if needed. 5184 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) 5185 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT); 5186 5187 // The mask of the VSELECT should have integer elements. 5188 EVT ToMaskVT = VSelVT; 5189 if (!ToMaskVT.getScalarType().isInteger()) 5190 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger(); 5191 5192 SDValue Mask; 5193 if (isSETCCOp(Cond->getOpcode())) { 5194 EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond)); 5195 Mask = convertMask(Cond, MaskVT, ToMaskVT); 5196 } else if (isLogicalMaskOp(Cond->getOpcode()) && 5197 isSETCCOp(Cond->getOperand(0).getOpcode()) && 5198 isSETCCOp(Cond->getOperand(1).getOpcode())) { 5199 // Cond is (AND/OR/XOR (SETCC, SETCC)) 5200 SDValue SETCC0 = Cond->getOperand(0); 5201 SDValue SETCC1 = Cond->getOperand(1); 5202 EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0)); 5203 EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1)); 5204 unsigned ScalarBits0 = VT0.getScalarSizeInBits(); 5205 unsigned ScalarBits1 = VT1.getScalarSizeInBits(); 5206 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits(); 5207 EVT MaskVT; 5208 // If the two SETCCs have different VTs, either extend/truncate one of 5209 // them to the other "towards" ToMaskVT, or truncate one and extend the 5210 // other to ToMaskVT. 5211 if (ScalarBits0 != ScalarBits1) { 5212 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1); 5213 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0); 5214 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits()) 5215 MaskVT = WideVT; 5216 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits()) 5217 MaskVT = NarrowVT; 5218 else 5219 MaskVT = ToMaskVT; 5220 } else 5221 // If the two SETCCs have the same VT, don't change it. 5222 MaskVT = VT0; 5223 5224 // Make new SETCCs and logical nodes. 5225 SETCC0 = convertMask(SETCC0, VT0, MaskVT); 5226 SETCC1 = convertMask(SETCC1, VT1, MaskVT); 5227 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1); 5228 5229 // Convert the logical op for VSELECT if needed. 5230 Mask = convertMask(Cond, MaskVT, ToMaskVT); 5231 } else 5232 return SDValue(); 5233 5234 return Mask; 5235 } 5236 5237 SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) { 5238 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5239 ElementCount WidenEC = WidenVT.getVectorElementCount(); 5240 5241 SDValue Cond1 = N->getOperand(0); 5242 EVT CondVT = Cond1.getValueType(); 5243 unsigned Opcode = N->getOpcode(); 5244 if (CondVT.isVector()) { 5245 if (SDValue WideCond = WidenVSELECTMask(N)) { 5246 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 5247 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 5248 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 5249 return DAG.getNode(Opcode, SDLoc(N), WidenVT, WideCond, InOp1, InOp2); 5250 } 5251 5252 EVT CondEltVT = CondVT.getVectorElementType(); 5253 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenEC); 5254 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 5255 Cond1 = GetWidenedVector(Cond1); 5256 5257 // If we have to split the condition there is no point in widening the 5258 // select. This would result in an cycle of widening the select -> 5259 // widening the condition operand -> splitting the condition operand -> 5260 // splitting the select -> widening the select. Instead split this select 5261 // further and widen the resulting type. 5262 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) { 5263 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0); 5264 SDValue Res = ModifyToType(SplitSelect, WidenVT); 5265 return Res; 5266 } 5267 5268 if (Cond1.getValueType() != CondWidenVT) 5269 Cond1 = ModifyToType(Cond1, CondWidenVT); 5270 } 5271 5272 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 5273 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 5274 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 5275 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) 5276 return DAG.getNode(Opcode, SDLoc(N), WidenVT, Cond1, InOp1, InOp2, 5277 N->getOperand(3)); 5278 return DAG.getNode(Opcode, SDLoc(N), WidenVT, Cond1, InOp1, InOp2); 5279 } 5280 5281 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 5282 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 5283 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 5284 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 5285 InOp1.getValueType(), N->getOperand(0), 5286 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 5287 } 5288 5289 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 5290 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5291 return DAG.getUNDEF(WidenVT); 5292 } 5293 5294 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 5295 EVT VT = N->getValueType(0); 5296 SDLoc dl(N); 5297 5298 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5299 unsigned NumElts = VT.getVectorNumElements(); 5300 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 5301 5302 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 5303 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 5304 5305 // Adjust mask based on new input vector length. 5306 SmallVector<int, 16> NewMask; 5307 for (unsigned i = 0; i != NumElts; ++i) { 5308 int Idx = N->getMaskElt(i); 5309 if (Idx < (int)NumElts) 5310 NewMask.push_back(Idx); 5311 else 5312 NewMask.push_back(Idx - NumElts + WidenNumElts); 5313 } 5314 for (unsigned i = NumElts; i != WidenNumElts; ++i) 5315 NewMask.push_back(-1); 5316 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask); 5317 } 5318 5319 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 5320 assert(N->getValueType(0).isVector() && 5321 N->getOperand(0).getValueType().isVector() && 5322 "Operands must be vectors"); 5323 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5324 ElementCount WidenEC = WidenVT.getVectorElementCount(); 5325 5326 SDValue InOp1 = N->getOperand(0); 5327 EVT InVT = InOp1.getValueType(); 5328 assert(InVT.isVector() && "can not widen non-vector type"); 5329 EVT WidenInVT = 5330 EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenEC); 5331 5332 // The input and output types often differ here, and it could be that while 5333 // we'd prefer to widen the result type, the input operands have been split. 5334 // In this case, we also need to split the result of this node as well. 5335 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { 5336 SDValue SplitVSetCC = SplitVecOp_VSETCC(N); 5337 SDValue Res = ModifyToType(SplitVSetCC, WidenVT); 5338 return Res; 5339 } 5340 5341 // If the inputs also widen, handle them directly. Otherwise widen by hand. 5342 SDValue InOp2 = N->getOperand(1); 5343 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 5344 InOp1 = GetWidenedVector(InOp1); 5345 InOp2 = GetWidenedVector(InOp2); 5346 } else { 5347 InOp1 = DAG.WidenVector(InOp1, SDLoc(N)); 5348 InOp2 = DAG.WidenVector(InOp2, SDLoc(N)); 5349 } 5350 5351 // Assume that the input and output will be widen appropriately. If not, 5352 // we will have to unroll it at some point. 5353 assert(InOp1.getValueType() == WidenInVT && 5354 InOp2.getValueType() == WidenInVT && 5355 "Input not widened to expected type!"); 5356 (void)WidenInVT; 5357 if (N->getOpcode() == ISD::VP_SETCC) { 5358 SDValue Mask = 5359 GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount()); 5360 return DAG.getNode(ISD::VP_SETCC, SDLoc(N), WidenVT, InOp1, InOp2, 5361 N->getOperand(2), Mask, N->getOperand(4)); 5362 } 5363 return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT, InOp1, InOp2, 5364 N->getOperand(2)); 5365 } 5366 5367 SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) { 5368 assert(N->getValueType(0).isVector() && 5369 N->getOperand(1).getValueType().isVector() && 5370 "Operands must be vectors"); 5371 EVT VT = N->getValueType(0); 5372 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5373 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 5374 unsigned NumElts = VT.getVectorNumElements(); 5375 EVT EltVT = VT.getVectorElementType(); 5376 5377 SDLoc dl(N); 5378 SDValue Chain = N->getOperand(0); 5379 SDValue LHS = N->getOperand(1); 5380 SDValue RHS = N->getOperand(2); 5381 SDValue CC = N->getOperand(3); 5382 EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 5383 5384 // Fully unroll and reassemble. 5385 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT)); 5386 SmallVector<SDValue, 8> Chains(NumElts); 5387 for (unsigned i = 0; i != NumElts; ++i) { 5388 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 5389 DAG.getVectorIdxConstant(i, dl)); 5390 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 5391 DAG.getVectorIdxConstant(i, dl)); 5392 5393 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other}, 5394 {Chain, LHSElem, RHSElem, CC}); 5395 Chains[i] = Scalars[i].getValue(1); 5396 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i], 5397 DAG.getBoolConstant(true, dl, EltVT, VT), 5398 DAG.getBoolConstant(false, dl, EltVT, VT)); 5399 } 5400 5401 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 5402 ReplaceValueWith(SDValue(N, 1), NewChain); 5403 5404 return DAG.getBuildVector(WidenVT, dl, Scalars); 5405 } 5406 5407 //===----------------------------------------------------------------------===// 5408 // Widen Vector Operand 5409 //===----------------------------------------------------------------------===// 5410 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 5411 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG); 5412 dbgs() << "\n"); 5413 SDValue Res = SDValue(); 5414 5415 // See if the target wants to custom widen this node. 5416 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 5417 return false; 5418 5419 switch (N->getOpcode()) { 5420 default: 5421 #ifndef NDEBUG 5422 dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 5423 N->dump(&DAG); 5424 dbgs() << "\n"; 5425 #endif 5426 llvm_unreachable("Do not know how to widen this operator's operand!"); 5427 5428 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 5429 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 5430 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break; 5431 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 5432 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 5433 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 5434 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break; 5435 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break; 5436 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break; 5437 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break; 5438 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break; 5439 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 5440 case ISD::STRICT_FSETCC: 5441 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break; 5442 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break; 5443 case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break; 5444 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break; 5445 5446 case ISD::ANY_EXTEND: 5447 case ISD::SIGN_EXTEND: 5448 case ISD::ZERO_EXTEND: 5449 Res = WidenVecOp_EXTEND(N); 5450 break; 5451 5452 case ISD::FP_EXTEND: 5453 case ISD::STRICT_FP_EXTEND: 5454 case ISD::FP_ROUND: 5455 case ISD::STRICT_FP_ROUND: 5456 case ISD::FP_TO_SINT: 5457 case ISD::STRICT_FP_TO_SINT: 5458 case ISD::FP_TO_UINT: 5459 case ISD::STRICT_FP_TO_UINT: 5460 case ISD::SINT_TO_FP: 5461 case ISD::STRICT_SINT_TO_FP: 5462 case ISD::UINT_TO_FP: 5463 case ISD::STRICT_UINT_TO_FP: 5464 case ISD::TRUNCATE: 5465 Res = WidenVecOp_Convert(N); 5466 break; 5467 5468 case ISD::FP_TO_SINT_SAT: 5469 case ISD::FP_TO_UINT_SAT: 5470 Res = WidenVecOp_FP_TO_XINT_SAT(N); 5471 break; 5472 5473 case ISD::VECREDUCE_FADD: 5474 case ISD::VECREDUCE_FMUL: 5475 case ISD::VECREDUCE_ADD: 5476 case ISD::VECREDUCE_MUL: 5477 case ISD::VECREDUCE_AND: 5478 case ISD::VECREDUCE_OR: 5479 case ISD::VECREDUCE_XOR: 5480 case ISD::VECREDUCE_SMAX: 5481 case ISD::VECREDUCE_SMIN: 5482 case ISD::VECREDUCE_UMAX: 5483 case ISD::VECREDUCE_UMIN: 5484 case ISD::VECREDUCE_FMAX: 5485 case ISD::VECREDUCE_FMIN: 5486 Res = WidenVecOp_VECREDUCE(N); 5487 break; 5488 case ISD::VECREDUCE_SEQ_FADD: 5489 case ISD::VECREDUCE_SEQ_FMUL: 5490 Res = WidenVecOp_VECREDUCE_SEQ(N); 5491 break; 5492 case ISD::VP_REDUCE_FADD: 5493 case ISD::VP_REDUCE_SEQ_FADD: 5494 case ISD::VP_REDUCE_FMUL: 5495 case ISD::VP_REDUCE_SEQ_FMUL: 5496 case ISD::VP_REDUCE_ADD: 5497 case ISD::VP_REDUCE_MUL: 5498 case ISD::VP_REDUCE_AND: 5499 case ISD::VP_REDUCE_OR: 5500 case ISD::VP_REDUCE_XOR: 5501 case ISD::VP_REDUCE_SMAX: 5502 case ISD::VP_REDUCE_SMIN: 5503 case ISD::VP_REDUCE_UMAX: 5504 case ISD::VP_REDUCE_UMIN: 5505 case ISD::VP_REDUCE_FMAX: 5506 case ISD::VP_REDUCE_FMIN: 5507 Res = WidenVecOp_VP_REDUCE(N); 5508 break; 5509 } 5510 5511 // If Res is null, the sub-method took care of registering the result. 5512 if (!Res.getNode()) return false; 5513 5514 // If the result is N, the sub-method updated N in place. Tell the legalizer 5515 // core about this. 5516 if (Res.getNode() == N) 5517 return true; 5518 5519 5520 if (N->isStrictFPOpcode()) 5521 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && 5522 "Invalid operand expansion"); 5523 else 5524 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 5525 "Invalid operand expansion"); 5526 5527 ReplaceValueWith(SDValue(N, 0), Res); 5528 return false; 5529 } 5530 5531 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) { 5532 SDLoc DL(N); 5533 EVT VT = N->getValueType(0); 5534 5535 SDValue InOp = N->getOperand(0); 5536 assert(getTypeAction(InOp.getValueType()) == 5537 TargetLowering::TypeWidenVector && 5538 "Unexpected type action"); 5539 InOp = GetWidenedVector(InOp); 5540 assert(VT.getVectorNumElements() < 5541 InOp.getValueType().getVectorNumElements() && 5542 "Input wasn't widened!"); 5543 5544 // We may need to further widen the operand until it has the same total 5545 // vector size as the result. 5546 EVT InVT = InOp.getValueType(); 5547 if (InVT.getSizeInBits() != VT.getSizeInBits()) { 5548 EVT InEltVT = InVT.getVectorElementType(); 5549 for (EVT FixedVT : MVT::vector_valuetypes()) { 5550 EVT FixedEltVT = FixedVT.getVectorElementType(); 5551 if (TLI.isTypeLegal(FixedVT) && 5552 FixedVT.getSizeInBits() == VT.getSizeInBits() && 5553 FixedEltVT == InEltVT) { 5554 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() && 5555 "Not enough elements in the fixed type for the operand!"); 5556 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() && 5557 "We can't have the same type as we started with!"); 5558 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements()) 5559 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT, 5560 DAG.getUNDEF(FixedVT), InOp, 5561 DAG.getVectorIdxConstant(0, DL)); 5562 else 5563 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp, 5564 DAG.getVectorIdxConstant(0, DL)); 5565 break; 5566 } 5567 } 5568 InVT = InOp.getValueType(); 5569 if (InVT.getSizeInBits() != VT.getSizeInBits()) 5570 // We couldn't find a legal vector type that was a widening of the input 5571 // and could be extended in-register to the result type, so we have to 5572 // scalarize. 5573 return WidenVecOp_Convert(N); 5574 } 5575 5576 // Use special DAG nodes to represent the operation of extending the 5577 // low lanes. 5578 switch (N->getOpcode()) { 5579 default: 5580 llvm_unreachable("Extend legalization on extend operation!"); 5581 case ISD::ANY_EXTEND: 5582 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp); 5583 case ISD::SIGN_EXTEND: 5584 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp); 5585 case ISD::ZERO_EXTEND: 5586 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp); 5587 } 5588 } 5589 5590 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) { 5591 // The result (and first input) is legal, but the second input is illegal. 5592 // We can't do much to fix that, so just unroll and let the extracts off of 5593 // the second input be widened as needed later. 5594 return DAG.UnrollVectorOp(N); 5595 } 5596 5597 SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) { 5598 SDLoc DL(N); 5599 EVT ResultVT = N->getValueType(0); 5600 SDValue Test = N->getOperand(1); 5601 SDValue WideArg = GetWidenedVector(N->getOperand(0)); 5602 5603 // Process this node similarly to SETCC. 5604 EVT WideResultVT = getSetCCResultType(WideArg.getValueType()); 5605 if (ResultVT.getScalarType() == MVT::i1) 5606 WideResultVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 5607 WideResultVT.getVectorNumElements()); 5608 5609 SDValue WideNode = DAG.getNode(ISD::IS_FPCLASS, DL, WideResultVT, 5610 {WideArg, Test}, N->getFlags()); 5611 5612 // Extract the needed results from the result vector. 5613 EVT ResVT = 5614 EVT::getVectorVT(*DAG.getContext(), WideResultVT.getVectorElementType(), 5615 ResultVT.getVectorNumElements()); 5616 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, WideNode, 5617 DAG.getVectorIdxConstant(0, DL)); 5618 5619 EVT OpVT = N->getOperand(0).getValueType(); 5620 ISD::NodeType ExtendCode = 5621 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 5622 return DAG.getNode(ExtendCode, DL, ResultVT, CC); 5623 } 5624 5625 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 5626 // Since the result is legal and the input is illegal. 5627 EVT VT = N->getValueType(0); 5628 EVT EltVT = VT.getVectorElementType(); 5629 SDLoc dl(N); 5630 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); 5631 assert(getTypeAction(InOp.getValueType()) == 5632 TargetLowering::TypeWidenVector && 5633 "Unexpected type action"); 5634 InOp = GetWidenedVector(InOp); 5635 EVT InVT = InOp.getValueType(); 5636 unsigned Opcode = N->getOpcode(); 5637 5638 // See if a widened result type would be legal, if so widen the node. 5639 // FIXME: This isn't safe for StrictFP. Other optimization here is needed. 5640 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 5641 InVT.getVectorElementCount()); 5642 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) { 5643 SDValue Res; 5644 if (N->isStrictFPOpcode()) { 5645 if (Opcode == ISD::STRICT_FP_ROUND) 5646 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, 5647 { N->getOperand(0), InOp, N->getOperand(2) }); 5648 else 5649 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, 5650 { N->getOperand(0), InOp }); 5651 // Legalize the chain result - switch anything that used the old chain to 5652 // use the new one. 5653 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 5654 } else { 5655 if (Opcode == ISD::FP_ROUND) 5656 Res = DAG.getNode(Opcode, dl, WideVT, InOp, N->getOperand(1)); 5657 else 5658 Res = DAG.getNode(Opcode, dl, WideVT, InOp); 5659 } 5660 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, 5661 DAG.getVectorIdxConstant(0, dl)); 5662 } 5663 5664 EVT InEltVT = InVT.getVectorElementType(); 5665 5666 // Unroll the convert into some scalar code and create a nasty build vector. 5667 unsigned NumElts = VT.getVectorNumElements(); 5668 SmallVector<SDValue, 16> Ops(NumElts); 5669 if (N->isStrictFPOpcode()) { 5670 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 5671 SmallVector<SDValue, 32> OpChains; 5672 for (unsigned i=0; i < NumElts; ++i) { 5673 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 5674 DAG.getVectorIdxConstant(i, dl)); 5675 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps); 5676 OpChains.push_back(Ops[i].getValue(1)); 5677 } 5678 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains); 5679 ReplaceValueWith(SDValue(N, 1), NewChain); 5680 } else { 5681 for (unsigned i = 0; i < NumElts; ++i) 5682 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 5683 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, 5684 InOp, DAG.getVectorIdxConstant(i, dl))); 5685 } 5686 5687 return DAG.getBuildVector(VT, dl, Ops); 5688 } 5689 5690 SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) { 5691 EVT DstVT = N->getValueType(0); 5692 SDValue Src = GetWidenedVector(N->getOperand(0)); 5693 EVT SrcVT = Src.getValueType(); 5694 ElementCount WideNumElts = SrcVT.getVectorElementCount(); 5695 SDLoc dl(N); 5696 5697 // See if a widened result type would be legal, if so widen the node. 5698 EVT WideDstVT = EVT::getVectorVT(*DAG.getContext(), 5699 DstVT.getVectorElementType(), WideNumElts); 5700 if (TLI.isTypeLegal(WideDstVT)) { 5701 SDValue Res = 5702 DAG.getNode(N->getOpcode(), dl, WideDstVT, Src, N->getOperand(1)); 5703 return DAG.getNode( 5704 ISD::EXTRACT_SUBVECTOR, dl, DstVT, Res, 5705 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 5706 } 5707 5708 // Give up and unroll. 5709 return DAG.UnrollVectorOp(N); 5710 } 5711 5712 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 5713 EVT VT = N->getValueType(0); 5714 SDValue InOp = GetWidenedVector(N->getOperand(0)); 5715 EVT InWidenVT = InOp.getValueType(); 5716 SDLoc dl(N); 5717 5718 // Check if we can convert between two legal vector types and extract. 5719 TypeSize InWidenSize = InWidenVT.getSizeInBits(); 5720 TypeSize Size = VT.getSizeInBits(); 5721 // x86mmx is not an acceptable vector element type, so don't try. 5722 if (!VT.isVector() && VT != MVT::x86mmx && 5723 InWidenSize.hasKnownScalarFactor(Size)) { 5724 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(Size); 5725 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 5726 if (TLI.isTypeLegal(NewVT)) { 5727 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 5728 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 5729 DAG.getVectorIdxConstant(0, dl)); 5730 } 5731 } 5732 5733 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened 5734 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not, 5735 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids 5736 // having to copy via memory. 5737 if (VT.isVector()) { 5738 EVT EltVT = VT.getVectorElementType(); 5739 unsigned EltSize = EltVT.getFixedSizeInBits(); 5740 if (InWidenSize.isKnownMultipleOf(EltSize)) { 5741 ElementCount NewNumElts = 5742 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits()) 5743 .divideCoefficientBy(EltSize); 5744 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts); 5745 if (TLI.isTypeLegal(NewVT)) { 5746 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 5747 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp, 5748 DAG.getVectorIdxConstant(0, dl)); 5749 } 5750 } 5751 } 5752 5753 return CreateStackStoreLoad(InOp, VT); 5754 } 5755 5756 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 5757 EVT VT = N->getValueType(0); 5758 EVT EltVT = VT.getVectorElementType(); 5759 EVT InVT = N->getOperand(0).getValueType(); 5760 SDLoc dl(N); 5761 5762 // If the widen width for this operand is the same as the width of the concat 5763 // and all but the first operand is undef, just use the widened operand. 5764 unsigned NumOperands = N->getNumOperands(); 5765 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 5766 unsigned i; 5767 for (i = 1; i < NumOperands; ++i) 5768 if (!N->getOperand(i).isUndef()) 5769 break; 5770 5771 if (i == NumOperands) 5772 return GetWidenedVector(N->getOperand(0)); 5773 } 5774 5775 // Otherwise, fall back to a nasty build vector. 5776 unsigned NumElts = VT.getVectorNumElements(); 5777 SmallVector<SDValue, 16> Ops(NumElts); 5778 5779 unsigned NumInElts = InVT.getVectorNumElements(); 5780 5781 unsigned Idx = 0; 5782 for (unsigned i=0; i < NumOperands; ++i) { 5783 SDValue InOp = N->getOperand(i); 5784 assert(getTypeAction(InOp.getValueType()) == 5785 TargetLowering::TypeWidenVector && 5786 "Unexpected type action"); 5787 InOp = GetWidenedVector(InOp); 5788 for (unsigned j = 0; j < NumInElts; ++j) 5789 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 5790 DAG.getVectorIdxConstant(j, dl)); 5791 } 5792 return DAG.getBuildVector(VT, dl, Ops); 5793 } 5794 5795 SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) { 5796 EVT VT = N->getValueType(0); 5797 SDValue SubVec = N->getOperand(1); 5798 SDValue InVec = N->getOperand(0); 5799 5800 if (getTypeAction(SubVec.getValueType()) == TargetLowering::TypeWidenVector) 5801 SubVec = GetWidenedVector(SubVec); 5802 5803 if (SubVec.getValueType().knownBitsLE(VT) && InVec.isUndef() && 5804 N->getConstantOperandVal(2) == 0) 5805 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, InVec, SubVec, 5806 N->getOperand(2)); 5807 5808 report_fatal_error("Don't know how to widen the operands for " 5809 "INSERT_SUBVECTOR"); 5810 } 5811 5812 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 5813 SDValue InOp = GetWidenedVector(N->getOperand(0)); 5814 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), 5815 N->getValueType(0), InOp, N->getOperand(1)); 5816 } 5817 5818 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 5819 SDValue InOp = GetWidenedVector(N->getOperand(0)); 5820 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 5821 N->getValueType(0), InOp, N->getOperand(1)); 5822 } 5823 5824 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 5825 // We have to widen the value, but we want only to store the original 5826 // vector type. 5827 StoreSDNode *ST = cast<StoreSDNode>(N); 5828 5829 if (!ST->getMemoryVT().getScalarType().isByteSized()) 5830 return TLI.scalarizeVectorStore(ST, DAG); 5831 5832 if (ST->isTruncatingStore()) 5833 return TLI.scalarizeVectorStore(ST, DAG); 5834 5835 SmallVector<SDValue, 16> StChain; 5836 if (GenWidenVectorStores(StChain, ST)) { 5837 if (StChain.size() == 1) 5838 return StChain[0]; 5839 5840 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain); 5841 } 5842 5843 // Generate a vector-predicated store if it is custom/legal on the target. 5844 // To avoid possible recursion, only do this if the widened mask type is 5845 // legal. 5846 // FIXME: Not all targets may support EVL in VP_STORE. These will have been 5847 // removed from the IR by the ExpandVectorPredication pass but we're 5848 // reintroducing them here. 5849 SDValue StVal = ST->getValue(); 5850 EVT StVT = StVal.getValueType(); 5851 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StVT); 5852 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 5853 WideVT.getVectorElementCount()); 5854 if (WideVT.isScalableVector() && 5855 TLI.isOperationLegalOrCustom(ISD::VP_STORE, WideVT) && 5856 TLI.isTypeLegal(WideMaskVT)) { 5857 // Widen the value. 5858 SDLoc DL(N); 5859 StVal = GetWidenedVector(StVal); 5860 SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT); 5861 MVT EVLVT = TLI.getVPExplicitVectorLengthTy(); 5862 unsigned NumVTElts = StVT.getVectorMinNumElements(); 5863 SDValue EVL = 5864 DAG.getVScale(DL, EVLVT, APInt(EVLVT.getScalarSizeInBits(), NumVTElts)); 5865 return DAG.getStoreVP(ST->getChain(), DL, StVal, ST->getBasePtr(), 5866 DAG.getUNDEF(ST->getBasePtr().getValueType()), Mask, 5867 EVL, StVal.getValueType(), ST->getMemOperand(), 5868 ST->getAddressingMode()); 5869 } 5870 5871 report_fatal_error("Unable to widen vector store"); 5872 } 5873 5874 SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) { 5875 assert((OpNo == 1 || OpNo == 3) && 5876 "Can widen only data or mask operand of vp_store"); 5877 VPStoreSDNode *ST = cast<VPStoreSDNode>(N); 5878 SDValue Mask = ST->getMask(); 5879 SDValue StVal = ST->getValue(); 5880 SDLoc dl(N); 5881 5882 if (OpNo == 1) { 5883 // Widen the value. 5884 StVal = GetWidenedVector(StVal); 5885 5886 // We only handle the case where the mask needs widening to an 5887 // identically-sized type as the vector inputs. 5888 assert(getTypeAction(Mask.getValueType()) == 5889 TargetLowering::TypeWidenVector && 5890 "Unable to widen VP store"); 5891 Mask = GetWidenedVector(Mask); 5892 } else { 5893 Mask = GetWidenedVector(Mask); 5894 5895 // We only handle the case where the stored value needs widening to an 5896 // identically-sized type as the mask. 5897 assert(getTypeAction(StVal.getValueType()) == 5898 TargetLowering::TypeWidenVector && 5899 "Unable to widen VP store"); 5900 StVal = GetWidenedVector(StVal); 5901 } 5902 5903 assert(Mask.getValueType().getVectorElementCount() == 5904 StVal.getValueType().getVectorElementCount() && 5905 "Mask and data vectors should have the same number of elements"); 5906 return DAG.getStoreVP(ST->getChain(), dl, StVal, ST->getBasePtr(), 5907 ST->getOffset(), Mask, ST->getVectorLength(), 5908 ST->getMemoryVT(), ST->getMemOperand(), 5909 ST->getAddressingMode(), ST->isTruncatingStore(), 5910 ST->isCompressingStore()); 5911 } 5912 5913 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) { 5914 assert((OpNo == 1 || OpNo == 3) && 5915 "Can widen only data or mask operand of mstore"); 5916 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); 5917 SDValue Mask = MST->getMask(); 5918 EVT MaskVT = Mask.getValueType(); 5919 SDValue StVal = MST->getValue(); 5920 SDLoc dl(N); 5921 5922 if (OpNo == 1) { 5923 // Widen the value. 5924 StVal = GetWidenedVector(StVal); 5925 5926 // The mask should be widened as well. 5927 EVT WideVT = StVal.getValueType(); 5928 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 5929 MaskVT.getVectorElementType(), 5930 WideVT.getVectorNumElements()); 5931 Mask = ModifyToType(Mask, WideMaskVT, true); 5932 } else { 5933 // Widen the mask. 5934 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT); 5935 Mask = ModifyToType(Mask, WideMaskVT, true); 5936 5937 EVT ValueVT = StVal.getValueType(); 5938 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), 5939 ValueVT.getVectorElementType(), 5940 WideMaskVT.getVectorNumElements()); 5941 StVal = ModifyToType(StVal, WideVT); 5942 } 5943 5944 assert(Mask.getValueType().getVectorNumElements() == 5945 StVal.getValueType().getVectorNumElements() && 5946 "Mask and data vectors should have the same number of elements"); 5947 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(), 5948 MST->getOffset(), Mask, MST->getMemoryVT(), 5949 MST->getMemOperand(), MST->getAddressingMode(), 5950 false, MST->isCompressingStore()); 5951 } 5952 5953 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) { 5954 assert(OpNo == 4 && "Can widen only the index of mgather"); 5955 auto *MG = cast<MaskedGatherSDNode>(N); 5956 SDValue DataOp = MG->getPassThru(); 5957 SDValue Mask = MG->getMask(); 5958 SDValue Scale = MG->getScale(); 5959 5960 // Just widen the index. It's allowed to have extra elements. 5961 SDValue Index = GetWidenedVector(MG->getIndex()); 5962 5963 SDLoc dl(N); 5964 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index, 5965 Scale}; 5966 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops, 5967 MG->getMemOperand(), MG->getIndexType(), 5968 MG->getExtensionType()); 5969 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 5970 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 5971 return SDValue(); 5972 } 5973 5974 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) { 5975 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N); 5976 SDValue DataOp = MSC->getValue(); 5977 SDValue Mask = MSC->getMask(); 5978 SDValue Index = MSC->getIndex(); 5979 SDValue Scale = MSC->getScale(); 5980 EVT WideMemVT = MSC->getMemoryVT(); 5981 5982 if (OpNo == 1) { 5983 DataOp = GetWidenedVector(DataOp); 5984 unsigned NumElts = DataOp.getValueType().getVectorNumElements(); 5985 5986 // Widen index. 5987 EVT IndexVT = Index.getValueType(); 5988 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(), 5989 IndexVT.getVectorElementType(), NumElts); 5990 Index = ModifyToType(Index, WideIndexVT); 5991 5992 // The mask should be widened as well. 5993 EVT MaskVT = Mask.getValueType(); 5994 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 5995 MaskVT.getVectorElementType(), NumElts); 5996 Mask = ModifyToType(Mask, WideMaskVT, true); 5997 5998 // Widen the MemoryType 5999 WideMemVT = EVT::getVectorVT(*DAG.getContext(), 6000 MSC->getMemoryVT().getScalarType(), NumElts); 6001 } else if (OpNo == 4) { 6002 // Just widen the index. It's allowed to have extra elements. 6003 Index = GetWidenedVector(Index); 6004 } else 6005 llvm_unreachable("Can't widen this operand of mscatter"); 6006 6007 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index, 6008 Scale}; 6009 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), 6010 Ops, MSC->getMemOperand(), MSC->getIndexType(), 6011 MSC->isTruncatingStore()); 6012 } 6013 6014 SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) { 6015 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(N); 6016 SDValue DataOp = VPSC->getValue(); 6017 SDValue Mask = VPSC->getMask(); 6018 SDValue Index = VPSC->getIndex(); 6019 SDValue Scale = VPSC->getScale(); 6020 EVT WideMemVT = VPSC->getMemoryVT(); 6021 6022 if (OpNo == 1) { 6023 DataOp = GetWidenedVector(DataOp); 6024 Index = GetWidenedVector(Index); 6025 const auto WideEC = DataOp.getValueType().getVectorElementCount(); 6026 Mask = GetWidenedMask(Mask, WideEC); 6027 WideMemVT = EVT::getVectorVT(*DAG.getContext(), 6028 VPSC->getMemoryVT().getScalarType(), WideEC); 6029 } else if (OpNo == 3) { 6030 // Just widen the index. It's allowed to have extra elements. 6031 Index = GetWidenedVector(Index); 6032 } else 6033 llvm_unreachable("Can't widen this operand of VP_SCATTER"); 6034 6035 SDValue Ops[] = { 6036 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask, 6037 VPSC->getVectorLength()}; 6038 return DAG.getScatterVP(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), Ops, 6039 VPSC->getMemOperand(), VPSC->getIndexType()); 6040 } 6041 6042 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 6043 SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 6044 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 6045 SDLoc dl(N); 6046 EVT VT = N->getValueType(0); 6047 6048 // WARNING: In this code we widen the compare instruction with garbage. 6049 // This garbage may contain denormal floats which may be slow. Is this a real 6050 // concern ? Should we zero the unused lanes if this is a float compare ? 6051 6052 // Get a new SETCC node to compare the newly widened operands. 6053 // Only some of the compared elements are legal. 6054 EVT SVT = getSetCCResultType(InOp0.getValueType()); 6055 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC. 6056 if (VT.getScalarType() == MVT::i1) 6057 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 6058 SVT.getVectorElementCount()); 6059 6060 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N), 6061 SVT, InOp0, InOp1, N->getOperand(2)); 6062 6063 // Extract the needed results from the result vector. 6064 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 6065 SVT.getVectorElementType(), 6066 VT.getVectorElementCount()); 6067 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC, 6068 DAG.getVectorIdxConstant(0, dl)); 6069 6070 EVT OpVT = N->getOperand(0).getValueType(); 6071 ISD::NodeType ExtendCode = 6072 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 6073 return DAG.getNode(ExtendCode, dl, VT, CC); 6074 } 6075 6076 SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) { 6077 SDValue Chain = N->getOperand(0); 6078 SDValue LHS = GetWidenedVector(N->getOperand(1)); 6079 SDValue RHS = GetWidenedVector(N->getOperand(2)); 6080 SDValue CC = N->getOperand(3); 6081 SDLoc dl(N); 6082 6083 EVT VT = N->getValueType(0); 6084 EVT EltVT = VT.getVectorElementType(); 6085 EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 6086 unsigned NumElts = VT.getVectorNumElements(); 6087 6088 // Unroll into a build vector. 6089 SmallVector<SDValue, 8> Scalars(NumElts); 6090 SmallVector<SDValue, 8> Chains(NumElts); 6091 6092 for (unsigned i = 0; i != NumElts; ++i) { 6093 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 6094 DAG.getVectorIdxConstant(i, dl)); 6095 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 6096 DAG.getVectorIdxConstant(i, dl)); 6097 6098 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other}, 6099 {Chain, LHSElem, RHSElem, CC}); 6100 Chains[i] = Scalars[i].getValue(1); 6101 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i], 6102 DAG.getBoolConstant(true, dl, EltVT, VT), 6103 DAG.getBoolConstant(false, dl, EltVT, VT)); 6104 } 6105 6106 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 6107 ReplaceValueWith(SDValue(N, 1), NewChain); 6108 6109 return DAG.getBuildVector(VT, dl, Scalars); 6110 } 6111 6112 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) { 6113 SDLoc dl(N); 6114 SDValue Op = GetWidenedVector(N->getOperand(0)); 6115 EVT OrigVT = N->getOperand(0).getValueType(); 6116 EVT WideVT = Op.getValueType(); 6117 EVT ElemVT = OrigVT.getVectorElementType(); 6118 SDNodeFlags Flags = N->getFlags(); 6119 6120 unsigned Opc = N->getOpcode(); 6121 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc); 6122 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags); 6123 assert(NeutralElem && "Neutral element must exist"); 6124 6125 // Pad the vector with the neutral element. 6126 unsigned OrigElts = OrigVT.getVectorMinNumElements(); 6127 unsigned WideElts = WideVT.getVectorMinNumElements(); 6128 6129 if (WideVT.isScalableVector()) { 6130 unsigned GCD = greatestCommonDivisor(OrigElts, WideElts); 6131 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, 6132 ElementCount::getScalable(GCD)); 6133 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem); 6134 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD) 6135 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral, 6136 DAG.getVectorIdxConstant(Idx, dl)); 6137 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags); 6138 } 6139 6140 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) 6141 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem, 6142 DAG.getVectorIdxConstant(Idx, dl)); 6143 6144 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags); 6145 } 6146 6147 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) { 6148 SDLoc dl(N); 6149 SDValue AccOp = N->getOperand(0); 6150 SDValue VecOp = N->getOperand(1); 6151 SDValue Op = GetWidenedVector(VecOp); 6152 6153 EVT OrigVT = VecOp.getValueType(); 6154 EVT WideVT = Op.getValueType(); 6155 EVT ElemVT = OrigVT.getVectorElementType(); 6156 SDNodeFlags Flags = N->getFlags(); 6157 6158 unsigned Opc = N->getOpcode(); 6159 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc); 6160 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags); 6161 6162 // Pad the vector with the neutral element. 6163 unsigned OrigElts = OrigVT.getVectorMinNumElements(); 6164 unsigned WideElts = WideVT.getVectorMinNumElements(); 6165 6166 if (WideVT.isScalableVector()) { 6167 unsigned GCD = greatestCommonDivisor(OrigElts, WideElts); 6168 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, 6169 ElementCount::getScalable(GCD)); 6170 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem); 6171 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD) 6172 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral, 6173 DAG.getVectorIdxConstant(Idx, dl)); 6174 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags); 6175 } 6176 6177 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) 6178 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem, 6179 DAG.getVectorIdxConstant(Idx, dl)); 6180 6181 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags); 6182 } 6183 6184 SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) { 6185 assert(N->isVPOpcode() && "Expected VP opcode"); 6186 6187 SDLoc dl(N); 6188 SDValue Op = GetWidenedVector(N->getOperand(1)); 6189 SDValue Mask = GetWidenedMask(N->getOperand(2), 6190 Op.getValueType().getVectorElementCount()); 6191 6192 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), 6193 {N->getOperand(0), Op, Mask, N->getOperand(3)}, 6194 N->getFlags()); 6195 } 6196 6197 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) { 6198 // This only gets called in the case that the left and right inputs and 6199 // result are of a legal odd vector type, and the condition is illegal i1 of 6200 // the same odd width that needs widening. 6201 EVT VT = N->getValueType(0); 6202 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT)); 6203 6204 SDValue Cond = GetWidenedVector(N->getOperand(0)); 6205 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N)); 6206 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N)); 6207 SDLoc DL(N); 6208 6209 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond, 6210 LeftIn, RightIn); 6211 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Select, 6212 DAG.getVectorIdxConstant(0, DL)); 6213 } 6214 6215 //===----------------------------------------------------------------------===// 6216 // Vector Widening Utilities 6217 //===----------------------------------------------------------------------===// 6218 6219 // Utility function to find the type to chop up a widen vector for load/store 6220 // TLI: Target lowering used to determine legal types. 6221 // Width: Width left need to load/store. 6222 // WidenVT: The widen vector type to load to/store from 6223 // Align: If 0, don't allow use of a wider type 6224 // WidenEx: If Align is not 0, the amount additional we can load/store from. 6225 6226 static Optional<EVT> findMemType(SelectionDAG &DAG, const TargetLowering &TLI, 6227 unsigned Width, EVT WidenVT, 6228 unsigned Align = 0, unsigned WidenEx = 0) { 6229 EVT WidenEltVT = WidenVT.getVectorElementType(); 6230 const bool Scalable = WidenVT.isScalableVector(); 6231 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinSize(); 6232 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 6233 unsigned AlignInBits = Align*8; 6234 6235 // If we have one element to load/store, return it. 6236 EVT RetVT = WidenEltVT; 6237 if (!Scalable && Width == WidenEltWidth) 6238 return RetVT; 6239 6240 // Don't bother looking for an integer type if the vector is scalable, skip 6241 // to vector types. 6242 if (!Scalable) { 6243 // See if there is larger legal integer than the element type to load/store. 6244 for (EVT MemVT : reverse(MVT::integer_valuetypes())) { 6245 unsigned MemVTWidth = MemVT.getSizeInBits(); 6246 if (MemVT.getSizeInBits() <= WidenEltWidth) 6247 break; 6248 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); 6249 if ((Action == TargetLowering::TypeLegal || 6250 Action == TargetLowering::TypePromoteInteger) && 6251 (WidenWidth % MemVTWidth) == 0 && 6252 isPowerOf2_32(WidenWidth / MemVTWidth) && 6253 (MemVTWidth <= Width || 6254 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 6255 if (MemVTWidth == WidenWidth) 6256 return MemVT; 6257 RetVT = MemVT; 6258 break; 6259 } 6260 } 6261 } 6262 6263 // See if there is a larger vector type to load/store that has the same vector 6264 // element type and is evenly divisible with the WidenVT. 6265 for (EVT MemVT : reverse(MVT::vector_valuetypes())) { 6266 // Skip vector MVTs which don't match the scalable property of WidenVT. 6267 if (Scalable != MemVT.isScalableVector()) 6268 continue; 6269 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinSize(); 6270 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); 6271 if ((Action == TargetLowering::TypeLegal || 6272 Action == TargetLowering::TypePromoteInteger) && 6273 WidenEltVT == MemVT.getVectorElementType() && 6274 (WidenWidth % MemVTWidth) == 0 && 6275 isPowerOf2_32(WidenWidth / MemVTWidth) && 6276 (MemVTWidth <= Width || 6277 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 6278 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT) 6279 return MemVT; 6280 } 6281 } 6282 6283 // Using element-wise loads and stores for widening operations is not 6284 // supported for scalable vectors 6285 if (Scalable) 6286 return None; 6287 6288 return RetVT; 6289 } 6290 6291 // Builds a vector type from scalar loads 6292 // VecTy: Resulting Vector type 6293 // LDOps: Load operators to build a vector type 6294 // [Start,End) the list of loads to use. 6295 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 6296 SmallVectorImpl<SDValue> &LdOps, 6297 unsigned Start, unsigned End) { 6298 SDLoc dl(LdOps[Start]); 6299 EVT LdTy = LdOps[Start].getValueType(); 6300 unsigned Width = VecTy.getSizeInBits(); 6301 unsigned NumElts = Width / LdTy.getSizeInBits(); 6302 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 6303 6304 unsigned Idx = 1; 6305 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 6306 6307 for (unsigned i = Start + 1; i != End; ++i) { 6308 EVT NewLdTy = LdOps[i].getValueType(); 6309 if (NewLdTy != LdTy) { 6310 NumElts = Width / NewLdTy.getSizeInBits(); 6311 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 6312 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 6313 // Readjust position and vector position based on new load type. 6314 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 6315 LdTy = NewLdTy; 6316 } 6317 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 6318 DAG.getVectorIdxConstant(Idx++, dl)); 6319 } 6320 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 6321 } 6322 6323 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, 6324 LoadSDNode *LD) { 6325 // The strategy assumes that we can efficiently load power-of-two widths. 6326 // The routine chops the vector into the largest vector loads with the same 6327 // element type or scalar loads and then recombines it to the widen vector 6328 // type. 6329 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 6330 EVT LdVT = LD->getMemoryVT(); 6331 SDLoc dl(LD); 6332 assert(LdVT.isVector() && WidenVT.isVector()); 6333 assert(LdVT.isScalableVector() == WidenVT.isScalableVector()); 6334 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 6335 6336 // Load information 6337 SDValue Chain = LD->getChain(); 6338 SDValue BasePtr = LD->getBasePtr(); 6339 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 6340 AAMDNodes AAInfo = LD->getAAInfo(); 6341 6342 TypeSize LdWidth = LdVT.getSizeInBits(); 6343 TypeSize WidenWidth = WidenVT.getSizeInBits(); 6344 TypeSize WidthDiff = WidenWidth - LdWidth; 6345 // Allow wider loads if they are sufficiently aligned to avoid memory faults 6346 // and if the original load is simple. 6347 unsigned LdAlign = 6348 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value(); 6349 6350 // Find the vector type that can load from. 6351 Optional<EVT> FirstVT = 6352 findMemType(DAG, TLI, LdWidth.getKnownMinSize(), WidenVT, LdAlign, 6353 WidthDiff.getKnownMinSize()); 6354 6355 if (!FirstVT) 6356 return SDValue(); 6357 6358 SmallVector<EVT, 8> MemVTs; 6359 TypeSize FirstVTWidth = FirstVT->getSizeInBits(); 6360 6361 // Unless we're able to load in one instruction we must work out how to load 6362 // the remainder. 6363 if (!TypeSize::isKnownLE(LdWidth, FirstVTWidth)) { 6364 Optional<EVT> NewVT = FirstVT; 6365 TypeSize RemainingWidth = LdWidth; 6366 TypeSize NewVTWidth = FirstVTWidth; 6367 do { 6368 RemainingWidth -= NewVTWidth; 6369 if (TypeSize::isKnownLT(RemainingWidth, NewVTWidth)) { 6370 // The current type we are using is too large. Find a better size. 6371 NewVT = findMemType(DAG, TLI, RemainingWidth.getKnownMinSize(), WidenVT, 6372 LdAlign, WidthDiff.getKnownMinSize()); 6373 if (!NewVT) 6374 return SDValue(); 6375 NewVTWidth = NewVT->getSizeInBits(); 6376 } 6377 MemVTs.push_back(*NewVT); 6378 } while (TypeSize::isKnownGT(RemainingWidth, NewVTWidth)); 6379 } 6380 6381 SDValue LdOp = DAG.getLoad(*FirstVT, dl, Chain, BasePtr, LD->getPointerInfo(), 6382 LD->getOriginalAlign(), MMOFlags, AAInfo); 6383 LdChain.push_back(LdOp.getValue(1)); 6384 6385 // Check if we can load the element with one instruction. 6386 if (MemVTs.empty()) { 6387 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth)); 6388 if (!FirstVT->isVector()) { 6389 unsigned NumElts = 6390 WidenWidth.getFixedSize() / FirstVTWidth.getFixedSize(); 6391 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), *FirstVT, NumElts); 6392 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 6393 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 6394 } 6395 if (FirstVT == WidenVT) 6396 return LdOp; 6397 6398 // TODO: We don't currently have any tests that exercise this code path. 6399 assert(WidenWidth.getFixedSize() % FirstVTWidth.getFixedSize() == 0); 6400 unsigned NumConcat = 6401 WidenWidth.getFixedSize() / FirstVTWidth.getFixedSize(); 6402 SmallVector<SDValue, 16> ConcatOps(NumConcat); 6403 SDValue UndefVal = DAG.getUNDEF(*FirstVT); 6404 ConcatOps[0] = LdOp; 6405 for (unsigned i = 1; i != NumConcat; ++i) 6406 ConcatOps[i] = UndefVal; 6407 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps); 6408 } 6409 6410 // Load vector by using multiple loads from largest vector to scalar. 6411 SmallVector<SDValue, 16> LdOps; 6412 LdOps.push_back(LdOp); 6413 6414 uint64_t ScaledOffset = 0; 6415 MachinePointerInfo MPI = LD->getPointerInfo(); 6416 6417 // First incremement past the first load. 6418 IncrementPointer(cast<LoadSDNode>(LdOp), *FirstVT, MPI, BasePtr, 6419 &ScaledOffset); 6420 6421 for (EVT MemVT : MemVTs) { 6422 Align NewAlign = ScaledOffset == 0 6423 ? LD->getOriginalAlign() 6424 : commonAlignment(LD->getAlign(), ScaledOffset); 6425 SDValue L = 6426 DAG.getLoad(MemVT, dl, Chain, BasePtr, MPI, NewAlign, MMOFlags, AAInfo); 6427 6428 LdOps.push_back(L); 6429 LdChain.push_back(L.getValue(1)); 6430 IncrementPointer(cast<LoadSDNode>(L), MemVT, MPI, BasePtr, &ScaledOffset); 6431 } 6432 6433 // Build the vector from the load operations. 6434 unsigned End = LdOps.size(); 6435 if (!LdOps[0].getValueType().isVector()) 6436 // All the loads are scalar loads. 6437 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 6438 6439 // If the load contains vectors, build the vector using concat vector. 6440 // All of the vectors used to load are power-of-2, and the scalar loads can be 6441 // combined to make a power-of-2 vector. 6442 SmallVector<SDValue, 16> ConcatOps(End); 6443 int i = End - 1; 6444 int Idx = End; 6445 EVT LdTy = LdOps[i].getValueType(); 6446 // First, combine the scalar loads to a vector. 6447 if (!LdTy.isVector()) { 6448 for (--i; i >= 0; --i) { 6449 LdTy = LdOps[i].getValueType(); 6450 if (LdTy.isVector()) 6451 break; 6452 } 6453 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End); 6454 } 6455 6456 ConcatOps[--Idx] = LdOps[i]; 6457 for (--i; i >= 0; --i) { 6458 EVT NewLdTy = LdOps[i].getValueType(); 6459 if (NewLdTy != LdTy) { 6460 // Create a larger vector. 6461 TypeSize LdTySize = LdTy.getSizeInBits(); 6462 TypeSize NewLdTySize = NewLdTy.getSizeInBits(); 6463 assert(NewLdTySize.isScalable() == LdTySize.isScalable() && 6464 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinSize())); 6465 unsigned NumOps = 6466 NewLdTySize.getKnownMinSize() / LdTySize.getKnownMinSize(); 6467 SmallVector<SDValue, 16> WidenOps(NumOps); 6468 unsigned j = 0; 6469 for (; j != End-Idx; ++j) 6470 WidenOps[j] = ConcatOps[Idx+j]; 6471 for (; j != NumOps; ++j) 6472 WidenOps[j] = DAG.getUNDEF(LdTy); 6473 6474 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 6475 WidenOps); 6476 Idx = End - 1; 6477 LdTy = NewLdTy; 6478 } 6479 ConcatOps[--Idx] = LdOps[i]; 6480 } 6481 6482 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx)) 6483 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 6484 makeArrayRef(&ConcatOps[Idx], End - Idx)); 6485 6486 // We need to fill the rest with undefs to build the vector. 6487 unsigned NumOps = 6488 WidenWidth.getKnownMinSize() / LdTy.getSizeInBits().getKnownMinSize(); 6489 SmallVector<SDValue, 16> WidenOps(NumOps); 6490 SDValue UndefVal = DAG.getUNDEF(LdTy); 6491 { 6492 unsigned i = 0; 6493 for (; i != End-Idx; ++i) 6494 WidenOps[i] = ConcatOps[Idx+i]; 6495 for (; i != NumOps; ++i) 6496 WidenOps[i] = UndefVal; 6497 } 6498 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps); 6499 } 6500 6501 SDValue 6502 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, 6503 LoadSDNode *LD, 6504 ISD::LoadExtType ExtType) { 6505 // For extension loads, it may not be more efficient to chop up the vector 6506 // and then extend it. Instead, we unroll the load and build a new vector. 6507 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 6508 EVT LdVT = LD->getMemoryVT(); 6509 SDLoc dl(LD); 6510 assert(LdVT.isVector() && WidenVT.isVector()); 6511 assert(LdVT.isScalableVector() == WidenVT.isScalableVector()); 6512 6513 // Load information 6514 SDValue Chain = LD->getChain(); 6515 SDValue BasePtr = LD->getBasePtr(); 6516 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 6517 AAMDNodes AAInfo = LD->getAAInfo(); 6518 6519 if (LdVT.isScalableVector()) 6520 report_fatal_error("Generating widen scalable extending vector loads is " 6521 "not yet supported"); 6522 6523 EVT EltVT = WidenVT.getVectorElementType(); 6524 EVT LdEltVT = LdVT.getVectorElementType(); 6525 unsigned NumElts = LdVT.getVectorNumElements(); 6526 6527 // Load each element and widen. 6528 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 6529 SmallVector<SDValue, 16> Ops(WidenNumElts); 6530 unsigned Increment = LdEltVT.getSizeInBits() / 8; 6531 Ops[0] = 6532 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(), 6533 LdEltVT, LD->getOriginalAlign(), MMOFlags, AAInfo); 6534 LdChain.push_back(Ops[0].getValue(1)); 6535 unsigned i = 0, Offset = Increment; 6536 for (i=1; i < NumElts; ++i, Offset += Increment) { 6537 SDValue NewBasePtr = 6538 DAG.getObjectPtrOffset(dl, BasePtr, TypeSize::Fixed(Offset)); 6539 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 6540 LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 6541 LD->getOriginalAlign(), MMOFlags, AAInfo); 6542 LdChain.push_back(Ops[i].getValue(1)); 6543 } 6544 6545 // Fill the rest with undefs. 6546 SDValue UndefVal = DAG.getUNDEF(EltVT); 6547 for (; i != WidenNumElts; ++i) 6548 Ops[i] = UndefVal; 6549 6550 return DAG.getBuildVector(WidenVT, dl, Ops); 6551 } 6552 6553 bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, 6554 StoreSDNode *ST) { 6555 // The strategy assumes that we can efficiently store power-of-two widths. 6556 // The routine chops the vector into the largest vector stores with the same 6557 // element type or scalar stores. 6558 SDValue Chain = ST->getChain(); 6559 SDValue BasePtr = ST->getBasePtr(); 6560 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); 6561 AAMDNodes AAInfo = ST->getAAInfo(); 6562 SDValue ValOp = GetWidenedVector(ST->getValue()); 6563 SDLoc dl(ST); 6564 6565 EVT StVT = ST->getMemoryVT(); 6566 TypeSize StWidth = StVT.getSizeInBits(); 6567 EVT ValVT = ValOp.getValueType(); 6568 TypeSize ValWidth = ValVT.getSizeInBits(); 6569 EVT ValEltVT = ValVT.getVectorElementType(); 6570 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits(); 6571 assert(StVT.getVectorElementType() == ValEltVT); 6572 assert(StVT.isScalableVector() == ValVT.isScalableVector() && 6573 "Mismatch between store and value types"); 6574 6575 int Idx = 0; // current index to store 6576 6577 MachinePointerInfo MPI = ST->getPointerInfo(); 6578 uint64_t ScaledOffset = 0; 6579 6580 // A breakdown of how to widen this vector store. Each element of the vector 6581 // is a memory VT combined with the number of times it is to be stored to, 6582 // e,g., v5i32 -> {{v2i32,2},{i32,1}} 6583 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs; 6584 6585 while (StWidth.isNonZero()) { 6586 // Find the largest vector type we can store with. 6587 Optional<EVT> NewVT = 6588 findMemType(DAG, TLI, StWidth.getKnownMinSize(), ValVT); 6589 if (!NewVT) 6590 return false; 6591 MemVTs.push_back({*NewVT, 0}); 6592 TypeSize NewVTWidth = NewVT->getSizeInBits(); 6593 6594 do { 6595 StWidth -= NewVTWidth; 6596 MemVTs.back().second++; 6597 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth)); 6598 } 6599 6600 for (const auto &Pair : MemVTs) { 6601 EVT NewVT = Pair.first; 6602 unsigned Count = Pair.second; 6603 TypeSize NewVTWidth = NewVT.getSizeInBits(); 6604 6605 if (NewVT.isVector()) { 6606 unsigned NumVTElts = NewVT.getVectorMinNumElements(); 6607 do { 6608 Align NewAlign = ScaledOffset == 0 6609 ? ST->getOriginalAlign() 6610 : commonAlignment(ST->getAlign(), ScaledOffset); 6611 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 6612 DAG.getVectorIdxConstant(Idx, dl)); 6613 SDValue PartStore = DAG.getStore(Chain, dl, EOp, BasePtr, MPI, NewAlign, 6614 MMOFlags, AAInfo); 6615 StChain.push_back(PartStore); 6616 6617 Idx += NumVTElts; 6618 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr, 6619 &ScaledOffset); 6620 } while (--Count); 6621 } else { 6622 // Cast the vector to the scalar type we can store. 6623 unsigned NumElts = ValWidth.getFixedSize() / NewVTWidth.getFixedSize(); 6624 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 6625 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 6626 // Readjust index position based on new vector type. 6627 Idx = Idx * ValEltWidth / NewVTWidth.getFixedSize(); 6628 do { 6629 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 6630 DAG.getVectorIdxConstant(Idx++, dl)); 6631 SDValue PartStore = 6632 DAG.getStore(Chain, dl, EOp, BasePtr, MPI, ST->getOriginalAlign(), 6633 MMOFlags, AAInfo); 6634 StChain.push_back(PartStore); 6635 6636 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr); 6637 } while (--Count); 6638 // Restore index back to be relative to the original widen element type. 6639 Idx = Idx * NewVTWidth.getFixedSize() / ValEltWidth; 6640 } 6641 } 6642 6643 return true; 6644 } 6645 6646 /// Modifies a vector input (widen or narrows) to a vector of NVT. The 6647 /// input vector must have the same element type as NVT. 6648 /// FillWithZeroes specifies that the vector should be widened with zeroes. 6649 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT, 6650 bool FillWithZeroes) { 6651 // Note that InOp might have been widened so it might already have 6652 // the right width or it might need be narrowed. 6653 EVT InVT = InOp.getValueType(); 6654 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 6655 "input and widen element type must match"); 6656 assert(InVT.isScalableVector() == NVT.isScalableVector() && 6657 "cannot modify scalable vectors in this way"); 6658 SDLoc dl(InOp); 6659 6660 // Check if InOp already has the right width. 6661 if (InVT == NVT) 6662 return InOp; 6663 6664 ElementCount InEC = InVT.getVectorElementCount(); 6665 ElementCount WidenEC = NVT.getVectorElementCount(); 6666 if (WidenEC.hasKnownScalarFactor(InEC)) { 6667 unsigned NumConcat = WidenEC.getKnownScalarFactor(InEC); 6668 SmallVector<SDValue, 16> Ops(NumConcat); 6669 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) : 6670 DAG.getUNDEF(InVT); 6671 Ops[0] = InOp; 6672 for (unsigned i = 1; i != NumConcat; ++i) 6673 Ops[i] = FillVal; 6674 6675 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops); 6676 } 6677 6678 if (InEC.hasKnownScalarFactor(WidenEC)) 6679 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 6680 DAG.getVectorIdxConstant(0, dl)); 6681 6682 assert(!InVT.isScalableVector() && !NVT.isScalableVector() && 6683 "Scalable vectors should have been handled already."); 6684 6685 unsigned InNumElts = InEC.getFixedValue(); 6686 unsigned WidenNumElts = WidenEC.getFixedValue(); 6687 6688 // Fall back to extract and build. 6689 SmallVector<SDValue, 16> Ops(WidenNumElts); 6690 EVT EltVT = NVT.getVectorElementType(); 6691 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 6692 unsigned Idx; 6693 for (Idx = 0; Idx < MinNumElts; ++Idx) 6694 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 6695 DAG.getVectorIdxConstant(Idx, dl)); 6696 6697 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) : 6698 DAG.getUNDEF(EltVT); 6699 for ( ; Idx < WidenNumElts; ++Idx) 6700 Ops[Idx] = FillVal; 6701 return DAG.getBuildVector(NVT, dl, Ops); 6702 } 6703