1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file performs vector type splitting and scalarization for LegalizeTypes. 10 // Scalarization is the act of changing a computation in an illegal one-element 11 // vector type to be a computation in its scalar element type. For example, 12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which 14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 15 // types. 16 // Splitting is the act of changing a computation in an invalid vector type to 17 // be a computation in two vectors of half the size. For example, implementing 18 // <128 x f32> operations in terms of two <64 x f32> operations. 19 // 20 //===----------------------------------------------------------------------===// 21 22 #include "LegalizeTypes.h" 23 #include "llvm/ADT/SmallBitVector.h" 24 #include "llvm/Analysis/MemoryLocation.h" 25 #include "llvm/Analysis/VectorUtils.h" 26 #include "llvm/IR/DataLayout.h" 27 #include "llvm/Support/ErrorHandling.h" 28 #include "llvm/Support/TypeSize.h" 29 #include "llvm/Support/raw_ostream.h" 30 #include <numeric> 31 32 using namespace llvm; 33 34 #define DEBUG_TYPE "legalize-types" 35 36 //===----------------------------------------------------------------------===// 37 // Result Vector Scalarization: <1 x ty> -> ty. 38 //===----------------------------------------------------------------------===// 39 40 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 41 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 42 dbgs() << "\n"); 43 SDValue R = SDValue(); 44 45 switch (N->getOpcode()) { 46 default: 47 #ifndef NDEBUG 48 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 49 N->dump(&DAG); 50 dbgs() << "\n"; 51 #endif 52 report_fatal_error("Do not know how to scalarize the result of this " 53 "operator!\n"); 54 55 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 56 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 57 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 58 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 59 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 60 case ISD::FPOWI: R = ScalarizeVecRes_ExpOp(N); break; 61 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 62 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 63 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 64 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 65 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 66 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 67 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 68 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 69 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 70 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 71 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break; 72 case ISD::ANY_EXTEND_VECTOR_INREG: 73 case ISD::SIGN_EXTEND_VECTOR_INREG: 74 case ISD::ZERO_EXTEND_VECTOR_INREG: 75 R = ScalarizeVecRes_VecInregOp(N); 76 break; 77 case ISD::ABS: 78 case ISD::ANY_EXTEND: 79 case ISD::BITREVERSE: 80 case ISD::BSWAP: 81 case ISD::CTLZ: 82 case ISD::CTLZ_ZERO_UNDEF: 83 case ISD::CTPOP: 84 case ISD::CTTZ: 85 case ISD::CTTZ_ZERO_UNDEF: 86 case ISD::FABS: 87 case ISD::FCEIL: 88 case ISD::FCOS: 89 case ISD::FEXP: 90 case ISD::FEXP2: 91 case ISD::FFLOOR: 92 case ISD::FLOG: 93 case ISD::FLOG10: 94 case ISD::FLOG2: 95 case ISD::FNEARBYINT: 96 case ISD::FNEG: 97 case ISD::FREEZE: 98 case ISD::ARITH_FENCE: 99 case ISD::FP_EXTEND: 100 case ISD::FP_TO_SINT: 101 case ISD::FP_TO_UINT: 102 case ISD::FRINT: 103 case ISD::FROUND: 104 case ISD::FROUNDEVEN: 105 case ISD::FSIN: 106 case ISD::FSQRT: 107 case ISD::FTRUNC: 108 case ISD::SIGN_EXTEND: 109 case ISD::SINT_TO_FP: 110 case ISD::TRUNCATE: 111 case ISD::UINT_TO_FP: 112 case ISD::ZERO_EXTEND: 113 case ISD::FCANONICALIZE: 114 R = ScalarizeVecRes_UnaryOp(N); 115 break; 116 case ISD::FFREXP: 117 R = ScalarizeVecRes_FFREXP(N, ResNo); 118 break; 119 case ISD::ADD: 120 case ISD::AND: 121 case ISD::FADD: 122 case ISD::FCOPYSIGN: 123 case ISD::FDIV: 124 case ISD::FMUL: 125 case ISD::FMINNUM: 126 case ISD::FMAXNUM: 127 case ISD::FMINNUM_IEEE: 128 case ISD::FMAXNUM_IEEE: 129 case ISD::FMINIMUM: 130 case ISD::FMAXIMUM: 131 case ISD::FLDEXP: 132 case ISD::SMIN: 133 case ISD::SMAX: 134 case ISD::UMIN: 135 case ISD::UMAX: 136 137 case ISD::SADDSAT: 138 case ISD::UADDSAT: 139 case ISD::SSUBSAT: 140 case ISD::USUBSAT: 141 case ISD::SSHLSAT: 142 case ISD::USHLSAT: 143 144 case ISD::FPOW: 145 case ISD::FREM: 146 case ISD::FSUB: 147 case ISD::MUL: 148 case ISD::MULHS: 149 case ISD::MULHU: 150 case ISD::OR: 151 case ISD::SDIV: 152 case ISD::SREM: 153 case ISD::SUB: 154 case ISD::UDIV: 155 case ISD::UREM: 156 case ISD::XOR: 157 case ISD::SHL: 158 case ISD::SRA: 159 case ISD::SRL: 160 case ISD::ROTL: 161 case ISD::ROTR: 162 R = ScalarizeVecRes_BinOp(N); 163 break; 164 case ISD::FMA: 165 case ISD::FSHL: 166 case ISD::FSHR: 167 R = ScalarizeVecRes_TernaryOp(N); 168 break; 169 170 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 171 case ISD::STRICT_##DAGN: 172 #include "llvm/IR/ConstrainedOps.def" 173 R = ScalarizeVecRes_StrictFPOp(N); 174 break; 175 176 case ISD::FP_TO_UINT_SAT: 177 case ISD::FP_TO_SINT_SAT: 178 R = ScalarizeVecRes_FP_TO_XINT_SAT(N); 179 break; 180 181 case ISD::UADDO: 182 case ISD::SADDO: 183 case ISD::USUBO: 184 case ISD::SSUBO: 185 case ISD::UMULO: 186 case ISD::SMULO: 187 R = ScalarizeVecRes_OverflowOp(N, ResNo); 188 break; 189 case ISD::SMULFIX: 190 case ISD::SMULFIXSAT: 191 case ISD::UMULFIX: 192 case ISD::UMULFIXSAT: 193 case ISD::SDIVFIX: 194 case ISD::SDIVFIXSAT: 195 case ISD::UDIVFIX: 196 case ISD::UDIVFIXSAT: 197 R = ScalarizeVecRes_FIX(N); 198 break; 199 } 200 201 // If R is null, the sub-method took care of registering the result. 202 if (R.getNode()) 203 SetScalarizedVector(SDValue(N, ResNo), R); 204 } 205 206 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 207 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 208 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 209 return DAG.getNode(N->getOpcode(), SDLoc(N), 210 LHS.getValueType(), LHS, RHS, N->getFlags()); 211 } 212 213 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 214 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 215 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 216 SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 217 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1, 218 Op2, N->getFlags()); 219 } 220 221 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) { 222 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 223 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 224 SDValue Op2 = N->getOperand(2); 225 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1, 226 Op2, N->getFlags()); 227 } 228 229 SDValue DAGTypeLegalizer::ScalarizeVecRes_FFREXP(SDNode *N, unsigned ResNo) { 230 assert(N->getValueType(0).getVectorNumElements() == 1 && 231 "Unexpected vector type!"); 232 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 233 234 EVT VT0 = N->getValueType(0); 235 EVT VT1 = N->getValueType(1); 236 SDLoc dl(N); 237 238 SDNode *ScalarNode = 239 DAG.getNode(N->getOpcode(), dl, 240 {VT0.getScalarType(), VT1.getScalarType()}, Elt) 241 .getNode(); 242 243 // Replace the other vector result not being explicitly scalarized here. 244 unsigned OtherNo = 1 - ResNo; 245 EVT OtherVT = N->getValueType(OtherNo); 246 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) { 247 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo)); 248 } else { 249 SDValue OtherVal = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, OtherVT, 250 SDValue(ScalarNode, OtherNo)); 251 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 252 } 253 254 return SDValue(ScalarNode, ResNo); 255 } 256 257 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) { 258 EVT VT = N->getValueType(0).getVectorElementType(); 259 unsigned NumOpers = N->getNumOperands(); 260 SDValue Chain = N->getOperand(0); 261 EVT ValueVTs[] = {VT, MVT::Other}; 262 SDLoc dl(N); 263 264 SmallVector<SDValue, 4> Opers(NumOpers); 265 266 // The Chain is the first operand. 267 Opers[0] = Chain; 268 269 // Now process the remaining operands. 270 for (unsigned i = 1; i < NumOpers; ++i) { 271 SDValue Oper = N->getOperand(i); 272 EVT OperVT = Oper.getValueType(); 273 274 if (OperVT.isVector()) { 275 if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector) 276 Oper = GetScalarizedVector(Oper); 277 else 278 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 279 OperVT.getVectorElementType(), Oper, 280 DAG.getVectorIdxConstant(0, dl)); 281 } 282 283 Opers[i] = Oper; 284 } 285 286 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs), 287 Opers, N->getFlags()); 288 289 // Legalize the chain result - switch anything that used the old chain to 290 // use the new one. 291 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 292 return Result; 293 } 294 295 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N, 296 unsigned ResNo) { 297 SDLoc DL(N); 298 EVT ResVT = N->getValueType(0); 299 EVT OvVT = N->getValueType(1); 300 301 SDValue ScalarLHS, ScalarRHS; 302 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) { 303 ScalarLHS = GetScalarizedVector(N->getOperand(0)); 304 ScalarRHS = GetScalarizedVector(N->getOperand(1)); 305 } else { 306 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS; 307 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS); 308 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS); 309 ScalarLHS = ElemsLHS[0]; 310 ScalarRHS = ElemsRHS[0]; 311 } 312 313 SDVTList ScalarVTs = DAG.getVTList( 314 ResVT.getVectorElementType(), OvVT.getVectorElementType()); 315 SDNode *ScalarNode = DAG.getNode( 316 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode(); 317 ScalarNode->setFlags(N->getFlags()); 318 319 // Replace the other vector result not being explicitly scalarized here. 320 unsigned OtherNo = 1 - ResNo; 321 EVT OtherVT = N->getValueType(OtherNo); 322 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) { 323 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo)); 324 } else { 325 SDValue OtherVal = DAG.getNode( 326 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo)); 327 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 328 } 329 330 return SDValue(ScalarNode, ResNo); 331 } 332 333 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 334 unsigned ResNo) { 335 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 336 return GetScalarizedVector(Op); 337 } 338 339 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 340 SDValue Op = N->getOperand(0); 341 if (Op.getValueType().isVector() 342 && Op.getValueType().getVectorNumElements() == 1 343 && !isSimpleLegalType(Op.getValueType())) 344 Op = GetScalarizedVector(Op); 345 EVT NewVT = N->getValueType(0).getVectorElementType(); 346 return DAG.getNode(ISD::BITCAST, SDLoc(N), 347 NewVT, Op); 348 } 349 350 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 351 EVT EltVT = N->getValueType(0).getVectorElementType(); 352 SDValue InOp = N->getOperand(0); 353 // The BUILD_VECTOR operands may be of wider element types and 354 // we may need to truncate them back to the requested return type. 355 if (EltVT.isInteger()) 356 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 357 return InOp; 358 } 359 360 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 361 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 362 N->getValueType(0).getVectorElementType(), 363 N->getOperand(0), N->getOperand(1)); 364 } 365 366 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 367 SDLoc DL(N); 368 SDValue Op = N->getOperand(0); 369 EVT OpVT = Op.getValueType(); 370 // The result needs scalarizing, but it's not a given that the source does. 371 // See similar logic in ScalarizeVecRes_UnaryOp. 372 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 373 Op = GetScalarizedVector(Op); 374 } else { 375 EVT VT = OpVT.getVectorElementType(); 376 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, 377 DAG.getVectorIdxConstant(0, DL)); 378 } 379 return DAG.getNode(ISD::FP_ROUND, DL, 380 N->getValueType(0).getVectorElementType(), Op, 381 N->getOperand(1)); 382 } 383 384 SDValue DAGTypeLegalizer::ScalarizeVecRes_ExpOp(SDNode *N) { 385 SDValue Op = GetScalarizedVector(N->getOperand(0)); 386 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op, 387 N->getOperand(1)); 388 } 389 390 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 391 // The value to insert may have a wider type than the vector element type, 392 // so be sure to truncate it to the element type if necessary. 393 SDValue Op = N->getOperand(1); 394 EVT EltVT = N->getValueType(0).getVectorElementType(); 395 if (Op.getValueType() != EltVT) 396 // FIXME: Can this happen for floating point types? 397 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op); 398 return Op; 399 } 400 401 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 402 assert(N->isUnindexed() && "Indexed vector load?"); 403 404 SDValue Result = DAG.getLoad( 405 ISD::UNINDEXED, N->getExtensionType(), 406 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(), 407 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()), 408 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(), 409 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo()); 410 411 // Legalize the chain result - switch anything that used the old chain to 412 // use the new one. 413 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 414 return Result; 415 } 416 417 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 418 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 419 EVT DestVT = N->getValueType(0).getVectorElementType(); 420 SDValue Op = N->getOperand(0); 421 EVT OpVT = Op.getValueType(); 422 SDLoc DL(N); 423 // The result needs scalarizing, but it's not a given that the source does. 424 // This is a workaround for targets where it's impossible to scalarize the 425 // result of a conversion, because the source type is legal. 426 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32} 427 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is 428 // legal and was not scalarized. 429 // See the similar logic in ScalarizeVecRes_SETCC 430 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 431 Op = GetScalarizedVector(Op); 432 } else { 433 EVT VT = OpVT.getVectorElementType(); 434 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, 435 DAG.getVectorIdxConstant(0, DL)); 436 } 437 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags()); 438 } 439 440 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 441 EVT EltVT = N->getValueType(0).getVectorElementType(); 442 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 443 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 444 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT, 445 LHS, DAG.getValueType(ExtVT)); 446 } 447 448 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) { 449 SDLoc DL(N); 450 SDValue Op = N->getOperand(0); 451 452 EVT OpVT = Op.getValueType(); 453 EVT OpEltVT = OpVT.getVectorElementType(); 454 EVT EltVT = N->getValueType(0).getVectorElementType(); 455 456 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 457 Op = GetScalarizedVector(Op); 458 } else { 459 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op, 460 DAG.getVectorIdxConstant(0, DL)); 461 } 462 463 switch (N->getOpcode()) { 464 case ISD::ANY_EXTEND_VECTOR_INREG: 465 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op); 466 case ISD::SIGN_EXTEND_VECTOR_INREG: 467 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op); 468 case ISD::ZERO_EXTEND_VECTOR_INREG: 469 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op); 470 } 471 472 llvm_unreachable("Illegal extend_vector_inreg opcode"); 473 } 474 475 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 476 // If the operand is wider than the vector element type then it is implicitly 477 // truncated. Make that explicit here. 478 EVT EltVT = N->getValueType(0).getVectorElementType(); 479 SDValue InOp = N->getOperand(0); 480 if (InOp.getValueType() != EltVT) 481 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 482 return InOp; 483 } 484 485 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 486 SDValue Cond = N->getOperand(0); 487 EVT OpVT = Cond.getValueType(); 488 SDLoc DL(N); 489 // The vselect result and true/value operands needs scalarizing, but it's 490 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal. 491 // See the similar logic in ScalarizeVecRes_SETCC 492 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 493 Cond = GetScalarizedVector(Cond); 494 } else { 495 EVT VT = OpVT.getVectorElementType(); 496 Cond = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond, 497 DAG.getVectorIdxConstant(0, DL)); 498 } 499 500 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 501 TargetLowering::BooleanContent ScalarBool = 502 TLI.getBooleanContents(false, false); 503 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false); 504 505 // If integer and float booleans have different contents then we can't 506 // reliably optimize in all cases. There is a full explanation for this in 507 // DAGCombiner::visitSELECT() where the same issue affects folding 508 // (select C, 0, 1) to (xor C, 1). 509 if (TLI.getBooleanContents(false, false) != 510 TLI.getBooleanContents(false, true)) { 511 // At least try the common case where the boolean is generated by a 512 // comparison. 513 if (Cond->getOpcode() == ISD::SETCC) { 514 EVT OpVT = Cond->getOperand(0).getValueType(); 515 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType()); 516 VecBool = TLI.getBooleanContents(OpVT); 517 } else 518 ScalarBool = TargetLowering::UndefinedBooleanContent; 519 } 520 521 EVT CondVT = Cond.getValueType(); 522 if (ScalarBool != VecBool) { 523 switch (ScalarBool) { 524 case TargetLowering::UndefinedBooleanContent: 525 break; 526 case TargetLowering::ZeroOrOneBooleanContent: 527 assert(VecBool == TargetLowering::UndefinedBooleanContent || 528 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 529 // Vector read from all ones, scalar expects a single 1 so mask. 530 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT, 531 Cond, DAG.getConstant(1, SDLoc(N), CondVT)); 532 break; 533 case TargetLowering::ZeroOrNegativeOneBooleanContent: 534 assert(VecBool == TargetLowering::UndefinedBooleanContent || 535 VecBool == TargetLowering::ZeroOrOneBooleanContent); 536 // Vector reads from a one, scalar from all ones so sign extend. 537 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT, 538 Cond, DAG.getValueType(MVT::i1)); 539 break; 540 } 541 } 542 543 // Truncate the condition if needed 544 auto BoolVT = getSetCCResultType(CondVT); 545 if (BoolVT.bitsLT(CondVT)) 546 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond); 547 548 return DAG.getSelect(SDLoc(N), 549 LHS.getValueType(), Cond, LHS, 550 GetScalarizedVector(N->getOperand(2))); 551 } 552 553 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 554 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 555 return DAG.getSelect(SDLoc(N), 556 LHS.getValueType(), N->getOperand(0), LHS, 557 GetScalarizedVector(N->getOperand(2))); 558 } 559 560 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 561 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 562 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(), 563 N->getOperand(0), N->getOperand(1), 564 LHS, GetScalarizedVector(N->getOperand(3)), 565 N->getOperand(4)); 566 } 567 568 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 569 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 570 } 571 572 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 573 // Figure out if the scalar is the LHS or RHS and return it. 574 SDValue Arg = N->getOperand(2).getOperand(0); 575 if (Arg.isUndef()) 576 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 577 unsigned Op = !cast<ConstantSDNode>(Arg)->isZero(); 578 return GetScalarizedVector(N->getOperand(Op)); 579 } 580 581 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) { 582 SDValue Src = N->getOperand(0); 583 EVT SrcVT = Src.getValueType(); 584 SDLoc dl(N); 585 586 // Handle case where result is scalarized but operand is not 587 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector) 588 Src = GetScalarizedVector(Src); 589 else 590 Src = DAG.getNode( 591 ISD::EXTRACT_VECTOR_ELT, dl, SrcVT.getVectorElementType(), Src, 592 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 593 594 EVT DstVT = N->getValueType(0).getVectorElementType(); 595 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1)); 596 } 597 598 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 599 assert(N->getValueType(0).isVector() && 600 N->getOperand(0).getValueType().isVector() && 601 "Operand types must be vectors"); 602 SDValue LHS = N->getOperand(0); 603 SDValue RHS = N->getOperand(1); 604 EVT OpVT = LHS.getValueType(); 605 EVT NVT = N->getValueType(0).getVectorElementType(); 606 SDLoc DL(N); 607 608 // The result needs scalarizing, but it's not a given that the source does. 609 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 610 LHS = GetScalarizedVector(LHS); 611 RHS = GetScalarizedVector(RHS); 612 } else { 613 EVT VT = OpVT.getVectorElementType(); 614 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS, 615 DAG.getVectorIdxConstant(0, DL)); 616 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS, 617 DAG.getVectorIdxConstant(0, DL)); 618 } 619 620 // Turn it into a scalar SETCC. 621 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 622 N->getOperand(2)); 623 // Vectors may have a different boolean contents to scalars. Promote the 624 // value appropriately. 625 ISD::NodeType ExtendCode = 626 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 627 return DAG.getNode(ExtendCode, DL, NVT, Res); 628 } 629 630 SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) { 631 SDLoc DL(N); 632 SDValue Arg = N->getOperand(0); 633 SDValue Test = N->getOperand(1); 634 EVT ArgVT = Arg.getValueType(); 635 EVT ResultVT = N->getValueType(0).getVectorElementType(); 636 637 if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) { 638 Arg = GetScalarizedVector(Arg); 639 } else { 640 EVT VT = ArgVT.getVectorElementType(); 641 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Arg, 642 DAG.getVectorIdxConstant(0, DL)); 643 } 644 645 SDValue Res = 646 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags()); 647 // Vectors may have a different boolean contents to scalars. Promote the 648 // value appropriately. 649 ISD::NodeType ExtendCode = 650 TargetLowering::getExtendForContent(TLI.getBooleanContents(ArgVT)); 651 return DAG.getNode(ExtendCode, DL, ResultVT, Res); 652 } 653 654 //===----------------------------------------------------------------------===// 655 // Operand Vector Scalarization <1 x ty> -> ty. 656 //===----------------------------------------------------------------------===// 657 658 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 659 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 660 dbgs() << "\n"); 661 SDValue Res = SDValue(); 662 663 switch (N->getOpcode()) { 664 default: 665 #ifndef NDEBUG 666 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 667 N->dump(&DAG); 668 dbgs() << "\n"; 669 #endif 670 report_fatal_error("Do not know how to scalarize this operator's " 671 "operand!\n"); 672 case ISD::BITCAST: 673 Res = ScalarizeVecOp_BITCAST(N); 674 break; 675 case ISD::ANY_EXTEND: 676 case ISD::ZERO_EXTEND: 677 case ISD::SIGN_EXTEND: 678 case ISD::TRUNCATE: 679 case ISD::FP_TO_SINT: 680 case ISD::FP_TO_UINT: 681 case ISD::SINT_TO_FP: 682 case ISD::UINT_TO_FP: 683 Res = ScalarizeVecOp_UnaryOp(N); 684 break; 685 case ISD::STRICT_SINT_TO_FP: 686 case ISD::STRICT_UINT_TO_FP: 687 case ISD::STRICT_FP_TO_SINT: 688 case ISD::STRICT_FP_TO_UINT: 689 Res = ScalarizeVecOp_UnaryOp_StrictFP(N); 690 break; 691 case ISD::CONCAT_VECTORS: 692 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 693 break; 694 case ISD::EXTRACT_VECTOR_ELT: 695 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 696 break; 697 case ISD::VSELECT: 698 Res = ScalarizeVecOp_VSELECT(N); 699 break; 700 case ISD::SETCC: 701 Res = ScalarizeVecOp_VSETCC(N); 702 break; 703 case ISD::STORE: 704 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 705 break; 706 case ISD::STRICT_FP_ROUND: 707 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo); 708 break; 709 case ISD::FP_ROUND: 710 Res = ScalarizeVecOp_FP_ROUND(N, OpNo); 711 break; 712 case ISD::STRICT_FP_EXTEND: 713 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N); 714 break; 715 case ISD::FP_EXTEND: 716 Res = ScalarizeVecOp_FP_EXTEND(N); 717 break; 718 case ISD::VECREDUCE_FADD: 719 case ISD::VECREDUCE_FMUL: 720 case ISD::VECREDUCE_ADD: 721 case ISD::VECREDUCE_MUL: 722 case ISD::VECREDUCE_AND: 723 case ISD::VECREDUCE_OR: 724 case ISD::VECREDUCE_XOR: 725 case ISD::VECREDUCE_SMAX: 726 case ISD::VECREDUCE_SMIN: 727 case ISD::VECREDUCE_UMAX: 728 case ISD::VECREDUCE_UMIN: 729 case ISD::VECREDUCE_FMAX: 730 case ISD::VECREDUCE_FMIN: 731 case ISD::VECREDUCE_FMAXIMUM: 732 case ISD::VECREDUCE_FMINIMUM: 733 Res = ScalarizeVecOp_VECREDUCE(N); 734 break; 735 case ISD::VECREDUCE_SEQ_FADD: 736 case ISD::VECREDUCE_SEQ_FMUL: 737 Res = ScalarizeVecOp_VECREDUCE_SEQ(N); 738 break; 739 } 740 741 // If the result is null, the sub-method took care of registering results etc. 742 if (!Res.getNode()) return false; 743 744 // If the result is N, the sub-method updated N in place. Tell the legalizer 745 // core about this. 746 if (Res.getNode() == N) 747 return true; 748 749 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 750 "Invalid operand expansion"); 751 752 ReplaceValueWith(SDValue(N, 0), Res); 753 return false; 754 } 755 756 /// If the value to convert is a vector that needs to be scalarized, it must be 757 /// <1 x ty>. Convert the element instead. 758 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 759 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 760 return DAG.getNode(ISD::BITCAST, SDLoc(N), 761 N->getValueType(0), Elt); 762 } 763 764 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>. 765 /// Do the operation on the element instead. 766 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) { 767 assert(N->getValueType(0).getVectorNumElements() == 1 && 768 "Unexpected vector type!"); 769 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 770 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N), 771 N->getValueType(0).getScalarType(), Elt); 772 // Revectorize the result so the types line up with what the uses of this 773 // expression expect. 774 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op); 775 } 776 777 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>. 778 /// Do the strict FP operation on the element instead. 779 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) { 780 assert(N->getValueType(0).getVectorNumElements() == 1 && 781 "Unexpected vector type!"); 782 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 783 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), 784 { N->getValueType(0).getScalarType(), MVT::Other }, 785 { N->getOperand(0), Elt }); 786 // Legalize the chain result - switch anything that used the old chain to 787 // use the new one. 788 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 789 // Revectorize the result so the types line up with what the uses of this 790 // expression expect. 791 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 792 793 // Do our own replacement and return SDValue() to tell the caller that we 794 // handled all replacements since caller can only handle a single result. 795 ReplaceValueWith(SDValue(N, 0), Res); 796 return SDValue(); 797 } 798 799 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead. 800 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 801 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 802 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 803 Ops[i] = GetScalarizedVector(N->getOperand(i)); 804 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops); 805 } 806 807 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>, 808 /// so just return the element, ignoring the index. 809 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 810 EVT VT = N->getValueType(0); 811 SDValue Res = GetScalarizedVector(N->getOperand(0)); 812 if (Res.getValueType() != VT) 813 Res = VT.isFloatingPoint() 814 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res) 815 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res); 816 return Res; 817 } 818 819 /// If the input condition is a vector that needs to be scalarized, it must be 820 /// <1 x i1>, so just convert to a normal ISD::SELECT 821 /// (still with vector output type since that was acceptable if we got here). 822 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) { 823 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0)); 824 EVT VT = N->getValueType(0); 825 826 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1), 827 N->getOperand(2)); 828 } 829 830 /// If the operand is a vector that needs to be scalarized then the 831 /// result must be v1i1, so just convert to a scalar SETCC and wrap 832 /// with a scalar_to_vector since the res type is legal if we got here 833 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) { 834 assert(N->getValueType(0).isVector() && 835 N->getOperand(0).getValueType().isVector() && 836 "Operand types must be vectors"); 837 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type"); 838 839 EVT VT = N->getValueType(0); 840 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 841 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 842 843 EVT OpVT = N->getOperand(0).getValueType(); 844 EVT NVT = VT.getVectorElementType(); 845 SDLoc DL(N); 846 // Turn it into a scalar SETCC. 847 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 848 N->getOperand(2)); 849 850 // Vectors may have a different boolean contents to scalars. Promote the 851 // value appropriately. 852 ISD::NodeType ExtendCode = 853 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 854 855 Res = DAG.getNode(ExtendCode, DL, NVT, Res); 856 857 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res); 858 } 859 860 /// If the value to store is a vector that needs to be scalarized, it must be 861 /// <1 x ty>. Just store the element. 862 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 863 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 864 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 865 SDLoc dl(N); 866 867 if (N->isTruncatingStore()) 868 return DAG.getTruncStore( 869 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 870 N->getBasePtr(), N->getPointerInfo(), 871 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(), 872 N->getMemOperand()->getFlags(), N->getAAInfo()); 873 874 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 875 N->getBasePtr(), N->getPointerInfo(), 876 N->getOriginalAlign(), N->getMemOperand()->getFlags(), 877 N->getAAInfo()); 878 } 879 880 /// If the value to round is a vector that needs to be scalarized, it must be 881 /// <1 x ty>. Convert the element instead. 882 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { 883 assert(OpNo == 0 && "Wrong operand for scalarization!"); 884 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 885 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N), 886 N->getValueType(0).getVectorElementType(), Elt, 887 N->getOperand(1)); 888 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 889 } 890 891 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, 892 unsigned OpNo) { 893 assert(OpNo == 1 && "Wrong operand for scalarization!"); 894 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 895 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 896 { N->getValueType(0).getVectorElementType(), 897 MVT::Other }, 898 { N->getOperand(0), Elt, N->getOperand(2) }); 899 // Legalize the chain result - switch anything that used the old chain to 900 // use the new one. 901 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 902 903 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 904 905 // Do our own replacement and return SDValue() to tell the caller that we 906 // handled all replacements since caller can only handle a single result. 907 ReplaceValueWith(SDValue(N, 0), Res); 908 return SDValue(); 909 } 910 911 /// If the value to extend is a vector that needs to be scalarized, it must be 912 /// <1 x ty>. Convert the element instead. 913 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) { 914 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 915 SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), 916 N->getValueType(0).getVectorElementType(), Elt); 917 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 918 } 919 920 /// If the value to extend is a vector that needs to be scalarized, it must be 921 /// <1 x ty>. Convert the element instead. 922 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) { 923 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 924 SDValue Res = 925 DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 926 {N->getValueType(0).getVectorElementType(), MVT::Other}, 927 {N->getOperand(0), Elt}); 928 // Legalize the chain result - switch anything that used the old chain to 929 // use the new one. 930 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 931 932 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 933 934 // Do our own replacement and return SDValue() to tell the caller that we 935 // handled all replacements since caller can only handle a single result. 936 ReplaceValueWith(SDValue(N, 0), Res); 937 return SDValue(); 938 } 939 940 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) { 941 SDValue Res = GetScalarizedVector(N->getOperand(0)); 942 // Result type may be wider than element type. 943 if (Res.getValueType() != N->getValueType(0)) 944 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res); 945 return Res; 946 } 947 948 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) { 949 SDValue AccOp = N->getOperand(0); 950 SDValue VecOp = N->getOperand(1); 951 952 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode()); 953 954 SDValue Op = GetScalarizedVector(VecOp); 955 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0), 956 AccOp, Op, N->getFlags()); 957 } 958 959 //===----------------------------------------------------------------------===// 960 // Result Vector Splitting 961 //===----------------------------------------------------------------------===// 962 963 /// This method is called when the specified result of the specified node is 964 /// found to need vector splitting. At this point, the node may also have 965 /// invalid operands or may have other results that need legalization, we just 966 /// know that (at least) one result needs vector splitting. 967 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 968 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n"); 969 SDValue Lo, Hi; 970 971 // See if the target wants to custom expand this node. 972 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 973 return; 974 975 switch (N->getOpcode()) { 976 default: 977 #ifndef NDEBUG 978 dbgs() << "SplitVectorResult #" << ResNo << ": "; 979 N->dump(&DAG); 980 dbgs() << "\n"; 981 #endif 982 report_fatal_error("Do not know how to split the result of this " 983 "operator!\n"); 984 985 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 986 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break; 987 case ISD::VSELECT: 988 case ISD::SELECT: 989 case ISD::VP_MERGE: 990 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break; 991 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 992 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 993 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 994 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 995 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 996 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 997 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break; 998 case ISD::FPOWI: 999 case ISD::FLDEXP: 1000 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break; 1001 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break; 1002 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 1003 case ISD::SPLAT_VECTOR: 1004 case ISD::SCALAR_TO_VECTOR: 1005 SplitVecRes_ScalarOp(N, Lo, Hi); 1006 break; 1007 case ISD::STEP_VECTOR: 1008 SplitVecRes_STEP_VECTOR(N, Lo, Hi); 1009 break; 1010 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 1011 case ISD::LOAD: 1012 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 1013 break; 1014 case ISD::VP_LOAD: 1015 SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi); 1016 break; 1017 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 1018 SplitVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N), Lo, Hi); 1019 break; 1020 case ISD::MLOAD: 1021 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi); 1022 break; 1023 case ISD::MGATHER: 1024 case ISD::VP_GATHER: 1025 SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true); 1026 break; 1027 case ISD::SETCC: 1028 case ISD::VP_SETCC: 1029 SplitVecRes_SETCC(N, Lo, Hi); 1030 break; 1031 case ISD::VECTOR_REVERSE: 1032 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi); 1033 break; 1034 case ISD::VECTOR_SHUFFLE: 1035 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 1036 break; 1037 case ISD::VECTOR_SPLICE: 1038 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi); 1039 break; 1040 case ISD::VECTOR_DEINTERLEAVE: 1041 SplitVecRes_VECTOR_DEINTERLEAVE(N); 1042 return; 1043 case ISD::VECTOR_INTERLEAVE: 1044 SplitVecRes_VECTOR_INTERLEAVE(N); 1045 return; 1046 case ISD::VAARG: 1047 SplitVecRes_VAARG(N, Lo, Hi); 1048 break; 1049 1050 case ISD::ANY_EXTEND_VECTOR_INREG: 1051 case ISD::SIGN_EXTEND_VECTOR_INREG: 1052 case ISD::ZERO_EXTEND_VECTOR_INREG: 1053 SplitVecRes_ExtVecInRegOp(N, Lo, Hi); 1054 break; 1055 1056 case ISD::ABS: 1057 case ISD::VP_ABS: 1058 case ISD::BITREVERSE: 1059 case ISD::VP_BITREVERSE: 1060 case ISD::BSWAP: 1061 case ISD::VP_BSWAP: 1062 case ISD::CTLZ: 1063 case ISD::VP_CTLZ: 1064 case ISD::CTTZ: 1065 case ISD::VP_CTTZ: 1066 case ISD::CTLZ_ZERO_UNDEF: 1067 case ISD::VP_CTLZ_ZERO_UNDEF: 1068 case ISD::CTTZ_ZERO_UNDEF: 1069 case ISD::VP_CTTZ_ZERO_UNDEF: 1070 case ISD::CTPOP: 1071 case ISD::VP_CTPOP: 1072 case ISD::FABS: case ISD::VP_FABS: 1073 case ISD::FCEIL: 1074 case ISD::VP_FCEIL: 1075 case ISD::FCOS: 1076 case ISD::FEXP: 1077 case ISD::FEXP2: 1078 case ISD::FFLOOR: 1079 case ISD::VP_FFLOOR: 1080 case ISD::FLOG: 1081 case ISD::FLOG10: 1082 case ISD::FLOG2: 1083 case ISD::FNEARBYINT: 1084 case ISD::VP_FNEARBYINT: 1085 case ISD::FNEG: case ISD::VP_FNEG: 1086 case ISD::FREEZE: 1087 case ISD::ARITH_FENCE: 1088 case ISD::FP_EXTEND: 1089 case ISD::VP_FP_EXTEND: 1090 case ISD::FP_ROUND: 1091 case ISD::VP_FP_ROUND: 1092 case ISD::FP_TO_SINT: 1093 case ISD::VP_FP_TO_SINT: 1094 case ISD::FP_TO_UINT: 1095 case ISD::VP_FP_TO_UINT: 1096 case ISD::FRINT: 1097 case ISD::VP_FRINT: 1098 case ISD::FROUND: 1099 case ISD::VP_FROUND: 1100 case ISD::FROUNDEVEN: 1101 case ISD::VP_FROUNDEVEN: 1102 case ISD::FSIN: 1103 case ISD::FSQRT: case ISD::VP_SQRT: 1104 case ISD::FTRUNC: 1105 case ISD::VP_FROUNDTOZERO: 1106 case ISD::SINT_TO_FP: 1107 case ISD::VP_SINT_TO_FP: 1108 case ISD::TRUNCATE: 1109 case ISD::VP_TRUNCATE: 1110 case ISD::UINT_TO_FP: 1111 case ISD::VP_UINT_TO_FP: 1112 case ISD::FCANONICALIZE: 1113 SplitVecRes_UnaryOp(N, Lo, Hi); 1114 break; 1115 case ISD::FFREXP: 1116 SplitVecRes_FFREXP(N, ResNo, Lo, Hi); 1117 break; 1118 1119 case ISD::ANY_EXTEND: 1120 case ISD::SIGN_EXTEND: 1121 case ISD::ZERO_EXTEND: 1122 case ISD::VP_SIGN_EXTEND: 1123 case ISD::VP_ZERO_EXTEND: 1124 SplitVecRes_ExtendOp(N, Lo, Hi); 1125 break; 1126 1127 case ISD::ADD: case ISD::VP_ADD: 1128 case ISD::SUB: case ISD::VP_SUB: 1129 case ISD::MUL: case ISD::VP_MUL: 1130 case ISD::MULHS: 1131 case ISD::MULHU: 1132 case ISD::FADD: case ISD::VP_FADD: 1133 case ISD::FSUB: case ISD::VP_FSUB: 1134 case ISD::FMUL: case ISD::VP_FMUL: 1135 case ISD::FMINNUM: case ISD::VP_FMINNUM: 1136 case ISD::FMAXNUM: case ISD::VP_FMAXNUM: 1137 case ISD::FMINIMUM: 1138 case ISD::FMAXIMUM: 1139 case ISD::SDIV: case ISD::VP_SDIV: 1140 case ISD::UDIV: case ISD::VP_UDIV: 1141 case ISD::FDIV: case ISD::VP_FDIV: 1142 case ISD::FPOW: 1143 case ISD::AND: case ISD::VP_AND: 1144 case ISD::OR: case ISD::VP_OR: 1145 case ISD::XOR: case ISD::VP_XOR: 1146 case ISD::SHL: case ISD::VP_SHL: 1147 case ISD::SRA: case ISD::VP_ASHR: 1148 case ISD::SRL: case ISD::VP_LSHR: 1149 case ISD::UREM: case ISD::VP_UREM: 1150 case ISD::SREM: case ISD::VP_SREM: 1151 case ISD::FREM: case ISD::VP_FREM: 1152 case ISD::SMIN: case ISD::VP_SMIN: 1153 case ISD::SMAX: case ISD::VP_SMAX: 1154 case ISD::UMIN: case ISD::VP_UMIN: 1155 case ISD::UMAX: case ISD::VP_UMAX: 1156 case ISD::SADDSAT: 1157 case ISD::UADDSAT: 1158 case ISD::SSUBSAT: 1159 case ISD::USUBSAT: 1160 case ISD::SSHLSAT: 1161 case ISD::USHLSAT: 1162 case ISD::ROTL: 1163 case ISD::ROTR: 1164 case ISD::VP_FCOPYSIGN: 1165 SplitVecRes_BinOp(N, Lo, Hi); 1166 break; 1167 case ISD::FMA: case ISD::VP_FMA: 1168 case ISD::FSHL: 1169 case ISD::VP_FSHL: 1170 case ISD::FSHR: 1171 case ISD::VP_FSHR: 1172 SplitVecRes_TernaryOp(N, Lo, Hi); 1173 break; 1174 1175 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 1176 case ISD::STRICT_##DAGN: 1177 #include "llvm/IR/ConstrainedOps.def" 1178 SplitVecRes_StrictFPOp(N, Lo, Hi); 1179 break; 1180 1181 case ISD::FP_TO_UINT_SAT: 1182 case ISD::FP_TO_SINT_SAT: 1183 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi); 1184 break; 1185 1186 case ISD::UADDO: 1187 case ISD::SADDO: 1188 case ISD::USUBO: 1189 case ISD::SSUBO: 1190 case ISD::UMULO: 1191 case ISD::SMULO: 1192 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi); 1193 break; 1194 case ISD::SMULFIX: 1195 case ISD::SMULFIXSAT: 1196 case ISD::UMULFIX: 1197 case ISD::UMULFIXSAT: 1198 case ISD::SDIVFIX: 1199 case ISD::SDIVFIXSAT: 1200 case ISD::UDIVFIX: 1201 case ISD::UDIVFIXSAT: 1202 SplitVecRes_FIX(N, Lo, Hi); 1203 break; 1204 } 1205 1206 // If Lo/Hi is null, the sub-method took care of registering results etc. 1207 if (Lo.getNode()) 1208 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 1209 } 1210 1211 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT, 1212 MachinePointerInfo &MPI, SDValue &Ptr, 1213 uint64_t *ScaledOffset) { 1214 SDLoc DL(N); 1215 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8; 1216 1217 if (MemVT.isScalableVector()) { 1218 SDNodeFlags Flags; 1219 SDValue BytesIncrement = DAG.getVScale( 1220 DL, Ptr.getValueType(), 1221 APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize)); 1222 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); 1223 Flags.setNoUnsignedWrap(true); 1224 if (ScaledOffset) 1225 *ScaledOffset += IncrementSize; 1226 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement, 1227 Flags); 1228 } else { 1229 MPI = N->getPointerInfo().getWithOffset(IncrementSize); 1230 // Increment the pointer to the other half. 1231 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize)); 1232 } 1233 } 1234 1235 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) { 1236 return SplitMask(Mask, SDLoc(Mask)); 1237 } 1238 1239 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask, 1240 const SDLoc &DL) { 1241 SDValue MaskLo, MaskHi; 1242 EVT MaskVT = Mask.getValueType(); 1243 if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector) 1244 GetSplitVector(Mask, MaskLo, MaskHi); 1245 else 1246 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 1247 return std::make_pair(MaskLo, MaskHi); 1248 } 1249 1250 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) { 1251 SDValue LHSLo, LHSHi; 1252 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1253 SDValue RHSLo, RHSHi; 1254 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 1255 SDLoc dl(N); 1256 1257 const SDNodeFlags Flags = N->getFlags(); 1258 unsigned Opcode = N->getOpcode(); 1259 if (N->getNumOperands() == 2) { 1260 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags); 1261 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags); 1262 return; 1263 } 1264 1265 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 1266 assert(N->isVPOpcode() && "Expected VP opcode"); 1267 1268 SDValue MaskLo, MaskHi; 1269 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2)); 1270 1271 SDValue EVLLo, EVLHi; 1272 std::tie(EVLLo, EVLHi) = 1273 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl); 1274 1275 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), 1276 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags); 1277 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), 1278 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags); 1279 } 1280 1281 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 1282 SDValue &Hi) { 1283 SDValue Op0Lo, Op0Hi; 1284 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 1285 SDValue Op1Lo, Op1Hi; 1286 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 1287 SDValue Op2Lo, Op2Hi; 1288 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 1289 SDLoc dl(N); 1290 1291 const SDNodeFlags Flags = N->getFlags(); 1292 unsigned Opcode = N->getOpcode(); 1293 if (N->getNumOperands() == 3) { 1294 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags); 1295 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags); 1296 return; 1297 } 1298 1299 assert(N->getNumOperands() == 5 && "Unexpected number of operands!"); 1300 assert(N->isVPOpcode() && "Expected VP opcode"); 1301 1302 SDValue MaskLo, MaskHi; 1303 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3)); 1304 1305 SDValue EVLLo, EVLHi; 1306 std::tie(EVLLo, EVLHi) = 1307 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), dl); 1308 1309 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), 1310 {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags); 1311 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), 1312 {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags); 1313 } 1314 1315 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) { 1316 SDValue LHSLo, LHSHi; 1317 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1318 SDValue RHSLo, RHSHi; 1319 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 1320 SDLoc dl(N); 1321 SDValue Op2 = N->getOperand(2); 1322 1323 unsigned Opcode = N->getOpcode(); 1324 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2, 1325 N->getFlags()); 1326 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2, 1327 N->getFlags()); 1328 } 1329 1330 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 1331 SDValue &Hi) { 1332 // We know the result is a vector. The input may be either a vector or a 1333 // scalar value. 1334 EVT LoVT, HiVT; 1335 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1336 SDLoc dl(N); 1337 1338 SDValue InOp = N->getOperand(0); 1339 EVT InVT = InOp.getValueType(); 1340 1341 // Handle some special cases efficiently. 1342 switch (getTypeAction(InVT)) { 1343 case TargetLowering::TypeLegal: 1344 case TargetLowering::TypePromoteInteger: 1345 case TargetLowering::TypePromoteFloat: 1346 case TargetLowering::TypeSoftPromoteHalf: 1347 case TargetLowering::TypeSoftenFloat: 1348 case TargetLowering::TypeScalarizeVector: 1349 case TargetLowering::TypeWidenVector: 1350 break; 1351 case TargetLowering::TypeExpandInteger: 1352 case TargetLowering::TypeExpandFloat: 1353 // A scalar to vector conversion, where the scalar needs expansion. 1354 // If the vector is being split in two then we can just convert the 1355 // expanded pieces. 1356 if (LoVT == HiVT) { 1357 GetExpandedOp(InOp, Lo, Hi); 1358 if (DAG.getDataLayout().isBigEndian()) 1359 std::swap(Lo, Hi); 1360 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1361 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1362 return; 1363 } 1364 break; 1365 case TargetLowering::TypeSplitVector: 1366 // If the input is a vector that needs to be split, convert each split 1367 // piece of the input now. 1368 GetSplitVector(InOp, Lo, Hi); 1369 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1370 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1371 return; 1372 case TargetLowering::TypeScalarizeScalableVector: 1373 report_fatal_error("Scalarization of scalable vectors is not supported."); 1374 } 1375 1376 // In the general case, convert the input to an integer and split it by hand. 1377 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 1378 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 1379 if (DAG.getDataLayout().isBigEndian()) 1380 std::swap(LoIntVT, HiIntVT); 1381 1382 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 1383 1384 if (DAG.getDataLayout().isBigEndian()) 1385 std::swap(Lo, Hi); 1386 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1387 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1388 } 1389 1390 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 1391 SDValue &Hi) { 1392 EVT LoVT, HiVT; 1393 SDLoc dl(N); 1394 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1395 unsigned LoNumElts = LoVT.getVectorNumElements(); 1396 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 1397 Lo = DAG.getBuildVector(LoVT, dl, LoOps); 1398 1399 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 1400 Hi = DAG.getBuildVector(HiVT, dl, HiOps); 1401 } 1402 1403 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 1404 SDValue &Hi) { 1405 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 1406 SDLoc dl(N); 1407 unsigned NumSubvectors = N->getNumOperands() / 2; 1408 if (NumSubvectors == 1) { 1409 Lo = N->getOperand(0); 1410 Hi = N->getOperand(1); 1411 return; 1412 } 1413 1414 EVT LoVT, HiVT; 1415 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1416 1417 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 1418 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps); 1419 1420 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 1421 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps); 1422 } 1423 1424 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 1425 SDValue &Hi) { 1426 SDValue Vec = N->getOperand(0); 1427 SDValue Idx = N->getOperand(1); 1428 SDLoc dl(N); 1429 1430 EVT LoVT, HiVT; 1431 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1432 1433 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 1434 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1435 Hi = DAG.getNode( 1436 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 1437 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl)); 1438 } 1439 1440 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, 1441 SDValue &Hi) { 1442 SDValue Vec = N->getOperand(0); 1443 SDValue SubVec = N->getOperand(1); 1444 SDValue Idx = N->getOperand(2); 1445 SDLoc dl(N); 1446 GetSplitVector(Vec, Lo, Hi); 1447 1448 EVT VecVT = Vec.getValueType(); 1449 EVT LoVT = Lo.getValueType(); 1450 EVT SubVecVT = SubVec.getValueType(); 1451 unsigned VecElems = VecVT.getVectorMinNumElements(); 1452 unsigned SubElems = SubVecVT.getVectorMinNumElements(); 1453 unsigned LoElems = LoVT.getVectorMinNumElements(); 1454 1455 // If we know the index is in the first half, and we know the subvector 1456 // doesn't cross the boundary between the halves, we can avoid spilling the 1457 // vector, and insert into the lower half of the split vector directly. 1458 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1459 if (IdxVal + SubElems <= LoElems) { 1460 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx); 1461 return; 1462 } 1463 // Similarly if the subvector is fully in the high half, but mind that we 1464 // can't tell whether a fixed-length subvector is fully within the high half 1465 // of a scalable vector. 1466 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() && 1467 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) { 1468 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec, 1469 DAG.getVectorIdxConstant(IdxVal - LoElems, dl)); 1470 return; 1471 } 1472 1473 // Spill the vector to the stack. 1474 // In cases where the vector is illegal it will be broken down into parts 1475 // and stored in parts - we should use the alignment for the smallest part. 1476 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false); 1477 SDValue StackPtr = 1478 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign); 1479 auto &MF = DAG.getMachineFunction(); 1480 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1481 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 1482 1483 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1484 SmallestAlign); 1485 1486 // Store the new subvector into the specified index. 1487 SDValue SubVecPtr = 1488 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx); 1489 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, 1490 MachinePointerInfo::getUnknownStack(MF)); 1491 1492 // Load the Lo part from the stack slot. 1493 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo, 1494 SmallestAlign); 1495 1496 // Increment the pointer to the other part. 1497 auto *Load = cast<LoadSDNode>(Lo); 1498 MachinePointerInfo MPI = Load->getPointerInfo(); 1499 IncrementPointer(Load, LoVT, MPI, StackPtr); 1500 1501 // Load the Hi part from the stack slot. 1502 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign); 1503 } 1504 1505 // Handle splitting an FP where the second operand does not match the first 1506 // type. The second operand may be a scalar, or a vector that has exactly as 1507 // many elements as the first 1508 void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo, 1509 SDValue &Hi) { 1510 SDValue LHSLo, LHSHi; 1511 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1512 SDLoc DL(N); 1513 1514 SDValue RHSLo, RHSHi; 1515 SDValue RHS = N->getOperand(1); 1516 EVT RHSVT = RHS.getValueType(); 1517 if (RHSVT.isVector()) { 1518 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector) 1519 GetSplitVector(RHS, RHSLo, RHSHi); 1520 else 1521 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS)); 1522 1523 Lo = DAG.getNode(N->getOpcode(), DL, LHSLo.getValueType(), LHSLo, RHSLo); 1524 Hi = DAG.getNode(N->getOpcode(), DL, LHSHi.getValueType(), LHSHi, RHSHi); 1525 } else { 1526 Lo = DAG.getNode(N->getOpcode(), DL, LHSLo.getValueType(), LHSLo, RHS); 1527 Hi = DAG.getNode(N->getOpcode(), DL, LHSHi.getValueType(), LHSHi, RHS); 1528 } 1529 } 1530 1531 void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo, 1532 SDValue &Hi) { 1533 SDLoc DL(N); 1534 SDValue ArgLo, ArgHi; 1535 SDValue Test = N->getOperand(1); 1536 SDValue FpValue = N->getOperand(0); 1537 if (getTypeAction(FpValue.getValueType()) == TargetLowering::TypeSplitVector) 1538 GetSplitVector(FpValue, ArgLo, ArgHi); 1539 else 1540 std::tie(ArgLo, ArgHi) = DAG.SplitVector(FpValue, SDLoc(FpValue)); 1541 EVT LoVT, HiVT; 1542 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1543 1544 Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags()); 1545 Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags()); 1546 } 1547 1548 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 1549 SDValue &Hi) { 1550 SDValue LHSLo, LHSHi; 1551 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1552 SDLoc dl(N); 1553 1554 EVT LoVT, HiVT; 1555 std::tie(LoVT, HiVT) = 1556 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT()); 1557 1558 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 1559 DAG.getValueType(LoVT)); 1560 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 1561 DAG.getValueType(HiVT)); 1562 } 1563 1564 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, 1565 SDValue &Hi) { 1566 unsigned Opcode = N->getOpcode(); 1567 SDValue N0 = N->getOperand(0); 1568 1569 SDLoc dl(N); 1570 SDValue InLo, InHi; 1571 1572 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector) 1573 GetSplitVector(N0, InLo, InHi); 1574 else 1575 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0); 1576 1577 EVT InLoVT = InLo.getValueType(); 1578 unsigned InNumElements = InLoVT.getVectorNumElements(); 1579 1580 EVT OutLoVT, OutHiVT; 1581 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1582 unsigned OutNumElements = OutLoVT.getVectorNumElements(); 1583 assert((2 * OutNumElements) <= InNumElements && 1584 "Illegal extend vector in reg split"); 1585 1586 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the 1587 // input vector (i.e. we only use InLo): 1588 // OutLo will extend the first OutNumElements from InLo. 1589 // OutHi will extend the next OutNumElements from InLo. 1590 1591 // Shuffle the elements from InLo for OutHi into the bottom elements to 1592 // create a 'fake' InHi. 1593 SmallVector<int, 8> SplitHi(InNumElements, -1); 1594 for (unsigned i = 0; i != OutNumElements; ++i) 1595 SplitHi[i] = i + OutNumElements; 1596 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi); 1597 1598 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo); 1599 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi); 1600 } 1601 1602 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, 1603 SDValue &Hi) { 1604 unsigned NumOps = N->getNumOperands(); 1605 SDValue Chain = N->getOperand(0); 1606 EVT LoVT, HiVT; 1607 SDLoc dl(N); 1608 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1609 1610 SmallVector<SDValue, 4> OpsLo(NumOps); 1611 SmallVector<SDValue, 4> OpsHi(NumOps); 1612 1613 // The Chain is the first operand. 1614 OpsLo[0] = Chain; 1615 OpsHi[0] = Chain; 1616 1617 // Now process the remaining operands. 1618 for (unsigned i = 1; i < NumOps; ++i) { 1619 SDValue Op = N->getOperand(i); 1620 SDValue OpLo = Op; 1621 SDValue OpHi = Op; 1622 1623 EVT InVT = Op.getValueType(); 1624 if (InVT.isVector()) { 1625 // If the input also splits, handle it directly for a 1626 // compile time speedup. Otherwise split it by hand. 1627 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 1628 GetSplitVector(Op, OpLo, OpHi); 1629 else 1630 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i); 1631 } 1632 1633 OpsLo[i] = OpLo; 1634 OpsHi[i] = OpHi; 1635 } 1636 1637 EVT LoValueVTs[] = {LoVT, MVT::Other}; 1638 EVT HiValueVTs[] = {HiVT, MVT::Other}; 1639 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo, 1640 N->getFlags()); 1641 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi, 1642 N->getFlags()); 1643 1644 // Build a factor node to remember that this Op is independent of the 1645 // other one. 1646 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1647 Lo.getValue(1), Hi.getValue(1)); 1648 1649 // Legalize the chain result - switch anything that used the old chain to 1650 // use the new one. 1651 ReplaceValueWith(SDValue(N, 1), Chain); 1652 } 1653 1654 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { 1655 SDValue Chain = N->getOperand(0); 1656 EVT VT = N->getValueType(0); 1657 unsigned NE = VT.getVectorNumElements(); 1658 EVT EltVT = VT.getVectorElementType(); 1659 SDLoc dl(N); 1660 1661 SmallVector<SDValue, 8> Scalars; 1662 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 1663 1664 // If ResNE is 0, fully unroll the vector op. 1665 if (ResNE == 0) 1666 ResNE = NE; 1667 else if (NE > ResNE) 1668 NE = ResNE; 1669 1670 //The results of each unrolled operation, including the chain. 1671 EVT ChainVTs[] = {EltVT, MVT::Other}; 1672 SmallVector<SDValue, 8> Chains; 1673 1674 unsigned i; 1675 for (i = 0; i != NE; ++i) { 1676 Operands[0] = Chain; 1677 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) { 1678 SDValue Operand = N->getOperand(j); 1679 EVT OperandVT = Operand.getValueType(); 1680 if (OperandVT.isVector()) { 1681 EVT OperandEltVT = OperandVT.getVectorElementType(); 1682 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, 1683 Operand, DAG.getVectorIdxConstant(i, dl)); 1684 } else { 1685 Operands[j] = Operand; 1686 } 1687 } 1688 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands); 1689 Scalar.getNode()->setFlags(N->getFlags()); 1690 1691 //Add in the scalar as well as its chain value to the 1692 //result vectors. 1693 Scalars.push_back(Scalar); 1694 Chains.push_back(Scalar.getValue(1)); 1695 } 1696 1697 for (; i < ResNE; ++i) 1698 Scalars.push_back(DAG.getUNDEF(EltVT)); 1699 1700 // Build a new factor node to connect the chain back together. 1701 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 1702 ReplaceValueWith(SDValue(N, 1), Chain); 1703 1704 // Create a new BUILD_VECTOR node 1705 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE); 1706 return DAG.getBuildVector(VecVT, dl, Scalars); 1707 } 1708 1709 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, 1710 SDValue &Lo, SDValue &Hi) { 1711 SDLoc dl(N); 1712 EVT ResVT = N->getValueType(0); 1713 EVT OvVT = N->getValueType(1); 1714 EVT LoResVT, HiResVT, LoOvVT, HiOvVT; 1715 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT); 1716 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT); 1717 1718 SDValue LoLHS, HiLHS, LoRHS, HiRHS; 1719 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) { 1720 GetSplitVector(N->getOperand(0), LoLHS, HiLHS); 1721 GetSplitVector(N->getOperand(1), LoRHS, HiRHS); 1722 } else { 1723 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0); 1724 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1); 1725 } 1726 1727 unsigned Opcode = N->getOpcode(); 1728 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT); 1729 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT); 1730 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode(); 1731 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode(); 1732 LoNode->setFlags(N->getFlags()); 1733 HiNode->setFlags(N->getFlags()); 1734 1735 Lo = SDValue(LoNode, ResNo); 1736 Hi = SDValue(HiNode, ResNo); 1737 1738 // Replace the other vector result not being explicitly split here. 1739 unsigned OtherNo = 1 - ResNo; 1740 EVT OtherVT = N->getValueType(OtherNo); 1741 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) { 1742 SetSplitVector(SDValue(N, OtherNo), 1743 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo)); 1744 } else { 1745 SDValue OtherVal = DAG.getNode( 1746 ISD::CONCAT_VECTORS, dl, OtherVT, 1747 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo)); 1748 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 1749 } 1750 } 1751 1752 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 1753 SDValue &Hi) { 1754 SDValue Vec = N->getOperand(0); 1755 SDValue Elt = N->getOperand(1); 1756 SDValue Idx = N->getOperand(2); 1757 SDLoc dl(N); 1758 GetSplitVector(Vec, Lo, Hi); 1759 1760 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 1761 unsigned IdxVal = CIdx->getZExtValue(); 1762 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements(); 1763 if (IdxVal < LoNumElts) { 1764 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 1765 Lo.getValueType(), Lo, Elt, Idx); 1766 return; 1767 } else if (!Vec.getValueType().isScalableVector()) { 1768 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 1769 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl)); 1770 return; 1771 } 1772 } 1773 1774 // See if the target wants to custom expand this node. 1775 if (CustomLowerNode(N, N->getValueType(0), true)) 1776 return; 1777 1778 // Make the vector elements byte-addressable if they aren't already. 1779 EVT VecVT = Vec.getValueType(); 1780 EVT EltVT = VecVT.getVectorElementType(); 1781 if (VecVT.getScalarSizeInBits() < 8) { 1782 EltVT = MVT::i8; 1783 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 1784 VecVT.getVectorElementCount()); 1785 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec); 1786 // Extend the element type to match if needed. 1787 if (EltVT.bitsGT(Elt.getValueType())) 1788 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt); 1789 } 1790 1791 // Spill the vector to the stack. 1792 // In cases where the vector is illegal it will be broken down into parts 1793 // and stored in parts - we should use the alignment for the smallest part. 1794 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false); 1795 SDValue StackPtr = 1796 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign); 1797 auto &MF = DAG.getMachineFunction(); 1798 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1799 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 1800 1801 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1802 SmallestAlign); 1803 1804 // Store the new element. This may be larger than the vector element type, 1805 // so use a truncating store. 1806 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 1807 Store = DAG.getTruncStore( 1808 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT, 1809 commonAlignment(SmallestAlign, 1810 EltVT.getFixedSizeInBits() / 8)); 1811 1812 EVT LoVT, HiVT; 1813 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT); 1814 1815 // Load the Lo part from the stack slot. 1816 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign); 1817 1818 // Increment the pointer to the other part. 1819 auto Load = cast<LoadSDNode>(Lo); 1820 MachinePointerInfo MPI = Load->getPointerInfo(); 1821 IncrementPointer(Load, LoVT, MPI, StackPtr); 1822 1823 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign); 1824 1825 // If we adjusted the original type, we need to truncate the results. 1826 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1827 if (LoVT != Lo.getValueType()) 1828 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo); 1829 if (HiVT != Hi.getValueType()) 1830 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi); 1831 } 1832 1833 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, 1834 SDValue &Hi) { 1835 EVT LoVT, HiVT; 1836 SDLoc dl(N); 1837 assert(N->getValueType(0).isScalableVector() && 1838 "Only scalable vectors are supported for STEP_VECTOR"); 1839 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1840 SDValue Step = N->getOperand(0); 1841 1842 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step); 1843 1844 // Hi = Lo + (EltCnt * Step) 1845 EVT EltVT = Step.getValueType(); 1846 APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue(); 1847 SDValue StartOfHi = 1848 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements()); 1849 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType()); 1850 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi); 1851 1852 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step); 1853 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi); 1854 } 1855 1856 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, 1857 SDValue &Hi) { 1858 EVT LoVT, HiVT; 1859 SDLoc dl(N); 1860 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1861 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0)); 1862 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { 1863 Hi = DAG.getUNDEF(HiVT); 1864 } else { 1865 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode"); 1866 Hi = Lo; 1867 } 1868 } 1869 1870 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 1871 SDValue &Hi) { 1872 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 1873 EVT LoVT, HiVT; 1874 SDLoc dl(LD); 1875 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0)); 1876 1877 ISD::LoadExtType ExtType = LD->getExtensionType(); 1878 SDValue Ch = LD->getChain(); 1879 SDValue Ptr = LD->getBasePtr(); 1880 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 1881 EVT MemoryVT = LD->getMemoryVT(); 1882 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 1883 AAMDNodes AAInfo = LD->getAAInfo(); 1884 1885 EVT LoMemVT, HiMemVT; 1886 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1887 1888 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) { 1889 SDValue Value, NewChain; 1890 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG); 1891 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl); 1892 ReplaceValueWith(SDValue(LD, 1), NewChain); 1893 return; 1894 } 1895 1896 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 1897 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(), 1898 MMOFlags, AAInfo); 1899 1900 MachinePointerInfo MPI; 1901 IncrementPointer(LD, LoMemVT, MPI, Ptr); 1902 1903 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI, 1904 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo); 1905 1906 // Build a factor node to remember that this load is independent of the 1907 // other one. 1908 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1909 Hi.getValue(1)); 1910 1911 // Legalize the chain result - switch anything that used the old chain to 1912 // use the new one. 1913 ReplaceValueWith(SDValue(LD, 1), Ch); 1914 } 1915 1916 void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo, 1917 SDValue &Hi) { 1918 assert(LD->isUnindexed() && "Indexed VP load during type legalization!"); 1919 EVT LoVT, HiVT; 1920 SDLoc dl(LD); 1921 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0)); 1922 1923 ISD::LoadExtType ExtType = LD->getExtensionType(); 1924 SDValue Ch = LD->getChain(); 1925 SDValue Ptr = LD->getBasePtr(); 1926 SDValue Offset = LD->getOffset(); 1927 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset"); 1928 Align Alignment = LD->getOriginalAlign(); 1929 SDValue Mask = LD->getMask(); 1930 SDValue EVL = LD->getVectorLength(); 1931 EVT MemoryVT = LD->getMemoryVT(); 1932 1933 EVT LoMemVT, HiMemVT; 1934 bool HiIsEmpty = false; 1935 std::tie(LoMemVT, HiMemVT) = 1936 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty); 1937 1938 // Split Mask operand 1939 SDValue MaskLo, MaskHi; 1940 if (Mask.getOpcode() == ISD::SETCC) { 1941 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 1942 } else { 1943 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1944 GetSplitVector(Mask, MaskLo, MaskHi); 1945 else 1946 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1947 } 1948 1949 // Split EVL operand 1950 SDValue EVLLo, EVLHi; 1951 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl); 1952 1953 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 1954 LD->getPointerInfo(), MachineMemOperand::MOLoad, 1955 MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges()); 1956 1957 Lo = 1958 DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset, 1959 MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad()); 1960 1961 if (HiIsEmpty) { 1962 // The hi vp_load has zero storage size. We therefore simply set it to 1963 // the low vp_load and rely on subsequent removal from the chain. 1964 Hi = Lo; 1965 } else { 1966 // Generate hi vp_load. 1967 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG, 1968 LD->isExpandingLoad()); 1969 1970 MachinePointerInfo MPI; 1971 if (LoMemVT.isScalableVector()) 1972 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace()); 1973 else 1974 MPI = LD->getPointerInfo().getWithOffset( 1975 LoMemVT.getStoreSize().getFixedValue()); 1976 1977 MMO = DAG.getMachineFunction().getMachineMemOperand( 1978 MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment, 1979 LD->getAAInfo(), LD->getRanges()); 1980 1981 Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr, 1982 Offset, MaskHi, EVLHi, HiMemVT, MMO, 1983 LD->isExpandingLoad()); 1984 } 1985 1986 // Build a factor node to remember that this load is independent of the 1987 // other one. 1988 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1989 Hi.getValue(1)); 1990 1991 // Legalize the chain result - switch anything that used the old chain to 1992 // use the new one. 1993 ReplaceValueWith(SDValue(LD, 1), Ch); 1994 } 1995 1996 void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD, 1997 SDValue &Lo, SDValue &Hi) { 1998 assert(SLD->isUnindexed() && 1999 "Indexed VP strided load during type legalization!"); 2000 assert(SLD->getOffset().isUndef() && 2001 "Unexpected indexed variable-length load offset"); 2002 2003 SDLoc DL(SLD); 2004 2005 EVT LoVT, HiVT; 2006 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(SLD->getValueType(0)); 2007 2008 EVT LoMemVT, HiMemVT; 2009 bool HiIsEmpty = false; 2010 std::tie(LoMemVT, HiMemVT) = 2011 DAG.GetDependentSplitDestVTs(SLD->getMemoryVT(), LoVT, &HiIsEmpty); 2012 2013 SDValue Mask = SLD->getMask(); 2014 SDValue LoMask, HiMask; 2015 if (Mask.getOpcode() == ISD::SETCC) { 2016 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask); 2017 } else { 2018 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 2019 GetSplitVector(Mask, LoMask, HiMask); 2020 else 2021 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL); 2022 } 2023 2024 SDValue LoEVL, HiEVL; 2025 std::tie(LoEVL, HiEVL) = 2026 DAG.SplitEVL(SLD->getVectorLength(), SLD->getValueType(0), DL); 2027 2028 // Generate the low vp_strided_load 2029 Lo = DAG.getStridedLoadVP( 2030 SLD->getAddressingMode(), SLD->getExtensionType(), LoVT, DL, 2031 SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(), SLD->getStride(), 2032 LoMask, LoEVL, LoMemVT, SLD->getMemOperand(), SLD->isExpandingLoad()); 2033 2034 if (HiIsEmpty) { 2035 // The high vp_strided_load has zero storage size. We therefore simply set 2036 // it to the low vp_strided_load and rely on subsequent removal from the 2037 // chain. 2038 Hi = Lo; 2039 } else { 2040 // Generate the high vp_strided_load. 2041 // To calculate the high base address, we need to sum to the low base 2042 // address stride number of bytes for each element already loaded by low, 2043 // that is: Ptr = Ptr + (LoEVL * Stride) 2044 EVT PtrVT = SLD->getBasePtr().getValueType(); 2045 SDValue Increment = 2046 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL, 2047 DAG.getSExtOrTrunc(SLD->getStride(), DL, PtrVT)); 2048 SDValue Ptr = 2049 DAG.getNode(ISD::ADD, DL, PtrVT, SLD->getBasePtr(), Increment); 2050 2051 Align Alignment = SLD->getOriginalAlign(); 2052 if (LoMemVT.isScalableVector()) 2053 Alignment = commonAlignment( 2054 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8); 2055 2056 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 2057 MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()), 2058 MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment, 2059 SLD->getAAInfo(), SLD->getRanges()); 2060 2061 Hi = DAG.getStridedLoadVP(SLD->getAddressingMode(), SLD->getExtensionType(), 2062 HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(), 2063 SLD->getStride(), HiMask, HiEVL, HiMemVT, MMO, 2064 SLD->isExpandingLoad()); 2065 } 2066 2067 // Build a factor node to remember that this load is independent of the 2068 // other one. 2069 SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1), 2070 Hi.getValue(1)); 2071 2072 // Legalize the chain result - switch anything that used the old chain to 2073 // use the new one. 2074 ReplaceValueWith(SDValue(SLD, 1), Ch); 2075 } 2076 2077 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, 2078 SDValue &Lo, SDValue &Hi) { 2079 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!"); 2080 EVT LoVT, HiVT; 2081 SDLoc dl(MLD); 2082 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0)); 2083 2084 SDValue Ch = MLD->getChain(); 2085 SDValue Ptr = MLD->getBasePtr(); 2086 SDValue Offset = MLD->getOffset(); 2087 assert(Offset.isUndef() && "Unexpected indexed masked load offset"); 2088 SDValue Mask = MLD->getMask(); 2089 SDValue PassThru = MLD->getPassThru(); 2090 Align Alignment = MLD->getOriginalAlign(); 2091 ISD::LoadExtType ExtType = MLD->getExtensionType(); 2092 2093 // Split Mask operand 2094 SDValue MaskLo, MaskHi; 2095 if (Mask.getOpcode() == ISD::SETCC) { 2096 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 2097 } else { 2098 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 2099 GetSplitVector(Mask, MaskLo, MaskHi); 2100 else 2101 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 2102 } 2103 2104 EVT MemoryVT = MLD->getMemoryVT(); 2105 EVT LoMemVT, HiMemVT; 2106 bool HiIsEmpty = false; 2107 std::tie(LoMemVT, HiMemVT) = 2108 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty); 2109 2110 SDValue PassThruLo, PassThruHi; 2111 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector) 2112 GetSplitVector(PassThru, PassThruLo, PassThruHi); 2113 else 2114 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); 2115 2116 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 2117 MLD->getPointerInfo(), MachineMemOperand::MOLoad, 2118 MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(), 2119 MLD->getRanges()); 2120 2121 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT, 2122 MMO, MLD->getAddressingMode(), ExtType, 2123 MLD->isExpandingLoad()); 2124 2125 if (HiIsEmpty) { 2126 // The hi masked load has zero storage size. We therefore simply set it to 2127 // the low masked load and rely on subsequent removal from the chain. 2128 Hi = Lo; 2129 } else { 2130 // Generate hi masked load. 2131 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG, 2132 MLD->isExpandingLoad()); 2133 2134 MachinePointerInfo MPI; 2135 if (LoMemVT.isScalableVector()) 2136 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace()); 2137 else 2138 MPI = MLD->getPointerInfo().getWithOffset( 2139 LoMemVT.getStoreSize().getFixedValue()); 2140 2141 MMO = DAG.getMachineFunction().getMachineMemOperand( 2142 MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment, 2143 MLD->getAAInfo(), MLD->getRanges()); 2144 2145 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi, 2146 HiMemVT, MMO, MLD->getAddressingMode(), ExtType, 2147 MLD->isExpandingLoad()); 2148 } 2149 2150 // Build a factor node to remember that this load is independent of the 2151 // other one. 2152 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2153 Hi.getValue(1)); 2154 2155 // Legalize the chain result - switch anything that used the old chain to 2156 // use the new one. 2157 ReplaceValueWith(SDValue(MLD, 1), Ch); 2158 2159 } 2160 2161 void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo, 2162 SDValue &Hi, bool SplitSETCC) { 2163 EVT LoVT, HiVT; 2164 SDLoc dl(N); 2165 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 2166 2167 SDValue Ch = N->getChain(); 2168 SDValue Ptr = N->getBasePtr(); 2169 struct Operands { 2170 SDValue Mask; 2171 SDValue Index; 2172 SDValue Scale; 2173 } Ops = [&]() -> Operands { 2174 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) { 2175 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()}; 2176 } 2177 auto *VPSC = cast<VPGatherSDNode>(N); 2178 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()}; 2179 }(); 2180 2181 EVT MemoryVT = N->getMemoryVT(); 2182 Align Alignment = N->getOriginalAlign(); 2183 2184 // Split Mask operand 2185 SDValue MaskLo, MaskHi; 2186 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) { 2187 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi); 2188 } else { 2189 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl); 2190 } 2191 2192 EVT LoMemVT, HiMemVT; 2193 // Split MemoryVT 2194 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 2195 2196 SDValue IndexHi, IndexLo; 2197 if (getTypeAction(Ops.Index.getValueType()) == 2198 TargetLowering::TypeSplitVector) 2199 GetSplitVector(Ops.Index, IndexLo, IndexHi); 2200 else 2201 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl); 2202 2203 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 2204 N->getPointerInfo(), MachineMemOperand::MOLoad, 2205 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges()); 2206 2207 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) { 2208 SDValue PassThru = MGT->getPassThru(); 2209 SDValue PassThruLo, PassThruHi; 2210 if (getTypeAction(PassThru.getValueType()) == 2211 TargetLowering::TypeSplitVector) 2212 GetSplitVector(PassThru, PassThruLo, PassThruHi); 2213 else 2214 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); 2215 2216 ISD::LoadExtType ExtType = MGT->getExtensionType(); 2217 ISD::MemIndexType IndexTy = MGT->getIndexType(); 2218 2219 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale}; 2220 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, 2221 OpsLo, MMO, IndexTy, ExtType); 2222 2223 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale}; 2224 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, 2225 OpsHi, MMO, IndexTy, ExtType); 2226 } else { 2227 auto *VPGT = cast<VPGatherSDNode>(N); 2228 SDValue EVLLo, EVLHi; 2229 std::tie(EVLLo, EVLHi) = 2230 DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl); 2231 2232 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo}; 2233 Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo, 2234 MMO, VPGT->getIndexType()); 2235 2236 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi}; 2237 Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi, 2238 MMO, VPGT->getIndexType()); 2239 } 2240 2241 // Build a factor node to remember that this load is independent of the 2242 // other one. 2243 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2244 Hi.getValue(1)); 2245 2246 // Legalize the chain result - switch anything that used the old chain to 2247 // use the new one. 2248 ReplaceValueWith(SDValue(N, 1), Ch); 2249 } 2250 2251 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 2252 assert(N->getValueType(0).isVector() && 2253 N->getOperand(0).getValueType().isVector() && 2254 "Operand types must be vectors"); 2255 2256 EVT LoVT, HiVT; 2257 SDLoc DL(N); 2258 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 2259 2260 // If the input also splits, handle it directly. Otherwise split it by hand. 2261 SDValue LL, LH, RL, RH; 2262 if (getTypeAction(N->getOperand(0).getValueType()) == 2263 TargetLowering::TypeSplitVector) 2264 GetSplitVector(N->getOperand(0), LL, LH); 2265 else 2266 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0); 2267 2268 if (getTypeAction(N->getOperand(1).getValueType()) == 2269 TargetLowering::TypeSplitVector) 2270 GetSplitVector(N->getOperand(1), RL, RH); 2271 else 2272 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1); 2273 2274 if (N->getOpcode() == ISD::SETCC) { 2275 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 2276 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 2277 } else { 2278 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode"); 2279 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 2280 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3)); 2281 std::tie(EVLLo, EVLHi) = 2282 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL); 2283 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2), MaskLo, 2284 EVLLo); 2285 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2), MaskHi, 2286 EVLHi); 2287 } 2288 } 2289 2290 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 2291 SDValue &Hi) { 2292 // Get the dest types - they may not match the input types, e.g. int_to_fp. 2293 EVT LoVT, HiVT; 2294 SDLoc dl(N); 2295 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 2296 2297 // If the input also splits, handle it directly for a compile time speedup. 2298 // Otherwise split it by hand. 2299 EVT InVT = N->getOperand(0).getValueType(); 2300 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 2301 GetSplitVector(N->getOperand(0), Lo, Hi); 2302 else 2303 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); 2304 2305 const SDNodeFlags Flags = N->getFlags(); 2306 unsigned Opcode = N->getOpcode(); 2307 if (N->getNumOperands() <= 2) { 2308 if (Opcode == ISD::FP_ROUND) { 2309 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags); 2310 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags); 2311 } else { 2312 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags); 2313 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags); 2314 } 2315 return; 2316 } 2317 2318 assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); 2319 assert(N->isVPOpcode() && "Expected VP opcode"); 2320 2321 SDValue MaskLo, MaskHi; 2322 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 2323 2324 SDValue EVLLo, EVLHi; 2325 std::tie(EVLLo, EVLHi) = 2326 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); 2327 2328 Lo = DAG.getNode(Opcode, dl, LoVT, {Lo, MaskLo, EVLLo}, Flags); 2329 Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags); 2330 } 2331 2332 void DAGTypeLegalizer::SplitVecRes_FFREXP(SDNode *N, unsigned ResNo, 2333 SDValue &Lo, SDValue &Hi) { 2334 SDLoc dl(N); 2335 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(N->getValueType(0)); 2336 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(N->getValueType(1)); 2337 2338 // If the input also splits, handle it directly for a compile time speedup. 2339 // Otherwise split it by hand. 2340 EVT InVT = N->getOperand(0).getValueType(); 2341 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 2342 GetSplitVector(N->getOperand(0), Lo, Hi); 2343 else 2344 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); 2345 2346 Lo = DAG.getNode(N->getOpcode(), dl, {LoVT, LoVT1}, Lo); 2347 Hi = DAG.getNode(N->getOpcode(), dl, {HiVT, HiVT1}, Hi); 2348 Lo->setFlags(N->getFlags()); 2349 Hi->setFlags(N->getFlags()); 2350 2351 SDNode *HiNode = Hi.getNode(); 2352 SDNode *LoNode = Lo.getNode(); 2353 2354 // Replace the other vector result not being explicitly split here. 2355 unsigned OtherNo = 1 - ResNo; 2356 EVT OtherVT = N->getValueType(OtherNo); 2357 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) { 2358 SetSplitVector(SDValue(N, OtherNo), SDValue(LoNode, OtherNo), 2359 SDValue(HiNode, OtherNo)); 2360 } else { 2361 SDValue OtherVal = 2362 DAG.getNode(ISD::CONCAT_VECTORS, dl, OtherVT, SDValue(LoNode, OtherNo), 2363 SDValue(HiNode, OtherNo)); 2364 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 2365 } 2366 } 2367 2368 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, 2369 SDValue &Hi) { 2370 SDLoc dl(N); 2371 EVT SrcVT = N->getOperand(0).getValueType(); 2372 EVT DestVT = N->getValueType(0); 2373 EVT LoVT, HiVT; 2374 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT); 2375 2376 // We can do better than a generic split operation if the extend is doing 2377 // more than just doubling the width of the elements and the following are 2378 // true: 2379 // - The number of vector elements is even, 2380 // - the source type is legal, 2381 // - the type of a split source is illegal, 2382 // - the type of an extended (by doubling element size) source is legal, and 2383 // - the type of that extended source when split is legal. 2384 // 2385 // This won't necessarily completely legalize the operation, but it will 2386 // more effectively move in the right direction and prevent falling down 2387 // to scalarization in many cases due to the input vector being split too 2388 // far. 2389 if (SrcVT.getVectorElementCount().isKnownEven() && 2390 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) { 2391 LLVMContext &Ctx = *DAG.getContext(); 2392 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx); 2393 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx); 2394 2395 EVT SplitLoVT, SplitHiVT; 2396 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT); 2397 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) && 2398 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) { 2399 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:"; 2400 N->dump(&DAG); dbgs() << "\n"); 2401 if (!N->isVPOpcode()) { 2402 // Extend the source vector by one step. 2403 SDValue NewSrc = 2404 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0)); 2405 // Get the low and high halves of the new, extended one step, vector. 2406 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl); 2407 // Extend those vector halves the rest of the way. 2408 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 2409 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 2410 return; 2411 } 2412 2413 // Extend the source vector by one step. 2414 SDValue NewSrc = 2415 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0), 2416 N->getOperand(1), N->getOperand(2)); 2417 // Get the low and high halves of the new, extended one step, vector. 2418 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl); 2419 2420 SDValue MaskLo, MaskHi; 2421 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 2422 2423 SDValue EVLLo, EVLHi; 2424 std::tie(EVLLo, EVLHi) = 2425 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); 2426 // Extend those vector halves the rest of the way. 2427 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo}); 2428 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi}); 2429 return; 2430 } 2431 } 2432 // Fall back to the generic unary operator splitting otherwise. 2433 SplitVecRes_UnaryOp(N, Lo, Hi); 2434 } 2435 2436 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 2437 SDValue &Lo, SDValue &Hi) { 2438 // The low and high parts of the original input give four input vectors. 2439 SDValue Inputs[4]; 2440 SDLoc DL(N); 2441 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 2442 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 2443 EVT NewVT = Inputs[0].getValueType(); 2444 unsigned NewElts = NewVT.getVectorNumElements(); 2445 2446 auto &&IsConstant = [](const SDValue &N) { 2447 APInt SplatValue; 2448 return N.getResNo() == 0 && 2449 (ISD::isConstantSplatVector(N.getNode(), SplatValue) || 2450 ISD::isBuildVectorOfConstantSDNodes(N.getNode())); 2451 }; 2452 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1, 2453 SDValue &Input2, 2454 ArrayRef<int> Mask) { 2455 assert(Input1->getOpcode() == ISD::BUILD_VECTOR && 2456 Input2->getOpcode() == ISD::BUILD_VECTOR && 2457 "Expected build vector node."); 2458 EVT EltVT = NewVT.getVectorElementType(); 2459 SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT)); 2460 for (unsigned I = 0; I < NewElts; ++I) { 2461 if (Mask[I] == PoisonMaskElem) 2462 continue; 2463 unsigned Idx = Mask[I]; 2464 if (Idx >= NewElts) 2465 Ops[I] = Input2.getOperand(Idx - NewElts); 2466 else 2467 Ops[I] = Input1.getOperand(Idx); 2468 // Make the type of all elements the same as the element type. 2469 if (Ops[I].getValueType().bitsGT(EltVT)) 2470 Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]); 2471 } 2472 return DAG.getBuildVector(NewVT, DL, Ops); 2473 }; 2474 2475 // If Lo or Hi uses elements from at most two of the four input vectors, then 2476 // express it as a vector shuffle of those two inputs. Otherwise extract the 2477 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 2478 SmallVector<int> OrigMask(N->getMask()); 2479 // Try to pack incoming shuffles/inputs. 2480 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts, 2481 &DL](SmallVectorImpl<int> &Mask) { 2482 // Check if all inputs are shuffles of the same operands or non-shuffles. 2483 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs; 2484 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) { 2485 SDValue Input = Inputs[Idx]; 2486 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode()); 2487 if (!Shuffle || 2488 Input.getOperand(0).getValueType() != Input.getValueType()) 2489 continue; 2490 ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))] 2491 .push_back(Idx); 2492 ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))] 2493 .push_back(Idx); 2494 } 2495 for (auto &P : ShufflesIdxs) { 2496 if (P.second.size() < 2) 2497 continue; 2498 // Use shuffles operands instead of shuffles themselves. 2499 // 1. Adjust mask. 2500 for (int &Idx : Mask) { 2501 if (Idx == PoisonMaskElem) 2502 continue; 2503 unsigned SrcRegIdx = Idx / NewElts; 2504 if (Inputs[SrcRegIdx].isUndef()) { 2505 Idx = PoisonMaskElem; 2506 continue; 2507 } 2508 auto *Shuffle = 2509 dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode()); 2510 if (!Shuffle || !is_contained(P.second, SrcRegIdx)) 2511 continue; 2512 int MaskElt = Shuffle->getMaskElt(Idx % NewElts); 2513 if (MaskElt == PoisonMaskElem) { 2514 Idx = PoisonMaskElem; 2515 continue; 2516 } 2517 Idx = MaskElt % NewElts + 2518 P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first 2519 ? 0 2520 : 1] * 2521 NewElts; 2522 } 2523 // 2. Update inputs. 2524 Inputs[P.second[0]] = P.first.first; 2525 Inputs[P.second[1]] = P.first.second; 2526 // Clear the pair data. 2527 P.second.clear(); 2528 ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear(); 2529 } 2530 // Check if any concat_vectors can be simplified. 2531 SmallBitVector UsedSubVector(2 * std::size(Inputs)); 2532 for (int &Idx : Mask) { 2533 if (Idx == PoisonMaskElem) 2534 continue; 2535 unsigned SrcRegIdx = Idx / NewElts; 2536 if (Inputs[SrcRegIdx].isUndef()) { 2537 Idx = PoisonMaskElem; 2538 continue; 2539 } 2540 TargetLowering::LegalizeTypeAction TypeAction = 2541 getTypeAction(Inputs[SrcRegIdx].getValueType()); 2542 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS && 2543 Inputs[SrcRegIdx].getNumOperands() == 2 && 2544 !Inputs[SrcRegIdx].getOperand(1).isUndef() && 2545 (TypeAction == TargetLowering::TypeLegal || 2546 TypeAction == TargetLowering::TypeWidenVector)) 2547 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2)); 2548 } 2549 if (UsedSubVector.count() > 1) { 2550 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs; 2551 for (unsigned I = 0; I < std::size(Inputs); ++I) { 2552 if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1)) 2553 continue; 2554 if (Pairs.empty() || Pairs.back().size() == 2) 2555 Pairs.emplace_back(); 2556 if (UsedSubVector.test(2 * I)) { 2557 Pairs.back().emplace_back(I, 0); 2558 } else { 2559 assert(UsedSubVector.test(2 * I + 1) && 2560 "Expected to be used one of the subvectors."); 2561 Pairs.back().emplace_back(I, 1); 2562 } 2563 } 2564 if (!Pairs.empty() && Pairs.front().size() > 1) { 2565 // Adjust mask. 2566 for (int &Idx : Mask) { 2567 if (Idx == PoisonMaskElem) 2568 continue; 2569 unsigned SrcRegIdx = Idx / NewElts; 2570 auto *It = find_if( 2571 Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) { 2572 return Idxs.front().first == SrcRegIdx || 2573 Idxs.back().first == SrcRegIdx; 2574 }); 2575 if (It == Pairs.end()) 2576 continue; 2577 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) + 2578 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2)); 2579 } 2580 // Adjust inputs. 2581 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) { 2582 Inputs[Idxs.front().first] = DAG.getNode( 2583 ISD::CONCAT_VECTORS, DL, 2584 Inputs[Idxs.front().first].getValueType(), 2585 Inputs[Idxs.front().first].getOperand(Idxs.front().second), 2586 Inputs[Idxs.back().first].getOperand(Idxs.back().second)); 2587 } 2588 } 2589 } 2590 bool Changed; 2591 do { 2592 // Try to remove extra shuffles (except broadcasts) and shuffles with the 2593 // reused operands. 2594 Changed = false; 2595 for (unsigned I = 0; I < std::size(Inputs); ++I) { 2596 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode()); 2597 if (!Shuffle) 2598 continue; 2599 if (Shuffle->getOperand(0).getValueType() != NewVT) 2600 continue; 2601 int Op = -1; 2602 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() && 2603 !Shuffle->isSplat()) { 2604 Op = 0; 2605 } else if (!Inputs[I].hasOneUse() && 2606 !Shuffle->getOperand(1).isUndef()) { 2607 // Find the only used operand, if possible. 2608 for (int &Idx : Mask) { 2609 if (Idx == PoisonMaskElem) 2610 continue; 2611 unsigned SrcRegIdx = Idx / NewElts; 2612 if (SrcRegIdx != I) 2613 continue; 2614 int MaskElt = Shuffle->getMaskElt(Idx % NewElts); 2615 if (MaskElt == PoisonMaskElem) { 2616 Idx = PoisonMaskElem; 2617 continue; 2618 } 2619 int OpIdx = MaskElt / NewElts; 2620 if (Op == -1) { 2621 Op = OpIdx; 2622 continue; 2623 } 2624 if (Op != OpIdx) { 2625 Op = -1; 2626 break; 2627 } 2628 } 2629 } 2630 if (Op < 0) { 2631 // Try to check if one of the shuffle operands is used already. 2632 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) { 2633 if (Shuffle->getOperand(OpIdx).isUndef()) 2634 continue; 2635 auto *It = find(Inputs, Shuffle->getOperand(OpIdx)); 2636 if (It == std::end(Inputs)) 2637 continue; 2638 int FoundOp = std::distance(std::begin(Inputs), It); 2639 // Found that operand is used already. 2640 // 1. Fix the mask for the reused operand. 2641 for (int &Idx : Mask) { 2642 if (Idx == PoisonMaskElem) 2643 continue; 2644 unsigned SrcRegIdx = Idx / NewElts; 2645 if (SrcRegIdx != I) 2646 continue; 2647 int MaskElt = Shuffle->getMaskElt(Idx % NewElts); 2648 if (MaskElt == PoisonMaskElem) { 2649 Idx = PoisonMaskElem; 2650 continue; 2651 } 2652 int MaskIdx = MaskElt / NewElts; 2653 if (OpIdx == MaskIdx) 2654 Idx = MaskElt % NewElts + FoundOp * NewElts; 2655 } 2656 // 2. Set Op to the unused OpIdx. 2657 Op = (OpIdx + 1) % 2; 2658 break; 2659 } 2660 } 2661 if (Op >= 0) { 2662 Changed = true; 2663 Inputs[I] = Shuffle->getOperand(Op); 2664 // Adjust mask. 2665 for (int &Idx : Mask) { 2666 if (Idx == PoisonMaskElem) 2667 continue; 2668 unsigned SrcRegIdx = Idx / NewElts; 2669 if (SrcRegIdx != I) 2670 continue; 2671 int MaskElt = Shuffle->getMaskElt(Idx % NewElts); 2672 int OpIdx = MaskElt / NewElts; 2673 if (OpIdx != Op) 2674 continue; 2675 Idx = MaskElt % NewElts + SrcRegIdx * NewElts; 2676 } 2677 } 2678 } 2679 } while (Changed); 2680 }; 2681 TryPeekThroughShufflesInputs(OrigMask); 2682 // Proces unique inputs. 2683 auto &&MakeUniqueInputs = [&Inputs, &IsConstant, 2684 NewElts](SmallVectorImpl<int> &Mask) { 2685 SetVector<SDValue> UniqueInputs; 2686 SetVector<SDValue> UniqueConstantInputs; 2687 for (const auto &I : Inputs) { 2688 if (IsConstant(I)) 2689 UniqueConstantInputs.insert(I); 2690 else if (!I.isUndef()) 2691 UniqueInputs.insert(I); 2692 } 2693 // Adjust mask in case of reused inputs. Also, need to insert constant 2694 // inputs at first, otherwise it affects the final outcome. 2695 if (UniqueInputs.size() != std::size(Inputs)) { 2696 auto &&UniqueVec = UniqueInputs.takeVector(); 2697 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector(); 2698 unsigned ConstNum = UniqueConstantVec.size(); 2699 for (int &Idx : Mask) { 2700 if (Idx == PoisonMaskElem) 2701 continue; 2702 unsigned SrcRegIdx = Idx / NewElts; 2703 if (Inputs[SrcRegIdx].isUndef()) { 2704 Idx = PoisonMaskElem; 2705 continue; 2706 } 2707 const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]); 2708 if (It != UniqueConstantVec.end()) { 2709 Idx = (Idx % NewElts) + 2710 NewElts * std::distance(UniqueConstantVec.begin(), It); 2711 assert(Idx >= 0 && "Expected defined mask idx."); 2712 continue; 2713 } 2714 const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]); 2715 assert(RegIt != UniqueVec.end() && "Cannot find non-const value."); 2716 Idx = (Idx % NewElts) + 2717 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum); 2718 assert(Idx >= 0 && "Expected defined mask idx."); 2719 } 2720 copy(UniqueConstantVec, std::begin(Inputs)); 2721 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum)); 2722 } 2723 }; 2724 MakeUniqueInputs(OrigMask); 2725 SDValue OrigInputs[4]; 2726 copy(Inputs, std::begin(OrigInputs)); 2727 for (unsigned High = 0; High < 2; ++High) { 2728 SDValue &Output = High ? Hi : Lo; 2729 2730 // Build a shuffle mask for the output, discovering on the fly which 2731 // input vectors to use as shuffle operands. 2732 unsigned FirstMaskIdx = High * NewElts; 2733 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem); 2734 copy(ArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin()); 2735 assert(!Output && "Expected default initialized initial value."); 2736 TryPeekThroughShufflesInputs(Mask); 2737 MakeUniqueInputs(Mask); 2738 SDValue TmpInputs[4]; 2739 copy(Inputs, std::begin(TmpInputs)); 2740 // Track changes in the output registers. 2741 int UsedIdx = -1; 2742 bool SecondIteration = false; 2743 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) { 2744 if (UsedIdx < 0) { 2745 UsedIdx = Idx; 2746 return false; 2747 } 2748 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx) 2749 SecondIteration = true; 2750 return SecondIteration; 2751 }; 2752 processShuffleMasks( 2753 Mask, std::size(Inputs), std::size(Inputs), 2754 /*NumOfUsedRegs=*/1, 2755 [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); }, 2756 [&Output, &DAG = DAG, NewVT, &DL, &Inputs, 2757 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) { 2758 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR) 2759 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask); 2760 else 2761 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx], 2762 DAG.getUNDEF(NewVT), Mask); 2763 Inputs[Idx] = Output; 2764 }, 2765 [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs, 2766 &TmpInputs, 2767 &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) { 2768 if (AccumulateResults(Idx1)) { 2769 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR && 2770 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR) 2771 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask); 2772 else 2773 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1], 2774 Inputs[Idx2], Mask); 2775 } else { 2776 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR && 2777 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR) 2778 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask); 2779 else 2780 Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1], 2781 TmpInputs[Idx2], Mask); 2782 } 2783 Inputs[Idx1] = Output; 2784 }); 2785 copy(OrigInputs, std::begin(Inputs)); 2786 } 2787 } 2788 2789 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { 2790 EVT OVT = N->getValueType(0); 2791 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext()); 2792 SDValue Chain = N->getOperand(0); 2793 SDValue Ptr = N->getOperand(1); 2794 SDValue SV = N->getOperand(2); 2795 SDLoc dl(N); 2796 2797 const Align Alignment = 2798 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext())); 2799 2800 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value()); 2801 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value()); 2802 Chain = Hi.getValue(1); 2803 2804 // Modified the chain - switch anything that used the old chain to use 2805 // the new one. 2806 ReplaceValueWith(SDValue(N, 1), Chain); 2807 } 2808 2809 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, 2810 SDValue &Hi) { 2811 EVT DstVTLo, DstVTHi; 2812 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0)); 2813 SDLoc dl(N); 2814 2815 SDValue SrcLo, SrcHi; 2816 EVT SrcVT = N->getOperand(0).getValueType(); 2817 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector) 2818 GetSplitVector(N->getOperand(0), SrcLo, SrcHi); 2819 else 2820 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0); 2821 2822 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1)); 2823 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1)); 2824 } 2825 2826 void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo, 2827 SDValue &Hi) { 2828 SDValue InLo, InHi; 2829 GetSplitVector(N->getOperand(0), InLo, InHi); 2830 SDLoc DL(N); 2831 2832 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi); 2833 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo); 2834 } 2835 2836 void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo, 2837 SDValue &Hi) { 2838 EVT VT = N->getValueType(0); 2839 SDLoc DL(N); 2840 2841 EVT LoVT, HiVT; 2842 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); 2843 2844 SDValue Expanded = TLI.expandVectorSplice(N, DAG); 2845 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded, 2846 DAG.getVectorIdxConstant(0, DL)); 2847 Hi = 2848 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded, 2849 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL)); 2850 } 2851 2852 void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) { 2853 2854 SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi; 2855 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 2856 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 2857 EVT VT = Op0Lo.getValueType(); 2858 SDLoc DL(N); 2859 SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, 2860 DAG.getVTList(VT, VT), Op0Lo, Op0Hi); 2861 SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, 2862 DAG.getVTList(VT, VT), Op1Lo, Op1Hi); 2863 2864 SetSplitVector(SDValue(N, 0), ResLo.getValue(0), ResHi.getValue(0)); 2865 SetSplitVector(SDValue(N, 1), ResLo.getValue(1), ResHi.getValue(1)); 2866 } 2867 2868 void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) { 2869 SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi; 2870 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 2871 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 2872 EVT VT = Op0Lo.getValueType(); 2873 SDLoc DL(N); 2874 SDValue Res[] = {DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, 2875 DAG.getVTList(VT, VT), Op0Lo, Op1Lo), 2876 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, 2877 DAG.getVTList(VT, VT), Op0Hi, Op1Hi)}; 2878 2879 SetSplitVector(SDValue(N, 0), Res[0].getValue(0), Res[0].getValue(1)); 2880 SetSplitVector(SDValue(N, 1), Res[1].getValue(0), Res[1].getValue(1)); 2881 } 2882 2883 //===----------------------------------------------------------------------===// 2884 // Operand Vector Splitting 2885 //===----------------------------------------------------------------------===// 2886 2887 /// This method is called when the specified operand of the specified node is 2888 /// found to need vector splitting. At this point, all of the result types of 2889 /// the node are known to be legal, but other operands of the node may need 2890 /// legalization as well as the specified one. 2891 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 2892 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n"); 2893 SDValue Res = SDValue(); 2894 2895 // See if the target wants to custom split this node. 2896 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2897 return false; 2898 2899 switch (N->getOpcode()) { 2900 default: 2901 #ifndef NDEBUG 2902 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 2903 N->dump(&DAG); 2904 dbgs() << "\n"; 2905 #endif 2906 report_fatal_error("Do not know how to split this operator's " 2907 "operand!\n"); 2908 2909 case ISD::VP_SETCC: 2910 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 2911 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 2912 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 2913 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break; 2914 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 2915 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 2916 case ISD::VP_TRUNCATE: 2917 case ISD::TRUNCATE: 2918 Res = SplitVecOp_TruncateHelper(N); 2919 break; 2920 case ISD::STRICT_FP_ROUND: 2921 case ISD::VP_FP_ROUND: 2922 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 2923 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break; 2924 case ISD::STORE: 2925 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 2926 break; 2927 case ISD::VP_STORE: 2928 Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo); 2929 break; 2930 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 2931 Res = SplitVecOp_VP_STRIDED_STORE(cast<VPStridedStoreSDNode>(N), OpNo); 2932 break; 2933 case ISD::MSTORE: 2934 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo); 2935 break; 2936 case ISD::MSCATTER: 2937 case ISD::VP_SCATTER: 2938 Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo); 2939 break; 2940 case ISD::MGATHER: 2941 case ISD::VP_GATHER: 2942 Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo); 2943 break; 2944 case ISD::VSELECT: 2945 Res = SplitVecOp_VSELECT(N, OpNo); 2946 break; 2947 case ISD::STRICT_SINT_TO_FP: 2948 case ISD::STRICT_UINT_TO_FP: 2949 case ISD::SINT_TO_FP: 2950 case ISD::UINT_TO_FP: 2951 case ISD::VP_SINT_TO_FP: 2952 case ISD::VP_UINT_TO_FP: 2953 if (N->getValueType(0).bitsLT( 2954 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType())) 2955 Res = SplitVecOp_TruncateHelper(N); 2956 else 2957 Res = SplitVecOp_UnaryOp(N); 2958 break; 2959 case ISD::FP_TO_SINT_SAT: 2960 case ISD::FP_TO_UINT_SAT: 2961 Res = SplitVecOp_FP_TO_XINT_SAT(N); 2962 break; 2963 case ISD::FP_TO_SINT: 2964 case ISD::FP_TO_UINT: 2965 case ISD::VP_FP_TO_SINT: 2966 case ISD::VP_FP_TO_UINT: 2967 case ISD::STRICT_FP_TO_SINT: 2968 case ISD::STRICT_FP_TO_UINT: 2969 case ISD::STRICT_FP_EXTEND: 2970 case ISD::FP_EXTEND: 2971 case ISD::SIGN_EXTEND: 2972 case ISD::ZERO_EXTEND: 2973 case ISD::ANY_EXTEND: 2974 case ISD::FTRUNC: 2975 Res = SplitVecOp_UnaryOp(N); 2976 break; 2977 case ISD::FLDEXP: 2978 Res = SplitVecOp_FPOpDifferentTypes(N); 2979 break; 2980 2981 case ISD::ANY_EXTEND_VECTOR_INREG: 2982 case ISD::SIGN_EXTEND_VECTOR_INREG: 2983 case ISD::ZERO_EXTEND_VECTOR_INREG: 2984 Res = SplitVecOp_ExtVecInRegOp(N); 2985 break; 2986 2987 case ISD::VECREDUCE_FADD: 2988 case ISD::VECREDUCE_FMUL: 2989 case ISD::VECREDUCE_ADD: 2990 case ISD::VECREDUCE_MUL: 2991 case ISD::VECREDUCE_AND: 2992 case ISD::VECREDUCE_OR: 2993 case ISD::VECREDUCE_XOR: 2994 case ISD::VECREDUCE_SMAX: 2995 case ISD::VECREDUCE_SMIN: 2996 case ISD::VECREDUCE_UMAX: 2997 case ISD::VECREDUCE_UMIN: 2998 case ISD::VECREDUCE_FMAX: 2999 case ISD::VECREDUCE_FMIN: 3000 case ISD::VECREDUCE_FMAXIMUM: 3001 case ISD::VECREDUCE_FMINIMUM: 3002 Res = SplitVecOp_VECREDUCE(N, OpNo); 3003 break; 3004 case ISD::VECREDUCE_SEQ_FADD: 3005 case ISD::VECREDUCE_SEQ_FMUL: 3006 Res = SplitVecOp_VECREDUCE_SEQ(N); 3007 break; 3008 case ISD::VP_REDUCE_FADD: 3009 case ISD::VP_REDUCE_SEQ_FADD: 3010 case ISD::VP_REDUCE_FMUL: 3011 case ISD::VP_REDUCE_SEQ_FMUL: 3012 case ISD::VP_REDUCE_ADD: 3013 case ISD::VP_REDUCE_MUL: 3014 case ISD::VP_REDUCE_AND: 3015 case ISD::VP_REDUCE_OR: 3016 case ISD::VP_REDUCE_XOR: 3017 case ISD::VP_REDUCE_SMAX: 3018 case ISD::VP_REDUCE_SMIN: 3019 case ISD::VP_REDUCE_UMAX: 3020 case ISD::VP_REDUCE_UMIN: 3021 case ISD::VP_REDUCE_FMAX: 3022 case ISD::VP_REDUCE_FMIN: 3023 Res = SplitVecOp_VP_REDUCE(N, OpNo); 3024 break; 3025 } 3026 3027 // If the result is null, the sub-method took care of registering results etc. 3028 if (!Res.getNode()) return false; 3029 3030 // If the result is N, the sub-method updated N in place. Tell the legalizer 3031 // core about this. 3032 if (Res.getNode() == N) 3033 return true; 3034 3035 if (N->isStrictFPOpcode()) 3036 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && 3037 "Invalid operand expansion"); 3038 else 3039 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 3040 "Invalid operand expansion"); 3041 3042 ReplaceValueWith(SDValue(N, 0), Res); 3043 return false; 3044 } 3045 3046 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 3047 // The only possibility for an illegal operand is the mask, since result type 3048 // legalization would have handled this node already otherwise. 3049 assert(OpNo == 0 && "Illegal operand must be mask"); 3050 3051 SDValue Mask = N->getOperand(0); 3052 SDValue Src0 = N->getOperand(1); 3053 SDValue Src1 = N->getOperand(2); 3054 EVT Src0VT = Src0.getValueType(); 3055 SDLoc DL(N); 3056 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?"); 3057 3058 SDValue Lo, Hi; 3059 GetSplitVector(N->getOperand(0), Lo, Hi); 3060 assert(Lo.getValueType() == Hi.getValueType() && 3061 "Lo and Hi have differing types"); 3062 3063 EVT LoOpVT, HiOpVT; 3064 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT); 3065 assert(LoOpVT == HiOpVT && "Asymmetric vector split?"); 3066 3067 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask; 3068 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL); 3069 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL); 3070 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL); 3071 3072 SDValue LoSelect = 3073 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 3074 SDValue HiSelect = 3075 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 3076 3077 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 3078 } 3079 3080 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) { 3081 EVT ResVT = N->getValueType(0); 3082 SDValue Lo, Hi; 3083 SDLoc dl(N); 3084 3085 SDValue VecOp = N->getOperand(OpNo); 3086 EVT VecVT = VecOp.getValueType(); 3087 assert(VecVT.isVector() && "Can only split reduce vector operand"); 3088 GetSplitVector(VecOp, Lo, Hi); 3089 EVT LoOpVT, HiOpVT; 3090 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT); 3091 3092 // Use the appropriate scalar instruction on the split subvectors before 3093 // reducing the now partially reduced smaller vector. 3094 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode()); 3095 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags()); 3096 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags()); 3097 } 3098 3099 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) { 3100 EVT ResVT = N->getValueType(0); 3101 SDValue Lo, Hi; 3102 SDLoc dl(N); 3103 3104 SDValue AccOp = N->getOperand(0); 3105 SDValue VecOp = N->getOperand(1); 3106 SDNodeFlags Flags = N->getFlags(); 3107 3108 EVT VecVT = VecOp.getValueType(); 3109 assert(VecVT.isVector() && "Can only split reduce vector operand"); 3110 GetSplitVector(VecOp, Lo, Hi); 3111 EVT LoOpVT, HiOpVT; 3112 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT); 3113 3114 // Reduce low half. 3115 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags); 3116 3117 // Reduce high half, using low half result as initial value. 3118 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags); 3119 } 3120 3121 SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) { 3122 assert(N->isVPOpcode() && "Expected VP opcode"); 3123 assert(OpNo == 1 && "Can only split reduce vector operand"); 3124 3125 unsigned Opc = N->getOpcode(); 3126 EVT ResVT = N->getValueType(0); 3127 SDValue Lo, Hi; 3128 SDLoc dl(N); 3129 3130 SDValue VecOp = N->getOperand(OpNo); 3131 EVT VecVT = VecOp.getValueType(); 3132 assert(VecVT.isVector() && "Can only split reduce vector operand"); 3133 GetSplitVector(VecOp, Lo, Hi); 3134 3135 SDValue MaskLo, MaskHi; 3136 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2)); 3137 3138 SDValue EVLLo, EVLHi; 3139 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl); 3140 3141 const SDNodeFlags Flags = N->getFlags(); 3142 3143 SDValue ResLo = 3144 DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags); 3145 return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags); 3146 } 3147 3148 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 3149 // The result has a legal vector type, but the input needs splitting. 3150 EVT ResVT = N->getValueType(0); 3151 SDValue Lo, Hi; 3152 SDLoc dl(N); 3153 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); 3154 EVT InVT = Lo.getValueType(); 3155 3156 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 3157 InVT.getVectorElementCount()); 3158 3159 if (N->isStrictFPOpcode()) { 3160 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 3161 { N->getOperand(0), Lo }); 3162 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 3163 { N->getOperand(0), Hi }); 3164 3165 // Build a factor node to remember that this operation is independent 3166 // of the other one. 3167 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 3168 Hi.getValue(1)); 3169 3170 // Legalize the chain result - switch anything that used the old chain to 3171 // use the new one. 3172 ReplaceValueWith(SDValue(N, 1), Ch); 3173 } else if (N->getNumOperands() == 3) { 3174 assert(N->isVPOpcode() && "Expected VP opcode"); 3175 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 3176 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 3177 std::tie(EVLLo, EVLHi) = 3178 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); 3179 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo, MaskLo, EVLLo); 3180 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi, MaskHi, EVLHi); 3181 } else { 3182 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 3183 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 3184 } 3185 3186 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 3187 } 3188 3189 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 3190 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 3191 // end up being split all the way down to individual components. Convert the 3192 // split pieces into integers and reassemble. 3193 SDValue Lo, Hi; 3194 GetSplitVector(N->getOperand(0), Lo, Hi); 3195 Lo = BitConvertToInteger(Lo); 3196 Hi = BitConvertToInteger(Hi); 3197 3198 if (DAG.getDataLayout().isBigEndian()) 3199 std::swap(Lo, Hi); 3200 3201 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), 3202 JoinIntegers(Lo, Hi)); 3203 } 3204 3205 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N, 3206 unsigned OpNo) { 3207 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec."); 3208 // We know that the result type is legal. 3209 EVT ResVT = N->getValueType(0); 3210 3211 SDValue Vec = N->getOperand(0); 3212 SDValue SubVec = N->getOperand(1); 3213 SDValue Idx = N->getOperand(2); 3214 SDLoc dl(N); 3215 3216 SDValue Lo, Hi; 3217 GetSplitVector(SubVec, Lo, Hi); 3218 3219 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 3220 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); 3221 3222 SDValue FirstInsertion = 3223 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx); 3224 SDValue SecondInsertion = 3225 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi, 3226 DAG.getVectorIdxConstant(IdxVal + LoElts, dl)); 3227 3228 return SecondInsertion; 3229 } 3230 3231 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 3232 // We know that the extracted result type is legal. 3233 EVT SubVT = N->getValueType(0); 3234 SDValue Idx = N->getOperand(1); 3235 SDLoc dl(N); 3236 SDValue Lo, Hi; 3237 3238 GetSplitVector(N->getOperand(0), Lo, Hi); 3239 3240 uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements(); 3241 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 3242 3243 if (IdxVal < LoEltsMin) { 3244 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin && 3245 "Extracted subvector crosses vector split!"); 3246 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 3247 } else if (SubVT.isScalableVector() == 3248 N->getOperand(0).getValueType().isScalableVector()) 3249 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 3250 DAG.getVectorIdxConstant(IdxVal - LoEltsMin, dl)); 3251 3252 // After this point the DAG node only permits extracting fixed-width 3253 // subvectors from scalable vectors. 3254 assert(SubVT.isFixedLengthVector() && 3255 "Extracting scalable subvector from fixed-width unsupported"); 3256 3257 // If the element type is i1 and we're not promoting the result, then we may 3258 // end up loading the wrong data since the bits are packed tightly into 3259 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal) 3260 // type at index 4, then we will load a byte starting at index 0. 3261 if (SubVT.getScalarType() == MVT::i1) 3262 report_fatal_error("Don't know how to extract fixed-width predicate " 3263 "subvector from a scalable predicate vector"); 3264 3265 // Spill the vector to the stack. We should use the alignment for 3266 // the smallest part. 3267 SDValue Vec = N->getOperand(0); 3268 EVT VecVT = Vec.getValueType(); 3269 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false); 3270 SDValue StackPtr = 3271 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign); 3272 auto &MF = DAG.getMachineFunction(); 3273 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 3274 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 3275 3276 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 3277 SmallestAlign); 3278 3279 // Extract the subvector by loading the correct part. 3280 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVT, Idx); 3281 3282 return DAG.getLoad( 3283 SubVT, dl, Store, StackPtr, 3284 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction())); 3285 } 3286 3287 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 3288 SDValue Vec = N->getOperand(0); 3289 SDValue Idx = N->getOperand(1); 3290 EVT VecVT = Vec.getValueType(); 3291 3292 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Idx)) { 3293 uint64_t IdxVal = Index->getZExtValue(); 3294 3295 SDValue Lo, Hi; 3296 GetSplitVector(Vec, Lo, Hi); 3297 3298 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); 3299 3300 if (IdxVal < LoElts) 3301 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 3302 else if (!Vec.getValueType().isScalableVector()) 3303 return SDValue(DAG.UpdateNodeOperands(N, Hi, 3304 DAG.getConstant(IdxVal - LoElts, SDLoc(N), 3305 Idx.getValueType())), 0); 3306 } 3307 3308 // See if the target wants to custom expand this node. 3309 if (CustomLowerNode(N, N->getValueType(0), true)) 3310 return SDValue(); 3311 3312 // Make the vector elements byte-addressable if they aren't already. 3313 SDLoc dl(N); 3314 EVT EltVT = VecVT.getVectorElementType(); 3315 if (VecVT.getScalarSizeInBits() < 8) { 3316 EltVT = MVT::i8; 3317 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 3318 VecVT.getVectorElementCount()); 3319 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec); 3320 } 3321 3322 // Store the vector to the stack. 3323 // In cases where the vector is illegal it will be broken down into parts 3324 // and stored in parts - we should use the alignment for the smallest part. 3325 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false); 3326 SDValue StackPtr = 3327 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign); 3328 auto &MF = DAG.getMachineFunction(); 3329 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 3330 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 3331 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 3332 SmallestAlign); 3333 3334 // Load back the required element. 3335 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 3336 3337 // FIXME: This is to handle i1 vectors with elements promoted to i8. 3338 // i1 vector handling needs general improvement. 3339 if (N->getValueType(0).bitsLT(EltVT)) { 3340 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr, 3341 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction())); 3342 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0)); 3343 } 3344 3345 return DAG.getExtLoad( 3346 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 3347 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT, 3348 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8)); 3349 } 3350 3351 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) { 3352 SDValue Lo, Hi; 3353 3354 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so 3355 // splitting the result has the same effect as splitting the input operand. 3356 SplitVecRes_ExtVecInRegOp(N, Lo, Hi); 3357 3358 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi); 3359 } 3360 3361 SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) { 3362 (void)OpNo; 3363 SDValue Lo, Hi; 3364 SplitVecRes_Gather(N, Lo, Hi); 3365 3366 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi); 3367 ReplaceValueWith(SDValue(N, 0), Res); 3368 return SDValue(); 3369 } 3370 3371 SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) { 3372 assert(N->isUnindexed() && "Indexed vp_store of vector?"); 3373 SDValue Ch = N->getChain(); 3374 SDValue Ptr = N->getBasePtr(); 3375 SDValue Offset = N->getOffset(); 3376 assert(Offset.isUndef() && "Unexpected VP store offset"); 3377 SDValue Mask = N->getMask(); 3378 SDValue EVL = N->getVectorLength(); 3379 SDValue Data = N->getValue(); 3380 Align Alignment = N->getOriginalAlign(); 3381 SDLoc DL(N); 3382 3383 SDValue DataLo, DataHi; 3384 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 3385 // Split Data operand 3386 GetSplitVector(Data, DataLo, DataHi); 3387 else 3388 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL); 3389 3390 // Split Mask operand 3391 SDValue MaskLo, MaskHi; 3392 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) { 3393 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 3394 } else { 3395 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 3396 GetSplitVector(Mask, MaskLo, MaskHi); 3397 else 3398 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 3399 } 3400 3401 EVT MemoryVT = N->getMemoryVT(); 3402 EVT LoMemVT, HiMemVT; 3403 bool HiIsEmpty = false; 3404 std::tie(LoMemVT, HiMemVT) = 3405 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty); 3406 3407 // Split EVL 3408 SDValue EVLLo, EVLHi; 3409 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL); 3410 3411 SDValue Lo, Hi; 3412 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 3413 N->getPointerInfo(), MachineMemOperand::MOStore, 3414 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges()); 3415 3416 Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO, 3417 N->getAddressingMode(), N->isTruncatingStore(), 3418 N->isCompressingStore()); 3419 3420 // If the hi vp_store has zero storage size, only the lo vp_store is needed. 3421 if (HiIsEmpty) 3422 return Lo; 3423 3424 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG, 3425 N->isCompressingStore()); 3426 3427 MachinePointerInfo MPI; 3428 if (LoMemVT.isScalableVector()) { 3429 Alignment = commonAlignment(Alignment, 3430 LoMemVT.getSizeInBits().getKnownMinValue() / 8); 3431 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); 3432 } else 3433 MPI = N->getPointerInfo().getWithOffset( 3434 LoMemVT.getStoreSize().getFixedValue()); 3435 3436 MMO = DAG.getMachineFunction().getMachineMemOperand( 3437 MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment, 3438 N->getAAInfo(), N->getRanges()); 3439 3440 Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO, 3441 N->getAddressingMode(), N->isTruncatingStore(), 3442 N->isCompressingStore()); 3443 3444 // Build a factor node to remember that this store is independent of the 3445 // other one. 3446 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 3447 } 3448 3449 SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N, 3450 unsigned OpNo) { 3451 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?"); 3452 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset"); 3453 3454 SDLoc DL(N); 3455 3456 SDValue Data = N->getValue(); 3457 SDValue LoData, HiData; 3458 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 3459 GetSplitVector(Data, LoData, HiData); 3460 else 3461 std::tie(LoData, HiData) = DAG.SplitVector(Data, DL); 3462 3463 EVT LoMemVT, HiMemVT; 3464 bool HiIsEmpty = false; 3465 std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs( 3466 N->getMemoryVT(), LoData.getValueType(), &HiIsEmpty); 3467 3468 SDValue Mask = N->getMask(); 3469 SDValue LoMask, HiMask; 3470 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) 3471 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask); 3472 else if (getTypeAction(Mask.getValueType()) == 3473 TargetLowering::TypeSplitVector) 3474 GetSplitVector(Mask, LoMask, HiMask); 3475 else 3476 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL); 3477 3478 SDValue LoEVL, HiEVL; 3479 std::tie(LoEVL, HiEVL) = 3480 DAG.SplitEVL(N->getVectorLength(), Data.getValueType(), DL); 3481 3482 // Generate the low vp_strided_store 3483 SDValue Lo = DAG.getStridedStoreVP( 3484 N->getChain(), DL, LoData, N->getBasePtr(), N->getOffset(), 3485 N->getStride(), LoMask, LoEVL, LoMemVT, N->getMemOperand(), 3486 N->getAddressingMode(), N->isTruncatingStore(), N->isCompressingStore()); 3487 3488 // If the high vp_strided_store has zero storage size, only the low 3489 // vp_strided_store is needed. 3490 if (HiIsEmpty) 3491 return Lo; 3492 3493 // Generate the high vp_strided_store. 3494 // To calculate the high base address, we need to sum to the low base 3495 // address stride number of bytes for each element already stored by low, 3496 // that is: Ptr = Ptr + (LoEVL * Stride) 3497 EVT PtrVT = N->getBasePtr().getValueType(); 3498 SDValue Increment = 3499 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL, 3500 DAG.getSExtOrTrunc(N->getStride(), DL, PtrVT)); 3501 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, N->getBasePtr(), Increment); 3502 3503 Align Alignment = N->getOriginalAlign(); 3504 if (LoMemVT.isScalableVector()) 3505 Alignment = commonAlignment(Alignment, 3506 LoMemVT.getSizeInBits().getKnownMinValue() / 8); 3507 3508 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 3509 MachinePointerInfo(N->getPointerInfo().getAddrSpace()), 3510 MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment, 3511 N->getAAInfo(), N->getRanges()); 3512 3513 SDValue Hi = DAG.getStridedStoreVP( 3514 N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask, 3515 HiEVL, HiMemVT, MMO, N->getAddressingMode(), N->isTruncatingStore(), 3516 N->isCompressingStore()); 3517 3518 // Build a factor node to remember that this store is independent of the 3519 // other one. 3520 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 3521 } 3522 3523 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N, 3524 unsigned OpNo) { 3525 assert(N->isUnindexed() && "Indexed masked store of vector?"); 3526 SDValue Ch = N->getChain(); 3527 SDValue Ptr = N->getBasePtr(); 3528 SDValue Offset = N->getOffset(); 3529 assert(Offset.isUndef() && "Unexpected indexed masked store offset"); 3530 SDValue Mask = N->getMask(); 3531 SDValue Data = N->getValue(); 3532 Align Alignment = N->getOriginalAlign(); 3533 SDLoc DL(N); 3534 3535 SDValue DataLo, DataHi; 3536 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 3537 // Split Data operand 3538 GetSplitVector(Data, DataLo, DataHi); 3539 else 3540 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL); 3541 3542 // Split Mask operand 3543 SDValue MaskLo, MaskHi; 3544 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) { 3545 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 3546 } else { 3547 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 3548 GetSplitVector(Mask, MaskLo, MaskHi); 3549 else 3550 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 3551 } 3552 3553 EVT MemoryVT = N->getMemoryVT(); 3554 EVT LoMemVT, HiMemVT; 3555 bool HiIsEmpty = false; 3556 std::tie(LoMemVT, HiMemVT) = 3557 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty); 3558 3559 SDValue Lo, Hi, Res; 3560 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 3561 N->getPointerInfo(), MachineMemOperand::MOStore, 3562 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges()); 3563 3564 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO, 3565 N->getAddressingMode(), N->isTruncatingStore(), 3566 N->isCompressingStore()); 3567 3568 if (HiIsEmpty) { 3569 // The hi masked store has zero storage size. 3570 // Only the lo masked store is needed. 3571 Res = Lo; 3572 } else { 3573 3574 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG, 3575 N->isCompressingStore()); 3576 3577 MachinePointerInfo MPI; 3578 if (LoMemVT.isScalableVector()) { 3579 Alignment = commonAlignment( 3580 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8); 3581 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); 3582 } else 3583 MPI = N->getPointerInfo().getWithOffset( 3584 LoMemVT.getStoreSize().getFixedValue()); 3585 3586 MMO = DAG.getMachineFunction().getMachineMemOperand( 3587 MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment, 3588 N->getAAInfo(), N->getRanges()); 3589 3590 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO, 3591 N->getAddressingMode(), N->isTruncatingStore(), 3592 N->isCompressingStore()); 3593 3594 // Build a factor node to remember that this store is independent of the 3595 // other one. 3596 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 3597 } 3598 3599 return Res; 3600 } 3601 3602 SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) { 3603 SDValue Ch = N->getChain(); 3604 SDValue Ptr = N->getBasePtr(); 3605 EVT MemoryVT = N->getMemoryVT(); 3606 Align Alignment = N->getOriginalAlign(); 3607 SDLoc DL(N); 3608 struct Operands { 3609 SDValue Mask; 3610 SDValue Index; 3611 SDValue Scale; 3612 SDValue Data; 3613 } Ops = [&]() -> Operands { 3614 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) { 3615 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(), 3616 MSC->getValue()}; 3617 } 3618 auto *VPSC = cast<VPScatterSDNode>(N); 3619 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(), 3620 VPSC->getValue()}; 3621 }(); 3622 // Split all operands 3623 3624 EVT LoMemVT, HiMemVT; 3625 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 3626 3627 SDValue DataLo, DataHi; 3628 if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector) 3629 // Split Data operand 3630 GetSplitVector(Ops.Data, DataLo, DataHi); 3631 else 3632 std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL); 3633 3634 // Split Mask operand 3635 SDValue MaskLo, MaskHi; 3636 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) { 3637 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi); 3638 } else { 3639 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL); 3640 } 3641 3642 SDValue IndexHi, IndexLo; 3643 if (getTypeAction(Ops.Index.getValueType()) == 3644 TargetLowering::TypeSplitVector) 3645 GetSplitVector(Ops.Index, IndexLo, IndexHi); 3646 else 3647 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL); 3648 3649 SDValue Lo; 3650 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 3651 N->getPointerInfo(), MachineMemOperand::MOStore, 3652 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges()); 3653 3654 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) { 3655 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale}; 3656 Lo = 3657 DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO, 3658 MSC->getIndexType(), MSC->isTruncatingStore()); 3659 3660 // The order of the Scatter operation after split is well defined. The "Hi" 3661 // part comes after the "Lo". So these two operations should be chained one 3662 // after another. 3663 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale}; 3664 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, 3665 MMO, MSC->getIndexType(), 3666 MSC->isTruncatingStore()); 3667 } 3668 auto *VPSC = cast<VPScatterSDNode>(N); 3669 SDValue EVLLo, EVLHi; 3670 std::tie(EVLLo, EVLHi) = 3671 DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL); 3672 3673 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo}; 3674 Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO, 3675 VPSC->getIndexType()); 3676 3677 // The order of the Scatter operation after split is well defined. The "Hi" 3678 // part comes after the "Lo". So these two operations should be chained one 3679 // after another. 3680 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi}; 3681 return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO, 3682 VPSC->getIndexType()); 3683 } 3684 3685 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 3686 assert(N->isUnindexed() && "Indexed store of vector?"); 3687 assert(OpNo == 1 && "Can only split the stored value"); 3688 SDLoc DL(N); 3689 3690 bool isTruncating = N->isTruncatingStore(); 3691 SDValue Ch = N->getChain(); 3692 SDValue Ptr = N->getBasePtr(); 3693 EVT MemoryVT = N->getMemoryVT(); 3694 Align Alignment = N->getOriginalAlign(); 3695 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 3696 AAMDNodes AAInfo = N->getAAInfo(); 3697 SDValue Lo, Hi; 3698 GetSplitVector(N->getOperand(1), Lo, Hi); 3699 3700 EVT LoMemVT, HiMemVT; 3701 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 3702 3703 // Scalarize if the split halves are not byte-sized. 3704 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) 3705 return TLI.scalarizeVectorStore(N, DAG); 3706 3707 if (isTruncating) 3708 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT, 3709 Alignment, MMOFlags, AAInfo); 3710 else 3711 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags, 3712 AAInfo); 3713 3714 MachinePointerInfo MPI; 3715 IncrementPointer(N, LoMemVT, MPI, Ptr); 3716 3717 if (isTruncating) 3718 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI, 3719 HiMemVT, Alignment, MMOFlags, AAInfo); 3720 else 3721 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo); 3722 3723 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 3724 } 3725 3726 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 3727 SDLoc DL(N); 3728 3729 // The input operands all must have the same type, and we know the result 3730 // type is valid. Convert this to a buildvector which extracts all the 3731 // input elements. 3732 // TODO: If the input elements are power-two vectors, we could convert this to 3733 // a new CONCAT_VECTORS node with elements that are half-wide. 3734 SmallVector<SDValue, 32> Elts; 3735 EVT EltVT = N->getValueType(0).getVectorElementType(); 3736 for (const SDValue &Op : N->op_values()) { 3737 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 3738 i != e; ++i) { 3739 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op, 3740 DAG.getVectorIdxConstant(i, DL))); 3741 } 3742 } 3743 3744 return DAG.getBuildVector(N->getValueType(0), DL, Elts); 3745 } 3746 3747 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) { 3748 // The result type is legal, but the input type is illegal. If splitting 3749 // ends up with the result type of each half still being legal, just 3750 // do that. If, however, that would result in an illegal result type, 3751 // we can try to get more clever with power-two vectors. Specifically, 3752 // split the input type, but also widen the result element size, then 3753 // concatenate the halves and truncate again. For example, consider a target 3754 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit 3755 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: 3756 // %inlo = v4i32 extract_subvector %in, 0 3757 // %inhi = v4i32 extract_subvector %in, 4 3758 // %lo16 = v4i16 trunc v4i32 %inlo 3759 // %hi16 = v4i16 trunc v4i32 %inhi 3760 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 3761 // %res = v8i8 trunc v8i16 %in16 3762 // 3763 // Without this transform, the original truncate would end up being 3764 // scalarized, which is pretty much always a last resort. 3765 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 3766 SDValue InVec = N->getOperand(OpNo); 3767 EVT InVT = InVec->getValueType(0); 3768 EVT OutVT = N->getValueType(0); 3769 ElementCount NumElements = OutVT.getVectorElementCount(); 3770 bool IsFloat = OutVT.isFloatingPoint(); 3771 3772 unsigned InElementSize = InVT.getScalarSizeInBits(); 3773 unsigned OutElementSize = OutVT.getScalarSizeInBits(); 3774 3775 // Determine the split output VT. If its legal we can just split dirctly. 3776 EVT LoOutVT, HiOutVT; 3777 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT); 3778 assert(LoOutVT == HiOutVT && "Unequal split?"); 3779 3780 // If the input elements are only 1/2 the width of the result elements, 3781 // just use the normal splitting. Our trick only work if there's room 3782 // to split more than once. 3783 if (isTypeLegal(LoOutVT) || 3784 InElementSize <= OutElementSize * 2) 3785 return SplitVecOp_UnaryOp(N); 3786 SDLoc DL(N); 3787 3788 // Don't touch if this will be scalarized. 3789 EVT FinalVT = InVT; 3790 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector) 3791 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext()); 3792 3793 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector) 3794 return SplitVecOp_UnaryOp(N); 3795 3796 // Get the split input vector. 3797 SDValue InLoVec, InHiVec; 3798 GetSplitVector(InVec, InLoVec, InHiVec); 3799 3800 // Truncate them to 1/2 the element size. 3801 // 3802 // This assumes the number of elements is a power of two; any vector that 3803 // isn't should be widened, not split. 3804 EVT HalfElementVT = IsFloat ? 3805 EVT::getFloatingPointVT(InElementSize/2) : 3806 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); 3807 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, 3808 NumElements.divideCoefficientBy(2)); 3809 3810 SDValue HalfLo; 3811 SDValue HalfHi; 3812 SDValue Chain; 3813 if (N->isStrictFPOpcode()) { 3814 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other}, 3815 {N->getOperand(0), InLoVec}); 3816 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other}, 3817 {N->getOperand(0), InHiVec}); 3818 // Legalize the chain result - switch anything that used the old chain to 3819 // use the new one. 3820 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1), 3821 HalfHi.getValue(1)); 3822 } else { 3823 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec); 3824 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec); 3825 } 3826 3827 // Concatenate them to get the full intermediate truncation result. 3828 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); 3829 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, 3830 HalfHi); 3831 // Now finish up by truncating all the way down to the original result 3832 // type. This should normally be something that ends up being legal directly, 3833 // but in theory if a target has very wide vectors and an annoyingly 3834 // restricted set of legal types, this split can chain to build things up. 3835 3836 if (N->isStrictFPOpcode()) { 3837 SDValue Res = DAG.getNode( 3838 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other}, 3839 {Chain, InterVec, 3840 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))}); 3841 // Relink the chain 3842 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 3843 return Res; 3844 } 3845 3846 return IsFloat 3847 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec, 3848 DAG.getTargetConstant( 3849 0, DL, TLI.getPointerTy(DAG.getDataLayout()))) 3850 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); 3851 } 3852 3853 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 3854 assert(N->getValueType(0).isVector() && 3855 N->getOperand(0).getValueType().isVector() && 3856 "Operand types must be vectors"); 3857 // The result has a legal vector type, but the input needs splitting. 3858 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 3859 SDLoc DL(N); 3860 GetSplitVector(N->getOperand(0), Lo0, Hi0); 3861 GetSplitVector(N->getOperand(1), Lo1, Hi1); 3862 auto PartEltCnt = Lo0.getValueType().getVectorElementCount(); 3863 3864 LLVMContext &Context = *DAG.getContext(); 3865 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt); 3866 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2); 3867 3868 if (N->getOpcode() == ISD::SETCC) { 3869 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 3870 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 3871 } else { 3872 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode"); 3873 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 3874 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3)); 3875 std::tie(EVLLo, EVLHi) = 3876 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL); 3877 LoRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Lo0, Lo1, 3878 N->getOperand(2), MaskLo, EVLLo); 3879 HiRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Hi0, Hi1, 3880 N->getOperand(2), MaskHi, EVLHi); 3881 } 3882 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 3883 3884 EVT OpVT = N->getOperand(0).getValueType(); 3885 ISD::NodeType ExtendCode = 3886 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 3887 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con); 3888 } 3889 3890 3891 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 3892 // The result has a legal vector type, but the input needs splitting. 3893 EVT ResVT = N->getValueType(0); 3894 SDValue Lo, Hi; 3895 SDLoc DL(N); 3896 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); 3897 EVT InVT = Lo.getValueType(); 3898 3899 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 3900 InVT.getVectorElementCount()); 3901 3902 if (N->isStrictFPOpcode()) { 3903 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, 3904 { N->getOperand(0), Lo, N->getOperand(2) }); 3905 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, 3906 { N->getOperand(0), Hi, N->getOperand(2) }); 3907 // Legalize the chain result - switch anything that used the old chain to 3908 // use the new one. 3909 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 3910 Lo.getValue(1), Hi.getValue(1)); 3911 ReplaceValueWith(SDValue(N, 1), NewChain); 3912 } else if (N->getOpcode() == ISD::VP_FP_ROUND) { 3913 SDValue MaskLo, MaskHi, EVLLo, EVLHi; 3914 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); 3915 std::tie(EVLLo, EVLHi) = 3916 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), DL); 3917 Lo = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Lo, MaskLo, EVLLo); 3918 Hi = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Hi, MaskHi, EVLHi); 3919 } else { 3920 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 3921 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 3922 } 3923 3924 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 3925 } 3926 3927 // Split a vector type in an FP binary operation where the second operand has a 3928 // different type from the first. 3929 // 3930 // The result (and the first input) has a legal vector type, but the second 3931 // input needs splitting. 3932 SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) { 3933 SDLoc DL(N); 3934 3935 EVT LHSLoVT, LHSHiVT; 3936 std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 3937 3938 if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT)) 3939 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements()); 3940 3941 SDValue LHSLo, LHSHi; 3942 std::tie(LHSLo, LHSHi) = 3943 DAG.SplitVector(N->getOperand(0), DL, LHSLoVT, LHSHiVT); 3944 3945 SDValue RHSLo, RHSHi; 3946 std::tie(RHSLo, RHSHi) = DAG.SplitVector(N->getOperand(1), DL); 3947 3948 SDValue Lo = DAG.getNode(N->getOpcode(), DL, LHSLoVT, LHSLo, RHSLo); 3949 SDValue Hi = DAG.getNode(N->getOpcode(), DL, LHSHiVT, LHSHi, RHSHi); 3950 3951 return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), Lo, Hi); 3952 } 3953 3954 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) { 3955 EVT ResVT = N->getValueType(0); 3956 SDValue Lo, Hi; 3957 SDLoc dl(N); 3958 GetSplitVector(N->getOperand(0), Lo, Hi); 3959 EVT InVT = Lo.getValueType(); 3960 3961 EVT NewResVT = 3962 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 3963 InVT.getVectorElementCount()); 3964 3965 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1)); 3966 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1)); 3967 3968 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 3969 } 3970 3971 //===----------------------------------------------------------------------===// 3972 // Result Vector Widening 3973 //===----------------------------------------------------------------------===// 3974 3975 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 3976 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG); 3977 dbgs() << "\n"); 3978 3979 // See if the target wants to custom widen this node. 3980 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 3981 return; 3982 3983 SDValue Res = SDValue(); 3984 3985 auto unrollExpandedOp = [&]() { 3986 // We're going to widen this vector op to a legal type by padding with undef 3987 // elements. If the wide vector op is eventually going to be expanded to 3988 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary 3989 // libcalls on the undef elements. 3990 EVT VT = N->getValueType(0); 3991 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3992 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) && 3993 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) { 3994 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements()); 3995 return true; 3996 } 3997 return false; 3998 }; 3999 4000 switch (N->getOpcode()) { 4001 default: 4002 #ifndef NDEBUG 4003 dbgs() << "WidenVectorResult #" << ResNo << ": "; 4004 N->dump(&DAG); 4005 dbgs() << "\n"; 4006 #endif 4007 report_fatal_error("Do not know how to widen the result of this operator!"); 4008 4009 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 4010 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break; 4011 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 4012 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 4013 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 4014 case ISD::INSERT_SUBVECTOR: 4015 Res = WidenVecRes_INSERT_SUBVECTOR(N); 4016 break; 4017 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 4018 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 4019 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 4020 case ISD::STEP_VECTOR: 4021 case ISD::SPLAT_VECTOR: 4022 case ISD::SCALAR_TO_VECTOR: 4023 Res = WidenVecRes_ScalarOp(N); 4024 break; 4025 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 4026 case ISD::VSELECT: 4027 case ISD::SELECT: 4028 case ISD::VP_SELECT: 4029 case ISD::VP_MERGE: 4030 Res = WidenVecRes_Select(N); 4031 break; 4032 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 4033 case ISD::VP_SETCC: 4034 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 4035 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 4036 case ISD::VECTOR_SHUFFLE: 4037 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 4038 break; 4039 case ISD::VP_LOAD: 4040 Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(N)); 4041 break; 4042 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 4043 Res = WidenVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N)); 4044 break; 4045 case ISD::MLOAD: 4046 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N)); 4047 break; 4048 case ISD::MGATHER: 4049 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N)); 4050 break; 4051 case ISD::VP_GATHER: 4052 Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(N)); 4053 break; 4054 case ISD::VECTOR_REVERSE: 4055 Res = WidenVecRes_VECTOR_REVERSE(N); 4056 break; 4057 4058 case ISD::ADD: case ISD::VP_ADD: 4059 case ISD::AND: case ISD::VP_AND: 4060 case ISD::MUL: case ISD::VP_MUL: 4061 case ISD::MULHS: 4062 case ISD::MULHU: 4063 case ISD::OR: case ISD::VP_OR: 4064 case ISD::SUB: case ISD::VP_SUB: 4065 case ISD::XOR: case ISD::VP_XOR: 4066 case ISD::SHL: case ISD::VP_SHL: 4067 case ISD::SRA: case ISD::VP_ASHR: 4068 case ISD::SRL: case ISD::VP_LSHR: 4069 case ISD::FMINNUM: case ISD::VP_FMINNUM: 4070 case ISD::FMAXNUM: case ISD::VP_FMAXNUM: 4071 case ISD::FMINIMUM: 4072 case ISD::FMAXIMUM: 4073 case ISD::SMIN: case ISD::VP_SMIN: 4074 case ISD::SMAX: case ISD::VP_SMAX: 4075 case ISD::UMIN: case ISD::VP_UMIN: 4076 case ISD::UMAX: case ISD::VP_UMAX: 4077 case ISD::UADDSAT: 4078 case ISD::SADDSAT: 4079 case ISD::USUBSAT: 4080 case ISD::SSUBSAT: 4081 case ISD::SSHLSAT: 4082 case ISD::USHLSAT: 4083 case ISD::ROTL: 4084 case ISD::ROTR: 4085 case ISD::AVGFLOORS: 4086 case ISD::AVGFLOORU: 4087 case ISD::AVGCEILS: 4088 case ISD::AVGCEILU: 4089 // Vector-predicated binary op widening. Note that -- unlike the 4090 // unpredicated versions -- we don't have to worry about trapping on 4091 // operations like UDIV, FADD, etc., as we pass on the original vector 4092 // length parameter. This means the widened elements containing garbage 4093 // aren't active. 4094 case ISD::VP_SDIV: 4095 case ISD::VP_UDIV: 4096 case ISD::VP_SREM: 4097 case ISD::VP_UREM: 4098 case ISD::VP_FADD: 4099 case ISD::VP_FSUB: 4100 case ISD::VP_FMUL: 4101 case ISD::VP_FDIV: 4102 case ISD::VP_FREM: 4103 case ISD::VP_FCOPYSIGN: 4104 Res = WidenVecRes_Binary(N); 4105 break; 4106 4107 case ISD::FPOW: 4108 case ISD::FREM: 4109 if (unrollExpandedOp()) 4110 break; 4111 // If the target has custom/legal support for the scalar FP intrinsic ops 4112 // (they are probably not destined to become libcalls), then widen those 4113 // like any other binary ops. 4114 [[fallthrough]]; 4115 4116 case ISD::FADD: 4117 case ISD::FMUL: 4118 case ISD::FSUB: 4119 case ISD::FDIV: 4120 case ISD::SDIV: 4121 case ISD::UDIV: 4122 case ISD::SREM: 4123 case ISD::UREM: 4124 Res = WidenVecRes_BinaryCanTrap(N); 4125 break; 4126 4127 case ISD::SMULFIX: 4128 case ISD::SMULFIXSAT: 4129 case ISD::UMULFIX: 4130 case ISD::UMULFIXSAT: 4131 // These are binary operations, but with an extra operand that shouldn't 4132 // be widened (the scale). 4133 Res = WidenVecRes_BinaryWithExtraScalarOp(N); 4134 break; 4135 4136 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 4137 case ISD::STRICT_##DAGN: 4138 #include "llvm/IR/ConstrainedOps.def" 4139 Res = WidenVecRes_StrictFP(N); 4140 break; 4141 4142 case ISD::UADDO: 4143 case ISD::SADDO: 4144 case ISD::USUBO: 4145 case ISD::SSUBO: 4146 case ISD::UMULO: 4147 case ISD::SMULO: 4148 Res = WidenVecRes_OverflowOp(N, ResNo); 4149 break; 4150 4151 case ISD::FCOPYSIGN: 4152 Res = WidenVecRes_FCOPYSIGN(N); 4153 break; 4154 4155 case ISD::IS_FPCLASS: 4156 Res = WidenVecRes_IS_FPCLASS(N); 4157 break; 4158 4159 case ISD::FLDEXP: 4160 case ISD::FPOWI: 4161 if (!unrollExpandedOp()) 4162 Res = WidenVecRes_ExpOp(N); 4163 break; 4164 4165 case ISD::ANY_EXTEND_VECTOR_INREG: 4166 case ISD::SIGN_EXTEND_VECTOR_INREG: 4167 case ISD::ZERO_EXTEND_VECTOR_INREG: 4168 Res = WidenVecRes_EXTEND_VECTOR_INREG(N); 4169 break; 4170 4171 case ISD::ANY_EXTEND: 4172 case ISD::FP_EXTEND: 4173 case ISD::VP_FP_EXTEND: 4174 case ISD::FP_ROUND: 4175 case ISD::VP_FP_ROUND: 4176 case ISD::FP_TO_SINT: 4177 case ISD::VP_FP_TO_SINT: 4178 case ISD::FP_TO_UINT: 4179 case ISD::VP_FP_TO_UINT: 4180 case ISD::SIGN_EXTEND: 4181 case ISD::VP_SIGN_EXTEND: 4182 case ISD::SINT_TO_FP: 4183 case ISD::VP_SINT_TO_FP: 4184 case ISD::VP_TRUNCATE: 4185 case ISD::TRUNCATE: 4186 case ISD::UINT_TO_FP: 4187 case ISD::VP_UINT_TO_FP: 4188 case ISD::ZERO_EXTEND: 4189 case ISD::VP_ZERO_EXTEND: 4190 Res = WidenVecRes_Convert(N); 4191 break; 4192 4193 case ISD::FP_TO_SINT_SAT: 4194 case ISD::FP_TO_UINT_SAT: 4195 Res = WidenVecRes_FP_TO_XINT_SAT(N); 4196 break; 4197 4198 case ISD::FABS: 4199 case ISD::FCEIL: 4200 case ISD::FCOS: 4201 case ISD::FEXP: 4202 case ISD::FEXP2: 4203 case ISD::FFLOOR: 4204 case ISD::FLOG: 4205 case ISD::FLOG10: 4206 case ISD::FLOG2: 4207 case ISD::FNEARBYINT: 4208 case ISD::FRINT: 4209 case ISD::FROUND: 4210 case ISD::FROUNDEVEN: 4211 case ISD::FSIN: 4212 case ISD::FSQRT: 4213 case ISD::FTRUNC: 4214 if (unrollExpandedOp()) 4215 break; 4216 // If the target has custom/legal support for the scalar FP intrinsic ops 4217 // (they are probably not destined to become libcalls), then widen those 4218 // like any other unary ops. 4219 [[fallthrough]]; 4220 4221 case ISD::ABS: 4222 case ISD::VP_ABS: 4223 case ISD::BITREVERSE: 4224 case ISD::VP_BITREVERSE: 4225 case ISD::BSWAP: 4226 case ISD::VP_BSWAP: 4227 case ISD::CTLZ: 4228 case ISD::VP_CTLZ: 4229 case ISD::CTLZ_ZERO_UNDEF: 4230 case ISD::VP_CTLZ_ZERO_UNDEF: 4231 case ISD::CTPOP: 4232 case ISD::VP_CTPOP: 4233 case ISD::CTTZ: 4234 case ISD::VP_CTTZ: 4235 case ISD::CTTZ_ZERO_UNDEF: 4236 case ISD::VP_CTTZ_ZERO_UNDEF: 4237 case ISD::FNEG: case ISD::VP_FNEG: 4238 case ISD::VP_FABS: 4239 case ISD::VP_SQRT: 4240 case ISD::VP_FCEIL: 4241 case ISD::VP_FFLOOR: 4242 case ISD::VP_FRINT: 4243 case ISD::VP_FNEARBYINT: 4244 case ISD::VP_FROUND: 4245 case ISD::VP_FROUNDEVEN: 4246 case ISD::VP_FROUNDTOZERO: 4247 case ISD::FREEZE: 4248 case ISD::ARITH_FENCE: 4249 case ISD::FCANONICALIZE: 4250 Res = WidenVecRes_Unary(N); 4251 break; 4252 case ISD::FMA: case ISD::VP_FMA: 4253 case ISD::FSHL: 4254 case ISD::VP_FSHL: 4255 case ISD::FSHR: 4256 case ISD::VP_FSHR: 4257 Res = WidenVecRes_Ternary(N); 4258 break; 4259 } 4260 4261 // If Res is null, the sub-method took care of registering the result. 4262 if (Res.getNode()) 4263 SetWidenedVector(SDValue(N, ResNo), Res); 4264 } 4265 4266 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 4267 // Ternary op widening. 4268 SDLoc dl(N); 4269 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4270 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 4271 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 4272 SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 4273 if (N->getNumOperands() == 3) 4274 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 4275 4276 assert(N->getNumOperands() == 5 && "Unexpected number of operands!"); 4277 assert(N->isVPOpcode() && "Expected VP opcode"); 4278 4279 SDValue Mask = 4280 GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount()); 4281 return DAG.getNode(N->getOpcode(), dl, WidenVT, 4282 {InOp1, InOp2, InOp3, Mask, N->getOperand(4)}); 4283 } 4284 4285 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 4286 // Binary op widening. 4287 SDLoc dl(N); 4288 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4289 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 4290 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 4291 if (N->getNumOperands() == 2) 4292 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, 4293 N->getFlags()); 4294 4295 assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); 4296 assert(N->isVPOpcode() && "Expected VP opcode"); 4297 4298 SDValue Mask = 4299 GetWidenedMask(N->getOperand(2), WidenVT.getVectorElementCount()); 4300 return DAG.getNode(N->getOpcode(), dl, WidenVT, 4301 {InOp1, InOp2, Mask, N->getOperand(3)}, N->getFlags()); 4302 } 4303 4304 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) { 4305 // Binary op widening, but with an extra operand that shouldn't be widened. 4306 SDLoc dl(N); 4307 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4308 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 4309 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 4310 SDValue InOp3 = N->getOperand(2); 4311 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3, 4312 N->getFlags()); 4313 } 4314 4315 // Given a vector of operations that have been broken up to widen, see 4316 // if we can collect them together into the next widest legal VT. This 4317 // implementation is trap-safe. 4318 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, 4319 SmallVectorImpl<SDValue> &ConcatOps, 4320 unsigned ConcatEnd, EVT VT, EVT MaxVT, 4321 EVT WidenVT) { 4322 // Check to see if we have a single operation with the widen type. 4323 if (ConcatEnd == 1) { 4324 VT = ConcatOps[0].getValueType(); 4325 if (VT == WidenVT) 4326 return ConcatOps[0]; 4327 } 4328 4329 SDLoc dl(ConcatOps[0]); 4330 EVT WidenEltVT = WidenVT.getVectorElementType(); 4331 4332 // while (Some element of ConcatOps is not of type MaxVT) { 4333 // From the end of ConcatOps, collect elements of the same type and put 4334 // them into an op of the next larger supported type 4335 // } 4336 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 4337 int Idx = ConcatEnd - 1; 4338 VT = ConcatOps[Idx--].getValueType(); 4339 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 4340 Idx--; 4341 4342 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 4343 EVT NextVT; 4344 do { 4345 NextSize *= 2; 4346 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 4347 } while (!TLI.isTypeLegal(NextVT)); 4348 4349 if (!VT.isVector()) { 4350 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 4351 SDValue VecOp = DAG.getUNDEF(NextVT); 4352 unsigned NumToInsert = ConcatEnd - Idx - 1; 4353 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 4354 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, 4355 ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl)); 4356 } 4357 ConcatOps[Idx+1] = VecOp; 4358 ConcatEnd = Idx + 2; 4359 } else { 4360 // Vector type, create a CONCAT_VECTORS of type NextVT 4361 SDValue undefVec = DAG.getUNDEF(VT); 4362 unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 4363 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 4364 unsigned RealVals = ConcatEnd - Idx - 1; 4365 unsigned SubConcatEnd = 0; 4366 unsigned SubConcatIdx = Idx + 1; 4367 while (SubConcatEnd < RealVals) 4368 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 4369 while (SubConcatEnd < OpsToConcat) 4370 SubConcatOps[SubConcatEnd++] = undefVec; 4371 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 4372 NextVT, SubConcatOps); 4373 ConcatEnd = SubConcatIdx + 1; 4374 } 4375 } 4376 4377 // Check to see if we have a single operation with the widen type. 4378 if (ConcatEnd == 1) { 4379 VT = ConcatOps[0].getValueType(); 4380 if (VT == WidenVT) 4381 return ConcatOps[0]; 4382 } 4383 4384 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 4385 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 4386 if (NumOps != ConcatEnd ) { 4387 SDValue UndefVal = DAG.getUNDEF(MaxVT); 4388 for (unsigned j = ConcatEnd; j < NumOps; ++j) 4389 ConcatOps[j] = UndefVal; 4390 } 4391 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 4392 ArrayRef(ConcatOps.data(), NumOps)); 4393 } 4394 4395 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) { 4396 // Binary op widening for operations that can trap. 4397 unsigned Opcode = N->getOpcode(); 4398 SDLoc dl(N); 4399 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4400 EVT WidenEltVT = WidenVT.getVectorElementType(); 4401 EVT VT = WidenVT; 4402 unsigned NumElts = VT.getVectorMinNumElements(); 4403 const SDNodeFlags Flags = N->getFlags(); 4404 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 4405 NumElts = NumElts / 2; 4406 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 4407 } 4408 4409 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 4410 // Operation doesn't trap so just widen as normal. 4411 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 4412 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 4413 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags); 4414 } 4415 4416 // FIXME: Improve support for scalable vectors. 4417 assert(!VT.isScalableVector() && "Scalable vectors not handled yet."); 4418 4419 // No legal vector version so unroll the vector operation and then widen. 4420 if (NumElts == 1) 4421 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 4422 4423 // Since the operation can trap, apply operation on the original vector. 4424 EVT MaxVT = VT; 4425 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 4426 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 4427 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 4428 4429 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 4430 unsigned ConcatEnd = 0; // Current ConcatOps index. 4431 int Idx = 0; // Current Idx into input vectors. 4432 4433 // NumElts := greatest legal vector size (at most WidenVT) 4434 // while (orig. vector has unhandled elements) { 4435 // take munches of size NumElts from the beginning and add to ConcatOps 4436 // NumElts := next smaller supported vector size or 1 4437 // } 4438 while (CurNumElts != 0) { 4439 while (CurNumElts >= NumElts) { 4440 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 4441 DAG.getVectorIdxConstant(Idx, dl)); 4442 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 4443 DAG.getVectorIdxConstant(Idx, dl)); 4444 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags); 4445 Idx += NumElts; 4446 CurNumElts -= NumElts; 4447 } 4448 do { 4449 NumElts = NumElts / 2; 4450 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 4451 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 4452 4453 if (NumElts == 1) { 4454 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 4455 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 4456 InOp1, DAG.getVectorIdxConstant(Idx, dl)); 4457 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 4458 InOp2, DAG.getVectorIdxConstant(Idx, dl)); 4459 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 4460 EOp1, EOp2, Flags); 4461 } 4462 CurNumElts = 0; 4463 } 4464 } 4465 4466 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); 4467 } 4468 4469 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) { 4470 switch (N->getOpcode()) { 4471 case ISD::STRICT_FSETCC: 4472 case ISD::STRICT_FSETCCS: 4473 return WidenVecRes_STRICT_FSETCC(N); 4474 case ISD::STRICT_FP_EXTEND: 4475 case ISD::STRICT_FP_ROUND: 4476 case ISD::STRICT_FP_TO_SINT: 4477 case ISD::STRICT_FP_TO_UINT: 4478 case ISD::STRICT_SINT_TO_FP: 4479 case ISD::STRICT_UINT_TO_FP: 4480 return WidenVecRes_Convert_StrictFP(N); 4481 default: 4482 break; 4483 } 4484 4485 // StrictFP op widening for operations that can trap. 4486 unsigned NumOpers = N->getNumOperands(); 4487 unsigned Opcode = N->getOpcode(); 4488 SDLoc dl(N); 4489 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4490 EVT WidenEltVT = WidenVT.getVectorElementType(); 4491 EVT VT = WidenVT; 4492 unsigned NumElts = VT.getVectorNumElements(); 4493 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 4494 NumElts = NumElts / 2; 4495 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 4496 } 4497 4498 // No legal vector version so unroll the vector operation and then widen. 4499 if (NumElts == 1) 4500 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements()); 4501 4502 // Since the operation can trap, apply operation on the original vector. 4503 EVT MaxVT = VT; 4504 SmallVector<SDValue, 4> InOps; 4505 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 4506 4507 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 4508 SmallVector<SDValue, 16> Chains; 4509 unsigned ConcatEnd = 0; // Current ConcatOps index. 4510 int Idx = 0; // Current Idx into input vectors. 4511 4512 // The Chain is the first operand. 4513 InOps.push_back(N->getOperand(0)); 4514 4515 // Now process the remaining operands. 4516 for (unsigned i = 1; i < NumOpers; ++i) { 4517 SDValue Oper = N->getOperand(i); 4518 4519 EVT OpVT = Oper.getValueType(); 4520 if (OpVT.isVector()) { 4521 if (getTypeAction(OpVT) == TargetLowering::TypeWidenVector) 4522 Oper = GetWidenedVector(Oper); 4523 else { 4524 EVT WideOpVT = 4525 EVT::getVectorVT(*DAG.getContext(), OpVT.getVectorElementType(), 4526 WidenVT.getVectorElementCount()); 4527 Oper = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, 4528 DAG.getUNDEF(WideOpVT), Oper, 4529 DAG.getVectorIdxConstant(0, dl)); 4530 } 4531 } 4532 4533 InOps.push_back(Oper); 4534 } 4535 4536 // NumElts := greatest legal vector size (at most WidenVT) 4537 // while (orig. vector has unhandled elements) { 4538 // take munches of size NumElts from the beginning and add to ConcatOps 4539 // NumElts := next smaller supported vector size or 1 4540 // } 4541 while (CurNumElts != 0) { 4542 while (CurNumElts >= NumElts) { 4543 SmallVector<SDValue, 4> EOps; 4544 4545 for (unsigned i = 0; i < NumOpers; ++i) { 4546 SDValue Op = InOps[i]; 4547 4548 EVT OpVT = Op.getValueType(); 4549 if (OpVT.isVector()) { 4550 EVT OpExtractVT = 4551 EVT::getVectorVT(*DAG.getContext(), OpVT.getVectorElementType(), 4552 VT.getVectorElementCount()); 4553 Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpExtractVT, Op, 4554 DAG.getVectorIdxConstant(Idx, dl)); 4555 } 4556 4557 EOps.push_back(Op); 4558 } 4559 4560 EVT OperVT[] = {VT, MVT::Other}; 4561 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps); 4562 ConcatOps[ConcatEnd++] = Oper; 4563 Chains.push_back(Oper.getValue(1)); 4564 Idx += NumElts; 4565 CurNumElts -= NumElts; 4566 } 4567 do { 4568 NumElts = NumElts / 2; 4569 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 4570 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 4571 4572 if (NumElts == 1) { 4573 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 4574 SmallVector<SDValue, 4> EOps; 4575 4576 for (unsigned i = 0; i < NumOpers; ++i) { 4577 SDValue Op = InOps[i]; 4578 4579 EVT OpVT = Op.getValueType(); 4580 if (OpVT.isVector()) 4581 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 4582 OpVT.getVectorElementType(), Op, 4583 DAG.getVectorIdxConstant(Idx, dl)); 4584 4585 EOps.push_back(Op); 4586 } 4587 4588 EVT WidenVT[] = {WidenEltVT, MVT::Other}; 4589 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps); 4590 ConcatOps[ConcatEnd++] = Oper; 4591 Chains.push_back(Oper.getValue(1)); 4592 } 4593 CurNumElts = 0; 4594 } 4595 } 4596 4597 // Build a factor node to remember all the Ops that have been created. 4598 SDValue NewChain; 4599 if (Chains.size() == 1) 4600 NewChain = Chains[0]; 4601 else 4602 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 4603 ReplaceValueWith(SDValue(N, 1), NewChain); 4604 4605 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); 4606 } 4607 4608 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) { 4609 SDLoc DL(N); 4610 EVT ResVT = N->getValueType(0); 4611 EVT OvVT = N->getValueType(1); 4612 EVT WideResVT, WideOvVT; 4613 SDValue WideLHS, WideRHS; 4614 4615 // TODO: This might result in a widen/split loop. 4616 if (ResNo == 0) { 4617 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT); 4618 WideOvVT = EVT::getVectorVT( 4619 *DAG.getContext(), OvVT.getVectorElementType(), 4620 WideResVT.getVectorNumElements()); 4621 4622 WideLHS = GetWidenedVector(N->getOperand(0)); 4623 WideRHS = GetWidenedVector(N->getOperand(1)); 4624 } else { 4625 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT); 4626 WideResVT = EVT::getVectorVT( 4627 *DAG.getContext(), ResVT.getVectorElementType(), 4628 WideOvVT.getVectorNumElements()); 4629 4630 SDValue Zero = DAG.getVectorIdxConstant(0, DL); 4631 WideLHS = DAG.getNode( 4632 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT), 4633 N->getOperand(0), Zero); 4634 WideRHS = DAG.getNode( 4635 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT), 4636 N->getOperand(1), Zero); 4637 } 4638 4639 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT); 4640 SDNode *WideNode = DAG.getNode( 4641 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode(); 4642 4643 // Replace the other vector result not being explicitly widened here. 4644 unsigned OtherNo = 1 - ResNo; 4645 EVT OtherVT = N->getValueType(OtherNo); 4646 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) { 4647 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo)); 4648 } else { 4649 SDValue Zero = DAG.getVectorIdxConstant(0, DL); 4650 SDValue OtherVal = DAG.getNode( 4651 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero); 4652 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 4653 } 4654 4655 return SDValue(WideNode, ResNo); 4656 } 4657 4658 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 4659 LLVMContext &Ctx = *DAG.getContext(); 4660 SDValue InOp = N->getOperand(0); 4661 SDLoc DL(N); 4662 4663 EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0)); 4664 ElementCount WidenEC = WidenVT.getVectorElementCount(); 4665 4666 EVT InVT = InOp.getValueType(); 4667 4668 unsigned Opcode = N->getOpcode(); 4669 const SDNodeFlags Flags = N->getFlags(); 4670 4671 // Handle the case of ZERO_EXTEND where the promoted InVT element size does 4672 // not equal that of WidenVT. 4673 if (N->getOpcode() == ISD::ZERO_EXTEND && 4674 getTypeAction(InVT) == TargetLowering::TypePromoteInteger && 4675 TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() != 4676 WidenVT.getScalarSizeInBits()) { 4677 InOp = ZExtPromotedInteger(InOp); 4678 InVT = InOp.getValueType(); 4679 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits()) 4680 Opcode = ISD::TRUNCATE; 4681 } 4682 4683 EVT InEltVT = InVT.getVectorElementType(); 4684 EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenEC); 4685 ElementCount InVTEC = InVT.getVectorElementCount(); 4686 4687 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 4688 InOp = GetWidenedVector(N->getOperand(0)); 4689 InVT = InOp.getValueType(); 4690 InVTEC = InVT.getVectorElementCount(); 4691 if (InVTEC == WidenEC) { 4692 if (N->getNumOperands() == 1) 4693 return DAG.getNode(Opcode, DL, WidenVT, InOp); 4694 if (N->getNumOperands() == 3) { 4695 assert(N->isVPOpcode() && "Expected VP opcode"); 4696 SDValue Mask = 4697 GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount()); 4698 return DAG.getNode(Opcode, DL, WidenVT, InOp, Mask, N->getOperand(2)); 4699 } 4700 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags); 4701 } 4702 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) { 4703 // If both input and result vector types are of same width, extend 4704 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which 4705 // accepts fewer elements in the result than in the input. 4706 if (Opcode == ISD::ANY_EXTEND) 4707 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 4708 if (Opcode == ISD::SIGN_EXTEND) 4709 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 4710 if (Opcode == ISD::ZERO_EXTEND) 4711 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 4712 } 4713 } 4714 4715 if (TLI.isTypeLegal(InWidenVT)) { 4716 // Because the result and the input are different vector types, widening 4717 // the result could create a legal type but widening the input might make 4718 // it an illegal type that might lead to repeatedly splitting the input 4719 // and then widening it. To avoid this, we widen the input only if 4720 // it results in a legal type. 4721 if (WidenEC.isKnownMultipleOf(InVTEC.getKnownMinValue())) { 4722 // Widen the input and call convert on the widened input vector. 4723 unsigned NumConcat = 4724 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue(); 4725 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT)); 4726 Ops[0] = InOp; 4727 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops); 4728 if (N->getNumOperands() == 1) 4729 return DAG.getNode(Opcode, DL, WidenVT, InVec); 4730 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags); 4731 } 4732 4733 if (InVTEC.isKnownMultipleOf(WidenEC.getKnownMinValue())) { 4734 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp, 4735 DAG.getVectorIdxConstant(0, DL)); 4736 // Extract the input and convert the shorten input vector. 4737 if (N->getNumOperands() == 1) 4738 return DAG.getNode(Opcode, DL, WidenVT, InVal); 4739 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags); 4740 } 4741 } 4742 4743 // Otherwise unroll into some nasty scalar code and rebuild the vector. 4744 EVT EltVT = WidenVT.getVectorElementType(); 4745 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(EltVT)); 4746 // Use the original element count so we don't do more scalar opts than 4747 // necessary. 4748 unsigned MinElts = N->getValueType(0).getVectorNumElements(); 4749 for (unsigned i=0; i < MinElts; ++i) { 4750 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 4751 DAG.getVectorIdxConstant(i, DL)); 4752 if (N->getNumOperands() == 1) 4753 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 4754 else 4755 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags); 4756 } 4757 4758 return DAG.getBuildVector(WidenVT, DL, Ops); 4759 } 4760 4761 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) { 4762 SDLoc dl(N); 4763 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4764 ElementCount WidenNumElts = WidenVT.getVectorElementCount(); 4765 4766 SDValue Src = N->getOperand(0); 4767 EVT SrcVT = Src.getValueType(); 4768 4769 // Also widen the input. 4770 if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) { 4771 Src = GetWidenedVector(Src); 4772 SrcVT = Src.getValueType(); 4773 } 4774 4775 // Input and output not widened to the same size, give up. 4776 if (WidenNumElts != SrcVT.getVectorElementCount()) 4777 return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue()); 4778 4779 return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1)); 4780 } 4781 4782 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) { 4783 SDValue InOp = N->getOperand(1); 4784 SDLoc DL(N); 4785 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 4786 4787 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4788 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4789 4790 EVT InVT = InOp.getValueType(); 4791 EVT InEltVT = InVT.getVectorElementType(); 4792 4793 unsigned Opcode = N->getOpcode(); 4794 4795 // FIXME: Optimizations need to be implemented here. 4796 4797 // Otherwise unroll into some nasty scalar code and rebuild the vector. 4798 EVT EltVT = WidenVT.getVectorElementType(); 4799 std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}}; 4800 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT)); 4801 SmallVector<SDValue, 32> OpChains; 4802 // Use the original element count so we don't do more scalar opts than 4803 // necessary. 4804 unsigned MinElts = N->getValueType(0).getVectorNumElements(); 4805 for (unsigned i=0; i < MinElts; ++i) { 4806 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 4807 DAG.getVectorIdxConstant(i, DL)); 4808 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps); 4809 OpChains.push_back(Ops[i].getValue(1)); 4810 } 4811 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains); 4812 ReplaceValueWith(SDValue(N, 1), NewChain); 4813 4814 return DAG.getBuildVector(WidenVT, DL, Ops); 4815 } 4816 4817 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) { 4818 unsigned Opcode = N->getOpcode(); 4819 SDValue InOp = N->getOperand(0); 4820 SDLoc DL(N); 4821 4822 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4823 EVT WidenSVT = WidenVT.getVectorElementType(); 4824 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4825 4826 EVT InVT = InOp.getValueType(); 4827 EVT InSVT = InVT.getVectorElementType(); 4828 unsigned InVTNumElts = InVT.getVectorNumElements(); 4829 4830 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 4831 InOp = GetWidenedVector(InOp); 4832 InVT = InOp.getValueType(); 4833 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) { 4834 switch (Opcode) { 4835 case ISD::ANY_EXTEND_VECTOR_INREG: 4836 case ISD::SIGN_EXTEND_VECTOR_INREG: 4837 case ISD::ZERO_EXTEND_VECTOR_INREG: 4838 return DAG.getNode(Opcode, DL, WidenVT, InOp); 4839 } 4840 } 4841 } 4842 4843 // Unroll, extend the scalars and rebuild the vector. 4844 SmallVector<SDValue, 16> Ops; 4845 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) { 4846 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp, 4847 DAG.getVectorIdxConstant(i, DL)); 4848 switch (Opcode) { 4849 case ISD::ANY_EXTEND_VECTOR_INREG: 4850 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val); 4851 break; 4852 case ISD::SIGN_EXTEND_VECTOR_INREG: 4853 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val); 4854 break; 4855 case ISD::ZERO_EXTEND_VECTOR_INREG: 4856 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val); 4857 break; 4858 default: 4859 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected"); 4860 } 4861 Ops.push_back(Val); 4862 } 4863 4864 while (Ops.size() != WidenNumElts) 4865 Ops.push_back(DAG.getUNDEF(WidenSVT)); 4866 4867 return DAG.getBuildVector(WidenVT, DL, Ops); 4868 } 4869 4870 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) { 4871 // If this is an FCOPYSIGN with same input types, we can treat it as a 4872 // normal (can trap) binary op. 4873 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType()) 4874 return WidenVecRes_BinaryCanTrap(N); 4875 4876 // If the types are different, fall back to unrolling. 4877 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4878 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 4879 } 4880 4881 SDValue DAGTypeLegalizer::WidenVecRes_IS_FPCLASS(SDNode *N) { 4882 SDValue FpValue = N->getOperand(0); 4883 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4884 if (getTypeAction(FpValue.getValueType()) != TargetLowering::TypeWidenVector) 4885 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 4886 SDValue Arg = GetWidenedVector(FpValue); 4887 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, {Arg, N->getOperand(1)}, 4888 N->getFlags()); 4889 } 4890 4891 SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) { 4892 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4893 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4894 SDValue RHS = N->getOperand(1); 4895 SDValue ExpOp = RHS.getValueType().isVector() ? GetWidenedVector(RHS) : RHS; 4896 4897 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ExpOp); 4898 } 4899 4900 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 4901 // Unary op widening. 4902 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4903 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4904 if (N->getNumOperands() == 1) 4905 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, N->getFlags()); 4906 4907 assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); 4908 assert(N->isVPOpcode() && "Expected VP opcode"); 4909 4910 SDValue Mask = 4911 GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount()); 4912 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, 4913 {InOp, Mask, N->getOperand(2)}); 4914 } 4915 4916 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 4917 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4918 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 4919 cast<VTSDNode>(N->getOperand(1))->getVT() 4920 .getVectorElementType(), 4921 WidenVT.getVectorNumElements()); 4922 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 4923 return DAG.getNode(N->getOpcode(), SDLoc(N), 4924 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 4925 } 4926 4927 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 4928 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 4929 return GetWidenedVector(WidenVec); 4930 } 4931 4932 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 4933 SDValue InOp = N->getOperand(0); 4934 EVT InVT = InOp.getValueType(); 4935 EVT VT = N->getValueType(0); 4936 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4937 SDLoc dl(N); 4938 4939 switch (getTypeAction(InVT)) { 4940 case TargetLowering::TypeLegal: 4941 break; 4942 case TargetLowering::TypeScalarizeScalableVector: 4943 report_fatal_error("Scalarization of scalable vectors is not supported."); 4944 case TargetLowering::TypePromoteInteger: { 4945 // If the incoming type is a vector that is being promoted, then 4946 // we know that the elements are arranged differently and that we 4947 // must perform the conversion using a stack slot. 4948 if (InVT.isVector()) 4949 break; 4950 4951 // If the InOp is promoted to the same size, convert it. Otherwise, 4952 // fall out of the switch and widen the promoted input. 4953 SDValue NInOp = GetPromotedInteger(InOp); 4954 EVT NInVT = NInOp.getValueType(); 4955 if (WidenVT.bitsEq(NInVT)) { 4956 // For big endian targets we need to shift the input integer or the 4957 // interesting bits will end up at the wrong place. 4958 if (DAG.getDataLayout().isBigEndian()) { 4959 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 4960 EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout()); 4961 assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!"); 4962 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp, 4963 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy)); 4964 } 4965 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp); 4966 } 4967 InOp = NInOp; 4968 InVT = NInVT; 4969 break; 4970 } 4971 case TargetLowering::TypeSoftenFloat: 4972 case TargetLowering::TypePromoteFloat: 4973 case TargetLowering::TypeSoftPromoteHalf: 4974 case TargetLowering::TypeExpandInteger: 4975 case TargetLowering::TypeExpandFloat: 4976 case TargetLowering::TypeScalarizeVector: 4977 case TargetLowering::TypeSplitVector: 4978 break; 4979 case TargetLowering::TypeWidenVector: 4980 // If the InOp is widened to the same size, convert it. Otherwise, fall 4981 // out of the switch and widen the widened input. 4982 InOp = GetWidenedVector(InOp); 4983 InVT = InOp.getValueType(); 4984 if (WidenVT.bitsEq(InVT)) 4985 // The input widens to the same size. Convert to the widen value. 4986 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 4987 break; 4988 } 4989 4990 unsigned WidenSize = WidenVT.getSizeInBits(); 4991 unsigned InSize = InVT.getSizeInBits(); 4992 unsigned InScalarSize = InVT.getScalarSizeInBits(); 4993 // x86mmx is not an acceptable vector element type, so don't try. 4994 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) { 4995 // Determine new input vector type. The new input vector type will use 4996 // the same element type (if its a vector) or use the input type as a 4997 // vector. It is the same size as the type to widen to. 4998 EVT NewInVT; 4999 unsigned NewNumParts = WidenSize / InSize; 5000 if (InVT.isVector()) { 5001 EVT InEltVT = InVT.getVectorElementType(); 5002 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 5003 WidenSize / InEltVT.getSizeInBits()); 5004 } else { 5005 // For big endian systems, using the promoted input scalar type 5006 // to produce the scalar_to_vector would put the desired bits into 5007 // the least significant byte(s) of the wider element zero. This 5008 // will mean that the users of the result vector are using incorrect 5009 // bits. Use the original input type instead. Although either input 5010 // type can be used on little endian systems, for consistency we 5011 // use the original type there as well. 5012 EVT OrigInVT = N->getOperand(0).getValueType(); 5013 NewNumParts = WidenSize / OrigInVT.getSizeInBits(); 5014 NewInVT = EVT::getVectorVT(*DAG.getContext(), OrigInVT, NewNumParts); 5015 } 5016 5017 if (TLI.isTypeLegal(NewInVT)) { 5018 SDValue NewVec; 5019 if (InVT.isVector()) { 5020 // Because the result and the input are different vector types, widening 5021 // the result could create a legal type but widening the input might 5022 // make it an illegal type that might lead to repeatedly splitting the 5023 // input and then widening it. To avoid this, we widen the input only if 5024 // it results in a legal type. 5025 if (WidenSize % InSize == 0) { 5026 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getUNDEF(InVT)); 5027 Ops[0] = InOp; 5028 5029 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops); 5030 } else { 5031 SmallVector<SDValue, 16> Ops; 5032 DAG.ExtractVectorElements(InOp, Ops); 5033 Ops.append(WidenSize / InScalarSize - Ops.size(), 5034 DAG.getUNDEF(InVT.getVectorElementType())); 5035 5036 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, Ops); 5037 } 5038 } else { 5039 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp); 5040 } 5041 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 5042 } 5043 } 5044 5045 return CreateStackStoreLoad(InOp, WidenVT); 5046 } 5047 5048 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 5049 SDLoc dl(N); 5050 // Build a vector with undefined for the new nodes. 5051 EVT VT = N->getValueType(0); 5052 5053 // Integer BUILD_VECTOR operands may be larger than the node's vector element 5054 // type. The UNDEFs need to have the same type as the existing operands. 5055 EVT EltVT = N->getOperand(0).getValueType(); 5056 unsigned NumElts = VT.getVectorNumElements(); 5057 5058 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5059 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 5060 5061 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 5062 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!"); 5063 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT)); 5064 5065 return DAG.getBuildVector(WidenVT, dl, NewOps); 5066 } 5067 5068 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 5069 EVT InVT = N->getOperand(0).getValueType(); 5070 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5071 SDLoc dl(N); 5072 unsigned NumOperands = N->getNumOperands(); 5073 5074 bool InputWidened = false; // Indicates we need to widen the input. 5075 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 5076 unsigned WidenNumElts = WidenVT.getVectorMinNumElements(); 5077 unsigned NumInElts = InVT.getVectorMinNumElements(); 5078 if (WidenNumElts % NumInElts == 0) { 5079 // Add undef vectors to widen to correct length. 5080 unsigned NumConcat = WidenNumElts / NumInElts; 5081 SDValue UndefVal = DAG.getUNDEF(InVT); 5082 SmallVector<SDValue, 16> Ops(NumConcat); 5083 for (unsigned i=0; i < NumOperands; ++i) 5084 Ops[i] = N->getOperand(i); 5085 for (unsigned i = NumOperands; i != NumConcat; ++i) 5086 Ops[i] = UndefVal; 5087 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops); 5088 } 5089 } else { 5090 InputWidened = true; 5091 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 5092 // The inputs and the result are widen to the same value. 5093 unsigned i; 5094 for (i=1; i < NumOperands; ++i) 5095 if (!N->getOperand(i).isUndef()) 5096 break; 5097 5098 if (i == NumOperands) 5099 // Everything but the first operand is an UNDEF so just return the 5100 // widened first operand. 5101 return GetWidenedVector(N->getOperand(0)); 5102 5103 if (NumOperands == 2) { 5104 assert(!WidenVT.isScalableVector() && 5105 "Cannot use vector shuffles to widen CONCAT_VECTOR result"); 5106 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 5107 unsigned NumInElts = InVT.getVectorNumElements(); 5108 5109 // Replace concat of two operands with a shuffle. 5110 SmallVector<int, 16> MaskOps(WidenNumElts, -1); 5111 for (unsigned i = 0; i < NumInElts; ++i) { 5112 MaskOps[i] = i; 5113 MaskOps[i + NumInElts] = i + WidenNumElts; 5114 } 5115 return DAG.getVectorShuffle(WidenVT, dl, 5116 GetWidenedVector(N->getOperand(0)), 5117 GetWidenedVector(N->getOperand(1)), 5118 MaskOps); 5119 } 5120 } 5121 } 5122 5123 assert(!WidenVT.isScalableVector() && 5124 "Cannot use build vectors to widen CONCAT_VECTOR result"); 5125 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 5126 unsigned NumInElts = InVT.getVectorNumElements(); 5127 5128 // Fall back to use extracts and build vector. 5129 EVT EltVT = WidenVT.getVectorElementType(); 5130 SmallVector<SDValue, 16> Ops(WidenNumElts); 5131 unsigned Idx = 0; 5132 for (unsigned i=0; i < NumOperands; ++i) { 5133 SDValue InOp = N->getOperand(i); 5134 if (InputWidened) 5135 InOp = GetWidenedVector(InOp); 5136 for (unsigned j = 0; j < NumInElts; ++j) 5137 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 5138 DAG.getVectorIdxConstant(j, dl)); 5139 } 5140 SDValue UndefVal = DAG.getUNDEF(EltVT); 5141 for (; Idx < WidenNumElts; ++Idx) 5142 Ops[Idx] = UndefVal; 5143 return DAG.getBuildVector(WidenVT, dl, Ops); 5144 } 5145 5146 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) { 5147 EVT VT = N->getValueType(0); 5148 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5149 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 5150 SDValue InOp2 = N->getOperand(1); 5151 SDValue Idx = N->getOperand(2); 5152 SDLoc dl(N); 5153 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WidenVT, InOp1, InOp2, Idx); 5154 } 5155 5156 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 5157 EVT VT = N->getValueType(0); 5158 EVT EltVT = VT.getVectorElementType(); 5159 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5160 SDValue InOp = N->getOperand(0); 5161 SDValue Idx = N->getOperand(1); 5162 SDLoc dl(N); 5163 5164 auto InOpTypeAction = getTypeAction(InOp.getValueType()); 5165 if (InOpTypeAction == TargetLowering::TypeWidenVector) 5166 InOp = GetWidenedVector(InOp); 5167 5168 EVT InVT = InOp.getValueType(); 5169 5170 // Check if we can just return the input vector after widening. 5171 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 5172 if (IdxVal == 0 && InVT == WidenVT) 5173 return InOp; 5174 5175 // Check if we can extract from the vector. 5176 unsigned WidenNumElts = WidenVT.getVectorMinNumElements(); 5177 unsigned InNumElts = InVT.getVectorMinNumElements(); 5178 unsigned VTNumElts = VT.getVectorMinNumElements(); 5179 assert(IdxVal % VTNumElts == 0 && 5180 "Expected Idx to be a multiple of subvector minimum vector length"); 5181 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 5182 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 5183 5184 if (VT.isScalableVector()) { 5185 // Try to split the operation up into smaller extracts and concat the 5186 // results together, e.g. 5187 // nxv6i64 extract_subvector(nxv12i64, 6) 5188 // <-> 5189 // nxv8i64 concat( 5190 // nxv2i64 extract_subvector(nxv16i64, 6) 5191 // nxv2i64 extract_subvector(nxv16i64, 8) 5192 // nxv2i64 extract_subvector(nxv16i64, 10) 5193 // undef) 5194 unsigned GCD = std::gcd(VTNumElts, WidenNumElts); 5195 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken " 5196 "down type's element count"); 5197 EVT PartVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 5198 ElementCount::getScalable(GCD)); 5199 // Avoid recursion around e.g. nxv1i8. 5200 if (getTypeAction(PartVT) != TargetLowering::TypeWidenVector) { 5201 SmallVector<SDValue> Parts; 5202 unsigned I = 0; 5203 for (; I < VTNumElts / GCD; ++I) 5204 Parts.push_back( 5205 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, PartVT, InOp, 5206 DAG.getVectorIdxConstant(IdxVal + I * GCD, dl))); 5207 for (; I < WidenNumElts / GCD; ++I) 5208 Parts.push_back(DAG.getUNDEF(PartVT)); 5209 5210 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Parts); 5211 } 5212 5213 report_fatal_error("Don't know how to widen the result of " 5214 "EXTRACT_SUBVECTOR for scalable vectors"); 5215 } 5216 5217 // We could try widening the input to the right length but for now, extract 5218 // the original elements, fill the rest with undefs and build a vector. 5219 SmallVector<SDValue, 16> Ops(WidenNumElts); 5220 unsigned i; 5221 for (i = 0; i < VTNumElts; ++i) 5222 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 5223 DAG.getVectorIdxConstant(IdxVal + i, dl)); 5224 5225 SDValue UndefVal = DAG.getUNDEF(EltVT); 5226 for (; i < WidenNumElts; ++i) 5227 Ops[i] = UndefVal; 5228 return DAG.getBuildVector(WidenVT, dl, Ops); 5229 } 5230 5231 SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) { 5232 SDValue InOp = ModifyToType( 5233 N->getOperand(0), 5234 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), true); 5235 return DAG.getNode(ISD::AssertZext, SDLoc(N), InOp.getValueType(), InOp, 5236 N->getOperand(1)); 5237 } 5238 5239 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 5240 SDValue InOp = GetWidenedVector(N->getOperand(0)); 5241 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), 5242 InOp.getValueType(), InOp, 5243 N->getOperand(1), N->getOperand(2)); 5244 } 5245 5246 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 5247 LoadSDNode *LD = cast<LoadSDNode>(N); 5248 ISD::LoadExtType ExtType = LD->getExtensionType(); 5249 5250 // A vector must always be stored in memory as-is, i.e. without any padding 5251 // between the elements, since various code depend on it, e.g. in the 5252 // handling of a bitcast of a vector type to int, which may be done with a 5253 // vector store followed by an integer load. A vector that does not have 5254 // elements that are byte-sized must therefore be stored as an integer 5255 // built out of the extracted vector elements. 5256 if (!LD->getMemoryVT().isByteSized()) { 5257 SDValue Value, NewChain; 5258 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG); 5259 ReplaceValueWith(SDValue(LD, 0), Value); 5260 ReplaceValueWith(SDValue(LD, 1), NewChain); 5261 return SDValue(); 5262 } 5263 5264 // Generate a vector-predicated load if it is custom/legal on the target. To 5265 // avoid possible recursion, only do this if the widened mask type is legal. 5266 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been 5267 // removed from the IR by the ExpandVectorPredication pass but we're 5268 // reintroducing them here. 5269 EVT LdVT = LD->getMemoryVT(); 5270 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), LdVT); 5271 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 5272 WideVT.getVectorElementCount()); 5273 if (ExtType == ISD::NON_EXTLOAD && 5274 TLI.isOperationLegalOrCustom(ISD::VP_LOAD, WideVT) && 5275 TLI.isTypeLegal(WideMaskVT)) { 5276 SDLoc DL(N); 5277 SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT); 5278 SDValue EVL = DAG.getElementCount(DL, TLI.getVPExplicitVectorLengthTy(), 5279 LdVT.getVectorElementCount()); 5280 const auto *MMO = LD->getMemOperand(); 5281 SDValue NewLoad = 5282 DAG.getLoadVP(WideVT, DL, LD->getChain(), LD->getBasePtr(), Mask, EVL, 5283 MMO->getPointerInfo(), MMO->getAlign(), MMO->getFlags(), 5284 MMO->getAAInfo()); 5285 5286 // Modified the chain - switch anything that used the old chain to use 5287 // the new one. 5288 ReplaceValueWith(SDValue(N, 1), NewLoad.getValue(1)); 5289 5290 return NewLoad; 5291 } 5292 5293 SDValue Result; 5294 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 5295 if (ExtType != ISD::NON_EXTLOAD) 5296 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 5297 else 5298 Result = GenWidenVectorLoads(LdChain, LD); 5299 5300 if (Result) { 5301 // If we generate a single load, we can use that for the chain. Otherwise, 5302 // build a factor node to remember the multiple loads are independent and 5303 // chain to that. 5304 SDValue NewChain; 5305 if (LdChain.size() == 1) 5306 NewChain = LdChain[0]; 5307 else 5308 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain); 5309 5310 // Modified the chain - switch anything that used the old chain to use 5311 // the new one. 5312 ReplaceValueWith(SDValue(N, 1), NewChain); 5313 5314 return Result; 5315 } 5316 5317 report_fatal_error("Unable to widen vector load"); 5318 } 5319 5320 SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) { 5321 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5322 SDValue Mask = N->getMask(); 5323 SDValue EVL = N->getVectorLength(); 5324 ISD::LoadExtType ExtType = N->getExtensionType(); 5325 SDLoc dl(N); 5326 5327 // The mask should be widened as well 5328 assert(getTypeAction(Mask.getValueType()) == 5329 TargetLowering::TypeWidenVector && 5330 "Unable to widen binary VP op"); 5331 Mask = GetWidenedVector(Mask); 5332 assert(Mask.getValueType().getVectorElementCount() == 5333 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType()) 5334 .getVectorElementCount() && 5335 "Unable to widen vector load"); 5336 5337 SDValue Res = 5338 DAG.getLoadVP(N->getAddressingMode(), ExtType, WidenVT, dl, N->getChain(), 5339 N->getBasePtr(), N->getOffset(), Mask, EVL, 5340 N->getMemoryVT(), N->getMemOperand(), N->isExpandingLoad()); 5341 // Legalize the chain result - switch anything that used the old chain to 5342 // use the new one. 5343 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 5344 return Res; 5345 } 5346 5347 SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) { 5348 SDLoc DL(N); 5349 5350 // The mask should be widened as well 5351 SDValue Mask = N->getMask(); 5352 assert(getTypeAction(Mask.getValueType()) == 5353 TargetLowering::TypeWidenVector && 5354 "Unable to widen VP strided load"); 5355 Mask = GetWidenedVector(Mask); 5356 5357 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5358 assert(Mask.getValueType().getVectorElementCount() == 5359 WidenVT.getVectorElementCount() && 5360 "Data and mask vectors should have the same number of elements"); 5361 5362 SDValue Res = DAG.getStridedLoadVP( 5363 N->getAddressingMode(), N->getExtensionType(), WidenVT, DL, N->getChain(), 5364 N->getBasePtr(), N->getOffset(), N->getStride(), Mask, 5365 N->getVectorLength(), N->getMemoryVT(), N->getMemOperand(), 5366 N->isExpandingLoad()); 5367 5368 // Legalize the chain result - switch anything that used the old chain to 5369 // use the new one. 5370 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 5371 return Res; 5372 } 5373 5374 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) { 5375 5376 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0)); 5377 SDValue Mask = N->getMask(); 5378 EVT MaskVT = Mask.getValueType(); 5379 SDValue PassThru = GetWidenedVector(N->getPassThru()); 5380 ISD::LoadExtType ExtType = N->getExtensionType(); 5381 SDLoc dl(N); 5382 5383 // The mask should be widened as well 5384 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 5385 MaskVT.getVectorElementType(), 5386 WidenVT.getVectorNumElements()); 5387 Mask = ModifyToType(Mask, WideMaskVT, true); 5388 5389 SDValue Res = DAG.getMaskedLoad( 5390 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, 5391 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(), 5392 ExtType, N->isExpandingLoad()); 5393 // Legalize the chain result - switch anything that used the old chain to 5394 // use the new one. 5395 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 5396 return Res; 5397 } 5398 5399 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) { 5400 5401 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5402 SDValue Mask = N->getMask(); 5403 EVT MaskVT = Mask.getValueType(); 5404 SDValue PassThru = GetWidenedVector(N->getPassThru()); 5405 SDValue Scale = N->getScale(); 5406 unsigned NumElts = WideVT.getVectorNumElements(); 5407 SDLoc dl(N); 5408 5409 // The mask should be widened as well 5410 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 5411 MaskVT.getVectorElementType(), 5412 WideVT.getVectorNumElements()); 5413 Mask = ModifyToType(Mask, WideMaskVT, true); 5414 5415 // Widen the Index operand 5416 SDValue Index = N->getIndex(); 5417 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(), 5418 Index.getValueType().getScalarType(), 5419 NumElts); 5420 Index = ModifyToType(Index, WideIndexVT); 5421 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index, 5422 Scale }; 5423 5424 // Widen the MemoryType 5425 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(), 5426 N->getMemoryVT().getScalarType(), NumElts); 5427 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other), 5428 WideMemVT, dl, Ops, N->getMemOperand(), 5429 N->getIndexType(), N->getExtensionType()); 5430 5431 // Legalize the chain result - switch anything that used the old chain to 5432 // use the new one. 5433 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 5434 return Res; 5435 } 5436 5437 SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) { 5438 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5439 SDValue Mask = N->getMask(); 5440 SDValue Scale = N->getScale(); 5441 ElementCount WideEC = WideVT.getVectorElementCount(); 5442 SDLoc dl(N); 5443 5444 SDValue Index = GetWidenedVector(N->getIndex()); 5445 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(), 5446 N->getMemoryVT().getScalarType(), WideEC); 5447 Mask = GetWidenedMask(Mask, WideEC); 5448 5449 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale, 5450 Mask, N->getVectorLength()}; 5451 SDValue Res = DAG.getGatherVP(DAG.getVTList(WideVT, MVT::Other), WideMemVT, 5452 dl, Ops, N->getMemOperand(), N->getIndexType()); 5453 5454 // Legalize the chain result - switch anything that used the old chain to 5455 // use the new one. 5456 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 5457 return Res; 5458 } 5459 5460 SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) { 5461 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5462 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0)); 5463 } 5464 5465 // Return true is this is a SETCC node or a strict version of it. 5466 static inline bool isSETCCOp(unsigned Opcode) { 5467 switch (Opcode) { 5468 case ISD::SETCC: 5469 case ISD::STRICT_FSETCC: 5470 case ISD::STRICT_FSETCCS: 5471 return true; 5472 } 5473 return false; 5474 } 5475 5476 // Return true if this is a node that could have two SETCCs as operands. 5477 static inline bool isLogicalMaskOp(unsigned Opcode) { 5478 switch (Opcode) { 5479 case ISD::AND: 5480 case ISD::OR: 5481 case ISD::XOR: 5482 return true; 5483 } 5484 return false; 5485 } 5486 5487 // If N is a SETCC or a strict variant of it, return the type 5488 // of the compare operands. 5489 static inline EVT getSETCCOperandType(SDValue N) { 5490 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 5491 return N->getOperand(OpNo).getValueType(); 5492 } 5493 5494 // This is used just for the assert in convertMask(). Check that this either 5495 // a SETCC or a previously handled SETCC by convertMask(). 5496 #ifndef NDEBUG 5497 static inline bool isSETCCorConvertedSETCC(SDValue N) { 5498 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) 5499 N = N.getOperand(0); 5500 else if (N.getOpcode() == ISD::CONCAT_VECTORS) { 5501 for (unsigned i = 1; i < N->getNumOperands(); ++i) 5502 if (!N->getOperand(i)->isUndef()) 5503 return false; 5504 N = N.getOperand(0); 5505 } 5506 5507 if (N.getOpcode() == ISD::TRUNCATE) 5508 N = N.getOperand(0); 5509 else if (N.getOpcode() == ISD::SIGN_EXTEND) 5510 N = N.getOperand(0); 5511 5512 if (isLogicalMaskOp(N.getOpcode())) 5513 return isSETCCorConvertedSETCC(N.getOperand(0)) && 5514 isSETCCorConvertedSETCC(N.getOperand(1)); 5515 5516 return (isSETCCOp(N.getOpcode()) || 5517 ISD::isBuildVectorOfConstantSDNodes(N.getNode())); 5518 } 5519 #endif 5520 5521 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT 5522 // to ToMaskVT if needed with vector extension or truncation. 5523 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT, 5524 EVT ToMaskVT) { 5525 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled. 5526 // FIXME: This code seems to be too restrictive, we might consider 5527 // generalizing it or dropping it. 5528 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument."); 5529 5530 // Make a new Mask node, with a legal result VT. 5531 SDValue Mask; 5532 SmallVector<SDValue, 4> Ops; 5533 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i) 5534 Ops.push_back(InMask->getOperand(i)); 5535 if (InMask->isStrictFPOpcode()) { 5536 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), 5537 { MaskVT, MVT::Other }, Ops); 5538 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1)); 5539 } 5540 else 5541 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops); 5542 5543 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign 5544 // extend or truncate is needed. 5545 LLVMContext &Ctx = *DAG.getContext(); 5546 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits(); 5547 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits(); 5548 if (MaskScalarBits < ToMaskScalBits) { 5549 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(), 5550 MaskVT.getVectorNumElements()); 5551 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask); 5552 } else if (MaskScalarBits > ToMaskScalBits) { 5553 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(), 5554 MaskVT.getVectorNumElements()); 5555 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask); 5556 } 5557 5558 assert(Mask->getValueType(0).getScalarSizeInBits() == 5559 ToMaskVT.getScalarSizeInBits() && 5560 "Mask should have the right element size by now."); 5561 5562 // Adjust Mask to the right number of elements. 5563 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements(); 5564 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) { 5565 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask)); 5566 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask, 5567 ZeroIdx); 5568 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) { 5569 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls); 5570 EVT SubVT = Mask->getValueType(0); 5571 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT)); 5572 SubOps[0] = Mask; 5573 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps); 5574 } 5575 5576 assert((Mask->getValueType(0) == ToMaskVT) && 5577 "A mask of ToMaskVT should have been produced by now."); 5578 5579 return Mask; 5580 } 5581 5582 // This method tries to handle some special cases for the vselect mask 5583 // and if needed adjusting the mask vector type to match that of the VSELECT. 5584 // Without it, many cases end up with scalarization of the SETCC, with many 5585 // unnecessary instructions. 5586 SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) { 5587 LLVMContext &Ctx = *DAG.getContext(); 5588 SDValue Cond = N->getOperand(0); 5589 5590 if (N->getOpcode() != ISD::VSELECT) 5591 return SDValue(); 5592 5593 if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode())) 5594 return SDValue(); 5595 5596 // If this is a splitted VSELECT that was previously already handled, do 5597 // nothing. 5598 EVT CondVT = Cond->getValueType(0); 5599 if (CondVT.getScalarSizeInBits() != 1) 5600 return SDValue(); 5601 5602 EVT VSelVT = N->getValueType(0); 5603 5604 // This method can't handle scalable vector types. 5605 // FIXME: This support could be added in the future. 5606 if (VSelVT.isScalableVector()) 5607 return SDValue(); 5608 5609 // Only handle vector types which are a power of 2. 5610 if (!isPowerOf2_64(VSelVT.getSizeInBits())) 5611 return SDValue(); 5612 5613 // Don't touch if this will be scalarized. 5614 EVT FinalVT = VSelVT; 5615 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector) 5616 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx); 5617 5618 if (FinalVT.getVectorNumElements() == 1) 5619 return SDValue(); 5620 5621 // If there is support for an i1 vector mask, don't touch. 5622 if (isSETCCOp(Cond.getOpcode())) { 5623 EVT SetCCOpVT = getSETCCOperandType(Cond); 5624 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal) 5625 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT); 5626 EVT SetCCResVT = getSetCCResultType(SetCCOpVT); 5627 if (SetCCResVT.getScalarSizeInBits() == 1) 5628 return SDValue(); 5629 } else if (CondVT.getScalarType() == MVT::i1) { 5630 // If there is support for an i1 vector mask (or only scalar i1 conditions), 5631 // don't touch. 5632 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal) 5633 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT); 5634 5635 if (CondVT.getScalarType() == MVT::i1) 5636 return SDValue(); 5637 } 5638 5639 // Widen the vselect result type if needed. 5640 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) 5641 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT); 5642 5643 // The mask of the VSELECT should have integer elements. 5644 EVT ToMaskVT = VSelVT; 5645 if (!ToMaskVT.getScalarType().isInteger()) 5646 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger(); 5647 5648 SDValue Mask; 5649 if (isSETCCOp(Cond->getOpcode())) { 5650 EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond)); 5651 Mask = convertMask(Cond, MaskVT, ToMaskVT); 5652 } else if (isLogicalMaskOp(Cond->getOpcode()) && 5653 isSETCCOp(Cond->getOperand(0).getOpcode()) && 5654 isSETCCOp(Cond->getOperand(1).getOpcode())) { 5655 // Cond is (AND/OR/XOR (SETCC, SETCC)) 5656 SDValue SETCC0 = Cond->getOperand(0); 5657 SDValue SETCC1 = Cond->getOperand(1); 5658 EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0)); 5659 EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1)); 5660 unsigned ScalarBits0 = VT0.getScalarSizeInBits(); 5661 unsigned ScalarBits1 = VT1.getScalarSizeInBits(); 5662 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits(); 5663 EVT MaskVT; 5664 // If the two SETCCs have different VTs, either extend/truncate one of 5665 // them to the other "towards" ToMaskVT, or truncate one and extend the 5666 // other to ToMaskVT. 5667 if (ScalarBits0 != ScalarBits1) { 5668 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1); 5669 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0); 5670 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits()) 5671 MaskVT = WideVT; 5672 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits()) 5673 MaskVT = NarrowVT; 5674 else 5675 MaskVT = ToMaskVT; 5676 } else 5677 // If the two SETCCs have the same VT, don't change it. 5678 MaskVT = VT0; 5679 5680 // Make new SETCCs and logical nodes. 5681 SETCC0 = convertMask(SETCC0, VT0, MaskVT); 5682 SETCC1 = convertMask(SETCC1, VT1, MaskVT); 5683 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1); 5684 5685 // Convert the logical op for VSELECT if needed. 5686 Mask = convertMask(Cond, MaskVT, ToMaskVT); 5687 } else 5688 return SDValue(); 5689 5690 return Mask; 5691 } 5692 5693 SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) { 5694 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5695 ElementCount WidenEC = WidenVT.getVectorElementCount(); 5696 5697 SDValue Cond1 = N->getOperand(0); 5698 EVT CondVT = Cond1.getValueType(); 5699 unsigned Opcode = N->getOpcode(); 5700 if (CondVT.isVector()) { 5701 if (SDValue WideCond = WidenVSELECTMask(N)) { 5702 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 5703 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 5704 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 5705 return DAG.getNode(Opcode, SDLoc(N), WidenVT, WideCond, InOp1, InOp2); 5706 } 5707 5708 EVT CondEltVT = CondVT.getVectorElementType(); 5709 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenEC); 5710 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 5711 Cond1 = GetWidenedVector(Cond1); 5712 5713 // If we have to split the condition there is no point in widening the 5714 // select. This would result in an cycle of widening the select -> 5715 // widening the condition operand -> splitting the condition operand -> 5716 // splitting the select -> widening the select. Instead split this select 5717 // further and widen the resulting type. 5718 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) { 5719 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0); 5720 SDValue Res = ModifyToType(SplitSelect, WidenVT); 5721 return Res; 5722 } 5723 5724 if (Cond1.getValueType() != CondWidenVT) 5725 Cond1 = ModifyToType(Cond1, CondWidenVT); 5726 } 5727 5728 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 5729 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 5730 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 5731 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) 5732 return DAG.getNode(Opcode, SDLoc(N), WidenVT, Cond1, InOp1, InOp2, 5733 N->getOperand(3)); 5734 return DAG.getNode(Opcode, SDLoc(N), WidenVT, Cond1, InOp1, InOp2); 5735 } 5736 5737 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 5738 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 5739 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 5740 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 5741 InOp1.getValueType(), N->getOperand(0), 5742 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 5743 } 5744 5745 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 5746 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5747 return DAG.getUNDEF(WidenVT); 5748 } 5749 5750 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 5751 EVT VT = N->getValueType(0); 5752 SDLoc dl(N); 5753 5754 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5755 unsigned NumElts = VT.getVectorNumElements(); 5756 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 5757 5758 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 5759 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 5760 5761 // Adjust mask based on new input vector length. 5762 SmallVector<int, 16> NewMask; 5763 for (unsigned i = 0; i != NumElts; ++i) { 5764 int Idx = N->getMaskElt(i); 5765 if (Idx < (int)NumElts) 5766 NewMask.push_back(Idx); 5767 else 5768 NewMask.push_back(Idx - NumElts + WidenNumElts); 5769 } 5770 for (unsigned i = NumElts; i != WidenNumElts; ++i) 5771 NewMask.push_back(-1); 5772 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask); 5773 } 5774 5775 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) { 5776 EVT VT = N->getValueType(0); 5777 EVT EltVT = VT.getVectorElementType(); 5778 SDLoc dl(N); 5779 5780 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5781 SDValue OpValue = GetWidenedVector(N->getOperand(0)); 5782 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type"); 5783 5784 SDValue ReverseVal = DAG.getNode(ISD::VECTOR_REVERSE, dl, WidenVT, OpValue); 5785 unsigned WidenNumElts = WidenVT.getVectorMinNumElements(); 5786 unsigned VTNumElts = VT.getVectorMinNumElements(); 5787 unsigned IdxVal = WidenNumElts - VTNumElts; 5788 5789 if (VT.isScalableVector()) { 5790 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the 5791 // results together, e.g.(nxv6i64 -> nxv8i64) 5792 // nxv8i64 vector_reverse 5793 // <-> 5794 // nxv8i64 concat( 5795 // nxv2i64 extract_subvector(nxv8i64, 2) 5796 // nxv2i64 extract_subvector(nxv8i64, 4) 5797 // nxv2i64 extract_subvector(nxv8i64, 6) 5798 // nxv2i64 undef) 5799 5800 unsigned GCD = std::gcd(VTNumElts, WidenNumElts); 5801 EVT PartVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 5802 ElementCount::getScalable(GCD)); 5803 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken " 5804 "down type's element count"); 5805 SmallVector<SDValue> Parts; 5806 unsigned i = 0; 5807 for (; i < VTNumElts / GCD; ++i) 5808 Parts.push_back( 5809 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, PartVT, ReverseVal, 5810 DAG.getVectorIdxConstant(IdxVal + i * GCD, dl))); 5811 for (; i < WidenNumElts / GCD; ++i) 5812 Parts.push_back(DAG.getUNDEF(PartVT)); 5813 5814 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Parts); 5815 } 5816 5817 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for 5818 // fixed-vectors. 5819 SmallVector<int, 16> Mask; 5820 for (unsigned i = 0; i != VTNumElts; ++i) { 5821 Mask.push_back(IdxVal + i); 5822 } 5823 for (unsigned i = VTNumElts; i != WidenNumElts; ++i) 5824 Mask.push_back(-1); 5825 5826 return DAG.getVectorShuffle(WidenVT, dl, ReverseVal, DAG.getUNDEF(WidenVT), 5827 Mask); 5828 } 5829 5830 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 5831 assert(N->getValueType(0).isVector() && 5832 N->getOperand(0).getValueType().isVector() && 5833 "Operands must be vectors"); 5834 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 5835 ElementCount WidenEC = WidenVT.getVectorElementCount(); 5836 5837 SDValue InOp1 = N->getOperand(0); 5838 EVT InVT = InOp1.getValueType(); 5839 assert(InVT.isVector() && "can not widen non-vector type"); 5840 EVT WidenInVT = 5841 EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenEC); 5842 5843 // The input and output types often differ here, and it could be that while 5844 // we'd prefer to widen the result type, the input operands have been split. 5845 // In this case, we also need to split the result of this node as well. 5846 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { 5847 SDValue SplitVSetCC = SplitVecOp_VSETCC(N); 5848 SDValue Res = ModifyToType(SplitVSetCC, WidenVT); 5849 return Res; 5850 } 5851 5852 // If the inputs also widen, handle them directly. Otherwise widen by hand. 5853 SDValue InOp2 = N->getOperand(1); 5854 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 5855 InOp1 = GetWidenedVector(InOp1); 5856 InOp2 = GetWidenedVector(InOp2); 5857 } else { 5858 InOp1 = DAG.WidenVector(InOp1, SDLoc(N)); 5859 InOp2 = DAG.WidenVector(InOp2, SDLoc(N)); 5860 } 5861 5862 // Assume that the input and output will be widen appropriately. If not, 5863 // we will have to unroll it at some point. 5864 assert(InOp1.getValueType() == WidenInVT && 5865 InOp2.getValueType() == WidenInVT && 5866 "Input not widened to expected type!"); 5867 (void)WidenInVT; 5868 if (N->getOpcode() == ISD::VP_SETCC) { 5869 SDValue Mask = 5870 GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount()); 5871 return DAG.getNode(ISD::VP_SETCC, SDLoc(N), WidenVT, InOp1, InOp2, 5872 N->getOperand(2), Mask, N->getOperand(4)); 5873 } 5874 return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT, InOp1, InOp2, 5875 N->getOperand(2)); 5876 } 5877 5878 SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) { 5879 assert(N->getValueType(0).isVector() && 5880 N->getOperand(1).getValueType().isVector() && 5881 "Operands must be vectors"); 5882 EVT VT = N->getValueType(0); 5883 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 5884 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 5885 unsigned NumElts = VT.getVectorNumElements(); 5886 EVT EltVT = VT.getVectorElementType(); 5887 5888 SDLoc dl(N); 5889 SDValue Chain = N->getOperand(0); 5890 SDValue LHS = N->getOperand(1); 5891 SDValue RHS = N->getOperand(2); 5892 SDValue CC = N->getOperand(3); 5893 EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 5894 5895 // Fully unroll and reassemble. 5896 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT)); 5897 SmallVector<SDValue, 8> Chains(NumElts); 5898 for (unsigned i = 0; i != NumElts; ++i) { 5899 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 5900 DAG.getVectorIdxConstant(i, dl)); 5901 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 5902 DAG.getVectorIdxConstant(i, dl)); 5903 5904 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other}, 5905 {Chain, LHSElem, RHSElem, CC}); 5906 Chains[i] = Scalars[i].getValue(1); 5907 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i], 5908 DAG.getBoolConstant(true, dl, EltVT, VT), 5909 DAG.getBoolConstant(false, dl, EltVT, VT)); 5910 } 5911 5912 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 5913 ReplaceValueWith(SDValue(N, 1), NewChain); 5914 5915 return DAG.getBuildVector(WidenVT, dl, Scalars); 5916 } 5917 5918 //===----------------------------------------------------------------------===// 5919 // Widen Vector Operand 5920 //===----------------------------------------------------------------------===// 5921 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 5922 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG); 5923 dbgs() << "\n"); 5924 SDValue Res = SDValue(); 5925 5926 // See if the target wants to custom widen this node. 5927 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 5928 return false; 5929 5930 switch (N->getOpcode()) { 5931 default: 5932 #ifndef NDEBUG 5933 dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 5934 N->dump(&DAG); 5935 dbgs() << "\n"; 5936 #endif 5937 report_fatal_error("Do not know how to widen this operator's operand!"); 5938 5939 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 5940 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 5941 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break; 5942 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 5943 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 5944 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 5945 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break; 5946 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 5947 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo); 5948 break; 5949 case ISD::ANY_EXTEND_VECTOR_INREG: 5950 case ISD::SIGN_EXTEND_VECTOR_INREG: 5951 case ISD::ZERO_EXTEND_VECTOR_INREG: 5952 Res = WidenVecOp_EXTEND_VECTOR_INREG(N); 5953 break; 5954 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break; 5955 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break; 5956 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break; 5957 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break; 5958 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 5959 case ISD::STRICT_FSETCC: 5960 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break; 5961 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break; 5962 case ISD::FLDEXP: 5963 case ISD::FCOPYSIGN: Res = WidenVecOp_UnrollVectorOp(N); break; 5964 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break; 5965 5966 case ISD::ANY_EXTEND: 5967 case ISD::SIGN_EXTEND: 5968 case ISD::ZERO_EXTEND: 5969 Res = WidenVecOp_EXTEND(N); 5970 break; 5971 5972 case ISD::FP_EXTEND: 5973 case ISD::STRICT_FP_EXTEND: 5974 case ISD::FP_ROUND: 5975 case ISD::STRICT_FP_ROUND: 5976 case ISD::FP_TO_SINT: 5977 case ISD::STRICT_FP_TO_SINT: 5978 case ISD::FP_TO_UINT: 5979 case ISD::STRICT_FP_TO_UINT: 5980 case ISD::SINT_TO_FP: 5981 case ISD::STRICT_SINT_TO_FP: 5982 case ISD::UINT_TO_FP: 5983 case ISD::STRICT_UINT_TO_FP: 5984 case ISD::TRUNCATE: 5985 Res = WidenVecOp_Convert(N); 5986 break; 5987 5988 case ISD::FP_TO_SINT_SAT: 5989 case ISD::FP_TO_UINT_SAT: 5990 Res = WidenVecOp_FP_TO_XINT_SAT(N); 5991 break; 5992 5993 case ISD::VECREDUCE_FADD: 5994 case ISD::VECREDUCE_FMUL: 5995 case ISD::VECREDUCE_ADD: 5996 case ISD::VECREDUCE_MUL: 5997 case ISD::VECREDUCE_AND: 5998 case ISD::VECREDUCE_OR: 5999 case ISD::VECREDUCE_XOR: 6000 case ISD::VECREDUCE_SMAX: 6001 case ISD::VECREDUCE_SMIN: 6002 case ISD::VECREDUCE_UMAX: 6003 case ISD::VECREDUCE_UMIN: 6004 case ISD::VECREDUCE_FMAX: 6005 case ISD::VECREDUCE_FMIN: 6006 case ISD::VECREDUCE_FMAXIMUM: 6007 case ISD::VECREDUCE_FMINIMUM: 6008 Res = WidenVecOp_VECREDUCE(N); 6009 break; 6010 case ISD::VECREDUCE_SEQ_FADD: 6011 case ISD::VECREDUCE_SEQ_FMUL: 6012 Res = WidenVecOp_VECREDUCE_SEQ(N); 6013 break; 6014 case ISD::VP_REDUCE_FADD: 6015 case ISD::VP_REDUCE_SEQ_FADD: 6016 case ISD::VP_REDUCE_FMUL: 6017 case ISD::VP_REDUCE_SEQ_FMUL: 6018 case ISD::VP_REDUCE_ADD: 6019 case ISD::VP_REDUCE_MUL: 6020 case ISD::VP_REDUCE_AND: 6021 case ISD::VP_REDUCE_OR: 6022 case ISD::VP_REDUCE_XOR: 6023 case ISD::VP_REDUCE_SMAX: 6024 case ISD::VP_REDUCE_SMIN: 6025 case ISD::VP_REDUCE_UMAX: 6026 case ISD::VP_REDUCE_UMIN: 6027 case ISD::VP_REDUCE_FMAX: 6028 case ISD::VP_REDUCE_FMIN: 6029 Res = WidenVecOp_VP_REDUCE(N); 6030 break; 6031 } 6032 6033 // If Res is null, the sub-method took care of registering the result. 6034 if (!Res.getNode()) return false; 6035 6036 // If the result is N, the sub-method updated N in place. Tell the legalizer 6037 // core about this. 6038 if (Res.getNode() == N) 6039 return true; 6040 6041 6042 if (N->isStrictFPOpcode()) 6043 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && 6044 "Invalid operand expansion"); 6045 else 6046 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 6047 "Invalid operand expansion"); 6048 6049 ReplaceValueWith(SDValue(N, 0), Res); 6050 return false; 6051 } 6052 6053 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) { 6054 SDLoc DL(N); 6055 EVT VT = N->getValueType(0); 6056 6057 SDValue InOp = N->getOperand(0); 6058 assert(getTypeAction(InOp.getValueType()) == 6059 TargetLowering::TypeWidenVector && 6060 "Unexpected type action"); 6061 InOp = GetWidenedVector(InOp); 6062 assert(VT.getVectorNumElements() < 6063 InOp.getValueType().getVectorNumElements() && 6064 "Input wasn't widened!"); 6065 6066 // We may need to further widen the operand until it has the same total 6067 // vector size as the result. 6068 EVT InVT = InOp.getValueType(); 6069 if (InVT.getSizeInBits() != VT.getSizeInBits()) { 6070 EVT InEltVT = InVT.getVectorElementType(); 6071 for (EVT FixedVT : MVT::vector_valuetypes()) { 6072 EVT FixedEltVT = FixedVT.getVectorElementType(); 6073 if (TLI.isTypeLegal(FixedVT) && 6074 FixedVT.getSizeInBits() == VT.getSizeInBits() && 6075 FixedEltVT == InEltVT) { 6076 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() && 6077 "Not enough elements in the fixed type for the operand!"); 6078 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() && 6079 "We can't have the same type as we started with!"); 6080 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements()) 6081 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT, 6082 DAG.getUNDEF(FixedVT), InOp, 6083 DAG.getVectorIdxConstant(0, DL)); 6084 else 6085 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp, 6086 DAG.getVectorIdxConstant(0, DL)); 6087 break; 6088 } 6089 } 6090 InVT = InOp.getValueType(); 6091 if (InVT.getSizeInBits() != VT.getSizeInBits()) 6092 // We couldn't find a legal vector type that was a widening of the input 6093 // and could be extended in-register to the result type, so we have to 6094 // scalarize. 6095 return WidenVecOp_Convert(N); 6096 } 6097 6098 // Use special DAG nodes to represent the operation of extending the 6099 // low lanes. 6100 switch (N->getOpcode()) { 6101 default: 6102 llvm_unreachable("Extend legalization on extend operation!"); 6103 case ISD::ANY_EXTEND: 6104 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp); 6105 case ISD::SIGN_EXTEND: 6106 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp); 6107 case ISD::ZERO_EXTEND: 6108 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp); 6109 } 6110 } 6111 6112 SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) { 6113 // The result (and first input) is legal, but the second input is illegal. 6114 // We can't do much to fix that, so just unroll and let the extracts off of 6115 // the second input be widened as needed later. 6116 return DAG.UnrollVectorOp(N); 6117 } 6118 6119 SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) { 6120 SDLoc DL(N); 6121 EVT ResultVT = N->getValueType(0); 6122 SDValue Test = N->getOperand(1); 6123 SDValue WideArg = GetWidenedVector(N->getOperand(0)); 6124 6125 // Process this node similarly to SETCC. 6126 EVT WideResultVT = getSetCCResultType(WideArg.getValueType()); 6127 if (ResultVT.getScalarType() == MVT::i1) 6128 WideResultVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 6129 WideResultVT.getVectorNumElements()); 6130 6131 SDValue WideNode = DAG.getNode(ISD::IS_FPCLASS, DL, WideResultVT, 6132 {WideArg, Test}, N->getFlags()); 6133 6134 // Extract the needed results from the result vector. 6135 EVT ResVT = 6136 EVT::getVectorVT(*DAG.getContext(), WideResultVT.getVectorElementType(), 6137 ResultVT.getVectorNumElements()); 6138 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, WideNode, 6139 DAG.getVectorIdxConstant(0, DL)); 6140 6141 EVT OpVT = N->getOperand(0).getValueType(); 6142 ISD::NodeType ExtendCode = 6143 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 6144 return DAG.getNode(ExtendCode, DL, ResultVT, CC); 6145 } 6146 6147 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 6148 // Since the result is legal and the input is illegal. 6149 EVT VT = N->getValueType(0); 6150 EVT EltVT = VT.getVectorElementType(); 6151 SDLoc dl(N); 6152 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); 6153 assert(getTypeAction(InOp.getValueType()) == 6154 TargetLowering::TypeWidenVector && 6155 "Unexpected type action"); 6156 InOp = GetWidenedVector(InOp); 6157 EVT InVT = InOp.getValueType(); 6158 unsigned Opcode = N->getOpcode(); 6159 6160 // See if a widened result type would be legal, if so widen the node. 6161 // FIXME: This isn't safe for StrictFP. Other optimization here is needed. 6162 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 6163 InVT.getVectorElementCount()); 6164 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) { 6165 SDValue Res; 6166 if (N->isStrictFPOpcode()) { 6167 if (Opcode == ISD::STRICT_FP_ROUND) 6168 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, 6169 { N->getOperand(0), InOp, N->getOperand(2) }); 6170 else 6171 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, 6172 { N->getOperand(0), InOp }); 6173 // Legalize the chain result - switch anything that used the old chain to 6174 // use the new one. 6175 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 6176 } else { 6177 if (Opcode == ISD::FP_ROUND) 6178 Res = DAG.getNode(Opcode, dl, WideVT, InOp, N->getOperand(1)); 6179 else 6180 Res = DAG.getNode(Opcode, dl, WideVT, InOp); 6181 } 6182 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, 6183 DAG.getVectorIdxConstant(0, dl)); 6184 } 6185 6186 EVT InEltVT = InVT.getVectorElementType(); 6187 6188 // Unroll the convert into some scalar code and create a nasty build vector. 6189 unsigned NumElts = VT.getVectorNumElements(); 6190 SmallVector<SDValue, 16> Ops(NumElts); 6191 if (N->isStrictFPOpcode()) { 6192 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 6193 SmallVector<SDValue, 32> OpChains; 6194 for (unsigned i=0; i < NumElts; ++i) { 6195 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 6196 DAG.getVectorIdxConstant(i, dl)); 6197 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps); 6198 OpChains.push_back(Ops[i].getValue(1)); 6199 } 6200 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains); 6201 ReplaceValueWith(SDValue(N, 1), NewChain); 6202 } else { 6203 for (unsigned i = 0; i < NumElts; ++i) 6204 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 6205 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, 6206 InOp, DAG.getVectorIdxConstant(i, dl))); 6207 } 6208 6209 return DAG.getBuildVector(VT, dl, Ops); 6210 } 6211 6212 SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) { 6213 EVT DstVT = N->getValueType(0); 6214 SDValue Src = GetWidenedVector(N->getOperand(0)); 6215 EVT SrcVT = Src.getValueType(); 6216 ElementCount WideNumElts = SrcVT.getVectorElementCount(); 6217 SDLoc dl(N); 6218 6219 // See if a widened result type would be legal, if so widen the node. 6220 EVT WideDstVT = EVT::getVectorVT(*DAG.getContext(), 6221 DstVT.getVectorElementType(), WideNumElts); 6222 if (TLI.isTypeLegal(WideDstVT)) { 6223 SDValue Res = 6224 DAG.getNode(N->getOpcode(), dl, WideDstVT, Src, N->getOperand(1)); 6225 return DAG.getNode( 6226 ISD::EXTRACT_SUBVECTOR, dl, DstVT, Res, 6227 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 6228 } 6229 6230 // Give up and unroll. 6231 return DAG.UnrollVectorOp(N); 6232 } 6233 6234 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 6235 EVT VT = N->getValueType(0); 6236 SDValue InOp = GetWidenedVector(N->getOperand(0)); 6237 EVT InWidenVT = InOp.getValueType(); 6238 SDLoc dl(N); 6239 6240 // Check if we can convert between two legal vector types and extract. 6241 TypeSize InWidenSize = InWidenVT.getSizeInBits(); 6242 TypeSize Size = VT.getSizeInBits(); 6243 // x86mmx is not an acceptable vector element type, so don't try. 6244 if (!VT.isVector() && VT != MVT::x86mmx && 6245 InWidenSize.hasKnownScalarFactor(Size)) { 6246 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(Size); 6247 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 6248 if (TLI.isTypeLegal(NewVT)) { 6249 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 6250 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 6251 DAG.getVectorIdxConstant(0, dl)); 6252 } 6253 } 6254 6255 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened 6256 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not, 6257 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids 6258 // having to copy via memory. 6259 if (VT.isVector()) { 6260 EVT EltVT = VT.getVectorElementType(); 6261 unsigned EltSize = EltVT.getFixedSizeInBits(); 6262 if (InWidenSize.isKnownMultipleOf(EltSize)) { 6263 ElementCount NewNumElts = 6264 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits()) 6265 .divideCoefficientBy(EltSize); 6266 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts); 6267 if (TLI.isTypeLegal(NewVT)) { 6268 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 6269 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp, 6270 DAG.getVectorIdxConstant(0, dl)); 6271 } 6272 } 6273 } 6274 6275 return CreateStackStoreLoad(InOp, VT); 6276 } 6277 6278 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 6279 EVT VT = N->getValueType(0); 6280 EVT EltVT = VT.getVectorElementType(); 6281 EVT InVT = N->getOperand(0).getValueType(); 6282 SDLoc dl(N); 6283 6284 // If the widen width for this operand is the same as the width of the concat 6285 // and all but the first operand is undef, just use the widened operand. 6286 unsigned NumOperands = N->getNumOperands(); 6287 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 6288 unsigned i; 6289 for (i = 1; i < NumOperands; ++i) 6290 if (!N->getOperand(i).isUndef()) 6291 break; 6292 6293 if (i == NumOperands) 6294 return GetWidenedVector(N->getOperand(0)); 6295 } 6296 6297 // Otherwise, fall back to a nasty build vector. 6298 unsigned NumElts = VT.getVectorNumElements(); 6299 SmallVector<SDValue, 16> Ops(NumElts); 6300 6301 unsigned NumInElts = InVT.getVectorNumElements(); 6302 6303 unsigned Idx = 0; 6304 for (unsigned i=0; i < NumOperands; ++i) { 6305 SDValue InOp = N->getOperand(i); 6306 assert(getTypeAction(InOp.getValueType()) == 6307 TargetLowering::TypeWidenVector && 6308 "Unexpected type action"); 6309 InOp = GetWidenedVector(InOp); 6310 for (unsigned j = 0; j < NumInElts; ++j) 6311 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 6312 DAG.getVectorIdxConstant(j, dl)); 6313 } 6314 return DAG.getBuildVector(VT, dl, Ops); 6315 } 6316 6317 SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) { 6318 EVT VT = N->getValueType(0); 6319 SDValue SubVec = N->getOperand(1); 6320 SDValue InVec = N->getOperand(0); 6321 6322 if (getTypeAction(SubVec.getValueType()) == TargetLowering::TypeWidenVector) 6323 SubVec = GetWidenedVector(SubVec); 6324 6325 if (SubVec.getValueType().knownBitsLE(VT) && InVec.isUndef() && 6326 N->getConstantOperandVal(2) == 0) 6327 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, InVec, SubVec, 6328 N->getOperand(2)); 6329 6330 report_fatal_error("Don't know how to widen the operands for " 6331 "INSERT_SUBVECTOR"); 6332 } 6333 6334 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 6335 SDValue InOp = GetWidenedVector(N->getOperand(0)); 6336 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), 6337 N->getValueType(0), InOp, N->getOperand(1)); 6338 } 6339 6340 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 6341 SDValue InOp = GetWidenedVector(N->getOperand(0)); 6342 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 6343 N->getValueType(0), InOp, N->getOperand(1)); 6344 } 6345 6346 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) { 6347 SDValue InOp = GetWidenedVector(N->getOperand(0)); 6348 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), InOp); 6349 } 6350 6351 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 6352 // We have to widen the value, but we want only to store the original 6353 // vector type. 6354 StoreSDNode *ST = cast<StoreSDNode>(N); 6355 6356 if (!ST->getMemoryVT().getScalarType().isByteSized()) 6357 return TLI.scalarizeVectorStore(ST, DAG); 6358 6359 if (ST->isTruncatingStore()) 6360 return TLI.scalarizeVectorStore(ST, DAG); 6361 6362 // Generate a vector-predicated store if it is custom/legal on the target. 6363 // To avoid possible recursion, only do this if the widened mask type is 6364 // legal. 6365 // FIXME: Not all targets may support EVL in VP_STORE. These will have been 6366 // removed from the IR by the ExpandVectorPredication pass but we're 6367 // reintroducing them here. 6368 SDValue StVal = ST->getValue(); 6369 EVT StVT = StVal.getValueType(); 6370 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StVT); 6371 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 6372 WideVT.getVectorElementCount()); 6373 6374 if (TLI.isOperationLegalOrCustom(ISD::VP_STORE, WideVT) && 6375 TLI.isTypeLegal(WideMaskVT)) { 6376 // Widen the value. 6377 SDLoc DL(N); 6378 StVal = GetWidenedVector(StVal); 6379 SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT); 6380 SDValue EVL = DAG.getElementCount(DL, TLI.getVPExplicitVectorLengthTy(), 6381 StVT.getVectorElementCount()); 6382 return DAG.getStoreVP(ST->getChain(), DL, StVal, ST->getBasePtr(), 6383 DAG.getUNDEF(ST->getBasePtr().getValueType()), Mask, 6384 EVL, StVT, ST->getMemOperand(), 6385 ST->getAddressingMode()); 6386 } 6387 6388 SmallVector<SDValue, 16> StChain; 6389 if (GenWidenVectorStores(StChain, ST)) { 6390 if (StChain.size() == 1) 6391 return StChain[0]; 6392 6393 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain); 6394 } 6395 6396 report_fatal_error("Unable to widen vector store"); 6397 } 6398 6399 SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) { 6400 assert((OpNo == 1 || OpNo == 3) && 6401 "Can widen only data or mask operand of vp_store"); 6402 VPStoreSDNode *ST = cast<VPStoreSDNode>(N); 6403 SDValue Mask = ST->getMask(); 6404 SDValue StVal = ST->getValue(); 6405 SDLoc dl(N); 6406 6407 if (OpNo == 1) { 6408 // Widen the value. 6409 StVal = GetWidenedVector(StVal); 6410 6411 // We only handle the case where the mask needs widening to an 6412 // identically-sized type as the vector inputs. 6413 assert(getTypeAction(Mask.getValueType()) == 6414 TargetLowering::TypeWidenVector && 6415 "Unable to widen VP store"); 6416 Mask = GetWidenedVector(Mask); 6417 } else { 6418 Mask = GetWidenedVector(Mask); 6419 6420 // We only handle the case where the stored value needs widening to an 6421 // identically-sized type as the mask. 6422 assert(getTypeAction(StVal.getValueType()) == 6423 TargetLowering::TypeWidenVector && 6424 "Unable to widen VP store"); 6425 StVal = GetWidenedVector(StVal); 6426 } 6427 6428 assert(Mask.getValueType().getVectorElementCount() == 6429 StVal.getValueType().getVectorElementCount() && 6430 "Mask and data vectors should have the same number of elements"); 6431 return DAG.getStoreVP(ST->getChain(), dl, StVal, ST->getBasePtr(), 6432 ST->getOffset(), Mask, ST->getVectorLength(), 6433 ST->getMemoryVT(), ST->getMemOperand(), 6434 ST->getAddressingMode(), ST->isTruncatingStore(), 6435 ST->isCompressingStore()); 6436 } 6437 6438 SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N, 6439 unsigned OpNo) { 6440 assert((OpNo == 1 || OpNo == 4) && 6441 "Can widen only data or mask operand of vp_strided_store"); 6442 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(N); 6443 SDValue Mask = SST->getMask(); 6444 SDValue StVal = SST->getValue(); 6445 SDLoc DL(N); 6446 6447 if (OpNo == 1) 6448 assert(getTypeAction(Mask.getValueType()) == 6449 TargetLowering::TypeWidenVector && 6450 "Unable to widen VP strided store"); 6451 else 6452 assert(getTypeAction(StVal.getValueType()) == 6453 TargetLowering::TypeWidenVector && 6454 "Unable to widen VP strided store"); 6455 6456 StVal = GetWidenedVector(StVal); 6457 Mask = GetWidenedVector(Mask); 6458 6459 assert(StVal.getValueType().getVectorElementCount() == 6460 Mask.getValueType().getVectorElementCount() && 6461 "Data and mask vectors should have the same number of elements"); 6462 6463 return DAG.getStridedStoreVP( 6464 SST->getChain(), DL, StVal, SST->getBasePtr(), SST->getOffset(), 6465 SST->getStride(), Mask, SST->getVectorLength(), SST->getMemoryVT(), 6466 SST->getMemOperand(), SST->getAddressingMode(), SST->isTruncatingStore(), 6467 SST->isCompressingStore()); 6468 } 6469 6470 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) { 6471 assert((OpNo == 1 || OpNo == 3) && 6472 "Can widen only data or mask operand of mstore"); 6473 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); 6474 SDValue Mask = MST->getMask(); 6475 EVT MaskVT = Mask.getValueType(); 6476 SDValue StVal = MST->getValue(); 6477 SDLoc dl(N); 6478 6479 if (OpNo == 1) { 6480 // Widen the value. 6481 StVal = GetWidenedVector(StVal); 6482 6483 // The mask should be widened as well. 6484 EVT WideVT = StVal.getValueType(); 6485 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 6486 MaskVT.getVectorElementType(), 6487 WideVT.getVectorNumElements()); 6488 Mask = ModifyToType(Mask, WideMaskVT, true); 6489 } else { 6490 // Widen the mask. 6491 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT); 6492 Mask = ModifyToType(Mask, WideMaskVT, true); 6493 6494 EVT ValueVT = StVal.getValueType(); 6495 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), 6496 ValueVT.getVectorElementType(), 6497 WideMaskVT.getVectorNumElements()); 6498 StVal = ModifyToType(StVal, WideVT); 6499 } 6500 6501 assert(Mask.getValueType().getVectorNumElements() == 6502 StVal.getValueType().getVectorNumElements() && 6503 "Mask and data vectors should have the same number of elements"); 6504 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(), 6505 MST->getOffset(), Mask, MST->getMemoryVT(), 6506 MST->getMemOperand(), MST->getAddressingMode(), 6507 false, MST->isCompressingStore()); 6508 } 6509 6510 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) { 6511 assert(OpNo == 4 && "Can widen only the index of mgather"); 6512 auto *MG = cast<MaskedGatherSDNode>(N); 6513 SDValue DataOp = MG->getPassThru(); 6514 SDValue Mask = MG->getMask(); 6515 SDValue Scale = MG->getScale(); 6516 6517 // Just widen the index. It's allowed to have extra elements. 6518 SDValue Index = GetWidenedVector(MG->getIndex()); 6519 6520 SDLoc dl(N); 6521 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index, 6522 Scale}; 6523 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops, 6524 MG->getMemOperand(), MG->getIndexType(), 6525 MG->getExtensionType()); 6526 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 6527 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 6528 return SDValue(); 6529 } 6530 6531 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) { 6532 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N); 6533 SDValue DataOp = MSC->getValue(); 6534 SDValue Mask = MSC->getMask(); 6535 SDValue Index = MSC->getIndex(); 6536 SDValue Scale = MSC->getScale(); 6537 EVT WideMemVT = MSC->getMemoryVT(); 6538 6539 if (OpNo == 1) { 6540 DataOp = GetWidenedVector(DataOp); 6541 unsigned NumElts = DataOp.getValueType().getVectorNumElements(); 6542 6543 // Widen index. 6544 EVT IndexVT = Index.getValueType(); 6545 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(), 6546 IndexVT.getVectorElementType(), NumElts); 6547 Index = ModifyToType(Index, WideIndexVT); 6548 6549 // The mask should be widened as well. 6550 EVT MaskVT = Mask.getValueType(); 6551 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 6552 MaskVT.getVectorElementType(), NumElts); 6553 Mask = ModifyToType(Mask, WideMaskVT, true); 6554 6555 // Widen the MemoryType 6556 WideMemVT = EVT::getVectorVT(*DAG.getContext(), 6557 MSC->getMemoryVT().getScalarType(), NumElts); 6558 } else if (OpNo == 4) { 6559 // Just widen the index. It's allowed to have extra elements. 6560 Index = GetWidenedVector(Index); 6561 } else 6562 llvm_unreachable("Can't widen this operand of mscatter"); 6563 6564 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index, 6565 Scale}; 6566 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), 6567 Ops, MSC->getMemOperand(), MSC->getIndexType(), 6568 MSC->isTruncatingStore()); 6569 } 6570 6571 SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) { 6572 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(N); 6573 SDValue DataOp = VPSC->getValue(); 6574 SDValue Mask = VPSC->getMask(); 6575 SDValue Index = VPSC->getIndex(); 6576 SDValue Scale = VPSC->getScale(); 6577 EVT WideMemVT = VPSC->getMemoryVT(); 6578 6579 if (OpNo == 1) { 6580 DataOp = GetWidenedVector(DataOp); 6581 Index = GetWidenedVector(Index); 6582 const auto WideEC = DataOp.getValueType().getVectorElementCount(); 6583 Mask = GetWidenedMask(Mask, WideEC); 6584 WideMemVT = EVT::getVectorVT(*DAG.getContext(), 6585 VPSC->getMemoryVT().getScalarType(), WideEC); 6586 } else if (OpNo == 3) { 6587 // Just widen the index. It's allowed to have extra elements. 6588 Index = GetWidenedVector(Index); 6589 } else 6590 llvm_unreachable("Can't widen this operand of VP_SCATTER"); 6591 6592 SDValue Ops[] = { 6593 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask, 6594 VPSC->getVectorLength()}; 6595 return DAG.getScatterVP(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), Ops, 6596 VPSC->getMemOperand(), VPSC->getIndexType()); 6597 } 6598 6599 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 6600 SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 6601 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 6602 SDLoc dl(N); 6603 EVT VT = N->getValueType(0); 6604 6605 // WARNING: In this code we widen the compare instruction with garbage. 6606 // This garbage may contain denormal floats which may be slow. Is this a real 6607 // concern ? Should we zero the unused lanes if this is a float compare ? 6608 6609 // Get a new SETCC node to compare the newly widened operands. 6610 // Only some of the compared elements are legal. 6611 EVT SVT = getSetCCResultType(InOp0.getValueType()); 6612 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC. 6613 if (VT.getScalarType() == MVT::i1) 6614 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 6615 SVT.getVectorElementCount()); 6616 6617 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N), 6618 SVT, InOp0, InOp1, N->getOperand(2)); 6619 6620 // Extract the needed results from the result vector. 6621 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 6622 SVT.getVectorElementType(), 6623 VT.getVectorElementCount()); 6624 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC, 6625 DAG.getVectorIdxConstant(0, dl)); 6626 6627 EVT OpVT = N->getOperand(0).getValueType(); 6628 ISD::NodeType ExtendCode = 6629 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 6630 return DAG.getNode(ExtendCode, dl, VT, CC); 6631 } 6632 6633 SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) { 6634 SDValue Chain = N->getOperand(0); 6635 SDValue LHS = GetWidenedVector(N->getOperand(1)); 6636 SDValue RHS = GetWidenedVector(N->getOperand(2)); 6637 SDValue CC = N->getOperand(3); 6638 SDLoc dl(N); 6639 6640 EVT VT = N->getValueType(0); 6641 EVT EltVT = VT.getVectorElementType(); 6642 EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 6643 unsigned NumElts = VT.getVectorNumElements(); 6644 6645 // Unroll into a build vector. 6646 SmallVector<SDValue, 8> Scalars(NumElts); 6647 SmallVector<SDValue, 8> Chains(NumElts); 6648 6649 for (unsigned i = 0; i != NumElts; ++i) { 6650 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 6651 DAG.getVectorIdxConstant(i, dl)); 6652 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 6653 DAG.getVectorIdxConstant(i, dl)); 6654 6655 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other}, 6656 {Chain, LHSElem, RHSElem, CC}); 6657 Chains[i] = Scalars[i].getValue(1); 6658 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i], 6659 DAG.getBoolConstant(true, dl, EltVT, VT), 6660 DAG.getBoolConstant(false, dl, EltVT, VT)); 6661 } 6662 6663 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 6664 ReplaceValueWith(SDValue(N, 1), NewChain); 6665 6666 return DAG.getBuildVector(VT, dl, Scalars); 6667 } 6668 6669 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) { 6670 SDLoc dl(N); 6671 SDValue Op = GetWidenedVector(N->getOperand(0)); 6672 EVT OrigVT = N->getOperand(0).getValueType(); 6673 EVT WideVT = Op.getValueType(); 6674 EVT ElemVT = OrigVT.getVectorElementType(); 6675 SDNodeFlags Flags = N->getFlags(); 6676 6677 unsigned Opc = N->getOpcode(); 6678 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc); 6679 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags); 6680 assert(NeutralElem && "Neutral element must exist"); 6681 6682 // Pad the vector with the neutral element. 6683 unsigned OrigElts = OrigVT.getVectorMinNumElements(); 6684 unsigned WideElts = WideVT.getVectorMinNumElements(); 6685 6686 if (WideVT.isScalableVector()) { 6687 unsigned GCD = std::gcd(OrigElts, WideElts); 6688 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, 6689 ElementCount::getScalable(GCD)); 6690 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem); 6691 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD) 6692 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral, 6693 DAG.getVectorIdxConstant(Idx, dl)); 6694 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags); 6695 } 6696 6697 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) 6698 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem, 6699 DAG.getVectorIdxConstant(Idx, dl)); 6700 6701 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags); 6702 } 6703 6704 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) { 6705 SDLoc dl(N); 6706 SDValue AccOp = N->getOperand(0); 6707 SDValue VecOp = N->getOperand(1); 6708 SDValue Op = GetWidenedVector(VecOp); 6709 6710 EVT OrigVT = VecOp.getValueType(); 6711 EVT WideVT = Op.getValueType(); 6712 EVT ElemVT = OrigVT.getVectorElementType(); 6713 SDNodeFlags Flags = N->getFlags(); 6714 6715 unsigned Opc = N->getOpcode(); 6716 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc); 6717 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags); 6718 6719 // Pad the vector with the neutral element. 6720 unsigned OrigElts = OrigVT.getVectorMinNumElements(); 6721 unsigned WideElts = WideVT.getVectorMinNumElements(); 6722 6723 if (WideVT.isScalableVector()) { 6724 unsigned GCD = std::gcd(OrigElts, WideElts); 6725 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, 6726 ElementCount::getScalable(GCD)); 6727 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem); 6728 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD) 6729 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral, 6730 DAG.getVectorIdxConstant(Idx, dl)); 6731 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags); 6732 } 6733 6734 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) 6735 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem, 6736 DAG.getVectorIdxConstant(Idx, dl)); 6737 6738 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags); 6739 } 6740 6741 SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) { 6742 assert(N->isVPOpcode() && "Expected VP opcode"); 6743 6744 SDLoc dl(N); 6745 SDValue Op = GetWidenedVector(N->getOperand(1)); 6746 SDValue Mask = GetWidenedMask(N->getOperand(2), 6747 Op.getValueType().getVectorElementCount()); 6748 6749 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), 6750 {N->getOperand(0), Op, Mask, N->getOperand(3)}, 6751 N->getFlags()); 6752 } 6753 6754 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) { 6755 // This only gets called in the case that the left and right inputs and 6756 // result are of a legal odd vector type, and the condition is illegal i1 of 6757 // the same odd width that needs widening. 6758 EVT VT = N->getValueType(0); 6759 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT)); 6760 6761 SDValue Cond = GetWidenedVector(N->getOperand(0)); 6762 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N)); 6763 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N)); 6764 SDLoc DL(N); 6765 6766 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond, 6767 LeftIn, RightIn); 6768 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Select, 6769 DAG.getVectorIdxConstant(0, DL)); 6770 } 6771 6772 //===----------------------------------------------------------------------===// 6773 // Vector Widening Utilities 6774 //===----------------------------------------------------------------------===// 6775 6776 // Utility function to find the type to chop up a widen vector for load/store 6777 // TLI: Target lowering used to determine legal types. 6778 // Width: Width left need to load/store. 6779 // WidenVT: The widen vector type to load to/store from 6780 // Align: If 0, don't allow use of a wider type 6781 // WidenEx: If Align is not 0, the amount additional we can load/store from. 6782 6783 static std::optional<EVT> findMemType(SelectionDAG &DAG, 6784 const TargetLowering &TLI, unsigned Width, 6785 EVT WidenVT, unsigned Align = 0, 6786 unsigned WidenEx = 0) { 6787 EVT WidenEltVT = WidenVT.getVectorElementType(); 6788 const bool Scalable = WidenVT.isScalableVector(); 6789 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue(); 6790 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 6791 unsigned AlignInBits = Align*8; 6792 6793 // If we have one element to load/store, return it. 6794 EVT RetVT = WidenEltVT; 6795 if (!Scalable && Width == WidenEltWidth) 6796 return RetVT; 6797 6798 // Don't bother looking for an integer type if the vector is scalable, skip 6799 // to vector types. 6800 if (!Scalable) { 6801 // See if there is larger legal integer than the element type to load/store. 6802 for (EVT MemVT : reverse(MVT::integer_valuetypes())) { 6803 unsigned MemVTWidth = MemVT.getSizeInBits(); 6804 if (MemVT.getSizeInBits() <= WidenEltWidth) 6805 break; 6806 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); 6807 if ((Action == TargetLowering::TypeLegal || 6808 Action == TargetLowering::TypePromoteInteger) && 6809 (WidenWidth % MemVTWidth) == 0 && 6810 isPowerOf2_32(WidenWidth / MemVTWidth) && 6811 (MemVTWidth <= Width || 6812 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 6813 if (MemVTWidth == WidenWidth) 6814 return MemVT; 6815 RetVT = MemVT; 6816 break; 6817 } 6818 } 6819 } 6820 6821 // See if there is a larger vector type to load/store that has the same vector 6822 // element type and is evenly divisible with the WidenVT. 6823 for (EVT MemVT : reverse(MVT::vector_valuetypes())) { 6824 // Skip vector MVTs which don't match the scalable property of WidenVT. 6825 if (Scalable != MemVT.isScalableVector()) 6826 continue; 6827 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue(); 6828 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); 6829 if ((Action == TargetLowering::TypeLegal || 6830 Action == TargetLowering::TypePromoteInteger) && 6831 WidenEltVT == MemVT.getVectorElementType() && 6832 (WidenWidth % MemVTWidth) == 0 && 6833 isPowerOf2_32(WidenWidth / MemVTWidth) && 6834 (MemVTWidth <= Width || 6835 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 6836 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT) 6837 return MemVT; 6838 } 6839 } 6840 6841 // Using element-wise loads and stores for widening operations is not 6842 // supported for scalable vectors 6843 if (Scalable) 6844 return std::nullopt; 6845 6846 return RetVT; 6847 } 6848 6849 // Builds a vector type from scalar loads 6850 // VecTy: Resulting Vector type 6851 // LDOps: Load operators to build a vector type 6852 // [Start,End) the list of loads to use. 6853 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 6854 SmallVectorImpl<SDValue> &LdOps, 6855 unsigned Start, unsigned End) { 6856 SDLoc dl(LdOps[Start]); 6857 EVT LdTy = LdOps[Start].getValueType(); 6858 unsigned Width = VecTy.getSizeInBits(); 6859 unsigned NumElts = Width / LdTy.getSizeInBits(); 6860 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 6861 6862 unsigned Idx = 1; 6863 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 6864 6865 for (unsigned i = Start + 1; i != End; ++i) { 6866 EVT NewLdTy = LdOps[i].getValueType(); 6867 if (NewLdTy != LdTy) { 6868 NumElts = Width / NewLdTy.getSizeInBits(); 6869 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 6870 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 6871 // Readjust position and vector position based on new load type. 6872 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 6873 LdTy = NewLdTy; 6874 } 6875 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 6876 DAG.getVectorIdxConstant(Idx++, dl)); 6877 } 6878 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 6879 } 6880 6881 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, 6882 LoadSDNode *LD) { 6883 // The strategy assumes that we can efficiently load power-of-two widths. 6884 // The routine chops the vector into the largest vector loads with the same 6885 // element type or scalar loads and then recombines it to the widen vector 6886 // type. 6887 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 6888 EVT LdVT = LD->getMemoryVT(); 6889 SDLoc dl(LD); 6890 assert(LdVT.isVector() && WidenVT.isVector()); 6891 assert(LdVT.isScalableVector() == WidenVT.isScalableVector()); 6892 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 6893 6894 // Load information 6895 SDValue Chain = LD->getChain(); 6896 SDValue BasePtr = LD->getBasePtr(); 6897 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 6898 AAMDNodes AAInfo = LD->getAAInfo(); 6899 6900 TypeSize LdWidth = LdVT.getSizeInBits(); 6901 TypeSize WidenWidth = WidenVT.getSizeInBits(); 6902 TypeSize WidthDiff = WidenWidth - LdWidth; 6903 // Allow wider loads if they are sufficiently aligned to avoid memory faults 6904 // and if the original load is simple. 6905 unsigned LdAlign = 6906 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value(); 6907 6908 // Find the vector type that can load from. 6909 std::optional<EVT> FirstVT = 6910 findMemType(DAG, TLI, LdWidth.getKnownMinValue(), WidenVT, LdAlign, 6911 WidthDiff.getKnownMinValue()); 6912 6913 if (!FirstVT) 6914 return SDValue(); 6915 6916 SmallVector<EVT, 8> MemVTs; 6917 TypeSize FirstVTWidth = FirstVT->getSizeInBits(); 6918 6919 // Unless we're able to load in one instruction we must work out how to load 6920 // the remainder. 6921 if (!TypeSize::isKnownLE(LdWidth, FirstVTWidth)) { 6922 std::optional<EVT> NewVT = FirstVT; 6923 TypeSize RemainingWidth = LdWidth; 6924 TypeSize NewVTWidth = FirstVTWidth; 6925 do { 6926 RemainingWidth -= NewVTWidth; 6927 if (TypeSize::isKnownLT(RemainingWidth, NewVTWidth)) { 6928 // The current type we are using is too large. Find a better size. 6929 NewVT = findMemType(DAG, TLI, RemainingWidth.getKnownMinValue(), 6930 WidenVT, LdAlign, WidthDiff.getKnownMinValue()); 6931 if (!NewVT) 6932 return SDValue(); 6933 NewVTWidth = NewVT->getSizeInBits(); 6934 } 6935 MemVTs.push_back(*NewVT); 6936 } while (TypeSize::isKnownGT(RemainingWidth, NewVTWidth)); 6937 } 6938 6939 SDValue LdOp = DAG.getLoad(*FirstVT, dl, Chain, BasePtr, LD->getPointerInfo(), 6940 LD->getOriginalAlign(), MMOFlags, AAInfo); 6941 LdChain.push_back(LdOp.getValue(1)); 6942 6943 // Check if we can load the element with one instruction. 6944 if (MemVTs.empty()) { 6945 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth)); 6946 if (!FirstVT->isVector()) { 6947 unsigned NumElts = 6948 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue(); 6949 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), *FirstVT, NumElts); 6950 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 6951 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 6952 } 6953 if (FirstVT == WidenVT) 6954 return LdOp; 6955 6956 // TODO: We don't currently have any tests that exercise this code path. 6957 assert(WidenWidth.getFixedValue() % FirstVTWidth.getFixedValue() == 0); 6958 unsigned NumConcat = 6959 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue(); 6960 SmallVector<SDValue, 16> ConcatOps(NumConcat); 6961 SDValue UndefVal = DAG.getUNDEF(*FirstVT); 6962 ConcatOps[0] = LdOp; 6963 for (unsigned i = 1; i != NumConcat; ++i) 6964 ConcatOps[i] = UndefVal; 6965 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps); 6966 } 6967 6968 // Load vector by using multiple loads from largest vector to scalar. 6969 SmallVector<SDValue, 16> LdOps; 6970 LdOps.push_back(LdOp); 6971 6972 uint64_t ScaledOffset = 0; 6973 MachinePointerInfo MPI = LD->getPointerInfo(); 6974 6975 // First incremement past the first load. 6976 IncrementPointer(cast<LoadSDNode>(LdOp), *FirstVT, MPI, BasePtr, 6977 &ScaledOffset); 6978 6979 for (EVT MemVT : MemVTs) { 6980 Align NewAlign = ScaledOffset == 0 6981 ? LD->getOriginalAlign() 6982 : commonAlignment(LD->getAlign(), ScaledOffset); 6983 SDValue L = 6984 DAG.getLoad(MemVT, dl, Chain, BasePtr, MPI, NewAlign, MMOFlags, AAInfo); 6985 6986 LdOps.push_back(L); 6987 LdChain.push_back(L.getValue(1)); 6988 IncrementPointer(cast<LoadSDNode>(L), MemVT, MPI, BasePtr, &ScaledOffset); 6989 } 6990 6991 // Build the vector from the load operations. 6992 unsigned End = LdOps.size(); 6993 if (!LdOps[0].getValueType().isVector()) 6994 // All the loads are scalar loads. 6995 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 6996 6997 // If the load contains vectors, build the vector using concat vector. 6998 // All of the vectors used to load are power-of-2, and the scalar loads can be 6999 // combined to make a power-of-2 vector. 7000 SmallVector<SDValue, 16> ConcatOps(End); 7001 int i = End - 1; 7002 int Idx = End; 7003 EVT LdTy = LdOps[i].getValueType(); 7004 // First, combine the scalar loads to a vector. 7005 if (!LdTy.isVector()) { 7006 for (--i; i >= 0; --i) { 7007 LdTy = LdOps[i].getValueType(); 7008 if (LdTy.isVector()) 7009 break; 7010 } 7011 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End); 7012 } 7013 7014 ConcatOps[--Idx] = LdOps[i]; 7015 for (--i; i >= 0; --i) { 7016 EVT NewLdTy = LdOps[i].getValueType(); 7017 if (NewLdTy != LdTy) { 7018 // Create a larger vector. 7019 TypeSize LdTySize = LdTy.getSizeInBits(); 7020 TypeSize NewLdTySize = NewLdTy.getSizeInBits(); 7021 assert(NewLdTySize.isScalable() == LdTySize.isScalable() && 7022 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue())); 7023 unsigned NumOps = 7024 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue(); 7025 SmallVector<SDValue, 16> WidenOps(NumOps); 7026 unsigned j = 0; 7027 for (; j != End-Idx; ++j) 7028 WidenOps[j] = ConcatOps[Idx+j]; 7029 for (; j != NumOps; ++j) 7030 WidenOps[j] = DAG.getUNDEF(LdTy); 7031 7032 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 7033 WidenOps); 7034 Idx = End - 1; 7035 LdTy = NewLdTy; 7036 } 7037 ConcatOps[--Idx] = LdOps[i]; 7038 } 7039 7040 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx)) 7041 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 7042 ArrayRef(&ConcatOps[Idx], End - Idx)); 7043 7044 // We need to fill the rest with undefs to build the vector. 7045 unsigned NumOps = 7046 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue(); 7047 SmallVector<SDValue, 16> WidenOps(NumOps); 7048 SDValue UndefVal = DAG.getUNDEF(LdTy); 7049 { 7050 unsigned i = 0; 7051 for (; i != End-Idx; ++i) 7052 WidenOps[i] = ConcatOps[Idx+i]; 7053 for (; i != NumOps; ++i) 7054 WidenOps[i] = UndefVal; 7055 } 7056 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps); 7057 } 7058 7059 SDValue 7060 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, 7061 LoadSDNode *LD, 7062 ISD::LoadExtType ExtType) { 7063 // For extension loads, it may not be more efficient to chop up the vector 7064 // and then extend it. Instead, we unroll the load and build a new vector. 7065 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 7066 EVT LdVT = LD->getMemoryVT(); 7067 SDLoc dl(LD); 7068 assert(LdVT.isVector() && WidenVT.isVector()); 7069 assert(LdVT.isScalableVector() == WidenVT.isScalableVector()); 7070 7071 // Load information 7072 SDValue Chain = LD->getChain(); 7073 SDValue BasePtr = LD->getBasePtr(); 7074 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 7075 AAMDNodes AAInfo = LD->getAAInfo(); 7076 7077 if (LdVT.isScalableVector()) 7078 report_fatal_error("Generating widen scalable extending vector loads is " 7079 "not yet supported"); 7080 7081 EVT EltVT = WidenVT.getVectorElementType(); 7082 EVT LdEltVT = LdVT.getVectorElementType(); 7083 unsigned NumElts = LdVT.getVectorNumElements(); 7084 7085 // Load each element and widen. 7086 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 7087 SmallVector<SDValue, 16> Ops(WidenNumElts); 7088 unsigned Increment = LdEltVT.getSizeInBits() / 8; 7089 Ops[0] = 7090 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(), 7091 LdEltVT, LD->getOriginalAlign(), MMOFlags, AAInfo); 7092 LdChain.push_back(Ops[0].getValue(1)); 7093 unsigned i = 0, Offset = Increment; 7094 for (i=1; i < NumElts; ++i, Offset += Increment) { 7095 SDValue NewBasePtr = 7096 DAG.getObjectPtrOffset(dl, BasePtr, TypeSize::Fixed(Offset)); 7097 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 7098 LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 7099 LD->getOriginalAlign(), MMOFlags, AAInfo); 7100 LdChain.push_back(Ops[i].getValue(1)); 7101 } 7102 7103 // Fill the rest with undefs. 7104 SDValue UndefVal = DAG.getUNDEF(EltVT); 7105 for (; i != WidenNumElts; ++i) 7106 Ops[i] = UndefVal; 7107 7108 return DAG.getBuildVector(WidenVT, dl, Ops); 7109 } 7110 7111 bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, 7112 StoreSDNode *ST) { 7113 // The strategy assumes that we can efficiently store power-of-two widths. 7114 // The routine chops the vector into the largest vector stores with the same 7115 // element type or scalar stores. 7116 SDValue Chain = ST->getChain(); 7117 SDValue BasePtr = ST->getBasePtr(); 7118 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); 7119 AAMDNodes AAInfo = ST->getAAInfo(); 7120 SDValue ValOp = GetWidenedVector(ST->getValue()); 7121 SDLoc dl(ST); 7122 7123 EVT StVT = ST->getMemoryVT(); 7124 TypeSize StWidth = StVT.getSizeInBits(); 7125 EVT ValVT = ValOp.getValueType(); 7126 TypeSize ValWidth = ValVT.getSizeInBits(); 7127 EVT ValEltVT = ValVT.getVectorElementType(); 7128 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits(); 7129 assert(StVT.getVectorElementType() == ValEltVT); 7130 assert(StVT.isScalableVector() == ValVT.isScalableVector() && 7131 "Mismatch between store and value types"); 7132 7133 int Idx = 0; // current index to store 7134 7135 MachinePointerInfo MPI = ST->getPointerInfo(); 7136 uint64_t ScaledOffset = 0; 7137 7138 // A breakdown of how to widen this vector store. Each element of the vector 7139 // is a memory VT combined with the number of times it is to be stored to, 7140 // e,g., v5i32 -> {{v2i32,2},{i32,1}} 7141 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs; 7142 7143 while (StWidth.isNonZero()) { 7144 // Find the largest vector type we can store with. 7145 std::optional<EVT> NewVT = 7146 findMemType(DAG, TLI, StWidth.getKnownMinValue(), ValVT); 7147 if (!NewVT) 7148 return false; 7149 MemVTs.push_back({*NewVT, 0}); 7150 TypeSize NewVTWidth = NewVT->getSizeInBits(); 7151 7152 do { 7153 StWidth -= NewVTWidth; 7154 MemVTs.back().second++; 7155 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth)); 7156 } 7157 7158 for (const auto &Pair : MemVTs) { 7159 EVT NewVT = Pair.first; 7160 unsigned Count = Pair.second; 7161 TypeSize NewVTWidth = NewVT.getSizeInBits(); 7162 7163 if (NewVT.isVector()) { 7164 unsigned NumVTElts = NewVT.getVectorMinNumElements(); 7165 do { 7166 Align NewAlign = ScaledOffset == 0 7167 ? ST->getOriginalAlign() 7168 : commonAlignment(ST->getAlign(), ScaledOffset); 7169 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 7170 DAG.getVectorIdxConstant(Idx, dl)); 7171 SDValue PartStore = DAG.getStore(Chain, dl, EOp, BasePtr, MPI, NewAlign, 7172 MMOFlags, AAInfo); 7173 StChain.push_back(PartStore); 7174 7175 Idx += NumVTElts; 7176 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr, 7177 &ScaledOffset); 7178 } while (--Count); 7179 } else { 7180 // Cast the vector to the scalar type we can store. 7181 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue(); 7182 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 7183 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 7184 // Readjust index position based on new vector type. 7185 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue(); 7186 do { 7187 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 7188 DAG.getVectorIdxConstant(Idx++, dl)); 7189 SDValue PartStore = 7190 DAG.getStore(Chain, dl, EOp, BasePtr, MPI, ST->getOriginalAlign(), 7191 MMOFlags, AAInfo); 7192 StChain.push_back(PartStore); 7193 7194 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr); 7195 } while (--Count); 7196 // Restore index back to be relative to the original widen element type. 7197 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth; 7198 } 7199 } 7200 7201 return true; 7202 } 7203 7204 /// Modifies a vector input (widen or narrows) to a vector of NVT. The 7205 /// input vector must have the same element type as NVT. 7206 /// FillWithZeroes specifies that the vector should be widened with zeroes. 7207 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT, 7208 bool FillWithZeroes) { 7209 // Note that InOp might have been widened so it might already have 7210 // the right width or it might need be narrowed. 7211 EVT InVT = InOp.getValueType(); 7212 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 7213 "input and widen element type must match"); 7214 assert(InVT.isScalableVector() == NVT.isScalableVector() && 7215 "cannot modify scalable vectors in this way"); 7216 SDLoc dl(InOp); 7217 7218 // Check if InOp already has the right width. 7219 if (InVT == NVT) 7220 return InOp; 7221 7222 ElementCount InEC = InVT.getVectorElementCount(); 7223 ElementCount WidenEC = NVT.getVectorElementCount(); 7224 if (WidenEC.hasKnownScalarFactor(InEC)) { 7225 unsigned NumConcat = WidenEC.getKnownScalarFactor(InEC); 7226 SmallVector<SDValue, 16> Ops(NumConcat); 7227 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) : 7228 DAG.getUNDEF(InVT); 7229 Ops[0] = InOp; 7230 for (unsigned i = 1; i != NumConcat; ++i) 7231 Ops[i] = FillVal; 7232 7233 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops); 7234 } 7235 7236 if (InEC.hasKnownScalarFactor(WidenEC)) 7237 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 7238 DAG.getVectorIdxConstant(0, dl)); 7239 7240 assert(!InVT.isScalableVector() && !NVT.isScalableVector() && 7241 "Scalable vectors should have been handled already."); 7242 7243 unsigned InNumElts = InEC.getFixedValue(); 7244 unsigned WidenNumElts = WidenEC.getFixedValue(); 7245 7246 // Fall back to extract and build (+ mask, if padding with zeros). 7247 SmallVector<SDValue, 16> Ops(WidenNumElts); 7248 EVT EltVT = NVT.getVectorElementType(); 7249 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 7250 unsigned Idx; 7251 for (Idx = 0; Idx < MinNumElts; ++Idx) 7252 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 7253 DAG.getVectorIdxConstant(Idx, dl)); 7254 7255 SDValue UndefVal = DAG.getUNDEF(EltVT); 7256 for (; Idx < WidenNumElts; ++Idx) 7257 Ops[Idx] = UndefVal; 7258 7259 SDValue Widened = DAG.getBuildVector(NVT, dl, Ops); 7260 if (!FillWithZeroes) 7261 return Widened; 7262 7263 assert(NVT.isInteger() && 7264 "We expect to never want to FillWithZeroes for non-integral types."); 7265 7266 SmallVector<SDValue, 16> MaskOps; 7267 MaskOps.append(MinNumElts, DAG.getAllOnesConstant(dl, EltVT)); 7268 MaskOps.append(WidenNumElts - MinNumElts, DAG.getConstant(0, dl, EltVT)); 7269 7270 return DAG.getNode(ISD::AND, dl, NVT, Widened, 7271 DAG.getBuildVector(NVT, dl, MaskOps)); 7272 } 7273